aboutsummaryrefslogtreecommitdiff
path: root/v_windows/v/old/vlib/x/json2
diff options
context:
space:
mode:
Diffstat (limited to 'v_windows/v/old/vlib/x/json2')
-rw-r--r--v_windows/v/old/vlib/x/json2/README.md175
-rw-r--r--v_windows/v/old/vlib/x/json2/any_test.v130
-rw-r--r--v_windows/v/old/vlib/x/json2/decoder.v200
-rw-r--r--v_windows/v/old/vlib/x/json2/decoder_test.v61
-rw-r--r--v_windows/v/old/vlib/x/json2/encoder.v179
-rw-r--r--v_windows/v/old/vlib/x/json2/encoder_test.v29
-rw-r--r--v_windows/v/old/vlib/x/json2/json2.v122
-rw-r--r--v_windows/v/old/vlib/x/json2/json2_test.v398
-rw-r--r--v_windows/v/old/vlib/x/json2/scanner.v306
-rw-r--r--v_windows/v/old/vlib/x/json2/scanner_test.v351
10 files changed, 1951 insertions, 0 deletions
diff --git a/v_windows/v/old/vlib/x/json2/README.md b/v_windows/v/old/vlib/x/json2/README.md
new file mode 100644
index 0000000..fcefbff
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/README.md
@@ -0,0 +1,175 @@
+> The name `json2` was chosen to avoid any unwanted potential conflicts with the
+> existing codegen tailored for the main `json` module which is powered by CJSON.
+
+`x.json2` is an experimental JSON parser written from scratch on V.
+
+## Usage
+```v oksyntax
+import x.json2
+import net.http
+
+fn main() {
+ // Decoding
+ resp := http.get('https://example.com') ?
+
+ // raw decode
+ raw_person := json2.raw_decode(resp.text) ?
+
+ // Casting `Any` type / Navigating
+ person := raw_person.as_map()
+ name := person['name'].str() // Bob
+ age := person['age'].int() // 19
+ pi := person['pi'].f64() // 3.14....
+
+ // Constructing an `Any` type
+ mut me := map[string]json2.Any{}
+ me['name'] = 'Bob'
+ me['age'] = 18
+
+ mut arr := []json2.Any{}
+ arr << 'rock'
+ arr << 'papers'
+ arr << json2.null
+ arr << 12
+
+ me['interests'] = arr
+
+ mut pets := map[string]json2.Any{}
+ pets['Sam'] = 'Maltese Shitzu'
+ me['pets'] = pets
+
+ // Stringify to JSON
+ println(me.str())
+ //{
+ // "name":"Bob",
+ // "age":18,
+ // "interests":["rock","papers","scissors",null,12],
+ // "pets":{"Sam":"Maltese"}
+ //}
+
+ // Encode a struct/type to JSON
+ encoded_json := json2.encode<Person>(person2)
+}
+```
+## Using `decode<T>` and `encode<T>`
+> Codegen for this feature is still WIP.
+> You need to manually define the methods before using the module to structs.
+
+In order to use the `decode<T>` and `encode<T>` function, you need to explicitly define
+two methods: `from_json` and `to_json`. `from_json` accepts a `json2.Any` argument
+and inside of it you need to map the fields you're going to put into the type.
+As for `to_json` method, you just need to map the values into `json2.Any`
+and turn it into a string.
+
+```v ignore
+struct Person {
+mut:
+ name string
+ age int = 20
+ pets []string
+}
+
+fn (mut p Person) from_json(f json2.Any) {
+ obj := f.as_map()
+ for k, v in obj {
+ match k {
+ 'name' { p.name = v.str() }
+ 'age' { p.age = v.int() }
+ 'pets' { p.pets = v.arr().map(it.str()) }
+ else {}
+ }
+ }
+}
+
+fn (p Person) to_json() string {
+ mut obj := map[string]json2.Any
+ obj['name'] = p.name
+ obj['age'] = p.age
+ obj['pets'] = p.pets
+ return obj.str()
+}
+
+fn main() {
+ resp := os.read_file('./person.json')?
+ person := json2.decode<Person>(resp)?
+ println(person) // Person{name: 'Bob', age: 28, pets: ['Floof']}
+ person_json := json2.encode<Person>(person)
+ println(person_json) // {"name": "Bob", "age": 28, "pets": ["Floof"]}
+}
+```
+
+## Using struct tags
+`x.json2` can access and use the struct field tags similar to the
+`json` module by using the comp-time `$for` for structs.
+
+```v ignore
+fn (mut p Person) from_json(f json2.Any) {
+ mp := an.as_map()
+ mut js_field_name := ''
+ $for field in Person.fields {
+ js_field_name = field.name
+
+ for attr in field.attrs {
+ if attr.starts_with('json:') {
+ js_field_name = attr.all_after('json:').trim_left(' ')
+ break
+ }
+ }
+
+ match field.name {
+ 'name' { p.name = mp[js_field_name].str() }
+ 'age' { u.age = mp[js_field_name].int() }
+ 'pets' { u.pets = mp[js_field_name].arr().map(it.str()) }
+ else {}
+ }
+ }
+}
+```
+
+### Null Values
+`x.json2` has a separate `null` type for differentiating an undefined value and a null value.
+To verify that the field you're accessing is a `null`, use `<typ> is json2.Null`.
+
+```v ignore
+fn (mut p Person) from_json(f json2.Any) {
+ obj := f.as_map()
+ if obj['age'] is json2.Null {
+ // use a default value
+ p.age = 10
+ }
+}
+```
+
+### Custom field names
+Aside from using struct tags, you can also just simply cast the base field into a map (`as_map()`)
+and access the field you wish to put into the struct/type.
+
+```v ignore
+fn (mut p Person) from_json(f json2.Any) {
+ obj := f.as_map()
+ p.name = obj['nickname'].str()
+}
+```
+
+```v oksyntax
+fn (mut p Person) to_json() string {
+ obj := f.as_map()
+ obj['nickname'] = p.name
+ return obj.str()
+}
+```
+
+### Undefined Values
+Getting undefined values has the same behavior as regular V types.
+If you're casting a base field into `map[string]json2.Any` and fetch an undefined entry/value,
+it simply returns empty. As for the `[]json2.Any`, it returns an index error.
+
+## Casting a value to an incompatible type
+`x.json2` provides methods for turning `Any` types into usable types.
+The following list shows the possible outputs when casting a value to an incompatible type.
+
+1. Casting non-array values as array (`arr()`) will return an array with the value as the content.
+2. Casting non-map values as map (`as_map()`) will return a map with the value as the content.
+3. Casting non-string values to string (`str()`) will return the
+JSON string representation of the value.
+4. Casting non-numeric values to int/float (`int()`/`i64()`/`f32()`/`f64()`) will return zero.
diff --git a/v_windows/v/old/vlib/x/json2/any_test.v b/v_windows/v/old/vlib/x/json2/any_test.v
new file mode 100644
index 0000000..6f86900
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/any_test.v
@@ -0,0 +1,130 @@
+import x.json2
+
+const (
+ sample_data = map{
+ 'int': json2.Any(int(1))
+ 'i64': json2.Any(i64(128))
+ 'f32': json2.Any(f32(2.0))
+ 'f64': json2.Any(f64(1.283))
+ 'bool': json2.Any(false)
+ 'str': json2.Any('test')
+ 'null': json2.Any(json2.null)
+ 'arr': json2.Any([json2.Any('lol')])
+ 'obj': json2.Any(map{
+ 'foo': json2.Any(10)
+ })
+ }
+)
+
+fn is_null(f json2.Any) bool {
+ match f {
+ json2.Null { return true }
+ else { return false }
+ }
+}
+
+fn test_f32() {
+ // valid conversions
+ assert sample_data['int'].f32() == 1.0
+ assert sample_data['i64'].f32() == 128.0
+ assert sample_data['f32'].f32() == 2.0
+ assert sample_data['f64'].f32() == 1.2829999923706055
+ // invalid conversions
+ assert sample_data['bool'].f32() == 0.0
+ assert sample_data['str'].f32() == 0.0
+ assert sample_data['null'].f32() == 0.0
+ assert sample_data['arr'].f32() == 0.0
+ assert sample_data['obj'].f32() == 0.0
+}
+
+fn test_f64() {
+ // valid conversions
+ assert sample_data['int'].f64() == 1.0
+ assert sample_data['i64'].f64() == 128.0
+ assert sample_data['f32'].f64() == 2.0
+ assert sample_data['f64'].f64() == 1.283
+ // invalid conversions
+ assert sample_data['bool'].f64() == 0.0
+ assert sample_data['str'].f64() == 0.0
+ assert sample_data['null'].f64() == 0.0
+ assert sample_data['arr'].f64() == 0.0
+ assert sample_data['obj'].f64() == 0.0
+}
+
+fn test_int() {
+ // valid conversions
+ assert sample_data['int'].int() == 1
+ assert sample_data['i64'].int() == 128
+ assert sample_data['f32'].int() == 2
+ assert sample_data['f64'].int() == 1
+ assert json2.Any(true).int() == 1
+ // invalid conversions
+ assert json2.Any('123').int() == 0
+ assert sample_data['null'].int() == 0
+ assert sample_data['arr'].int() == 0
+ assert sample_data['obj'].int() == 0
+}
+
+fn test_i64() {
+ // valid conversions
+ assert sample_data['int'].i64() == 1
+ assert sample_data['i64'].i64() == 128
+ assert sample_data['f32'].i64() == 2
+ assert sample_data['f64'].i64() == 1
+ assert json2.Any(true).i64() == 1
+ // invalid conversions
+ assert json2.Any('123').i64() == 0
+ assert sample_data['null'].i64() == 0
+ assert sample_data['arr'].i64() == 0
+ assert sample_data['obj'].i64() == 0
+}
+
+fn test_as_map() {
+ assert sample_data['int'].as_map()['0'].int() == 1
+ assert sample_data['i64'].as_map()['0'].i64() == 128.0
+ assert sample_data['f32'].as_map()['0'].f32() == 2.0
+ assert sample_data['f64'].as_map()['0'].f64() == 1.283
+ assert sample_data['bool'].as_map()['0'].bool() == false
+ assert sample_data['str'].as_map()['0'].str() == 'test'
+ assert is_null(sample_data['null'].as_map()['0']) == true
+ assert sample_data['arr'].as_map()['0'].str() == 'lol'
+ assert sample_data['obj'].as_map()['foo'].int() == 10
+}
+
+fn test_arr() {
+ assert sample_data['int'].arr()[0].int() == 1
+ assert sample_data['i64'].arr()[0].i64() == 128.0
+ assert sample_data['f32'].arr()[0].f32() == 2.0
+ assert sample_data['f64'].arr()[0].f64() == 1.283
+ assert sample_data['bool'].arr()[0].bool() == false
+ assert sample_data['str'].arr()[0].str() == 'test'
+ assert is_null(sample_data['null'].arr()[0]) == true
+ assert sample_data['arr'].arr()[0].str() == 'lol'
+ assert sample_data['obj'].arr()[0].int() == 10
+}
+
+fn test_bool() {
+ // valid conversions
+ assert sample_data['bool'].bool() == false
+ assert json2.Any('true').bool() == true
+ // invalid conversions
+ assert sample_data['int'].bool() == false
+ assert sample_data['i64'].bool() == false
+ assert sample_data['f32'].bool() == false
+ assert sample_data['f64'].bool() == false
+ assert sample_data['null'].bool() == false
+ assert sample_data['arr'].bool() == false
+ assert sample_data['obj'].bool() == false
+}
+
+fn test_str() {
+ assert sample_data['int'].str() == '1'
+ assert sample_data['i64'].str() == '128'
+ assert sample_data['f32'].str() == '2.0'
+ assert sample_data['f64'].str() == '1.283'
+ assert sample_data['bool'].str() == 'false'
+ assert sample_data['str'].str() == 'test'
+ assert sample_data['null'].str() == 'null'
+ assert sample_data['arr'].str() == '["lol"]'
+ assert sample_data.str() == '{"int":1,"i64":128,"f32":2.0,"f64":1.283,"bool":false,"str":"test","null":null,"arr":["lol"],"obj":{"foo":10}}'
+}
diff --git a/v_windows/v/old/vlib/x/json2/decoder.v b/v_windows/v/old/vlib/x/json2/decoder.v
new file mode 100644
index 0000000..a45a091
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/decoder.v
@@ -0,0 +1,200 @@
+// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module json2
+
+// `Any` is a sum type that lists the possible types to be decoded and used.
+pub type Any = Null | []Any | bool | f32 | f64 | i64 | int | map[string]Any | string |
+ u64
+
+// `Null` struct is a simple representation of the `null` value in JSON.
+pub struct Null {
+ is_null bool = true
+}
+
+struct Parser {
+mut:
+ scanner &Scanner
+ p_tok Token
+ tok Token
+ n_tok Token
+ n_level int
+ convert_type bool = true
+}
+
+struct InvalidTokenError {
+ msg string
+ code int
+}
+
+struct UnknownTokenError {
+ msg string
+ code int
+}
+
+fn (mut p Parser) next() {
+ p.p_tok = p.tok
+ p.tok = p.n_tok
+ p.n_tok = p.scanner.scan()
+}
+
+fn (mut p Parser) next_with_err() ? {
+ p.next()
+ if p.tok.kind == .error {
+ return error(p.emit_error(p.tok.lit.bytestr()))
+ }
+}
+
+fn (p Parser) emit_error(msg string) string {
+ line := p.tok.line
+ column := p.tok.col + p.tok.lit.len
+ return '[x.json2] $msg ($line:$column)'
+}
+
+// TODO: copied from v.util to avoid the entire module and its functions
+// from being imported. remove later once -skip-unused is enabled by default.
+fn skip_bom(file_content string) string {
+ mut raw_text := file_content
+ // BOM check
+ if raw_text.len >= 3 {
+ unsafe {
+ c_text := raw_text.str
+ if c_text[0] == 0xEF && c_text[1] == 0xBB && c_text[2] == 0xBF {
+ // skip three BOM bytes
+ offset_from_begin := 3
+ raw_text = tos(c_text[offset_from_begin], vstrlen(c_text) - offset_from_begin)
+ }
+ }
+ }
+ return raw_text
+}
+
+fn new_parser(srce string, convert_type bool) Parser {
+ src := skip_bom(srce)
+ return Parser{
+ scanner: &Scanner{
+ text: src.bytes()
+ }
+ convert_type: convert_type
+ }
+}
+
+fn (mut p Parser) decode() ?Any {
+ p.next()
+ p.next_with_err() ?
+ fi := p.decode_value() ?
+ if p.tok.kind != .eof {
+ return IError(&InvalidTokenError{
+ msg: p.emit_error('invalid token `$p.tok.kind`')
+ })
+ }
+ return fi
+}
+
+fn (mut p Parser) decode_value() ?Any {
+ if p.n_level + 1 == 500 {
+ return error(p.emit_error('reached maximum nesting level of 500'))
+ }
+ match p.tok.kind {
+ .lsbr {
+ return p.decode_array()
+ }
+ .lcbr {
+ return p.decode_object()
+ }
+ .int_, .float {
+ tl := p.tok.lit.bytestr()
+ kind := p.tok.kind
+ p.next_with_err() ?
+ if p.convert_type {
+ if kind == .float {
+ return Any(tl.f64())
+ }
+ return Any(tl.i64())
+ }
+ return Any(tl)
+ }
+ .bool_ {
+ lit := p.tok.lit.bytestr()
+ p.next_with_err() ?
+ if p.convert_type {
+ return Any(lit.bool())
+ }
+ return Any(lit)
+ }
+ .null {
+ p.next_with_err() ?
+ if p.convert_type {
+ return Any(null)
+ }
+ return Any('null')
+ }
+ .str_ {
+ str := p.tok.lit.bytestr()
+ p.next_with_err() ?
+ return Any(str)
+ }
+ else {
+ return IError(&InvalidTokenError{
+ msg: p.emit_error('invalid token `$p.tok.kind`')
+ })
+ }
+ }
+ return Any(null)
+}
+
+fn (mut p Parser) decode_array() ?Any {
+ mut items := []Any{}
+ p.next_with_err() ?
+ p.n_level++
+ for p.tok.kind != .rsbr {
+ item := p.decode_value() ?
+ items << item
+ if p.tok.kind == .comma {
+ p.next_with_err() ?
+ if p.tok.kind == .rsbr || p.tok.kind == .rcbr {
+ return IError(&InvalidTokenError{
+ msg: p.emit_error('invalid token `$p.tok.lit')
+ })
+ }
+ } else if p.tok.kind == .rsbr {
+ break
+ } else {
+ return IError(&UnknownTokenError{
+ msg: p.emit_error("unknown token '$p.tok.lit' when decoding array.")
+ })
+ }
+ }
+ p.next_with_err() ?
+ p.n_level--
+ return Any(items)
+}
+
+fn (mut p Parser) decode_object() ?Any {
+ mut fields := map[string]Any{}
+ p.next_with_err() ?
+ p.n_level++
+ for p.tok.kind != .rcbr {
+ is_key := p.tok.kind == .str_ && p.n_tok.kind == .colon
+ if !is_key {
+ return IError(&InvalidTokenError{
+ msg: p.emit_error('invalid token `$p.tok.kind`, expecting `str_`')
+ })
+ }
+ cur_key := p.tok.lit.bytestr()
+ p.next_with_err() ?
+ p.next_with_err() ?
+ fields[cur_key] = p.decode_value() ?
+ if p.tok.kind == .comma {
+ p.next_with_err() ?
+ if p.tok.kind != .str_ {
+ return IError(&UnknownTokenError{
+ msg: p.emit_error("unknown token '$p.tok.lit' when decoding object.")
+ })
+ }
+ }
+ }
+ p.next_with_err() ?
+ p.n_level--
+ return Any(fields)
+}
diff --git a/v_windows/v/old/vlib/x/json2/decoder_test.v b/v_windows/v/old/vlib/x/json2/decoder_test.v
new file mode 100644
index 0000000..f80f8b2
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/decoder_test.v
@@ -0,0 +1,61 @@
+module json2
+
+fn test_raw_decode_string() ? {
+ str := raw_decode('"Hello!"') ?
+ assert str.str() == 'Hello!'
+}
+
+fn test_raw_decode_number() ? {
+ num := raw_decode('123') ?
+ assert num.int() == 123
+}
+
+fn test_raw_decode_array() ? {
+ raw_arr := raw_decode('["Foo", 1]') ?
+ arr := raw_arr.arr()
+ assert arr[0].str() == 'Foo'
+ assert arr[1].int() == 1
+}
+
+fn test_raw_decode_bool() ? {
+ bol := raw_decode('false') ?
+ assert bol.bool() == false
+}
+
+fn test_raw_decode_map() ? {
+ raw_mp := raw_decode('{"name":"Bob","age":20}') ?
+ mp := raw_mp.as_map()
+ assert mp['name'].str() == 'Bob'
+ assert mp['age'].int() == 20
+}
+
+fn test_raw_decode_null() ? {
+ nul := raw_decode('null') ?
+ assert nul is Null
+}
+
+fn test_raw_decode_invalid() ? {
+ raw_decode('1z') or {
+ assert err.msg == '[x.json2] invalid token `z` (0:17)'
+ return
+ }
+ assert false
+}
+
+fn test_raw_decode_string_with_dollarsign() ? {
+ str := raw_decode(r'"Hello $world"') ?
+ assert str.str() == r'Hello $world'
+}
+
+fn test_raw_decode_map_with_whitespaces() ? {
+ raw_mp := raw_decode(' \n\t{"name":"Bob","age":20}\n\t') ?
+ mp := raw_mp.as_map()
+ assert mp['name'].str() == 'Bob'
+ assert mp['age'].int() == 20
+}
+
+fn test_nested_array_object() ? {
+ mut parser := new_parser(r'[[[[[],[],[]]]],{"Test":{}},[[]]]', false)
+ decoded := parser.decode() ?
+ assert parser.n_level == 0
+}
diff --git a/v_windows/v/old/vlib/x/json2/encoder.v b/v_windows/v/old/vlib/x/json2/encoder.v
new file mode 100644
index 0000000..b1ca0e4
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/encoder.v
@@ -0,0 +1,179 @@
+// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module json2
+
+import strings
+
+fn write_value(v Any, i int, len int, mut wr strings.Builder) {
+ str := v.json_str()
+ if v is string {
+ wr.write_string('"$str"')
+ } else {
+ wr.write_string(str)
+ }
+ if i >= len - 1 {
+ return
+ }
+ wr.write_b(`,`)
+}
+
+// str returns the string representation of the `map[string]Any`.
+pub fn (flds map[string]Any) str() string {
+ mut wr := strings.new_builder(200)
+ wr.write_b(`{`)
+ mut i := 0
+ for k, v in flds {
+ wr.write_string('"$k":')
+ write_value(v, i, flds.len, mut wr)
+ i++
+ }
+ wr.write_b(`}`)
+ defer {
+ unsafe { wr.free() }
+ }
+ res := wr.str()
+ return res
+}
+
+// str returns the string representation of the `[]Any`.
+pub fn (flds []Any) str() string {
+ mut wr := strings.new_builder(200)
+ wr.write_b(`[`)
+ for i, v in flds {
+ write_value(v, i, flds.len, mut wr)
+ }
+ wr.write_b(`]`)
+ defer {
+ unsafe { wr.free() }
+ }
+ res := wr.str()
+ return res
+}
+
+// str returns the string representation of the `Any` type. Use the `json_str` method
+// if you want to use the escaped str() version of the `Any` type.
+pub fn (f Any) str() string {
+ if f is string {
+ return f
+ } else {
+ return f.json_str()
+ }
+}
+
+// json_str returns the JSON string representation of the `Any` type.
+pub fn (f Any) json_str() string {
+ match f {
+ string {
+ return json_string(f)
+ }
+ int {
+ return f.str()
+ }
+ u64, i64 {
+ return f.str()
+ }
+ f32 {
+ str_f32 := f.str()
+ if str_f32.ends_with('.') {
+ return '${str_f32}0'
+ }
+ return str_f32
+ }
+ f64 {
+ str_f64 := f.str()
+ if str_f64.ends_with('.') {
+ return '${str_f64}0'
+ }
+ return str_f64
+ }
+ bool {
+ return f.str()
+ }
+ map[string]Any {
+ return f.str()
+ }
+ []Any {
+ return f.str()
+ }
+ Null {
+ return 'null'
+ }
+ }
+}
+
+// char_len_list is a modified version of builtin.utf8_str_len
+// that returns an array of character lengths. (e.g "t✔" => [1,2])
+fn char_len_list(s string) []int {
+ mut l := 1
+ mut ls := []int{}
+ for i := 0; i < s.len; i++ {
+ c := s[i]
+ if (c & (1 << 7)) != 0 {
+ for t := byte(1 << 6); (c & t) != 0; t >>= 1 {
+ l++
+ i++
+ }
+ }
+ ls << l
+ l = 1
+ }
+ return ls
+}
+
+const escaped_chars = [r'\b', r'\f', r'\n', r'\r', r'\t']
+
+// json_string returns the JSON spec-compliant version of the string.
+[manualfree]
+fn json_string(s string) string {
+ // not the best implementation but will revisit it soon
+ char_lens := char_len_list(s)
+ mut sb := strings.new_builder(s.len)
+ mut i := 0
+ defer {
+ unsafe {
+ char_lens.free()
+ // freeing string builder on defer after
+ // returning .str() still isn't working :(
+ // sb.free()
+ }
+ }
+ for char_len in char_lens {
+ if char_len == 1 {
+ chr := s[i]
+ if chr in important_escapable_chars {
+ for j := 0; j < important_escapable_chars.len; j++ {
+ if chr == important_escapable_chars[j] {
+ sb.write_string(json2.escaped_chars[j])
+ break
+ }
+ }
+ } else if chr == `"` || chr == `/` || chr == `\\` {
+ sb.write_string('\\' + chr.ascii_str())
+ } else {
+ sb.write_b(chr)
+ }
+ } else {
+ slice := s[i..i + char_len]
+ hex_code := slice.utf32_code().hex()
+ if hex_code.len < 4 {
+ // an utf8 codepoint
+ sb.write_string(slice)
+ } else if hex_code.len == 4 {
+ sb.write_string('\\u$hex_code')
+ } else {
+ // TODO: still figuring out what
+ // to do with more than 4 chars
+ sb.write_b(` `)
+ }
+ unsafe {
+ slice.free()
+ hex_code.free()
+ }
+ }
+ i += char_len
+ }
+ str := sb.str()
+ unsafe { sb.free() }
+ return str
+}
diff --git a/v_windows/v/old/vlib/x/json2/encoder_test.v b/v_windows/v/old/vlib/x/json2/encoder_test.v
new file mode 100644
index 0000000..8135172
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/encoder_test.v
@@ -0,0 +1,29 @@
+import x.json2
+
+fn test_json_string_characters() {
+ text := json2.raw_decode(r'"\n\r\b\f\t\\\"\/"') or { '' }
+ assert text.json_str() == '\\n\\r\\b\\f\\t\\\\\\"\\/'
+}
+
+fn test_json_string() {
+ text := json2.Any('te✔st')
+ assert text.json_str() == r'te\u2714st'
+}
+
+fn test_json_string_emoji() {
+ text := json2.Any('🐈')
+ assert text.json_str() == r' '
+}
+
+fn test_json_string_non_ascii() {
+ text := json2.Any('ひらがな')
+ assert text.json_str() == r'\u3072\u3089\u304c\u306a'
+}
+
+fn test_utf8_strings_are_not_modified() ? {
+ original := '{"s":"Schilddrüsenerkrankungen"}'
+ // dump(original)
+ deresult := json2.raw_decode(original) ?
+ // dump(deresult)
+ assert deresult.str() == original
+}
diff --git a/v_windows/v/old/vlib/x/json2/json2.v b/v_windows/v/old/vlib/x/json2/json2.v
new file mode 100644
index 0000000..0e5012c
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/json2.v
@@ -0,0 +1,122 @@
+// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module json2
+
+pub const (
+ null = Null{}
+)
+
+pub interface Serializable {
+ from_json(f Any)
+ to_json() string
+}
+
+// Decodes a JSON string into an `Any` type. Returns an option.
+pub fn raw_decode(src string) ?Any {
+ mut p := new_parser(src, true)
+ return p.decode()
+}
+
+// Same with `raw_decode`, but skips the type conversion for certain types when decoding a certain value.
+pub fn fast_raw_decode(src string) ?Any {
+ mut p := new_parser(src, false)
+ return p.decode()
+}
+
+// decode is a generic function that decodes a JSON string into the target type.
+pub fn decode<T>(src string) ?T {
+ res := raw_decode(src) ?
+ mut typ := T{}
+ typ.from_json(res)
+ return typ
+}
+
+// encode is a generic function that encodes a type into a JSON string.
+pub fn encode<T>(typ T) string {
+ return typ.to_json()
+}
+
+// as_map uses `Any` as a map.
+pub fn (f Any) as_map() map[string]Any {
+ if f is map[string]Any {
+ return f
+ } else if f is []Any {
+ mut mp := map[string]Any{}
+ for i, fi in f {
+ mp['$i'] = fi
+ }
+ return mp
+ }
+ return map{
+ '0': f
+ }
+}
+
+// int uses `Any` as an integer.
+pub fn (f Any) int() int {
+ match f {
+ int { return f }
+ i64, f32, f64, bool { return int(f) }
+ else { return 0 }
+ }
+}
+
+// i64 uses `Any` as a 64-bit integer.
+pub fn (f Any) i64() i64 {
+ match f {
+ i64 { return f }
+ int, f32, f64, bool { return i64(f) }
+ else { return 0 }
+ }
+}
+
+// u64 uses `Any` as a 64-bit unsigned integer.
+pub fn (f Any) u64() u64 {
+ match f {
+ u64 { return f }
+ int, i64, f32, f64, bool { return u64(f) }
+ else { return 0 }
+ }
+}
+
+// f32 uses `Any` as a 32-bit float.
+pub fn (f Any) f32() f32 {
+ match f {
+ f32 { return f }
+ int, i64, f64 { return f32(f) }
+ else { return 0.0 }
+ }
+}
+
+// f64 uses `Any` as a float.
+pub fn (f Any) f64() f64 {
+ match f {
+ f64 { return f }
+ int, i64, f32 { return f64(f) }
+ else { return 0.0 }
+ }
+}
+
+// arr uses `Any` as an array.
+pub fn (f Any) arr() []Any {
+ if f is []Any {
+ return f
+ } else if f is map[string]Any {
+ mut arr := []Any{}
+ for _, v in f {
+ arr << v
+ }
+ return arr
+ }
+ return [f]
+}
+
+// bool uses `Any` as a bool
+pub fn (f Any) bool() bool {
+ match f {
+ bool { return f }
+ string { return f.bool() }
+ else { return false }
+ }
+}
diff --git a/v_windows/v/old/vlib/x/json2/json2_test.v b/v_windows/v/old/vlib/x/json2/json2_test.v
new file mode 100644
index 0000000..b808f42
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/json2_test.v
@@ -0,0 +1,398 @@
+import x.json2
+
+enum JobTitle {
+ manager
+ executive
+ worker
+}
+
+struct Employee {
+pub mut:
+ name string
+ age int
+ salary f32
+ title JobTitle
+}
+
+fn (e Employee) to_json() string {
+ mut mp := map[string]json2.Any{}
+ mp['name'] = e.name
+ mp['age'] = e.age
+ mp['salary'] = e.salary
+ mp['title'] = int(e.title)
+ /*
+ $for field in Employee.fields {
+ d := e.$(field.name)
+
+ $if field.typ is JobTitle {
+ mp[field.name] = json.encode<int>(d)
+ } $else {
+ mp[field.name] = d
+ }
+ }
+ */
+ return mp.str()
+}
+
+fn (mut e Employee) from_json(any json2.Any) {
+ mp := any.as_map()
+ e.name = mp['name'].str()
+ e.age = mp['age'].int()
+ e.salary = mp['salary'].f32()
+ e.title = JobTitle(mp['title'].int())
+}
+
+fn test_simple() {
+ x := Employee{'Peter', 28, 95000.5, .worker}
+ s := json2.encode<Employee>(x)
+ eprintln('Employee x: $s')
+ assert s == '{"name":"Peter","age":28,"salary":95000.5,"title":2}'
+ y := json2.decode<Employee>(s) or {
+ println(err)
+ assert false
+ return
+ }
+ eprintln('Employee y: $y')
+ assert y.name == 'Peter'
+ assert y.age == 28
+ assert y.salary == 95000.5
+ assert y.title == .worker
+}
+
+fn test_fast_raw_decode() {
+ s := '{"name":"Peter","age":28,"salary":95000.5,"title":2}'
+ o := json2.fast_raw_decode(s) or {
+ assert false
+ json2.Any(json2.null)
+ }
+ str := o.str()
+ assert str == '{"name":"Peter","age":"28","salary":"95000.5","title":"2"}'
+}
+
+fn test_character_unescape() {
+ message := r'{
+ "newline": "new\nline",
+ "tab": "\ttab",
+ "backslash": "back\\slash",
+ "quotes": "\"quotes\"",
+ "slash":"\/dev\/null"
+}'
+ mut obj := json2.raw_decode(message) or {
+ println(err)
+ assert false
+ return
+ }
+ lines := obj.as_map()
+ eprintln('$lines')
+ assert lines['newline'].str() == 'new\nline'
+ assert lines['tab'].str() == '\ttab'
+ assert lines['backslash'].str() == 'back\\slash'
+ assert lines['quotes'].str() == '"quotes"'
+ assert lines['slash'].str() == '/dev/null'
+}
+
+struct User2 {
+pub mut:
+ age int
+ nums []int
+}
+
+fn (mut u User2) from_json(an json2.Any) {
+ mp := an.as_map()
+ mut js_field_name := ''
+ $for field in User.fields {
+ js_field_name = field.name
+ for attr in field.attrs {
+ if attr.starts_with('json:') {
+ js_field_name = attr.all_after('json:').trim_left(' ')
+ break
+ }
+ }
+ match field.name {
+ 'age' { u.age = mp[js_field_name].int() }
+ 'nums' { u.nums = mp[js_field_name].arr().map(it.int()) }
+ else {}
+ }
+ }
+}
+
+// User struct needs to be `pub mut` for now in order to access and manipulate values
+struct User {
+pub mut:
+ age int
+ nums []int
+ last_name string [json: lastName]
+ is_registered bool [json: IsRegistered]
+ typ int [json: 'type']
+ pets string [json: 'pet_animals'; raw]
+}
+
+fn (mut u User) from_json(an json2.Any) {
+ mp := an.as_map()
+ mut js_field_name := ''
+ $for field in User.fields {
+ // FIXME: C error when initializing js_field_name inside comptime for
+ js_field_name = field.name
+ for attr in field.attrs {
+ if attr.starts_with('json:') {
+ js_field_name = attr.all_after('json:').trim_left(' ')
+ break
+ }
+ }
+ match field.name {
+ 'age' { u.age = mp[js_field_name].int() }
+ 'nums' { u.nums = mp[js_field_name].arr().map(it.int()) }
+ 'last_name' { u.last_name = mp[js_field_name].str() }
+ 'is_registered' { u.is_registered = mp[js_field_name].bool() }
+ 'typ' { u.typ = mp[js_field_name].int() }
+ 'pets' { u.pets = mp[js_field_name].str() }
+ else {}
+ }
+ }
+}
+
+fn (u User) to_json() string {
+ // TODO: derive from field
+ mut mp := map{
+ 'age': json2.Any(u.age)
+ }
+ mp['nums'] = u.nums.map(json2.Any(it))
+ mp['lastName'] = u.last_name
+ mp['IsRegistered'] = u.is_registered
+ mp['type'] = u.typ
+ mp['pet_animals'] = u.pets
+ return mp.str()
+}
+
+fn test_parse_user() {
+ s := '{"age": 10, "nums": [1,2,3], "type": 1, "lastName": "Johnson", "IsRegistered": true, "pet_animals": {"name": "Bob", "animal": "Dog"}}'
+ u2 := json2.decode<User2>(s) or {
+ println(err)
+ assert false
+ return
+ }
+ println(u2)
+ u := json2.decode<User>(s) or {
+ println(err)
+ assert false
+ return
+ }
+ assert u.age == 10
+ assert u.last_name == 'Johnson'
+ assert u.is_registered == true
+ assert u.nums.len == 3
+ assert u.nums[0] == 1
+ assert u.nums[1] == 2
+ assert u.nums[2] == 3
+ assert u.typ == 1
+ assert u.pets == '{"name":"Bob","animal":"Dog"}'
+}
+
+fn test_encode_user() {
+ usr := User{
+ age: 10
+ nums: [1, 2, 3]
+ last_name: 'Johnson'
+ is_registered: true
+ typ: 0
+ pets: 'foo'
+ }
+ expected := '{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"foo"}'
+ out := json2.encode<User>(usr)
+ assert out == expected
+}
+
+struct Color {
+pub mut:
+ space string
+ point string [raw]
+}
+
+fn (mut c Color) from_json(an json2.Any) {
+ mp := an.as_map()
+ $for field in Color.fields {
+ match field.name {
+ 'space' { c.space = mp[field.name].str() }
+ 'point' { c.point = mp[field.name].str() }
+ else {}
+ }
+ }
+}
+
+fn test_raw_json_field() {
+ color := json2.decode<Color>('{"space": "YCbCr", "point": {"Y": 123}}') or {
+ assert false
+ Color{}
+ }
+ assert color.point == '{"Y":123}'
+ assert color.space == 'YCbCr'
+}
+
+/*
+struct City {
+ name string
+}
+
+struct Country {
+ cities []City
+ name string
+}
+
+fn test_struct_in_struct() {
+ country := json.decode(Country, '{ "name": "UK", "cities": [{"name":"London"}, {"name":"Manchester"}]}') or {
+ assert false
+ exit(1)
+ }
+ assert country.name == 'UK'
+ assert country.cities.len == 2
+ assert country.cities[0].name == 'London'
+ assert country.cities[1].name == 'Manchester'
+ println(country.cities)
+}
+*/
+fn test_encode_map() {
+ expected := '{"one":1,"two":2,"three":3,"four":4}'
+ numbers := map{
+ 'one': json2.Any(1)
+ 'two': json2.Any(2)
+ 'three': json2.Any(3)
+ 'four': json2.Any(4)
+ }
+ out := numbers.str()
+ assert out == expected
+}
+
+/*
+fn test_parse_map() {
+ expected := {
+ 'one': 1
+ 'two': 2
+ 'three': 3
+ 'four': 4
+ }
+ out := json.decode<map[string]int>('{"one":1,"two":2,"three":3,"four":4}') or {
+ assert false
+ r := {
+ '': 0
+ }
+ r
+ }
+ println(out)
+ assert out == expected
+}
+
+struct Data {
+ countries []Country
+ users map[string]User
+ extra map[string]map[string]int
+}
+
+fn test_nested_type() {
+ data_expected := '{"countries":[{"cities":[{"name":"London"},{"name":"Manchester"}],"name":"UK"},{"cities":[{"name":"Donlon"},{"name":"Termanches"}],"name":"KU"}],"users":{"Foo":{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"little foo"},"Boo":{"age":20,"nums":[5,3,1],"lastName":"Smith","IsRegistered":false,"type":4,"pet_animals":"little boo"}},"extra":{"2":{"n1":2,"n2":4,"n3":8,"n4":16},"3":{"n1":3,"n2":9,"n3":27,"n4":81}}}'
+
+ data := Data{
+ countries: [
+ Country{
+ name: 'UK'
+ cities: [City{'London'},
+ City{'Manchester'},
+ ]
+ },
+ Country{
+ name: 'KU'
+ cities: [City{'Donlon'},
+ City{'Termanches'},
+ ]
+ },
+ ]
+ users: {
+ 'Foo': User{
+ age: 10
+ nums: [1, 2, 3]
+ last_name: 'Johnson'
+ is_registered: true
+ typ: 0
+ pets: 'little foo'
+ },
+ 'Boo': User{
+ age: 20
+ nums: [5, 3, 1]
+ last_name: 'Smith'
+ is_registered: false
+ typ: 4
+ pets: 'little boo'
+ }
+ },
+ extra: {
+ '2': {
+ 'n1': 2
+ 'n2': 4
+ 'n3': 8
+ 'n4': 16
+ },
+ '3': {
+ 'n1': 3
+ 'n2': 9
+ 'n3': 27
+ 'n4': 81
+ },
+ }
+ }
+ out := json.encode(data)
+ println(out)
+ assert out == data_expected
+
+ data2 := json.decode(Data, data_expected) or {
+ assert false
+ Data{}
+ }
+ assert data2.countries.len == data.countries.len
+ for i in 0..1 {
+ assert data2.countries[i].name == data.countries[i].name
+ assert data2.countries[i].cities.len == data.countries[i].cities.len
+ for j in 0..1 {
+ assert data2.countries[i].cities[j].name == data.countries[i].cities[j].name
+ }
+ }
+
+ for key, user in data.users {
+ assert data2.users[key].age == user.age
+ assert data2.users[key].nums == user.nums
+ assert data2.users[key].last_name == user.last_name
+ assert data2.users[key].is_registered == user.is_registered
+ assert data2.users[key].typ == user.typ
+ // assert data2.users[key].pets == user.pets // TODO FIX
+ }
+
+ for k, v in data.extra {
+ for k2, v2 in v {
+ assert data2.extra[k][k2] == v2
+ }
+ }
+}
+
+fn test_errors() {
+ invalid_array := fn () {
+ data := '{"countries":[{"cities":[{"name":"London"},{"name":"Manchester"}],"name":"UK"},{"cities":{"name":"Donlon"},"name":"KU"}],"users":{"Foo":{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"little foo"},"Boo":{"age":20,"nums":[5,3,1],"lastName":"Smith","IsRegistered":false,"type":4,"pet_animals":"little boo"}},"extra":{"2":{"n1":2,"n2":4,"n3":8,"n4":16},"3":{"n1":3,"n2":9,"n3":27,"n4":81}}}'
+
+ json.decode(Data, data) or {
+ println(err)
+ assert err.starts_with('Json element is not an array:')
+ return
+ }
+ assert false
+ }
+ invalid_object := fn() {
+ data := '{"countries":[{"cities":[{"name":"London"},{"name":"Manchester"}],"name":"UK"},{"cities":[{"name":"Donlon"},{"name":"Termanches"}],"name":"KU"}],"users":[{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"little foo"},{"age":20,"nums":[5,3,1],"lastName":"Smith","IsRegistered":false,"type":4,"pet_animals":"little boo"}],"extra":{"2":{"n1":2,"n2":4,"n3":8,"n4":16},"3":{"n1":3,"n2":9,"n3":27,"n4":81}}}'
+
+ json.decode(Data, data) or {
+ println(err)
+ assert err.starts_with('Json element is not an object:')
+ return
+ }
+ assert false
+ }
+ invalid_array()
+ invalid_object()
+}
+*/
diff --git a/v_windows/v/old/vlib/x/json2/scanner.v b/v_windows/v/old/vlib/x/json2/scanner.v
new file mode 100644
index 0000000..473a83b
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/scanner.v
@@ -0,0 +1,306 @@
+// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module json2
+
+import strconv
+
+struct Scanner {
+mut:
+ text []byte
+ pos int
+ line int
+ col int
+}
+
+enum TokenKind {
+ none_
+ error
+ str_
+ float
+ int_
+ null
+ bool_
+ eof
+ comma = 44
+ colon = 58
+ lsbr = 91
+ rsbr = 93
+ lcbr = 123
+ rcbr = 125
+}
+
+struct Token {
+ lit []byte
+ kind TokenKind
+ line int
+ col int
+}
+
+const (
+ // list of characters commonly used in JSON.
+ char_list = [`{`, `}`, `[`, `]`, `,`, `:`]
+ // list of newlines to check when moving to a new position.
+ newlines = [`\r`, `\n`, `\t`]
+ // list of escapable that needs to be escaped inside a JSON string.
+ // double quotes and forward slashes are excluded intentionally since
+ // they have their own separate checks for it in order to pass the
+ // JSON test suite (https://github.com/nst/JSONTestSuite/).
+ important_escapable_chars = [`\b`, `\f`, `\n`, `\r`, `\t`]
+ // list of valid unicode escapes aside from \u{4-hex digits}
+ valid_unicode_escapes = [`b`, `f`, `n`, `r`, `t`, `\\`, `"`, `/`]
+ // used for transforming escapes into valid unicode (eg. n => \n)
+ unicode_transform_escapes = map{
+ 98: `\b`
+ 102: `\f`
+ 110: `\n`
+ 114: `\r`
+ 116: `\t`
+ 92: `\\`
+ 34: `"`
+ 47: `/`
+ }
+ exp_signs = [byte(`-`), `+`]
+)
+
+// move_pos proceeds to the next position.
+fn (mut s Scanner) move() {
+ s.move_pos(true, true)
+}
+
+// move_pos_with_newlines is the same as move_pos but only enables newline checking.
+fn (mut s Scanner) move_pos_with_newlines() {
+ s.move_pos(false, true)
+}
+
+fn (mut s Scanner) move_pos(include_space bool, include_newlines bool) {
+ s.pos++
+ if s.pos < s.text.len {
+ if include_newlines && s.text[s.pos] in json2.newlines {
+ s.line++
+ s.col = 0
+ if s.text[s.pos] == `\r` && s.pos + 1 < s.text.len && s.text[s.pos + 1] == `\n` {
+ s.pos++
+ }
+ for s.pos < s.text.len && s.text[s.pos] in json2.newlines {
+ s.move()
+ }
+ } else if include_space && s.text[s.pos] == ` ` {
+ s.pos++
+ s.col++
+ for s.pos < s.text.len && s.text[s.pos] == ` ` {
+ s.move()
+ }
+ }
+ } else {
+ s.col++
+ }
+}
+
+// error returns an error token.
+fn (s Scanner) error(description string) Token {
+ return s.tokenize(description.bytes(), .error)
+}
+
+// tokenize returns a token based on the given lit and kind.
+fn (s Scanner) tokenize(lit []byte, kind TokenKind) Token {
+ return Token{
+ lit: lit
+ kind: kind
+ col: s.col
+ line: s.line
+ }
+}
+
+// text_scan scans and returns a string token.
+[manualfree]
+fn (mut s Scanner) text_scan() Token {
+ mut has_closed := false
+ mut chrs := []byte{}
+ for {
+ s.pos++
+ s.col++
+ if s.pos >= s.text.len {
+ break
+ }
+ ch := s.text[s.pos]
+ if (s.pos - 1 >= 0 && s.text[s.pos - 1] != `\\`) && ch == `"` {
+ has_closed = true
+ break
+ } else if (s.pos - 1 >= 0 && s.text[s.pos - 1] != `\\`)
+ && ch in json2.important_escapable_chars {
+ return s.error('character must be escaped with a backslash')
+ } else if (s.pos == s.text.len - 1 && ch == `\\`) || ch == byte(0) {
+ return s.error('invalid backslash escape')
+ } else if s.pos + 1 < s.text.len && ch == `\\` {
+ peek := s.text[s.pos + 1]
+ if peek in json2.valid_unicode_escapes {
+ chrs << json2.unicode_transform_escapes[int(peek)]
+ s.pos++
+ s.col++
+ continue
+ } else if peek == `u` {
+ if s.pos + 5 < s.text.len {
+ s.pos++
+ s.col++
+ mut codepoint := []byte{}
+ codepoint_start := s.pos
+ for s.pos < s.text.len && s.pos < codepoint_start + 4 {
+ s.pos++
+ s.col++
+ if s.text[s.pos] == `"` {
+ break
+ } else if !s.text[s.pos].is_hex_digit() {
+ x := s.text[s.pos].ascii_str()
+ return s.error('`$x` is not a hex digit')
+ }
+ codepoint << s.text[s.pos]
+ }
+ if codepoint.len != 4 {
+ return s.error('unicode escape must have 4 hex digits')
+ }
+ val := u32(strconv.parse_uint(codepoint.bytestr(), 16, 32) or { 0 })
+ converted := utf32_to_str(val)
+ converted_bytes := converted.bytes()
+ chrs << converted_bytes
+ unsafe {
+ converted.free()
+ converted_bytes.free()
+ codepoint.free()
+ }
+ continue
+ } else {
+ return s.error('incomplete unicode escape')
+ }
+ } else if peek == `U` {
+ return s.error('unicode endpoints must be in lowercase `u`')
+ } else if peek == byte(229) {
+ return s.error('unicode endpoint not allowed')
+ } else {
+ return s.error('invalid backslash escape')
+ }
+ }
+ chrs << ch
+ }
+ tok := s.tokenize(chrs, .str_)
+ s.move()
+ if !has_closed {
+ return s.error('missing double quotes in string closing')
+ }
+ return tok
+}
+
+// num_scan scans and returns an int/float token.
+fn (mut s Scanner) num_scan() Token {
+ // analyze json number structure
+ // -[digit][?[dot][digit]][?[E/e][?-/+][digit]]
+ mut is_fl := false
+ mut dot_index := -1
+ mut digits := []byte{}
+ if s.text[s.pos] == `-` {
+ digits << `-`
+ if !s.text[s.pos + 1].is_digit() {
+ return s.invalid_token()
+ }
+ s.move_pos_with_newlines()
+ }
+ if s.text[s.pos] == `0` && (s.pos + 1 < s.text.len && s.text[s.pos + 1].is_digit()) {
+ return s.error('leading zeroes in a number are not allowed')
+ }
+ for s.pos < s.text.len && (s.text[s.pos].is_digit() || (!is_fl && s.text[s.pos] == `.`)) {
+ digits << s.text[s.pos]
+ if s.text[s.pos] == `.` {
+ is_fl = true
+ dot_index = digits.len - 1
+ }
+ s.move_pos_with_newlines()
+ }
+ if dot_index + 1 < s.text.len && digits[dot_index + 1..].len == 0 {
+ return s.error('invalid float')
+ }
+ if s.pos < s.text.len && (s.text[s.pos] == `e` || s.text[s.pos] == `E`) {
+ digits << s.text[s.pos]
+ s.move_pos_with_newlines()
+ if s.pos < s.text.len && s.text[s.pos] in json2.exp_signs {
+ digits << s.text[s.pos]
+ s.move_pos_with_newlines()
+ }
+ mut exp_digits_count := 0
+ for s.pos < s.text.len && s.text[s.pos].is_digit() {
+ digits << s.text[s.pos]
+ exp_digits_count++
+ s.move_pos_with_newlines()
+ }
+ if exp_digits_count == 0 {
+ return s.error('invalid exponent')
+ }
+ }
+ kind := if is_fl { TokenKind.float } else { TokenKind.int_ }
+ return s.tokenize(digits, kind)
+}
+
+// invalid_token returns an error token with the invalid token message.
+fn (s Scanner) invalid_token() Token {
+ if s.text[s.pos] >= 32 && s.text[s.pos] <= 126 {
+ x := s.text[s.pos].ascii_str()
+ return s.error('invalid token `$x`')
+ } else {
+ x := s.text[s.pos].str_escaped()
+ return s.error('invalid token `$x`')
+ }
+}
+
+// scan returns a token based on the scanner's current position.
+[manualfree]
+fn (mut s Scanner) scan() Token {
+ if s.pos < s.text.len && (s.text[s.pos] == ` ` || s.text[s.pos] in json2.newlines) {
+ s.move()
+ }
+ if s.pos >= s.text.len {
+ return s.tokenize([]byte{}, .eof)
+ } else if s.pos + 3 < s.text.len && (s.text[s.pos] == `t` || s.text[s.pos] == `n`) {
+ ident := s.text[s.pos..s.pos + 4].bytestr()
+ if ident == 'true' || ident == 'null' {
+ mut kind := TokenKind.null
+ if ident == 'true' {
+ kind = .bool_
+ }
+ unsafe { ident.free() }
+ val := s.text[s.pos..s.pos + 4]
+ tok := s.tokenize(val, kind)
+ s.move() // n / t
+ s.move() // u / r
+ s.move() // l / u
+ s.move() // l / e
+ return tok
+ }
+ unsafe { ident.free() }
+ return s.invalid_token()
+ } else if s.pos + 4 < s.text.len && s.text[s.pos] == `f` {
+ ident := s.text[s.pos..s.pos + 5].bytestr()
+ if ident == 'false' {
+ unsafe { ident.free() }
+ val := s.text[s.pos..s.pos + 5]
+ tok := s.tokenize(val, .bool_)
+ s.move() // f
+ s.move() // a
+ s.move() // l
+ s.move() // s
+ s.move() // e
+ return tok
+ }
+ unsafe { ident.free() }
+ return s.invalid_token()
+ } else if s.text[s.pos] in json2.char_list {
+ chr := s.text[s.pos]
+ tok := s.tokenize([]byte{}, TokenKind(int(chr)))
+ s.move()
+ return tok
+ } else if s.text[s.pos] == `"` {
+ return s.text_scan()
+ } else if s.text[s.pos].is_digit() || s.text[s.pos] == `-` {
+ return s.num_scan()
+ } else {
+ return s.invalid_token()
+ }
+}
diff --git a/v_windows/v/old/vlib/x/json2/scanner_test.v b/v_windows/v/old/vlib/x/json2/scanner_test.v
new file mode 100644
index 0000000..73f4d79
--- /dev/null
+++ b/v_windows/v/old/vlib/x/json2/scanner_test.v
@@ -0,0 +1,351 @@
+module json2
+
+fn test_str() {
+ mut sc := Scanner{
+ text: '"test"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .str_
+ assert tok.lit.len == 4
+ assert tok.lit.bytestr() == 'test'
+}
+
+fn test_str_valid_unicode_escape() {
+ mut sc := Scanner{
+ text: r'"\u0048"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .str_
+ assert tok.lit.len == 1
+ assert tok.lit.bytestr() == 'H'
+}
+
+fn test_str_valid_unicode_escape_2() {
+ mut sc := Scanner{
+ text: r'"\u2714"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .str_
+ assert tok.lit.len == 3
+ assert tok.lit.bytestr() == '✔'
+}
+
+fn test_str_invalid_escape() {
+ mut sc := Scanner{
+ text: r'"\z"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid backslash escape'
+}
+
+fn test_str_invalid_must_be_escape() {
+ for char in important_escapable_chars {
+ mut sc := Scanner{
+ text: [byte(`"`), `t`, char, `"`]
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'character must be escaped with a backslash'
+ }
+}
+
+fn test_str_invalid_unicode_escape() {
+ mut sc := Scanner{
+ text: r'"\u010G"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == '`G` is not a hex digit'
+}
+
+fn test_str_invalid_unicode_escape_len() {
+ mut sc := Scanner{
+ text: r'"\u001"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'unicode escape must have 4 hex digits'
+}
+
+fn test_str_invalid_uppercase_u() {
+ mut sc := Scanner{
+ text: r'"\U0000"'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'unicode endpoints must be in lowercase `u`'
+}
+
+fn test_str_missing_closing_bracket() {
+ mut sc := Scanner{
+ text: '"incomplete'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'missing double quotes in string closing'
+}
+
+fn test_int() {
+ mut sc := Scanner{
+ text: '10'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .int_
+ assert tok.lit.len == 2
+ assert tok.lit.bytestr() == '10'
+}
+
+fn test_int_negative() {
+ mut sc := Scanner{
+ text: '-10'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .int_
+ assert tok.lit.len == 3
+ assert tok.lit.bytestr() == '-10'
+}
+
+fn test_float() {
+ mut sc := Scanner{
+ text: '123.400'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .float
+ assert tok.lit.len == 7
+ assert tok.lit.bytestr() == '123.400'
+}
+
+fn test_float_negative() {
+ mut sc := Scanner{
+ text: '-123.400'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .float
+ assert tok.lit.len == 8
+ assert tok.lit.bytestr() == '-123.400'
+}
+
+fn test_int_exp() {
+ mut sc := Scanner{
+ text: '1E22'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .int_
+ assert tok.lit.len == 4
+ assert tok.lit.bytestr() == '1E22'
+}
+
+fn test_int_exp_negative() {
+ mut sc := Scanner{
+ text: '1E-2'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .int_
+ assert tok.lit.len == 4
+ assert tok.lit.bytestr() == '1E-2'
+}
+
+fn test_int_exp_positive() {
+ mut sc := Scanner{
+ text: '1E+2'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .int_
+ assert tok.lit.len == 4
+ assert tok.lit.bytestr() == '1E+2'
+}
+
+fn test_float_exp() {
+ mut sc := Scanner{
+ text: '123.456e78'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .float
+ assert tok.lit.len == 10
+ assert tok.lit.bytestr() == '123.456e78'
+}
+
+fn test_float_exp_negative() {
+ mut sc := Scanner{
+ text: '20.56e-5'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .float
+ assert tok.lit.len == 8
+ assert tok.lit.bytestr() == '20.56e-5'
+}
+
+fn test_float_exp_positive() {
+ mut sc := Scanner{
+ text: '20.56e+5'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .float
+ assert tok.lit.len == 8
+ assert tok.lit.bytestr() == '20.56e+5'
+}
+
+fn test_number_with_space() {
+ mut sc := Scanner{
+ text: ' 4'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .int_
+ assert tok.lit.len == 1
+ assert tok.lit.bytestr() == '4'
+}
+
+fn test_number_invalid_leading_zero() {
+ mut sc := Scanner{
+ text: '0010'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed'
+}
+
+fn test_number_invalid_leading_zero_negative() {
+ mut sc := Scanner{
+ text: '-0010'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed'
+}
+
+fn test_number_invalid_start_char() {
+ mut sc := Scanner{
+ text: '+1'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid token `+`'
+}
+
+fn test_number_invalid_char() {
+ mut sc := Scanner{
+ text: '122x'.bytes()
+ }
+ sc.scan()
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid token `x`'
+}
+
+fn test_number_invalid_char_float() {
+ mut sc := Scanner{
+ text: '122x.1'.bytes()
+ }
+ sc.scan()
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid token `x`'
+}
+
+fn test_number_invalid_multiple_dot() {
+ mut sc := Scanner{
+ text: '122.108.10'.bytes()
+ }
+ sc.scan()
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid token `.`'
+}
+
+fn test_number_invalid_exp() {
+ mut sc := Scanner{
+ text: '0.3e'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid exponent'
+}
+
+fn test_number_invalid_exp_with_sign() {
+ mut sc := Scanner{
+ text: '0.3e+'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid exponent'
+}
+
+fn test_number_invalid_zero_exp() {
+ mut sc := Scanner{
+ text: '0e'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid exponent'
+}
+
+fn test_number_invalid_dot_exp() {
+ mut sc := Scanner{
+ text: '0.e'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid float'
+}
+
+fn test_number_invalid_double_exp() {
+ mut sc := Scanner{
+ text: '2eE'.bytes()
+ }
+ sc.scan()
+ tok := sc.scan()
+ assert tok.kind == .error
+ assert tok.lit.bytestr() == 'invalid token `E`'
+}
+
+fn test_null() {
+ mut sc := Scanner{
+ text: 'null'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .null
+ assert tok.lit.len == 4
+ assert tok.lit.bytestr() == 'null'
+}
+
+fn test_bool_true() {
+ mut sc := Scanner{
+ text: 'true'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .bool_
+ assert tok.lit.len == 4
+ assert tok.lit.bytestr() == 'true'
+}
+
+fn test_bool_false() {
+ mut sc := Scanner{
+ text: 'false'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .bool_
+ assert tok.lit.len == 5
+ assert tok.lit.bytestr() == 'false'
+}
+
+fn test_json_with_whitespace_start() {
+ mut sc := Scanner{
+ text: ' \n \n\t {'.bytes()
+ }
+ tok := sc.scan()
+ eprintln(tok)
+ assert tok.kind == .lcbr
+ assert tok.lit.len == 0
+}
+
+fn test_json_with_whitespace_end() {
+ mut sc := Scanner{
+ text: '} \n\t'.bytes()
+ }
+ tok := sc.scan()
+ assert tok.kind == .rcbr
+ tok2 := sc.scan()
+ eprintln(tok2)
+ assert tok2.kind == .eof
+}