From f5c4671bfbad96bf346bd7e9a21fc4317b4959df Mon Sep 17 00:00:00 2001 From: Indrajith K L Date: Sat, 3 Dec 2022 17:00:20 +0530 Subject: Adds most of the tools --- v_windows/v/old/vlib/x/json2/README.md | 175 ++++++++++++ v_windows/v/old/vlib/x/json2/any_test.v | 130 +++++++++ v_windows/v/old/vlib/x/json2/decoder.v | 200 ++++++++++++++ v_windows/v/old/vlib/x/json2/decoder_test.v | 61 +++++ v_windows/v/old/vlib/x/json2/encoder.v | 179 +++++++++++++ v_windows/v/old/vlib/x/json2/encoder_test.v | 29 ++ v_windows/v/old/vlib/x/json2/json2.v | 122 +++++++++ v_windows/v/old/vlib/x/json2/json2_test.v | 398 ++++++++++++++++++++++++++++ v_windows/v/old/vlib/x/json2/scanner.v | 306 +++++++++++++++++++++ v_windows/v/old/vlib/x/json2/scanner_test.v | 351 ++++++++++++++++++++++++ 10 files changed, 1951 insertions(+) create mode 100644 v_windows/v/old/vlib/x/json2/README.md create mode 100644 v_windows/v/old/vlib/x/json2/any_test.v create mode 100644 v_windows/v/old/vlib/x/json2/decoder.v create mode 100644 v_windows/v/old/vlib/x/json2/decoder_test.v create mode 100644 v_windows/v/old/vlib/x/json2/encoder.v create mode 100644 v_windows/v/old/vlib/x/json2/encoder_test.v create mode 100644 v_windows/v/old/vlib/x/json2/json2.v create mode 100644 v_windows/v/old/vlib/x/json2/json2_test.v create mode 100644 v_windows/v/old/vlib/x/json2/scanner.v create mode 100644 v_windows/v/old/vlib/x/json2/scanner_test.v (limited to 'v_windows/v/old/vlib/x/json2') diff --git a/v_windows/v/old/vlib/x/json2/README.md b/v_windows/v/old/vlib/x/json2/README.md new file mode 100644 index 0000000..fcefbff --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/README.md @@ -0,0 +1,175 @@ +> The name `json2` was chosen to avoid any unwanted potential conflicts with the +> existing codegen tailored for the main `json` module which is powered by CJSON. + +`x.json2` is an experimental JSON parser written from scratch on V. + +## Usage +```v oksyntax +import x.json2 +import net.http + +fn main() { + // Decoding + resp := http.get('https://example.com') ? + + // raw decode + raw_person := json2.raw_decode(resp.text) ? + + // Casting `Any` type / Navigating + person := raw_person.as_map() + name := person['name'].str() // Bob + age := person['age'].int() // 19 + pi := person['pi'].f64() // 3.14.... + + // Constructing an `Any` type + mut me := map[string]json2.Any{} + me['name'] = 'Bob' + me['age'] = 18 + + mut arr := []json2.Any{} + arr << 'rock' + arr << 'papers' + arr << json2.null + arr << 12 + + me['interests'] = arr + + mut pets := map[string]json2.Any{} + pets['Sam'] = 'Maltese Shitzu' + me['pets'] = pets + + // Stringify to JSON + println(me.str()) + //{ + // "name":"Bob", + // "age":18, + // "interests":["rock","papers","scissors",null,12], + // "pets":{"Sam":"Maltese"} + //} + + // Encode a struct/type to JSON + encoded_json := json2.encode(person2) +} +``` +## Using `decode` and `encode` +> Codegen for this feature is still WIP. +> You need to manually define the methods before using the module to structs. + +In order to use the `decode` and `encode` function, you need to explicitly define +two methods: `from_json` and `to_json`. `from_json` accepts a `json2.Any` argument +and inside of it you need to map the fields you're going to put into the type. +As for `to_json` method, you just need to map the values into `json2.Any` +and turn it into a string. + +```v ignore +struct Person { +mut: + name string + age int = 20 + pets []string +} + +fn (mut p Person) from_json(f json2.Any) { + obj := f.as_map() + for k, v in obj { + match k { + 'name' { p.name = v.str() } + 'age' { p.age = v.int() } + 'pets' { p.pets = v.arr().map(it.str()) } + else {} + } + } +} + +fn (p Person) to_json() string { + mut obj := map[string]json2.Any + obj['name'] = p.name + obj['age'] = p.age + obj['pets'] = p.pets + return obj.str() +} + +fn main() { + resp := os.read_file('./person.json')? + person := json2.decode(resp)? + println(person) // Person{name: 'Bob', age: 28, pets: ['Floof']} + person_json := json2.encode(person) + println(person_json) // {"name": "Bob", "age": 28, "pets": ["Floof"]} +} +``` + +## Using struct tags +`x.json2` can access and use the struct field tags similar to the +`json` module by using the comp-time `$for` for structs. + +```v ignore +fn (mut p Person) from_json(f json2.Any) { + mp := an.as_map() + mut js_field_name := '' + $for field in Person.fields { + js_field_name = field.name + + for attr in field.attrs { + if attr.starts_with('json:') { + js_field_name = attr.all_after('json:').trim_left(' ') + break + } + } + + match field.name { + 'name' { p.name = mp[js_field_name].str() } + 'age' { u.age = mp[js_field_name].int() } + 'pets' { u.pets = mp[js_field_name].arr().map(it.str()) } + else {} + } + } +} +``` + +### Null Values +`x.json2` has a separate `null` type for differentiating an undefined value and a null value. +To verify that the field you're accessing is a `null`, use ` is json2.Null`. + +```v ignore +fn (mut p Person) from_json(f json2.Any) { + obj := f.as_map() + if obj['age'] is json2.Null { + // use a default value + p.age = 10 + } +} +``` + +### Custom field names +Aside from using struct tags, you can also just simply cast the base field into a map (`as_map()`) +and access the field you wish to put into the struct/type. + +```v ignore +fn (mut p Person) from_json(f json2.Any) { + obj := f.as_map() + p.name = obj['nickname'].str() +} +``` + +```v oksyntax +fn (mut p Person) to_json() string { + obj := f.as_map() + obj['nickname'] = p.name + return obj.str() +} +``` + +### Undefined Values +Getting undefined values has the same behavior as regular V types. +If you're casting a base field into `map[string]json2.Any` and fetch an undefined entry/value, +it simply returns empty. As for the `[]json2.Any`, it returns an index error. + +## Casting a value to an incompatible type +`x.json2` provides methods for turning `Any` types into usable types. +The following list shows the possible outputs when casting a value to an incompatible type. + +1. Casting non-array values as array (`arr()`) will return an array with the value as the content. +2. Casting non-map values as map (`as_map()`) will return a map with the value as the content. +3. Casting non-string values to string (`str()`) will return the +JSON string representation of the value. +4. Casting non-numeric values to int/float (`int()`/`i64()`/`f32()`/`f64()`) will return zero. diff --git a/v_windows/v/old/vlib/x/json2/any_test.v b/v_windows/v/old/vlib/x/json2/any_test.v new file mode 100644 index 0000000..6f86900 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/any_test.v @@ -0,0 +1,130 @@ +import x.json2 + +const ( + sample_data = map{ + 'int': json2.Any(int(1)) + 'i64': json2.Any(i64(128)) + 'f32': json2.Any(f32(2.0)) + 'f64': json2.Any(f64(1.283)) + 'bool': json2.Any(false) + 'str': json2.Any('test') + 'null': json2.Any(json2.null) + 'arr': json2.Any([json2.Any('lol')]) + 'obj': json2.Any(map{ + 'foo': json2.Any(10) + }) + } +) + +fn is_null(f json2.Any) bool { + match f { + json2.Null { return true } + else { return false } + } +} + +fn test_f32() { + // valid conversions + assert sample_data['int'].f32() == 1.0 + assert sample_data['i64'].f32() == 128.0 + assert sample_data['f32'].f32() == 2.0 + assert sample_data['f64'].f32() == 1.2829999923706055 + // invalid conversions + assert sample_data['bool'].f32() == 0.0 + assert sample_data['str'].f32() == 0.0 + assert sample_data['null'].f32() == 0.0 + assert sample_data['arr'].f32() == 0.0 + assert sample_data['obj'].f32() == 0.0 +} + +fn test_f64() { + // valid conversions + assert sample_data['int'].f64() == 1.0 + assert sample_data['i64'].f64() == 128.0 + assert sample_data['f32'].f64() == 2.0 + assert sample_data['f64'].f64() == 1.283 + // invalid conversions + assert sample_data['bool'].f64() == 0.0 + assert sample_data['str'].f64() == 0.0 + assert sample_data['null'].f64() == 0.0 + assert sample_data['arr'].f64() == 0.0 + assert sample_data['obj'].f64() == 0.0 +} + +fn test_int() { + // valid conversions + assert sample_data['int'].int() == 1 + assert sample_data['i64'].int() == 128 + assert sample_data['f32'].int() == 2 + assert sample_data['f64'].int() == 1 + assert json2.Any(true).int() == 1 + // invalid conversions + assert json2.Any('123').int() == 0 + assert sample_data['null'].int() == 0 + assert sample_data['arr'].int() == 0 + assert sample_data['obj'].int() == 0 +} + +fn test_i64() { + // valid conversions + assert sample_data['int'].i64() == 1 + assert sample_data['i64'].i64() == 128 + assert sample_data['f32'].i64() == 2 + assert sample_data['f64'].i64() == 1 + assert json2.Any(true).i64() == 1 + // invalid conversions + assert json2.Any('123').i64() == 0 + assert sample_data['null'].i64() == 0 + assert sample_data['arr'].i64() == 0 + assert sample_data['obj'].i64() == 0 +} + +fn test_as_map() { + assert sample_data['int'].as_map()['0'].int() == 1 + assert sample_data['i64'].as_map()['0'].i64() == 128.0 + assert sample_data['f32'].as_map()['0'].f32() == 2.0 + assert sample_data['f64'].as_map()['0'].f64() == 1.283 + assert sample_data['bool'].as_map()['0'].bool() == false + assert sample_data['str'].as_map()['0'].str() == 'test' + assert is_null(sample_data['null'].as_map()['0']) == true + assert sample_data['arr'].as_map()['0'].str() == 'lol' + assert sample_data['obj'].as_map()['foo'].int() == 10 +} + +fn test_arr() { + assert sample_data['int'].arr()[0].int() == 1 + assert sample_data['i64'].arr()[0].i64() == 128.0 + assert sample_data['f32'].arr()[0].f32() == 2.0 + assert sample_data['f64'].arr()[0].f64() == 1.283 + assert sample_data['bool'].arr()[0].bool() == false + assert sample_data['str'].arr()[0].str() == 'test' + assert is_null(sample_data['null'].arr()[0]) == true + assert sample_data['arr'].arr()[0].str() == 'lol' + assert sample_data['obj'].arr()[0].int() == 10 +} + +fn test_bool() { + // valid conversions + assert sample_data['bool'].bool() == false + assert json2.Any('true').bool() == true + // invalid conversions + assert sample_data['int'].bool() == false + assert sample_data['i64'].bool() == false + assert sample_data['f32'].bool() == false + assert sample_data['f64'].bool() == false + assert sample_data['null'].bool() == false + assert sample_data['arr'].bool() == false + assert sample_data['obj'].bool() == false +} + +fn test_str() { + assert sample_data['int'].str() == '1' + assert sample_data['i64'].str() == '128' + assert sample_data['f32'].str() == '2.0' + assert sample_data['f64'].str() == '1.283' + assert sample_data['bool'].str() == 'false' + assert sample_data['str'].str() == 'test' + assert sample_data['null'].str() == 'null' + assert sample_data['arr'].str() == '["lol"]' + assert sample_data.str() == '{"int":1,"i64":128,"f32":2.0,"f64":1.283,"bool":false,"str":"test","null":null,"arr":["lol"],"obj":{"foo":10}}' +} diff --git a/v_windows/v/old/vlib/x/json2/decoder.v b/v_windows/v/old/vlib/x/json2/decoder.v new file mode 100644 index 0000000..a45a091 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/decoder.v @@ -0,0 +1,200 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module json2 + +// `Any` is a sum type that lists the possible types to be decoded and used. +pub type Any = Null | []Any | bool | f32 | f64 | i64 | int | map[string]Any | string | + u64 + +// `Null` struct is a simple representation of the `null` value in JSON. +pub struct Null { + is_null bool = true +} + +struct Parser { +mut: + scanner &Scanner + p_tok Token + tok Token + n_tok Token + n_level int + convert_type bool = true +} + +struct InvalidTokenError { + msg string + code int +} + +struct UnknownTokenError { + msg string + code int +} + +fn (mut p Parser) next() { + p.p_tok = p.tok + p.tok = p.n_tok + p.n_tok = p.scanner.scan() +} + +fn (mut p Parser) next_with_err() ? { + p.next() + if p.tok.kind == .error { + return error(p.emit_error(p.tok.lit.bytestr())) + } +} + +fn (p Parser) emit_error(msg string) string { + line := p.tok.line + column := p.tok.col + p.tok.lit.len + return '[x.json2] $msg ($line:$column)' +} + +// TODO: copied from v.util to avoid the entire module and its functions +// from being imported. remove later once -skip-unused is enabled by default. +fn skip_bom(file_content string) string { + mut raw_text := file_content + // BOM check + if raw_text.len >= 3 { + unsafe { + c_text := raw_text.str + if c_text[0] == 0xEF && c_text[1] == 0xBB && c_text[2] == 0xBF { + // skip three BOM bytes + offset_from_begin := 3 + raw_text = tos(c_text[offset_from_begin], vstrlen(c_text) - offset_from_begin) + } + } + } + return raw_text +} + +fn new_parser(srce string, convert_type bool) Parser { + src := skip_bom(srce) + return Parser{ + scanner: &Scanner{ + text: src.bytes() + } + convert_type: convert_type + } +} + +fn (mut p Parser) decode() ?Any { + p.next() + p.next_with_err() ? + fi := p.decode_value() ? + if p.tok.kind != .eof { + return IError(&InvalidTokenError{ + msg: p.emit_error('invalid token `$p.tok.kind`') + }) + } + return fi +} + +fn (mut p Parser) decode_value() ?Any { + if p.n_level + 1 == 500 { + return error(p.emit_error('reached maximum nesting level of 500')) + } + match p.tok.kind { + .lsbr { + return p.decode_array() + } + .lcbr { + return p.decode_object() + } + .int_, .float { + tl := p.tok.lit.bytestr() + kind := p.tok.kind + p.next_with_err() ? + if p.convert_type { + if kind == .float { + return Any(tl.f64()) + } + return Any(tl.i64()) + } + return Any(tl) + } + .bool_ { + lit := p.tok.lit.bytestr() + p.next_with_err() ? + if p.convert_type { + return Any(lit.bool()) + } + return Any(lit) + } + .null { + p.next_with_err() ? + if p.convert_type { + return Any(null) + } + return Any('null') + } + .str_ { + str := p.tok.lit.bytestr() + p.next_with_err() ? + return Any(str) + } + else { + return IError(&InvalidTokenError{ + msg: p.emit_error('invalid token `$p.tok.kind`') + }) + } + } + return Any(null) +} + +fn (mut p Parser) decode_array() ?Any { + mut items := []Any{} + p.next_with_err() ? + p.n_level++ + for p.tok.kind != .rsbr { + item := p.decode_value() ? + items << item + if p.tok.kind == .comma { + p.next_with_err() ? + if p.tok.kind == .rsbr || p.tok.kind == .rcbr { + return IError(&InvalidTokenError{ + msg: p.emit_error('invalid token `$p.tok.lit') + }) + } + } else if p.tok.kind == .rsbr { + break + } else { + return IError(&UnknownTokenError{ + msg: p.emit_error("unknown token '$p.tok.lit' when decoding array.") + }) + } + } + p.next_with_err() ? + p.n_level-- + return Any(items) +} + +fn (mut p Parser) decode_object() ?Any { + mut fields := map[string]Any{} + p.next_with_err() ? + p.n_level++ + for p.tok.kind != .rcbr { + is_key := p.tok.kind == .str_ && p.n_tok.kind == .colon + if !is_key { + return IError(&InvalidTokenError{ + msg: p.emit_error('invalid token `$p.tok.kind`, expecting `str_`') + }) + } + cur_key := p.tok.lit.bytestr() + p.next_with_err() ? + p.next_with_err() ? + fields[cur_key] = p.decode_value() ? + if p.tok.kind == .comma { + p.next_with_err() ? + if p.tok.kind != .str_ { + return IError(&UnknownTokenError{ + msg: p.emit_error("unknown token '$p.tok.lit' when decoding object.") + }) + } + } + } + p.next_with_err() ? + p.n_level-- + return Any(fields) +} diff --git a/v_windows/v/old/vlib/x/json2/decoder_test.v b/v_windows/v/old/vlib/x/json2/decoder_test.v new file mode 100644 index 0000000..f80f8b2 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/decoder_test.v @@ -0,0 +1,61 @@ +module json2 + +fn test_raw_decode_string() ? { + str := raw_decode('"Hello!"') ? + assert str.str() == 'Hello!' +} + +fn test_raw_decode_number() ? { + num := raw_decode('123') ? + assert num.int() == 123 +} + +fn test_raw_decode_array() ? { + raw_arr := raw_decode('["Foo", 1]') ? + arr := raw_arr.arr() + assert arr[0].str() == 'Foo' + assert arr[1].int() == 1 +} + +fn test_raw_decode_bool() ? { + bol := raw_decode('false') ? + assert bol.bool() == false +} + +fn test_raw_decode_map() ? { + raw_mp := raw_decode('{"name":"Bob","age":20}') ? + mp := raw_mp.as_map() + assert mp['name'].str() == 'Bob' + assert mp['age'].int() == 20 +} + +fn test_raw_decode_null() ? { + nul := raw_decode('null') ? + assert nul is Null +} + +fn test_raw_decode_invalid() ? { + raw_decode('1z') or { + assert err.msg == '[x.json2] invalid token `z` (0:17)' + return + } + assert false +} + +fn test_raw_decode_string_with_dollarsign() ? { + str := raw_decode(r'"Hello $world"') ? + assert str.str() == r'Hello $world' +} + +fn test_raw_decode_map_with_whitespaces() ? { + raw_mp := raw_decode(' \n\t{"name":"Bob","age":20}\n\t') ? + mp := raw_mp.as_map() + assert mp['name'].str() == 'Bob' + assert mp['age'].int() == 20 +} + +fn test_nested_array_object() ? { + mut parser := new_parser(r'[[[[[],[],[]]]],{"Test":{}},[[]]]', false) + decoded := parser.decode() ? + assert parser.n_level == 0 +} diff --git a/v_windows/v/old/vlib/x/json2/encoder.v b/v_windows/v/old/vlib/x/json2/encoder.v new file mode 100644 index 0000000..b1ca0e4 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/encoder.v @@ -0,0 +1,179 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module json2 + +import strings + +fn write_value(v Any, i int, len int, mut wr strings.Builder) { + str := v.json_str() + if v is string { + wr.write_string('"$str"') + } else { + wr.write_string(str) + } + if i >= len - 1 { + return + } + wr.write_b(`,`) +} + +// str returns the string representation of the `map[string]Any`. +pub fn (flds map[string]Any) str() string { + mut wr := strings.new_builder(200) + wr.write_b(`{`) + mut i := 0 + for k, v in flds { + wr.write_string('"$k":') + write_value(v, i, flds.len, mut wr) + i++ + } + wr.write_b(`}`) + defer { + unsafe { wr.free() } + } + res := wr.str() + return res +} + +// str returns the string representation of the `[]Any`. +pub fn (flds []Any) str() string { + mut wr := strings.new_builder(200) + wr.write_b(`[`) + for i, v in flds { + write_value(v, i, flds.len, mut wr) + } + wr.write_b(`]`) + defer { + unsafe { wr.free() } + } + res := wr.str() + return res +} + +// str returns the string representation of the `Any` type. Use the `json_str` method +// if you want to use the escaped str() version of the `Any` type. +pub fn (f Any) str() string { + if f is string { + return f + } else { + return f.json_str() + } +} + +// json_str returns the JSON string representation of the `Any` type. +pub fn (f Any) json_str() string { + match f { + string { + return json_string(f) + } + int { + return f.str() + } + u64, i64 { + return f.str() + } + f32 { + str_f32 := f.str() + if str_f32.ends_with('.') { + return '${str_f32}0' + } + return str_f32 + } + f64 { + str_f64 := f.str() + if str_f64.ends_with('.') { + return '${str_f64}0' + } + return str_f64 + } + bool { + return f.str() + } + map[string]Any { + return f.str() + } + []Any { + return f.str() + } + Null { + return 'null' + } + } +} + +// char_len_list is a modified version of builtin.utf8_str_len +// that returns an array of character lengths. (e.g "t✔" => [1,2]) +fn char_len_list(s string) []int { + mut l := 1 + mut ls := []int{} + for i := 0; i < s.len; i++ { + c := s[i] + if (c & (1 << 7)) != 0 { + for t := byte(1 << 6); (c & t) != 0; t >>= 1 { + l++ + i++ + } + } + ls << l + l = 1 + } + return ls +} + +const escaped_chars = [r'\b', r'\f', r'\n', r'\r', r'\t'] + +// json_string returns the JSON spec-compliant version of the string. +[manualfree] +fn json_string(s string) string { + // not the best implementation but will revisit it soon + char_lens := char_len_list(s) + mut sb := strings.new_builder(s.len) + mut i := 0 + defer { + unsafe { + char_lens.free() + // freeing string builder on defer after + // returning .str() still isn't working :( + // sb.free() + } + } + for char_len in char_lens { + if char_len == 1 { + chr := s[i] + if chr in important_escapable_chars { + for j := 0; j < important_escapable_chars.len; j++ { + if chr == important_escapable_chars[j] { + sb.write_string(json2.escaped_chars[j]) + break + } + } + } else if chr == `"` || chr == `/` || chr == `\\` { + sb.write_string('\\' + chr.ascii_str()) + } else { + sb.write_b(chr) + } + } else { + slice := s[i..i + char_len] + hex_code := slice.utf32_code().hex() + if hex_code.len < 4 { + // an utf8 codepoint + sb.write_string(slice) + } else if hex_code.len == 4 { + sb.write_string('\\u$hex_code') + } else { + // TODO: still figuring out what + // to do with more than 4 chars + sb.write_b(` `) + } + unsafe { + slice.free() + hex_code.free() + } + } + i += char_len + } + str := sb.str() + unsafe { sb.free() } + return str +} diff --git a/v_windows/v/old/vlib/x/json2/encoder_test.v b/v_windows/v/old/vlib/x/json2/encoder_test.v new file mode 100644 index 0000000..8135172 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/encoder_test.v @@ -0,0 +1,29 @@ +import x.json2 + +fn test_json_string_characters() { + text := json2.raw_decode(r'"\n\r\b\f\t\\\"\/"') or { '' } + assert text.json_str() == '\\n\\r\\b\\f\\t\\\\\\"\\/' +} + +fn test_json_string() { + text := json2.Any('te✔st') + assert text.json_str() == r'te\u2714st' +} + +fn test_json_string_emoji() { + text := json2.Any('🐈') + assert text.json_str() == r' ' +} + +fn test_json_string_non_ascii() { + text := json2.Any('ひらがな') + assert text.json_str() == r'\u3072\u3089\u304c\u306a' +} + +fn test_utf8_strings_are_not_modified() ? { + original := '{"s":"Schilddrüsenerkrankungen"}' + // dump(original) + deresult := json2.raw_decode(original) ? + // dump(deresult) + assert deresult.str() == original +} diff --git a/v_windows/v/old/vlib/x/json2/json2.v b/v_windows/v/old/vlib/x/json2/json2.v new file mode 100644 index 0000000..0e5012c --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/json2.v @@ -0,0 +1,122 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module json2 + +pub const ( + null = Null{} +) + +pub interface Serializable { + from_json(f Any) + to_json() string +} + +// Decodes a JSON string into an `Any` type. Returns an option. +pub fn raw_decode(src string) ?Any { + mut p := new_parser(src, true) + return p.decode() +} + +// Same with `raw_decode`, but skips the type conversion for certain types when decoding a certain value. +pub fn fast_raw_decode(src string) ?Any { + mut p := new_parser(src, false) + return p.decode() +} + +// decode is a generic function that decodes a JSON string into the target type. +pub fn decode(src string) ?T { + res := raw_decode(src) ? + mut typ := T{} + typ.from_json(res) + return typ +} + +// encode is a generic function that encodes a type into a JSON string. +pub fn encode(typ T) string { + return typ.to_json() +} + +// as_map uses `Any` as a map. +pub fn (f Any) as_map() map[string]Any { + if f is map[string]Any { + return f + } else if f is []Any { + mut mp := map[string]Any{} + for i, fi in f { + mp['$i'] = fi + } + return mp + } + return map{ + '0': f + } +} + +// int uses `Any` as an integer. +pub fn (f Any) int() int { + match f { + int { return f } + i64, f32, f64, bool { return int(f) } + else { return 0 } + } +} + +// i64 uses `Any` as a 64-bit integer. +pub fn (f Any) i64() i64 { + match f { + i64 { return f } + int, f32, f64, bool { return i64(f) } + else { return 0 } + } +} + +// u64 uses `Any` as a 64-bit unsigned integer. +pub fn (f Any) u64() u64 { + match f { + u64 { return f } + int, i64, f32, f64, bool { return u64(f) } + else { return 0 } + } +} + +// f32 uses `Any` as a 32-bit float. +pub fn (f Any) f32() f32 { + match f { + f32 { return f } + int, i64, f64 { return f32(f) } + else { return 0.0 } + } +} + +// f64 uses `Any` as a float. +pub fn (f Any) f64() f64 { + match f { + f64 { return f } + int, i64, f32 { return f64(f) } + else { return 0.0 } + } +} + +// arr uses `Any` as an array. +pub fn (f Any) arr() []Any { + if f is []Any { + return f + } else if f is map[string]Any { + mut arr := []Any{} + for _, v in f { + arr << v + } + return arr + } + return [f] +} + +// bool uses `Any` as a bool +pub fn (f Any) bool() bool { + match f { + bool { return f } + string { return f.bool() } + else { return false } + } +} diff --git a/v_windows/v/old/vlib/x/json2/json2_test.v b/v_windows/v/old/vlib/x/json2/json2_test.v new file mode 100644 index 0000000..b808f42 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/json2_test.v @@ -0,0 +1,398 @@ +import x.json2 + +enum JobTitle { + manager + executive + worker +} + +struct Employee { +pub mut: + name string + age int + salary f32 + title JobTitle +} + +fn (e Employee) to_json() string { + mut mp := map[string]json2.Any{} + mp['name'] = e.name + mp['age'] = e.age + mp['salary'] = e.salary + mp['title'] = int(e.title) + /* + $for field in Employee.fields { + d := e.$(field.name) + + $if field.typ is JobTitle { + mp[field.name] = json.encode(d) + } $else { + mp[field.name] = d + } + } + */ + return mp.str() +} + +fn (mut e Employee) from_json(any json2.Any) { + mp := any.as_map() + e.name = mp['name'].str() + e.age = mp['age'].int() + e.salary = mp['salary'].f32() + e.title = JobTitle(mp['title'].int()) +} + +fn test_simple() { + x := Employee{'Peter', 28, 95000.5, .worker} + s := json2.encode(x) + eprintln('Employee x: $s') + assert s == '{"name":"Peter","age":28,"salary":95000.5,"title":2}' + y := json2.decode(s) or { + println(err) + assert false + return + } + eprintln('Employee y: $y') + assert y.name == 'Peter' + assert y.age == 28 + assert y.salary == 95000.5 + assert y.title == .worker +} + +fn test_fast_raw_decode() { + s := '{"name":"Peter","age":28,"salary":95000.5,"title":2}' + o := json2.fast_raw_decode(s) or { + assert false + json2.Any(json2.null) + } + str := o.str() + assert str == '{"name":"Peter","age":"28","salary":"95000.5","title":"2"}' +} + +fn test_character_unescape() { + message := r'{ + "newline": "new\nline", + "tab": "\ttab", + "backslash": "back\\slash", + "quotes": "\"quotes\"", + "slash":"\/dev\/null" +}' + mut obj := json2.raw_decode(message) or { + println(err) + assert false + return + } + lines := obj.as_map() + eprintln('$lines') + assert lines['newline'].str() == 'new\nline' + assert lines['tab'].str() == '\ttab' + assert lines['backslash'].str() == 'back\\slash' + assert lines['quotes'].str() == '"quotes"' + assert lines['slash'].str() == '/dev/null' +} + +struct User2 { +pub mut: + age int + nums []int +} + +fn (mut u User2) from_json(an json2.Any) { + mp := an.as_map() + mut js_field_name := '' + $for field in User.fields { + js_field_name = field.name + for attr in field.attrs { + if attr.starts_with('json:') { + js_field_name = attr.all_after('json:').trim_left(' ') + break + } + } + match field.name { + 'age' { u.age = mp[js_field_name].int() } + 'nums' { u.nums = mp[js_field_name].arr().map(it.int()) } + else {} + } + } +} + +// User struct needs to be `pub mut` for now in order to access and manipulate values +struct User { +pub mut: + age int + nums []int + last_name string [json: lastName] + is_registered bool [json: IsRegistered] + typ int [json: 'type'] + pets string [json: 'pet_animals'; raw] +} + +fn (mut u User) from_json(an json2.Any) { + mp := an.as_map() + mut js_field_name := '' + $for field in User.fields { + // FIXME: C error when initializing js_field_name inside comptime for + js_field_name = field.name + for attr in field.attrs { + if attr.starts_with('json:') { + js_field_name = attr.all_after('json:').trim_left(' ') + break + } + } + match field.name { + 'age' { u.age = mp[js_field_name].int() } + 'nums' { u.nums = mp[js_field_name].arr().map(it.int()) } + 'last_name' { u.last_name = mp[js_field_name].str() } + 'is_registered' { u.is_registered = mp[js_field_name].bool() } + 'typ' { u.typ = mp[js_field_name].int() } + 'pets' { u.pets = mp[js_field_name].str() } + else {} + } + } +} + +fn (u User) to_json() string { + // TODO: derive from field + mut mp := map{ + 'age': json2.Any(u.age) + } + mp['nums'] = u.nums.map(json2.Any(it)) + mp['lastName'] = u.last_name + mp['IsRegistered'] = u.is_registered + mp['type'] = u.typ + mp['pet_animals'] = u.pets + return mp.str() +} + +fn test_parse_user() { + s := '{"age": 10, "nums": [1,2,3], "type": 1, "lastName": "Johnson", "IsRegistered": true, "pet_animals": {"name": "Bob", "animal": "Dog"}}' + u2 := json2.decode(s) or { + println(err) + assert false + return + } + println(u2) + u := json2.decode(s) or { + println(err) + assert false + return + } + assert u.age == 10 + assert u.last_name == 'Johnson' + assert u.is_registered == true + assert u.nums.len == 3 + assert u.nums[0] == 1 + assert u.nums[1] == 2 + assert u.nums[2] == 3 + assert u.typ == 1 + assert u.pets == '{"name":"Bob","animal":"Dog"}' +} + +fn test_encode_user() { + usr := User{ + age: 10 + nums: [1, 2, 3] + last_name: 'Johnson' + is_registered: true + typ: 0 + pets: 'foo' + } + expected := '{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"foo"}' + out := json2.encode(usr) + assert out == expected +} + +struct Color { +pub mut: + space string + point string [raw] +} + +fn (mut c Color) from_json(an json2.Any) { + mp := an.as_map() + $for field in Color.fields { + match field.name { + 'space' { c.space = mp[field.name].str() } + 'point' { c.point = mp[field.name].str() } + else {} + } + } +} + +fn test_raw_json_field() { + color := json2.decode('{"space": "YCbCr", "point": {"Y": 123}}') or { + assert false + Color{} + } + assert color.point == '{"Y":123}' + assert color.space == 'YCbCr' +} + +/* +struct City { + name string +} + +struct Country { + cities []City + name string +} + +fn test_struct_in_struct() { + country := json.decode(Country, '{ "name": "UK", "cities": [{"name":"London"}, {"name":"Manchester"}]}') or { + assert false + exit(1) + } + assert country.name == 'UK' + assert country.cities.len == 2 + assert country.cities[0].name == 'London' + assert country.cities[1].name == 'Manchester' + println(country.cities) +} +*/ +fn test_encode_map() { + expected := '{"one":1,"two":2,"three":3,"four":4}' + numbers := map{ + 'one': json2.Any(1) + 'two': json2.Any(2) + 'three': json2.Any(3) + 'four': json2.Any(4) + } + out := numbers.str() + assert out == expected +} + +/* +fn test_parse_map() { + expected := { + 'one': 1 + 'two': 2 + 'three': 3 + 'four': 4 + } + out := json.decode('{"one":1,"two":2,"three":3,"four":4}') or { + assert false + r := { + '': 0 + } + r + } + println(out) + assert out == expected +} + +struct Data { + countries []Country + users map[string]User + extra map[string]map[string]int +} + +fn test_nested_type() { + data_expected := '{"countries":[{"cities":[{"name":"London"},{"name":"Manchester"}],"name":"UK"},{"cities":[{"name":"Donlon"},{"name":"Termanches"}],"name":"KU"}],"users":{"Foo":{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"little foo"},"Boo":{"age":20,"nums":[5,3,1],"lastName":"Smith","IsRegistered":false,"type":4,"pet_animals":"little boo"}},"extra":{"2":{"n1":2,"n2":4,"n3":8,"n4":16},"3":{"n1":3,"n2":9,"n3":27,"n4":81}}}' + + data := Data{ + countries: [ + Country{ + name: 'UK' + cities: [City{'London'}, + City{'Manchester'}, + ] + }, + Country{ + name: 'KU' + cities: [City{'Donlon'}, + City{'Termanches'}, + ] + }, + ] + users: { + 'Foo': User{ + age: 10 + nums: [1, 2, 3] + last_name: 'Johnson' + is_registered: true + typ: 0 + pets: 'little foo' + }, + 'Boo': User{ + age: 20 + nums: [5, 3, 1] + last_name: 'Smith' + is_registered: false + typ: 4 + pets: 'little boo' + } + }, + extra: { + '2': { + 'n1': 2 + 'n2': 4 + 'n3': 8 + 'n4': 16 + }, + '3': { + 'n1': 3 + 'n2': 9 + 'n3': 27 + 'n4': 81 + }, + } + } + out := json.encode(data) + println(out) + assert out == data_expected + + data2 := json.decode(Data, data_expected) or { + assert false + Data{} + } + assert data2.countries.len == data.countries.len + for i in 0..1 { + assert data2.countries[i].name == data.countries[i].name + assert data2.countries[i].cities.len == data.countries[i].cities.len + for j in 0..1 { + assert data2.countries[i].cities[j].name == data.countries[i].cities[j].name + } + } + + for key, user in data.users { + assert data2.users[key].age == user.age + assert data2.users[key].nums == user.nums + assert data2.users[key].last_name == user.last_name + assert data2.users[key].is_registered == user.is_registered + assert data2.users[key].typ == user.typ + // assert data2.users[key].pets == user.pets // TODO FIX + } + + for k, v in data.extra { + for k2, v2 in v { + assert data2.extra[k][k2] == v2 + } + } +} + +fn test_errors() { + invalid_array := fn () { + data := '{"countries":[{"cities":[{"name":"London"},{"name":"Manchester"}],"name":"UK"},{"cities":{"name":"Donlon"},"name":"KU"}],"users":{"Foo":{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"little foo"},"Boo":{"age":20,"nums":[5,3,1],"lastName":"Smith","IsRegistered":false,"type":4,"pet_animals":"little boo"}},"extra":{"2":{"n1":2,"n2":4,"n3":8,"n4":16},"3":{"n1":3,"n2":9,"n3":27,"n4":81}}}' + + json.decode(Data, data) or { + println(err) + assert err.starts_with('Json element is not an array:') + return + } + assert false + } + invalid_object := fn() { + data := '{"countries":[{"cities":[{"name":"London"},{"name":"Manchester"}],"name":"UK"},{"cities":[{"name":"Donlon"},{"name":"Termanches"}],"name":"KU"}],"users":[{"age":10,"nums":[1,2,3],"lastName":"Johnson","IsRegistered":true,"type":0,"pet_animals":"little foo"},{"age":20,"nums":[5,3,1],"lastName":"Smith","IsRegistered":false,"type":4,"pet_animals":"little boo"}],"extra":{"2":{"n1":2,"n2":4,"n3":8,"n4":16},"3":{"n1":3,"n2":9,"n3":27,"n4":81}}}' + + json.decode(Data, data) or { + println(err) + assert err.starts_with('Json element is not an object:') + return + } + assert false + } + invalid_array() + invalid_object() +} +*/ diff --git a/v_windows/v/old/vlib/x/json2/scanner.v b/v_windows/v/old/vlib/x/json2/scanner.v new file mode 100644 index 0000000..473a83b --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/scanner.v @@ -0,0 +1,306 @@ +// Copyright (c) 2019-2021 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module json2 + +import strconv + +struct Scanner { +mut: + text []byte + pos int + line int + col int +} + +enum TokenKind { + none_ + error + str_ + float + int_ + null + bool_ + eof + comma = 44 + colon = 58 + lsbr = 91 + rsbr = 93 + lcbr = 123 + rcbr = 125 +} + +struct Token { + lit []byte + kind TokenKind + line int + col int +} + +const ( + // list of characters commonly used in JSON. + char_list = [`{`, `}`, `[`, `]`, `,`, `:`] + // list of newlines to check when moving to a new position. + newlines = [`\r`, `\n`, `\t`] + // list of escapable that needs to be escaped inside a JSON string. + // double quotes and forward slashes are excluded intentionally since + // they have their own separate checks for it in order to pass the + // JSON test suite (https://github.com/nst/JSONTestSuite/). + important_escapable_chars = [`\b`, `\f`, `\n`, `\r`, `\t`] + // list of valid unicode escapes aside from \u{4-hex digits} + valid_unicode_escapes = [`b`, `f`, `n`, `r`, `t`, `\\`, `"`, `/`] + // used for transforming escapes into valid unicode (eg. n => \n) + unicode_transform_escapes = map{ + 98: `\b` + 102: `\f` + 110: `\n` + 114: `\r` + 116: `\t` + 92: `\\` + 34: `"` + 47: `/` + } + exp_signs = [byte(`-`), `+`] +) + +// move_pos proceeds to the next position. +fn (mut s Scanner) move() { + s.move_pos(true, true) +} + +// move_pos_with_newlines is the same as move_pos but only enables newline checking. +fn (mut s Scanner) move_pos_with_newlines() { + s.move_pos(false, true) +} + +fn (mut s Scanner) move_pos(include_space bool, include_newlines bool) { + s.pos++ + if s.pos < s.text.len { + if include_newlines && s.text[s.pos] in json2.newlines { + s.line++ + s.col = 0 + if s.text[s.pos] == `\r` && s.pos + 1 < s.text.len && s.text[s.pos + 1] == `\n` { + s.pos++ + } + for s.pos < s.text.len && s.text[s.pos] in json2.newlines { + s.move() + } + } else if include_space && s.text[s.pos] == ` ` { + s.pos++ + s.col++ + for s.pos < s.text.len && s.text[s.pos] == ` ` { + s.move() + } + } + } else { + s.col++ + } +} + +// error returns an error token. +fn (s Scanner) error(description string) Token { + return s.tokenize(description.bytes(), .error) +} + +// tokenize returns a token based on the given lit and kind. +fn (s Scanner) tokenize(lit []byte, kind TokenKind) Token { + return Token{ + lit: lit + kind: kind + col: s.col + line: s.line + } +} + +// text_scan scans and returns a string token. +[manualfree] +fn (mut s Scanner) text_scan() Token { + mut has_closed := false + mut chrs := []byte{} + for { + s.pos++ + s.col++ + if s.pos >= s.text.len { + break + } + ch := s.text[s.pos] + if (s.pos - 1 >= 0 && s.text[s.pos - 1] != `\\`) && ch == `"` { + has_closed = true + break + } else if (s.pos - 1 >= 0 && s.text[s.pos - 1] != `\\`) + && ch in json2.important_escapable_chars { + return s.error('character must be escaped with a backslash') + } else if (s.pos == s.text.len - 1 && ch == `\\`) || ch == byte(0) { + return s.error('invalid backslash escape') + } else if s.pos + 1 < s.text.len && ch == `\\` { + peek := s.text[s.pos + 1] + if peek in json2.valid_unicode_escapes { + chrs << json2.unicode_transform_escapes[int(peek)] + s.pos++ + s.col++ + continue + } else if peek == `u` { + if s.pos + 5 < s.text.len { + s.pos++ + s.col++ + mut codepoint := []byte{} + codepoint_start := s.pos + for s.pos < s.text.len && s.pos < codepoint_start + 4 { + s.pos++ + s.col++ + if s.text[s.pos] == `"` { + break + } else if !s.text[s.pos].is_hex_digit() { + x := s.text[s.pos].ascii_str() + return s.error('`$x` is not a hex digit') + } + codepoint << s.text[s.pos] + } + if codepoint.len != 4 { + return s.error('unicode escape must have 4 hex digits') + } + val := u32(strconv.parse_uint(codepoint.bytestr(), 16, 32) or { 0 }) + converted := utf32_to_str(val) + converted_bytes := converted.bytes() + chrs << converted_bytes + unsafe { + converted.free() + converted_bytes.free() + codepoint.free() + } + continue + } else { + return s.error('incomplete unicode escape') + } + } else if peek == `U` { + return s.error('unicode endpoints must be in lowercase `u`') + } else if peek == byte(229) { + return s.error('unicode endpoint not allowed') + } else { + return s.error('invalid backslash escape') + } + } + chrs << ch + } + tok := s.tokenize(chrs, .str_) + s.move() + if !has_closed { + return s.error('missing double quotes in string closing') + } + return tok +} + +// num_scan scans and returns an int/float token. +fn (mut s Scanner) num_scan() Token { + // analyze json number structure + // -[digit][?[dot][digit]][?[E/e][?-/+][digit]] + mut is_fl := false + mut dot_index := -1 + mut digits := []byte{} + if s.text[s.pos] == `-` { + digits << `-` + if !s.text[s.pos + 1].is_digit() { + return s.invalid_token() + } + s.move_pos_with_newlines() + } + if s.text[s.pos] == `0` && (s.pos + 1 < s.text.len && s.text[s.pos + 1].is_digit()) { + return s.error('leading zeroes in a number are not allowed') + } + for s.pos < s.text.len && (s.text[s.pos].is_digit() || (!is_fl && s.text[s.pos] == `.`)) { + digits << s.text[s.pos] + if s.text[s.pos] == `.` { + is_fl = true + dot_index = digits.len - 1 + } + s.move_pos_with_newlines() + } + if dot_index + 1 < s.text.len && digits[dot_index + 1..].len == 0 { + return s.error('invalid float') + } + if s.pos < s.text.len && (s.text[s.pos] == `e` || s.text[s.pos] == `E`) { + digits << s.text[s.pos] + s.move_pos_with_newlines() + if s.pos < s.text.len && s.text[s.pos] in json2.exp_signs { + digits << s.text[s.pos] + s.move_pos_with_newlines() + } + mut exp_digits_count := 0 + for s.pos < s.text.len && s.text[s.pos].is_digit() { + digits << s.text[s.pos] + exp_digits_count++ + s.move_pos_with_newlines() + } + if exp_digits_count == 0 { + return s.error('invalid exponent') + } + } + kind := if is_fl { TokenKind.float } else { TokenKind.int_ } + return s.tokenize(digits, kind) +} + +// invalid_token returns an error token with the invalid token message. +fn (s Scanner) invalid_token() Token { + if s.text[s.pos] >= 32 && s.text[s.pos] <= 126 { + x := s.text[s.pos].ascii_str() + return s.error('invalid token `$x`') + } else { + x := s.text[s.pos].str_escaped() + return s.error('invalid token `$x`') + } +} + +// scan returns a token based on the scanner's current position. +[manualfree] +fn (mut s Scanner) scan() Token { + if s.pos < s.text.len && (s.text[s.pos] == ` ` || s.text[s.pos] in json2.newlines) { + s.move() + } + if s.pos >= s.text.len { + return s.tokenize([]byte{}, .eof) + } else if s.pos + 3 < s.text.len && (s.text[s.pos] == `t` || s.text[s.pos] == `n`) { + ident := s.text[s.pos..s.pos + 4].bytestr() + if ident == 'true' || ident == 'null' { + mut kind := TokenKind.null + if ident == 'true' { + kind = .bool_ + } + unsafe { ident.free() } + val := s.text[s.pos..s.pos + 4] + tok := s.tokenize(val, kind) + s.move() // n / t + s.move() // u / r + s.move() // l / u + s.move() // l / e + return tok + } + unsafe { ident.free() } + return s.invalid_token() + } else if s.pos + 4 < s.text.len && s.text[s.pos] == `f` { + ident := s.text[s.pos..s.pos + 5].bytestr() + if ident == 'false' { + unsafe { ident.free() } + val := s.text[s.pos..s.pos + 5] + tok := s.tokenize(val, .bool_) + s.move() // f + s.move() // a + s.move() // l + s.move() // s + s.move() // e + return tok + } + unsafe { ident.free() } + return s.invalid_token() + } else if s.text[s.pos] in json2.char_list { + chr := s.text[s.pos] + tok := s.tokenize([]byte{}, TokenKind(int(chr))) + s.move() + return tok + } else if s.text[s.pos] == `"` { + return s.text_scan() + } else if s.text[s.pos].is_digit() || s.text[s.pos] == `-` { + return s.num_scan() + } else { + return s.invalid_token() + } +} diff --git a/v_windows/v/old/vlib/x/json2/scanner_test.v b/v_windows/v/old/vlib/x/json2/scanner_test.v new file mode 100644 index 0000000..73f4d79 --- /dev/null +++ b/v_windows/v/old/vlib/x/json2/scanner_test.v @@ -0,0 +1,351 @@ +module json2 + +fn test_str() { + mut sc := Scanner{ + text: '"test"'.bytes() + } + tok := sc.scan() + assert tok.kind == .str_ + assert tok.lit.len == 4 + assert tok.lit.bytestr() == 'test' +} + +fn test_str_valid_unicode_escape() { + mut sc := Scanner{ + text: r'"\u0048"'.bytes() + } + tok := sc.scan() + assert tok.kind == .str_ + assert tok.lit.len == 1 + assert tok.lit.bytestr() == 'H' +} + +fn test_str_valid_unicode_escape_2() { + mut sc := Scanner{ + text: r'"\u2714"'.bytes() + } + tok := sc.scan() + assert tok.kind == .str_ + assert tok.lit.len == 3 + assert tok.lit.bytestr() == '✔' +} + +fn test_str_invalid_escape() { + mut sc := Scanner{ + text: r'"\z"'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid backslash escape' +} + +fn test_str_invalid_must_be_escape() { + for char in important_escapable_chars { + mut sc := Scanner{ + text: [byte(`"`), `t`, char, `"`] + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'character must be escaped with a backslash' + } +} + +fn test_str_invalid_unicode_escape() { + mut sc := Scanner{ + text: r'"\u010G"'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == '`G` is not a hex digit' +} + +fn test_str_invalid_unicode_escape_len() { + mut sc := Scanner{ + text: r'"\u001"'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'unicode escape must have 4 hex digits' +} + +fn test_str_invalid_uppercase_u() { + mut sc := Scanner{ + text: r'"\U0000"'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'unicode endpoints must be in lowercase `u`' +} + +fn test_str_missing_closing_bracket() { + mut sc := Scanner{ + text: '"incomplete'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'missing double quotes in string closing' +} + +fn test_int() { + mut sc := Scanner{ + text: '10'.bytes() + } + tok := sc.scan() + assert tok.kind == .int_ + assert tok.lit.len == 2 + assert tok.lit.bytestr() == '10' +} + +fn test_int_negative() { + mut sc := Scanner{ + text: '-10'.bytes() + } + tok := sc.scan() + assert tok.kind == .int_ + assert tok.lit.len == 3 + assert tok.lit.bytestr() == '-10' +} + +fn test_float() { + mut sc := Scanner{ + text: '123.400'.bytes() + } + tok := sc.scan() + assert tok.kind == .float + assert tok.lit.len == 7 + assert tok.lit.bytestr() == '123.400' +} + +fn test_float_negative() { + mut sc := Scanner{ + text: '-123.400'.bytes() + } + tok := sc.scan() + assert tok.kind == .float + assert tok.lit.len == 8 + assert tok.lit.bytestr() == '-123.400' +} + +fn test_int_exp() { + mut sc := Scanner{ + text: '1E22'.bytes() + } + tok := sc.scan() + assert tok.kind == .int_ + assert tok.lit.len == 4 + assert tok.lit.bytestr() == '1E22' +} + +fn test_int_exp_negative() { + mut sc := Scanner{ + text: '1E-2'.bytes() + } + tok := sc.scan() + assert tok.kind == .int_ + assert tok.lit.len == 4 + assert tok.lit.bytestr() == '1E-2' +} + +fn test_int_exp_positive() { + mut sc := Scanner{ + text: '1E+2'.bytes() + } + tok := sc.scan() + assert tok.kind == .int_ + assert tok.lit.len == 4 + assert tok.lit.bytestr() == '1E+2' +} + +fn test_float_exp() { + mut sc := Scanner{ + text: '123.456e78'.bytes() + } + tok := sc.scan() + assert tok.kind == .float + assert tok.lit.len == 10 + assert tok.lit.bytestr() == '123.456e78' +} + +fn test_float_exp_negative() { + mut sc := Scanner{ + text: '20.56e-5'.bytes() + } + tok := sc.scan() + assert tok.kind == .float + assert tok.lit.len == 8 + assert tok.lit.bytestr() == '20.56e-5' +} + +fn test_float_exp_positive() { + mut sc := Scanner{ + text: '20.56e+5'.bytes() + } + tok := sc.scan() + assert tok.kind == .float + assert tok.lit.len == 8 + assert tok.lit.bytestr() == '20.56e+5' +} + +fn test_number_with_space() { + mut sc := Scanner{ + text: ' 4'.bytes() + } + tok := sc.scan() + assert tok.kind == .int_ + assert tok.lit.len == 1 + assert tok.lit.bytestr() == '4' +} + +fn test_number_invalid_leading_zero() { + mut sc := Scanner{ + text: '0010'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed' +} + +fn test_number_invalid_leading_zero_negative() { + mut sc := Scanner{ + text: '-0010'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed' +} + +fn test_number_invalid_start_char() { + mut sc := Scanner{ + text: '+1'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid token `+`' +} + +fn test_number_invalid_char() { + mut sc := Scanner{ + text: '122x'.bytes() + } + sc.scan() + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid token `x`' +} + +fn test_number_invalid_char_float() { + mut sc := Scanner{ + text: '122x.1'.bytes() + } + sc.scan() + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid token `x`' +} + +fn test_number_invalid_multiple_dot() { + mut sc := Scanner{ + text: '122.108.10'.bytes() + } + sc.scan() + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid token `.`' +} + +fn test_number_invalid_exp() { + mut sc := Scanner{ + text: '0.3e'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid exponent' +} + +fn test_number_invalid_exp_with_sign() { + mut sc := Scanner{ + text: '0.3e+'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid exponent' +} + +fn test_number_invalid_zero_exp() { + mut sc := Scanner{ + text: '0e'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid exponent' +} + +fn test_number_invalid_dot_exp() { + mut sc := Scanner{ + text: '0.e'.bytes() + } + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid float' +} + +fn test_number_invalid_double_exp() { + mut sc := Scanner{ + text: '2eE'.bytes() + } + sc.scan() + tok := sc.scan() + assert tok.kind == .error + assert tok.lit.bytestr() == 'invalid token `E`' +} + +fn test_null() { + mut sc := Scanner{ + text: 'null'.bytes() + } + tok := sc.scan() + assert tok.kind == .null + assert tok.lit.len == 4 + assert tok.lit.bytestr() == 'null' +} + +fn test_bool_true() { + mut sc := Scanner{ + text: 'true'.bytes() + } + tok := sc.scan() + assert tok.kind == .bool_ + assert tok.lit.len == 4 + assert tok.lit.bytestr() == 'true' +} + +fn test_bool_false() { + mut sc := Scanner{ + text: 'false'.bytes() + } + tok := sc.scan() + assert tok.kind == .bool_ + assert tok.lit.len == 5 + assert tok.lit.bytestr() == 'false' +} + +fn test_json_with_whitespace_start() { + mut sc := Scanner{ + text: ' \n \n\t {'.bytes() + } + tok := sc.scan() + eprintln(tok) + assert tok.kind == .lcbr + assert tok.lit.len == 0 +} + +fn test_json_with_whitespace_end() { + mut sc := Scanner{ + text: '} \n\t'.bytes() + } + tok := sc.scan() + assert tok.kind == .rcbr + tok2 := sc.scan() + eprintln(tok2) + assert tok2.kind == .eof +} -- cgit v1.2.3