omlish 0.0.0.dev436__tar.gz → 0.0.0.dev438__tar.gz

{omlish-0.0.0.dev436/omlish.egg-info → omlish-0.0.0.dev438}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omlish
-Version: 0.0.0.dev436
+Version: 0.0.0.dev438
 Summary: omlish
 Author: wrmsr
 License-Expression: BSD-3-Clause

{omlish-0.0.0.dev436 → omlish-0.0.0.dev438}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev436'
-__revision__ = '97bde773ba1e7d8a5c4245f8d7ed6b0492842e64'
+__version__ = '0.0.0.dev438'
+__revision__ = '5451232ca0015e84e38bd53d3c368a9d659ea8ec'
 
 
 #

{omlish-0.0.0.dev436 → omlish-0.0.0.dev438}/omlish/formats/json/stream/__init__.py

@@ -8,6 +8,7 @@ from .errors import (  # noqa
 
 
 from .lexing import (  # noqa
+    IdentTokenKind,
     ValueTokenKind,
     VALUE_TOKEN_KINDS,
     ControlTokenKind,
@@ -22,7 +23,7 @@ from .lexing import (  # noqa
     Token,
 
     CONTROL_TOKENS,
-    CONST_TOKENS,
+    CONST_IDENT_VALUES,
 
     JsonStreamLexError,
     JsonStreamLexer,
@@ -54,4 +55,5 @@ from .utils import (  # noqa
 
     stream_parse_values,
     stream_parse_one_value,
+    stream_parse_exactly_one_value,
 )

{omlish-0.0.0.dev436 → omlish-0.0.0.dev438}/omlish/formats/json/stream/lexing.py

@@ -13,20 +13,23 @@ import re
 import typing as ta
 
 from .... import check
+from .... import lang
 from ....funcs.genmachine import GenMachine
 from .errors import JsonStreamError
 
 
+with lang.auto_proxy_import(globals()):
+    import unicodedata
+
+
 ##
 
 
+IdentTokenKind: ta.TypeAlias = ta.Literal['IDENT']
+
 ValueTokenKind: ta.TypeAlias = ta.Literal[
     'STRING',
     'NUMBER',
-
-    'SPECIAL_NUMBER',
-    'BOOLEAN',
-    'NULL',
 ]
 
 VALUE_TOKEN_KINDS = frozenset(check.isinstance(a, str) for a in ta.get_args(ValueTokenKind))
@@ -45,6 +48,7 @@ SpaceTokenKind: ta.TypeAlias = ta.Literal['SPACE']
 CommentTokenKind: ta.TypeAlias = ta.Literal['COMMENT']
 
 TokenKind: ta.TypeAlias = ta.Union[  # noqa
+    IdentTokenKind,
     ValueTokenKind,
     ControlTokenKind,
     SpaceTokenKind,
@@ -93,16 +97,19 @@ CONTROL_TOKENS: ta.Mapping[str, TokenKind] = {
     ':': 'COLON',
 }
 
-CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
-    'NaN': ('SPECIAL_NUMBER', float('nan')),
-    'Infinity': ('SPECIAL_NUMBER', float('inf')),
-    '-Infinity': ('SPECIAL_NUMBER', float('-inf')),
+CONST_IDENT_VALUES: ta.Mapping[str, str | float | None] = {
+    'NaN': float('nan'),
+    '-NaN': float('-nan'),  # distinguished in parsing even if indistinguishable in value
+    'Infinity': float('inf'),
+    '-Infinity': float('-inf'),
 
-    'true': ('BOOLEAN', True),
-    'false': ('BOOLEAN', False),
-    'null': ('NULL', None),
+    'true': True,
+    'false': False,
+    'null': None,
 }
 
+MAX_CONST_IDENT_LEN = max(map(len, CONST_IDENT_VALUES))
+
 
 ##
 
@@ -119,21 +126,39 @@ class JsonStreamLexer(GenMachine[str, Token]):
             self,
             *,
             include_raw: bool = False,
+
+            allow_extended_space: bool = False,
             include_space: bool = False,
+
             allow_comments: bool = False,
             include_comments: bool = False,
+
             allow_single_quotes: bool = False,
             string_literal_parser: ta.Callable[[str], str] | None = None,
+
+            allow_extended_number_literals: bool = False,
+            number_literal_parser: ta.Callable[[str], ta.Any] | None = None,
+
+            allow_extended_idents: bool = False,
     ) -> None:
         self._include_raw = include_raw
+
+        self._allow_extended_space = allow_extended_space
         self._include_space = include_space
+
         self._allow_comments = allow_comments
         self._include_comments = include_comments
+
         self._allow_single_quotes = allow_single_quotes
         if string_literal_parser is None:
-            string_literal_parser = json.loads
+            string_literal_parser = json.loads  # noqa
         self._string_literal_parser = string_literal_parser
 
+        self._allow_extended_number_literals = allow_extended_number_literals
+        self._number_literal_parser = number_literal_parser
+
+        self._allow_extended_idents = allow_extended_idents
+
         self._ofs = 0
         self._line = 1
         self._col = 0
@@ -199,7 +224,33 @@ class JsonStreamLexer(GenMachine[str, Token]):
             if not c:
                 return None
 
-            if c.isspace():
+            if c.isspace() or (self._allow_extended_space and c in (
+                    '\u0009'
+                    '\u000A'
+                    '\u000B'
+                    '\u000C'
+                    '\u000D'
+                    '\u0020'
+                    '\u00A0'
+                    '\u2028'
+                    '\u2029'
+                    '\uFEFF'
+                    '\u1680'
+                    '\u2000'
+                    '\u2001'
+                    '\u2002'
+                    '\u2003'
+                    '\u2004'
+                    '\u2005'
+                    '\u2006'
+                    '\u2007'
+                    '\u2008'
+                    '\u2009'
+                    '\u200A'
+                    '\u202F'
+                    '\u205F'
+                    '\u3000'
+            )):
                 if self._include_space:
                     yield self._make_tok('SPACE', c, c, self.pos)
                 continue
@@ -211,16 +262,18 @@ class JsonStreamLexer(GenMachine[str, Token]):
             if c == '"' or (self._allow_single_quotes and c == "'"):
                 return self._do_string(c)
 
-            if c.isdigit() or c == '-':
+            if c.isdigit() or c == '-' or (self._allow_extended_number_literals and c in '.+'):
                 return self._do_number(c)
 
+            if self._allow_comments and c == '/':
+                return self._do_comment()
+
+            if self._allow_extended_idents:
+                return self._do_extended_ident(c)
+
             if c in 'tfnIN':
                 return self._do_const(c)
 
-            if self._allow_comments and c == '/':
-                yield from self._do_comment()
-                continue
-
             self._raise(f'Unexpected character: {c}')
 
     def _do_string(self, q: str):
@@ -269,7 +322,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
             if not c:
                 break
 
-            if not (c.isdigit() or c in '.eE+-'):
+            if not (c.isdigit() or c in '.eE+-' or (self._allow_extended_number_literals and c in 'xXabcdefABCDEF')):
                 break
             self._buf.write(c)
 
@@ -277,33 +330,58 @@ class JsonStreamLexer(GenMachine[str, Token]):
 
         #
 
-        if not NUMBER_PAT.fullmatch(raw):
-            # Can only be -Infinity
+        if self._allow_extended_number_literals:
+            p = 1 if raw[0] in '+-' else 0
+            if (len(raw) - p) > 1 and raw[p] == '0' and raw[p + 1].isdigit():
+                self._raise('Invalid number literal')
 
-            if not c:
-                self._raise('Unexpected end of input')
+        if raw == '-' or (self._allow_extended_number_literals and raw == '+'):
+            for svs in [
+                'Infinity',
+                *(['NaN'] if self._allow_extended_number_literals else []),
+            ]:
+                if c != svs[0]:
+                    continue
 
-            raw += c
-            try:
-                for _ in range(7):
-                    raw += self._char_in((yield None))  # noqa
-            except GeneratorExit:
-                self._raise('Unexpected end of input')
+                if not c:
+                    self._raise('Unexpected end of input')
 
-            if raw != '-Infinity':
-                self._raise(f'Invalid number format: {raw}')
+                raw += c
+                try:
+                    for _ in range(len(svs) - 1):
+                        c = self._char_in((yield None))  # noqa
+                        if not c:
+                            break
+                        raw += c
+                except GeneratorExit:
+                    self._raise('Unexpected end of input')
+
+                if raw[1:] != svs:
+                    self._raise(f'Invalid number format: {raw}')
 
-            tk, tv = CONST_TOKENS[raw]
-            yield self._make_tok(tk, tv, raw, pos)
+                if raw[0] == '+':
+                    raw = raw[1:]
 
-            return self._do_main()
+                yield self._make_tok('IDENT', raw, raw, pos)
+
+                return self._do_main()
 
         #
 
-        if '.' in raw or 'e' in raw or 'E' in raw:
-            nv = float(raw)
+        nv: ta.Any
+
+        if (np := self._number_literal_parser) is not None:
+            nv = np(raw)
+
         else:
-            nv = int(raw)
+            if not NUMBER_PAT.fullmatch(raw):
+                self._raise(f'Invalid number format: {raw}')
+
+            if '.' in raw or 'e' in raw or 'E' in raw:
+                nv = float(raw)
+            else:
+                nv = int(raw)
+
         yield self._make_tok('NUMBER', nv, raw, pos)
 
         #
@@ -322,17 +400,79 @@ class JsonStreamLexer(GenMachine[str, Token]):
             except GeneratorExit:
                 self._raise('Unexpected end of input')
 
-            if raw in CONST_TOKENS:
+            if raw in CONST_IDENT_VALUES:
                 break
 
-            if len(raw) > 8:  # None of the keywords are longer than 8 characters
+            if len(raw) > MAX_CONST_IDENT_LEN:
                 self._raise(f'Invalid literal: {raw}')
 
-        tk, tv = CONST_TOKENS[raw]
-        yield self._make_tok(tk, tv, raw, pos)
+        yield self._make_tok('IDENT', raw, raw, pos)
 
         return self._do_main()
 
+    def _do_unicode_escape(self):
+        try:
+            c = self._char_in((yield None))  # noqa
+        except GeneratorExit:
+            self._raise('Unexpected end of input')
+
+        if c != 'u':
+            self._raise('Illegal identifier escape')
+
+        ux = []
+        for _ in range(4):
+            try:
+                c = self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if c not in '0123456789abcdefABCDEF':
+                self._raise('Illegal identifier escape')
+
+            ux.append(c)
+
+        return chr(int(''.join(ux), 16))
+
+    def _do_extended_ident(self, c: str):
+        check.state(self._buf.tell() == 0)
+
+        if c == '\\':
+            c = yield from self._do_unicode_escape()
+
+        elif not (c in '$_' or unicodedata.category(c).startswith('L')):
+            self._raise('Illegal identifier start')
+
+        self._buf.write(c)
+
+        pos = self.pos
+
+        while True:
+            try:
+                c = self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if c == '\\':
+                c = yield from self._do_unicode_escape()
+                self._buf.write(c)
+                continue
+
+            if not c:
+                break
+
+            if c not in '$_\u200C\u200D':
+                uc = unicodedata.category(c)
+                if not (uc.startswith(('L', 'M', 'N')) or uc == 'Pc'):
+                    break
+
+            self._buf.write(c)
+
+        raw = self._flip_buf()
+
+        yield self._make_tok('IDENT', raw, raw, pos)
+
+        return self._do_main(c)
+
     def _do_comment(self):
         check.state(self._buf.tell() == 0)
 
@@ -349,7 +489,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
                 except GeneratorExit:
                     self._raise('Unexpected end of input')
 
-                if ic == '\n':
+                if not ic or ic == '\n':
                     break
 
                 if self._include_comments:
@@ -360,6 +500,9 @@ class JsonStreamLexer(GenMachine[str, Token]):
                 raw = f'//{cmt}\n' if self._include_raw else None
                 yield self._make_tok('COMMENT', cmt, raw, pos)
 
+            if not ic:
+                return
+
         elif oc == '*':
             lc: str | None = None
             while True:
@@ -382,3 +525,5 @@ class JsonStreamLexer(GenMachine[str, Token]):
 
         else:
             self._raise(f'Unexpected character after comment start: {oc}')
+
+        return self._do_main()

{omlish-0.0.0.dev436 → omlish-0.0.0.dev438}/omlish/formats/json/stream/parsing.py

@@ -4,6 +4,7 @@ import typing as ta
 from .... import lang
 from ....funcs.genmachine import GenMachine
 from .errors import JsonStreamError
+from .lexing import CONST_IDENT_VALUES
 from .lexing import SCALAR_VALUE_TYPES
 from .lexing import VALUE_TOKEN_KINDS
 from .lexing import Position
@@ -95,11 +96,21 @@ class JsonStreamObject(list):
 
 
 class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
-    def __init__(self) -> None:
-        super().__init__(self._do_value())
+    def __init__(
+            self,
+            *,
+            allow_trailing_commas: bool = False,
+
+            allow_extended_idents: bool = False,
+    ) -> None:
+        self._allow_trailing_commas = allow_trailing_commas
+
+        self._allow_extended_idents = allow_extended_idents
 
         self._stack: list[ta.Literal['OBJECT', 'KEY', 'ARRAY']] = []
 
+        super().__init__(self._do_value())
+
     #
 
     def _next_tok(self):
@@ -144,12 +155,23 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
                 raise JsonStreamParseError('Expected value') from None
             else:
                 raise
+        # except Exception as e:
+        #     raise
 
         if tok.kind in VALUE_TOKEN_KINDS:
             y, r = self._emit_event(tok.value)
             yield y
             return r
 
+        elif tok.kind == 'IDENT':
+            try:
+                cv = CONST_IDENT_VALUES[tok.value]
+            except KeyError:
+                raise JsonStreamParseError('Expected value', tok.pos) from None
+            y, r = self._emit_event(cv)
+            yield y
+            return r
+
         elif tok.kind == 'LBRACE':
             y, r = self._emit_begin_object()
             yield y
@@ -193,7 +215,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         except GeneratorExit:
             raise JsonStreamParseError('Expected object body') from None
 
-        if tok.kind == 'STRING':
+        if tok.kind == 'STRING' or (self._allow_trailing_commas and tok.kind == 'IDENT'):
             k = tok.value
 
             try:
@@ -225,7 +247,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_object_body(must_be_present=True)
+            return self._do_object_body(must_be_present=not self._allow_trailing_commas)
 
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
@@ -258,7 +280,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_value(must_be_present=True)
+            return self._do_value(must_be_present=not self._allow_trailing_commas)
 
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()

omlish-0.0.0.dev438/omlish/formats/json/stream/utils.py

@@ -0,0 +1,158 @@
+r"""
+TODO:
+ - gson switches
+  - Strictness.LEGACY_STRICT
+   - JsonReader allows the literals true, false and null to have any capitalization, for example fAlSe or NULL
+   - JsonReader supports the escape sequence \', representing a ' (single-quote)
+   - JsonReader supports the escape sequence \LF (with LF being the Unicode character U+000A), resulting in a LF within
+     the read JSON string
+   - JsonReader allows unescaped control characters (U+0000 through U+001F)
+  - Strictness.LENIENT
+   - In lenient mode, all input that is accepted in legacy strict mode is accepted in addition to the following
+     departures from RFC 8259:
+   - Streams that start with the non-execute prefix, ")]'\n"}
+   - Streams that include multiple top-level values. With legacy strict or strict parsing, each stream must contain
+     exactly one top-level value.
+   - Numbers may be NaNs or infinities represented by NaN and (-)Infinity respectively.
+   - End of line comments starting with // or # and ending with a newline character.
+   - C-style comments starting with /* and ending with */. Such comments may not be nested.
+   - Names that are unquoted or 'single quoted'.
+   - Strings that are unquoted or 'single quoted'.
+   - Array elements separated by ; instead of ,.
+   - Unnecessary array separators. These are interpreted as if null was the omitted value.
+   - Names and values separated by = or => instead of :.
+   - Name/value pairs separated by ; instead of ,.
+"""
+import itertools
+import typing as ta
+
+from .... import lang
+from .building import JsonValueBuilder
+from .errors import JsonStreamError
+from .lexing import JsonStreamLexer
+from .lexing import Token
+from .parsing import JsonStreamParser
+from .parsing import JsonStreamParserEvent
+
+
+##
+
+
+class JsonStreamValueParser(lang.ExitStacked):
+    class Machinery(ta.NamedTuple):
+        lex: JsonStreamLexer
+        parse: JsonStreamParser
+        build: JsonValueBuilder
+
+    def __init__(self, m: Machinery) -> None:
+        super().__init__()
+
+        self._m = m
+
+    #
+
+    def _enter_contexts(self) -> None:
+        self._enter_context(self._m.lex)
+        self._enter_context(self._m.parse)
+
+    def feed(self, i: ta.Iterable[str]) -> ta.Iterator[ta.Any]:
+        for c in i:
+            for t in self._m.lex(c):
+                for e in self._m.parse(t):
+                    for v in self._m.build(e):  # noqa
+                        yield v
+
+    #
+
+    @classmethod
+    def parse_values(
+            cls,
+            m: Machinery,
+            i: ta.Iterable[str],
+    ) -> ta.Iterator[ta.Any]:
+        with cls(m) as p:
+            yield from p.feed(itertools.chain(i, ['']))
+
+    @classmethod
+    def parse_one_value(
+            cls,
+            m: Machinery,
+            i: ta.Iterable[str],
+    ) -> ta.Any:
+        with cls(m) as p:
+            return next(p.feed(itertools.chain(i, [''])))
+
+    @classmethod
+    def parse_exactly_one_value(
+            cls,
+            m: Machinery,
+            i: ta.Iterable[str],
+    ) -> ta.Any:
+        r: ta.Any
+        r = not_set = object()
+        with cls(m) as p:
+            for v in p.feed(itertools.chain(i, [''])):
+                if r is not_set:
+                    r = v
+                else:
+                    raise JsonStreamError('Unexpected input')
+        if r is not_set:
+            raise JsonStreamError('No value')
+        return r
+
+
+##
+
+
+class DebugJsonStreamValueParser(JsonStreamValueParser):
+    def __init__(self, m: JsonStreamValueParser.Machinery) -> None:
+        super().__init__(m)
+
+        self._chars: list[str] = []
+        self._tokens: list[Token] = []
+        self._events: list[JsonStreamParserEvent] = []
+        self._values: list[ta.Any] = []
+
+    def feed(self, i: ta.Iterable[str]) -> ta.Iterator[ta.Any]:
+        for c in i:
+            self._chars.append(c)
+            for t in self._m.lex(c):
+                self._tokens.append(t)
+                for e in self._m.parse(t):
+                    self._events.append(e)
+                    for v in self._m.build(e):
+                        self._values.append(v)
+                        yield v
+
+
+##
+
+
+def make_machinery(
+        *,
+        include_raw: bool = False,
+        yield_object_lists: bool = False,
+) -> JsonStreamValueParser.Machinery:
+    return JsonStreamValueParser.Machinery(
+        JsonStreamLexer(
+            include_raw=include_raw,
+        ),
+
+        JsonStreamParser(),
+
+        JsonValueBuilder(
+            yield_object_lists=yield_object_lists,
+        ),
+    )
+
+
+def stream_parse_values(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Iterator[ta.Any]:
+    return JsonStreamValueParser.parse_values(make_machinery(**kwargs), i)
+
+
+def stream_parse_one_value(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Any:
+    return JsonStreamValueParser.parse_one_value(make_machinery(**kwargs), i)
+
+
+def stream_parse_exactly_one_value(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Any:
+    return JsonStreamValueParser.parse_exactly_one_value(make_machinery(**kwargs), i)

{omlish-0.0.0.dev436 → omlish-0.0.0.dev438}/omlish/formats/json5/literals.py

@@ -124,7 +124,10 @@ def parse_string_literal(s: str) -> str:
 def parse_number_literal(s: str) -> int | float:
     s = s.lower()
 
-    if 'x' in s:
-        return int(s, 16)
-    else:
-        return float(s)
+    try:
+        if 'x' in s:
+            return int(s, 16)
+        else:
+            return float(s)
+    except ValueError as e:
+        raise Json5Error from e