omlish 0.0.0.dev437__py3-none-any.whl → 0.0.0.dev439__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev437'
-__revision__ = '6e7eba22d4b3eb0dc36e24d1612b485ffa1e3c00'
+__version__ = '0.0.0.dev439'
+__revision__ = '3f7c136d6cdb4f4c2e749cf404632d93585559e3'
 
 
 #

omlish/dataclasses/impl/api/classes/make.py

@@ -142,7 +142,7 @@ def make_dataclass(  # noqa
 
     if module is None:
         try:
-            module = sys._getframemodulename(_frame_offset) or '__main__'  # type: ignore  # noqa
+            module = sys._getframemodulename(_frame_offset) or '__main__'  # noqa
         except AttributeError:
             with contextlib.suppress(AttributeError, ValueError):
                 module = sys._getframe(_frame_offset).f_globals.get('__name__', '__main__')  # noqa

omlish/dataclasses/tools/static.py

@@ -142,7 +142,7 @@ class Static(lang.Abstract):
                     n_md[_ExtraFieldParamsMetadata] = {
                         fs_f.name: getattr(x_fs, fs_f.name)
                         for fs_f in dc.fields(FieldSpec)  # noqa
-                        if fs_f not in dc.Field.__slots__  # type: ignore[attr-defined]
+                        if fs_f not in dc.Field.__slots__
                         and fs_f.name not in ('default', 'default_factory')
                     }
                     new_fld.metadata = n_md  # type: ignore[assignment]

omlish/formats/json/stream/__init__.py

@@ -8,6 +8,7 @@ from .errors import (  # noqa
 
 
 from .lexing import (  # noqa
+    IdentTokenKind,
     ValueTokenKind,
     VALUE_TOKEN_KINDS,
     ControlTokenKind,
@@ -22,7 +23,7 @@ from .lexing import (  # noqa
     Token,
 
     CONTROL_TOKENS,
-    CONST_TOKENS,
+    CONST_IDENT_VALUES,
 
     JsonStreamLexError,
     JsonStreamLexer,
@@ -35,8 +36,8 @@ from .parsing import (  # noqa
     BeginArray,
     EndArray,
 
-    JsonStreamParserEvent,
-    JsonStreamParserEvents,
+    Event,
+    Events,
 
     yield_parser_events,
 
@@ -54,4 +55,5 @@ from .utils import (  # noqa
 
     stream_parse_values,
     stream_parse_one_value,
+    stream_parse_exactly_one_value,
 )

omlish/formats/json/stream/building.py

@@ -5,8 +5,8 @@ from .parsing import BeginArray
 from .parsing import BeginObject
 from .parsing import EndArray
 from .parsing import EndObject
+from .parsing import Event
 from .parsing import JsonStreamObject
-from .parsing import JsonStreamParserEvent
 from .parsing import Key
 
 
@@ -62,7 +62,7 @@ class JsonValueBuilder:
         else:
             raise self.StateError
 
-    def __call__(self, e: JsonStreamParserEvent) -> ta.Iterable[ta.Any]:
+    def __call__(self, e: Event) -> ta.Iterable[ta.Any]:
         stk = self._stack
 
         #

omlish/formats/json/stream/lexing.py

@@ -13,20 +13,23 @@ import re
 import typing as ta
 
 from .... import check
+from .... import lang
 from ....funcs.genmachine import GenMachine
 from .errors import JsonStreamError
 
 
+with lang.auto_proxy_import(globals()):
+    import unicodedata
+
+
 ##
 
 
+IdentTokenKind: ta.TypeAlias = ta.Literal['IDENT']
+
 ValueTokenKind: ta.TypeAlias = ta.Literal[
     'STRING',
     'NUMBER',
-
-    'SPECIAL_NUMBER',
-    'BOOLEAN',
-    'NULL',
 ]
 
 VALUE_TOKEN_KINDS = frozenset(check.isinstance(a, str) for a in ta.get_args(ValueTokenKind))
@@ -45,6 +48,7 @@ SpaceTokenKind: ta.TypeAlias = ta.Literal['SPACE']
 CommentTokenKind: ta.TypeAlias = ta.Literal['COMMENT']
 
 TokenKind: ta.TypeAlias = ta.Union[  # noqa
+    IdentTokenKind,
     ValueTokenKind,
     ControlTokenKind,
     SpaceTokenKind,
@@ -93,16 +97,19 @@ CONTROL_TOKENS: ta.Mapping[str, TokenKind] = {
     ':': 'COLON',
 }
 
-CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
-    'NaN': ('SPECIAL_NUMBER', float('nan')),
-    'Infinity': ('SPECIAL_NUMBER', float('inf')),
-    '-Infinity': ('SPECIAL_NUMBER', float('-inf')),
+CONST_IDENT_VALUES: ta.Mapping[str, str | float | None] = {
+    'NaN': float('nan'),
+    '-NaN': float('-nan'),  # distinguished in parsing even if indistinguishable in value
+    'Infinity': float('inf'),
+    '-Infinity': float('-inf'),
 
-    'true': ('BOOLEAN', True),
-    'false': ('BOOLEAN', False),
-    'null': ('NULL', None),
+    'true': True,
+    'false': False,
+    'null': None,
 }
 
+MAX_CONST_IDENT_LEN = max(map(len, CONST_IDENT_VALUES))
+
 
 ##
 
@@ -119,21 +126,39 @@ class JsonStreamLexer(GenMachine[str, Token]):
             self,
             *,
             include_raw: bool = False,
+
+            allow_extended_space: bool = False,
             include_space: bool = False,
+
             allow_comments: bool = False,
             include_comments: bool = False,
+
             allow_single_quotes: bool = False,
             string_literal_parser: ta.Callable[[str], str] | None = None,
+
+            allow_extended_number_literals: bool = False,
+            number_literal_parser: ta.Callable[[str], ta.Any] | None = None,
+
+            allow_extended_idents: bool = False,
     ) -> None:
         self._include_raw = include_raw
+
+        self._allow_extended_space = allow_extended_space
         self._include_space = include_space
+
         self._allow_comments = allow_comments
         self._include_comments = include_comments
+
         self._allow_single_quotes = allow_single_quotes
         if string_literal_parser is None:
-            string_literal_parser = json.loads
+            string_literal_parser = json.loads  # noqa
         self._string_literal_parser = string_literal_parser
 
+        self._allow_extended_number_literals = allow_extended_number_literals
+        self._number_literal_parser = number_literal_parser
+
+        self._allow_extended_idents = allow_extended_idents
+
         self._ofs = 0
         self._line = 1
         self._col = 0
@@ -199,7 +224,33 @@ class JsonStreamLexer(GenMachine[str, Token]):
             if not c:
                 return None
 
-            if c.isspace():
+            if c.isspace() or (self._allow_extended_space and c in (
+                    '\u0009'
+                    '\u000A'
+                    '\u000B'
+                    '\u000C'
+                    '\u000D'
+                    '\u0020'
+                    '\u00A0'
+                    '\u2028'
+                    '\u2029'
+                    '\uFEFF'
+                    '\u1680'
+                    '\u2000'
+                    '\u2001'
+                    '\u2002'
+                    '\u2003'
+                    '\u2004'
+                    '\u2005'
+                    '\u2006'
+                    '\u2007'
+                    '\u2008'
+                    '\u2009'
+                    '\u200A'
+                    '\u202F'
+                    '\u205F'
+                    '\u3000'
+            )):
                 if self._include_space:
                     yield self._make_tok('SPACE', c, c, self.pos)
                 continue
@@ -211,16 +262,18 @@ class JsonStreamLexer(GenMachine[str, Token]):
             if c == '"' or (self._allow_single_quotes and c == "'"):
                 return self._do_string(c)
 
-            if c.isdigit() or c == '-':
+            if c.isdigit() or c == '-' or (self._allow_extended_number_literals and c in '.+'):
                 return self._do_number(c)
 
+            if self._allow_comments and c == '/':
+                return self._do_comment()
+
+            if self._allow_extended_idents:
+                return self._do_extended_ident(c)
+
             if c in 'tfnIN':
                 return self._do_const(c)
 
-            if self._allow_comments and c == '/':
-                yield from self._do_comment()
-                continue
-
             self._raise(f'Unexpected character: {c}')
 
     def _do_string(self, q: str):
@@ -269,7 +322,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
             if not c:
                 break
 
-            if not (c.isdigit() or c in '.eE+-'):
+            if not (c.isdigit() or c in '.eE+-' or (self._allow_extended_number_literals and c in 'xXabcdefABCDEF')):
                 break
             self._buf.write(c)
 
@@ -277,33 +330,58 @@ class JsonStreamLexer(GenMachine[str, Token]):
 
         #
 
-        if not NUMBER_PAT.fullmatch(raw):
-            # Can only be -Infinity
+        if self._allow_extended_number_literals:
+            p = 1 if raw[0] in '+-' else 0
+            if (len(raw) - p) > 1 and raw[p] == '0' and raw[p + 1].isdigit():
+                self._raise('Invalid number literal')
 
-            if not c:
-                self._raise('Unexpected end of input')
+        if raw == '-' or (self._allow_extended_number_literals and raw == '+'):
+            for svs in [
+                'Infinity',
+                *(['NaN'] if self._allow_extended_number_literals else []),
+            ]:
+                if c != svs[0]:
+                    continue
 
-            raw += c
-            try:
-                for _ in range(7):
-                    raw += self._char_in((yield None))  # noqa
-            except GeneratorExit:
-                self._raise('Unexpected end of input')
+                if not c:
+                    self._raise('Unexpected end of input')
 
-            if raw != '-Infinity':
-                self._raise(f'Invalid number format: {raw}')
+                raw += c
+                try:
+                    for _ in range(len(svs) - 1):
+                        c = self._char_in((yield None))  # noqa
+                        if not c:
+                            break
+                        raw += c
+                except GeneratorExit:
+                    self._raise('Unexpected end of input')
+
+                if raw[1:] != svs:
+                    self._raise(f'Invalid number format: {raw}')
 
-            tk, tv = CONST_TOKENS[raw]
-            yield self._make_tok(tk, tv, raw, pos)
+                if raw[0] == '+':
+                    raw = raw[1:]
 
-            return self._do_main()
+                yield self._make_tok('IDENT', raw, raw, pos)
+
+                return self._do_main()
 
         #
 
-        if '.' in raw or 'e' in raw or 'E' in raw:
-            nv = float(raw)
+        nv: ta.Any
+
+        if (np := self._number_literal_parser) is not None:
+            nv = np(raw)
+
         else:
-            nv = int(raw)
+            if not NUMBER_PAT.fullmatch(raw):
+                self._raise(f'Invalid number format: {raw}')
+
+            if '.' in raw or 'e' in raw or 'E' in raw:
+                nv = float(raw)
+            else:
+                nv = int(raw)
+
         yield self._make_tok('NUMBER', nv, raw, pos)
 
         #
@@ -322,17 +400,79 @@ class JsonStreamLexer(GenMachine[str, Token]):
             except GeneratorExit:
                 self._raise('Unexpected end of input')
 
-            if raw in CONST_TOKENS:
+            if raw in CONST_IDENT_VALUES:
                 break
 
-            if len(raw) > 8:  # None of the keywords are longer than 8 characters
+            if len(raw) > MAX_CONST_IDENT_LEN:
                 self._raise(f'Invalid literal: {raw}')
 
-        tk, tv = CONST_TOKENS[raw]
-        yield self._make_tok(tk, tv, raw, pos)
+        yield self._make_tok('IDENT', raw, raw, pos)
 
         return self._do_main()
 
+    def _do_unicode_escape(self):
+        try:
+            c = self._char_in((yield None))  # noqa
+        except GeneratorExit:
+            self._raise('Unexpected end of input')
+
+        if c != 'u':
+            self._raise('Illegal identifier escape')
+
+        ux = []
+        for _ in range(4):
+            try:
+                c = self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if c not in '0123456789abcdefABCDEF':
+                self._raise('Illegal identifier escape')
+
+            ux.append(c)
+
+        return chr(int(''.join(ux), 16))
+
+    def _do_extended_ident(self, c: str):
+        check.state(self._buf.tell() == 0)
+
+        if c == '\\':
+            c = yield from self._do_unicode_escape()
+
+        elif not (c in '$_' or unicodedata.category(c).startswith('L')):
+            self._raise('Illegal identifier start')
+
+        self._buf.write(c)
+
+        pos = self.pos
+
+        while True:
+            try:
+                c = self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if c == '\\':
+                c = yield from self._do_unicode_escape()
+                self._buf.write(c)
+                continue
+
+            if not c:
+                break
+
+            if c not in '$_\u200C\u200D':
+                uc = unicodedata.category(c)
+                if not (uc.startswith(('L', 'M', 'N')) or uc == 'Pc'):
+                    break
+
+            self._buf.write(c)
+
+        raw = self._flip_buf()
+
+        yield self._make_tok('IDENT', raw, raw, pos)
+
+        return self._do_main(c)
+
     def _do_comment(self):
         check.state(self._buf.tell() == 0)
 
@@ -349,7 +489,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
                 except GeneratorExit:
                     self._raise('Unexpected end of input')
 
-                if ic == '\n':
+                if not ic or ic == '\n':
                     break
 
                 if self._include_comments:
@@ -360,6 +500,9 @@ class JsonStreamLexer(GenMachine[str, Token]):
                 raw = f'//{cmt}\n' if self._include_raw else None
                 yield self._make_tok('COMMENT', cmt, raw, pos)
 
+            if not ic:
+                return
+
         elif oc == '*':
             lc: str | None = None
             while True:
@@ -382,3 +525,5 @@ class JsonStreamLexer(GenMachine[str, Token]):
 
         else:
             self._raise(f'Unexpected character after comment start: {oc}')
+
+        return self._do_main()

omlish/formats/json/stream/parsing.py

@@ -4,6 +4,7 @@ import typing as ta
 from .... import lang
 from ....funcs.genmachine import GenMachine
 from .errors import JsonStreamError
+from .lexing import CONST_IDENT_VALUES
 from .lexing import SCALAR_VALUE_TYPES
 from .lexing import VALUE_TOKEN_KINDS
 from .lexing import Position
@@ -34,7 +35,7 @@ class EndArray(lang.Marker):
     pass
 
 
-JsonStreamParserEvent: ta.TypeAlias = ta.Union[  # noqa
+Event: ta.TypeAlias = ta.Union[  # noqa
     type[BeginObject],
     Key,
     type[EndObject],
@@ -46,7 +47,7 @@ JsonStreamParserEvent: ta.TypeAlias = ta.Union[  # noqa
 ]
 
 
-class JsonStreamParserEvents(lang.Namespace):
+class Events(lang.Namespace):
     BeginObject = BeginObject
     Key = Key
     EndObject = EndObject
@@ -58,7 +59,7 @@ class JsonStreamParserEvents(lang.Namespace):
 ##
 
 
-def yield_parser_events(obj: ta.Any) -> ta.Generator[JsonStreamParserEvent]:
+def yield_parser_events(obj: ta.Any) -> ta.Iterator[Event]:
     if isinstance(obj, SCALAR_VALUE_TYPES):
         yield obj  # type: ignore
 
@@ -94,12 +95,22 @@ class JsonStreamObject(list):
         return f'{self.__class__.__name__}({super().__repr__()})'
 
 
-class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
-    def __init__(self) -> None:
-        super().__init__(self._do_value())
+class JsonStreamParser(GenMachine[Token, Event]):
+    def __init__(
+            self,
+            *,
+            allow_trailing_commas: bool = False,
+
+            allow_extended_idents: bool = False,
+    ) -> None:
+        self._allow_trailing_commas = allow_trailing_commas
+
+        self._allow_extended_idents = allow_extended_idents
 
         self._stack: list[ta.Literal['OBJECT', 'KEY', 'ARRAY']] = []
 
+        super().__init__(self._do_value())
+
     #
 
     def _next_tok(self):
@@ -144,12 +155,23 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
                 raise JsonStreamParseError('Expected value') from None
             else:
                 raise
+        # except Exception as e:
+        #     raise
 
         if tok.kind in VALUE_TOKEN_KINDS:
             y, r = self._emit_event(tok.value)
             yield y
             return r
 
+        elif tok.kind == 'IDENT':
+            try:
+                cv = CONST_IDENT_VALUES[tok.value]
+            except KeyError:
+                raise JsonStreamParseError('Expected value', tok.pos) from None
+            y, r = self._emit_event(cv)
+            yield y
+            return r
+
         elif tok.kind == 'LBRACE':
             y, r = self._emit_begin_object()
             yield y
@@ -193,7 +215,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         except GeneratorExit:
             raise JsonStreamParseError('Expected object body') from None
 
-        if tok.kind == 'STRING':
+        if tok.kind == 'STRING' or (self._allow_trailing_commas and tok.kind == 'IDENT'):
             k = tok.value
 
             try:
@@ -225,7 +247,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_object_body(must_be_present=True)
+            return self._do_object_body(must_be_present=not self._allow_trailing_commas)
 
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
@@ -258,7 +280,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_value(must_be_present=True)
+            return self._do_value(must_be_present=not self._allow_trailing_commas)
 
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()

omlish/formats/json/stream/rendering.py

@@ -7,14 +7,14 @@ from .parsing import BeginArray
 from .parsing import BeginObject
 from .parsing import EndArray
 from .parsing import EndObject
-from .parsing import JsonStreamParserEvent
+from .parsing import Event
 from .parsing import Key
 
 
 ##
 
 
-class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]]):
+class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[Event]]):
     def __init__(
             self,
             *,
@@ -36,7 +36,7 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
             self,
             o: ta.Any,
             state: AbstractJsonRenderer.State = AbstractJsonRenderer.State.VALUE,
-    ) -> ta.Generator[str]:
+    ) -> ta.Iterator[str]:
         if self._style is not None:
             pre, post = self._style(o, state)
             yield pre
@@ -52,7 +52,7 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
         if post:
             yield post
 
-    def _render(self, e: JsonStreamParserEvent) -> ta.Generator[str]:
+    def _render(self, e: Event) -> ta.Iterator[str]:
         if self._need_delimit:
             yield self._delimiter
             self._need_delimit = False
@@ -124,12 +124,12 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
         else:
             raise TypeError(e)
 
-    def render(self, events: ta.Iterable[JsonStreamParserEvent]) -> ta.Generator[str]:
+    def render(self, events: ta.Iterable[Event]) -> ta.Iterator[str]:
         for e in events:
             yield from self._render(e)
 
     @classmethod
-    def render_str(cls, i: ta.Iterable[JsonStreamParserEvent], /, **kwargs: ta.Any) -> str:
+    def render_str(cls, i: ta.Iterable[Event], /, **kwargs: ta.Any) -> str:
         out = io.StringIO()
         for s in cls(**kwargs).render(i):
             out.write(s)