omlish 0.0.0.dev423__py3-none-any.whl → 0.0.0.dev484__py3-none-any.whl

omlish/formats/json/stream/parsing.py

@@ -4,6 +4,7 @@ import typing as ta
 from .... import lang
 from ....funcs.genmachine import GenMachine
 from .errors import JsonStreamError
+from .lexing import CONST_IDENT_VALUES
 from .lexing import SCALAR_VALUE_TYPES
 from .lexing import VALUE_TOKEN_KINDS
 from .lexing import Position
@@ -34,7 +35,7 @@ class EndArray(lang.Marker):
     pass
 
 
-JsonStreamParserEvent: ta.TypeAlias = ta.Union[  # noqa
+Event: ta.TypeAlias = ta.Union[  # noqa
     type[BeginObject],
     Key,
     type[EndObject],
@@ -46,7 +47,7 @@ JsonStreamParserEvent: ta.TypeAlias = ta.Union[  # noqa
 ]
 
 
-class JsonStreamParserEvents(lang.Namespace):
+class Events(lang.Namespace):
     BeginObject = BeginObject
     Key = Key
     EndObject = EndObject
@@ -58,7 +59,7 @@ class JsonStreamParserEvents(lang.Namespace):
 ##
 
 
-def yield_parser_events(obj: ta.Any) -> ta.Generator[JsonStreamParserEvent]:
+def yield_parser_events(obj: ta.Any) -> ta.Iterator[Event]:
     if isinstance(obj, SCALAR_VALUE_TYPES):
         yield obj  # type: ignore
 
@@ -94,19 +95,29 @@ class JsonStreamObject(list):
         return f'{self.__class__.__name__}({super().__repr__()})'
 
 
-class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
-    def __init__(self) -> None:
-        super().__init__(self._do_value())
+class JsonStreamParser(GenMachine[Token, Event]):
+    def __init__(
+            self,
+            *,
+            allow_trailing_commas: bool = False,
+
+            allow_extended_idents: bool = False,
+    ) -> None:
+        self._allow_trailing_commas = allow_trailing_commas
+
+        self._allow_extended_idents = allow_extended_idents
 
         self._stack: list[ta.Literal['OBJECT', 'KEY', 'ARRAY']] = []
 
+        super().__init__(self._do_value())
+
     #
 
     def _next_tok(self):
         while True:
             tok = yield None
 
-            if tok.kind != 'SPACE':
+            if tok.kind != 'SPACE' and tok.kind != 'COMMENT':
                 return tok
 
     #
@@ -144,12 +155,23 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
                 raise JsonStreamParseError('Expected value') from None
             else:
                 raise
+        # except Exception as e:
+        #     raise
 
         if tok.kind in VALUE_TOKEN_KINDS:
             y, r = self._emit_event(tok.value)
             yield y
             return r
 
+        elif tok.kind == 'IDENT':
+            try:
+                cv = CONST_IDENT_VALUES[tok.value]
+            except KeyError:
+                raise JsonStreamParseError('Expected value', tok.pos) from None
+            y, r = self._emit_event(cv)
+            yield y
+            return r
+
         elif tok.kind == 'LBRACE':
             y, r = self._emit_begin_object()
             yield y
@@ -193,7 +215,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         except GeneratorExit:
             raise JsonStreamParseError('Expected object body') from None
 
-        if tok.kind == 'STRING':
+        if tok.kind == 'STRING' or (self._allow_trailing_commas and tok.kind == 'IDENT'):
             k = tok.value
 
             try:
@@ -225,7 +247,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_object_body(must_be_present=True)
+            return self._do_object_body(must_be_present=not self._allow_trailing_commas)
 
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
@@ -258,7 +280,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_value(must_be_present=True)
+            return self._do_value(must_be_present=not self._allow_trailing_commas)
 
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()

omlish/formats/json/stream/rendering.py

@@ -7,14 +7,14 @@ from .parsing import BeginArray
 from .parsing import BeginObject
 from .parsing import EndArray
 from .parsing import EndObject
-from .parsing import JsonStreamParserEvent
+from .parsing import Event
 from .parsing import Key
 
 
 ##
 
 
-class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]]):
+class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[Event]]):
     def __init__(
             self,
             *,
@@ -36,7 +36,7 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
             self,
             o: ta.Any,
             state: AbstractJsonRenderer.State = AbstractJsonRenderer.State.VALUE,
-    ) -> ta.Generator[str]:
+    ) -> ta.Iterator[str]:
         if self._style is not None:
             pre, post = self._style(o, state)
             yield pre
@@ -52,7 +52,7 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
         if post:
             yield post
 
-    def _render(self, e: JsonStreamParserEvent) -> ta.Generator[str]:
+    def _render(self, e: Event) -> ta.Iterator[str]:
         if self._need_delimit:
             yield self._delimiter
             self._need_delimit = False
@@ -124,12 +124,12 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
         else:
             raise TypeError(e)
 
-    def render(self, events: ta.Iterable[JsonStreamParserEvent]) -> ta.Generator[str]:
+    def render(self, events: ta.Iterable[Event]) -> ta.Iterator[str]:
         for e in events:
             yield from self._render(e)
 
     @classmethod
-    def render_str(cls, i: ta.Iterable[JsonStreamParserEvent], /, **kwargs: ta.Any) -> str:
+    def render_str(cls, i: ta.Iterable[Event], /, **kwargs: ta.Any) -> str:
         out = io.StringIO()
         for s in cls(**kwargs).render(i):
             out.write(s)

omlish/formats/json/stream/utils.py

@@ -1,56 +1,158 @@
-import dataclasses as dc
+r"""
+TODO:
+ - gson switches
+  - Strictness.LEGACY_STRICT
+   - JsonReader allows the literals true, false and null to have any capitalization, for example fAlSe or NULL
+   - JsonReader supports the escape sequence \', representing a ' (single-quote)
+   - JsonReader supports the escape sequence \LF (with LF being the Unicode character U+000A), resulting in a LF within
+     the read JSON string
+   - JsonReader allows unescaped control characters (U+0000 through U+001F)
+  - Strictness.LENIENT
+   - In lenient mode, all input that is accepted in legacy strict mode is accepted in addition to the following
+     departures from RFC 8259:
+   - Streams that start with the non-execute prefix, ")]'\n"}
+   - Streams that include multiple top-level values. With legacy strict or strict parsing, each stream must contain
+     exactly one top-level value.
+   - Numbers may be NaNs or infinities represented by NaN and (-)Infinity respectively.
+   - End of line comments starting with // or # and ending with a newline character.
+   - C-style comments starting with /* and ending with */. Such comments may not be nested.
+   - Names that are unquoted or 'single quoted'.
+   - Strings that are unquoted or 'single quoted'.
+   - Array elements separated by ; instead of ,.
+   - Unnecessary array separators. These are interpreted as if null was the omitted value.
+   - Names and values separated by = or => instead of :.
+   - Name/value pairs separated by ; instead of ,.
+"""
+import itertools
 import typing as ta
 
 from .... import lang
 from .building import JsonValueBuilder
+from .errors import JsonStreamError
 from .lexing import JsonStreamLexer
+from .lexing import Token
+from .parsing import Event
 from .parsing import JsonStreamParser
 
 
 ##
 
 
-@dc.dataclass(kw_only=True)
 class JsonStreamValueParser(lang.ExitStacked):
-    include_raw: bool = False
-    yield_object_lists: bool = False
+    class Machinery(ta.NamedTuple):
+        lex: JsonStreamLexer
+        parse: JsonStreamParser
+        build: JsonValueBuilder
 
-    #
+    def __init__(self, m: Machinery) -> None:
+        super().__init__()
+
+        self._m = m
 
-    _lex: JsonStreamLexer = dc.field(init=False)
-    _parse: JsonStreamParser = dc.field(init=False)
-    _build: JsonValueBuilder = dc.field(init=False)
+    #
 
     def _enter_contexts(self) -> None:
-        self._lex = JsonStreamLexer(
-            include_raw=self.include_raw,
-        )
+        self._enter_context(self._m.lex)
+        self._enter_context(self._m.parse)
 
-        self._parse = JsonStreamParser()
+    def feed(self, i: ta.Iterable[str]) -> ta.Iterator[ta.Any]:
+        for c in i:
+            for t in self._m.lex(c):
+                for e in self._m.parse(t):
+                    for v in self._m.build(e):  # noqa
+                        yield v
 
-        self._build = JsonValueBuilder(
-            yield_object_lists=self.yield_object_lists,
-        )
+    #
+
+    @classmethod
+    def parse_values(
+            cls,
+            m: Machinery,
+            i: ta.Iterable[str],
+    ) -> ta.Iterator[ta.Any]:
+        with cls(m) as p:
+            yield from p.feed(itertools.chain(i, ['']))
+
+    @classmethod
+    def parse_one_value(
+            cls,
+            m: Machinery,
+            i: ta.Iterable[str],
+    ) -> ta.Any:
+        with cls(m) as p:
+            return next(p.feed(itertools.chain(i, [''])))
+
+    @classmethod
+    def parse_exactly_one_value(
+            cls,
+            m: Machinery,
+            i: ta.Iterable[str],
+    ) -> ta.Any:
+        r: ta.Any
+        r = not_set = object()
+        with cls(m) as p:
+            for v in p.feed(itertools.chain(i, [''])):
+                if r is not_set:
+                    r = v
+                else:
+                    raise JsonStreamError('Unexpected input')
+        if r is not_set:
+            raise JsonStreamError('No value')
+        return r
+
+
+##
+
+
+class DebugJsonStreamValueParser(JsonStreamValueParser):
+    def __init__(self, m: JsonStreamValueParser.Machinery) -> None:
+        super().__init__(m)
+
+        self._chars: list[str] = []
+        self._tokens: list[Token] = []
+        self._events: list[Event] = []
+        self._values: list[ta.Any] = []
 
     def feed(self, i: ta.Iterable[str]) -> ta.Iterator[ta.Any]:
         for c in i:
-            for t in self._lex(c):
-                for e in self._parse(t):
-                    for v in self._build(e):  # noqa
+            self._chars.append(c)
+            for t in self._m.lex(c):
+                self._tokens.append(t)
+                for e in self._m.parse(t):
+                    self._events.append(e)
+                    for v in self._m.build(e):
+                        self._values.append(v)
                         yield v
 
 
-def stream_parse_values(
-        i: ta.Iterable[str],
-        **kwargs: ta.Any,
-) -> ta.Generator[ta.Any]:
-    with JsonStreamValueParser(**kwargs) as p:
-        yield from p.feed(i)
+##
+
+
+def make_machinery(
+        *,
+        include_raw: bool = False,
+        yield_object_lists: bool = False,
+) -> JsonStreamValueParser.Machinery:
+    return JsonStreamValueParser.Machinery(
+        JsonStreamLexer(
+            include_raw=include_raw,
+        ),
+
+        JsonStreamParser(),
+
+        JsonValueBuilder(
+            yield_object_lists=yield_object_lists,
+        ),
+    )
+
+
+def stream_parse_values(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Iterator[ta.Any]:
+    return JsonStreamValueParser.parse_values(make_machinery(**kwargs), i)
+
+
+def stream_parse_one_value(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Any:
+    return JsonStreamValueParser.parse_one_value(make_machinery(**kwargs), i)
 
 
-def stream_parse_one_value(
-        i: ta.Iterable[str],
-        **kwargs: ta.Any,
-) -> ta.Any:
-    with JsonStreamValueParser(**kwargs) as p:
-        return next(p.feed(i))
+def stream_parse_exactly_one_value(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Any:
+    return JsonStreamValueParser.parse_exactly_one_value(make_machinery(**kwargs), i)

omlish/formats/json5/literals.py

@@ -124,7 +124,10 @@ def parse_string_literal(s: str) -> str:
 def parse_number_literal(s: str) -> int | float:
     s = s.lower()
 
-    if 'x' in s:
-        return int(s, 16)
-    else:
-        return float(s)
+    try:
+        if 'x' in s:
+            return int(s, 16)
+        else:
+            return float(s)
+    except ValueError as e:
+        raise Json5Error from e

omlish/formats/json5/parsing.py

@@ -1,100 +1,54 @@
-# ruff: noqa: N802 N803
 import typing as ta
 
-from ...text import antlr
-from ._antlr.Json5Lexer import Json5Lexer  # type: ignore
-from ._antlr.Json5Parser import Json5Parser  # type: ignore
-from ._antlr.Json5Visitor import Json5Visitor  # type: ignore
+from ..json.stream.errors import JsonStreamError
+from ..json.stream.utils import DebugJsonStreamValueParser
+from ..json.stream.utils import JsonStreamValueParser
 from .errors import Json5Error
-from .literals import LITERAL_VALUES
-from .literals import parse_number_literal
-from .literals import parse_string_literal
+from .stream import make_machinery
 
 
 ##
 
 
-class Json5ParseVisitor(antlr.parsing.StandardParseTreeVisitor, Json5Visitor):
-    def visitArr(self, ctx: Json5Parser.ArrContext):
-        return [self.visit(e) for e in ctx.value()]
+def parse(
+        buf: str,
+        *,
+        debug: bool = False,
+) -> ta.Any:
+    m = make_machinery()
 
-    def visitKey(self, ctx: Json5Parser.KeyContext):
-        if (s := ctx.STRING()) is not None:
-            return parse_string_literal(s.getText())
+    if debug:
+        vc: type[JsonStreamValueParser] = DebugJsonStreamValueParser
+    else:
+        vc = JsonStreamValueParser
 
-        elif (i := ctx.IDENTIFIER()) is not None:
-            return parse_string_literal(''.join(['"', i.getText(), '"']))
-
-        elif (l := ctx.LITERAL()) is not None:
-            return LITERAL_VALUES[l.getText()]
-
-        elif (n := ctx.NUMERIC_LITERAL()) is not None:
-            return n.getText()
-
-        else:
-            raise RuntimeError(ctx)
-
-    def visitNumber(self, ctx: Json5Parser.NumberContext):
-        return parse_number_literal(ctx.getText())
-
-    def visitObj(self, ctx: Json5Parser.ObjContext):
-        dct: dict[ta.Any, ta.Any] = {}
-        for pair in ctx.pair():
-            key, value = self.visit(pair)
-            dct[key] = value
-        return dct
-
-    def visitPair(self, ctx: Json5Parser.PairContext):
-        key = self.visit(ctx.key())
-        value = self.visit(ctx.value())
-        return (key, value)
-
-    def visitValue(self, ctx: Json5Parser.ValueContext):
-        if (s := ctx.STRING()) is not None:
-            return parse_string_literal(s.getText())
-
-        elif (n := ctx.LITERAL()) is not None:
-            return LITERAL_VALUES[n.getText()]
-
-        else:
-            return super().visitChildren(ctx)
-
-
-def _make_parser(buf: str) -> Json5Parser:
-    return antlr.parsing.make_parser(
-        buf,
-        Json5Lexer,
-        Json5Parser,
-        silent_errors=True,
-    )
-
-
-def parse(buf: str) -> ta.Any:
     try:
-        root = _make_parser(buf).json5()
+        return vc.parse_exactly_one_value(m, buf)
+
+    except Json5Error:
+        raise
 
-    except antlr.errors.ParseError as e:
+    except JsonStreamError as e:
         raise Json5Error from e
 
-    if antlr.parsing.is_eof_context(root):
-        raise Json5Error('Empty input')
 
-    visitor = Json5ParseVisitor()
-    return visitor.visit(root)
+def parse_many(
+        buf: str,
+        *,
+        debug: bool = False,
+) -> ta.Iterator[ta.Any]:
+    m = make_machinery()
 
+    if debug:
+        vc: type[JsonStreamValueParser] = DebugJsonStreamValueParser
+    else:
+        vc = JsonStreamValueParser
 
-def parse_many(buf: str) -> ta.Generator[ta.Any]:
     try:
-        parser = _make_parser(buf)
-
-        while True:
-            if parser.getInputStream().LT(1).type == antlr.runtime.Token.EOF:
-                break
-
-            value = parser.value()
+        yield from vc.parse_values(m, buf)
 
-            visitor = Json5ParseVisitor()
-            yield visitor.visit(value)
+    except Json5Error:
+        raise
 
-    except antlr.errors.ParseError as e:
+    except JsonStreamError as e:
         raise Json5Error from e

omlish/formats/json5/stream.py

@@ -0,0 +1,77 @@
+"""
+Objects:
+ + Object keys may be an ECMAScript 5.1 IdentifierName.
+ + Objects may have a single trailing comma.
+Arrays:
+ + Arrays may have a single trailing comma.
+Strings:
+ + Strings may be single quoted.
+ + Strings may span multiple lines by escaping new line characters.
+ + Strings may include character escapes.
+Numbers:
+ + Numbers may be hexadecimal.
+ + Numbers may have a leading or trailing decimal point.
+ + Numbers may be IEEE 754 positive infinity, negative infinity, and NaN.
+ + Numbers may begin with an explicit plus sign.
+Comments:
+ + Single and multi-line comments are allowed.
+White Space:
+ + Additional white space characters are allowed.
+"""
+import typing as ta
+
+from ..json.stream.building import JsonValueBuilder
+from ..json.stream.lexing import JsonStreamLexer
+from ..json.stream.parsing import JsonStreamParser
+from ..json.stream.utils import JsonStreamValueParser
+from .literals import parse_number_literal
+from .literals import parse_string_literal
+
+
+##
+
+
+def make_machinery(
+        *,
+        include_raw: bool = False,
+        yield_object_lists: bool = False,
+) -> JsonStreamValueParser.Machinery:
+    return JsonStreamValueParser.Machinery(
+        JsonStreamLexer(
+            include_raw=include_raw,
+
+            allow_extended_space=True,
+
+            allow_comments=True,
+
+            allow_single_quotes=True,
+            string_literal_parser=parse_string_literal,
+
+            allow_extended_number_literals=True,
+            number_literal_parser=parse_number_literal,
+
+            allow_extended_idents=True,
+        ),
+
+        JsonStreamParser(
+            allow_trailing_commas=True,
+
+            allow_extended_idents=True,
+        ),
+
+        JsonValueBuilder(
+            yield_object_lists=yield_object_lists,
+        ),
+    )
+
+
+def stream_parse_values(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Iterator[ta.Any]:
+    return JsonStreamValueParser.parse_values(make_machinery(**kwargs), i)
+
+
+def stream_parse_one_value(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Any:
+    return JsonStreamValueParser.parse_one_value(make_machinery(**kwargs), i)
+
+
+def stream_parse_exactly_one_value(i: ta.Iterable[str], **kwargs: ta.Any) -> ta.Any:
+    return JsonStreamValueParser.parse_exactly_one_value(make_machinery(**kwargs), i)

omlish/formats/logfmt.py

@@ -20,7 +20,10 @@ def logfmt_encode(
 ) -> str:
     def encode(s: str) -> str:
         if _LOGFMT_ENCODE_WS_PAT.search(s) is not None:
-            return '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"'
+            s = s.replace('\\', '\\\\')
+            s = s.replace('\n', '\\n')
+            s = s.replace('"', '\\"')
+            return f'"{s}"'
         else:
             return s
 
@@ -67,7 +70,10 @@ def logfmt_decode(
         if s.startswith('"'):
             if len(s) < 2 or not s.endswith('"'):
                 raise ValueError(s)
-            s = s[1:-1].replace('\\"', '"').replace('\\\\', '\\')
+            s = s[1:-1]
+            s = s.replace('\\"', '"')
+            s = s.replace('\\n', '\n')
+            s = s.replace('\\\\', '\\')
         return s
 
     if value_decoder is None:

omlish/funcs/genmachine.py

@@ -45,7 +45,7 @@ class GenMachine(ta.Generic[I, O]):
     _gen: MachineGen | None
 
     def __repr__(self) -> str:
-        return f'{self.__class__.__name__}@{hex(id(self))[2:]}<{self.state}>'
+        return f'{self.__class__.__name__}@{id(self):x}<{self.state}>'
 
     #
 
@@ -88,7 +88,7 @@ class GenMachine(ta.Generic[I, O]):
 
     #
 
-    def __call__(self, i: I) -> ta.Iterable[O]:
+    def __call__(self, i: I) -> ta.Iterator[O]:
         if self._gen is None:
             raise GenMachine.ClosedError