omlish 0.0.0.dev81__py3-none-any.whl → 0.0.0.dev83__py3-none-any.whl

omlish/formats/json/consts.py

@@ -0,0 +1,22 @@
+import typing as ta
+
+
+##
+
+
+PRETTY_INDENT = 2
+
+PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=PRETTY_INDENT,
+)
+
+
+##
+
+
+COMPACT_SEPARATORS = (',', ':')
+
+COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=None,
+    separators=COMPACT_SEPARATORS,
+)

omlish/formats/json/encoding.py

@@ -0,0 +1,17 @@
+import json
+
+
+detect_encoding = json.detect_encoding
+
+
+def decodes(s: str | bytes | bytearray) -> str:
+    if isinstance(s, str):
+        if s.startswith('\ufeff'):
+            raise json.JSONDecodeError('Unexpected UTF-8 BOM (decode using utf-8-sig)', s, 0)
+        return s
+
+    elif isinstance(s, (bytes, bytearray)):
+        return s.decode(detect_encoding(s), 'surrogatepass')
+
+    else:
+        raise TypeError(f'the JSON object must be str, bytes or bytearray, not {s.__class__.__name__}')

omlish/formats/json/json.py

@@ -1,47 +1,17 @@
-"""
-TODO:
- - backend abstr
- - streaming
-"""
-import functools
-import json as _json
-import typing as ta
+from .backends import DEFAULT_BACKED
 
 
 ##
 
 
-dump = _json.dump
-dumps = _json.dumps
+dump = DEFAULT_BACKED.dump
+dumps = DEFAULT_BACKED.dumps
 
-detect_encoding = _json.detect_encoding
+load = DEFAULT_BACKED.load
+loads = DEFAULT_BACKED.loads
 
-load = _json.load
-loads = _json.loads
+dump_pretty = DEFAULT_BACKED.dump_pretty
+dumps_pretty = DEFAULT_BACKED.dumps_pretty
 
-
-##
-
-
-PRETTY_INDENT = 2
-
-PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
-    indent=PRETTY_INDENT,
-)
-
-dump_pretty: ta.Callable[..., bytes] = functools.partial(dump, **PRETTY_KWARGS)  # type: ignore
-dumps_pretty: ta.Callable[..., str] = functools.partial(dumps, **PRETTY_KWARGS)
-
-
-##
-
-
-COMPACT_SEPARATORS = (',', ':')
-
-COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
-    indent=None,
-    separators=COMPACT_SEPARATORS,
-)
-
-dump_compact: ta.Callable[..., bytes] = functools.partial(dump, **COMPACT_KWARGS)  # type: ignore
-dumps_compact: ta.Callable[..., str] = functools.partial(dumps, **COMPACT_KWARGS)
+dump_compact = DEFAULT_BACKED.dump_compact
+dumps_compact = DEFAULT_BACKED.dumps_compact

omlish/formats/json/render.py

@@ -1,44 +1,55 @@
-"""
-TODO:
- - genmachine...
-"""
+import abc
+import dataclasses as dc
 import enum
 import io
 import json
 import typing as ta
 
+from ... import lang
 
-class JsonRenderer:
+
+I = ta.TypeVar('I')
+
+
+class JsonRendererOut(ta.Protocol):
+    def write(self, s: str) -> ta.Any: ...
+
+
+class AbstractJsonRenderer(lang.Abstract, ta.Generic[I]):
     class State(enum.Enum):
         VALUE = enum.auto()
         KEY = enum.auto()
 
+    @dc.dataclass(frozen=True, kw_only=True)
+    class Options:
+        indent: int | str | None = None
+        separators: tuple[str, str] | None = None
+        sort_keys: bool = False
+        style: ta.Callable[[ta.Any, 'AbstractJsonRenderer.State'], tuple[str, str]] | None = None
+
     def __init__(
             self,
-            out: ta.TextIO,
-            *,
-            indent: int | str | None = None,
-            separators: tuple[str, str] | None = None,
-            sort_keys: bool = False,
-            style: ta.Callable[[ta.Any, State], tuple[str, str]] | None = None,
+            out: JsonRendererOut,
+            opts: Options = Options(),
     ) -> None:
         super().__init__()
 
         self._out = out
-        if isinstance(indent, (str, int)):
-            self._indent = (' ' * indent) if isinstance(indent, int) else indent
+        self._opts = opts
+
+        separators = opts.separators
+        if isinstance(opts.indent, (str, int)):
+            self._indent = (' ' * opts.indent) if isinstance(opts.indent, int) else opts.indent
             self._endl = '\n'
             if separators is None:
                 separators = (',', ': ')
-        elif indent is None:
+        elif opts.indent is None:
             self._indent = self._endl = ''
             if separators is None:
                 separators = (', ', ': ')
         else:
-            raise TypeError(indent)
+            raise TypeError(opts.indent)
         self._comma, self._colon = separators
-        self._sort_keys = sort_keys
-        self._style = style
 
         self._level = 0
 
@@ -58,9 +69,25 @@ class JsonRenderer:
             if self._level:
                 self._write(self._indent * self._level)
 
-    def _render(self, o: ta.Any, state: State = State.VALUE) -> None:
-        if self._style is not None:
-            pre, post = self._style(o, state)
+    @abc.abstractmethod
+    def render(self, i: I) -> None:
+        raise NotImplementedError
+
+    @classmethod
+    def render_str(cls, i: I, **kwargs: ta.Any) -> str:
+        out = io.StringIO()
+        cls(out, cls.Options(**kwargs)).render(i)
+        return out.getvalue()
+
+
+class JsonRenderer(AbstractJsonRenderer[ta.Any]):
+    def _render(
+            self,
+            o: ta.Any,
+            state: AbstractJsonRenderer.State = AbstractJsonRenderer.State.VALUE,
+    ) -> None:
+        if self._opts.style is not None:
+            pre, post = self._opts.style(o, state)
             self._write(pre)
         else:
             post = None
@@ -75,13 +102,13 @@ class JsonRenderer:
             self._write('{')
             self._level += 1
             items = list(o.items())
-            if self._sort_keys:
+            if self._opts.sort_keys:
                 items.sort(key=lambda t: t[0])
             for i, (k, v) in enumerate(items):
                 if i:
                     self._write(self._comma)
                 self._write_indent()
-                self._render(k, JsonRenderer.State.KEY)
+                self._render(k, AbstractJsonRenderer.State.KEY)
                 self._write(self._colon)
                 self._render(v)
             self._level -= 1
@@ -110,9 +137,3 @@ class JsonRenderer:
 
     def render(self, o: ta.Any) -> None:
         self._render(o)
-
-    @classmethod
-    def render_str(cls, o: ta.Any, **kwargs: ta.Any) -> str:
-        out = io.StringIO()
-        cls(out, **kwargs).render(o)
-        return out.getvalue()

omlish/formats/json/stream/build.py

@@ -0,0 +1,113 @@
+import typing as ta
+
+from ....genmachine import GenMachine
+from .lex import SCALAR_VALUE_TYPES
+from .parse import BeginArray
+from .parse import BeginObject
+from .parse import EndArray
+from .parse import EndObject
+from .parse import JsonStreamObject
+from .parse import JsonStreamParserEvent
+from .parse import Key
+
+
+##
+
+
+class JsonObjectBuilder(GenMachine[JsonStreamParserEvent, ta.Any]):
+    def __init__(
+            self,
+            *,
+            yield_object_lists: bool = False,
+    ) -> None:
+        self._stack: list[JsonStreamObject | list | Key] = []
+        self._yield_object_lists = yield_object_lists
+
+        super().__init__(self._do())
+
+    def _do(self):
+        stk = self._stack
+
+        def emit_value(v):
+            if not stk:
+                return (v,)
+
+            tv = stk[-1]
+            if isinstance(tv, Key):
+                stk.pop()
+                if not stk:
+                    raise self.StateError
+
+                tv2 = stk[-1]
+                if not isinstance(tv2, JsonStreamObject):
+                    raise self.StateError
+
+                tv2.append((tv.key, v))
+                return ()
+
+            elif isinstance(tv, list):
+                tv.append(v)
+                return ()
+
+            else:
+                raise self.StateError
+
+        while True:
+            try:
+                e = yield None
+            except GeneratorExit:
+                if stk:
+                    raise self.StateError from None
+                else:
+                    raise
+
+            #
+
+            if isinstance(e, SCALAR_VALUE_TYPES):
+                if t := emit_value(e):
+                    yield t
+                continue
+
+            #
+
+            elif e is BeginObject:
+                stk.append(JsonStreamObject())
+                continue
+
+            elif isinstance(e, Key):
+                if not stk or not isinstance(stk[-1], JsonStreamObject):
+                    raise self.StateError
+
+                stk.append(e)
+                continue
+
+            elif e is EndObject:
+                tv: ta.Any
+                if not stk or not isinstance(tv := stk.pop(), JsonStreamObject):
+                    raise self.StateError
+
+                if not self._yield_object_lists:
+                    tv = dict(tv)
+
+                if t := emit_value(tv):
+                    yield t
+                continue
+
+            #
+
+            elif e is BeginArray:
+                stk.append([])
+                continue
+
+            elif e is EndArray:
+                if not stk or not isinstance(tv := stk.pop(), list):
+                    raise self.StateError
+
+                if t := emit_value(tv):
+                    yield t
+                continue
+
+            #
+
+            else:
+                raise TypeError(e)

omlish/formats/json/{stream.py → stream/lex.py}

@@ -1,11 +1,20 @@
+"""
+TODO:
+ - max buf size
+ - max recursion depth
+ - mark start pos of tokens, currently returning end
+"""
 import dataclasses as dc
 import io
 import json
 import re
 import typing as ta
 
-from ... import check
-from ...genmachine import GenMachine
+from .... import check
+from ....genmachine import GenMachine
+
+
+##
 
 
 ValueTokenKind: ta.TypeAlias = ta.Literal[
@@ -17,6 +26,8 @@ ValueTokenKind: ta.TypeAlias = ta.Literal[
     'NULL',
 ]
 
+VALUE_TOKEN_KINDS = frozenset(check.isinstance(a, str) for a in ta.get_args(ValueTokenKind))
+
 ControlTokenKind: ta.TypeAlias = ta.Literal[
     'LBRACE',
     'RBRACE',
@@ -28,13 +39,23 @@ ControlTokenKind: ta.TypeAlias = ta.Literal[
 
 TokenKind: ta.TypeAlias = ValueTokenKind | ControlTokenKind
 
-TokenValue: ta.TypeAlias = str | float | int | None
+#
+
+ScalarValue: ta.TypeAlias = str | float | int | None
+
+SCALAR_VALUE_TYPES: tuple[type, ...] = tuple(
+    check.isinstance(e, type) if e is not None else type(None)
+    for e in ta.get_args(ScalarValue)
+)
+
+
+##
 
 
 class Token(ta.NamedTuple):
     kind: TokenKind
-    value: TokenValue
-    raw: str
+    value: ScalarValue
+    raw: str | None
 
     ofs: int
     line: int
@@ -46,8 +67,6 @@ class Token(ta.NamedTuple):
 
 NUMBER_PAT = re.compile(r'-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?')
 
-VALUE_TOKEN_KINDS = frozenset(check.isinstance(a, str) for a in ta.get_args(ValueTokenKind))
-
 CONTROL_TOKENS: ta.Mapping[str, TokenKind] = {
     '{': 'LBRACE',
     '}': 'RBRACE',
@@ -68,6 +87,9 @@ CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
 }
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class JsonLexError(Exception):
     message: str
@@ -78,7 +100,13 @@ class JsonLexError(Exception):
 
 
 class JsonStreamLexer(GenMachine[str, Token]):
-    def __init__(self) -> None:
+    def __init__(
+            self,
+            *,
+            include_raw: bool = False,
+    ) -> None:
+        self._include_raw = include_raw
+
         self._ofs = 0
         self._line = 0
         self._col = 0
@@ -88,7 +116,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
         super().__init__(self._do_main())
 
     def _char_in(self, c: str) -> str:
-        if len(c) != 1:
+        if c and len(c) != 1:
             raise ValueError(c)
 
         self._ofs += 1
@@ -104,13 +132,13 @@ class JsonStreamLexer(GenMachine[str, Token]):
     def _make_tok(
             self,
             kind: TokenKind,
-            value: TokenValue,
+            value: ScalarValue,
             raw: str,
     ) -> ta.Sequence[Token]:
         tok = Token(
             kind,
             value,
-            raw,
+            raw if self._include_raw else None,
             self._ofs,
             self._line,
             self._col,
@@ -130,6 +158,9 @@ class JsonStreamLexer(GenMachine[str, Token]):
         while True:
             c = self._char_in((yield None))  # noqa
 
+            if not c:
+                return None
+
             if c.isspace():
                 continue
 
@@ -158,6 +189,9 @@ class JsonStreamLexer(GenMachine[str, Token]):
             except GeneratorExit:
                 self._raise('Unexpected end of input')
 
+            if not c:
+                raise NotImplementedError
+
             self._buf.write(c)
             if c == '"' and last != '\\':
                 break
@@ -178,12 +212,23 @@ class JsonStreamLexer(GenMachine[str, Token]):
             except GeneratorExit:
                 self._raise('Unexpected end of input')
 
+            if not c:
+                break
+
             if not (c.isdigit() or c in '.eE+-'):
                 break
             self._buf.write(c)
 
         raw = self._flip_buf()
+
+        #
+
         if not NUMBER_PAT.fullmatch(raw):
+            # Can only be -Infinity
+
+            if not c:
+                self._raise('Unexpected end of input')
+
             raw += c
             try:
                 for _ in range(7):
@@ -199,11 +244,22 @@ class JsonStreamLexer(GenMachine[str, Token]):
 
             return self._do_main()
 
-        nv = float(raw) if '.' in raw or 'e' in raw or 'E' in raw else int(raw)
+        #
+
+        if '.' in raw or 'e' in raw or 'E' in raw:
+            nv = float(raw)
+        else:
+            nv = int(raw)
         yield self._make_tok('NUMBER', nv, raw)
 
+        #
+
+        if not c:
+            return None
+
         if c in CONTROL_TOKENS:
             yield self._make_tok(CONTROL_TOKENS[c], c, c)
+
         elif not c.isspace():
             self._raise(f'Unexpected character after number: {c}')
 
@@ -227,163 +283,3 @@ class JsonStreamLexer(GenMachine[str, Token]):
         yield self._make_tok(tk, tv, raw)
 
         return self._do_main()
-
-
-class JsonStreamObject(list):
-    def __repr__(self) -> str:
-        return f'{self.__class__.__name__}({super().__repr__()})'
-
-
-class JsonStreamValueBuilder(GenMachine[Token, ta.Any]):
-    def __init__(
-            self,
-            *,
-            yield_object_lists: bool = False,
-    ) -> None:
-        super().__init__(self._do_value())
-
-        self._yield_object_lists = yield_object_lists
-
-        self._stack: list[
-            tuple[ta.Literal['OBJECT'], JsonStreamObject] |
-            tuple[ta.Literal['PAIR'], str] |
-            tuple[ta.Literal['ARRAY'], list]
-        ] = []
-
-    #
-
-    def _emit_value(self, v):
-        if not self._stack:
-            return ((v,), self._do_value())
-
-        tt, tv = self._stack[-1]
-        if tt == 'PAIR':
-            self._stack.pop()
-            if not self._stack:
-                raise self.StateError
-
-            tt2, tv2 = self._stack[-1]
-            if tt2 == 'OBJECT':
-                tv2.append((tv, v))  # type: ignore
-                return ((), self._do_after_pair())
-
-            else:
-                raise self.StateError
-
-        elif tt == 'ARRAY':
-            tv.append(v)  # type: ignore
-            return ((), self._do_after_element())
-
-        else:
-            raise self.StateError
-
-    #
-
-    def _do_value(self):
-        try:
-            tok = yield None
-        except GeneratorExit:
-            if self._stack:
-                raise self.StateError from None
-            else:
-                raise
-
-        if tok.kind in VALUE_TOKEN_KINDS:
-            y, r = self._emit_value(tok.value)
-            yield y
-            return r
-
-        elif tok.kind == 'LBRACE':
-            return self._do_object()
-
-        elif tok.kind == 'LBRACKET':
-            return self._do_array()
-
-        else:
-            raise self.StateError
-
-    #
-
-    def _do_object(self):
-        self._stack.append(('OBJECT', JsonStreamObject()))
-        return self._do_object_body()
-
-    def _do_object_body(self):
-        try:
-            tok = yield None
-        except GeneratorExit:
-            raise self.StateError from None
-
-        if tok.kind == 'STRING':
-            k = tok.value
-
-            try:
-                tok = yield None
-            except GeneratorExit:
-                raise self.StateError from None
-            if tok.kind != 'COLON':
-                raise self.StateError
-
-            self._stack.append(('PAIR', k))
-            return self._do_value()
-
-        else:
-            raise self.StateError
-
-    def _do_after_pair(self):
-        try:
-            tok = yield None
-        except GeneratorExit:
-            raise self.StateError from None
-
-        if tok.kind == 'COMMA':
-            return self._do_object_body()
-
-        elif tok.kind == 'RBRACE':
-            if not self._stack:
-                raise self.StateError
-
-            tv: ta.Any
-            tt, tv = self._stack.pop()
-            if tt != 'OBJECT':
-                raise self.StateError
-
-            if not self._yield_object_lists:
-                tv = dict(tv)
-
-            y, r = self._emit_value(tv)
-            yield y
-            return r
-
-        else:
-            raise self.StateError
-
-    #
-
-    def _do_array(self):
-        self._stack.append(('ARRAY', []))
-        return self._do_value()
-
-    def _do_after_element(self):
-        try:
-            tok = yield None
-        except GeneratorExit:
-            raise self.StateError from None
-
-        if tok.kind == 'COMMA':
-            return self._do_value()
-
-        elif tok.kind == 'RBRACKET':
-            if not self._stack:
-                raise self.StateError
-
-            tt, tv = self._stack.pop()
-            if tt != 'ARRAY':
-                raise self.StateError
-
-            y, r = self._emit_value(tv)
-            yield y
-            return r
-
-        else:
-            raise self.StateError