omlish 0.0.0.dev79__tar.gz → 0.0.0.dev81__tar.gz

{omlish-0.0.0.dev79/omlish.egg-info → omlish-0.0.0.dev81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev79
+Version: 0.0.0.dev81
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -19,7 +19,7 @@ Requires-Dist: greenlet~=3.1; extra == "all"
 Requires-Dist: trio~=0.27; extra == "all"
 Requires-Dist: trio-asyncio~=0.15; extra == "all"
 Requires-Dist: lz4~=4.3; extra == "all"
-Requires-Dist: python-snappy~=0.7; python_version < "3.13" and extra == "all"
+Requires-Dist: python-snappy~=0.7; extra == "all"
 Requires-Dist: zstd~=1.5; extra == "all"
 Requires-Dist: asttokens~=2.4; extra == "all"
 Requires-Dist: executing~=2.1; extra == "all"
@@ -38,7 +38,7 @@ Requires-Dist: pg8000~=1.31; extra == "all"
 Requires-Dist: pymysql~=1.1; extra == "all"
 Requires-Dist: aiomysql~=0.2; extra == "all"
 Requires-Dist: aiosqlite~=0.20; extra == "all"
-Requires-Dist: asyncpg~=0.30; python_version < "3.13" and extra == "all"
+Requires-Dist: asyncpg~=0.30; extra == "all"
 Requires-Dist: apsw~=3.46; extra == "all"
 Requires-Dist: sqlean.py~=3.45; python_version < "3.13" and extra == "all"
 Requires-Dist: duckdb~=1.1; extra == "all"
@@ -58,7 +58,7 @@ Requires-Dist: trio~=0.27; extra == "async"
 Requires-Dist: trio-asyncio~=0.15; extra == "async"
 Provides-Extra: compress
 Requires-Dist: lz4~=4.3; extra == "compress"
-Requires-Dist: python-snappy~=0.7; python_version < "3.13" and extra == "compress"
+Requires-Dist: python-snappy~=0.7; extra == "compress"
 Requires-Dist: zstd~=1.5; extra == "compress"
 Provides-Extra: diag
 Requires-Dist: asttokens~=2.4; extra == "diag"
@@ -84,7 +84,7 @@ Requires-Dist: pg8000~=1.31; extra == "sqldrivers"
 Requires-Dist: pymysql~=1.1; extra == "sqldrivers"
 Requires-Dist: aiomysql~=0.2; extra == "sqldrivers"
 Requires-Dist: aiosqlite~=0.20; extra == "sqldrivers"
-Requires-Dist: asyncpg~=0.30; python_version < "3.13" and extra == "sqldrivers"
+Requires-Dist: asyncpg~=0.30; extra == "sqldrivers"
 Requires-Dist: apsw~=3.46; extra == "sqldrivers"
 Requires-Dist: sqlean.py~=3.45; python_version < "3.13" and extra == "sqldrivers"
 Requires-Dist: duckdb~=1.1; extra == "sqldrivers"

{omlish-0.0.0.dev79 → omlish-0.0.0.dev81}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev79'
-__revision__ = '8512759fd4d12a86592f3c891711e4200a150442'
+__version__ = '0.0.0.dev81'
+__revision__ = 'a6b71055e7603077b28ba1999d389905e5663aac'
 
 
 #
@@ -45,7 +45,7 @@ class Project(ProjectBase):
             'lz4 ~= 4.3',
             # 'lz4 @ git+https://github.com/wrmsr/python-lz4@wrmsr_20240830_GIL_NOT_USED'
 
-            'python-snappy ~= 0.7; python_version < "3.13"',
+            'python-snappy ~= 0.7',
 
             'zstd ~= 1.5',
         ],
@@ -97,7 +97,7 @@ class Project(ProjectBase):
 
             'aiomysql ~= 0.2',
             'aiosqlite ~= 0.20',
-            'asyncpg ~= 0.30; python_version < "3.13"',
+            'asyncpg ~= 0.30',
 
             'apsw ~= 3.46',
 

{omlish-0.0.0.dev79 → omlish-0.0.0.dev81}/omlish/formats/json/cli.py

@@ -1,15 +1,21 @@
 import argparse
+import codecs
 import contextlib
 import dataclasses as dc
 import enum
+import io
 import json
+import os
 import subprocess
 import sys
 import typing as ta
 
+from ... import check
 from ... import lang
 from ... import term
 from .render import JsonRenderer
+from .stream import JsonStreamLexer
+from .stream import JsonStreamValueBuilder
 
 
 if ta.TYPE_CHECKING:
@@ -67,15 +73,25 @@ def _main() -> None:
     parser = argparse.ArgumentParser()
 
     parser.add_argument('file', nargs='?')
+
+    parser.add_argument('--stream', action='store_true')
+    parser.add_argument('--stream-buffer-size', type=int, default=0x1000)
+
     parser.add_argument('-f', '--format')
+
     parser.add_argument('-z', '--compact', action='store_true')
     parser.add_argument('-p', '--pretty', action='store_true')
     parser.add_argument('-i', '--indent')
     parser.add_argument('-s', '--sort-keys', action='store_true')
+
     parser.add_argument('-c', '--color', action='store_true')
+
     parser.add_argument('-l', '--less', action='store_true')
+
     args = parser.parse_args()
 
+    #
+
     separators = None
     if args.compact:
         separators = (',', ':')
@@ -89,6 +105,28 @@ def _main() -> None:
         except ValueError:
             indent = args.indent
 
+    kw: dict[str, ta.Any] = dict(
+        indent=indent,
+        separators=separators,
+        sort_keys=args.sort_keys,
+    )
+
+    def render_one(v: ta.Any) -> str:
+        if args.color:
+            return JsonRenderer.render_str(
+                v,
+                **kw,
+                style=term_color,
+            )
+
+        else:
+            return json.dumps(
+                v,
+                **kw,
+            )
+
+    #
+
     fmt_name = args.format
     if fmt_name is None:
         if args.file is not None:
@@ -99,45 +137,71 @@ def _main() -> None:
         fmt_name = 'json'
     fmt = FORMATS_BY_NAME[fmt_name]
 
+    if args.stream:
+        check.arg(fmt is Formats.JSON.value)
+
+    #
+
     with contextlib.ExitStack() as es:
         if args.file is None:
-            in_file = sys.stdin
+            in_file = sys.stdin.buffer
+
         else:
-            in_file = es.enter_context(open(args.file))
+            in_file = es.enter_context(open(args.file, 'rb'))
 
-        data = fmt.load(in_file)
+        #
 
-    kw: dict[str, ta.Any] = dict(
-        indent=indent,
-        separators=separators,
-        sort_keys=args.sort_keys,
-    )
+        if args.less:
+            less = subprocess.Popen(
+                [
+                    'less',
+                    *(['-R'] if args.color else []),
+                ],
+                stdin=subprocess.PIPE,
+                encoding='utf-8',
+            )
+            out = check.not_none(less.stdin)
 
-    if args.color:
-        out = JsonRenderer.render_str(
-            data,
-            **kw,
-            style=term_color,
-        )
+            def close_less():
+                out.close()
+                less.wait()
 
-    else:
-        out = json.dumps(
-            data,
-            **kw,
-        )
-
-    if args.less:
-        subprocess.run(
-            [
-                'less',
-                *(['-R'] if args.color else []),
-            ],
-            input=out.encode(),
-            check=True,
-        )
+            es.enter_context(lang.defer(close_less))  # noqa
 
-    else:
-        print(out)
+        else:
+            out = sys.stdout
+
+        #
+
+        if args.stream:
+            fd = in_file.fileno()
+            decoder = codecs.getincrementaldecoder('utf-8')()
+
+            with contextlib.ExitStack() as es2:
+                lex = es2.enter_context(JsonStreamLexer())
+                vb = es2.enter_context(JsonStreamValueBuilder())
+
+                while True:
+                    buf = os.read(fd, args.stream_buffer_size)
+
+                    for s in decoder.decode(buf, not buf):
+                        n = 0
+                        for c in s:
+                            for t in lex(c):
+                                for v in vb(t):
+                                    print(render_one(v), file=out)
+                                    n += 1
+
+                        if n:
+                            out.flush()
+
+                    if not buf:
+                        break
+
+        else:
+            with io.TextIOWrapper(in_file) as tw:
+                v = fmt.load(tw)
+            print(render_one(v), file=out)
 
 
 if __name__ == '__main__':

{omlish-0.0.0.dev79 → omlish-0.0.0.dev81}/omlish/formats/json/render.py

@@ -1,3 +1,7 @@
+"""
+TODO:
+ - genmachine...
+"""
 import enum
 import io
 import json

omlish-0.0.0.dev81/omlish/formats/json/stream.py

@@ -0,0 +1,389 @@
+import dataclasses as dc
+import io
+import json
+import re
+import typing as ta
+
+from ... import check
+from ...genmachine import GenMachine
+
+
+ValueTokenKind: ta.TypeAlias = ta.Literal[
+    'STRING',
+    'NUMBER',
+
+    'SPECIAL_NUMBER',
+    'BOOLEAN',
+    'NULL',
+]
+
+ControlTokenKind: ta.TypeAlias = ta.Literal[
+    'LBRACE',
+    'RBRACE',
+    'LBRACKET',
+    'RBRACKET',
+    'COMMA',
+    'COLON',
+]
+
+TokenKind: ta.TypeAlias = ValueTokenKind | ControlTokenKind
+
+TokenValue: ta.TypeAlias = str | float | int | None
+
+
+class Token(ta.NamedTuple):
+    kind: TokenKind
+    value: TokenValue
+    raw: str
+
+    ofs: int
+    line: int
+    col: int
+
+    def __iter__(self):
+        raise TypeError
+
+
+NUMBER_PAT = re.compile(r'-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?')
+
+VALUE_TOKEN_KINDS = frozenset(check.isinstance(a, str) for a in ta.get_args(ValueTokenKind))
+
+CONTROL_TOKENS: ta.Mapping[str, TokenKind] = {
+    '{': 'LBRACE',
+    '}': 'RBRACE',
+    '[': 'LBRACKET',
+    ']': 'RBRACKET',
+    ',': 'COMMA',
+    ':': 'COLON',
+}
+
+CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
+    'NaN': ('SPECIAL_NUMBER', float('nan')),
+    'Infinity': ('SPECIAL_NUMBER', float('inf')),
+    '-Infinity': ('SPECIAL_NUMBER', float('-inf')),
+
+    'true': ('BOOLEAN', True),
+    'false': ('BOOLEAN', False),
+    'null': ('NULL', None),
+}
+
+
+@dc.dataclass(frozen=True)
+class JsonLexError(Exception):
+    message: str
+
+    ofs: int
+    line: int
+    col: int
+
+
+class JsonStreamLexer(GenMachine[str, Token]):
+    def __init__(self) -> None:
+        self._ofs = 0
+        self._line = 0
+        self._col = 0
+
+        self._buf = io.StringIO()
+
+        super().__init__(self._do_main())
+
+    def _char_in(self, c: str) -> str:
+        if len(c) != 1:
+            raise ValueError(c)
+
+        self._ofs += 1
+
+        if c == '\n':
+            self._line += 1
+            self._col = 0
+        else:
+            self._col += 1
+
+        return c
+
+    def _make_tok(
+            self,
+            kind: TokenKind,
+            value: TokenValue,
+            raw: str,
+    ) -> ta.Sequence[Token]:
+        tok = Token(
+            kind,
+            value,
+            raw,
+            self._ofs,
+            self._line,
+            self._col,
+        )
+        return (tok,)
+
+    def _flip_buf(self) -> str:
+        raw = self._buf.getvalue()
+        self._buf.seek(0)
+        self._buf.truncate()
+        return raw
+
+    def _raise(self, msg: str) -> ta.NoReturn:
+        raise JsonLexError(msg, self._ofs, self._line, self._col)
+
+    def _do_main(self):
+        while True:
+            c = self._char_in((yield None))  # noqa
+
+            if c.isspace():
+                continue
+
+            if c in CONTROL_TOKENS:
+                yield self._make_tok(CONTROL_TOKENS[c], c, c)
+                continue
+
+            if c == '"':
+                return self._do_string()
+
+            if c.isdigit() or c == '-':
+                return self._do_number(c)
+
+            if c in 'tfnIN':
+                return self._do_const(c)
+
+            self._raise(f'Unexpected character: {c}')
+
+    def _do_string(self):
+        self._buf.write('"')
+
+        last = None
+        while True:
+            try:
+                c = self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            self._buf.write(c)
+            if c == '"' and last != '\\':
+                break
+            last = c
+
+        raw = self._flip_buf()
+        sv = json.loads(raw)
+        yield self._make_tok('STRING', sv, raw)
+
+        return self._do_main()
+
+    def _do_number(self, c: str):
+        self._buf.write(c)
+
+        while True:
+            try:
+                c = self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if not (c.isdigit() or c in '.eE+-'):
+                break
+            self._buf.write(c)
+
+        raw = self._flip_buf()
+        if not NUMBER_PAT.fullmatch(raw):
+            raw += c
+            try:
+                for _ in range(7):
+                    raw += self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if raw != '-Infinity':
+                self._raise(f'Invalid number format: {raw}')
+
+            tk, tv = CONST_TOKENS[raw]
+            yield self._make_tok(tk, tv, raw)
+
+            return self._do_main()
+
+        nv = float(raw) if '.' in raw or 'e' in raw or 'E' in raw else int(raw)
+        yield self._make_tok('NUMBER', nv, raw)
+
+        if c in CONTROL_TOKENS:
+            yield self._make_tok(CONTROL_TOKENS[c], c, c)
+        elif not c.isspace():
+            self._raise(f'Unexpected character after number: {c}')
+
+        return self._do_main()
+
+    def _do_const(self, c: str):
+        raw = c
+        while True:
+            try:
+                raw += self._char_in((yield None))  # noqa
+            except GeneratorExit:
+                self._raise('Unexpected end of input')
+
+            if raw in CONST_TOKENS:
+                break
+
+            if len(raw) > 8:  # None of the keywords are longer than 8 characters
+                self._raise(f'Invalid literal: {raw}')
+
+        tk, tv = CONST_TOKENS[raw]
+        yield self._make_tok(tk, tv, raw)
+
+        return self._do_main()
+
+
+class JsonStreamObject(list):
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({super().__repr__()})'
+
+
+class JsonStreamValueBuilder(GenMachine[Token, ta.Any]):
+    def __init__(
+            self,
+            *,
+            yield_object_lists: bool = False,
+    ) -> None:
+        super().__init__(self._do_value())
+
+        self._yield_object_lists = yield_object_lists
+
+        self._stack: list[
+            tuple[ta.Literal['OBJECT'], JsonStreamObject] |
+            tuple[ta.Literal['PAIR'], str] |
+            tuple[ta.Literal['ARRAY'], list]
+        ] = []
+
+    #
+
+    def _emit_value(self, v):
+        if not self._stack:
+            return ((v,), self._do_value())
+
+        tt, tv = self._stack[-1]
+        if tt == 'PAIR':
+            self._stack.pop()
+            if not self._stack:
+                raise self.StateError
+
+            tt2, tv2 = self._stack[-1]
+            if tt2 == 'OBJECT':
+                tv2.append((tv, v))  # type: ignore
+                return ((), self._do_after_pair())
+
+            else:
+                raise self.StateError
+
+        elif tt == 'ARRAY':
+            tv.append(v)  # type: ignore
+            return ((), self._do_after_element())
+
+        else:
+            raise self.StateError
+
+    #
+
+    def _do_value(self):
+        try:
+            tok = yield None
+        except GeneratorExit:
+            if self._stack:
+                raise self.StateError from None
+            else:
+                raise
+
+        if tok.kind in VALUE_TOKEN_KINDS:
+            y, r = self._emit_value(tok.value)
+            yield y
+            return r
+
+        elif tok.kind == 'LBRACE':
+            return self._do_object()
+
+        elif tok.kind == 'LBRACKET':
+            return self._do_array()
+
+        else:
+            raise self.StateError
+
+    #
+
+    def _do_object(self):
+        self._stack.append(('OBJECT', JsonStreamObject()))
+        return self._do_object_body()
+
+    def _do_object_body(self):
+        try:
+            tok = yield None
+        except GeneratorExit:
+            raise self.StateError from None
+
+        if tok.kind == 'STRING':
+            k = tok.value
+
+            try:
+                tok = yield None
+            except GeneratorExit:
+                raise self.StateError from None
+            if tok.kind != 'COLON':
+                raise self.StateError
+
+            self._stack.append(('PAIR', k))
+            return self._do_value()
+
+        else:
+            raise self.StateError
+
+    def _do_after_pair(self):
+        try:
+            tok = yield None
+        except GeneratorExit:
+            raise self.StateError from None
+
+        if tok.kind == 'COMMA':
+            return self._do_object_body()
+
+        elif tok.kind == 'RBRACE':
+            if not self._stack:
+                raise self.StateError
+
+            tv: ta.Any
+            tt, tv = self._stack.pop()
+            if tt != 'OBJECT':
+                raise self.StateError
+
+            if not self._yield_object_lists:
+                tv = dict(tv)
+
+            y, r = self._emit_value(tv)
+            yield y
+            return r
+
+        else:
+            raise self.StateError
+
+    #
+
+    def _do_array(self):
+        self._stack.append(('ARRAY', []))
+        return self._do_value()
+
+    def _do_after_element(self):
+        try:
+            tok = yield None
+        except GeneratorExit:
+            raise self.StateError from None
+
+        if tok.kind == 'COMMA':
+            return self._do_value()
+
+        elif tok.kind == 'RBRACKET':
+            if not self._stack:
+                raise self.StateError
+
+            tt, tv = self._stack.pop()
+            if tt != 'ARRAY':
+                raise self.StateError
+
+            y, r = self._emit_value(tv)
+            yield y
+            return r
+
+        else:
+            raise self.StateError