omlish 0.0.0.dev80__py3-none-any.whl → 0.0.0.dev82__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev80'
-__revision__ = '7640968fc4c06ed0d54ea53c36daae3574c9071f'
+__version__ = '0.0.0.dev82'
+__revision__ = '025f44593a2c0c59d06340e1d7dbcd15b1ab7684'
 
 
 #
@@ -45,7 +45,7 @@ class Project(ProjectBase):
             'lz4 ~= 4.3',
             # 'lz4 @ git+https://github.com/wrmsr/python-lz4@wrmsr_20240830_GIL_NOT_USED'
 
-            'python-snappy ~= 0.7; python_version < "3.13"',
+            'python-snappy ~= 0.7',
 
             'zstd ~= 1.5',
         ],
@@ -97,7 +97,7 @@ class Project(ProjectBase):
 
             'aiomysql ~= 0.2',
             'aiosqlite ~= 0.20',
-            'asyncpg ~= 0.30; python_version < "3.13"',
+            'asyncpg ~= 0.30',
 
             'apsw ~= 3.46',
 

omlish/dataclasses/impl/__init__.py

@@ -22,4 +22,12 @@ TODO:
  - enums
  - nodal
  - embedding? forward kwargs in general? or only for replace?
+
+TODO refs:
+ - batch up exec calls
+  - https://github.com/python/cpython/commit/8945b7ff55b87d11c747af2dad0e3e4d631e62d6
+ - add doc parameter to dataclasses.field
+  - https://github.com/python/cpython/commit/9c7657f09914254724683d91177aed7947637be5
+ - add decorator argument to make_dataclass
+  - https://github.com/python/cpython/commit/3e3a4d231518f91ff2f3c5a085b3849e32f1d548
 """

omlish/dataclasses/impl/params.py

@@ -12,6 +12,9 @@ class Field_:
     metadata: Metadata | None = None
     kw_only: bool | MISSING = MISSING
 
+    if sys.version_info >= (3, 13):
+        doc: str | None = None
+
     _field_type: Any = None
 
 

omlish/dataclasses/impl/slots.py

@@ -1,5 +1,7 @@
 import dataclasses as dc
+import inspect
 import itertools
+import types
 
 
 MISSING = dc.MISSING
@@ -25,7 +27,7 @@ def _get_slots(cls):
             slots = []
             if getattr(cls, '__weakrefoffset__', -1) != 0:
                 slots.append('__weakref__')
-            if getattr(cls, '__dictrefoffset__', -1) != 0:
+            if getattr(cls, '__dictoffset__', -1) != 0:
                 slots.append('__dict__')
             yield from slots
         case str(slot):
@@ -37,44 +39,96 @@ def _get_slots(cls):
             raise TypeError(f"Slots of '{cls.__name__}' cannot be determined")
 
 
+def _update_func_cell_for__class__(f, oldcls, newcls):
+    # Returns True if we update a cell, else False.
+    if f is None:
+        # f will be None in the case of a property where not all of fget, fset, and fdel are used.  Nothing to do in
+        # that case.
+        return False
+    try:
+        idx = f.__code__.co_freevars.index('__class__')
+    except ValueError:
+        # This function doesn't reference __class__, so nothing to do.
+        return False
+    # Fix the cell to point to the new class, if it's already pointing at the old class.  I'm not convinced that the "is
+    # oldcls" test is needed, but other than performance can't hurt.
+    closure = f.__closure__[idx]
+    if closure.cell_contents is oldcls:
+        closure.cell_contents = newcls
+        return True
+    return False
+
+
 def add_slots(
         cls: type,
         is_frozen: bool,
         weakref_slot: bool,
 ) -> type:
+    # Need to create a new class, since we can't set __slots__ after a class has been created, and the @dataclass
+    # decorator is called after the class is created.
+
+    # Make sure __slots__ isn't already set.
     if '__slots__' in cls.__dict__:
         raise TypeError(f'{cls.__name__} already specifies __slots__')
 
+    # Create a new dict for our new class.
     cls_dict = dict(cls.__dict__)
     field_names = tuple(f.name for f in dc.fields(cls))  # noqa
 
+    # Make sure slots don't overlap with those in base classes.
     inherited_slots = set(itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])))
 
+    # The slots for our class.  Remove slots from our base classes.  Add '__weakref__' if weakref_slot was given, unless
+    # it is already present.
     cls_dict['__slots__'] = tuple(
         itertools.filterfalse(
             inherited_slots.__contains__,
             itertools.chain(
                 field_names,
+                # gh-93521: '__weakref__' also needs to be filtered out if already present in inherited_slots
                 ('__weakref__',) if weakref_slot else (),
             ),
         ),
     )
 
     for field_name in field_names:
+        # Remove our attributes, if present. They'll still be available in _MARKER.
         cls_dict.pop(field_name, None)
 
+    # Remove __dict__ itself.
     cls_dict.pop('__dict__', None)
+
+    # Clear existing `__weakref__` descriptor, it belongs to a previous type:
     cls_dict.pop('__weakref__', None)
 
     qualname = getattr(cls, '__qualname__', None)
-    cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
+    newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
     if qualname is not None:
-        cls.__qualname__ = qualname
+        newcls.__qualname__ = qualname
 
     if is_frozen:
+        # Need this for pickling frozen classes with slots.
         if '__getstate__' not in cls_dict:
-            cls.__getstate__ = _dataclass_getstate  # type: ignore
+            newcls.__getstate__ = _dataclass_getstate  # type: ignore
         if '__setstate__' not in cls_dict:
-            cls.__setstate__ = _dataclass_setstate  # type: ignore
-
-    return cls
+            newcls.__setstate__ = _dataclass_setstate  # type: ignore
+
+    # Fix up any closures which reference __class__.  This is used to fix zero argument super so that it points to the
+    # correct class (the newly created one, which we're returning) and not the original class.  We can break out of this
+    # loop as soon as we make an update, since all closures for a class will share a given cell.
+    for member in newcls.__dict__.values():
+        # If this is a wrapped function, unwrap it.
+        member = inspect.unwrap(member)
+        if isinstance(member, types.FunctionType):
+            if _update_func_cell_for__class__(member, cls, newcls):
+                break
+
+        elif isinstance(member, property):
+            if (
+                    _update_func_cell_for__class__(member.fget, cls, newcls) or
+                    _update_func_cell_for__class__(member.fset, cls, newcls) or
+                    _update_func_cell_for__class__(member.fdel, cls, newcls)
+            ):
+                break
+
+    return newcls

omlish/formats/json/__init__.py

@@ -1,9 +1,16 @@
-from .json import (  # noqa
+from .consts import (  # noqa
     COMPACT_KWARGS,
     COMPACT_SEPARATORS,
     PRETTY_INDENT,
     PRETTY_KWARGS,
+)
+
+from .encoding import (  # noqa
+    decodes,
     detect_encoding,
+)
+
+from .json import (  # noqa
     dump,
     dump_compact,
     dump_pretty,

omlish/formats/json/backends/__init__.py

@@ -0,0 +1,7 @@
+from .base import (  # noqa
+    Backend,
+)
+
+from .default import (  # noqa
+    DEFAULT_BACKED,
+)

omlish/formats/json/backends/base.py

@@ -0,0 +1,38 @@
+import abc
+import typing as ta
+
+from .... import lang
+
+
+class Backend(lang.Abstract):
+    @abc.abstractmethod
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError

omlish/formats/json/backends/default.py

@@ -0,0 +1,10 @@
+from .base import Backend
+from .std import STD_BACKEND
+from .ujson import UJSON_BACKEND
+
+
+DEFAULT_BACKED: Backend
+if UJSON_BACKEND is not None:
+    DEFAULT_BACKED = UJSON_BACKEND
+else:
+    DEFAULT_BACKED = STD_BACKEND

omlish/formats/json/backends/jiter.py

@@ -0,0 +1,25 @@
+"""
+from_json(
+    json_data: bytes,
+    /,
+    *,
+    allow_inf_nan: bool = True,
+    cache_mode: Literal[True, False, "all", "keys", "none"] = "all",
+    partial_mode: Literal[True, False, "off", "on", "trailing-strings"] = False,
+    catch_duplicate_keys: bool = False,
+    lossless_floats: bool = False,
+) -> Any:
+    json_data: The JSON data to parse
+    allow_inf_nan: Whether to allow infinity (`Infinity` an `-Infinity`) and `NaN` values to float fields.
+    Defaults to True.
+    cache_mode: cache Python strings to improve performance at the cost of some memory usage
+    - True / 'all' - cache all strings
+    - 'keys' - cache only object keys
+    - False / 'none' - cache nothing
+    partial_mode: How to handle incomplete strings:
+    - False / 'off' - raise an exception if the input is incomplete
+    - True / 'on' - allow incomplete JSON but discard the last string if it is incomplete
+    - 'trailing-strings' - allow incomplete JSON, and include the last incomplete string in the output
+    catch_duplicate_keys: if True, raise an exception if objects contain the same key multiple times
+    lossless_floats: if True, preserve full detail on floats using `LosslessFloat`
+"""

omlish/formats/json/backends/orjson.py

@@ -1,11 +1,16 @@
 """
-def loads(obj: str | bytes | bytearray | memoryview) -> ta.Any | oj.JSONDEcodeError
-def dumps(obj: ta.Any, **DumpOpts) -> bytes
+loads(obj: str | bytes | bytearray | memoryview) -> ta.Any | oj.JSONDEcodeError
+dumps(
+    obj: ta.Any,
+    default: ta.Callable[[ta.Any], Ata.ny] | None = ...,
+    option: int | None = ...,
+) -> bytes
 """
 import dataclasses as dc
 import typing as ta
 
 from .... import lang
+from .base import Backend
 
 
 if ta.TYPE_CHECKING:
@@ -14,6 +19,9 @@ else:
     oj = lang.proxy_import('orjson')
 
 
+##
+
+
 @dc.dataclass(frozen=True, kw_only=True)
 class Options:
     append_newline: bool = False  # append \n to the output
@@ -70,3 +78,39 @@ class Options:
 class DumpOpts:
     default: ta.Callable[[ta.Any], ta.Any] | None = None
     option: Options = Options()
+
+
+##
+
+
+class OrjsonBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        fp.write(self.dumps(obj, **kwargs))
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return oj.dumps(obj, **kwargs).decode('utf-8')
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return oj.loads(fp.read(), **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return oj.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        fp.write(self.dumps_pretty(obj, **kwargs))
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return self.dumps(obj, option=kwargs.pop('option', 0) | oj.OPT_INDENT_2, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        return self.dump(obj, fp, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return self.dumps(obj, **kwargs)
+
+
+ORJSON_BACKEND: OrjsonBackend | None
+if lang.can_import('orjson'):
+    ORJSON_BACKEND = OrjsonBackend()
+else:
+    ORJSON_BACKEND = None

omlish/formats/json/backends/std.py

@@ -8,6 +8,13 @@ import dataclasses as dc
 import json
 import typing as ta
 
+from ..consts import COMPACT_KWARGS
+from ..consts import PRETTY_KWARGS
+from .base import Backend
+
+
+##
+
 
 @dc.dataclass(frozen=True, kw_only=True)
 class DumpOpts:
@@ -35,3 +42,35 @@ class LoadOpts:
 
     # called with the result of any object literal decoded with an ordered list of pairs, by default dict  # noqa
     object_pairs_hook: ta.Callable[[list[tuple[str, ta.Any]]], ta.Any] | None = None
+
+
+##
+
+
+class StdBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **kwargs)
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **kwargs)
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return json.load(fp, **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return json.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **PRETTY_KWARGS, **kwargs)
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **PRETTY_KWARGS, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **COMPACT_KWARGS, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **COMPACT_KWARGS, **kwargs)
+
+
+STD_BACKEND = StdBackend()

omlish/formats/json/backends/ujson.py

@@ -7,6 +7,19 @@ dumps(obj: ta.Any, **DumpOpts) -> None
 import dataclasses as dc
 import typing as ta
 
+from .... import lang
+from ..consts import PRETTY_INDENT
+from .base import Backend
+
+
+if ta.TYPE_CHECKING:
+    import ujson as uj
+else:
+    uj = lang.proxy_import('ujson')
+
+
+##
+
 
 @dc.dataclass(frozen=True, kw_only=True)
 class DumpOpts:
@@ -23,3 +36,39 @@ class DumpOpts:
     reject_bytes: bool = True
     default: ta.Callable[[ta.Any], ta.Any] | None = None  # should return a serializable version of obj or raise TypeError  # noqa
     separators: tuple[str, str] | None = None
+
+
+##
+
+
+class UjsonBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, **kwargs)
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, **kwargs)
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return uj.load(fp, **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return uj.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, indent=PRETTY_INDENT, **kwargs)
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, indent=PRETTY_INDENT, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, **kwargs)
+
+
+UJSON_BACKEND: UjsonBackend | None
+if lang.can_import('ujson'):
+    UJSON_BACKEND = UjsonBackend()
+else:
+    UJSON_BACKEND = None

omlish/formats/json/cli.py

@@ -1,15 +1,28 @@
+"""
+TODO:
+ - xml - [{"att", {"el", {"cdata", ...
+ - csv - dict if headers, array if not
+"""
 import argparse
+import codecs
 import contextlib
 import dataclasses as dc
 import enum
+import io
 import json
+import os
 import subprocess
 import sys
 import typing as ta
 
+from ... import check
 from ... import lang
 from ... import term
 from .render import JsonRenderer
+from .stream.build import JsonObjectBuilder
+from .stream.lex import JsonStreamLexer
+from .stream.parse import JsonStreamParser
+from .stream.render import StreamJsonRenderer
 
 
 if ta.TYPE_CHECKING:
@@ -67,15 +80,26 @@ def _main() -> None:
     parser = argparse.ArgumentParser()
 
     parser.add_argument('file', nargs='?')
+
+    parser.add_argument('--stream', action='store_true')
+    parser.add_argument('--stream-build', action='store_true')
+    parser.add_argument('--stream-buffer-size', type=int, default=0x4000)
+
     parser.add_argument('-f', '--format')
+
     parser.add_argument('-z', '--compact', action='store_true')
     parser.add_argument('-p', '--pretty', action='store_true')
     parser.add_argument('-i', '--indent')
     parser.add_argument('-s', '--sort-keys', action='store_true')
+
     parser.add_argument('-c', '--color', action='store_true')
+
     parser.add_argument('-l', '--less', action='store_true')
+
     args = parser.parse_args()
 
+    #
+
     separators = None
     if args.compact:
         separators = (',', ':')
@@ -89,6 +113,28 @@ def _main() -> None:
         except ValueError:
             indent = args.indent
 
+    kw: dict[str, ta.Any] = dict(
+        indent=indent,
+        separators=separators,
+        sort_keys=args.sort_keys,
+    )
+
+    def render_one(v: ta.Any) -> str:
+        if args.color:
+            return JsonRenderer.render_str(
+                v,
+                **kw,
+                style=term_color,
+            )
+
+        else:
+            return json.dumps(
+                v,
+                **kw,
+            )
+
+    #
+
     fmt_name = args.format
     if fmt_name is None:
         if args.file is not None:
@@ -99,45 +145,93 @@ def _main() -> None:
         fmt_name = 'json'
     fmt = FORMATS_BY_NAME[fmt_name]
 
+    if args.stream:
+        check.arg(fmt is Formats.JSON.value)
+
+    #
+
     with contextlib.ExitStack() as es:
         if args.file is None:
-            in_file = sys.stdin
+            in_file = sys.stdin.buffer
+
         else:
-            in_file = es.enter_context(open(args.file))
+            in_file = es.enter_context(open(args.file, 'rb'))
 
-        data = fmt.load(in_file)
+        #
 
-    kw: dict[str, ta.Any] = dict(
-        indent=indent,
-        separators=separators,
-        sort_keys=args.sort_keys,
-    )
+        if args.less:
+            less = subprocess.Popen(
+                [
+                    'less',
+                    *(['-R'] if args.color else []),
+                ],
+                stdin=subprocess.PIPE,
+                encoding='utf-8',
+            )
+            out = check.not_none(less.stdin)
 
-    if args.color:
-        out = JsonRenderer.render_str(
-            data,
-            **kw,
-            style=term_color,
-        )
+            def close_less():
+                out.close()
+                less.wait()
 
-    else:
-        out = json.dumps(
-            data,
-            **kw,
-        )
-
-    if args.less:
-        subprocess.run(
-            [
-                'less',
-                *(['-R'] if args.color else []),
-            ],
-            input=out.encode(),
-            check=True,
-        )
+            es.enter_context(lang.defer(close_less))  # noqa
 
-    else:
-        print(out)
+        else:
+            out = sys.stdout
+
+        #
+
+        if args.stream:
+            fd = in_file.fileno()
+            decoder = codecs.getincrementaldecoder('utf-8')()
+
+            with contextlib.ExitStack() as es2:
+                lex = es2.enter_context(JsonStreamLexer())
+                parse = es2.enter_context(JsonStreamParser())
+
+                if args.stream_build:
+                    build = es2.enter_context(JsonObjectBuilder())
+                    renderer = None
+
+                else:
+                    renderer = StreamJsonRenderer(
+                        out,
+                        StreamJsonRenderer.Options(
+                            **kw,
+                            style=term_color if args.color else None,
+                        ),
+                    )
+                    build = None
+
+                while True:
+                    buf = os.read(fd, args.stream_buffer_size)
+
+                    for s in decoder.decode(buf, not buf):
+                        n = 0
+                        for c in s:
+                            for t in lex(c):
+                                for e in parse(t):
+                                    if renderer is not None:
+                                        renderer.render((e,))
+
+                                    if build is not None:
+                                        for v in build(e):
+                                            print(render_one(v), file=out)
+
+                                    n += 1
+
+                        if n:
+                            out.flush()
+
+                    if not buf:
+                        break
+
+                out.write('\n')
+
+        else:
+            with io.TextIOWrapper(in_file) as tw:
+                v = fmt.load(tw)
+            print(render_one(v), file=out)
 
 
 if __name__ == '__main__':

omlish/formats/json/consts.py

@@ -0,0 +1,22 @@
+import typing as ta
+
+
+##
+
+
+PRETTY_INDENT = 2
+
+PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=PRETTY_INDENT,
+)
+
+
+##
+
+
+COMPACT_SEPARATORS = (',', ':')
+
+COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=None,
+    separators=COMPACT_SEPARATORS,
+)