omlish 0.0.0.dev81__py3-none-any.whl → 0.0.0.dev82__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev81'
-__revision__ = 'a6b71055e7603077b28ba1999d389905e5663aac'
+__version__ = '0.0.0.dev82'
+__revision__ = '025f44593a2c0c59d06340e1d7dbcd15b1ab7684'
 
 
 #

omlish/dataclasses/impl/__init__.py

@@ -22,4 +22,12 @@ TODO:
  - enums
  - nodal
  - embedding? forward kwargs in general? or only for replace?
+
+TODO refs:
+ - batch up exec calls
+  - https://github.com/python/cpython/commit/8945b7ff55b87d11c747af2dad0e3e4d631e62d6
+ - add doc parameter to dataclasses.field
+  - https://github.com/python/cpython/commit/9c7657f09914254724683d91177aed7947637be5
+ - add decorator argument to make_dataclass
+  - https://github.com/python/cpython/commit/3e3a4d231518f91ff2f3c5a085b3849e32f1d548
 """

omlish/dataclasses/impl/params.py

@@ -12,6 +12,9 @@ class Field_:
     metadata: Metadata | None = None
     kw_only: bool | MISSING = MISSING
 
+    if sys.version_info >= (3, 13):
+        doc: str | None = None
+
     _field_type: Any = None
 
 

omlish/dataclasses/impl/slots.py

@@ -1,5 +1,7 @@
 import dataclasses as dc
+import inspect
 import itertools
+import types
 
 
 MISSING = dc.MISSING
@@ -25,7 +27,7 @@ def _get_slots(cls):
             slots = []
             if getattr(cls, '__weakrefoffset__', -1) != 0:
                 slots.append('__weakref__')
-            if getattr(cls, '__dictrefoffset__', -1) != 0:
+            if getattr(cls, '__dictoffset__', -1) != 0:
                 slots.append('__dict__')
             yield from slots
         case str(slot):
@@ -37,44 +39,96 @@ def _get_slots(cls):
             raise TypeError(f"Slots of '{cls.__name__}' cannot be determined")
 
 
+def _update_func_cell_for__class__(f, oldcls, newcls):
+    # Returns True if we update a cell, else False.
+    if f is None:
+        # f will be None in the case of a property where not all of fget, fset, and fdel are used.  Nothing to do in
+        # that case.
+        return False
+    try:
+        idx = f.__code__.co_freevars.index('__class__')
+    except ValueError:
+        # This function doesn't reference __class__, so nothing to do.
+        return False
+    # Fix the cell to point to the new class, if it's already pointing at the old class.  I'm not convinced that the "is
+    # oldcls" test is needed, but other than performance can't hurt.
+    closure = f.__closure__[idx]
+    if closure.cell_contents is oldcls:
+        closure.cell_contents = newcls
+        return True
+    return False
+
+
 def add_slots(
         cls: type,
         is_frozen: bool,
         weakref_slot: bool,
 ) -> type:
+    # Need to create a new class, since we can't set __slots__ after a class has been created, and the @dataclass
+    # decorator is called after the class is created.
+
+    # Make sure __slots__ isn't already set.
     if '__slots__' in cls.__dict__:
         raise TypeError(f'{cls.__name__} already specifies __slots__')
 
+    # Create a new dict for our new class.
     cls_dict = dict(cls.__dict__)
     field_names = tuple(f.name for f in dc.fields(cls))  # noqa
 
+    # Make sure slots don't overlap with those in base classes.
     inherited_slots = set(itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])))
 
+    # The slots for our class.  Remove slots from our base classes.  Add '__weakref__' if weakref_slot was given, unless
+    # it is already present.
     cls_dict['__slots__'] = tuple(
         itertools.filterfalse(
             inherited_slots.__contains__,
             itertools.chain(
                 field_names,
+                # gh-93521: '__weakref__' also needs to be filtered out if already present in inherited_slots
                 ('__weakref__',) if weakref_slot else (),
             ),
         ),
     )
 
     for field_name in field_names:
+        # Remove our attributes, if present. They'll still be available in _MARKER.
         cls_dict.pop(field_name, None)
 
+    # Remove __dict__ itself.
     cls_dict.pop('__dict__', None)
+
+    # Clear existing `__weakref__` descriptor, it belongs to a previous type:
     cls_dict.pop('__weakref__', None)
 
     qualname = getattr(cls, '__qualname__', None)
-    cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
+    newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
     if qualname is not None:
-        cls.__qualname__ = qualname
+        newcls.__qualname__ = qualname
 
     if is_frozen:
+        # Need this for pickling frozen classes with slots.
         if '__getstate__' not in cls_dict:
-            cls.__getstate__ = _dataclass_getstate  # type: ignore
+            newcls.__getstate__ = _dataclass_getstate  # type: ignore
         if '__setstate__' not in cls_dict:
-            cls.__setstate__ = _dataclass_setstate  # type: ignore
-
-    return cls
+            newcls.__setstate__ = _dataclass_setstate  # type: ignore
+
+    # Fix up any closures which reference __class__.  This is used to fix zero argument super so that it points to the
+    # correct class (the newly created one, which we're returning) and not the original class.  We can break out of this
+    # loop as soon as we make an update, since all closures for a class will share a given cell.
+    for member in newcls.__dict__.values():
+        # If this is a wrapped function, unwrap it.
+        member = inspect.unwrap(member)
+        if isinstance(member, types.FunctionType):
+            if _update_func_cell_for__class__(member, cls, newcls):
+                break
+
+        elif isinstance(member, property):
+            if (
+                    _update_func_cell_for__class__(member.fget, cls, newcls) or
+                    _update_func_cell_for__class__(member.fset, cls, newcls) or
+                    _update_func_cell_for__class__(member.fdel, cls, newcls)
+            ):
+                break
+
+    return newcls

omlish/formats/json/__init__.py

@@ -1,9 +1,16 @@
-from .json import (  # noqa
+from .consts import (  # noqa
     COMPACT_KWARGS,
     COMPACT_SEPARATORS,
     PRETTY_INDENT,
     PRETTY_KWARGS,
+)
+
+from .encoding import (  # noqa
+    decodes,
     detect_encoding,
+)
+
+from .json import (  # noqa
     dump,
     dump_compact,
     dump_pretty,

omlish/formats/json/backends/__init__.py

@@ -0,0 +1,7 @@
+from .base import (  # noqa
+    Backend,
+)
+
+from .default import (  # noqa
+    DEFAULT_BACKED,
+)

omlish/formats/json/backends/base.py

@@ -0,0 +1,38 @@
+import abc
+import typing as ta
+
+from .... import lang
+
+
+class Backend(lang.Abstract):
+    @abc.abstractmethod
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError

omlish/formats/json/backends/default.py

@@ -0,0 +1,10 @@
+from .base import Backend
+from .std import STD_BACKEND
+from .ujson import UJSON_BACKEND
+
+
+DEFAULT_BACKED: Backend
+if UJSON_BACKEND is not None:
+    DEFAULT_BACKED = UJSON_BACKEND
+else:
+    DEFAULT_BACKED = STD_BACKEND

omlish/formats/json/backends/jiter.py

@@ -0,0 +1,25 @@
+"""
+from_json(
+    json_data: bytes,
+    /,
+    *,
+    allow_inf_nan: bool = True,
+    cache_mode: Literal[True, False, "all", "keys", "none"] = "all",
+    partial_mode: Literal[True, False, "off", "on", "trailing-strings"] = False,
+    catch_duplicate_keys: bool = False,
+    lossless_floats: bool = False,
+) -> Any:
+    json_data: The JSON data to parse
+    allow_inf_nan: Whether to allow infinity (`Infinity` an `-Infinity`) and `NaN` values to float fields.
+    Defaults to True.
+    cache_mode: cache Python strings to improve performance at the cost of some memory usage
+    - True / 'all' - cache all strings
+    - 'keys' - cache only object keys
+    - False / 'none' - cache nothing
+    partial_mode: How to handle incomplete strings:
+    - False / 'off' - raise an exception if the input is incomplete
+    - True / 'on' - allow incomplete JSON but discard the last string if it is incomplete
+    - 'trailing-strings' - allow incomplete JSON, and include the last incomplete string in the output
+    catch_duplicate_keys: if True, raise an exception if objects contain the same key multiple times
+    lossless_floats: if True, preserve full detail on floats using `LosslessFloat`
+"""

omlish/formats/json/backends/orjson.py

@@ -1,11 +1,16 @@
 """
-def loads(obj: str | bytes | bytearray | memoryview) -> ta.Any | oj.JSONDEcodeError
-def dumps(obj: ta.Any, **DumpOpts) -> bytes
+loads(obj: str | bytes | bytearray | memoryview) -> ta.Any | oj.JSONDEcodeError
+dumps(
+    obj: ta.Any,
+    default: ta.Callable[[ta.Any], Ata.ny] | None = ...,
+    option: int | None = ...,
+) -> bytes
 """
 import dataclasses as dc
 import typing as ta
 
 from .... import lang
+from .base import Backend
 
 
 if ta.TYPE_CHECKING:
@@ -14,6 +19,9 @@ else:
     oj = lang.proxy_import('orjson')
 
 
+##
+
+
 @dc.dataclass(frozen=True, kw_only=True)
 class Options:
     append_newline: bool = False  # append \n to the output
@@ -70,3 +78,39 @@ class Options:
 class DumpOpts:
     default: ta.Callable[[ta.Any], ta.Any] | None = None
     option: Options = Options()
+
+
+##
+
+
+class OrjsonBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        fp.write(self.dumps(obj, **kwargs))
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return oj.dumps(obj, **kwargs).decode('utf-8')
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return oj.loads(fp.read(), **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return oj.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        fp.write(self.dumps_pretty(obj, **kwargs))
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return self.dumps(obj, option=kwargs.pop('option', 0) | oj.OPT_INDENT_2, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        return self.dump(obj, fp, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return self.dumps(obj, **kwargs)
+
+
+ORJSON_BACKEND: OrjsonBackend | None
+if lang.can_import('orjson'):
+    ORJSON_BACKEND = OrjsonBackend()
+else:
+    ORJSON_BACKEND = None

omlish/formats/json/backends/std.py

@@ -8,6 +8,13 @@ import dataclasses as dc
 import json
 import typing as ta
 
+from ..consts import COMPACT_KWARGS
+from ..consts import PRETTY_KWARGS
+from .base import Backend
+
+
+##
+
 
 @dc.dataclass(frozen=True, kw_only=True)
 class DumpOpts:
@@ -35,3 +42,35 @@ class LoadOpts:
 
     # called with the result of any object literal decoded with an ordered list of pairs, by default dict  # noqa
     object_pairs_hook: ta.Callable[[list[tuple[str, ta.Any]]], ta.Any] | None = None
+
+
+##
+
+
+class StdBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **kwargs)
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **kwargs)
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return json.load(fp, **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return json.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **PRETTY_KWARGS, **kwargs)
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **PRETTY_KWARGS, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **COMPACT_KWARGS, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **COMPACT_KWARGS, **kwargs)
+
+
+STD_BACKEND = StdBackend()

omlish/formats/json/backends/ujson.py

@@ -7,6 +7,19 @@ dumps(obj: ta.Any, **DumpOpts) -> None
 import dataclasses as dc
 import typing as ta
 
+from .... import lang
+from ..consts import PRETTY_INDENT
+from .base import Backend
+
+
+if ta.TYPE_CHECKING:
+    import ujson as uj
+else:
+    uj = lang.proxy_import('ujson')
+
+
+##
+
 
 @dc.dataclass(frozen=True, kw_only=True)
 class DumpOpts:
@@ -23,3 +36,39 @@ class DumpOpts:
     reject_bytes: bool = True
     default: ta.Callable[[ta.Any], ta.Any] | None = None  # should return a serializable version of obj or raise TypeError  # noqa
     separators: tuple[str, str] | None = None
+
+
+##
+
+
+class UjsonBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, **kwargs)
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, **kwargs)
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return uj.load(fp, **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return uj.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, indent=PRETTY_INDENT, **kwargs)
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, indent=PRETTY_INDENT, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, **kwargs)
+
+
+UJSON_BACKEND: UjsonBackend | None
+if lang.can_import('ujson'):
+    UJSON_BACKEND = UjsonBackend()
+else:
+    UJSON_BACKEND = None

omlish/formats/json/cli.py

@@ -1,3 +1,8 @@
+"""
+TODO:
+ - xml - [{"att", {"el", {"cdata", ...
+ - csv - dict if headers, array if not
+"""
 import argparse
 import codecs
 import contextlib
@@ -14,8 +19,10 @@ from ... import check
 from ... import lang
 from ... import term
 from .render import JsonRenderer
-from .stream import JsonStreamLexer
-from .stream import JsonStreamValueBuilder
+from .stream.build import JsonObjectBuilder
+from .stream.lex import JsonStreamLexer
+from .stream.parse import JsonStreamParser
+from .stream.render import StreamJsonRenderer
 
 
 if ta.TYPE_CHECKING:
@@ -75,7 +82,8 @@ def _main() -> None:
     parser.add_argument('file', nargs='?')
 
     parser.add_argument('--stream', action='store_true')
-    parser.add_argument('--stream-buffer-size', type=int, default=0x1000)
+    parser.add_argument('--stream-build', action='store_true')
+    parser.add_argument('--stream-buffer-size', type=int, default=0x4000)
 
     parser.add_argument('-f', '--format')
 
@@ -179,7 +187,21 @@ def _main() -> None:
 
             with contextlib.ExitStack() as es2:
                 lex = es2.enter_context(JsonStreamLexer())
-                vb = es2.enter_context(JsonStreamValueBuilder())
+                parse = es2.enter_context(JsonStreamParser())
+
+                if args.stream_build:
+                    build = es2.enter_context(JsonObjectBuilder())
+                    renderer = None
+
+                else:
+                    renderer = StreamJsonRenderer(
+                        out,
+                        StreamJsonRenderer.Options(
+                            **kw,
+                            style=term_color if args.color else None,
+                        ),
+                    )
+                    build = None
 
                 while True:
                     buf = os.read(fd, args.stream_buffer_size)
@@ -188,8 +210,14 @@ def _main() -> None:
                         n = 0
                         for c in s:
                             for t in lex(c):
-                                for v in vb(t):
-                                    print(render_one(v), file=out)
+                                for e in parse(t):
+                                    if renderer is not None:
+                                        renderer.render((e,))
+
+                                    if build is not None:
+                                        for v in build(e):
+                                            print(render_one(v), file=out)
+
                                     n += 1
 
                         if n:
@@ -198,6 +226,8 @@ def _main() -> None:
                     if not buf:
                         break
 
+                out.write('\n')
+
         else:
             with io.TextIOWrapper(in_file) as tw:
                 v = fmt.load(tw)

omlish/formats/json/consts.py

@@ -0,0 +1,22 @@
+import typing as ta
+
+
+##
+
+
+PRETTY_INDENT = 2
+
+PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=PRETTY_INDENT,
+)
+
+
+##
+
+
+COMPACT_SEPARATORS = (',', ':')
+
+COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=None,
+    separators=COMPACT_SEPARATORS,
+)

omlish/formats/json/encoding.py

@@ -0,0 +1,17 @@
+import json
+
+
+detect_encoding = json.detect_encoding
+
+
+def decodes(s: str | bytes | bytearray) -> str:
+    if isinstance(s, str):
+        if s.startswith('\ufeff'):
+            raise json.JSONDecodeError('Unexpected UTF-8 BOM (decode using utf-8-sig)', s, 0)
+        return s
+
+    elif isinstance(s, (bytes, bytearray)):
+        return s.decode(detect_encoding(s), 'surrogatepass')
+
+    else:
+        raise TypeError(f'the JSON object must be str, bytes or bytearray, not {s.__class__.__name__}')

omlish/formats/json/json.py

@@ -1,47 +1,17 @@
-"""
-TODO:
- - backend abstr
- - streaming
-"""
-import functools
-import json as _json
-import typing as ta
+from .backends import DEFAULT_BACKED
 
 
 ##
 
 
-dump = _json.dump
-dumps = _json.dumps
+dump = DEFAULT_BACKED.dump
+dumps = DEFAULT_BACKED.dumps
 
-detect_encoding = _json.detect_encoding
+load = DEFAULT_BACKED.load
+loads = DEFAULT_BACKED.loads
 
-load = _json.load
-loads = _json.loads
+dump_pretty = DEFAULT_BACKED.dump_pretty
+dumps_pretty = DEFAULT_BACKED.dumps_pretty
 
-
-##
-
-
-PRETTY_INDENT = 2
-
-PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
-    indent=PRETTY_INDENT,
-)
-
-dump_pretty: ta.Callable[..., bytes] = functools.partial(dump, **PRETTY_KWARGS)  # type: ignore
-dumps_pretty: ta.Callable[..., str] = functools.partial(dumps, **PRETTY_KWARGS)
-
-
-##
-
-
-COMPACT_SEPARATORS = (',', ':')
-
-COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
-    indent=None,
-    separators=COMPACT_SEPARATORS,
-)
-
-dump_compact: ta.Callable[..., bytes] = functools.partial(dump, **COMPACT_KWARGS)  # type: ignore
-dumps_compact: ta.Callable[..., str] = functools.partial(dumps, **COMPACT_KWARGS)
+dump_compact = DEFAULT_BACKED.dump_compact
+dumps_compact = DEFAULT_BACKED.dumps_compact