omlish 0.0.0.dev81__py3-none-any.whl → 0.0.0.dev83__py3-none-any.whl

omlish/.manifests.json

@@ -36,14 +36,14 @@
     }
   },
   {
-    "module": ".formats.json.__main__",
+    "module": ".formats.json.cli.__main__",
     "attr": "_CLI_MODULE",
-    "file": "omlish/formats/json/__main__.py",
+    "file": "omlish/formats/json/cli/__main__.py",
     "line": 1,
     "value": {
       "$omdev.cli.types.CliModule": {
         "cmd_name": "json",
-        "mod_name": "omlish.formats.json.__main__"
+        "mod_name": "omlish.formats.json.cli.__main__"
       }
     }
   },

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev81'
-__revision__ = 'a6b71055e7603077b28ba1999d389905e5663aac'
+__version__ = '0.0.0.dev83'
+__revision__ = '25a87321f2e19f536595fa049cd2f53fb5497448'
 
 
 #

omlish/dataclasses/impl/__init__.py

@@ -22,4 +22,12 @@ TODO:
  - enums
  - nodal
  - embedding? forward kwargs in general? or only for replace?
+
+TODO refs:
+ - batch up exec calls
+  - https://github.com/python/cpython/commit/8945b7ff55b87d11c747af2dad0e3e4d631e62d6
+ - add doc parameter to dataclasses.field
+  - https://github.com/python/cpython/commit/9c7657f09914254724683d91177aed7947637be5
+ - add decorator argument to make_dataclass
+  - https://github.com/python/cpython/commit/3e3a4d231518f91ff2f3c5a085b3849e32f1d548
 """

omlish/dataclasses/impl/params.py

@@ -12,6 +12,9 @@ class Field_:
     metadata: Metadata | None = None
     kw_only: bool | MISSING = MISSING
 
+    if sys.version_info >= (3, 13):
+        doc: str | None = None
+
     _field_type: Any = None
 
 

omlish/dataclasses/impl/slots.py

@@ -1,5 +1,7 @@
 import dataclasses as dc
+import inspect
 import itertools
+import types
 
 
 MISSING = dc.MISSING
@@ -25,7 +27,7 @@ def _get_slots(cls):
             slots = []
             if getattr(cls, '__weakrefoffset__', -1) != 0:
                 slots.append('__weakref__')
-            if getattr(cls, '__dictrefoffset__', -1) != 0:
+            if getattr(cls, '__dictoffset__', -1) != 0:
                 slots.append('__dict__')
             yield from slots
         case str(slot):
@@ -37,44 +39,96 @@ def _get_slots(cls):
             raise TypeError(f"Slots of '{cls.__name__}' cannot be determined")
 
 
+def _update_func_cell_for__class__(f, oldcls, newcls):
+    # Returns True if we update a cell, else False.
+    if f is None:
+        # f will be None in the case of a property where not all of fget, fset, and fdel are used.  Nothing to do in
+        # that case.
+        return False
+    try:
+        idx = f.__code__.co_freevars.index('__class__')
+    except ValueError:
+        # This function doesn't reference __class__, so nothing to do.
+        return False
+    # Fix the cell to point to the new class, if it's already pointing at the old class.  I'm not convinced that the "is
+    # oldcls" test is needed, but other than performance can't hurt.
+    closure = f.__closure__[idx]
+    if closure.cell_contents is oldcls:
+        closure.cell_contents = newcls
+        return True
+    return False
+
+
 def add_slots(
         cls: type,
         is_frozen: bool,
         weakref_slot: bool,
 ) -> type:
+    # Need to create a new class, since we can't set __slots__ after a class has been created, and the @dataclass
+    # decorator is called after the class is created.
+
+    # Make sure __slots__ isn't already set.
     if '__slots__' in cls.__dict__:
         raise TypeError(f'{cls.__name__} already specifies __slots__')
 
+    # Create a new dict for our new class.
     cls_dict = dict(cls.__dict__)
     field_names = tuple(f.name for f in dc.fields(cls))  # noqa
 
+    # Make sure slots don't overlap with those in base classes.
     inherited_slots = set(itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])))
 
+    # The slots for our class.  Remove slots from our base classes.  Add '__weakref__' if weakref_slot was given, unless
+    # it is already present.
     cls_dict['__slots__'] = tuple(
         itertools.filterfalse(
             inherited_slots.__contains__,
             itertools.chain(
                 field_names,
+                # gh-93521: '__weakref__' also needs to be filtered out if already present in inherited_slots
                 ('__weakref__',) if weakref_slot else (),
             ),
         ),
     )
 
     for field_name in field_names:
+        # Remove our attributes, if present. They'll still be available in _MARKER.
         cls_dict.pop(field_name, None)
 
+    # Remove __dict__ itself.
     cls_dict.pop('__dict__', None)
+
+    # Clear existing `__weakref__` descriptor, it belongs to a previous type:
     cls_dict.pop('__weakref__', None)
 
     qualname = getattr(cls, '__qualname__', None)
-    cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
+    newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
     if qualname is not None:
-        cls.__qualname__ = qualname
+        newcls.__qualname__ = qualname
 
     if is_frozen:
+        # Need this for pickling frozen classes with slots.
         if '__getstate__' not in cls_dict:
-            cls.__getstate__ = _dataclass_getstate  # type: ignore
+            newcls.__getstate__ = _dataclass_getstate  # type: ignore
         if '__setstate__' not in cls_dict:
-            cls.__setstate__ = _dataclass_setstate  # type: ignore
-
-    return cls
+            newcls.__setstate__ = _dataclass_setstate  # type: ignore
+
+    # Fix up any closures which reference __class__.  This is used to fix zero argument super so that it points to the
+    # correct class (the newly created one, which we're returning) and not the original class.  We can break out of this
+    # loop as soon as we make an update, since all closures for a class will share a given cell.
+    for member in newcls.__dict__.values():
+        # If this is a wrapped function, unwrap it.
+        member = inspect.unwrap(member)
+        if isinstance(member, types.FunctionType):
+            if _update_func_cell_for__class__(member, cls, newcls):
+                break
+
+        elif isinstance(member, property):
+            if (
+                    _update_func_cell_for__class__(member.fget, cls, newcls) or
+                    _update_func_cell_for__class__(member.fset, cls, newcls) or
+                    _update_func_cell_for__class__(member.fdel, cls, newcls)
+            ):
+                break
+
+    return newcls

omlish/formats/json/__init__.py

@@ -1,9 +1,16 @@
-from .json import (  # noqa
+from .consts import (  # noqa
     COMPACT_KWARGS,
     COMPACT_SEPARATORS,
     PRETTY_INDENT,
     PRETTY_KWARGS,
+)
+
+from .encoding import (  # noqa
+    decodes,
     detect_encoding,
+)
+
+from .json import (  # noqa
     dump,
     dump_compact,
     dump_pretty,

omlish/formats/json/backends/__init__.py

@@ -0,0 +1,7 @@
+from .base import (  # noqa
+    Backend,
+)
+
+from .default import (  # noqa
+    DEFAULT_BACKED,
+)

omlish/formats/json/backends/base.py

@@ -0,0 +1,38 @@
+import abc
+import typing as ta
+
+from .... import lang
+
+
+class Backend(lang.Abstract):
+    @abc.abstractmethod
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        raise NotImplementedError

omlish/formats/json/backends/default.py

@@ -0,0 +1,10 @@
+from .base import Backend
+from .std import STD_BACKEND
+from .ujson import UJSON_BACKEND
+
+
+DEFAULT_BACKED: Backend
+if UJSON_BACKEND is not None:
+    DEFAULT_BACKED = UJSON_BACKEND
+else:
+    DEFAULT_BACKED = STD_BACKEND

omlish/formats/json/backends/jiter.py

@@ -0,0 +1,25 @@
+"""
+from_json(
+    json_data: bytes,
+    /,
+    *,
+    allow_inf_nan: bool = True,
+    cache_mode: Literal[True, False, "all", "keys", "none"] = "all",
+    partial_mode: Literal[True, False, "off", "on", "trailing-strings"] = False,
+    catch_duplicate_keys: bool = False,
+    lossless_floats: bool = False,
+) -> Any:
+    json_data: The JSON data to parse
+    allow_inf_nan: Whether to allow infinity (`Infinity` an `-Infinity`) and `NaN` values to float fields.
+    Defaults to True.
+    cache_mode: cache Python strings to improve performance at the cost of some memory usage
+    - True / 'all' - cache all strings
+    - 'keys' - cache only object keys
+    - False / 'none' - cache nothing
+    partial_mode: How to handle incomplete strings:
+    - False / 'off' - raise an exception if the input is incomplete
+    - True / 'on' - allow incomplete JSON but discard the last string if it is incomplete
+    - 'trailing-strings' - allow incomplete JSON, and include the last incomplete string in the output
+    catch_duplicate_keys: if True, raise an exception if objects contain the same key multiple times
+    lossless_floats: if True, preserve full detail on floats using `LosslessFloat`
+"""

omlish/formats/json/backends/orjson.py

@@ -1,11 +1,16 @@
 """
-def loads(obj: str | bytes | bytearray | memoryview) -> ta.Any | oj.JSONDEcodeError
-def dumps(obj: ta.Any, **DumpOpts) -> bytes
+loads(obj: str | bytes | bytearray | memoryview) -> ta.Any | oj.JSONDEcodeError
+dumps(
+    obj: ta.Any,
+    default: ta.Callable[[ta.Any], Ata.ny] | None = ...,
+    option: int | None = ...,
+) -> bytes
 """
 import dataclasses as dc
 import typing as ta
 
 from .... import lang
+from .base import Backend
 
 
 if ta.TYPE_CHECKING:
@@ -14,6 +19,9 @@ else:
     oj = lang.proxy_import('orjson')
 
 
+##
+
+
 @dc.dataclass(frozen=True, kw_only=True)
 class Options:
     append_newline: bool = False  # append \n to the output
@@ -70,3 +78,39 @@ class Options:
 class DumpOpts:
     default: ta.Callable[[ta.Any], ta.Any] | None = None
     option: Options = Options()
+
+
+##
+
+
+class OrjsonBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        fp.write(self.dumps(obj, **kwargs))
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return oj.dumps(obj, **kwargs).decode('utf-8')
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return oj.loads(fp.read(), **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return oj.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        fp.write(self.dumps_pretty(obj, **kwargs))
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return self.dumps(obj, option=kwargs.pop('option', 0) | oj.OPT_INDENT_2, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        return self.dump(obj, fp, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return self.dumps(obj, **kwargs)
+
+
+ORJSON_BACKEND: OrjsonBackend | None
+if lang.can_import('orjson'):
+    ORJSON_BACKEND = OrjsonBackend()
+else:
+    ORJSON_BACKEND = None

omlish/formats/json/backends/std.py

@@ -8,6 +8,13 @@ import dataclasses as dc
 import json
 import typing as ta
 
+from ..consts import COMPACT_KWARGS
+from ..consts import PRETTY_KWARGS
+from .base import Backend
+
+
+##
+
 
 @dc.dataclass(frozen=True, kw_only=True)
 class DumpOpts:
@@ -35,3 +42,35 @@ class LoadOpts:
 
     # called with the result of any object literal decoded with an ordered list of pairs, by default dict  # noqa
     object_pairs_hook: ta.Callable[[list[tuple[str, ta.Any]]], ta.Any] | None = None
+
+
+##
+
+
+class StdBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **kwargs)
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **kwargs)
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return json.load(fp, **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return json.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **PRETTY_KWARGS, **kwargs)
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **PRETTY_KWARGS, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        json.dump(obj, fp, **COMPACT_KWARGS, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return json.dumps(obj, **COMPACT_KWARGS, **kwargs)
+
+
+STD_BACKEND = StdBackend()

omlish/formats/json/backends/ujson.py

@@ -7,6 +7,19 @@ dumps(obj: ta.Any, **DumpOpts) -> None
 import dataclasses as dc
 import typing as ta
 
+from .... import lang
+from ..consts import PRETTY_INDENT
+from .base import Backend
+
+
+if ta.TYPE_CHECKING:
+    import ujson as uj
+else:
+    uj = lang.proxy_import('ujson')
+
+
+##
+
 
 @dc.dataclass(frozen=True, kw_only=True)
 class DumpOpts:
@@ -23,3 +36,39 @@ class DumpOpts:
     reject_bytes: bool = True
     default: ta.Callable[[ta.Any], ta.Any] | None = None  # should return a serializable version of obj or raise TypeError  # noqa
     separators: tuple[str, str] | None = None
+
+
+##
+
+
+class UjsonBackend(Backend):
+    def dump(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, **kwargs)
+
+    def dumps(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, **kwargs)
+
+    def load(self, fp: ta.Any, **kwargs: ta.Any) -> ta.Any:
+        return uj.load(fp, **kwargs)
+
+    def loads(self, s: str | bytes | bytearray, **kwargs: ta.Any) -> ta.Any:
+        return uj.loads(s, **kwargs)
+
+    def dump_pretty(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, indent=PRETTY_INDENT, **kwargs)
+
+    def dumps_pretty(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, indent=PRETTY_INDENT, **kwargs)
+
+    def dump_compact(self, obj: ta.Any, fp: ta.Any, **kwargs: ta.Any) -> None:
+        uj.dump(obj, fp, **kwargs)
+
+    def dumps_compact(self, obj: ta.Any, **kwargs: ta.Any) -> str:
+        return uj.dumps(obj, **kwargs)
+
+
+UJSON_BACKEND: UjsonBackend | None
+if lang.can_import('ujson'):
+    UJSON_BACKEND = UjsonBackend()
+else:
+    UJSON_BACKEND = None

omlish/formats/json/{cli.py → cli/cli.py}

@@ -1,8 +1,6 @@
 import argparse
 import codecs
 import contextlib
-import dataclasses as dc
-import enum
 import io
 import json
 import os
@@ -10,31 +8,25 @@ import subprocess
 import sys
 import typing as ta
 
-from ... import check
-from ... import lang
-from ... import term
-from .render import JsonRenderer
-from .stream import JsonStreamLexer
-from .stream import JsonStreamValueBuilder
+from .... import check
+from .... import lang
+from .... import term
+from ..render import JsonRenderer
+from ..stream.build import JsonObjectBuilder
+from ..stream.lex import JsonStreamLexer
+from ..stream.parse import JsonStreamParser
+from ..stream.render import StreamJsonRenderer
+from .formats import FORMATS_BY_NAME
+from .formats import Formats
 
 
 if ta.TYPE_CHECKING:
-    import ast
-    import tomllib
-
-    import yaml
-
-    from .. import dotenv
-    from .. import props
-
+    from ....specs import jmespath
 else:
-    ast = lang.proxy_import('ast')
-    tomllib = lang.proxy_import('tomllib')
+    jmespath = lang.proxy_import('....specs.jmespath', __package__)
 
-    yaml = lang.proxy_import('yaml')
 
-    dotenv = lang.proxy_import('..dotenv', __package__)
-    props = lang.proxy_import('..props', __package__)
+##
 
 
 def term_color(o: ta.Any, state: JsonRenderer.State) -> tuple[str, str]:
@@ -46,39 +38,19 @@ def term_color(o: ta.Any, state: JsonRenderer.State) -> tuple[str, str]:
         return '', ''
 
 
-@dc.dataclass(frozen=True)
-class Format:
-    names: ta.Sequence[str]
-    load: ta.Callable[[ta.TextIO], ta.Any]
-
-
-class Formats(enum.Enum):
-    JSON = Format(['json'], json.load)
-    YAML = Format(['yaml', 'yml'], lambda f: yaml.safe_load(f))
-    TOML = Format(['toml'], lambda f: tomllib.loads(f.read()))
-    ENV = Format(['env', 'dotenv'], lambda f: dotenv.dotenv_values(stream=f))
-    PROPS = Format(['properties', 'props'], lambda f: dict(props.Properties().load(f.read())))
-    PY = Format(['py', 'python', 'repr'], lambda f: ast.literal_eval(f.read()))
-
-
-FORMATS_BY_NAME: ta.Mapping[str, Format] = {
-    n: f
-    for e in Formats
-    for f in [e.value]
-    for n in f.names
-}
-
-
 def _main() -> None:
     parser = argparse.ArgumentParser()
 
     parser.add_argument('file', nargs='?')
 
     parser.add_argument('--stream', action='store_true')
-    parser.add_argument('--stream-buffer-size', type=int, default=0x1000)
+    parser.add_argument('--stream-build', action='store_true')
+    parser.add_argument('--stream-buffer-size', type=int, default=0x4000)
 
     parser.add_argument('-f', '--format')
 
+    parser.add_argument('-x', '--jmespath-expr')
+
     parser.add_argument('-z', '--compact', action='store_true')
     parser.add_argument('-p', '--pretty', action='store_true')
     parser.add_argument('-i', '--indent')
@@ -111,7 +83,15 @@ def _main() -> None:
         sort_keys=args.sort_keys,
     )
 
+    if args.jmespath_expr is not None:
+        jp_expr = jmespath.compile(args.jmespath_expr)
+    else:
+        jp_expr = None
+
     def render_one(v: ta.Any) -> str:
+        if jp_expr is not None:
+            v = jp_expr.search(v)
+
         if args.color:
             return JsonRenderer.render_str(
                 v,
@@ -179,7 +159,21 @@ def _main() -> None:
 
             with contextlib.ExitStack() as es2:
                 lex = es2.enter_context(JsonStreamLexer())
-                vb = es2.enter_context(JsonStreamValueBuilder())
+                parse = es2.enter_context(JsonStreamParser())
+
+                if args.stream_build:
+                    build = es2.enter_context(JsonObjectBuilder())
+                    renderer = None
+
+                else:
+                    renderer = StreamJsonRenderer(
+                        out,
+                        StreamJsonRenderer.Options(
+                            **kw,
+                            style=term_color if args.color else None,
+                        ),
+                    )
+                    build = None
 
                 while True:
                     buf = os.read(fd, args.stream_buffer_size)
@@ -188,8 +182,14 @@ def _main() -> None:
                         n = 0
                         for c in s:
                             for t in lex(c):
-                                for v in vb(t):
-                                    print(render_one(v), file=out)
+                                for e in parse(t):
+                                    if renderer is not None:
+                                        renderer.render((e,))
+
+                                    if build is not None:
+                                        for v in build(e):
+                                            print(render_one(v), file=out)
+
                                     n += 1
 
                         if n:
@@ -198,6 +198,8 @@ def _main() -> None:
                     if not buf:
                         break
 
+                out.write('\n')
+
         else:
             with io.TextIOWrapper(in_file) as tw:
                 v = fmt.load(tw)

omlish/formats/json/cli/formats.py

@@ -0,0 +1,64 @@
+"""
+TODO:
+ - options lol - csv header, newline, etc
+"""
+import dataclasses as dc
+import enum
+import json
+import typing as ta
+
+from .... import lang
+
+
+if ta.TYPE_CHECKING:
+    import ast
+    import csv
+    import tomllib
+
+    import yaml
+
+    from ... import dotenv
+    from ... import props
+    from ... import xml
+
+else:
+    ast = lang.proxy_import('ast')
+    csv = lang.proxy_import('csv')
+    tomllib = lang.proxy_import('tomllib')
+
+    yaml = lang.proxy_import('yaml')
+
+    dotenv = lang.proxy_import('...dotenv', __package__)
+    props = lang.proxy_import('...props', __package__)
+    xml = lang.proxy_import('...xml', __package__)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Format:
+    names: ta.Sequence[str]
+    load: ta.Callable[[ta.TextIO], ta.Any]
+
+
+class Formats(enum.Enum):
+    JSON = Format(['json'], json.load)
+    YAML = Format(['yaml', 'yml'], lambda f: yaml.safe_load(f))
+    TOML = Format(['toml'], lambda f: tomllib.loads(f.read()))
+    ENV = Format(['env', 'dotenv'], lambda f: dotenv.dotenv_values(stream=f))
+    PROPS = Format(['properties', 'props'], lambda f: dict(props.Properties().load(f.read())))
+    PY = Format(['py', 'python', 'repr'], lambda f: ast.literal_eval(f.read()))
+    XML = Format(['xml'], lambda f: xml.build_simple_element(xml.parse_tree(f.read()).getroot()).as_dict())
+    CSV = Format(['csv'], lambda f: list(csv.DictReader(f)))
+    TSV = Format(['tsv'], lambda f: list(csv.DictReader(f, delimiter='\t')))
+    FLAT_CSV = Format(['fcsv'], lambda f: list(csv.reader(f)))
+    FLAT_TSV = Format(['ftsv'], lambda f: list(csv.reader(f, delimiter='\t')))
+
+
+FORMATS_BY_NAME: ta.Mapping[str, Format] = {
+    n: f
+    for e in Formats
+    for f in [e.value]
+    for n in f.names
+}