omlish 0.0.0.dev164__py3-none-any.whl → 0.0.0.dev166__py3-none-any.whl

omlish/.manifests.json

@@ -37,6 +37,20 @@
       }
     }
   },
+  {
+    "module": ".formats.cloudpickle",
+    "attr": "_CLOUDPICKLE_LAZY_CODEC",
+    "file": "omlish/formats/cloudpickle.py",
+    "line": 30,
+    "value": {
+      "$.codecs.base.LazyLoadedCodec": {
+        "mod_name": "omlish.formats.cloudpickle",
+        "attr_name": "CLOUDPICKLE_CODEC",
+        "name": "cloudpickle",
+        "aliases": null
+      }
+    }
+  },
   {
     "module": ".formats.json.codecs",
     "attr": "_JSON_LAZY_CODEC",
@@ -55,7 +69,7 @@
     "module": ".formats.json.codecs",
     "attr": "_JSON_COMPACT_LAZY_CODEC",
     "file": "omlish/formats/json/codecs.py",
-    "line": 23,
+    "line": 21,
     "value": {
       "$.codecs.base.LazyLoadedCodec": {
         "mod_name": "omlish.formats.json.codecs",
@@ -69,7 +83,7 @@
     "module": ".formats.json.codecs",
     "attr": "_JSON_PRETTY_LAZY_CODEC",
     "file": "omlish/formats/json/codecs.py",
-    "line": 32,
+    "line": 28,
     "value": {
       "$.codecs.base.LazyLoadedCodec": {
         "mod_name": "omlish.formats.json.codecs",
@@ -137,6 +151,20 @@
       }
     }
   },
+  {
+    "module": ".formats.yaml",
+    "attr": "_YAML_UNSAFE_LAZY_CODEC",
+    "file": "omlish/formats/yaml.py",
+    "line": 265,
+    "value": {
+      "$.codecs.base.LazyLoadedCodec": {
+        "mod_name": "omlish.formats.yaml",
+        "attr_name": "YAML_UNSAFE_CODEC",
+        "name": "yaml-unsafe",
+        "aliases": null
+      }
+    }
+  },
   {
     "module": ".io.compress.brotli",
     "attr": "_BROTLI_LAZY_CODEC",

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev164'
-__revision__ = '72b4cfb1086b384ec55cc221069d5bb2be6b3c10'
+__version__ = '0.0.0.dev166'
+__revision__ = 'e832ee32347c3f4c51e8ead2186def228e3aac1c'
 
 
 #

omlish/codecs/__init__.py

@@ -30,6 +30,9 @@ from .chain import (  # noqa
 
 from .funcs import (  # noqa
     FnPairEagerCodec,
+
+    of_pair,
+    of,
 )
 
 from .registry import (  # noqa

omlish/codecs/base.py

@@ -9,6 +9,7 @@ from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
 from omlish import reflect as rfl
+from omlish.funcs import pairs as fps
 
 
 I = ta.TypeVar('I')
@@ -27,6 +28,9 @@ class EagerCodec(lang.Abstract, ta.Generic[I, O]):
     def decode(self, o: O) -> I:
         raise NotImplementedError
 
+    def as_pair(self) -> fps.FnPair[I, O]:
+        return fps.of(self.encode, self.decode)
+
 
 class IncrementalCodec(lang.Abstract, ta.Generic[I, O]):
     @abc.abstractmethod

omlish/codecs/funcs.py

@@ -26,3 +26,14 @@ class FnPairEagerCodec(EagerCodec[I, O]):
             decode: ta.Callable[[O], I],
     ) -> 'FnPairEagerCodec[I, O]':
         return cls(fps.of(encode, decode))
+
+
+def of_pair(fp: fps.FnPair[I, O]) -> FnPairEagerCodec[I, O]:
+    return FnPairEagerCodec(fp)
+
+
+def of(
+        encode: ta.Callable[[I], O],
+        decode: ta.Callable[[O], I],
+) -> FnPairEagerCodec[I, O]:
+    return FnPairEagerCodec(fps.of(encode, decode))

omlish/codecs/text.py

@@ -46,8 +46,8 @@ TextEncodingErrors: ta.TypeAlias = ta.Literal[
     'namereplace',
 
     ##
-    # In addition, the following error handler is specific to the given codecs:
-    # utf-8, utf-16, utf-32, utf-16-be, utf-16-le, utf-32-be, utf-32-le
+    # In addition, the following error handler is specific to the given codecs: utf-8, utf-16, utf-32, utf-16-be,
+    # utf-16-le, utf-32-be, utf-32-le
 
     # Allow encoding and decoding surrogate code point (U+D800 - U+DFFF) as normal code point. Otherwise these codecs
     # treat the presence of surrogate code point in str as an error.

omlish/formats/cloudpickle.py

@@ -0,0 +1,31 @@
+import typing as ta
+
+from .. import lang
+from .codecs import make_bytes_object_codec
+from .codecs import make_object_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import cloudpickle
+else:
+    cloudpickle = lang.proxy_import('cloudpickle')
+
+
+##
+
+
+def dump(obj: ta.Any) -> bytes:
+    return cloudpickle.dumps(obj)
+
+
+def load(s: bytes) -> ta.Any:
+    return cloudpickle.loads(s)
+
+
+##
+
+
+CLOUDPICKLE_CODEC = make_bytes_object_codec('cloudpickle', dump, load)
+
+# @omlish-manifest
+_CLOUDPICKLE_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'CLOUDPICKLE_CODEC', CLOUDPICKLE_CODEC)

omlish/formats/json/codecs.py

@@ -14,19 +14,15 @@ JSON_CODEC = make_str_object_codec('json', dumps, loads)
 # @omlish-manifest
 _JSON_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'JSON_CODEC', JSON_CODEC)
 
-
 #
 
-
 JSON_COMPACT_CODEC = make_str_object_codec('json-compact', dumps_compact, loads)
 
 # @omlish-manifest
 _JSON_COMPACT_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'JSON_COMPACT_CODEC', JSON_COMPACT_CODEC)
 
-
 #
 
-
 JSON_PRETTY_CODEC = make_str_object_codec('json-pretty', dumps_pretty, loads)
 
 # @omlish-manifest

omlish/formats/json/delimted.py

@@ -0,0 +1,4 @@
+"""
+TODO:
+ - jsonl codec
+"""

omlish/formats/yaml.py

@@ -257,3 +257,10 @@ YAML_CODEC = make_str_object_codec('yaml', dump, safe_load, aliases=['yml'])
 
 # @omlish-manifest
 _YAML_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'YAML_CODEC', YAML_CODEC)
+
+#
+
+YAML_UNSAFE_CODEC = make_str_object_codec('yaml-unsafe', dump, full_load)
+
+# @omlish-manifest
+_YAML_UNSAFE_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'YAML_UNSAFE_CODEC', YAML_UNSAFE_CODEC)

omlish/funcs/pairs.py

@@ -1,22 +1,4 @@
-"""
-TODO:
- - objects
-  - csv
-  - csvloader
- - wrapped (wait for usecase)
- - streams / incremental
-  - fileobj -> fileobj?
- - swap zstandard for zstd
-
-Compression choice:
- - lzma if-available minimal-space
- - lz4 if-available read-heavy
- - zstd if-available
- - bz2 read-heavy (but no parallel decompress)
- - gz
-"""
 import abc
-import codecs
 import dataclasses as dc
 import typing as ta
 
@@ -24,40 +6,9 @@ from .. import lang
 
 
 if ta.TYPE_CHECKING:
-    import bz2 as _bz2
-    import gzip as _gzip
-    import lzma as _lzma
-    import pickle as _pickle
     import struct as _struct
-    import tomllib as _tomllib
-
-    import cbor2 as _cbor2
-    import cloudpickle as _cloudpickle
-    import json5 as _json5
-    import lz4.frame as _lz4_frame
-    import snappy as _snappy
-    import yaml as _yaml
-    import zstandard as _zstandard
-
-    from ..formats import json as _json
-
 else:
-    _bz2 = lang.proxy_import('bz2')
-    _gzip = lang.proxy_import('gzip')
-    _lzma = lang.proxy_import('lzma')
-    _pickle = lang.proxy_import('pickle')
     _struct = lang.proxy_import('struct')
-    _tomllib = lang.proxy_import('tomllib')
-
-    _cbor2 = lang.proxy_import('cbor2')
-    _cloudpickle = lang.proxy_import('cloudpickle')
-    _json5 = lang.proxy_import('json5')
-    _lz4_frame = lang.proxy_import('lz4.frame')
-    _snappy = lang.proxy_import('snappy')
-    _yaml = lang.proxy_import('yaml')
-    _zstandard = lang.proxy_import('zstandard')
-
-    _json = lang.proxy_import('..formats.json', __package__)
 
 
 ##
@@ -211,36 +162,6 @@ def compose(*ps):
 ##
 
 
-@dc.dataclass(frozen=True)
-class Text(FnPair[str, bytes]):
-    ci: codecs.CodecInfo
-    encode_errors: str = dc.field(default='strict', kw_only=True)
-    decode_errors: str = dc.field(default='strict', kw_only=True)
-
-    def forward(self, f: str) -> bytes:
-        # Python ignores the returned length:
-        #   https://github.com/python/cpython/blob/7431c3799efbd06ed03ee70b64420f45e83b3667/Python/codecs.c#L424
-        t, _ = self.ci.encode(f, self.encode_errors)
-        return t
-
-    def backward(self, t: bytes) -> str:
-        f, _ = self.ci.decode(t, self.decode_errors)
-        return f
-
-
-def text(name: str, *, encode_errors: str = 'strict', decode_errors: str = 'strict') -> Text:
-    ci = codecs.lookup(name)
-    if not ci._is_text_encoding:  # noqa
-        raise TypeError(f'must be text codec: {name}')
-    return Text(ci, encode_errors=encode_errors, decode_errors=decode_errors)
-
-
-UTF8 = text('utf-8')
-
-
-#
-
-
 @dc.dataclass(frozen=True)
 class Optional(FnPair[F | None, T | None]):
     fp: FnPair[F, T]
@@ -263,99 +184,6 @@ class Lines(FnPair[ta.Sequence[str], str]):
 ##
 
 
-_EXTENSION_REGISTRY: dict[str, type[FnPair]] = {}
-
-
-def _register_extension(*ss):
-    def inner(cls):
-        for s in ss:
-            if s in _EXTENSION_REGISTRY:
-                raise KeyError(s)
-            _EXTENSION_REGISTRY[s] = cls
-        return cls
-    return inner
-
-
-def get_for_extension(ext: str) -> FnPair:
-    return compose(*[_EXTENSION_REGISTRY[p]() for p in ext.split('.')])
-
-
-##
-
-
-class Compression(FnPair[bytes, bytes], abc.ABC):
-    pass
-
-
-@_register_extension('bz2')
-@dc.dataclass(frozen=True)
-class Bz2(Compression):
-    compresslevel: int = 9
-
-    def forward(self, f: bytes) -> bytes:
-        return _bz2.compress(f, compresslevel=self.compresslevel)
-
-    def backward(self, t: bytes) -> bytes:
-        return _bz2.decompress(t)
-
-
-@_register_extension('gz')
-@dc.dataclass(frozen=True)
-class Gzip(Compression):
-    compresslevel: int = 9
-
-    def forward(self, f: bytes) -> bytes:
-        return _gzip.compress(f, compresslevel=self.compresslevel)
-
-    def backward(self, t: bytes) -> bytes:
-        return _gzip.decompress(t)
-
-
-@_register_extension('lzma')
-class Lzma(Compression):
-    def forward(self, f: bytes) -> bytes:
-        return _lzma.compress(f)
-
-    def backward(self, t: bytes) -> bytes:
-        return _lzma.decompress(t)
-
-
-#
-
-
-@_register_extension('lz4')
-@dc.dataclass(frozen=True)
-class Lz4(Compression):
-    compression_level: int = 0
-
-    def forward(self, f: bytes) -> bytes:
-        return _lz4_frame.compress(f, compression_level=self.compression_level)
-
-    def backward(self, t: bytes) -> bytes:
-        return _lz4_frame.decompress(t)
-
-
-@_register_extension('snappy')
-class Snappy(Compression):
-    def forward(self, f: bytes) -> bytes:
-        return _snappy.compress(f)
-
-    def backward(self, t: bytes) -> bytes:
-        return _snappy.decompress(t)
-
-
-@_register_extension('zstd')
-class Zstd(Compression):
-    def forward(self, f: bytes) -> bytes:
-        return _zstandard.compress(f)
-
-    def backward(self, t: bytes) -> bytes:
-        return _zstandard.decompress(t)
-
-
-##
-
-
 @dc.dataclass(frozen=True)
 class Struct(FnPair[tuple, bytes]):
     fmt: str
@@ -385,112 +213,3 @@ class ObjectStr_(Object_[str], lang.Abstract):  # noqa
 
 class ObjectBytes_(Object_[bytes], lang.Abstract):  # noqa
     pass
-
-
-#
-
-
-@_register_extension('pkl')
-@dc.dataclass(frozen=True)
-class Pickle(ObjectBytes_):
-    protocol: int | None = None
-
-    def forward(self, f: ta.Any) -> bytes:
-        return _pickle.dumps(f, protocol=self.protocol)
-
-    def backward(self, t: bytes) -> ta.Any:
-        return _pickle.loads(t)
-
-
-class _Json(ObjectStr_, lang.Abstract):  # noqa
-    def backward(self, t: str) -> ta.Any:
-        return _json.loads(t)
-
-
-@_register_extension('json')
-class Json(_Json):
-    def forward(self, f: ta.Any) -> str:
-        return _json.dumps(f)
-
-
-class JsonPretty(_Json):
-    def forward(self, f: ta.Any) -> str:
-        return _json.dumps_pretty(f)
-
-
-class JsonCompact(_Json):
-    def forward(self, f: ta.Any) -> str:
-        return _json.dumps_compact(f)
-
-
-JSON = Json()
-PRETTY_JSON = JsonPretty()
-COMPACT_JSON = JsonCompact()
-
-
-@_register_extension('jsonl')
-class JsonLines(FnPair[ta.Sequence[ta.Any], str]):
-    def forward(self, f: ta.Sequence[ta.Any]) -> str:
-        return '\n'.join(_json.dumps(e) for e in f)
-
-    def backward(self, t: str) -> ta.Sequence[ta.Any]:
-        return [_json.loads(l) for l in t.splitlines()]
-
-
-@_register_extension('toml')
-class Toml(ObjectStr_):
-    def forward(self, f: ta.Any) -> str:
-        raise NotImplementedError
-
-    def backward(self, t: str) -> ta.Any:
-        return _tomllib.loads(t)
-
-
-#
-
-
-@_register_extension('cbor')
-class Cbor(ObjectBytes_):
-    def forward(self, f: ta.Any) -> bytes:
-        return _cbor2.dumps(f)
-
-    def backward(self, t: bytes) -> ta.Any:
-        return _cbor2.loads(t)
-
-
-@_register_extension('clpkl')
-@dc.dataclass(frozen=True)
-class Cloudpickle(ObjectBytes_):
-    protocol: int | None = None
-
-    def forward(self, f: ta.Any) -> bytes:
-        return _cloudpickle.dumps(f, protocol=self.protocol)
-
-    def backward(self, t: bytes) -> ta.Any:
-        return _cloudpickle.loads(t)
-
-
-@_register_extension('json5')
-class Json5(ObjectStr_):
-    def forward(self, f: ta.Any) -> str:
-        return _json5.dumps(f)
-
-    def backward(self, t: str) -> ta.Any:
-        return _json5.loads(t)
-
-
-@_register_extension('yml', 'yaml')
-class Yaml(ObjectStr_):
-    def forward(self, f: ta.Any) -> str:
-        return _yaml.dump(f)
-
-    def backward(self, t: str) -> ta.Any:
-        return _yaml.safe_load(t)
-
-
-class YamlUnsafe(ObjectStr_):
-    def forward(self, f: ta.Any) -> str:
-        return _yaml.dump(f)
-
-    def backward(self, t: str) -> ta.Any:
-        return _yaml.load(t, _yaml.FullLoader)

omlish/io/compress/codecs.py

@@ -2,7 +2,9 @@ import dataclasses as dc
 import typing as ta
 
 from ... import codecs
+from ..generators import buffer_bytes_stepped_reader_generator
 from .base import Compression
+from .base import IncrementalCompression
 
 
 ##
@@ -22,6 +24,20 @@ class CompressionEagerCodec(codecs.EagerCodec[bytes, bytes]):
 ##
 
 
+@dc.dataclass(frozen=True)
+class CompressionIncrementalCodec(codecs.IncrementalCodec[bytes, bytes]):
+    compression: IncrementalCompression
+
+    def encode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return self.compression.compress_incremental()
+
+    def decode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return buffer_bytes_stepped_reader_generator(self.compression.decompress_incremental())
+
+
+##
+
+
 class CompressionCodec(codecs.Codec):
     pass
 
@@ -40,6 +56,10 @@ def make_compression_codec(
         output=bytes,
 
         new=lambda *args, **kwargs: CompressionEagerCodec(cls(*args, **kwargs)),
+
+        new_incremental=(
+            lambda *args, **kwargs: CompressionIncrementalCodec(cls(*args, **kwargs))  # noqa
+        ) if issubclass(cls, IncrementalCompression) else None,
     )
 
 

omlish/io/generators/__init__.py

@@ -50,4 +50,7 @@ from .stepped import (  # noqa
     joined_str_stepped_generator,
 
     read_into_bytes_stepped_generator,
+    read_into_str_stepped_generator,
+
+    buffer_bytes_stepped_reader_generator,
 )

omlish/io/generators/stepped.py

@@ -151,18 +151,32 @@ def read_into_str_stepped_generator(
 def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> BytesSteppedGenerator:
     o = g.send(None)
     buf: ta.Any = None
+    eof = False
 
     while True:
+        if eof:
+            raise EOFError
+
         if not buf:
             buf = check.isinstance((yield None), bytes)
+            if not buf:
+                eof = True
 
-        if o is None or not buf:
+        if o is None:
             i = buf
+            buf = None
+
         elif isinstance(o, int):
-            if len(buf) < o:
-                raise NotImplementedError
+            while len(buf) < o:
+                more = check.isinstance((yield None), bytes)
+                if not more:
+                    raise EOFError
+                # FIXME: lol - share guts with readers
+                buf += more
+
             i = buf[:o]
             buf = buf[o:]
+
         else:
             raise TypeError(o)
 
@@ -171,5 +185,7 @@ def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> Byt
             i = None
             if isinstance(o, bytes):
                 check.none((yield o))
+                if not o:
+                    return
             else:
                 break

omlish/iterators/__init__.py

@@ -0,0 +1,24 @@
+from .iterators import (  # noqa
+    PeekIterator,
+    PrefetchIterator,
+    ProxyIterator,
+    RetainIterator,
+)
+
+from .recipes import (  # noqa
+    sliding_window,
+)
+
+from .tools import (  # noqa
+    chunk,
+    expand_indexed_pairs,
+    merge_on,
+    take,
+    unzip,
+)
+
+from .unique import (  # noqa
+    UniqueItem,
+    UniqueIterator,
+    UniqueStats,
+)