omlish 0.0.0.dev164__tar.gz → 0.0.0.dev166__tar.gz

{omlish-0.0.0.dev164/omlish.egg-info → omlish-0.0.0.dev166}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev164
+Version: 0.0.0.dev166
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/.manifests.json

@@ -37,6 +37,20 @@
       }
     }
   },
+  {
+    "module": ".formats.cloudpickle",
+    "attr": "_CLOUDPICKLE_LAZY_CODEC",
+    "file": "omlish/formats/cloudpickle.py",
+    "line": 30,
+    "value": {
+      "$.codecs.base.LazyLoadedCodec": {
+        "mod_name": "omlish.formats.cloudpickle",
+        "attr_name": "CLOUDPICKLE_CODEC",
+        "name": "cloudpickle",
+        "aliases": null
+      }
+    }
+  },
   {
     "module": ".formats.json.codecs",
     "attr": "_JSON_LAZY_CODEC",
@@ -55,7 +69,7 @@
     "module": ".formats.json.codecs",
     "attr": "_JSON_COMPACT_LAZY_CODEC",
     "file": "omlish/formats/json/codecs.py",
-    "line": 23,
+    "line": 21,
     "value": {
       "$.codecs.base.LazyLoadedCodec": {
         "mod_name": "omlish.formats.json.codecs",
@@ -69,7 +83,7 @@
     "module": ".formats.json.codecs",
     "attr": "_JSON_PRETTY_LAZY_CODEC",
     "file": "omlish/formats/json/codecs.py",
-    "line": 32,
+    "line": 28,
     "value": {
       "$.codecs.base.LazyLoadedCodec": {
         "mod_name": "omlish.formats.json.codecs",
@@ -137,6 +151,20 @@
       }
     }
   },
+  {
+    "module": ".formats.yaml",
+    "attr": "_YAML_UNSAFE_LAZY_CODEC",
+    "file": "omlish/formats/yaml.py",
+    "line": 265,
+    "value": {
+      "$.codecs.base.LazyLoadedCodec": {
+        "mod_name": "omlish.formats.yaml",
+        "attr_name": "YAML_UNSAFE_CODEC",
+        "name": "yaml-unsafe",
+        "aliases": null
+      }
+    }
+  },
   {
     "module": ".io.compress.brotli",
     "attr": "_BROTLI_LAZY_CODEC",

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev164'
-__revision__ = '72b4cfb1086b384ec55cc221069d5bb2be6b3c10'
+__version__ = '0.0.0.dev166'
+__revision__ = 'e832ee32347c3f4c51e8ead2186def228e3aac1c'
 
 
 #

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/codecs/__init__.py

@@ -30,6 +30,9 @@ from .chain import (  # noqa
 
 from .funcs import (  # noqa
     FnPairEagerCodec,
+
+    of_pair,
+    of,
 )
 
 from .registry import (  # noqa

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/codecs/base.py

@@ -9,6 +9,7 @@ from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
 from omlish import reflect as rfl
+from omlish.funcs import pairs as fps
 
 
 I = ta.TypeVar('I')
@@ -27,6 +28,9 @@ class EagerCodec(lang.Abstract, ta.Generic[I, O]):
     def decode(self, o: O) -> I:
         raise NotImplementedError
 
+    def as_pair(self) -> fps.FnPair[I, O]:
+        return fps.of(self.encode, self.decode)
+
 
 class IncrementalCodec(lang.Abstract, ta.Generic[I, O]):
     @abc.abstractmethod

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/codecs/funcs.py

@@ -26,3 +26,14 @@ class FnPairEagerCodec(EagerCodec[I, O]):
             decode: ta.Callable[[O], I],
     ) -> 'FnPairEagerCodec[I, O]':
         return cls(fps.of(encode, decode))
+
+
+def of_pair(fp: fps.FnPair[I, O]) -> FnPairEagerCodec[I, O]:
+    return FnPairEagerCodec(fp)
+
+
+def of(
+        encode: ta.Callable[[I], O],
+        decode: ta.Callable[[O], I],
+) -> FnPairEagerCodec[I, O]:
+    return FnPairEagerCodec(fps.of(encode, decode))

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/codecs/text.py

@@ -46,8 +46,8 @@ TextEncodingErrors: ta.TypeAlias = ta.Literal[
     'namereplace',
 
     ##
-    # In addition, the following error handler is specific to the given codecs:
-    # utf-8, utf-16, utf-32, utf-16-be, utf-16-le, utf-32-be, utf-32-le
+    # In addition, the following error handler is specific to the given codecs: utf-8, utf-16, utf-32, utf-16-be,
+    # utf-16-le, utf-32-be, utf-32-le
 
     # Allow encoding and decoding surrogate code point (U+D800 - U+DFFF) as normal code point. Otherwise these codecs
     # treat the presence of surrogate code point in str as an error.

omlish-0.0.0.dev166/omlish/formats/cloudpickle.py

@@ -0,0 +1,31 @@
+import typing as ta
+
+from .. import lang
+from .codecs import make_bytes_object_codec
+from .codecs import make_object_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import cloudpickle
+else:
+    cloudpickle = lang.proxy_import('cloudpickle')
+
+
+##
+
+
+def dump(obj: ta.Any) -> bytes:
+    return cloudpickle.dumps(obj)
+
+
+def load(s: bytes) -> ta.Any:
+    return cloudpickle.loads(s)
+
+
+##
+
+
+CLOUDPICKLE_CODEC = make_bytes_object_codec('cloudpickle', dump, load)
+
+# @omlish-manifest
+_CLOUDPICKLE_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'CLOUDPICKLE_CODEC', CLOUDPICKLE_CODEC)

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/formats/json/codecs.py

@@ -14,19 +14,15 @@ JSON_CODEC = make_str_object_codec('json', dumps, loads)
 # @omlish-manifest
 _JSON_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'JSON_CODEC', JSON_CODEC)
 
-
 #
 
-
 JSON_COMPACT_CODEC = make_str_object_codec('json-compact', dumps_compact, loads)
 
 # @omlish-manifest
 _JSON_COMPACT_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'JSON_COMPACT_CODEC', JSON_COMPACT_CODEC)
 
-
 #
 
-
 JSON_PRETTY_CODEC = make_str_object_codec('json-pretty', dumps_pretty, loads)
 
 # @omlish-manifest

omlish-0.0.0.dev166/omlish/formats/json/delimted.py

@@ -0,0 +1,4 @@
+"""
+TODO:
+ - jsonl codec
+"""

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/formats/yaml.py

@@ -257,3 +257,10 @@ YAML_CODEC = make_str_object_codec('yaml', dump, safe_load, aliases=['yml'])
 
 # @omlish-manifest
 _YAML_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'YAML_CODEC', YAML_CODEC)
+
+#
+
+YAML_UNSAFE_CODEC = make_str_object_codec('yaml-unsafe', dump, full_load)
+
+# @omlish-manifest
+_YAML_UNSAFE_LAZY_CODEC = make_object_lazy_loaded_codec(__name__, 'YAML_UNSAFE_CODEC', YAML_UNSAFE_CODEC)

omlish-0.0.0.dev166/omlish/funcs/pairs.py

@@ -0,0 +1,215 @@
+import abc
+import dataclasses as dc
+import typing as ta
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import struct as _struct
+else:
+    _struct = lang.proxy_import('struct')
+
+
+##
+
+
+F = ta.TypeVar('F')
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
+
+
+class FnPair(ta.Generic[F, T], abc.ABC):
+    @abc.abstractmethod
+    def forward(self, f: F) -> T:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def backward(self, t: T) -> F:
+        raise NotImplementedError
+
+    ##
+
+    def __call__(self, f: F) -> T:
+        return self.forward(f)
+
+    def invert(self) -> 'FnPair[T, F]':
+        if isinstance(self, Inverted):
+            return self.fp
+        return Inverted(self)
+
+    def compose(self, nxt: 'FnPair[T, U]') -> 'FnPair[F, U]':
+        return Composite((self, nxt))
+
+
+##
+
+
+@lang.unabstract_class(['forward', 'backward'])
+@dc.dataclass(frozen=True)
+class Simple(FnPair[F, T]):
+    forward: ta.Callable[[F], T]  # type: ignore
+    backward: ta.Callable[[T], F]  # type: ignore
+
+
+of = Simple
+
+NOP: FnPair[ta.Any, ta.Any] = of(lang.identity, lang.identity)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Inverted(FnPair[F, T]):
+    fp: FnPair[T, F]
+
+    def forward(self, f: F) -> T:
+        return self.fp.backward(f)
+
+    def backward(self, t: T) -> F:
+        return self.fp.forward(t)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Composite(FnPair[F, T]):
+    children: ta.Sequence[FnPair]
+
+    def forward(self, f: F) -> T:
+        for c in self.children:
+            f = c.forward(f)
+        return ta.cast(T, f)
+
+    def backward(self, t: T) -> F:
+        for c in reversed(self.children):
+            t = c.backward(t)
+        return ta.cast(F, t)
+
+
+I0 = ta.TypeVar('I0')
+I1 = ta.TypeVar('I1')
+I2 = ta.TypeVar('I2')
+I3 = ta.TypeVar('I3')
+I4 = ta.TypeVar('I4')
+
+
+@ta.overload
+def compose(
+        fp0: FnPair[F, I0],
+        f01: FnPair[I0, T],
+) -> FnPair[F, T]:
+    ...
+
+
+@ta.overload
+def compose(
+        fp0: FnPair[F, I0],
+        f01: FnPair[I0, I1],
+        fp2: FnPair[I1, T],
+) -> FnPair[F, T]:
+    ...
+
+
+@ta.overload
+def compose(
+        fp0: FnPair[F, I0],
+        f01: FnPair[I0, I1],
+        fp2: FnPair[I1, I2],
+        fp3: FnPair[I2, T],
+) -> FnPair[F, T]:
+    ...
+
+
+@ta.overload
+def compose(
+        fp0: FnPair[F, I0],
+        f01: FnPair[I0, I1],
+        fp2: FnPair[I1, I2],
+        fp3: FnPair[I2, I3],
+        fp4: FnPair[I3, T],
+) -> FnPair[F, T]:
+    ...
+
+
+@ta.overload
+def compose(
+        fp0: FnPair[F, I0],
+        f01: FnPair[I0, I1],
+        fp2: FnPair[I1, I2],
+        fp3: FnPair[I2, I3],
+        fp4: FnPair[I3, I4],
+        fp5: FnPair[I4, T],
+) -> FnPair[F, T]:
+    ...
+
+
+@ta.overload
+def compose(*ps: FnPair) -> FnPair:
+    ...
+
+
+def compose(*ps):
+    if not ps:
+        return NOP
+    if len(ps) == 1:
+        return ps[0]
+    return Composite(ps)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Optional(FnPair[F | None, T | None]):
+    fp: FnPair[F, T]
+
+    def forward(self, f: F | None) -> T | None:
+        return None if f is None else self.fp.forward(f)
+
+    def backward(self, t: T | None) -> F | None:
+        return None if t is None else self.fp.backward(t)
+
+
+class Lines(FnPair[ta.Sequence[str], str]):
+    def forward(self, f: ta.Sequence[str]) -> str:
+        return '\n'.join(f)
+
+    def backward(self, t: str) -> ta.Sequence[str]:
+        return t.splitlines()
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Struct(FnPair[tuple, bytes]):
+    fmt: str
+
+    def forward(self, f: tuple) -> bytes:
+        return _struct.pack(self.fmt, *f)
+
+    def backward(self, t: bytes) -> tuple:
+        return _struct.unpack(self.fmt, t)
+
+
+##
+
+
+Object: ta.TypeAlias = FnPair[ta.Any, T]
+ObjectStr: ta.TypeAlias = Object[str]
+ObjectBytes: ta.TypeAlias = Object[bytes]
+
+
+class Object_(FnPair[ta.Any, T], lang.Abstract):  # noqa
+    pass
+
+
+class ObjectStr_(Object_[str], lang.Abstract):  # noqa
+    pass
+
+
+class ObjectBytes_(Object_[bytes], lang.Abstract):  # noqa
+    pass

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/io/compress/codecs.py

@@ -2,7 +2,9 @@ import dataclasses as dc
 import typing as ta
 
 from ... import codecs
+from ..generators import buffer_bytes_stepped_reader_generator
 from .base import Compression
+from .base import IncrementalCompression
 
 
 ##
@@ -22,6 +24,20 @@ class CompressionEagerCodec(codecs.EagerCodec[bytes, bytes]):
 ##
 
 
+@dc.dataclass(frozen=True)
+class CompressionIncrementalCodec(codecs.IncrementalCodec[bytes, bytes]):
+    compression: IncrementalCompression
+
+    def encode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return self.compression.compress_incremental()
+
+    def decode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return buffer_bytes_stepped_reader_generator(self.compression.decompress_incremental())
+
+
+##
+
+
 class CompressionCodec(codecs.Codec):
     pass
 
@@ -40,6 +56,10 @@ def make_compression_codec(
         output=bytes,
 
         new=lambda *args, **kwargs: CompressionEagerCodec(cls(*args, **kwargs)),
+
+        new_incremental=(
+            lambda *args, **kwargs: CompressionIncrementalCodec(cls(*args, **kwargs))  # noqa
+        ) if issubclass(cls, IncrementalCompression) else None,
     )
 
 

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/io/generators/__init__.py

@@ -50,4 +50,7 @@ from .stepped import (  # noqa
     joined_str_stepped_generator,
 
     read_into_bytes_stepped_generator,
+    read_into_str_stepped_generator,
+
+    buffer_bytes_stepped_reader_generator,
 )

{omlish-0.0.0.dev164 → omlish-0.0.0.dev166}/omlish/io/generators/stepped.py

@@ -151,18 +151,32 @@ def read_into_str_stepped_generator(
 def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> BytesSteppedGenerator:
     o = g.send(None)
     buf: ta.Any = None
+    eof = False
 
     while True:
+        if eof:
+            raise EOFError
+
         if not buf:
             buf = check.isinstance((yield None), bytes)
+            if not buf:
+                eof = True
 
-        if o is None or not buf:
+        if o is None:
             i = buf
+            buf = None
+
         elif isinstance(o, int):
-            if len(buf) < o:
-                raise NotImplementedError
+            while len(buf) < o:
+                more = check.isinstance((yield None), bytes)
+                if not more:
+                    raise EOFError
+                # FIXME: lol - share guts with readers
+                buf += more
+
             i = buf[:o]
             buf = buf[o:]
+
         else:
             raise TypeError(o)
 
@@ -171,5 +185,7 @@ def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> Byt
             i = None
             if isinstance(o, bytes):
                 check.none((yield o))
+                if not o:
+                    return
             else:
                 break

omlish-0.0.0.dev166/omlish/iterators/__init__.py

@@ -0,0 +1,24 @@
+from .iterators import (  # noqa
+    PeekIterator,
+    PrefetchIterator,
+    ProxyIterator,
+    RetainIterator,
+)
+
+from .recipes import (  # noqa
+    sliding_window,
+)
+
+from .tools import (  # noqa
+    chunk,
+    expand_indexed_pairs,
+    merge_on,
+    take,
+    unzip,
+)
+
+from .unique import (  # noqa
+    UniqueItem,
+    UniqueIterator,
+    UniqueStats,
+)

omlish-0.0.0.dev166/omlish/iterators/iterators.py

@@ -0,0 +1,132 @@
+import collections
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+_MISSING = object()
+
+
+class PeekIterator(ta.Iterator[T]):
+
+    def __init__(self, it: ta.Iterable[T]) -> None:
+        super().__init__()
+
+        self._it = iter(it)
+        self._pos = -1
+        self._next_item: ta.Any = _MISSING
+
+    _item: T
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    @property
+    def done(self) -> bool:
+        try:
+            self.peek()
+        except StopIteration:
+            return True
+        else:
+            return False
+
+    def __next__(self) -> T:
+        if self._next_item is not _MISSING:
+            self._item = ta.cast(T, self._next_item)
+            self._next_item = _MISSING
+        else:
+            self._item = next(self._it)
+        self._pos += 1
+        return self._item
+
+    def peek(self) -> T:
+        if self._next_item is not _MISSING:
+            return ta.cast(T, self._next_item)
+        self._next_item = next(self._it)
+        return self._next_item
+
+    def next_peek(self) -> T:
+        next(self)
+        return self.peek()
+
+    def takewhile(self, fn: ta.Callable[[T], bool]) -> ta.Iterator[T]:
+        while fn(self.peek()):
+            yield next(self)
+
+    def skipwhile(self, fn: ta.Callable[[T], bool]) -> None:
+        while fn(self.peek()):
+            next(self)
+
+    def takeuntil(self, fn: ta.Callable[[T], bool]) -> ta.Iterator[T]:
+        return self.takewhile(lambda e: not fn(e))
+
+    def skipuntil(self, fn: ta.Callable[[T], bool]) -> None:
+        self.skipwhile(lambda e: not fn(e))
+
+    def takethrough(self, pos: int) -> ta.Iterator[T]:
+        return self.takewhile(lambda _: self._pos < pos)
+
+    def skipthrough(self, pos: int) -> None:
+        self.skipwhile(lambda _: self._pos < pos)
+
+    def taketo(self, pos: int) -> ta.Iterator[T]:
+        return self.takethrough(pos - 1)
+
+    def skipto(self, pos: int) -> None:
+        self.skipthrough(pos - 1)
+
+
+class ProxyIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T]) -> None:
+        self._fn = fn
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def __next__(self) -> T:
+        return self._fn()
+
+
+class PrefetchIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T] | None = None) -> None:
+        super().__init__()
+
+        self._fn = fn
+        self._deque: collections.deque[T] = collections.deque()
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def push(self, item) -> None:
+        self._deque.append(item)
+
+    def __next__(self) -> T:
+        try:
+            return self._deque.popleft()
+        except IndexError:
+            if self._fn is None:
+                raise StopIteration from None
+        return self._fn()
+
+
+class RetainIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T]) -> None:
+        super().__init__()
+
+        self._fn = fn
+        self._deque: collections.deque[T] = collections.deque()
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def pop(self) -> None:
+        self._deque.popleft()
+
+    def __next__(self) -> T:
+        item = self._fn()
+        self._deque.append(item)
+        return item

omlish-0.0.0.dev166/omlish/iterators/recipes.py

@@ -0,0 +1,18 @@
+"""
+https://docs.python.org/3/library/itertools.html#itertools-recipes
+"""
+import collections
+import itertools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def sliding_window(it: ta.Iterable[T], n: int) -> ta.Iterator[tuple[T, ...]]:
+    # sliding_window('ABCDEFG', 4) -> ABCD BCDE CDEF DEFG
+    iterator = iter(it)
+    window = collections.deque(itertools.islice(iterator, n - 1), maxlen=n)
+    for x in iterator:
+        window.append(x)
+        yield tuple(window)