omlish 0.0.0.dev141__py3-none-any.whl → 0.0.0.dev143__py3-none-any.whl

omlish/io/compress/zstd.py

@@ -0,0 +1,30 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import lang
+from .base import Compression
+
+
+if ta.TYPE_CHECKING:
+    import zstandard
+else:
+    zstandard = lang.proxy_import('zstandard')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class ZstdCompression(Compression):
+    level: int | None = None
+
+    max_output_size: int = 0
+
+    def compress(self, d: bytes) -> bytes:
+        return zstandard.compress(
+            d,
+            **(dict(level=self.level) if self.level is not None else {}),
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return zstandard.decompress(
+            d,
+            max_output_size=self.max_output_size,
+        )

omlish/io/generators/__init__.py

@@ -0,0 +1,53 @@
+from .consts import (  # noqa
+    DEFAULT_BUFFER_SIZE,
+)
+
+from .direct import (  # noqa
+    DirectGenerator,
+
+    BytesDirectGenerator,
+    StrDirectGenerator,
+)
+
+from .readers import (  # noqa
+    ReaderGenerator,
+    BytesReaderGenerator,
+    StrReaderGenerator,
+
+    ExactReaderGenerator,
+    BytesExactReaderGenerator,
+    StrExactReaderGenerator,
+
+    GeneratorReader,
+
+    PrependableGeneratorReader,
+    PrependableBytesGeneratorReader,
+    PrependableStrGeneratorReader,
+    prependable_bytes_generator_reader,
+    prependable_str_generator_reader,
+
+    BufferedGeneratorReader,
+    BufferedBytesGeneratorReader,
+    BufferedStrGeneratorReader,
+    buffered_bytes_generator_reader,
+    buffered_str_generator_reader,
+)
+
+from .stepped import (  # noqa
+    SteppedGenerator,
+    BytesSteppedGenerator,
+    StrSteppedGenerator,
+    BytesToStrSteppedGenerator,
+    StrToBytesSteppedGenerator,
+
+    SteppedReaderGenerator,
+    BytesSteppedReaderGenerator,
+    StrSteppedReaderGenerator,
+
+    flatmap_stepped_generator,
+
+    joined_bytes_stepped_generator,
+    joined_str_stepped_generator,
+
+    read_into_bytes_stepped_generator,
+)

omlish/io/generators/consts.py

@@ -0,0 +1 @@
+DEFAULT_BUFFER_SIZE = 4 * 0x1000

omlish/io/generators/direct.py

@@ -0,0 +1,13 @@
+import typing as ta
+
+
+O = ta.TypeVar('O')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+
+
+# Direct generators yield outputs 1:1 with inputs.
+DirectGenerator: ta.TypeAlias = ta.Generator[O, I, R]
+
+BytesDirectGenerator: ta.TypeAlias = DirectGenerator[bytes, bytes, R]
+StrDirectGenerator: ta.TypeAlias = DirectGenerator[str, str, R]

omlish/io/generators/readers.py

@@ -8,21 +8,29 @@ import abc
 import typing as ta
 
 from ... import check
+from .consts import DEFAULT_BUFFER_SIZE
 
 
 T = ta.TypeVar('T')
+
+O = ta.TypeVar('O')
 I = ta.TypeVar('I')
 R = ta.TypeVar('R')
+
 AnyT = ta.TypeVar('AnyT', bound=ta.Any)
 
 
+# Reader generators yield ints of amounts of data needed, or None for needing any amount of data.
 ReaderGenerator: ta.TypeAlias = ta.Generator[int | None, I, R]
-ExactReaderGenerator: ta.TypeAlias = ta.Generator[int, I, R]
 
 BytesReaderGenerator: ta.TypeAlias = ReaderGenerator[bytes, R]
-BytesExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[bytes, R]
-
 StrReaderGenerator: ta.TypeAlias = ReaderGenerator[str, R]
+
+
+# Exact reader generators always specify read sizes.
+ExactReaderGenerator: ta.TypeAlias = ta.Generator[int, I, R]
+
+BytesExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[bytes, R]
 StrExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[str, R]
 
 
@@ -44,10 +52,10 @@ class _StrJoiner:
 
 class GeneratorReader(abc.ABC, ta.Generic[T]):
     @abc.abstractmethod
-    def read(self, sz: int | None) -> ta.Generator[int | None, T, T]:
+    def read(self, sz: int | None) -> ReaderGenerator[T, T]:
         raise NotImplementedError
 
-    def read_exact(self, sz: int) -> ta.Generator[int | None, T, T]:
+    def read_exact(self, sz: int) -> ReaderGenerator[T, T]:
         d: ta.Any = yield from self.read(sz)
         if len(d) != sz:
             raise EOFError(f'GeneratorReader got {len(d)}, expected {sz}')
@@ -67,7 +75,7 @@ class PrependableGeneratorReader(GeneratorReader[AnyT]):
     def _join(self, lst: list[AnyT]) -> AnyT:
         raise NotImplementedError
 
-    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+    def read(self, sz: int | None) -> ReaderGenerator[AnyT, AnyT]:
         if not self._queue:
             d: AnyT = check.not_none((yield sz))
             return d
@@ -126,8 +134,6 @@ prependable_str_generator_reader = PrependableStrGeneratorReader
 
 
 class BufferedGeneratorReader(PrependableGeneratorReader[AnyT], abc.ABC):
-    DEFAULT_BUFFER_SIZE = 4 * 0x1000
-
     def __init__(
             self,
             buffer_size: int = DEFAULT_BUFFER_SIZE,
@@ -138,7 +144,7 @@ class BufferedGeneratorReader(PrependableGeneratorReader[AnyT], abc.ABC):
 
         self._buffer_size = buffer_size
 
-    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+    def read(self, sz: int | None) -> ReaderGenerator[AnyT, AnyT]:
         g = super().read(sz)
         i: ta.Any = None
         while True:

omlish/io/generators/stepped.py

@@ -1,18 +1,40 @@
 import typing as ta
 
+from ... import check
 from ... import lang
+from .consts import DEFAULT_BUFFER_SIZE
+from .direct import BytesDirectGenerator
+from .direct import StrDirectGenerator
 
 
 T = ta.TypeVar('T')
-I = ta.TypeVar('I')
+
 O = ta.TypeVar('O')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+
 OF = ta.TypeVar('OF')
 OT = ta.TypeVar('OT')
-R = ta.TypeVar('R')
 
 
+# Stepped generators accept a non-None input, then in response yield zero or more non-None outputs, until yielding None
+# to signal they need more input again.
 SteppedGenerator: ta.TypeAlias = ta.Generator[O | None, I | None, R]
 
+# Conventionally, these are sent and themselves yield an empty value to signify termination.
+BytesSteppedGenerator: ta.TypeAlias = SteppedGenerator[bytes, bytes, R]
+StrSteppedGenerator: ta.TypeAlias = SteppedGenerator[str, str, R]
+
+BytesToStrSteppedGenerator: ta.TypeAlias = SteppedGenerator[str, bytes, R]
+StrToBytesSteppedGenerator: ta.TypeAlias = SteppedGenerator[bytes, str, R]
+
+
+# Stepped reader generators emit either an int or None to request input, or emit some other kind of output.
+SteppedReaderGenerator: ta.TypeAlias = ta.Generator[int | None | O, I | None, R]
+
+BytesSteppedReaderGenerator: ta.TypeAlias = SteppedReaderGenerator[bytes, bytes, R]
+StrSteppedReaderGenerator: ta.TypeAlias = SteppedReaderGenerator[str, str, R]
+
 
 ##
 
@@ -25,10 +47,8 @@ def flatmap_stepped_generator(
         terminate: ta.Callable[[OF], bool] | None = None,
 ) -> ta.Generator[OT, I, lang.Maybe[R]]:
     """
-    Given a 'stepped generator' - a generator which accepts input items and yields zero or more non-None values in
-    response until it signals it's ready for the next input by yielding None - and a function taking a list, returns a
-    1:1 generator which accepts input, builds a list of yielded generator output, calls the given function with that
-    list, and yields the result.
+    Given a stepped generator and a function taking a list, returns a direct (1:1) generator which accepts input, builds
+    a list of yielded generator output, calls the given function with that list, and yields the result.
 
     An optional terminate function may be provided which will cause this function to return early if it returns true for
     an encountered yielded value. The encountered value causing termination will be included in the list sent to the
@@ -89,16 +109,67 @@ def _is_empty(o: T) -> bool:
     return len(o) < 1  # type: ignore
 
 
+def joined_bytes_stepped_generator(g: BytesSteppedGenerator[R]) -> BytesDirectGenerator[R]:
+    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+
+
+def joined_str_stepped_generator(g: StrSteppedGenerator[R]) -> StrDirectGenerator[R]:
+    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)
+
+
 ##
 
 
-def joined_bytes_stepped_generator(
-        g: ta.Generator[bytes | None, bytes | None, R],
-) -> ta.Generator[bytes, bytes, R]:
-    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+def read_into_bytes_stepped_generator(
+        g: BytesSteppedGenerator,
+        f: ta.IO,
+        *,
+        read_size: int = DEFAULT_BUFFER_SIZE,
+) -> ta.Iterator[bytes]:
+    yield from lang.genmap(  # type: ignore[misc]
+        joined_bytes_stepped_generator(g),
+        lang.readiter(f, read_size),
+    )
 
 
-def joined_str_stepped_generator(
-        g: ta.Generator[str | None, str | None, R],
-) -> ta.Generator[str, str, R]:
-    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)
+def read_into_str_stepped_generator(
+        g: StrSteppedGenerator,
+        f: ta.TextIO,
+        *,
+        read_size: int = DEFAULT_BUFFER_SIZE,
+) -> ta.Iterator[str]:
+    yield from lang.genmap(
+        joined_str_stepped_generator(g),
+        lang.readiter(f, read_size),
+    )
+
+
+##
+
+
+@lang.autostart
+def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> BytesSteppedGenerator:
+    o = g.send(None)
+    buf: ta.Any = None
+
+    while True:
+        if not buf:
+            buf = check.isinstance((yield None), bytes)
+
+        if o is None or not buf:
+            i = buf
+        elif isinstance(o, int):
+            if len(buf) < o:
+                raise NotImplementedError
+            i = buf[:o]
+            buf = buf[o:]
+        else:
+            raise TypeError(o)
+
+        while True:
+            o = g.send(i)
+            i = None
+            if isinstance(o, bytes):
+                check.none((yield o))
+            else:
+                break

omlish/io/pyio.py

@@ -4,7 +4,7 @@
 """
 Python implementation of the io module.
 
-https://github.com/python/cpython/blob/8fa4dc4ba8646c59f945f2451c53e2919f066065/Lib/_pyio.py
+https://github.com/python/cpython/blob/8b3cccf3f9508572d85b0044519f2bd5715dacad/Lib/_pyio.py
 """
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
 # --------------------------------------------
@@ -2580,8 +2580,11 @@ class TextIOWrapper(TextIOBase):
         decoder = self._decoder or self._get_decoder()
 
         if size < 0:
+            chunk = self.buffer.read()
+            if chunk is None:
+                raise BlockingIOError("Read returned None.")
             # Read everything.
-            result = self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)
+            result = self._get_decoded_chars() + decoder.decode(chunk, final=True)
             if self._snapshot is not None:
                 self._set_decoded_chars('')
                 self._snapshot = None

omlish/lang/__init__.py

@@ -156,6 +156,7 @@ from .iterables import (  # noqa
     itergen,
     peek,
     prodrange,
+    readiter,
     renumerate,
     take,
 )

omlish/lang/iterables.py

@@ -1,4 +1,5 @@
 import dataclasses as dc
+import functools
 import itertools
 import typing as ta
 
@@ -46,6 +47,25 @@ def interleave(vs: ta.Iterable[T], d: T) -> ta.Iterable[T]:
         yield v
 
 
+@ta.overload
+def readiter(f: ta.TextIO, sz: int) -> ta.Iterator[str]:
+    ...
+
+
+@ta.overload
+def readiter(f: ta.BinaryIO, sz: int) -> ta.Iterator[bytes]:
+    ...
+
+
+@ta.overload
+def readiter(f: ta.IO, sz: int) -> ta.Iterator[ta.AnyStr]:
+    ...
+
+
+def readiter(f, sz):
+    return iter(functools.partial(f.read, sz), None)
+
+
 @dc.dataclass(frozen=True)
 class IterGen(ta.Generic[T]):
     fn: ta.Callable[[], ta.Iterable[T]]

omlish/lite/docker.py

@@ -2,6 +2,9 @@ import re
 import sys
 
 
+DOCKER_FOR_MAC_HOSTNAME = 'docker.for.mac.localhost'
+
+
 _LIKELY_IN_DOCKER_PATTERN = re.compile(r'^overlay / .*/(docker|desktop-containerd)/')