omlish 0.0.0.dev141__py3-none-any.whl → 0.0.0.dev142__py3-none-any.whl

omlish/io/generators/__init__.py

@@ -0,0 +1,53 @@
+from .consts import (  # noqa
+    DEFAULT_BUFFER_SIZE,
+)
+
+from .direct import (  # noqa
+    DirectGenerator,
+
+    BytesDirectGenerator,
+    StrDirectGenerator,
+)
+
+from .readers import (  # noqa
+    ReaderGenerator,
+    BytesReaderGenerator,
+    StrReaderGenerator,
+
+    ExactReaderGenerator,
+    BytesExactReaderGenerator,
+    StrExactReaderGenerator,
+
+    GeneratorReader,
+
+    PrependableGeneratorReader,
+    PrependableBytesGeneratorReader,
+    PrependableStrGeneratorReader,
+    prependable_bytes_generator_reader,
+    prependable_str_generator_reader,
+
+    BufferedGeneratorReader,
+    BufferedBytesGeneratorReader,
+    BufferedStrGeneratorReader,
+    buffered_bytes_generator_reader,
+    buffered_str_generator_reader,
+)
+
+from .stepped import (  # noqa
+    SteppedGenerator,
+    BytesSteppedGenerator,
+    StrSteppedGenerator,
+    BytesToStrSteppedGenerator,
+    StrToBytesSteppedGenerator,
+
+    SteppedReaderGenerator,
+    BytesSteppedReaderGenerator,
+    StrSteppedReaderGenerator,
+
+    flatmap_stepped_generator,
+
+    joined_bytes_stepped_generator,
+    joined_str_stepped_generator,
+
+    read_into_bytes_stepped_generator,
+)

omlish/io/generators/consts.py

@@ -0,0 +1 @@
+DEFAULT_BUFFER_SIZE = 4 * 0x1000

omlish/io/generators/direct.py

@@ -0,0 +1,13 @@
+import typing as ta
+
+
+O = ta.TypeVar('O')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+
+
+# Direct generators yield outputs 1:1 with inputs.
+DirectGenerator: ta.TypeAlias = ta.Generator[O, I, R]
+
+BytesDirectGenerator: ta.TypeAlias = DirectGenerator[bytes, bytes, R]
+StrDirectGenerator: ta.TypeAlias = DirectGenerator[str, str, R]

omlish/io/generators/readers.py

@@ -8,21 +8,29 @@ import abc
 import typing as ta
 
 from ... import check
+from .consts import DEFAULT_BUFFER_SIZE
 
 
 T = ta.TypeVar('T')
+
+O = ta.TypeVar('O')
 I = ta.TypeVar('I')
 R = ta.TypeVar('R')
+
 AnyT = ta.TypeVar('AnyT', bound=ta.Any)
 
 
+# Reader generators yield ints of amounts of data needed, or None for needing any amount of data.
 ReaderGenerator: ta.TypeAlias = ta.Generator[int | None, I, R]
-ExactReaderGenerator: ta.TypeAlias = ta.Generator[int, I, R]
 
 BytesReaderGenerator: ta.TypeAlias = ReaderGenerator[bytes, R]
-BytesExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[bytes, R]
-
 StrReaderGenerator: ta.TypeAlias = ReaderGenerator[str, R]
+
+
+# Exact reader generators always specify read sizes.
+ExactReaderGenerator: ta.TypeAlias = ta.Generator[int, I, R]
+
+BytesExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[bytes, R]
 StrExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[str, R]
 
 
@@ -44,10 +52,10 @@ class _StrJoiner:
 
 class GeneratorReader(abc.ABC, ta.Generic[T]):
     @abc.abstractmethod
-    def read(self, sz: int | None) -> ta.Generator[int | None, T, T]:
+    def read(self, sz: int | None) -> ReaderGenerator[T, T]:
         raise NotImplementedError
 
-    def read_exact(self, sz: int) -> ta.Generator[int | None, T, T]:
+    def read_exact(self, sz: int) -> ReaderGenerator[T, T]:
         d: ta.Any = yield from self.read(sz)
         if len(d) != sz:
             raise EOFError(f'GeneratorReader got {len(d)}, expected {sz}')
@@ -67,7 +75,7 @@ class PrependableGeneratorReader(GeneratorReader[AnyT]):
     def _join(self, lst: list[AnyT]) -> AnyT:
         raise NotImplementedError
 
-    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+    def read(self, sz: int | None) -> ReaderGenerator[AnyT, AnyT]:
         if not self._queue:
             d: AnyT = check.not_none((yield sz))
             return d
@@ -126,8 +134,6 @@ prependable_str_generator_reader = PrependableStrGeneratorReader
 
 
 class BufferedGeneratorReader(PrependableGeneratorReader[AnyT], abc.ABC):
-    DEFAULT_BUFFER_SIZE = 4 * 0x1000
-
     def __init__(
             self,
             buffer_size: int = DEFAULT_BUFFER_SIZE,
@@ -138,7 +144,7 @@ class BufferedGeneratorReader(PrependableGeneratorReader[AnyT], abc.ABC):
 
         self._buffer_size = buffer_size
 
-    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+    def read(self, sz: int | None) -> ReaderGenerator[AnyT, AnyT]:
         g = super().read(sz)
         i: ta.Any = None
         while True:

omlish/io/generators/stepped.py

@@ -1,18 +1,40 @@
 import typing as ta
 
+from ... import check
 from ... import lang
+from .consts import DEFAULT_BUFFER_SIZE
+from .direct import BytesDirectGenerator
+from .direct import StrDirectGenerator
 
 
 T = ta.TypeVar('T')
-I = ta.TypeVar('I')
+
 O = ta.TypeVar('O')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+
 OF = ta.TypeVar('OF')
 OT = ta.TypeVar('OT')
-R = ta.TypeVar('R')
 
 
+# Stepped generators accept a non-None input, then in response yield zero or more non-None outputs, until yielding None
+# to signal they need more input again.
 SteppedGenerator: ta.TypeAlias = ta.Generator[O | None, I | None, R]
 
+# Conventionally, these are sent and themselves yield an empty value to signify termination.
+BytesSteppedGenerator: ta.TypeAlias = SteppedGenerator[bytes, bytes, R]
+StrSteppedGenerator: ta.TypeAlias = SteppedGenerator[str, str, R]
+
+BytesToStrSteppedGenerator: ta.TypeAlias = SteppedGenerator[str, bytes, R]
+StrToBytesSteppedGenerator: ta.TypeAlias = SteppedGenerator[bytes, str, R]
+
+
+# Stepped reader generators emit either an int or None to request input, or emit some other kind of output.
+SteppedReaderGenerator: ta.TypeAlias = ta.Generator[int | None | O, I | None, R]
+
+BytesSteppedReaderGenerator: ta.TypeAlias = SteppedReaderGenerator[bytes, bytes, R]
+StrSteppedReaderGenerator: ta.TypeAlias = SteppedReaderGenerator[str, str, R]
+
 
 ##
 
@@ -25,10 +47,8 @@ def flatmap_stepped_generator(
         terminate: ta.Callable[[OF], bool] | None = None,
 ) -> ta.Generator[OT, I, lang.Maybe[R]]:
     """
-    Given a 'stepped generator' - a generator which accepts input items and yields zero or more non-None values in
-    response until it signals it's ready for the next input by yielding None - and a function taking a list, returns a
-    1:1 generator which accepts input, builds a list of yielded generator output, calls the given function with that
-    list, and yields the result.
+    Given a stepped generator and a function taking a list, returns a direct (1:1) generator which accepts input, builds
+    a list of yielded generator output, calls the given function with that list, and yields the result.
 
     An optional terminate function may be provided which will cause this function to return early if it returns true for
     an encountered yielded value. The encountered value causing termination will be included in the list sent to the
@@ -89,16 +109,67 @@ def _is_empty(o: T) -> bool:
     return len(o) < 1  # type: ignore
 
 
+def joined_bytes_stepped_generator(g: BytesSteppedGenerator[R]) -> BytesDirectGenerator[R]:
+    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+
+
+def joined_str_stepped_generator(g: StrSteppedGenerator[R]) -> StrDirectGenerator[R]:
+    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)
+
+
 ##
 
 
-def joined_bytes_stepped_generator(
-        g: ta.Generator[bytes | None, bytes | None, R],
-) -> ta.Generator[bytes, bytes, R]:
-    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+def read_into_bytes_stepped_generator(
+        g: BytesSteppedGenerator,
+        f: ta.IO,
+        *,
+        read_size: int = DEFAULT_BUFFER_SIZE,
+) -> ta.Iterator[bytes]:
+    yield from lang.genmap(  # type: ignore[misc]
+        joined_bytes_stepped_generator(g),
+        lang.readiter(f, read_size),
+    )
 
 
-def joined_str_stepped_generator(
-        g: ta.Generator[str | None, str | None, R],
-) -> ta.Generator[str, str, R]:
-    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)
+def read_into_str_stepped_generator(
+        g: StrSteppedGenerator,
+        f: ta.TextIO,
+        *,
+        read_size: int = DEFAULT_BUFFER_SIZE,
+) -> ta.Iterator[str]:
+    yield from lang.genmap(
+        joined_str_stepped_generator(g),
+        lang.readiter(f, read_size),
+    )
+
+
+##
+
+
+@lang.autostart
+def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> BytesSteppedGenerator:
+    o = g.send(None)
+    buf: ta.Any = None
+
+    while True:
+        if not buf:
+            buf = check.isinstance((yield None), bytes)
+
+        if o is None or not buf:
+            i = buf
+        elif isinstance(o, int):
+            if len(buf) < o:
+                raise NotImplementedError
+            i = buf[:o]
+            buf = buf[o:]
+        else:
+            raise TypeError(o)
+
+        while True:
+            o = g.send(i)
+            i = None
+            if isinstance(o, bytes):
+                check.none((yield o))
+            else:
+                break

omlish/io/pyio.py

@@ -4,7 +4,7 @@
 """
 Python implementation of the io module.
 
-https://github.com/python/cpython/blob/8fa4dc4ba8646c59f945f2451c53e2919f066065/Lib/_pyio.py
+https://github.com/python/cpython/blob/8b3cccf3f9508572d85b0044519f2bd5715dacad/Lib/_pyio.py
 """
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
 # --------------------------------------------
@@ -2580,8 +2580,11 @@ class TextIOWrapper(TextIOBase):
         decoder = self._decoder or self._get_decoder()
 
         if size < 0:
+            chunk = self.buffer.read()
+            if chunk is None:
+                raise BlockingIOError("Read returned None.")
             # Read everything.
-            result = self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)
+            result = self._get_decoded_chars() + decoder.decode(chunk, final=True)
             if self._snapshot is not None:
                 self._set_decoded_chars('')
                 self._snapshot = None

omlish/lang/__init__.py

@@ -156,6 +156,7 @@ from .iterables import (  # noqa
     itergen,
     peek,
     prodrange,
+    readiter,
     renumerate,
     take,
 )

omlish/lang/iterables.py

@@ -1,4 +1,5 @@
 import dataclasses as dc
+import functools
 import itertools
 import typing as ta
 
@@ -46,6 +47,25 @@ def interleave(vs: ta.Iterable[T], d: T) -> ta.Iterable[T]:
         yield v
 
 
+@ta.overload
+def readiter(f: ta.TextIO, sz: int) -> ta.Iterator[str]:
+    ...
+
+
+@ta.overload
+def readiter(f: ta.BinaryIO, sz: int) -> ta.Iterator[bytes]:
+    ...
+
+
+@ta.overload
+def readiter(f: ta.IO, sz: int) -> ta.Iterator[ta.AnyStr]:
+    ...
+
+
+def readiter(f, sz):
+    return iter(functools.partial(f.read, sz), None)
+
+
 @dc.dataclass(frozen=True)
 class IterGen(ta.Generic[T]):
     fn: ta.Callable[[], ta.Iterable[T]]

omlish/lite/marshal.py

@@ -251,117 +251,141 @@ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
 }
 
 
-def _make_obj_marshaler(
-        ty: ta.Any,
-        rec: ta.Callable[[ta.Any], ObjMarshaler],
-        *,
-        nonstrict_dataclasses: bool = False,
-) -> ObjMarshaler:
-    if isinstance(ty, type):
-        if abc.ABC in ty.__bases__:
-            return PolymorphicObjMarshaler.of([  # type: ignore
-                PolymorphicObjMarshaler.Impl(
-                    ity,
-                    ity.__qualname__,
-                    rec(ity),
-                )
-                for ity in deep_subclasses(ty)
-                if abc.ABC not in ity.__bases__
-            ])
-
-        if issubclass(ty, enum.Enum):
-            return EnumObjMarshaler(ty)
-
-        if dc.is_dataclass(ty):
-            return DataclassObjMarshaler(
-                ty,
-                {f.name: rec(f.type) for f in dc.fields(ty)},
-                nonstrict=nonstrict_dataclasses,
-            )
-
-    if is_generic_alias(ty):
-        try:
-            mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
-        except KeyError:
-            pass
-        else:
-            k, v = ta.get_args(ty)
-            return MappingObjMarshaler(mt, rec(k), rec(v))
-
-        try:
-            st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
-        except KeyError:
-            pass
-        else:
-            [e] = ta.get_args(ty)
-            return IterableObjMarshaler(st, rec(e))
-
-        if is_union_alias(ty):
-            return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
-
-    raise TypeError(ty)
-
-
 ##
 
 
-_OBJ_MARSHALERS_LOCK = threading.RLock()
-
-_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
-
-_OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
-
+class ObjMarshalerManager:
+    def __init__(
+            self,
+            *,
+            default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS,  # noqa
+            generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES,  # noqa
+            generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES,  # noqa
+    ) -> None:
+        super().__init__()
+
+        self._obj_marshalers = dict(default_obj_marshalers)
+        self._generic_mapping_types = generic_mapping_types
+        self._generic_iterable_types = generic_iterable_types
+
+        self._lock = threading.RLock()
+        self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
+        self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
+
+    #
+
+    def make_obj_marshaler(
+            self,
+            ty: ta.Any,
+            rec: ta.Callable[[ta.Any], ObjMarshaler],
+            *,
+            nonstrict_dataclasses: bool = False,
+    ) -> ObjMarshaler:
+        if isinstance(ty, type):
+            if abc.ABC in ty.__bases__:
+                return PolymorphicObjMarshaler.of([  # type: ignore
+                    PolymorphicObjMarshaler.Impl(
+                        ity,
+                        ity.__qualname__,
+                        rec(ity),
+                    )
+                    for ity in deep_subclasses(ty)
+                    if abc.ABC not in ity.__bases__
+                ])
+
+            if issubclass(ty, enum.Enum):
+                return EnumObjMarshaler(ty)
+
+            if dc.is_dataclass(ty):
+                return DataclassObjMarshaler(
+                    ty,
+                    {f.name: rec(f.type) for f in dc.fields(ty)},
+                    nonstrict=nonstrict_dataclasses,
+                )
 
-def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
-    with _OBJ_MARSHALERS_LOCK:
-        if ty in _OBJ_MARSHALERS:
-            raise KeyError(ty)
-        _OBJ_MARSHALERS[ty] = m
+        if is_generic_alias(ty):
+            try:
+                mt = self._generic_mapping_types[ta.get_origin(ty)]
+            except KeyError:
+                pass
+            else:
+                k, v = ta.get_args(ty)
+                return MappingObjMarshaler(mt, rec(k), rec(v))
 
+            try:
+                st = self._generic_iterable_types[ta.get_origin(ty)]
+            except KeyError:
+                pass
+            else:
+                [e] = ta.get_args(ty)
+                return IterableObjMarshaler(st, rec(e))
+
+            if is_union_alias(ty):
+                return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
+
+        raise TypeError(ty)
+
+    #
+
+    def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
+        with self._lock:
+            if ty in self._obj_marshalers:
+                raise KeyError(ty)
+            self._obj_marshalers[ty] = m
+
+    def get_obj_marshaler(
+            self,
+            ty: ta.Any,
+            *,
+            no_cache: bool = False,
+            **kwargs: ta.Any,
+    ) -> ObjMarshaler:
+        with self._lock:
+            if not no_cache:
+                try:
+                    return self._obj_marshalers[ty]
+                except KeyError:
+                    pass
 
-def get_obj_marshaler(
-        ty: ta.Any,
-        *,
-        no_cache: bool = False,
-        **kwargs: ta.Any,
-) -> ObjMarshaler:
-    with _OBJ_MARSHALERS_LOCK:
-        if not no_cache:
             try:
-                return _OBJ_MARSHALERS[ty]
+                return self._proxies[ty]
             except KeyError:
                 pass
 
-        try:
-            return _OBJ_MARSHALER_PROXIES[ty]
-        except KeyError:
-            pass
+            rec = functools.partial(
+                self.get_obj_marshaler,
+                no_cache=no_cache,
+                **kwargs,
+            )
 
-        rec = functools.partial(
-            get_obj_marshaler,
-            no_cache=no_cache,
-            **kwargs,
-        )
+            p = ProxyObjMarshaler()
+            self._proxies[ty] = p
+            try:
+                m = self.make_obj_marshaler(ty, rec, **kwargs)
+            finally:
+                del self._proxies[ty]
+            p.m = m
+
+            if not no_cache:
+                self._obj_marshalers[ty] = m
+            return m
+
+    #
 
-        p = ProxyObjMarshaler()
-        _OBJ_MARSHALER_PROXIES[ty] = p
-        try:
-            m = _make_obj_marshaler(ty, rec, **kwargs)
-        finally:
-            del _OBJ_MARSHALER_PROXIES[ty]
-        p.m = m
+    def marshal_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Any:
+        return self.get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
 
-        if not no_cache:
-            _OBJ_MARSHALERS[ty] = m
-        return m
+    def unmarshal_obj(self, o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
+        return self.get_obj_marshaler(ty).unmarshal(o)
 
 
 ##
 
 
-def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
-    return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
+OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
 
+register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
+get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
 
-def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
-    return get_obj_marshaler(ty).unmarshal(o)
+marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
+unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj

omlish/lite/subprocesses.py

@@ -10,6 +10,19 @@ from .logs import log
 from .runtime import is_debugger_attached
 
 
+SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']
+
+
+##
+
+
+SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
+    'pipe': subprocess.PIPE,
+    'stdout': subprocess.STDOUT,
+    'devnull': subprocess.DEVNULL,
+}
+
+
 ##
 
 

{omlish-0.0.0.dev141.dist-info → omlish-0.0.0.dev142.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev141
+Version: 0.0.0.dev142
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -21,6 +21,7 @@ Requires-Dist: trio-asyncio~=0.15; extra == "all"
 Requires-Dist: lz4~=4.3; extra == "all"
 Requires-Dist: python-snappy~=0.7; extra == "all"
 Requires-Dist: zstandard~=0.23; extra == "all"
+Requires-Dist: brotli~=1.1; extra == "all"
 Requires-Dist: asttokens~=3.0; extra == "all"
 Requires-Dist: executing~=2.1; extra == "all"
 Requires-Dist: psutil~=6.0; extra == "all"
@@ -60,6 +61,7 @@ Provides-Extra: compress
 Requires-Dist: lz4~=4.3; extra == "compress"
 Requires-Dist: python-snappy~=0.7; extra == "compress"
 Requires-Dist: zstandard~=0.23; extra == "compress"
+Requires-Dist: brotli~=1.1; extra == "compress"
 Provides-Extra: diag
 Requires-Dist: asttokens~=3.0; extra == "diag"
 Requires-Dist: executing~=2.1; extra == "diag"