omlish 0.0.0.dev133__py3-none-any.whl → 0.0.0.dev177__py3-none-any.whl

omlish/io/generators/stepped.py

@@ -0,0 +1,191 @@
+import typing as ta
+
+from ... import check
+from ... import lang
+from .consts import DEFAULT_BUFFER_SIZE
+from .direct import BytesDirectGenerator
+from .direct import StrDirectGenerator
+
+
+T = ta.TypeVar('T')
+
+O = ta.TypeVar('O')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+
+OF = ta.TypeVar('OF')
+OT = ta.TypeVar('OT')
+
+
+# Stepped generators accept a non-None input, then in response yield zero or more non-None outputs, until yielding None
+# to signal they need more input again.
+SteppedGenerator: ta.TypeAlias = ta.Generator[O | None, I | None, R]
+
+# Conventionally, these are sent and themselves yield an empty value to signify termination.
+BytesSteppedGenerator: ta.TypeAlias = SteppedGenerator[bytes, bytes, R]
+StrSteppedGenerator: ta.TypeAlias = SteppedGenerator[str, str, R]
+
+BytesToStrSteppedGenerator: ta.TypeAlias = SteppedGenerator[str, bytes, R]
+StrToBytesSteppedGenerator: ta.TypeAlias = SteppedGenerator[bytes, str, R]
+
+
+# Stepped reader generators emit either an int or None to request input, or emit some other kind of output.
+SteppedReaderGenerator: ta.TypeAlias = ta.Generator[int | None | O, I | None, R]
+
+BytesSteppedReaderGenerator: ta.TypeAlias = SteppedReaderGenerator[bytes, bytes, R]
+StrSteppedReaderGenerator: ta.TypeAlias = SteppedReaderGenerator[str, str, R]
+
+
+##
+
+
+@lang.autostart
+def flatmap_stepped_generator(
+        fn: ta.Callable[[list[OF]], OT],
+        g: SteppedGenerator[OF, I, R],
+        *,
+        terminate: ta.Callable[[OF], bool] | None = None,
+) -> ta.Generator[OT, I, lang.Maybe[R]]:
+    """
+    Given a stepped generator and a function taking a list, returns a direct (1:1) generator which accepts input, builds
+    a list of yielded generator output, calls the given function with that list, and yields the result.
+
+    An optional terminate function may be provided which will cause this function to return early if it returns true for
+    an encountered yielded value. The encountered value causing termination will be included in the list sent to the
+    given fn.
+
+    Returns a Maybe of either the given generator's return value or empty if the terminator was encountered.
+    """
+
+    l: list[OF]
+    i: I | None = yield  # type: ignore
+    while True:
+        l = []
+
+        while True:
+            try:
+                o = g.send(i)
+            except StopIteration as e:
+                if l:
+                    yield fn(l)
+                return lang.just(e.value)
+
+            i = None
+
+            if o is None:
+                break
+
+            l.append(o)
+
+            if terminate is not None and terminate(o):
+                yield fn(l)
+                return lang.empty()
+
+        i = yield fn(l)
+
+
+##
+
+
+def _join_bytes(l: ta.Sequence[bytes]) -> bytes:
+    if not l:
+        return b''
+    elif len(l) == 1:
+        return l[0]
+    else:
+        return b''.join(l)
+
+
+def _join_str(l: ta.Sequence[str]) -> str:
+    if not l:
+        return ''
+    elif len(l) == 1:
+        return l[0]
+    else:
+        return ''.join(l)
+
+
+def _is_empty(o: T) -> bool:
+    return len(o) < 1  # type: ignore
+
+
+def joined_bytes_stepped_generator(g: BytesSteppedGenerator[R]) -> BytesDirectGenerator[R]:
+    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+
+
+def joined_str_stepped_generator(g: StrSteppedGenerator[R]) -> StrDirectGenerator[R]:
+    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)
+
+
+##
+
+
+def read_into_bytes_stepped_generator(
+        g: BytesSteppedGenerator,
+        f: ta.IO,
+        *,
+        read_size: int = DEFAULT_BUFFER_SIZE,
+) -> ta.Iterator[bytes]:
+    yield from lang.genmap(  # type: ignore[misc]
+        joined_bytes_stepped_generator(g),
+        lang.readiter(f, read_size),
+    )
+
+
+def read_into_str_stepped_generator(
+        g: StrSteppedGenerator,
+        f: ta.TextIO,
+        *,
+        read_size: int = DEFAULT_BUFFER_SIZE,
+) -> ta.Iterator[str]:
+    yield from lang.genmap(
+        joined_str_stepped_generator(g),
+        lang.readiter(f, read_size),
+    )
+
+
+##
+
+
+@lang.autostart
+def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> BytesSteppedGenerator:
+    o = g.send(None)
+    buf: ta.Any = None
+    eof = False
+
+    while True:
+        if eof:
+            raise EOFError
+
+        if not buf:
+            buf = check.isinstance((yield None), bytes)
+            if not buf:
+                eof = True
+
+        if o is None:
+            i = buf
+            buf = None
+
+        elif isinstance(o, int):
+            while len(buf) < o:
+                more = check.isinstance((yield None), bytes)
+                if not more:
+                    raise EOFError
+                # FIXME: lol - share guts with readers
+                buf += more
+
+            i = buf[:o]
+            buf = buf[o:]
+
+        else:
+            raise TypeError(o)
+
+        while True:
+            o = g.send(i)
+            i = None
+            if isinstance(o, bytes):
+                check.none((yield o))
+                if not o:
+                    return
+            else:
+                break

omlish/io/pyio.py

@@ -4,7 +4,7 @@
 """
 Python implementation of the io module.
 
-https://github.com/python/cpython/blob/8fa4dc4ba8646c59f945f2451c53e2919f066065/Lib/_pyio.py
+https://github.com/python/cpython/blob/8b3cccf3f9508572d85b0044519f2bd5715dacad/Lib/_pyio.py
 """
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
 # --------------------------------------------
@@ -2580,8 +2580,11 @@ class TextIOWrapper(TextIOBase):
         decoder = self._decoder or self._get_decoder()
 
         if size < 0:
+            chunk = self.buffer.read()
+            if chunk is None:
+                raise BlockingIOError("Read returned None.")
             # Read everything.
-            result = self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)
+            result = self._get_decoded_chars() + decoder.decode(chunk, final=True)
             if self._snapshot is not None:
                 self._set_decoded_chars('')
                 self._snapshot = None

omlish/iterators/__init__.py

@@ -0,0 +1,24 @@
+from .iterators import (  # noqa
+    PeekIterator,
+    PrefetchIterator,
+    ProxyIterator,
+    RetainIterator,
+)
+
+from .recipes import (  # noqa
+    sliding_window,
+)
+
+from .tools import (  # noqa
+    chunk,
+    expand_indexed_pairs,
+    merge_on,
+    take,
+    unzip,
+)
+
+from .unique import (  # noqa
+    UniqueItem,
+    UniqueIterator,
+    UniqueStats,
+)

omlish/iterators/iterators.py

@@ -0,0 +1,132 @@
+import collections
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+_MISSING = object()
+
+
+class PeekIterator(ta.Iterator[T]):
+
+    def __init__(self, it: ta.Iterable[T]) -> None:
+        super().__init__()
+
+        self._it = iter(it)
+        self._pos = -1
+        self._next_item: ta.Any = _MISSING
+
+    _item: T
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    @property
+    def done(self) -> bool:
+        try:
+            self.peek()
+        except StopIteration:
+            return True
+        else:
+            return False
+
+    def __next__(self) -> T:
+        if self._next_item is not _MISSING:
+            self._item = ta.cast(T, self._next_item)
+            self._next_item = _MISSING
+        else:
+            self._item = next(self._it)
+        self._pos += 1
+        return self._item
+
+    def peek(self) -> T:
+        if self._next_item is not _MISSING:
+            return ta.cast(T, self._next_item)
+        self._next_item = next(self._it)
+        return self._next_item
+
+    def next_peek(self) -> T:
+        next(self)
+        return self.peek()
+
+    def takewhile(self, fn: ta.Callable[[T], bool]) -> ta.Iterator[T]:
+        while fn(self.peek()):
+            yield next(self)
+
+    def skipwhile(self, fn: ta.Callable[[T], bool]) -> None:
+        while fn(self.peek()):
+            next(self)
+
+    def takeuntil(self, fn: ta.Callable[[T], bool]) -> ta.Iterator[T]:
+        return self.takewhile(lambda e: not fn(e))
+
+    def skipuntil(self, fn: ta.Callable[[T], bool]) -> None:
+        self.skipwhile(lambda e: not fn(e))
+
+    def takethrough(self, pos: int) -> ta.Iterator[T]:
+        return self.takewhile(lambda _: self._pos < pos)
+
+    def skipthrough(self, pos: int) -> None:
+        self.skipwhile(lambda _: self._pos < pos)
+
+    def taketo(self, pos: int) -> ta.Iterator[T]:
+        return self.takethrough(pos - 1)
+
+    def skipto(self, pos: int) -> None:
+        self.skipthrough(pos - 1)
+
+
+class ProxyIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T]) -> None:
+        self._fn = fn
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def __next__(self) -> T:
+        return self._fn()
+
+
+class PrefetchIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T] | None = None) -> None:
+        super().__init__()
+
+        self._fn = fn
+        self._deque: collections.deque[T] = collections.deque()
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def push(self, item) -> None:
+        self._deque.append(item)
+
+    def __next__(self) -> T:
+        try:
+            return self._deque.popleft()
+        except IndexError:
+            if self._fn is None:
+                raise StopIteration from None
+        return self._fn()
+
+
+class RetainIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T]) -> None:
+        super().__init__()
+
+        self._fn = fn
+        self._deque: collections.deque[T] = collections.deque()
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def pop(self) -> None:
+        self._deque.popleft()
+
+    def __next__(self) -> T:
+        item = self._fn()
+        self._deque.append(item)
+        return item

omlish/iterators/recipes.py

@@ -0,0 +1,18 @@
+"""
+https://docs.python.org/3/library/itertools.html#itertools-recipes
+"""
+import collections
+import itertools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def sliding_window(it: ta.Iterable[T], n: int) -> ta.Iterator[tuple[T, ...]]:
+    # sliding_window('ABCDEFG', 4) -> ABCD BCDE CDEF DEFG
+    iterator = iter(it)
+    window = collections.deque(itertools.islice(iterator, n - 1), maxlen=n)
+    for x in iterator:
+        window.append(x)
+        yield tuple(window)

omlish/iterators/tools.py

@@ -0,0 +1,96 @@
+import functools
+import heapq
+import itertools
+import typing as ta
+
+from .iterators import PeekIterator
+from .iterators import PrefetchIterator
+
+
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
+
+
+def unzip(it: ta.Iterable[T], width: int | None = None) -> list:
+    if width is None:
+        if not isinstance(it, PeekIterator):
+            it = PeekIterator(iter(it))
+        try:
+            width = len(it.peek())
+        except StopIteration:
+            return []
+
+    its: list[PrefetchIterator[T]] = []
+    running = True
+
+    def next_fn(idx):
+        nonlocal running
+        if not running:
+            raise StopIteration
+        try:
+            items = next(it)  # type: ignore
+        except StopIteration:
+            running = False
+            raise
+        for item_idx, item in enumerate(items):
+            its[item_idx].push(item)
+        return next(its[idx])
+
+    its.extend(PrefetchIterator(functools.partial(next_fn, idx)) for idx in range(width))
+    return its
+
+
+def take(n: int, iterable: ta.Iterable[T]) -> list[T]:
+    return list(itertools.islice(iterable, n))
+
+
+def chunk(n: int, iterable: ta.Iterable[T], strict: bool = False) -> ta.Iterator[list[T]]:
+    iterator = iter(functools.partial(take, n, iter(iterable)), [])
+    if strict:
+        def ret():
+            for chunk in iterator:
+                if len(chunk) != n:
+                    raise ValueError('iterable is not divisible by n.')
+                yield chunk
+        return iter(ret())
+    else:
+        return iterator
+
+
+def merge_on(
+        function: ta.Callable[[T], U],
+        *its: ta.Iterable[T],
+) -> ta.Iterator[tuple[U, list[tuple[int, T]]]]:
+    indexed_its = [
+        (
+            (function(item), it_idx, item)
+            for it_idx, item in zip(itertools.repeat(it_idx), it)
+        )
+        for it_idx, it in enumerate(its)
+    ]
+
+    grouped_indexed_its = itertools.groupby(
+        heapq.merge(*indexed_its),
+        key=lambda item_tuple: item_tuple[0],
+    )
+
+    return (
+        (fn_item, [(it_idx, item) for _, it_idx, item in grp])
+        for fn_item, grp in grouped_indexed_its
+    )
+
+
+def expand_indexed_pairs(
+        seq: ta.Iterable[tuple[int, T]],
+        default: T,
+        *,
+        width: int | None = None,
+) -> list[T]:
+    width_ = width
+    if width_ is None:
+        width_ = (max(idx for idx, _ in seq) + 1) if seq else 0
+    result = [default] * width_
+    for idx, value in seq:
+        if idx < width_:
+            result[idx] = value
+    return result

omlish/iterators/unique.py

@@ -0,0 +1,67 @@
+import dataclasses as dc
+import typing as ta
+
+from .. import lang
+
+
+T = ta.TypeVar('T')
+
+
+@dc.dataclass()
+class UniqueStats:
+    key: ta.Any
+    num_seen: int
+    first_idx: int
+    last_idx: int
+
+
+@dc.dataclass(frozen=True)
+class UniqueItem(ta.Generic[T]):
+    idx: int
+    item: T
+    stats: UniqueStats
+    out: lang.Maybe[T]
+
+
+class UniqueIterator(ta.Iterator[UniqueItem[T]]):
+    def __init__(
+            self,
+            it: ta.Iterable[T],
+            keyer: ta.Callable[[T], ta.Any] = lang.identity,
+    ) -> None:
+        super().__init__()
+        self._it = enumerate(it)
+        self._keyer = keyer
+
+        self.stats: dict[ta.Any, UniqueStats] = {}
+
+    def __next__(self) -> UniqueItem[T]:
+        idx, item = next(self._it)
+        key = self._keyer(item)
+
+        try:
+            stats = self.stats[key]
+
+        except KeyError:
+            stats = self.stats[key] = UniqueStats(
+                key,
+                num_seen=1,
+                first_idx=idx,
+                last_idx=idx,
+            )
+            return UniqueItem(
+                idx,
+                item,
+                stats,
+                lang.just(item),
+            )
+
+        else:
+            stats.num_seen += 1
+            stats.last_idx = idx
+            return UniqueItem(
+                idx,
+                item,
+                stats,
+                lang.empty(),
+            )

omlish/lang/__init__.py

@@ -112,6 +112,7 @@ from .functions import (  # noqa
     issubclass_of,
     maybe_call,
     opt_coalesce,
+    opt_fn,
     periodically,
     raise_,
     raising,
@@ -120,6 +121,17 @@ from .functions import (  # noqa
     void,
 )
 
+from .generators import (  # noqa
+    CoroutineGenerator,
+    Generator,
+    GeneratorLike,
+    GeneratorMappedIterator,
+    autostart,
+    corogen,
+    genmap,
+    nextgen,
+)
+
 from .imports import (  # noqa
     can_import,
     import_all,
@@ -136,7 +148,6 @@ from .imports import (  # noqa
 
 from .iterables import (  # noqa
     BUILTIN_SCALAR_ITERABLE_TYPES,
-    Generator,
     asrange,
     exhaust,
     flatmap,
@@ -146,6 +157,7 @@ from .iterables import (  # noqa
     itergen,
     peek,
     prodrange,
+    readiter,
     renumerate,
     take,
 )

omlish/lang/functions.py

@@ -4,6 +4,7 @@ import time
 import typing as ta
 
 
+F = ta.TypeVar('F')
 T = ta.TypeVar('T')
 P = ta.ParamSpec('P')
 CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -81,8 +82,17 @@ def identity(obj: T) -> T:
     return obj
 
 
-class constant(ta.Generic[T]):  # noqa
+def opt_fn(fn: ta.Callable[[F], T]) -> ta.Callable[[F | None], T | None]:
+    @functools.wraps(fn)
+    def inner(v: F | None) -> T | None:
+        if v is not None:
+            return fn(v)
+        else:
+            return None
+    return inner
 
+
+class constant(ta.Generic[T]):  # noqa
     def __init__(self, obj: T) -> None:
         super().__init__()
 
@@ -116,7 +126,6 @@ class VoidError(Exception):
 
 
 class Void:
-
     def __new__(cls, *args: ta.Any, **kwargs: ta.Any) -> None:  # type: ignore  # noqa
         raise VoidError