omlish 0.0.0.dev165__tar.gz → 0.0.0.dev167__tar.gz

{omlish-0.0.0.dev165/omlish.egg-info → omlish-0.0.0.dev167}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev165
+Version: 0.0.0.dev167
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev165'
-__revision__ = '5e68eb26c319c7641fcdbec0513840b576224ff3'
+__version__ = '0.0.0.dev167'
+__revision__ = '4dc487c3620d4629b8a2895a84511a4be478a801'
 
 
 #

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/codecs/text.py

@@ -46,8 +46,8 @@ TextEncodingErrors: ta.TypeAlias = ta.Literal[
     'namereplace',
 
     ##
-    # In addition, the following error handler is specific to the given codecs:
-    # utf-8, utf-16, utf-32, utf-16-be, utf-16-le, utf-32-be, utf-32-le
+    # In addition, the following error handler is specific to the given codecs: utf-8, utf-16, utf-32, utf-16-be,
+    # utf-16-le, utf-32-be, utf-32-le
 
     # Allow encoding and decoding surrogate code point (U+D800 - U+DFFF) as normal code point. Otherwise these codecs
     # treat the presence of surrogate code point in str as an error.

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/io/compress/codecs.py

@@ -2,7 +2,9 @@ import dataclasses as dc
 import typing as ta
 
 from ... import codecs
+from ..generators import buffer_bytes_stepped_reader_generator
 from .base import Compression
+from .base import IncrementalCompression
 
 
 ##
@@ -22,6 +24,20 @@ class CompressionEagerCodec(codecs.EagerCodec[bytes, bytes]):
 ##
 
 
+@dc.dataclass(frozen=True)
+class CompressionIncrementalCodec(codecs.IncrementalCodec[bytes, bytes]):
+    compression: IncrementalCompression
+
+    def encode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return self.compression.compress_incremental()
+
+    def decode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return buffer_bytes_stepped_reader_generator(self.compression.decompress_incremental())
+
+
+##
+
+
 class CompressionCodec(codecs.Codec):
     pass
 
@@ -40,6 +56,10 @@ def make_compression_codec(
         output=bytes,
 
         new=lambda *args, **kwargs: CompressionEagerCodec(cls(*args, **kwargs)),
+
+        new_incremental=(
+            lambda *args, **kwargs: CompressionIncrementalCodec(cls(*args, **kwargs))  # noqa
+        ) if issubclass(cls, IncrementalCompression) else None,
     )
 
 

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/io/generators/__init__.py

@@ -50,4 +50,7 @@ from .stepped import (  # noqa
     joined_str_stepped_generator,
 
     read_into_bytes_stepped_generator,
+    read_into_str_stepped_generator,
+
+    buffer_bytes_stepped_reader_generator,
 )

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/io/generators/stepped.py

@@ -151,18 +151,32 @@ def read_into_str_stepped_generator(
 def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> BytesSteppedGenerator:
     o = g.send(None)
     buf: ta.Any = None
+    eof = False
 
     while True:
+        if eof:
+            raise EOFError
+
         if not buf:
             buf = check.isinstance((yield None), bytes)
+            if not buf:
+                eof = True
 
-        if o is None or not buf:
+        if o is None:
             i = buf
+            buf = None
+
         elif isinstance(o, int):
-            if len(buf) < o:
-                raise NotImplementedError
+            while len(buf) < o:
+                more = check.isinstance((yield None), bytes)
+                if not more:
+                    raise EOFError
+                # FIXME: lol - share guts with readers
+                buf += more
+
             i = buf[:o]
             buf = buf[o:]
+
         else:
             raise TypeError(o)
 
@@ -171,5 +185,7 @@ def buffer_bytes_stepped_reader_generator(g: BytesSteppedReaderGenerator) -> Byt
             i = None
             if isinstance(o, bytes):
                 check.none((yield o))
+                if not o:
+                    return
             else:
                 break

omlish-0.0.0.dev167/omlish/iterators/__init__.py

@@ -0,0 +1,24 @@
+from .iterators import (  # noqa
+    PeekIterator,
+    PrefetchIterator,
+    ProxyIterator,
+    RetainIterator,
+)
+
+from .recipes import (  # noqa
+    sliding_window,
+)
+
+from .tools import (  # noqa
+    chunk,
+    expand_indexed_pairs,
+    merge_on,
+    take,
+    unzip,
+)
+
+from .unique import (  # noqa
+    UniqueItem,
+    UniqueIterator,
+    UniqueStats,
+)

omlish-0.0.0.dev167/omlish/iterators/iterators.py

@@ -0,0 +1,132 @@
+import collections
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+_MISSING = object()
+
+
+class PeekIterator(ta.Iterator[T]):
+
+    def __init__(self, it: ta.Iterable[T]) -> None:
+        super().__init__()
+
+        self._it = iter(it)
+        self._pos = -1
+        self._next_item: ta.Any = _MISSING
+
+    _item: T
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    @property
+    def done(self) -> bool:
+        try:
+            self.peek()
+        except StopIteration:
+            return True
+        else:
+            return False
+
+    def __next__(self) -> T:
+        if self._next_item is not _MISSING:
+            self._item = ta.cast(T, self._next_item)
+            self._next_item = _MISSING
+        else:
+            self._item = next(self._it)
+        self._pos += 1
+        return self._item
+
+    def peek(self) -> T:
+        if self._next_item is not _MISSING:
+            return ta.cast(T, self._next_item)
+        self._next_item = next(self._it)
+        return self._next_item
+
+    def next_peek(self) -> T:
+        next(self)
+        return self.peek()
+
+    def takewhile(self, fn: ta.Callable[[T], bool]) -> ta.Iterator[T]:
+        while fn(self.peek()):
+            yield next(self)
+
+    def skipwhile(self, fn: ta.Callable[[T], bool]) -> None:
+        while fn(self.peek()):
+            next(self)
+
+    def takeuntil(self, fn: ta.Callable[[T], bool]) -> ta.Iterator[T]:
+        return self.takewhile(lambda e: not fn(e))
+
+    def skipuntil(self, fn: ta.Callable[[T], bool]) -> None:
+        self.skipwhile(lambda e: not fn(e))
+
+    def takethrough(self, pos: int) -> ta.Iterator[T]:
+        return self.takewhile(lambda _: self._pos < pos)
+
+    def skipthrough(self, pos: int) -> None:
+        self.skipwhile(lambda _: self._pos < pos)
+
+    def taketo(self, pos: int) -> ta.Iterator[T]:
+        return self.takethrough(pos - 1)
+
+    def skipto(self, pos: int) -> None:
+        self.skipthrough(pos - 1)
+
+
+class ProxyIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T]) -> None:
+        self._fn = fn
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def __next__(self) -> T:
+        return self._fn()
+
+
+class PrefetchIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T] | None = None) -> None:
+        super().__init__()
+
+        self._fn = fn
+        self._deque: collections.deque[T] = collections.deque()
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def push(self, item) -> None:
+        self._deque.append(item)
+
+    def __next__(self) -> T:
+        try:
+            return self._deque.popleft()
+        except IndexError:
+            if self._fn is None:
+                raise StopIteration from None
+        return self._fn()
+
+
+class RetainIterator(ta.Iterator[T]):
+
+    def __init__(self, fn: ta.Callable[[], T]) -> None:
+        super().__init__()
+
+        self._fn = fn
+        self._deque: collections.deque[T] = collections.deque()
+
+    def __iter__(self) -> ta.Self:
+        return self
+
+    def pop(self) -> None:
+        self._deque.popleft()
+
+    def __next__(self) -> T:
+        item = self._fn()
+        self._deque.append(item)
+        return item

omlish-0.0.0.dev167/omlish/iterators/recipes.py

@@ -0,0 +1,18 @@
+"""
+https://docs.python.org/3/library/itertools.html#itertools-recipes
+"""
+import collections
+import itertools
+import typing as ta
+
+
+T = ta.TypeVar('T')
+
+
+def sliding_window(it: ta.Iterable[T], n: int) -> ta.Iterator[tuple[T, ...]]:
+    # sliding_window('ABCDEFG', 4) -> ABCD BCDE CDEF DEFG
+    iterator = iter(it)
+    window = collections.deque(itertools.islice(iterator, n - 1), maxlen=n)
+    for x in iterator:
+        window.append(x)
+        yield tuple(window)

omlish-0.0.0.dev167/omlish/iterators/tools.py

@@ -0,0 +1,96 @@
+import functools
+import heapq
+import itertools
+import typing as ta
+
+from .iterators import PeekIterator
+from .iterators import PrefetchIterator
+
+
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
+
+
+def unzip(it: ta.Iterable[T], width: int | None = None) -> list:
+    if width is None:
+        if not isinstance(it, PeekIterator):
+            it = PeekIterator(iter(it))
+        try:
+            width = len(it.peek())
+        except StopIteration:
+            return []
+
+    its: list[PrefetchIterator[T]] = []
+    running = True
+
+    def next_fn(idx):
+        nonlocal running
+        if not running:
+            raise StopIteration
+        try:
+            items = next(it)  # type: ignore
+        except StopIteration:
+            running = False
+            raise
+        for item_idx, item in enumerate(items):
+            its[item_idx].push(item)
+        return next(its[idx])
+
+    its.extend(PrefetchIterator(functools.partial(next_fn, idx)) for idx in range(width))
+    return its
+
+
+def take(n: int, iterable: ta.Iterable[T]) -> list[T]:
+    return list(itertools.islice(iterable, n))
+
+
+def chunk(n: int, iterable: ta.Iterable[T], strict: bool = False) -> ta.Iterator[list[T]]:
+    iterator = iter(functools.partial(take, n, iter(iterable)), [])
+    if strict:
+        def ret():
+            for chunk in iterator:
+                if len(chunk) != n:
+                    raise ValueError('iterable is not divisible by n.')
+                yield chunk
+        return iter(ret())
+    else:
+        return iterator
+
+
+def merge_on(
+        function: ta.Callable[[T], U],
+        *its: ta.Iterable[T],
+) -> ta.Iterator[tuple[U, list[tuple[int, T]]]]:
+    indexed_its = [
+        (
+            (function(item), it_idx, item)
+            for it_idx, item in zip(itertools.repeat(it_idx), it)
+        )
+        for it_idx, it in enumerate(its)
+    ]
+
+    grouped_indexed_its = itertools.groupby(
+        heapq.merge(*indexed_its),
+        key=lambda item_tuple: item_tuple[0],
+    )
+
+    return (
+        (fn_item, [(it_idx, item) for _, it_idx, item in grp])
+        for fn_item, grp in grouped_indexed_its
+    )
+
+
+def expand_indexed_pairs(
+        seq: ta.Iterable[tuple[int, T]],
+        default: T,
+        *,
+        width: int | None = None,
+) -> list[T]:
+    width_ = width
+    if width_ is None:
+        width_ = (max(idx for idx, _ in seq) + 1) if seq else 0
+    result = [default] * width_
+    for idx, value in seq:
+        if idx < width_:
+            result[idx] = value
+    return result

omlish-0.0.0.dev167/omlish/iterators/unique.py

@@ -0,0 +1,67 @@
+import dataclasses as dc
+import typing as ta
+
+from .. import lang
+
+
+T = ta.TypeVar('T')
+
+
+@dc.dataclass()
+class UniqueStats:
+    key: ta.Any
+    num_seen: int
+    first_idx: int
+    last_idx: int
+
+
+@dc.dataclass(frozen=True)
+class UniqueItem(ta.Generic[T]):
+    idx: int
+    item: T
+    stats: UniqueStats
+    out: lang.Maybe[T]
+
+
+class UniqueIterator(ta.Iterator[UniqueItem[T]]):
+    def __init__(
+            self,
+            it: ta.Iterable[T],
+            keyer: ta.Callable[[T], ta.Any] = lang.identity,
+    ) -> None:
+        super().__init__()
+        self._it = enumerate(it)
+        self._keyer = keyer
+
+        self.stats: dict[ta.Any, UniqueStats] = {}
+
+    def __next__(self) -> UniqueItem[T]:
+        idx, item = next(self._it)
+        key = self._keyer(item)
+
+        try:
+            stats = self.stats[key]
+
+        except KeyError:
+            stats = self.stats[key] = UniqueStats(
+                key,
+                num_seen=1,
+                first_idx=idx,
+                last_idx=idx,
+            )
+            return UniqueItem(
+                idx,
+                item,
+                stats,
+                lang.just(item),
+            )
+
+        else:
+            stats.num_seen += 1
+            stats.last_idx = idx
+            return UniqueItem(
+                idx,
+                item,
+                stats,
+                lang.empty(),
+            )

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/lite/strings.py

@@ -21,6 +21,15 @@ def snake_case(name: str) -> str:
 ##
 
 
+def strip_with_newline(s: str) -> str:
+    if not s:
+        return ''
+    return s.strip() + '\n'
+
+
+##
+
+
 def is_dunder(name: str) -> bool:
     return (
         name[:2] == name[-2:] == '__' and

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish/os/atomics.py

@@ -122,7 +122,7 @@ class AtomicPathSwapping(abc.ABC):
 ##
 
 
-class OsRenameAtomicPathSwap(AtomicPathSwap):
+class OsReplaceAtomicPathSwap(AtomicPathSwap):
     def __init__(
             self,
             kind: AtomicPathSwapKind,
@@ -150,7 +150,7 @@ class OsRenameAtomicPathSwap(AtomicPathSwap):
         return self._tmp_path
 
     def _commit(self) -> None:
-        os.rename(self._tmp_path, self._dst_path)
+        os.replace(self._tmp_path, self._dst_path)
 
     def _abort(self) -> None:
         shutil.rmtree(self._tmp_path, ignore_errors=True)
@@ -197,7 +197,7 @@ class TempDirAtomicPathSwapping(AtomicPathSwapping):
         else:
             raise TypeError(kind)
 
-        return OsRenameAtomicPathSwap(
+        return OsReplaceAtomicPathSwap(
             kind,
             dst_path,
             tmp_path,

omlish-0.0.0.dev167/omlish/os/paths.py

@@ -0,0 +1,32 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import os.path
+import typing as ta
+
+
+def abs_real_path(p: str) -> str:
+    return os.path.abspath(os.path.realpath(p))
+
+
+def is_path_in_dir(base_dir: str, target_path: str) -> bool:
+    base_dir = abs_real_path(base_dir)
+    target_path = abs_real_path(target_path)
+
+    return target_path.startswith(base_dir + os.path.sep)
+
+
+def relative_symlink(
+        src: str,
+        dst: str,
+        *,
+        target_is_directory: bool = False,
+        dir_fd: ta.Optional[int] = None,
+        **kwargs: ta.Any,
+) -> None:
+    os.symlink(
+        os.path.relpath(src, os.path.dirname(dst)),
+        dst,
+        target_is_directory=target_is_directory,
+        dir_fd=dir_fd,
+        **kwargs,
+    )

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev165
+Version: 0.0.0.dev167
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/omlish.egg-info/SOURCES.txt

@@ -11,7 +11,6 @@ omlish/check.py
 omlish/datetimes.py
 omlish/defs.py
 omlish/dynamic.py
-omlish/iterators.py
 omlish/libc.py
 omlish/multiprocessing.py
 omlish/runmodule.py
@@ -338,6 +337,11 @@ omlish/io/generators/consts.py
 omlish/io/generators/direct.py
 omlish/io/generators/readers.py
 omlish/io/generators/stepped.py
+omlish/iterators/__init__.py
+omlish/iterators/iterators.py
+omlish/iterators/recipes.py
+omlish/iterators/tools.py
+omlish/iterators/unique.py
 omlish/lang/__init__.py
 omlish/lang/cached.py
 omlish/lang/clsdct.py
@@ -444,6 +448,7 @@ omlish/os/deathsig.py
 omlish/os/files.py
 omlish/os/journald.py
 omlish/os/linux.py
+omlish/os/paths.py
 omlish/os/pidfile.py
 omlish/os/sizes.py
 omlish/reflect/__init__.py

{omlish-0.0.0.dev165 → omlish-0.0.0.dev167}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '>=3.12'
-version = '0.0.0.dev165'
+version = '0.0.0.dev167'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',