omlish 0.0.0.dev136__py3-none-any.whl → 0.0.0.dev138__py3-none-any.whl

omlish/io/compress/gzip.py

@@ -0,0 +1,306 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+import functools
+import os.path
+import struct
+import time
+import typing as ta
+
+from ... import cached
+from ... import check
+from ... import lang
+from ..generators.readers import PrependableBytesGeneratorReader
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import gzip
+    import zlib
+else:
+    gzip = lang.proxy_import('gzip')
+    zlib = lang.proxy_import('zlib')
+
+
+##
+
+
+COMPRESS_LEVEL_FAST = 1
+COMPRESS_LEVEL_TRADEOFF = 6
+COMPRESS_LEVEL_BEST = 9
+
+
+@cached.function
+def _zero_crc() -> int:
+    return zlib.crc32(b'')
+
+
+##
+
+
+class IncrementalGzipCompressor:
+    def __init__(
+            self,
+            *,
+            compresslevel: int = COMPRESS_LEVEL_BEST,
+            name: str | bytes | None = None,
+            mtime: float | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._name = name or ''
+        self._compresslevel = compresslevel
+        self._mtime = mtime
+
+    def _write_gzip_header(self) -> ta.Generator[bytes, None, None]:
+        check.none((yield b'\037\213'))  # magic header
+        check.none((yield b'\010'))  # compression method
+
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not include filenames that cannot be represented that
+            # way.
+            fname = os.path.basename(self._name)
+            if not isinstance(fname, bytes):
+                fname = fname.encode('latin-1')
+            if fname.endswith(b'.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = b''
+
+        flags = 0
+        if fname:
+            flags = gzip.FNAME
+        check.none((yield chr(flags).encode('latin-1')))
+
+        mtime = self._mtime
+        if mtime is None:
+            mtime = time.time()
+        check.none((yield struct.pack('<L', int(mtime))))
+
+        if self._compresslevel == COMPRESS_LEVEL_BEST:
+            xfl = b'\002'
+        elif self._compresslevel == COMPRESS_LEVEL_FAST:
+            xfl = b'\004'
+        else:
+            xfl = b'\000'
+        check.none((yield xfl))
+
+        check.none((yield b'\377'))
+
+        if fname:
+            check.none((yield fname + b'\000'))
+
+    @lang.autostart
+    def __call__(self) -> IncrementalCompressor:
+        crc = _zero_crc()
+        size = 0
+        offset = 0  # Current file offset for seek(), tell(), etc
+        wrote_header = False
+
+        compress = zlib.compressobj(
+            self._compresslevel,
+            zlib.DEFLATED,
+            -zlib.MAX_WBITS,
+            zlib.DEF_MEM_LEVEL,
+            0,
+        )
+
+        while True:
+            data: ta.Any = check.isinstance((yield None), bytes)
+
+            if not wrote_header:
+                yield from self._write_gzip_header()
+                wrote_header = True
+
+            if not data:
+                break
+
+            # Called by our self._buffer underlying BufferedWriterDelegate.
+            if isinstance(data, (bytes, bytearray)):
+                length = len(data)
+            else:
+                # accept any data that supports the buffer protocol
+                data = memoryview(data)
+                length = data.nbytes
+
+            if length > 0:
+                if (fl := compress.compress(data)):
+                    check.none((yield fl))
+                size += length
+                crc = zlib.crc32(data, crc)
+                offset += length
+
+        if (fl := compress.flush()):
+            check.none((yield fl))
+
+        yield struct.pack('<L', crc)
+        # size may exceed 2 GiB, or even 4 GiB
+        yield struct.pack('<L', size & 0xffffffff)
+
+        yield b''
+
+
+##
+
+
+class IncrementalGzipDecompressor:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._factory = functools.partial(
+            zlib.decompressobj,
+            wbits=-zlib.MAX_WBITS,
+        )
+
+    def _read_gzip_header(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+    ) -> ta.Generator[int | None, bytes, int | None]:
+        magic = yield from rdr.read(2)
+        if magic == b'':
+            return None
+
+        if magic != b'\037\213':
+            raise gzip.BadGzipFile(f'Not a gzipped file ({magic!r})')
+
+        buf = yield from rdr.read(8)
+        method, flag, last_mtime = struct.unpack('<BBIxx', buf)
+        if method != 8:
+            raise gzip.BadGzipFile('Unknown compression method')
+
+        if flag & gzip.FEXTRA:
+            # Read & discard the extra field, if present
+            buf = yield from rdr.read(2)
+            extra_len, = struct.unpack('<H', buf)
+            if extra_len:
+                yield from rdr.read(extra_len)
+
+        if flag & gzip.FNAME:
+            # Read and discard a null-terminated string containing the filename
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FCOMMENT:
+            # Read and discard a null-terminated string containing a comment
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FHCRC:
+            yield from rdr.read(2)  # Read & discard the 16-bit header CRC
+
+        return last_mtime
+
+    def _read_eof(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+            crc: int,
+            stream_size: int,
+    ) -> ta.Generator[int | None, bytes, None]:
+        # We've read to the end of the file.
+        # We check that the computed CRC and size of the uncompressed data matches the stored values. Note that the size
+        # stored is the true file size mod 2**32.
+        buf = yield from rdr.read(8)
+        crc32, isize = struct.unpack('<II', buf)
+        if crc32 != crc:
+            raise gzip.BadGzipFile(f'CRC check failed {hex(crc32)} != {hex(crc)}')
+        elif isize != (stream_size & 0xffffffff):
+            raise gzip.BadGzipFile('Incorrect length of data produced')
+
+        # Gzip files can be padded with zeroes and still have archives. Consume all zero bytes and set the file position
+        # to the first non-zero byte. See http://www.gzip.org/#faq8
+        c = b'\0'
+        while c == b'\0':
+            c = yield from rdr.read(1)
+        if c:
+            rdr.prepend(c)
+
+    def __call__(self) -> IncrementalDecompressor:
+        rdr = PrependableBytesGeneratorReader()
+
+        pos = 0  # Current offset in decompressed stream
+
+        crc = _zero_crc()
+        stream_size = 0  # Decompressed size of unconcatenated stream
+        new_member = True
+
+        decompressor = self._factory()
+
+        while True:
+            # For certain input data, a single call to decompress() may not return any data. In this case, retry until
+            # we get some data or reach EOF.
+            while True:
+                if decompressor.eof:
+                    # Ending case: we've come to the end of a member in the file, so finish up this member, and read a
+                    # new gzip header. Check the CRC and file size, and set the flag so we read a new member
+                    yield from self._read_eof(rdr, crc, stream_size)
+                    new_member = True
+                    decompressor = self._factory()
+
+                if new_member:
+                    # If the _new_member flag is set, we have to jump to the next member, if there is one.
+                    crc = _zero_crc()
+                    stream_size = 0  # Decompressed size of unconcatenated stream
+                    last_mtime = yield from self._read_gzip_header(rdr)
+                    if not last_mtime:
+                        check.none((yield b''))
+                        return
+                    new_member = False
+
+                # Read a chunk of data from the file
+                if not decompressor.unconsumed_tail:
+                    buf = yield from rdr.read(None)
+                    uncompress = decompressor.decompress(buf)
+                else:
+                    uncompress = decompressor.decompress(b'')
+
+                if decompressor.unused_data != b'':
+                    # Prepend the already read bytes to the fileobj so they can be seen by _read_eof() and
+                    # _read_gzip_header()
+                    rdr.prepend(decompressor.unused_data)
+
+                if uncompress != b'':
+                    break
+                if buf == b'':  # noqa
+                    raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+            crc = zlib.crc32(uncompress, crc)
+            stream_size += len(uncompress)
+            pos += len(uncompress)
+            check.none((yield uncompress))

omlish/io/compress/lzma.py

@@ -0,0 +1,32 @@
+import typing as ta
+
+from ... import lang
+from .adapters import CompressorIncrementalAdapter
+from .adapters import DecompressorIncrementalAdapter
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import lzma
+else:
+    lzma = lang.proxy_import('lzma')
+
+
+class IncrementalLzmaCompressor:
+    def __init__(self) -> None:
+        super().__init__()
+
+    @lang.autostart
+    def __call__(self) -> IncrementalCompressor:
+        return CompressorIncrementalAdapter(
+            lzma.LZMACompressor,  # type: ignore
+        )()
+
+
+class IncrementalLzmaDecompressor:
+    def __call__(self) -> IncrementalDecompressor:
+        return DecompressorIncrementalAdapter(
+            lzma.LZMADecompressor,  # type: ignore
+            trailing_error=lzma.LZMAError,
+        )()

omlish/io/compress/types.py

@@ -0,0 +1,29 @@
+# ruff: noqa: UP007
+import typing as ta
+
+
+IncrementalCompressor: ta.TypeAlias = ta.Generator[
+    ta.Union[
+        bytes,  # Compressed output
+        None,  # Need input
+    ],
+    ta.Union[
+        bytes,  # Input bytes
+        None,  # Need output
+    ],
+    None,
+]
+
+
+IncrementalDecompressor: ta.TypeAlias = ta.Generator[
+    ta.Union[
+        bytes,  # Uncompressed output
+        int,  # Need exactly n bytes
+        None,  # Need any amount of bytes
+    ],
+    ta.Union[
+        bytes,  # Input bytes
+        None,  # Need output
+    ],
+    None,
+]

omlish/io/generators/readers.py

@@ -0,0 +1,183 @@
+"""
+TODO:
+ - BufferedBytesGeneratorReader
+ - docstrings
+ - memoryviews
+"""
+import abc
+import typing as ta
+
+from ... import check
+
+
+T = ta.TypeVar('T')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+AnyT = ta.TypeVar('AnyT', bound=ta.Any)
+
+
+ReaderGenerator: ta.TypeAlias = ta.Generator[int | None, I, R]
+ExactReaderGenerator: ta.TypeAlias = ta.Generator[int, I, R]
+
+BytesReaderGenerator: ta.TypeAlias = ReaderGenerator[bytes, R]
+BytesExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[bytes, R]
+
+StrReaderGenerator: ta.TypeAlias = ReaderGenerator[str, R]
+StrExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[str, R]
+
+
+##
+
+
+class _BytesJoiner:
+    def _join(self, lst: list[bytes]) -> bytes:
+        return b''.join(lst)
+
+
+class _StrJoiner:
+    def _join(self, lst: list[str]) -> str:
+        return ''.join(lst)
+
+
+##
+
+
+class GeneratorReader(abc.ABC, ta.Generic[T]):
+    @abc.abstractmethod
+    def read(self, sz: int | None) -> ta.Generator[int | None, T, T]:
+        raise NotImplementedError
+
+    def read_exact(self, sz: int) -> ta.Generator[int | None, T, T]:
+        d: ta.Any = yield from self.read(sz)
+        if len(d) != sz:
+            raise EOFError(f'GeneratorReader got {len(d)}, expected {sz}')
+        return d
+
+
+##
+
+
+class PrependableGeneratorReader(GeneratorReader[AnyT]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._queue: list[tuple[AnyT, int]] = []
+
+    @abc.abstractmethod
+    def _join(self, lst: list[AnyT]) -> AnyT:
+        raise NotImplementedError
+
+    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+        if not self._queue:
+            d: AnyT = check.not_none((yield sz))
+            return d
+
+        if sz is None:
+            return self._queue.pop(0)[0]
+
+        lst: list[AnyT] = []
+        rem = sz
+        while rem > 0 and self._queue:
+            c, p = self._queue[0]
+
+            if len(c) - p > rem:
+                lst.append(c[p:p + rem])
+                self._queue[0] = (c, p + rem)
+                return self._join(lst)
+
+            lst.append(c[p:])
+            rem -= len(c) - p
+            self._queue.pop(0)
+
+        if rem:
+            d = check.not_none((yield rem))
+            if d:
+                lst.append(d)  # type: ignore[unreachable]
+
+        if len(lst) == 1:
+            return lst[0]
+        else:
+            return self._join(lst)
+
+    def prepend(self, d: AnyT, p: int | None = None) -> None:
+        if d:
+            self._queue.insert(0, (d, p or 0))
+
+
+class PrependableBytesGeneratorReader(
+    _BytesJoiner,
+    PrependableGeneratorReader[bytes],
+):
+    pass
+
+
+class PrependableStrGeneratorReader(
+    _StrJoiner,
+    PrependableGeneratorReader[str],
+):
+    pass
+
+
+prependable_bytes_generator_reader = PrependableBytesGeneratorReader
+prependable_str_generator_reader = PrependableStrGeneratorReader
+
+
+##
+
+
+class BufferedGeneratorReader(PrependableGeneratorReader[AnyT], abc.ABC):
+    DEFAULT_BUFFER_SIZE = 4 * 0x1000
+
+    def __init__(
+            self,
+            buffer_size: int = DEFAULT_BUFFER_SIZE,
+    ) -> None:
+        check.arg(buffer_size > 0)
+
+        super().__init__()
+
+        self._buffer_size = buffer_size
+
+    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+        g = super().read(sz)
+        i: ta.Any = None
+        while True:
+            try:
+                q = g.send(i)
+            except StopIteration as e:
+                return e.value
+
+            check.state(not self._queue)
+
+            if q is None:
+                i = check.not_none((yield None))
+                continue
+
+            r = max(q, self._buffer_size)
+            d: AnyT = check.not_none((yield r))
+            if len(d) < q:
+                i = d
+                continue
+
+            i = d[:q]
+            self.prepend(d, q)
+
+
+class BufferedBytesGeneratorReader(
+    _BytesJoiner,
+    BufferedGeneratorReader[bytes],
+    PrependableGeneratorReader[bytes],
+):
+    pass
+
+
+class BufferedStrGeneratorReader(
+    _StrJoiner,
+    BufferedGeneratorReader[str],
+    PrependableGeneratorReader[str],
+):
+    pass
+
+
+buffered_bytes_generator_reader = BufferedBytesGeneratorReader
+buffered_str_generator_reader = BufferedStrGeneratorReader

omlish/io/generators/stepped.py

@@ -0,0 +1,104 @@
+import typing as ta
+
+from ... import lang
+
+
+T = ta.TypeVar('T')
+I = ta.TypeVar('I')
+O = ta.TypeVar('O')
+OF = ta.TypeVar('OF')
+OT = ta.TypeVar('OT')
+R = ta.TypeVar('R')
+
+
+SteppedGenerator: ta.TypeAlias = ta.Generator[O | None, I | None, R]
+
+
+##
+
+
+@lang.autostart
+def flatmap_stepped_generator(
+        fn: ta.Callable[[list[OF]], OT],
+        g: SteppedGenerator[OF, I, R],
+        *,
+        terminate: ta.Callable[[OF], bool] | None = None,
+) -> ta.Generator[OT, I, lang.Maybe[R]]:
+    """
+    Given a 'stepped generator' - a generator which accepts input items and yields zero or more non-None values in
+    response until it signals it's ready for the next input by yielding None - and a function taking a list, returns a
+    1:1 generator which accepts input, builds a list of yielded generator output, calls the given function with that
+    list, and yields the result.
+
+    An optional terminate function may be provided which will cause this function to return early if it returns true for
+    an encountered yielded value. The encountered value causing termination will be included in the list sent to the
+    given fn.
+
+    Returns a Maybe of either the given generator's return value or empty if the terminator was encountered.
+    """
+
+    l: list[OF]
+    i: I | None = yield  # type: ignore
+    while True:
+        l = []
+
+        while True:
+            try:
+                o = g.send(i)
+            except StopIteration as e:
+                if l:
+                    yield fn(l)
+                return lang.just(e.value)
+
+            i = None
+
+            if o is None:
+                break
+
+            l.append(o)
+
+            if terminate is not None and terminate(o):
+                yield fn(l)
+                return lang.empty()
+
+        i = yield fn(l)
+
+
+##
+
+
+def _join_bytes(l: ta.Sequence[bytes]) -> bytes:
+    if not l:
+        return b''
+    elif len(l) == 1:
+        return l[0]
+    else:
+        return b''.join(l)
+
+
+def _join_str(l: ta.Sequence[str]) -> str:
+    if not l:
+        return ''
+    elif len(l) == 1:
+        return l[0]
+    else:
+        return ''.join(l)
+
+
+def _is_empty(o: T) -> bool:
+    return len(o) < 1  # type: ignore
+
+
+##
+
+
+def joined_bytes_stepped_generator(
+        g: ta.Generator[bytes | None, bytes | None, R],
+) -> ta.Generator[bytes, bytes, R]:
+    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+
+
+def joined_str_stepped_generator(
+        g: ta.Generator[str | None, str | None, R],
+) -> ta.Generator[str, str, R]:
+    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)

omlish/lang/__init__.py

@@ -120,6 +120,17 @@ from .functions import (  # noqa
     void,
 )
 
+from .generators import (  # noqa
+    CoroutineGenerator,
+    Generator,
+    GeneratorLike,
+    GeneratorMappedIterator,
+    autostart,
+    corogen,
+    genmap,
+    nextgen,
+)
+
 from .imports import (  # noqa
     can_import,
     import_all,
@@ -136,7 +147,6 @@ from .imports import (  # noqa
 
 from .iterables import (  # noqa
     BUILTIN_SCALAR_ITERABLE_TYPES,
-    Generator,
     asrange,
     exhaust,
     flatmap,

omlish/lang/functions.py

@@ -82,7 +82,6 @@ def identity(obj: T) -> T:
 
 
 class constant(ta.Generic[T]):  # noqa
-
     def __init__(self, obj: T) -> None:
         super().__init__()
 
@@ -116,7 +115,6 @@ class VoidError(Exception):
 
 
 class Void:
-
     def __new__(cls, *args: ta.Any, **kwargs: ta.Any) -> None:  # type: ignore  # noqa
         raise VoidError