omlish 0.0.0.dev134__py3-none-any.whl → 0.0.0.dev137__py3-none-any.whl

omlish/io/compress/gzip.py

@@ -0,0 +1,301 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+import functools
+import os.path
+import struct
+import time
+import typing as ta
+
+from ... import cached
+from ... import check
+from ... import lang
+from ..generators import PrependableBytesGeneratorReader
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import gzip
+    import zlib
+else:
+    gzip = lang.proxy_import('gzip')
+    zlib = lang.proxy_import('zlib')
+
+
+##
+
+
+COMPRESS_LEVEL_FAST = 1
+COMPRESS_LEVEL_TRADEOFF = 6
+COMPRESS_LEVEL_BEST = 9
+
+
+@cached.function
+def _zero_crc() -> int:
+    return zlib.crc32(b'')
+
+
+##
+
+
+class IncrementalGzipCompressor:
+    def __init__(
+            self,
+            *,
+            compresslevel: int = COMPRESS_LEVEL_BEST,
+            name: str | bytes | None = None,
+            mtime: float | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._name = name or ''
+        self._compresslevel = compresslevel
+        self._mtime = mtime
+
+    def _write_gzip_header(self) -> ta.Generator[bytes, None, None]:
+        check.none((yield b'\037\213'))  # magic header
+        check.none((yield b'\010'))  # compression method
+
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not include filenames that cannot be represented that
+            # way.
+            fname = os.path.basename(self._name)
+            if not isinstance(fname, bytes):
+                fname = fname.encode('latin-1')
+            if fname.endswith(b'.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = b''
+
+        flags = 0
+        if fname:
+            flags = gzip.FNAME
+        check.none((yield chr(flags).encode('latin-1')))
+
+        mtime = self._mtime
+        if mtime is None:
+            mtime = time.time()
+        check.none((yield struct.pack('<L', int(mtime))))
+
+        if self._compresslevel == COMPRESS_LEVEL_BEST:
+            xfl = b'\002'
+        elif self._compresslevel == COMPRESS_LEVEL_FAST:
+            xfl = b'\004'
+        else:
+            xfl = b'\000'
+        check.none((yield xfl))
+
+        check.none((yield b'\377'))
+
+        if fname:
+            check.none((yield fname + b'\000'))
+
+    def __call__(self) -> IncrementalCompressor:
+        crc = _zero_crc()
+        size = 0
+        offset = 0  # Current file offset for seek(), tell(), etc
+
+        compress = zlib.compressobj(
+            self._compresslevel,
+            zlib.DEFLATED,
+            -zlib.MAX_WBITS,
+            zlib.DEF_MEM_LEVEL,
+            0,
+        )
+
+        yield from self._write_gzip_header()
+
+        while True:
+            data: ta.Any = check.isinstance((yield None), bytes)
+            if not data:
+                break
+
+            # Called by our self._buffer underlying BufferedWriterDelegate.
+            if isinstance(data, (bytes, bytearray)):
+                length = len(data)
+            else:
+                # accept any data that supports the buffer protocol
+                data = memoryview(data)
+                length = data.nbytes
+
+            if length > 0:
+                if (fl := compress.compress(data)):
+                    check.none((yield fl))
+                size += length
+                crc = zlib.crc32(data, crc)
+                offset += length
+
+        if (fl := compress.flush()):
+            check.none((yield fl))
+
+        yield struct.pack('<L', crc)
+        # size may exceed 2 GiB, or even 4 GiB
+        yield struct.pack('<L', size & 0xffffffff)
+
+        yield b''
+
+
+##
+
+
+class IncrementalGzipDecompressor:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._factory = functools.partial(
+            zlib.decompressobj,
+            wbits=-zlib.MAX_WBITS,
+        )
+
+    def _read_gzip_header(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+    ) -> ta.Generator[int | None, bytes, int | None]:
+        magic = yield from rdr.read(2)
+        if magic == b'':
+            return None
+
+        if magic != b'\037\213':
+            raise gzip.BadGzipFile(f'Not a gzipped file ({magic!r})')
+
+        buf = yield from rdr.read(8)
+        method, flag, last_mtime = struct.unpack('<BBIxx', buf)
+        if method != 8:
+            raise gzip.BadGzipFile('Unknown compression method')
+
+        if flag & gzip.FEXTRA:
+            # Read & discard the extra field, if present
+            buf = yield from rdr.read(2)
+            extra_len, = struct.unpack('<H', buf)
+            if extra_len:
+                yield from rdr.read(extra_len)
+
+        if flag & gzip.FNAME:
+            # Read and discard a null-terminated string containing the filename
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FCOMMENT:
+            # Read and discard a null-terminated string containing a comment
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FHCRC:
+            yield from rdr.read(2)  # Read & discard the 16-bit header CRC
+
+        return last_mtime
+
+    def _read_eof(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+            crc: int,
+            stream_size: int,
+    ) -> ta.Generator[int | None, bytes, None]:
+        # We've read to the end of the file.
+        # We check that the computed CRC and size of the uncompressed data matches the stored values. Note that the size
+        # stored is the true file size mod 2**32.
+        buf = yield from rdr.read(8)
+        crc32, isize = struct.unpack('<II', buf)
+        if crc32 != crc:
+            raise gzip.BadGzipFile(f'CRC check failed {hex(crc32)} != {hex(crc)}')
+        elif isize != (stream_size & 0xffffffff):
+            raise gzip.BadGzipFile('Incorrect length of data produced')
+
+        # Gzip files can be padded with zeroes and still have archives. Consume all zero bytes and set the file position
+        # to the first non-zero byte. See http://www.gzip.org/#faq8
+        c = b'\0'
+        while c == b'\0':
+            c = yield from rdr.read(1)
+        if c:
+            rdr.prepend(c)
+
+    def __call__(self) -> IncrementalDecompressor:
+        rdr = PrependableBytesGeneratorReader()
+
+        pos = 0  # Current offset in decompressed stream
+
+        crc = _zero_crc()
+        stream_size = 0  # Decompressed size of unconcatenated stream
+        new_member = True
+
+        decompressor = self._factory()
+
+        while True:
+            # For certain input data, a single call to decompress() may not return any data. In this case, retry until
+            # we get some data or reach EOF.
+            while True:
+                if decompressor.eof:
+                    # Ending case: we've come to the end of a member in the file, so finish up this member, and read a
+                    # new gzip header. Check the CRC and file size, and set the flag so we read a new member
+                    yield from self._read_eof(rdr, crc, stream_size)
+                    new_member = True
+                    decompressor = self._factory()
+
+                if new_member:
+                    # If the _new_member flag is set, we have to jump to the next member, if there is one.
+                    crc = _zero_crc()
+                    stream_size = 0  # Decompressed size of unconcatenated stream
+                    last_mtime = yield from self._read_gzip_header(rdr)
+                    if not last_mtime:
+                        check.none((yield b''))
+                        return
+                    new_member = False
+
+                # Read a chunk of data from the file
+                if not decompressor.unconsumed_tail:
+                    buf = yield from rdr.read(None)
+                    uncompress = decompressor.decompress(buf)
+                else:
+                    uncompress = decompressor.decompress(b'')
+
+                if decompressor.unused_data != b'':
+                    # Prepend the already read bytes to the fileobj so they can be seen by _read_eof() and
+                    # _read_gzip_header()
+                    rdr.prepend(decompressor.unused_data)
+
+                if uncompress != b'':
+                    break
+                if buf == b'':  # noqa
+                    raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+            crc = zlib.crc32(uncompress, crc)
+            stream_size += len(uncompress)
+            pos += len(uncompress)
+            check.none((yield uncompress))

omlish/io/compress/lzma.py

@@ -0,0 +1,31 @@
+import typing as ta
+
+from ... import lang
+from .adapters import CompressorIncrementalAdapter
+from .adapters import DecompressorIncrementalAdapter
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import lzma
+else:
+    lzma = lang.proxy_import('lzma')
+
+
+class IncrementalLzmaCompressor:
+    def __init__(self) -> None:
+        super().__init__()
+
+    def __call__(self) -> IncrementalCompressor:
+        return CompressorIncrementalAdapter(
+            lzma.LZMACompressor,  # type: ignore
+        )()
+
+
+class IncrementalLzmaDecompressor:
+    def __call__(self) -> IncrementalDecompressor:
+        return DecompressorIncrementalAdapter(
+            lzma.LZMADecompressor,  # type: ignore
+            trailing_error=lzma.LZMAError,
+        )()

omlish/io/compress/types.py

@@ -0,0 +1,29 @@
+# ruff: noqa: UP007
+import typing as ta
+
+
+IncrementalCompressor: ta.TypeAlias = ta.Generator[
+    ta.Union[
+        bytes,  # Compressed output
+        None,  # Need input
+    ],
+    ta.Union[
+        bytes,  # Input bytes
+        None,  # Need output
+    ],
+    None,
+]
+
+
+IncrementalDecompressor: ta.TypeAlias = ta.Generator[
+    ta.Union[
+        bytes,  # Uncompressed output
+        int,  # Need exactly n bytes
+        None,  # Need any amount of bytes
+    ],
+    ta.Union[
+        bytes,  # Input bytes
+        None,  # Need output
+    ],
+    None,
+]

omlish/io/generators.py

@@ -0,0 +1,50 @@
+"""
+TODO:
+ - BufferedBytesGeneratorReader
+"""
+import typing as ta
+
+from .. import check
+
+
+class PrependableBytesGeneratorReader:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._p: list[bytes] = []
+
+    def read(self, sz: int | None) -> ta.Generator[int | None, bytes, bytes]:
+        if not self._p:
+            d = check.isinstance((yield sz), bytes)
+            return d
+
+        if sz is None:
+            return self._p.pop(0)
+
+        l: list[bytes] = []
+        r = sz
+        while r > 0 and self._p:
+            c = self._p[0]
+
+            if len(c) > r:
+                l.append(c[:r])
+                self._p[0] = c[r:]
+                return b''.join(l)
+
+            l.append(c)
+            r -= len(c)
+            self._p.pop(0)
+
+        if r:
+            c = check.isinstance((yield r), bytes)
+            if not c:
+                return b''
+            if len(c) != r:
+                raise EOFError(f'Reader got {len(c)} bytes, expected {r}')
+            l.append(c)
+
+        return b''.join(l)
+
+    def prepend(self, d: bytes) -> None:
+        if d:
+            self._p.append(d)

omlish/lang/__init__.py

@@ -120,6 +120,14 @@ from .functions import (  # noqa
     void,
 )
 
+from .generators import (  # noqa
+    CoroutineGenerator,
+    Generator,
+    GeneratorLike,
+    corogen,
+    nextgen,
+)
+
 from .imports import (  # noqa
     can_import,
     import_all,
@@ -136,7 +144,6 @@ from .imports import (  # noqa
 
 from .iterables import (  # noqa
     BUILTIN_SCALAR_ITERABLE_TYPES,
-    Generator,
     asrange,
     exhaust,
     flatmap,

omlish/lang/generators.py

@@ -0,0 +1,182 @@
+import abc
+import typing as ta
+
+from .maybes import Maybe
+
+
+T = ta.TypeVar('T')
+I = ta.TypeVar('I')
+O = ta.TypeVar('O')
+R = ta.TypeVar('R')
+I_contra = ta.TypeVar('I_contra', contravariant=True)
+O_co = ta.TypeVar('O_co', covariant=True)
+R_co = ta.TypeVar('R_co', covariant=True)
+
+
+##
+
+
+def nextgen(g: T) -> T:
+    next(g)  # type: ignore
+    return g
+
+
+##
+
+
+@ta.runtime_checkable
+class GeneratorLike(ta.Protocol[O_co, I_contra, R_co]):
+    def send(self, i: I_contra) -> O_co:  # Raises[StopIteration[R_co]]
+        ...
+
+    def close(self) -> None:
+        ...
+
+
+class GeneratorLike_(abc.ABC, ta.Generic[O, I, R]):  # noqa
+    @abc.abstractmethod
+    def send(self, i: I) -> O:  # Raises[StopIteration[R]]
+        raise NotImplementedError
+
+    def close(self) -> None:
+        pass
+
+
+@ta.overload
+def adapt_generator_like(gl: GeneratorLike_[O, I, R]) -> ta.Generator[O, I, R]:
+    ...
+
+
+@ta.overload
+def adapt_generator_like(gl: GeneratorLike[O, I, R]) -> ta.Generator[O, I, R]:
+    ...
+
+
+def adapt_generator_like(gl):
+    try:
+        i = yield
+        while True:
+            i = yield gl.send(i)
+    except StopIteration as e:
+        return e.value
+    finally:
+        gl.close()
+
+
+##
+
+
+class Generator(ta.Generator[O, I, R]):
+    def __init__(self, g: ta.Generator[O, I, R]) -> None:
+        super().__init__()
+        self._g = g
+
+    @property
+    def g(self) -> ta.Generator[O, I, R]:
+        return self._g
+
+    value: R
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            return next(self._g)
+        except StopIteration as e:
+            self.value = e.value
+            raise
+
+    def send(self, v):
+        try:
+            return self._g.send(v)
+        except StopIteration as e:
+            self.value = e.value
+            raise
+
+    def throw(self, *args):
+        try:
+            return self._g.throw(*args)
+        except StopIteration as e:
+            self.value = e.value
+            raise
+
+    def close(self):
+        self._g.close()
+
+
+##
+
+
+class CoroutineGenerator(ta.Generic[O, I, R]):
+    def __init__(self, g: ta.Generator[O, I, R]) -> None:
+        super().__init__()
+        self._g = g
+
+    @property
+    def g(self) -> ta.Generator[O, I, R]:
+        return self._g
+
+    #
+
+    def close(self) -> None:
+        self._g.close()
+
+    def __enter__(self) -> ta.Self:
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._g.close()
+
+    #
+
+    class Output(ta.NamedTuple, ta.Generic[T]):
+        v: T
+
+        @property
+        def is_return(self) -> bool:
+            raise NotImplementedError
+
+    class Yield(Output[T]):
+        @property
+        def is_return(self) -> bool:
+            return False
+
+    class Return(Output[T]):
+        @property
+        def is_return(self) -> bool:
+            return True
+
+    class Nothing:
+        def __new__(cls):
+            raise TypeError
+
+    #
+
+    def send(self, /, v: I | type[Nothing] = Nothing) -> Yield[O] | Return[R]:
+        try:
+            if v is self.Nothing:
+                o = next(self._g)
+            else:
+                o = self._g.send(v)  # type: ignore[arg-type]
+        except StopIteration as e:
+            return self.Return(e.value)
+        else:
+            return self.Yield(o)
+
+    def send_opt(self, v: I | None) -> Yield[O] | Return[R]:
+        return self.send(v if v is not None else self.Nothing)
+
+    def send_maybe(self, v: Maybe[I]) -> Yield[O] | Return[R]:
+        return self.send(v.or_else(self.Nothing))
+
+    def throw(self, v: BaseException) -> Yield[O] | Return[R]:
+        try:
+            o = self._g.throw(v)
+        except StopIteration as e:
+            return self.Return(e.value)
+        else:
+            return self.Yield(o)
+
+
+corogen = CoroutineGenerator