omlish 0.0.0.dev132__py3-none-any.whl → 0.0.0.dev177__py3-none-any.whl

omlish/io/compress/bz2.py

@@ -0,0 +1,61 @@
+import dataclasses as dc
+import functools
+import typing as ta
+
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from .adapters import CompressorObjectIncrementalAdapter
+from .adapters import DecompressorObjectIncrementalAdapter
+from .base import Compression
+from .base import IncrementalCompression
+from .codecs import make_compression_codec
+from .codecs import make_compression_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import bz2
+else:
+    bz2 = lang.proxy_import('bz2')
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Bz2Compression(Compression, IncrementalCompression):
+    level: int = 9
+
+    def compress(self, d: bytes) -> bytes:
+        return bz2.compress(
+            d,
+            self.level,
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return bz2.decompress(
+            d,
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(CompressorObjectIncrementalAdapter(
+            functools.partial(
+                bz2.BZ2Compressor,  # type: ignore
+                self.level,
+            ),
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return DecompressorObjectIncrementalAdapter(
+            bz2.BZ2Decompressor,  # type: ignore
+            trailing_error=OSError,
+        )()
+
+
+##
+
+
+BZ2_CODEC = make_compression_codec('bz2', Bz2Compression)
+
+# @omlish-manifest
+_BZ2_LAZY_CODEC = make_compression_lazy_loaded_codec(__name__, 'BZ2_CODEC', BZ2_CODEC)

omlish/io/compress/codecs.py

@@ -0,0 +1,78 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import codecs
+from ..generators import buffer_bytes_stepped_reader_generator
+from .base import Compression
+from .base import IncrementalCompression
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CompressionEagerCodec(codecs.EagerCodec[bytes, bytes]):
+    compression: Compression
+
+    def encode(self, i: bytes) -> bytes:
+        return self.compression.compress(i)
+
+    def decode(self, o: bytes) -> bytes:
+        return self.compression.decompress(o)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CompressionIncrementalCodec(codecs.IncrementalCodec[bytes, bytes]):
+    compression: IncrementalCompression
+
+    def encode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return self.compression.compress_incremental()
+
+    def decode_incremental(self) -> ta.Generator[bytes | None, bytes, None]:
+        return buffer_bytes_stepped_reader_generator(self.compression.decompress_incremental())
+
+
+##
+
+
+class CompressionCodec(codecs.Codec):
+    pass
+
+
+def make_compression_codec(
+        name: str,
+        cls: type[Compression],
+        *,
+        aliases: ta.Collection[str] | None = None,
+) -> CompressionCodec:
+    return CompressionCodec(
+        name=name,
+        aliases=aliases,
+
+        input=bytes,
+        output=bytes,
+
+        new=lambda *args, **kwargs: CompressionEagerCodec(cls(*args, **kwargs)),
+
+        new_incremental=(
+            lambda *args, **kwargs: CompressionIncrementalCodec(cls(*args, **kwargs))  # noqa
+        ) if issubclass(cls, IncrementalCompression) else None,
+    )
+
+
+##
+
+
+def make_compression_lazy_loaded_codec(
+        mod_name: str,
+        attr_name: str,
+        codec: CompressionCodec,
+) -> codecs.LazyLoadedCodec:
+    return codecs.LazyLoadedCodec.new(
+        mod_name,
+        attr_name,
+        codec,
+    )

omlish/io/compress/gzip.py

@@ -0,0 +1,350 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+import dataclasses as dc
+import functools
+import os.path
+import struct
+import time
+import typing as ta
+
+from ... import cached
+from ... import check
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from ..generators.readers import PrependableBytesGeneratorReader
+from .base import Compression
+from .base import IncrementalCompression
+from .codecs import make_compression_codec
+from .codecs import make_compression_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import gzip
+    import zlib
+else:
+    gzip = lang.proxy_import('gzip')
+    zlib = lang.proxy_import('zlib')
+
+
+##
+
+
+COMPRESS_LEVEL_FAST = 1
+COMPRESS_LEVEL_TRADEOFF = 6
+COMPRESS_LEVEL_BEST = 9
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class GzipCompression(Compression, IncrementalCompression):
+    level: int = COMPRESS_LEVEL_BEST
+
+    mtime: float | None = None
+
+    def compress(self, d: bytes) -> bytes:
+        return gzip.compress(
+            d,
+            self.level,
+            mtime=self.mtime,
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return gzip.decompress(
+            d,
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(IncrementalGzipCompressor(
+            level=self.level,
+            mtime=self.mtime,
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return IncrementalGzipDecompressor()()
+
+
+##
+
+
+@cached.function
+def _zero_crc() -> int:
+    return zlib.crc32(b'')
+
+
+##
+
+
+class IncrementalGzipCompressor:
+    def __init__(
+            self,
+            *,
+            level: int = COMPRESS_LEVEL_BEST,
+            name: str | bytes | None = None,
+            mtime: float | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._name = name or ''
+        self._level = level
+        self._mtime = mtime
+
+    def _write_gzip_header(self) -> ta.Generator[bytes, None, None]:
+        check.none((yield b'\037\213'))  # magic header
+        check.none((yield b'\010'))  # compression method
+
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not include filenames that cannot be represented that
+            # way.
+            fname = os.path.basename(self._name)
+            if not isinstance(fname, bytes):
+                fname = fname.encode('latin-1')
+            if fname.endswith(b'.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = b''
+
+        flags = 0
+        if fname:
+            flags = gzip.FNAME
+        check.none((yield chr(flags).encode('latin-1')))
+
+        mtime = self._mtime
+        if mtime is None:
+            mtime = time.time()
+        check.none((yield struct.pack('<L', int(mtime))))
+
+        if self._level == COMPRESS_LEVEL_BEST:
+            xfl = b'\002'
+        elif self._level == COMPRESS_LEVEL_FAST:
+            xfl = b'\004'
+        else:
+            xfl = b'\000'
+        check.none((yield xfl))
+
+        check.none((yield b'\377'))
+
+        if fname:
+            check.none((yield fname + b'\000'))
+
+    def __call__(self) -> BytesSteppedGenerator:
+        crc = _zero_crc()
+        size = 0
+        offset = 0  # Current file offset for seek(), tell(), etc
+        wrote_header = False
+
+        compress = zlib.compressobj(
+            self._level,
+            zlib.DEFLATED,
+            -zlib.MAX_WBITS,
+            zlib.DEF_MEM_LEVEL,
+            0,
+        )
+
+        while True:
+            data: ta.Any = check.isinstance((yield None), bytes)
+
+            if not wrote_header:
+                yield from self._write_gzip_header()
+                wrote_header = True
+
+            if not data:
+                break
+
+            # Called by our self._buffer underlying BufferedWriterDelegate.
+            if isinstance(data, (bytes, bytearray)):
+                length = len(data)
+            else:
+                # accept any data that supports the buffer protocol
+                data = memoryview(data)
+                length = data.nbytes
+
+            if length > 0:
+                if (fl := compress.compress(data)):
+                    check.none((yield fl))
+                size += length
+                crc = zlib.crc32(data, crc)
+                offset += length
+
+        if (fl := compress.flush()):
+            check.none((yield fl))
+
+        yield struct.pack('<L', crc)
+        # size may exceed 2 GiB, or even 4 GiB
+        yield struct.pack('<L', size & 0xffffffff)
+
+        yield b''
+
+
+##
+
+
+class IncrementalGzipDecompressor:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._factory = functools.partial(
+            zlib.decompressobj,
+            wbits=-zlib.MAX_WBITS,
+        )
+
+    def _read_gzip_header(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+    ) -> ta.Generator[int | None, bytes, int | None]:
+        magic = yield from rdr.read(2)
+        if magic == b'':
+            return None
+
+        if magic != b'\037\213':
+            raise gzip.BadGzipFile(f'Not a gzipped file ({magic!r})')
+
+        buf = yield from rdr.read(8)
+        method, flag, last_mtime = struct.unpack('<BBIxx', buf)
+        if method != 8:
+            raise gzip.BadGzipFile('Unknown compression method')
+
+        if flag & gzip.FEXTRA:
+            # Read & discard the extra field, if present
+            buf = yield from rdr.read(2)
+            extra_len, = struct.unpack('<H', buf)
+            if extra_len:
+                yield from rdr.read(extra_len)
+
+        if flag & gzip.FNAME:
+            # Read and discard a null-terminated string containing the filename
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FCOMMENT:
+            # Read and discard a null-terminated string containing a comment
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FHCRC:
+            yield from rdr.read(2)  # Read & discard the 16-bit header CRC
+
+        return last_mtime
+
+    def _read_eof(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+            crc: int,
+            stream_size: int,
+    ) -> ta.Generator[int | None, bytes, None]:
+        # We've read to the end of the file.
+        # We check that the computed CRC and size of the uncompressed data matches the stored values. Note that the size
+        # stored is the true file size mod 2**32.
+        buf = yield from rdr.read(8)
+        crc32, isize = struct.unpack('<II', buf)
+        if crc32 != crc:
+            raise gzip.BadGzipFile(f'CRC check failed {hex(crc32)} != {hex(crc)}')
+        elif isize != (stream_size & 0xffffffff):
+            raise gzip.BadGzipFile('Incorrect length of data produced')
+
+        # Gzip files can be padded with zeroes and still have archives. Consume all zero bytes and set the file position
+        # to the first non-zero byte. See http://www.gzip.org/#faq8
+        c = b'\0'
+        while c == b'\0':
+            c = yield from rdr.read(1)
+        if c:
+            rdr.prepend(c)
+
+    def __call__(self) -> BytesSteppedReaderGenerator:
+        rdr = PrependableBytesGeneratorReader()
+
+        pos = 0  # Current offset in decompressed stream
+
+        crc = _zero_crc()
+        stream_size = 0  # Decompressed size of unconcatenated stream
+        new_member = True
+
+        decompressor = self._factory()
+
+        while True:
+            # For certain input data, a single call to decompress() may not return any data. In this case, retry until
+            # we get some data or reach EOF.
+            while True:
+                if decompressor.eof:
+                    # Ending case: we've come to the end of a member in the file, so finish up this member, and read a
+                    # new gzip header. Check the CRC and file size, and set the flag so we read a new member
+                    yield from self._read_eof(rdr, crc, stream_size)
+                    new_member = True
+                    decompressor = self._factory()
+
+                if new_member:
+                    # If the _new_member flag is set, we have to jump to the next member, if there is one.
+                    crc = _zero_crc()
+                    stream_size = 0  # Decompressed size of unconcatenated stream
+                    last_mtime = yield from self._read_gzip_header(rdr)
+                    if not last_mtime:
+                        check.none((yield b''))
+                        return
+                    new_member = False
+
+                # Read a chunk of data from the file
+                if not decompressor.unconsumed_tail:
+                    buf = yield from rdr.read(None)
+                    uncompress = decompressor.decompress(buf)
+                else:
+                    uncompress = decompressor.decompress(b'')
+
+                if decompressor.unused_data != b'':
+                    # Prepend the already read bytes to the fileobj so they can be seen by _read_eof() and
+                    # _read_gzip_header()
+                    rdr.prepend(decompressor.unused_data)
+
+                if uncompress != b'':
+                    break
+                if buf == b'':  # noqa
+                    raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+            crc = zlib.crc32(uncompress, crc)
+            stream_size += len(uncompress)
+            pos += len(uncompress)
+            check.none((yield uncompress))
+
+
+##
+
+
+GZIP_CODEC = make_compression_codec('gzip', GzipCompression, aliases=['gz'])
+
+# @omlish-manifest
+_GZIP_LAZY_CODEC = make_compression_lazy_loaded_codec(__name__, 'GZIP_CODEC', GZIP_CODEC)

omlish/io/compress/lz4.py

@@ -0,0 +1,91 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import check
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from .base import Compression
+from .base import IncrementalCompression
+from .codecs import make_compression_codec
+from .codecs import make_compression_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import lz4.frame as lz4_frame
+else:
+    lz4_frame = lang.proxy_import('lz4.frame')
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Lz4Compression(Compression, IncrementalCompression):
+    level: int = 0
+
+    block_size: int = 0
+    block_linked: bool = True
+    block_checksum: bool = False
+    content_checksum: bool = False
+    store_size: bool = True
+    auto_flush: bool = False
+
+    def compress(self, d: bytes) -> bytes:
+        return lz4_frame.compress(
+            d,
+            compression_level=self.level,
+            block_size=self.block_size,
+            content_checksum=self.content_checksum,
+            block_linked=self.block_linked,
+            store_size=self.store_size,
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return lz4_frame.decompress(
+            d,
+        )
+
+    @lang.autostart
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        with lz4_frame.LZ4FrameCompressor(
+                compression_level=self.level,
+                block_size=self.block_size,
+                block_linked=self.block_linked,
+                block_checksum=self.block_checksum,
+                content_checksum=self.content_checksum,
+                auto_flush=self.auto_flush,
+        ) as compressor:
+            started = False
+            while True:
+                i = check.isinstance((yield None), bytes)
+                if not started:
+                    yield compressor.begin()
+                    started = True
+                if not i:
+                    yield compressor.flush()
+                    yield b''
+                    return
+                if (o := compressor.compress(i)):
+                    yield o
+
+    @lang.autostart
+    def decompress_incremental(self) -> BytesSteppedGenerator[None]:
+        # lz4 lib does internal buffering so this is simply a BytesSteppedGenerator not a BytesSteppedReaderGenerator as
+        # it only yields None, accepting any number of bytes at a time.
+        with lz4_frame.LZ4FrameDecompressor() as decompressor:
+            while True:
+                i = check.isinstance((yield None), bytes)
+                if not i:
+                    yield b''
+                    return
+                if (o := decompressor.decompress(i)):
+                    yield o
+
+
+##
+
+
+LZ4_CODEC = make_compression_codec('lz4', Lz4Compression)
+
+# @omlish-manifest
+_LZ4_LAZY_CODEC = make_compression_lazy_loaded_codec(__name__, 'LZ4_CODEC', LZ4_CODEC)

omlish/io/compress/lzma.py

@@ -0,0 +1,81 @@
+import dataclasses as dc
+import functools
+import typing as ta
+
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from .adapters import CompressorObjectIncrementalAdapter
+from .adapters import DecompressorObjectIncrementalAdapter
+from .base import Compression
+from .base import IncrementalCompression
+from .codecs import make_compression_codec
+from .codecs import make_compression_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import lzma
+else:
+    lzma = lang.proxy_import('lzma')
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class LzmaCompression(Compression, IncrementalCompression):
+    format: int | None = None
+
+    check: int = -1
+    preset: int | None = None
+    filters: dict | None = None
+
+    mem_limit: int | None = None
+
+    def compress(self, d: bytes) -> bytes:
+        return lzma.compress(
+            d,
+            format=self.format if self.format is not None else lzma.FORMAT_XZ,
+            check=self.check,
+            preset=self.preset,
+            filters=self.filters,  # type: ignore[arg-type]
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return lzma.decompress(
+            d,
+            format=self.format if self.format is not None else lzma.FORMAT_AUTO,
+            memlimit=self.mem_limit,
+            filters=self.filters,  # type: ignore[arg-type]
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(CompressorObjectIncrementalAdapter(
+            functools.partial(  # type: ignore
+                lzma.LZMACompressor,
+                format=self.format if self.format is not None else lzma.FORMAT_XZ,
+                check=self.check,
+                preset=self.preset,
+                filters=self.filters,  # type: ignore[arg-type]
+            ),
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return DecompressorObjectIncrementalAdapter(
+            functools.partial(  # type: ignore
+                lzma.LZMADecompressor,
+                format=self.format if self.format is not None else lzma.FORMAT_AUTO,
+                memlimit=self.mem_limit,
+                filters=self.filters,  # type: ignore[arg-type]
+            ),
+            trailing_error=lzma.LZMAError,
+        )()
+
+
+##
+
+
+LZMA_CODEC = make_compression_codec('lzma', LzmaCompression)
+
+# @omlish-manifest
+_LZMA_LAZY_CODEC = make_compression_lazy_loaded_codec(__name__, 'LZMA_CODEC', LZMA_CODEC)

omlish/io/compress/snappy.py

@@ -0,0 +1,34 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import lang
+from .base import Compression
+from .codecs import make_compression_codec
+from .codecs import make_compression_lazy_loaded_codec
+
+
+if ta.TYPE_CHECKING:
+    import snappy
+else:
+    snappy = lang.proxy_import('snappy')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SnappyCompression(Compression):
+    def compress(self, d: bytes) -> bytes:
+        return snappy.compress(d)
+
+    def decompress(self, d: bytes) -> bytes:
+        return snappy.decompress(d)
+
+
+##
+
+
+SNAPPY_CODEC = make_compression_codec('snappy', SnappyCompression)
+
+# @omlish-manifest
+_SNAPPY_LAZY_CODEC = make_compression_lazy_loaded_codec(__name__, 'SNAPPY_CODEC', SNAPPY_CODEC)