omlish 0.0.0.dev141__tar.gz → 0.0.0.dev142__tar.gz

{omlish-0.0.0.dev141/omlish.egg-info → omlish-0.0.0.dev142}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev141
+Version: 0.0.0.dev142
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -21,6 +21,7 @@ Requires-Dist: trio-asyncio~=0.15; extra == "all"
 Requires-Dist: lz4~=4.3; extra == "all"
 Requires-Dist: python-snappy~=0.7; extra == "all"
 Requires-Dist: zstandard~=0.23; extra == "all"
+Requires-Dist: brotli~=1.1; extra == "all"
 Requires-Dist: asttokens~=3.0; extra == "all"
 Requires-Dist: executing~=2.1; extra == "all"
 Requires-Dist: psutil~=6.0; extra == "all"
@@ -60,6 +61,7 @@ Provides-Extra: compress
 Requires-Dist: lz4~=4.3; extra == "compress"
 Requires-Dist: python-snappy~=0.7; extra == "compress"
 Requires-Dist: zstandard~=0.23; extra == "compress"
+Requires-Dist: brotli~=1.1; extra == "compress"
 Provides-Extra: diag
 Requires-Dist: asttokens~=3.0; extra == "diag"
 Requires-Dist: executing~=2.1; extra == "diag"

{omlish-0.0.0.dev141 → omlish-0.0.0.dev142}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev141'
-__revision__ = '3c5f18d26cf70e5f7c52546521cac21f20063258'
+__version__ = '0.0.0.dev142'
+__revision__ = 'fe684cfe6dc46f867fb6990935e8b145b67bbefa'
 
 
 #
@@ -48,6 +48,8 @@ class Project(ProjectBase):
             'python-snappy ~= 0.7',
 
             'zstandard ~= 0.23',
+
+            'brotli ~= 1.1',
         ],
 
         'diag': [

{omlish-0.0.0.dev141 → omlish-0.0.0.dev142}/omlish/check.py

@@ -144,6 +144,8 @@ def _raise(
 
 
 def _unpack_isinstance_spec(spec: ta.Any) -> tuple:
+    if _isinstance(spec, type):
+        return (spec,)
     if not _isinstance(spec, tuple):
         spec = (spec,)
     if None in spec:

{omlish-0.0.0.dev141 → omlish-0.0.0.dev142}/omlish/funcs/pairs.py

@@ -10,9 +10,9 @@ TODO:
 
 Compression choice:
  - lzma if-available minimal-space
- - lz4 if-available write-once
+ - lz4 if-available read-heavy
  - zstd if-available
- - bz2 write-once (but no parallel decompress)
+ - bz2 read-heavy (but no parallel decompress)
  - gz
 """
 import abc

omlish-0.0.0.dev142/omlish/io/compress/__init__.py

@@ -0,0 +1,9 @@
+"""
+TODO:
+ - zipfile?
+ - tarfile?
+ - lzf?
+ - blosc?
+ - python-lzo?
+ - zfp -> fp8?
+"""

{omlish-0.0.0.dev141 → omlish-0.0.0.dev142}/omlish/io/compress/abc.py

@@ -9,7 +9,7 @@ import abc
 ##
 
 
-class Compressor(abc.ABC):
+class CompressorObject(abc.ABC):
     @abc.abstractmethod
     def compress(self, data: bytes) -> bytes:
         """
@@ -36,7 +36,7 @@ class Compressor(abc.ABC):
 ##
 
 
-class Decompressor(abc.ABC):
+class DecompressorObject(abc.ABC):
     @property
     @abc.abstractmethod
     def unused_data(self) -> bytes:
@@ -68,7 +68,7 @@ class Decompressor(abc.ABC):
         raise NotImplementedError
 
 
-class NeedsInputDecompressor(Decompressor):
+class NeedsInputDecompressorObject(DecompressorObject):
     """
     Used by:
      - bz2.BZ2Decompressor
@@ -85,7 +85,7 @@ class NeedsInputDecompressor(Decompressor):
         raise NotImplementedError
 
 
-class UnconsumedTailDecompressor(Decompressor):
+class UnconsumedTailDecompressorObject(DecompressorObject):
     """
     Used by:
      - zlib.decompressobj

{omlish-0.0.0.dev141 → omlish-0.0.0.dev142}/omlish/io/compress/adapters.py

@@ -37,26 +37,26 @@
 import typing as ta
 
 from ... import check
-from .abc import Compressor
-from .abc import NeedsInputDecompressor
-from .abc import UnconsumedTailDecompressor
-from .types import IncrementalCompressor
-from .types import IncrementalDecompressor
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from .abc import CompressorObject
+from .abc import NeedsInputDecompressorObject
+from .abc import UnconsumedTailDecompressorObject
 
 
 ##
 
 
-class CompressorIncrementalAdapter:
+class CompressorObjectIncrementalAdapter:
     def __init__(
             self,
-            factory: ta.Callable[..., Compressor],
+            factory: ta.Callable[..., CompressorObject],
     ) -> None:
         super().__init__()
 
         self._factory = factory
 
-    def __call__(self) -> IncrementalCompressor:
+    def __call__(self) -> BytesSteppedGenerator:
         compressor = self._factory()
 
         while True:
@@ -77,10 +77,10 @@ class CompressorIncrementalAdapter:
 ##
 
 
-class DecompressorIncrementalAdapter:
+class DecompressorObjectIncrementalAdapter:
     def __init__(
             self,
-            factory: ta.Callable[..., NeedsInputDecompressor | UnconsumedTailDecompressor],
+            factory: ta.Callable[..., NeedsInputDecompressorObject | UnconsumedTailDecompressorObject],
             *,
             trailing_error: type[BaseException] | tuple[type[BaseException], ...] = (),
     ) -> None:
@@ -89,7 +89,7 @@ class DecompressorIncrementalAdapter:
         self._factory = factory
         self._trailing_error = trailing_error
 
-    def __call__(self) -> IncrementalDecompressor:
+    def __call__(self) -> BytesSteppedReaderGenerator:
         pos = 0
 
         data = None  # Default if EOF is encountered
@@ -145,3 +145,4 @@ class DecompressorIncrementalAdapter:
 
             pos += len(data)
             check.none((yield data))
+            data = None

omlish-0.0.0.dev142/omlish/io/compress/base.py

@@ -0,0 +1,24 @@
+import abc
+
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+
+
+class Compression(abc.ABC):
+    @abc.abstractmethod
+    def compress(self, d: bytes) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decompress(self, d: bytes) -> bytes:
+        raise NotImplementedError
+
+
+class IncrementalCompression(abc.ABC):
+    @abc.abstractmethod
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        raise NotImplementedError

omlish-0.0.0.dev142/omlish/io/compress/brotli.py

@@ -0,0 +1,33 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import lang
+from .base import Compression
+
+
+if ta.TYPE_CHECKING:
+    import brotli
+else:
+    brotli = lang.proxy_import('brotli')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class SnappyCompression(Compression):
+    mode: int | None = None
+    quality: int | None = None
+    lgwin: int | None = None
+    lgblock: int | None = None
+
+    def compress(self, d: bytes) -> bytes:
+        return brotli.compress(
+            d,
+            **(dict(mode=self.mode) if self.mode is not None else {}),
+            **(dict(mode=self.quality) if self.quality is not None else {}),
+            **(dict(mode=self.lgwin) if self.lgwin is not None else {}),
+            **(dict(mode=self.lgblock) if self.lgblock is not None else {}),
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return brotli.decompress(
+            d,
+        )

omlish-0.0.0.dev142/omlish/io/compress/bz2.py

@@ -0,0 +1,47 @@
+import dataclasses as dc
+import functools
+import typing as ta
+
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from .adapters import CompressorObjectIncrementalAdapter
+from .adapters import DecompressorObjectIncrementalAdapter
+from .base import Compression
+from .base import IncrementalCompression
+
+
+if ta.TYPE_CHECKING:
+    import bz2
+else:
+    bz2 = lang.proxy_import('bz2')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Bz2Compression(Compression, IncrementalCompression):
+    level: int = 9
+
+    def compress(self, d: bytes) -> bytes:
+        return bz2.compress(
+            d,
+            self.level,
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return bz2.decompress(
+            d,
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(CompressorObjectIncrementalAdapter(
+            functools.partial(
+                bz2.BZ2Compressor,  # type: ignore
+                self.level,
+            ),
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return DecompressorObjectIncrementalAdapter(
+            bz2.BZ2Decompressor,  # type: ignore
+            trailing_error=OSError,
+        )()

{omlish-0.0.0.dev141 → omlish-0.0.0.dev142}/omlish/io/compress/gzip.py

@@ -33,6 +33,7 @@
 #
 # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
 # License Agreement.
+import dataclasses as dc
 import functools
 import os.path
 import struct
@@ -42,9 +43,11 @@ import typing as ta
 from ... import cached
 from ... import check
 from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
 from ..generators.readers import PrependableBytesGeneratorReader
-from .types import IncrementalCompressor
-from .types import IncrementalDecompressor
+from .base import Compression
+from .base import IncrementalCompression
 
 
 if ta.TYPE_CHECKING:
@@ -63,6 +66,37 @@ COMPRESS_LEVEL_TRADEOFF = 6
 COMPRESS_LEVEL_BEST = 9
 
 
+@dc.dataclass(frozen=True, kw_only=True)
+class GzipCompression(Compression, IncrementalCompression):
+    level: int = COMPRESS_LEVEL_BEST
+
+    mtime: float | None = None
+
+    def compress(self, d: bytes) -> bytes:
+        return gzip.compress(
+            d,
+            self.level,
+            mtime=self.mtime,
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return gzip.decompress(
+            d,
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(IncrementalGzipCompressor(
+            level=self.level,
+            mtime=self.mtime,
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return IncrementalGzipDecompressor()()
+
+
+##
+
+
 @cached.function
 def _zero_crc() -> int:
     return zlib.crc32(b'')
@@ -75,14 +109,14 @@ class IncrementalGzipCompressor:
     def __init__(
             self,
             *,
-            compresslevel: int = COMPRESS_LEVEL_BEST,
+            level: int = COMPRESS_LEVEL_BEST,
             name: str | bytes | None = None,
             mtime: float | None = None,
     ) -> None:
         super().__init__()
 
         self._name = name or ''
-        self._compresslevel = compresslevel
+        self._level = level
         self._mtime = mtime
 
     def _write_gzip_header(self) -> ta.Generator[bytes, None, None]:
@@ -110,9 +144,9 @@ class IncrementalGzipCompressor:
             mtime = time.time()
         check.none((yield struct.pack('<L', int(mtime))))
 
-        if self._compresslevel == COMPRESS_LEVEL_BEST:
+        if self._level == COMPRESS_LEVEL_BEST:
             xfl = b'\002'
-        elif self._compresslevel == COMPRESS_LEVEL_FAST:
+        elif self._level == COMPRESS_LEVEL_FAST:
             xfl = b'\004'
         else:
             xfl = b'\000'
@@ -123,15 +157,14 @@ class IncrementalGzipCompressor:
         if fname:
             check.none((yield fname + b'\000'))
 
-    @lang.autostart
-    def __call__(self) -> IncrementalCompressor:
+    def __call__(self) -> BytesSteppedGenerator:
         crc = _zero_crc()
         size = 0
         offset = 0  # Current file offset for seek(), tell(), etc
         wrote_header = False
 
         compress = zlib.compressobj(
-            self._compresslevel,
+            self._level,
             zlib.DEFLATED,
             -zlib.MAX_WBITS,
             zlib.DEF_MEM_LEVEL,
@@ -251,7 +284,7 @@ class IncrementalGzipDecompressor:
         if c:
             rdr.prepend(c)
 
-    def __call__(self) -> IncrementalDecompressor:
+    def __call__(self) -> BytesSteppedReaderGenerator:
         rdr = PrependableBytesGeneratorReader()
 
         pos = 0  # Current offset in decompressed stream

omlish-0.0.0.dev142/omlish/io/compress/lz4.py

@@ -0,0 +1,77 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import check
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from .base import Compression
+from .base import IncrementalCompression
+
+
+if ta.TYPE_CHECKING:
+    import lz4.frame as lz4_frame
+else:
+    lz4_frame = lang.proxy_import('lz4.frame')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Lz4Compression(Compression, IncrementalCompression):
+    level: int = 0
+
+    block_size: int = 0
+    block_linked: bool = True
+    block_checksum: bool = False
+    content_checksum: bool = False
+    store_size: bool = True
+    auto_flush: bool = False
+
+    def compress(self, d: bytes) -> bytes:
+        return lz4_frame.compress(
+            d,
+            compression_level=self.level,
+            block_size=self.block_size,
+            content_checksum=self.content_checksum,
+            block_linked=self.block_linked,
+            store_size=self.store_size,
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return lz4_frame.decompress(
+            d,
+        )
+
+    @lang.autostart
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        with lz4_frame.LZ4FrameCompressor(
+                compression_level=self.level,
+                block_size=self.block_size,
+                block_linked=self.block_linked,
+                block_checksum=self.block_checksum,
+                content_checksum=self.content_checksum,
+                auto_flush=self.auto_flush,
+        ) as compressor:
+            started = False
+            while True:
+                i = check.isinstance((yield None), bytes)
+                if not started:
+                    yield compressor.begin()
+                    started = True
+                if not i:
+                    yield compressor.flush()
+                    yield b''
+                    return
+                if (o := compressor.compress(i)):
+                    yield o
+
+    @lang.autostart
+    def decompress_incremental(self) -> BytesSteppedGenerator[None]:
+        # lz4 lib does internal buffering so this is simply a BytesSteppedGenerator not a BytesSteppedReaderGenerator as
+        # it only yields None, accepting any number of bytes at a time.
+        with lz4_frame.LZ4FrameDecompressor() as decompressor:
+            while True:
+                i = check.isinstance((yield None), bytes)
+                if not i:
+                    yield b''
+                    return
+                if (o := decompressor.decompress(i)):
+                    yield o

omlish-0.0.0.dev142/omlish/io/compress/lzma.py

@@ -0,0 +1,67 @@
+import dataclasses as dc
+import functools
+import typing as ta
+
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from .adapters import CompressorObjectIncrementalAdapter
+from .adapters import DecompressorObjectIncrementalAdapter
+from .base import Compression
+from .base import IncrementalCompression
+
+
+if ta.TYPE_CHECKING:
+    import lzma
+else:
+    lzma = lang.proxy_import('lzma')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class LzmaCompression(Compression, IncrementalCompression):
+    format: int | None = None
+
+    check: int = -1
+    preset: int | None = None
+    filters: dict | None = None
+
+    mem_limit: int | None = None
+
+    def compress(self, d: bytes) -> bytes:
+        return lzma.compress(
+            d,
+            format=self.format if self.format is not None else lzma.FORMAT_XZ,
+            check=self.check,
+            preset=self.preset,
+            filters=self.filters,  # type: ignore[arg-type]
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return lzma.decompress(
+            d,
+            format=self.format if self.format is not None else lzma.FORMAT_AUTO,
+            memlimit=self.mem_limit,
+            filters=self.filters,  # type: ignore[arg-type]
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(CompressorObjectIncrementalAdapter(
+            functools.partial(  # type: ignore
+                lzma.LZMACompressor,
+                format=self.format if self.format is not None else lzma.FORMAT_XZ,
+                check=self.check,
+                preset=self.preset,
+                filters=self.filters,  # type: ignore[arg-type]
+            ),
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return DecompressorObjectIncrementalAdapter(
+            functools.partial(  # type: ignore
+                lzma.LZMADecompressor,
+                format=self.format if self.format is not None else lzma.FORMAT_AUTO,
+                memlimit=self.mem_limit,
+                filters=self.filters,  # type: ignore[arg-type]
+            ),
+            trailing_error=lzma.LZMAError,
+        )()

omlish-0.0.0.dev142/omlish/io/compress/snappy.py

@@ -0,0 +1,20 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import lang
+from .base import Compression
+
+
+if ta.TYPE_CHECKING:
+    import snappy
+else:
+    snappy = lang.proxy_import('snappy')
+
+
+@dc.dataclass(frozen=True)
+class SnappyCompression(Compression):
+    def compress(self, d: bytes) -> bytes:
+        return snappy.compress(d)
+
+    def decompress(self, d: bytes) -> bytes:
+        return snappy.decompress(d)

omlish-0.0.0.dev142/omlish/io/compress/zlib.py

@@ -0,0 +1,60 @@
+import dataclasses as dc
+import functools
+import typing as ta
+
+from ... import lang
+from ..generators import BytesSteppedGenerator
+from ..generators import BytesSteppedReaderGenerator
+from .adapters import CompressorObjectIncrementalAdapter
+from .adapters import DecompressorObjectIncrementalAdapter
+from .base import Compression
+from .base import IncrementalCompression
+
+
+if ta.TYPE_CHECKING:
+    import zlib
+else:
+    zlib = lang.proxy_import('zlib')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class ZlibCompression(Compression, IncrementalCompression):
+    level: int = 9
+
+    wbits: int | None = None
+    strategy: int | None = None
+    zdict: bytes | None = None
+
+    def compress(self, d: bytes) -> bytes:
+        return zlib.compress(
+            d,
+            self.level,
+            **(dict(wbits=self.wbits) if self.wbits is not None else {}),
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return zlib.decompress(
+            d,
+            **(dict(wbits=self.wbits) if self.wbits is not None else {}),
+        )
+
+    def compress_incremental(self) -> BytesSteppedGenerator[None]:
+        return lang.nextgen(CompressorObjectIncrementalAdapter(
+            functools.partial(
+                zlib.compressobj,  # type: ignore
+                self.level,
+                **(dict(wbits=self.wbits) if self.wbits is not None else {}),  # type: ignore[arg-type]
+                **(dict(strategy=self.strategy) if self.strategy is not None else {}),  # type: ignore[arg-type]
+                **(dict(zdict=self.zdict) if self.zdict is not None else {}),  # type: ignore[arg-type]
+            ),
+        )())
+
+    def decompress_incremental(self) -> BytesSteppedReaderGenerator[None]:
+        return DecompressorObjectIncrementalAdapter(
+            functools.partial(  # type: ignore
+                zlib.decompressobj,
+                **(dict(wbits=self.wbits) if self.wbits is not None else {}),  # type: ignore[arg-type]
+                **(dict(zdict=self.zdict) if self.zdict is not None else {}),  # type: ignore[arg-type]
+            ),
+            trailing_error=OSError,
+        )()

omlish-0.0.0.dev142/omlish/io/compress/zstd.py

@@ -0,0 +1,30 @@
+import dataclasses as dc
+import typing as ta
+
+from ... import lang
+from .base import Compression
+
+
+if ta.TYPE_CHECKING:
+    import zstandard
+else:
+    zstandard = lang.proxy_import('zstandard')
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class ZstdCompression(Compression):
+    level: int | None = None
+
+    max_output_size: int = 0
+
+    def compress(self, d: bytes) -> bytes:
+        return zstandard.compress(
+            d,
+            **(dict(level=self.level) if self.level is not None else {}),
+        )
+
+    def decompress(self, d: bytes) -> bytes:
+        return zstandard.decompress(
+            d,
+            max_output_size=self.max_output_size,
+        )