omlish 0.0.0.dev136__tar.gz → 0.0.0.dev137__tar.gz

{omlish-0.0.0.dev136/omlish.egg-info → omlish-0.0.0.dev137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev136
+Version: 0.0.0.dev137
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev136 → omlish-0.0.0.dev137}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev136'
-__revision__ = '345b1984bd3c96dc1b3ed4db8402e21d989f38ee'
+__version__ = '0.0.0.dev137'
+__revision__ = '895417d0442f51853415386ace94b1c41da0c990'
 
 
 #

{omlish-0.0.0.dev136 → omlish-0.0.0.dev137}/omlish/cached.py

@@ -10,9 +10,9 @@ builtins and thus not distinguish it from a normal property.
 
 """
 from .lang.cached import _CachedProperty  # noqa
-from .lang.cached import cached_function
+from .lang.cached import cached_function as _cached_function
 
-function = cached_function
+function = _cached_function
 
 property = property  # noqa
 

{omlish-0.0.0.dev136 → omlish-0.0.0.dev137}/omlish/collections/mappings.py

@@ -86,7 +86,7 @@ class DynamicTypeMap(ta.Generic[V]):
         self._items = list(items)
         self._weak = bool(weak)
 
-        self._cache: ta.MutableMapping[type, ta.Any] = weakref.WeakKeyDictionary()
+        self._cache: ta.MutableMapping[type, ta.Any] = weakref.WeakKeyDictionary() if weak else {}
 
     @property
     def items(self) -> ta.Sequence[V]:

{omlish-0.0.0.dev136 → omlish-0.0.0.dev137}/omlish/diag/_pycharm/runhack.py

@@ -3,7 +3,10 @@
 .venv/bin/python $(curl -LsSf https://raw.githubusercontent.com/wrmsr/omlish/master/omlish/diag/_pycharm/runhack.py -o $(mktemp) && echo "$_") install
 
 ==
+TODO:
+ - check for existing files - can't run regular dep entrypoints now
 
+==
 See:
  - https://github.com/JetBrains/intellij-community/blob/6400f70dde6f743e39a257a5a78cc51b644c835e/python/helpers/pycharm/_jb_pytest_runner.py
  - https://github.com/JetBrains/intellij-community/blob/5a4e584aa59767f2e7cf4bd377adfaaf7503984b/python/helpers/pycharm/_jb_runner_tools.py

{omlish-0.0.0.dev136 → omlish-0.0.0.dev137}/omlish/funcs/genmachine.py

@@ -12,7 +12,7 @@ import typing as ta
 I = ta.TypeVar('I')
 O = ta.TypeVar('O')
 
-# MachineGen: ta.TypeAlias = ta.Generator[ta.Iterable[O] | None, I, ta.Optional[MachineGen[I, O]]]
+# MachineGen: ta.TypeAlias = ta.Generator[ta.Iterable[O] | None, I | None, ta.Optional[MachineGen[I, O]]]
 MachineGen: ta.TypeAlias = ta.Generator[ta.Any, ta.Any, ta.Any]
 
 
@@ -93,8 +93,10 @@ class GenMachine(ta.Generic[I, O]):
         if self._gen is None:
             raise GenMachine.ClosedError
 
+        gi: I | None = i
         try:
-            while (o := self._gen.send(i)) is not None:
+            while (o := self._gen.send(gi)) is not None:
+                gi = None
                 yield from o
 
         except StopIteration as s:

omlish-0.0.0.dev137/omlish/io/compress/abc.py

@@ -0,0 +1,104 @@
+"""
+https://docs.python.org/3/library/bz2.html#bz2.BZ2Compressor
+https://docs.python.org/3/library/zlib.html#zlib.decompressobj
+https://docs.python.org/3/library/lzma.html#lzma.LZMADecompressor
+"""
+import abc
+
+
+##
+
+
+class Compressor(abc.ABC):
+    @abc.abstractmethod
+    def compress(self, data: bytes) -> bytes:
+        """
+        Provide data to the compressor object. Returns a chunk of compressed data if possible, or an empty byte string
+        otherwise.
+
+        When you have finished providing data to the compressor, call the flush() method to finish the compression
+        process.
+        """
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def flush(self) -> bytes:
+        """
+        Finish the compression process. Returns the compressed data left in internal buffers.
+
+        The compressor object may not be used after this method has been called.
+        """
+
+        raise NotImplementedError
+
+
+##
+
+
+class Decompressor(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def unused_data(self) -> bytes:
+        """
+        Data found after the end of the compressed stream.
+
+        If this attribute is accessed before the end of the stream has been reached, its value will be b''.
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def eof(self) -> bool:
+        """True if the end-of-stream marker has been reached."""
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decompress(self, data: bytes, *max_length: int) -> bytes:
+        """
+        Decompress data, returning a bytes object containing the uncompressed data corresponding to at least part of the
+        data in string. This data should be concatenated to the output produced by any preceding calls to the
+        decompress() method. Some of the input data may be preserved in internal buffers for later processing.
+
+        If the optional parameter max_length is non-zero then the return value will be no longer than max_length.
+        """
+
+        raise NotImplementedError
+
+
+class NeedsInputDecompressor(Decompressor):
+    """
+    Used by:
+     - bz2.BZ2Decompressor
+     - lzma.LZMADecompressor
+    """
+
+    @property
+    @abc.abstractmethod
+    def needs_input(self) -> bool:
+        """
+        False if the decompress() method can provide more decompressed data before requiring new uncompressed input.
+        """
+
+        raise NotImplementedError
+
+
+class UnconsumedTailDecompressor(Decompressor):
+    """
+    Used by:
+     - zlib.decompressobj
+    """
+
+    @property
+    @abc.abstractmethod
+    def unconsumed_tail(self) -> bytes:
+        """
+        A bytes object that contains any data that was not consumed by the last decompress() call because it exceeded
+        the limit for the uncompressed data buffer. This data has not yet been seen by the zlib machinery, so you must
+        feed it (possibly with further data concatenated to it) back to a subsequent decompress() method call in order
+        to get correct output.
+        """
+
+        raise NotImplementedError

omlish-0.0.0.dev137/omlish/io/compress/adapters.py

@@ -0,0 +1,147 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+# ~> https://github.com/python/cpython/blob/f19c50a4817ffebb26132182ed8cec6a72342cc0/Lib/_compression.py
+import typing as ta
+
+from ... import check
+from .abc import Compressor
+from .abc import NeedsInputDecompressor
+from .abc import UnconsumedTailDecompressor
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+##
+
+
+class CompressorIncrementalAdapter:
+    def __init__(
+            self,
+            factory: ta.Callable[..., Compressor],
+    ) -> None:
+        super().__init__()
+
+        self._factory = factory
+
+    def __call__(self) -> IncrementalCompressor:
+        compressor = self._factory()
+
+        while True:
+            data = check.isinstance((yield None), bytes)
+            if not data:
+                break
+
+            compressed = compressor.compress(data)
+            if compressed:
+                check.none((yield compressed))
+
+        if (fl := compressor.flush()):
+            check.none((yield fl))
+
+        check.none((yield b''))
+
+
+##
+
+
+class DecompressorIncrementalAdapter:
+    def __init__(
+            self,
+            factory: ta.Callable[..., NeedsInputDecompressor | UnconsumedTailDecompressor],
+            *,
+            trailing_error: type[BaseException] | tuple[type[BaseException], ...] = (),
+    ) -> None:
+        super().__init__()
+
+        self._factory = factory
+        self._trailing_error = trailing_error
+
+    def __call__(self) -> IncrementalDecompressor:
+        pos = 0
+
+        data = None  # Default if EOF is encountered
+
+        decompressor = self._factory()
+
+        while True:
+            # Depending on the input data, our call to the decompressor may not return any data. In this case, try again
+            # after reading another block.
+            while True:
+                if decompressor.eof:
+                    rawblock = decompressor.unused_data
+                    if not rawblock:
+                        rawblock = check.isinstance((yield None), bytes)
+                    if not rawblock:
+                        break
+
+                    # Continue to next stream.
+                    decompressor = self._factory()
+
+                    try:
+                        data = decompressor.decompress(rawblock)
+                    except self._trailing_error:
+                        # Trailing data isn't a valid compressed stream; ignore it.
+                        break
+
+                else:
+                    if hasattr(decompressor, 'needs_input'):
+                        if decompressor.needs_input:
+                            rawblock = check.isinstance((yield None), bytes)
+                            if not rawblock:
+                                raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+                        else:
+                            rawblock = b''
+
+                    elif hasattr(decompressor, 'unconsumed_tail'):
+                        if not (rawblock := decompressor.unconsumed_tail):
+                            rawblock = check.isinstance((yield None), bytes)
+                            if not rawblock:
+                                raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+                    else:
+                        raise TypeError(decompressor)
+
+                    data = decompressor.decompress(rawblock)
+
+                if data:
+                    break
+
+            if not data:
+                check.none((yield b''))
+                return
+
+            pos += len(data)
+            check.none((yield data))

omlish-0.0.0.dev137/omlish/io/compress/bz2.py

@@ -0,0 +1,41 @@
+import functools
+import typing as ta
+
+from ... import lang
+from .adapters import CompressorIncrementalAdapter
+from .adapters import DecompressorIncrementalAdapter
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import bz2
+else:
+    bz2 = lang.proxy_import('bz2')
+
+
+class IncrementalBz2Compressor:
+    def __init__(
+            self,
+            *,
+            compresslevel: int = 9,
+    ) -> None:
+        super().__init__()
+
+        self._compresslevel = compresslevel
+
+    def __call__(self) -> IncrementalCompressor:
+        return CompressorIncrementalAdapter(
+            functools.partial(
+                bz2.BZ2Compressor,  # type: ignore
+                self._compresslevel,
+            ),
+        )()
+
+
+class IncrementalBz2Decompressor:
+    def __call__(self) -> IncrementalDecompressor:
+        return DecompressorIncrementalAdapter(
+            bz2.BZ2Decompressor,  # type: ignore
+            trailing_error=OSError,
+        )()

omlish-0.0.0.dev137/omlish/io/compress/gzip.py

@@ -0,0 +1,301 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+import functools
+import os.path
+import struct
+import time
+import typing as ta
+
+from ... import cached
+from ... import check
+from ... import lang
+from ..generators import PrependableBytesGeneratorReader
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import gzip
+    import zlib
+else:
+    gzip = lang.proxy_import('gzip')
+    zlib = lang.proxy_import('zlib')
+
+
+##
+
+
+COMPRESS_LEVEL_FAST = 1
+COMPRESS_LEVEL_TRADEOFF = 6
+COMPRESS_LEVEL_BEST = 9
+
+
+@cached.function
+def _zero_crc() -> int:
+    return zlib.crc32(b'')
+
+
+##
+
+
+class IncrementalGzipCompressor:
+    def __init__(
+            self,
+            *,
+            compresslevel: int = COMPRESS_LEVEL_BEST,
+            name: str | bytes | None = None,
+            mtime: float | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._name = name or ''
+        self._compresslevel = compresslevel
+        self._mtime = mtime
+
+    def _write_gzip_header(self) -> ta.Generator[bytes, None, None]:
+        check.none((yield b'\037\213'))  # magic header
+        check.none((yield b'\010'))  # compression method
+
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not include filenames that cannot be represented that
+            # way.
+            fname = os.path.basename(self._name)
+            if not isinstance(fname, bytes):
+                fname = fname.encode('latin-1')
+            if fname.endswith(b'.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = b''
+
+        flags = 0
+        if fname:
+            flags = gzip.FNAME
+        check.none((yield chr(flags).encode('latin-1')))
+
+        mtime = self._mtime
+        if mtime is None:
+            mtime = time.time()
+        check.none((yield struct.pack('<L', int(mtime))))
+
+        if self._compresslevel == COMPRESS_LEVEL_BEST:
+            xfl = b'\002'
+        elif self._compresslevel == COMPRESS_LEVEL_FAST:
+            xfl = b'\004'
+        else:
+            xfl = b'\000'
+        check.none((yield xfl))
+
+        check.none((yield b'\377'))
+
+        if fname:
+            check.none((yield fname + b'\000'))
+
+    def __call__(self) -> IncrementalCompressor:
+        crc = _zero_crc()
+        size = 0
+        offset = 0  # Current file offset for seek(), tell(), etc
+
+        compress = zlib.compressobj(
+            self._compresslevel,
+            zlib.DEFLATED,
+            -zlib.MAX_WBITS,
+            zlib.DEF_MEM_LEVEL,
+            0,
+        )
+
+        yield from self._write_gzip_header()
+
+        while True:
+            data: ta.Any = check.isinstance((yield None), bytes)
+            if not data:
+                break
+
+            # Called by our self._buffer underlying BufferedWriterDelegate.
+            if isinstance(data, (bytes, bytearray)):
+                length = len(data)
+            else:
+                # accept any data that supports the buffer protocol
+                data = memoryview(data)
+                length = data.nbytes
+
+            if length > 0:
+                if (fl := compress.compress(data)):
+                    check.none((yield fl))
+                size += length
+                crc = zlib.crc32(data, crc)
+                offset += length
+
+        if (fl := compress.flush()):
+            check.none((yield fl))
+
+        yield struct.pack('<L', crc)
+        # size may exceed 2 GiB, or even 4 GiB
+        yield struct.pack('<L', size & 0xffffffff)
+
+        yield b''
+
+
+##
+
+
+class IncrementalGzipDecompressor:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._factory = functools.partial(
+            zlib.decompressobj,
+            wbits=-zlib.MAX_WBITS,
+        )
+
+    def _read_gzip_header(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+    ) -> ta.Generator[int | None, bytes, int | None]:
+        magic = yield from rdr.read(2)
+        if magic == b'':
+            return None
+
+        if magic != b'\037\213':
+            raise gzip.BadGzipFile(f'Not a gzipped file ({magic!r})')
+
+        buf = yield from rdr.read(8)
+        method, flag, last_mtime = struct.unpack('<BBIxx', buf)
+        if method != 8:
+            raise gzip.BadGzipFile('Unknown compression method')
+
+        if flag & gzip.FEXTRA:
+            # Read & discard the extra field, if present
+            buf = yield from rdr.read(2)
+            extra_len, = struct.unpack('<H', buf)
+            if extra_len:
+                yield from rdr.read(extra_len)
+
+        if flag & gzip.FNAME:
+            # Read and discard a null-terminated string containing the filename
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FCOMMENT:
+            # Read and discard a null-terminated string containing a comment
+            while True:
+                s = yield from rdr.read(1)
+                if not s or s == b'\000':
+                    break
+
+        if flag & gzip.FHCRC:
+            yield from rdr.read(2)  # Read & discard the 16-bit header CRC
+
+        return last_mtime
+
+    def _read_eof(
+            self,
+            rdr: PrependableBytesGeneratorReader,
+            crc: int,
+            stream_size: int,
+    ) -> ta.Generator[int | None, bytes, None]:
+        # We've read to the end of the file.
+        # We check that the computed CRC and size of the uncompressed data matches the stored values. Note that the size
+        # stored is the true file size mod 2**32.
+        buf = yield from rdr.read(8)
+        crc32, isize = struct.unpack('<II', buf)
+        if crc32 != crc:
+            raise gzip.BadGzipFile(f'CRC check failed {hex(crc32)} != {hex(crc)}')
+        elif isize != (stream_size & 0xffffffff):
+            raise gzip.BadGzipFile('Incorrect length of data produced')
+
+        # Gzip files can be padded with zeroes and still have archives. Consume all zero bytes and set the file position
+        # to the first non-zero byte. See http://www.gzip.org/#faq8
+        c = b'\0'
+        while c == b'\0':
+            c = yield from rdr.read(1)
+        if c:
+            rdr.prepend(c)
+
+    def __call__(self) -> IncrementalDecompressor:
+        rdr = PrependableBytesGeneratorReader()
+
+        pos = 0  # Current offset in decompressed stream
+
+        crc = _zero_crc()
+        stream_size = 0  # Decompressed size of unconcatenated stream
+        new_member = True
+
+        decompressor = self._factory()
+
+        while True:
+            # For certain input data, a single call to decompress() may not return any data. In this case, retry until
+            # we get some data or reach EOF.
+            while True:
+                if decompressor.eof:
+                    # Ending case: we've come to the end of a member in the file, so finish up this member, and read a
+                    # new gzip header. Check the CRC and file size, and set the flag so we read a new member
+                    yield from self._read_eof(rdr, crc, stream_size)
+                    new_member = True
+                    decompressor = self._factory()
+
+                if new_member:
+                    # If the _new_member flag is set, we have to jump to the next member, if there is one.
+                    crc = _zero_crc()
+                    stream_size = 0  # Decompressed size of unconcatenated stream
+                    last_mtime = yield from self._read_gzip_header(rdr)
+                    if not last_mtime:
+                        check.none((yield b''))
+                        return
+                    new_member = False
+
+                # Read a chunk of data from the file
+                if not decompressor.unconsumed_tail:
+                    buf = yield from rdr.read(None)
+                    uncompress = decompressor.decompress(buf)
+                else:
+                    uncompress = decompressor.decompress(b'')
+
+                if decompressor.unused_data != b'':
+                    # Prepend the already read bytes to the fileobj so they can be seen by _read_eof() and
+                    # _read_gzip_header()
+                    rdr.prepend(decompressor.unused_data)
+
+                if uncompress != b'':
+                    break
+                if buf == b'':  # noqa
+                    raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+            crc = zlib.crc32(uncompress, crc)
+            stream_size += len(uncompress)
+            pos += len(uncompress)
+            check.none((yield uncompress))