omlish 0.0.0.dev135__tar.gz → 0.0.0.dev137__tar.gz

{omlish-0.0.0.dev135/omlish.egg-info → omlish-0.0.0.dev137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev135
+Version: 0.0.0.dev137
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/.manifests.json

@@ -23,18 +23,6 @@
       }
     }
   },
-  {
-    "module": ".formats.json.cli.__main__",
-    "attr": "_CLI_MODULE",
-    "file": "omlish/formats/json/cli/__main__.py",
-    "line": 1,
-    "value": {
-      "$omdev.cli.types.CliModule": {
-        "cmd_name": "json",
-        "mod_name": "omlish.formats.json.cli.__main__"
-      }
-    }
-  },
   {
     "module": ".secrets.pwgen",
     "attr": "_CLI_MODULE",

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev135'
-__revision__ = 'bf47c29e2fe6b065c99ce565d89f133b798895d9'
+__version__ = '0.0.0.dev137'
+__revision__ = '895417d0442f51853415386ace94b1c41da0c990'
 
 
 #

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/cached.py

@@ -10,9 +10,9 @@ builtins and thus not distinguish it from a normal property.
 
 """
 from .lang.cached import _CachedProperty  # noqa
-from .lang.cached import cached_function
+from .lang.cached import cached_function as _cached_function
 
-function = cached_function
+function = _cached_function
 
 property = property  # noqa
 

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/collections/mappings.py

@@ -86,7 +86,7 @@ class DynamicTypeMap(ta.Generic[V]):
         self._items = list(items)
         self._weak = bool(weak)
 
-        self._cache: ta.MutableMapping[type, ta.Any] = weakref.WeakKeyDictionary()
+        self._cache: ta.MutableMapping[type, ta.Any] = weakref.WeakKeyDictionary() if weak else {}
 
     @property
     def items(self) -> ta.Sequence[V]:

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/diag/_pycharm/runhack.py

@@ -3,7 +3,10 @@
 .venv/bin/python $(curl -LsSf https://raw.githubusercontent.com/wrmsr/omlish/master/omlish/diag/_pycharm/runhack.py -o $(mktemp) && echo "$_") install
 
 ==
+TODO:
+ - check for existing files - can't run regular dep entrypoints now
 
+==
 See:
  - https://github.com/JetBrains/intellij-community/blob/6400f70dde6f743e39a257a5a78cc51b644c835e/python/helpers/pycharm/_jb_pytest_runner.py
  - https://github.com/JetBrains/intellij-community/blob/5a4e584aa59767f2e7cf4bd377adfaaf7503984b/python/helpers/pycharm/_jb_runner_tools.py

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/formats/json/stream/lex.py

@@ -11,7 +11,7 @@ import re
 import typing as ta
 
 from .... import check
-from ....genmachine import GenMachine
+from ....funcs.genmachine import GenMachine
 
 
 ##

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/formats/json/stream/parse.py

@@ -1,7 +1,7 @@
 import typing as ta
 
 from .... import lang
-from ....genmachine import GenMachine
+from ....funcs.genmachine import GenMachine
 from .lex import SCALAR_VALUE_TYPES
 from .lex import VALUE_TOKEN_KINDS
 from .lex import ScalarValue

{omlish-0.0.0.dev135/omlish → omlish-0.0.0.dev137/omlish/funcs}/genmachine.py

@@ -12,7 +12,7 @@ import typing as ta
 I = ta.TypeVar('I')
 O = ta.TypeVar('O')
 
-# MachineGen: ta.TypeAlias = ta.Generator[ta.Iterable[O] | None, I, ta.Optional[MachineGen[I, O]]]
+# MachineGen: ta.TypeAlias = ta.Generator[ta.Iterable[O] | None, I | None, ta.Optional[MachineGen[I, O]]]
 MachineGen: ta.TypeAlias = ta.Generator[ta.Any, ta.Any, ta.Any]
 
 
@@ -93,8 +93,10 @@ class GenMachine(ta.Generic[I, O]):
         if self._gen is None:
             raise GenMachine.ClosedError
 
+        gi: I | None = i
         try:
-            while (o := self._gen.send(i)) is not None:
+            while (o := self._gen.send(gi)) is not None:
+                gi = None
                 yield from o
 
         except StopIteration as s:

omlish-0.0.0.dev135/omlish/matchfns.py → omlish-0.0.0.dev137/omlish/funcs/match.py

@@ -8,7 +8,7 @@ import abc
 import dataclasses as dc
 import typing as ta
 
-from . import lang
+from .. import lang
 
 
 T = ta.TypeVar('T')

omlish-0.0.0.dev135/omlish/fnpairs.py → omlish-0.0.0.dev137/omlish/funcs/pairs.py

@@ -20,7 +20,7 @@ import codecs
 import dataclasses as dc
 import typing as ta
 
-from . import lang
+from .. import lang
 
 
 if ta.TYPE_CHECKING:
@@ -39,7 +39,7 @@ if ta.TYPE_CHECKING:
     import yaml as _yaml
     import zstandard as _zstandard
 
-    from .formats import json as _json
+    from ..formats import json as _json
 
 else:
     _bz2 = lang.proxy_import('bz2')
@@ -57,7 +57,7 @@ else:
     _yaml = lang.proxy_import('yaml')
     _zstandard = lang.proxy_import('zstandard')
 
-    _json = lang.proxy_import('.formats.json', __package__)
+    _json = lang.proxy_import('..formats.json', __package__)
 
 
 ##

{omlish-0.0.0.dev135 → omlish-0.0.0.dev137}/omlish/http/sessions.py

@@ -8,9 +8,9 @@ import time
 import typing as ta
 import zlib
 
-from .. import fnpairs as fpa
 from .. import lang
 from .. import secrets as sec
+from ..funcs import pairs as fpa
 from .cookies import dump_cookie
 from .cookies import parse_cookie
 from .json import JSON_TAGGER

omlish-0.0.0.dev137/omlish/io/compress/abc.py

@@ -0,0 +1,104 @@
+"""
+https://docs.python.org/3/library/bz2.html#bz2.BZ2Compressor
+https://docs.python.org/3/library/zlib.html#zlib.decompressobj
+https://docs.python.org/3/library/lzma.html#lzma.LZMADecompressor
+"""
+import abc
+
+
+##
+
+
+class Compressor(abc.ABC):
+    @abc.abstractmethod
+    def compress(self, data: bytes) -> bytes:
+        """
+        Provide data to the compressor object. Returns a chunk of compressed data if possible, or an empty byte string
+        otherwise.
+
+        When you have finished providing data to the compressor, call the flush() method to finish the compression
+        process.
+        """
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def flush(self) -> bytes:
+        """
+        Finish the compression process. Returns the compressed data left in internal buffers.
+
+        The compressor object may not be used after this method has been called.
+        """
+
+        raise NotImplementedError
+
+
+##
+
+
+class Decompressor(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def unused_data(self) -> bytes:
+        """
+        Data found after the end of the compressed stream.
+
+        If this attribute is accessed before the end of the stream has been reached, its value will be b''.
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def eof(self) -> bool:
+        """True if the end-of-stream marker has been reached."""
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decompress(self, data: bytes, *max_length: int) -> bytes:
+        """
+        Decompress data, returning a bytes object containing the uncompressed data corresponding to at least part of the
+        data in string. This data should be concatenated to the output produced by any preceding calls to the
+        decompress() method. Some of the input data may be preserved in internal buffers for later processing.
+
+        If the optional parameter max_length is non-zero then the return value will be no longer than max_length.
+        """
+
+        raise NotImplementedError
+
+
+class NeedsInputDecompressor(Decompressor):
+    """
+    Used by:
+     - bz2.BZ2Decompressor
+     - lzma.LZMADecompressor
+    """
+
+    @property
+    @abc.abstractmethod
+    def needs_input(self) -> bool:
+        """
+        False if the decompress() method can provide more decompressed data before requiring new uncompressed input.
+        """
+
+        raise NotImplementedError
+
+
+class UnconsumedTailDecompressor(Decompressor):
+    """
+    Used by:
+     - zlib.decompressobj
+    """
+
+    @property
+    @abc.abstractmethod
+    def unconsumed_tail(self) -> bytes:
+        """
+        A bytes object that contains any data that was not consumed by the last decompress() call because it exceeded
+        the limit for the uncompressed data buffer. This data has not yet been seen by the zlib machinery, so you must
+        feed it (possibly with further data concatenated to it) back to a subsequent decompress() method call in order
+        to get correct output.
+        """
+
+        raise NotImplementedError

omlish-0.0.0.dev137/omlish/io/compress/adapters.py

@@ -0,0 +1,147 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+# ~> https://github.com/python/cpython/blob/f19c50a4817ffebb26132182ed8cec6a72342cc0/Lib/_compression.py
+import typing as ta
+
+from ... import check
+from .abc import Compressor
+from .abc import NeedsInputDecompressor
+from .abc import UnconsumedTailDecompressor
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+##
+
+
+class CompressorIncrementalAdapter:
+    def __init__(
+            self,
+            factory: ta.Callable[..., Compressor],
+    ) -> None:
+        super().__init__()
+
+        self._factory = factory
+
+    def __call__(self) -> IncrementalCompressor:
+        compressor = self._factory()
+
+        while True:
+            data = check.isinstance((yield None), bytes)
+            if not data:
+                break
+
+            compressed = compressor.compress(data)
+            if compressed:
+                check.none((yield compressed))
+
+        if (fl := compressor.flush()):
+            check.none((yield fl))
+
+        check.none((yield b''))
+
+
+##
+
+
+class DecompressorIncrementalAdapter:
+    def __init__(
+            self,
+            factory: ta.Callable[..., NeedsInputDecompressor | UnconsumedTailDecompressor],
+            *,
+            trailing_error: type[BaseException] | tuple[type[BaseException], ...] = (),
+    ) -> None:
+        super().__init__()
+
+        self._factory = factory
+        self._trailing_error = trailing_error
+
+    def __call__(self) -> IncrementalDecompressor:
+        pos = 0
+
+        data = None  # Default if EOF is encountered
+
+        decompressor = self._factory()
+
+        while True:
+            # Depending on the input data, our call to the decompressor may not return any data. In this case, try again
+            # after reading another block.
+            while True:
+                if decompressor.eof:
+                    rawblock = decompressor.unused_data
+                    if not rawblock:
+                        rawblock = check.isinstance((yield None), bytes)
+                    if not rawblock:
+                        break
+
+                    # Continue to next stream.
+                    decompressor = self._factory()
+
+                    try:
+                        data = decompressor.decompress(rawblock)
+                    except self._trailing_error:
+                        # Trailing data isn't a valid compressed stream; ignore it.
+                        break
+
+                else:
+                    if hasattr(decompressor, 'needs_input'):
+                        if decompressor.needs_input:
+                            rawblock = check.isinstance((yield None), bytes)
+                            if not rawblock:
+                                raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+                        else:
+                            rawblock = b''
+
+                    elif hasattr(decompressor, 'unconsumed_tail'):
+                        if not (rawblock := decompressor.unconsumed_tail):
+                            rawblock = check.isinstance((yield None), bytes)
+                            if not rawblock:
+                                raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+                    else:
+                        raise TypeError(decompressor)
+
+                    data = decompressor.decompress(rawblock)
+
+                if data:
+                    break
+
+            if not data:
+                check.none((yield b''))
+                return
+
+            pos += len(data)
+            check.none((yield data))

omlish-0.0.0.dev137/omlish/io/compress/bz2.py

@@ -0,0 +1,41 @@
+import functools
+import typing as ta
+
+from ... import lang
+from .adapters import CompressorIncrementalAdapter
+from .adapters import DecompressorIncrementalAdapter
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import bz2
+else:
+    bz2 = lang.proxy_import('bz2')
+
+
+class IncrementalBz2Compressor:
+    def __init__(
+            self,
+            *,
+            compresslevel: int = 9,
+    ) -> None:
+        super().__init__()
+
+        self._compresslevel = compresslevel
+
+    def __call__(self) -> IncrementalCompressor:
+        return CompressorIncrementalAdapter(
+            functools.partial(
+                bz2.BZ2Compressor,  # type: ignore
+                self._compresslevel,
+            ),
+        )()
+
+
+class IncrementalBz2Decompressor:
+    def __call__(self) -> IncrementalDecompressor:
+        return DecompressorIncrementalAdapter(
+            bz2.BZ2Decompressor,  # type: ignore
+            trailing_error=OSError,
+        )()