omlish 0.0.0.dev136__tar.gz → 0.0.0.dev138__tar.gz

{omlish-0.0.0.dev136/omlish.egg-info → omlish-0.0.0.dev138}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev136
+Version: 0.0.0.dev138
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev136'
-__revision__ = '345b1984bd3c96dc1b3ed4db8402e21d989f38ee'
+__version__ = '0.0.0.dev138'
+__revision__ = 'd7ba57140bc56bdfb990684087dcda294928835f'
 
 
 #

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/cached.py

@@ -10,9 +10,9 @@ builtins and thus not distinguish it from a normal property.
 
 """
 from .lang.cached import _CachedProperty  # noqa
-from .lang.cached import cached_function
+from .lang.cached import cached_function as _cached_function
 
-function = cached_function
+function = _cached_function
 
 property = property  # noqa
 

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/collections/mappings.py

@@ -86,7 +86,7 @@ class DynamicTypeMap(ta.Generic[V]):
         self._items = list(items)
         self._weak = bool(weak)
 
-        self._cache: ta.MutableMapping[type, ta.Any] = weakref.WeakKeyDictionary()
+        self._cache: ta.MutableMapping[type, ta.Any] = weakref.WeakKeyDictionary() if weak else {}
 
     @property
     def items(self) -> ta.Sequence[V]:

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/configs/flattening.py

@@ -130,7 +130,7 @@ class Flattening:
                             .split(self._index_close + self._index_open):
                         yield int(p)
                 else:
-                    check.state(')' not in part)
+                    check.state(self._index_close not in part)
                     yield part
 
         for fk, v in flattened.items():

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/diag/_pycharm/runhack.py

@@ -3,7 +3,10 @@
 .venv/bin/python $(curl -LsSf https://raw.githubusercontent.com/wrmsr/omlish/master/omlish/diag/_pycharm/runhack.py -o $(mktemp) && echo "$_") install
 
 ==
+TODO:
+ - check for existing files - can't run regular dep entrypoints now
 
+==
 See:
  - https://github.com/JetBrains/intellij-community/blob/6400f70dde6f743e39a257a5a78cc51b644c835e/python/helpers/pycharm/_jb_pytest_runner.py
  - https://github.com/JetBrains/intellij-community/blob/5a4e584aa59767f2e7cf4bd377adfaaf7503984b/python/helpers/pycharm/_jb_runner_tools.py

omlish-0.0.0.dev138/omlish/formats/json/stream/errors.py

@@ -0,0 +1,2 @@
+class JsonStreamError(Exception):
+    pass

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/formats/json/stream/lex.py

@@ -3,6 +3,7 @@ TODO:
  - max buf size
  - max recursion depth
  - mark start pos of tokens, currently returning end
+ - _do_string inner loop optimization somehow
 """
 import dataclasses as dc
 import io
@@ -12,6 +13,7 @@ import typing as ta
 
 from .... import check
 from ....funcs.genmachine import GenMachine
+from .errors import JsonStreamError
 
 
 ##
@@ -95,7 +97,7 @@ CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
 
 
 @dc.dataclass()
-class JsonLexError(Exception):
+class JsonStreamLexError(JsonStreamError):
     message: str
 
     pos: Position
@@ -160,8 +162,8 @@ class JsonStreamLexer(GenMachine[str, Token]):
         self._buf.truncate()
         return raw
 
-    def _raise(self, msg: str) -> ta.NoReturn:
-        raise JsonLexError(msg, self.pos)
+    def _raise(self, msg: str, src: Exception | None = None) -> ta.NoReturn:
+        raise JsonStreamLexError(msg, self.pos) from src
 
     def _do_main(self):
         while True:
@@ -202,7 +204,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
                 self._raise('Unexpected end of input')
 
             if not c:
-                raise NotImplementedError
+                self._raise(f'Unterminated string literal: {self._buf.getvalue()}')
 
             self._buf.write(c)
             if c == '"' and last != '\\':
@@ -210,7 +212,11 @@ class JsonStreamLexer(GenMachine[str, Token]):
             last = c
 
         raw = self._flip_buf()
-        sv = json.loads(raw)
+        try:
+            sv = json.loads(raw)
+        except json.JSONDecodeError as e:
+            self._raise(f'Invalid string literal: {raw!r}', e)
+
         yield self._make_tok('STRING', sv, raw, pos)
 
         return self._do_main()

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/formats/json/stream/parse.py

@@ -1,9 +1,12 @@
+import dataclasses as dc
 import typing as ta
 
 from .... import lang
 from ....funcs.genmachine import GenMachine
+from .errors import JsonStreamError
 from .lex import SCALAR_VALUE_TYPES
 from .lex import VALUE_TOKEN_KINDS
+from .lex import Position
 from .lex import ScalarValue
 from .lex import Token
 
@@ -79,6 +82,13 @@ def yield_parser_events(obj: ta.Any) -> ta.Generator[JsonStreamParserEvent, None
 ##
 
 
+@dc.dataclass()
+class JsonStreamParseError(JsonStreamError):
+    message: str
+
+    pos: Position | None = None
+
+
 class JsonStreamObject(list):
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}({super().__repr__()})'
@@ -100,29 +110,29 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         if tt == 'KEY':
             self._stack.pop()
             if not self._stack:
-                raise self.StateError
+                raise JsonStreamParseError('Unexpected key')
 
             tt2 = self._stack[-1]
             if tt2 == 'OBJECT':
                 return ((v,), self._do_after_pair())
 
             else:
-                raise self.StateError
+                raise JsonStreamParseError('Unexpected key')
 
         elif tt == 'ARRAY':
             return ((v,), self._do_after_element())
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError(f'Unexpected value: {v!r}')
 
     #
 
-    def _do_value(self):
+    def _do_value(self, *, must_be_present: bool = False):
         try:
             tok = yield None
         except GeneratorExit:
             if self._stack:
-                raise self.StateError from None
+                raise JsonStreamParseError('Expected value') from None
             else:
                 raise
 
@@ -141,13 +151,16 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             yield y
             return r
 
+        elif must_be_present:
+            raise JsonStreamParseError('Expected value', tok.pos)
+
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()
             yield y
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected value', tok.pos)
 
     #
 
@@ -157,19 +170,19 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
 
     def _emit_end_object(self):
         if not self._stack:
-            raise self.StateError
+            raise JsonStreamParseError('Unexpected end object')
 
         tt = self._stack.pop()
         if tt != 'OBJECT':
-            raise self.StateError
+            raise JsonStreamParseError('Unexpected end object')
 
         return self._emit_event(EndObject)
 
-    def _do_object_body(self):
+    def _do_object_body(self, *, must_be_present: bool = False):
         try:
             tok = yield None
         except GeneratorExit:
-            raise self.StateError from None
+            raise JsonStreamParseError('Expected object body') from None
 
         if tok.kind == 'STRING':
             k = tok.value
@@ -177,30 +190,33 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             try:
                 tok = yield None
             except GeneratorExit:
-                raise self.StateError from None
+                raise JsonStreamParseError('Expected key') from None
             if tok.kind != 'COLON':
-                raise self.StateError
+                raise JsonStreamParseError('Expected colon', tok.pos)
 
             yield (Key(k),)
             self._stack.append('KEY')
             return self._do_value()
 
+        elif must_be_present:
+            raise JsonStreamParseError('Expected value', tok.pos)
+
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
             yield y
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected value', tok.pos)
 
     def _do_after_pair(self):
         try:
             tok = yield None
         except GeneratorExit:
-            raise self.StateError from None
+            raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_object_body()
+            return self._do_object_body(must_be_present=True)
 
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
@@ -208,7 +224,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected continuation', tok.pos)
 
     #
 
@@ -218,11 +234,11 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
 
     def _emit_end_array(self):
         if not self._stack:
-            raise self.StateError
+            raise JsonStreamParseError('Expected end array')
 
         tt = self._stack.pop()
         if tt != 'ARRAY':
-            raise self.StateError
+            raise JsonStreamParseError('Unexpected end array')
 
         return self._emit_event(EndArray)
 
@@ -230,10 +246,10 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         try:
             tok = yield None
         except GeneratorExit:
-            raise self.StateError from None
+            raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_value()
+            return self._do_value(must_be_present=True)
 
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()
@@ -241,4 +257,4 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected continuation', tok.pos)

{omlish-0.0.0.dev136 → omlish-0.0.0.dev138}/omlish/funcs/genmachine.py

@@ -12,7 +12,7 @@ import typing as ta
 I = ta.TypeVar('I')
 O = ta.TypeVar('O')
 
-# MachineGen: ta.TypeAlias = ta.Generator[ta.Iterable[O] | None, I, ta.Optional[MachineGen[I, O]]]
+# MachineGen: ta.TypeAlias = ta.Generator[ta.Iterable[O] | None, I | None, ta.Optional[MachineGen[I, O]]]
 MachineGen: ta.TypeAlias = ta.Generator[ta.Any, ta.Any, ta.Any]
 
 
@@ -67,8 +67,7 @@ class GenMachine(ta.Generic[I, O]):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is None:
-            self.close()
+        self.close()
 
     #
 
@@ -93,8 +92,10 @@ class GenMachine(ta.Generic[I, O]):
         if self._gen is None:
             raise GenMachine.ClosedError
 
+        gi: I | None = i
         try:
-            while (o := self._gen.send(i)) is not None:
+            while (o := self._gen.send(gi)) is not None:
+                gi = None
                 yield from o
 
         except StopIteration as s:

omlish-0.0.0.dev138/omlish/io/compress/abc.py

@@ -0,0 +1,104 @@
+"""
+https://docs.python.org/3/library/bz2.html#bz2.BZ2Compressor
+https://docs.python.org/3/library/zlib.html#zlib.decompressobj
+https://docs.python.org/3/library/lzma.html#lzma.LZMADecompressor
+"""
+import abc
+
+
+##
+
+
+class Compressor(abc.ABC):
+    @abc.abstractmethod
+    def compress(self, data: bytes) -> bytes:
+        """
+        Provide data to the compressor object. Returns a chunk of compressed data if possible, or an empty byte string
+        otherwise.
+
+        When you have finished providing data to the compressor, call the flush() method to finish the compression
+        process.
+        """
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def flush(self) -> bytes:
+        """
+        Finish the compression process. Returns the compressed data left in internal buffers.
+
+        The compressor object may not be used after this method has been called.
+        """
+
+        raise NotImplementedError
+
+
+##
+
+
+class Decompressor(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def unused_data(self) -> bytes:
+        """
+        Data found after the end of the compressed stream.
+
+        If this attribute is accessed before the end of the stream has been reached, its value will be b''.
+        """
+
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def eof(self) -> bool:
+        """True if the end-of-stream marker has been reached."""
+
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decompress(self, data: bytes, *max_length: int) -> bytes:
+        """
+        Decompress data, returning a bytes object containing the uncompressed data corresponding to at least part of the
+        data in string. This data should be concatenated to the output produced by any preceding calls to the
+        decompress() method. Some of the input data may be preserved in internal buffers for later processing.
+
+        If the optional parameter max_length is non-zero then the return value will be no longer than max_length.
+        """
+
+        raise NotImplementedError
+
+
+class NeedsInputDecompressor(Decompressor):
+    """
+    Used by:
+     - bz2.BZ2Decompressor
+     - lzma.LZMADecompressor
+    """
+
+    @property
+    @abc.abstractmethod
+    def needs_input(self) -> bool:
+        """
+        False if the decompress() method can provide more decompressed data before requiring new uncompressed input.
+        """
+
+        raise NotImplementedError
+
+
+class UnconsumedTailDecompressor(Decompressor):
+    """
+    Used by:
+     - zlib.decompressobj
+    """
+
+    @property
+    @abc.abstractmethod
+    def unconsumed_tail(self) -> bytes:
+        """
+        A bytes object that contains any data that was not consumed by the last decompress() call because it exceeded
+        the limit for the uncompressed data buffer. This data has not yet been seen by the zlib machinery, so you must
+        feed it (possibly with further data concatenated to it) back to a subsequent decompress() method call in order
+        to get correct output.
+        """
+
+        raise NotImplementedError

omlish-0.0.0.dev138/omlish/io/compress/adapters.py

@@ -0,0 +1,147 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights Reserved" are retained in Python
+# alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+# ~> https://github.com/python/cpython/blob/f19c50a4817ffebb26132182ed8cec6a72342cc0/Lib/_compression.py
+import typing as ta
+
+from ... import check
+from .abc import Compressor
+from .abc import NeedsInputDecompressor
+from .abc import UnconsumedTailDecompressor
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+##
+
+
+class CompressorIncrementalAdapter:
+    def __init__(
+            self,
+            factory: ta.Callable[..., Compressor],
+    ) -> None:
+        super().__init__()
+
+        self._factory = factory
+
+    def __call__(self) -> IncrementalCompressor:
+        compressor = self._factory()
+
+        while True:
+            data = check.isinstance((yield None), bytes)
+            if not data:
+                break
+
+            compressed = compressor.compress(data)
+            if compressed:
+                check.none((yield compressed))
+
+        if (fl := compressor.flush()):
+            check.none((yield fl))
+
+        check.none((yield b''))
+
+
+##
+
+
+class DecompressorIncrementalAdapter:
+    def __init__(
+            self,
+            factory: ta.Callable[..., NeedsInputDecompressor | UnconsumedTailDecompressor],
+            *,
+            trailing_error: type[BaseException] | tuple[type[BaseException], ...] = (),
+    ) -> None:
+        super().__init__()
+
+        self._factory = factory
+        self._trailing_error = trailing_error
+
+    def __call__(self) -> IncrementalDecompressor:
+        pos = 0
+
+        data = None  # Default if EOF is encountered
+
+        decompressor = self._factory()
+
+        while True:
+            # Depending on the input data, our call to the decompressor may not return any data. In this case, try again
+            # after reading another block.
+            while True:
+                if decompressor.eof:
+                    rawblock = decompressor.unused_data
+                    if not rawblock:
+                        rawblock = check.isinstance((yield None), bytes)
+                    if not rawblock:
+                        break
+
+                    # Continue to next stream.
+                    decompressor = self._factory()
+
+                    try:
+                        data = decompressor.decompress(rawblock)
+                    except self._trailing_error:
+                        # Trailing data isn't a valid compressed stream; ignore it.
+                        break
+
+                else:
+                    if hasattr(decompressor, 'needs_input'):
+                        if decompressor.needs_input:
+                            rawblock = check.isinstance((yield None), bytes)
+                            if not rawblock:
+                                raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+                        else:
+                            rawblock = b''
+
+                    elif hasattr(decompressor, 'unconsumed_tail'):
+                        if not (rawblock := decompressor.unconsumed_tail):
+                            rawblock = check.isinstance((yield None), bytes)
+                            if not rawblock:
+                                raise EOFError('Compressed file ended before the end-of-stream marker was reached')
+
+                    else:
+                        raise TypeError(decompressor)
+
+                    data = decompressor.decompress(rawblock)
+
+                if data:
+                    break
+
+            if not data:
+                check.none((yield b''))
+                return
+
+            pos += len(data)
+            check.none((yield data))

omlish-0.0.0.dev138/omlish/io/compress/bz2.py

@@ -0,0 +1,42 @@
+import functools
+import typing as ta
+
+from ... import lang
+from .adapters import CompressorIncrementalAdapter
+from .adapters import DecompressorIncrementalAdapter
+from .types import IncrementalCompressor
+from .types import IncrementalDecompressor
+
+
+if ta.TYPE_CHECKING:
+    import bz2
+else:
+    bz2 = lang.proxy_import('bz2')
+
+
+class IncrementalBz2Compressor:
+    def __init__(
+            self,
+            *,
+            compresslevel: int = 9,
+    ) -> None:
+        super().__init__()
+
+        self._compresslevel = compresslevel
+
+    @lang.autostart
+    def __call__(self) -> IncrementalCompressor:
+        return CompressorIncrementalAdapter(
+            functools.partial(
+                bz2.BZ2Compressor,  # type: ignore
+                self._compresslevel,
+            ),
+        )()
+
+
+class IncrementalBz2Decompressor:
+    def __call__(self) -> IncrementalDecompressor:
+        return DecompressorIncrementalAdapter(
+            bz2.BZ2Decompressor,  # type: ignore
+            trailing_error=OSError,
+        )()