omlish 0.0.0.dev137__tar.gz → 0.0.0.dev139__tar.gz

{omlish-0.0.0.dev137/omlish.egg-info → omlish-0.0.0.dev139}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev137
+Version: 0.0.0.dev139
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev137'
-__revision__ = '895417d0442f51853415386ace94b1c41da0c990'
+__version__ = '0.0.0.dev139'
+__revision__ = '943c8c72d1baaa0ef3088fcadb2933d4abc11f36'
 
 
 #

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/configs/flattening.py

@@ -130,7 +130,7 @@ class Flattening:
                             .split(self._index_close + self._index_open):
                         yield int(p)
                 else:
-                    check.state(')' not in part)
+                    check.state(self._index_close not in part)
                     yield part
 
         for fk, v in flattened.items():

omlish-0.0.0.dev139/omlish/formats/json/stream/errors.py

@@ -0,0 +1,2 @@
+class JsonStreamError(Exception):
+    pass

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/formats/json/stream/lex.py

@@ -3,6 +3,7 @@ TODO:
  - max buf size
  - max recursion depth
  - mark start pos of tokens, currently returning end
+ - _do_string inner loop optimization somehow
 """
 import dataclasses as dc
 import io
@@ -12,6 +13,7 @@ import typing as ta
 
 from .... import check
 from ....funcs.genmachine import GenMachine
+from .errors import JsonStreamError
 
 
 ##
@@ -95,7 +97,7 @@ CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
 
 
 @dc.dataclass()
-class JsonLexError(Exception):
+class JsonStreamLexError(JsonStreamError):
     message: str
 
     pos: Position
@@ -160,8 +162,8 @@ class JsonStreamLexer(GenMachine[str, Token]):
         self._buf.truncate()
         return raw
 
-    def _raise(self, msg: str) -> ta.NoReturn:
-        raise JsonLexError(msg, self.pos)
+    def _raise(self, msg: str, src: Exception | None = None) -> ta.NoReturn:
+        raise JsonStreamLexError(msg, self.pos) from src
 
     def _do_main(self):
         while True:
@@ -202,7 +204,7 @@ class JsonStreamLexer(GenMachine[str, Token]):
                 self._raise('Unexpected end of input')
 
             if not c:
-                raise NotImplementedError
+                self._raise(f'Unterminated string literal: {self._buf.getvalue()}')
 
             self._buf.write(c)
             if c == '"' and last != '\\':
@@ -210,7 +212,11 @@ class JsonStreamLexer(GenMachine[str, Token]):
             last = c
 
         raw = self._flip_buf()
-        sv = json.loads(raw)
+        try:
+            sv = json.loads(raw)
+        except json.JSONDecodeError as e:
+            self._raise(f'Invalid string literal: {raw!r}', e)
+
         yield self._make_tok('STRING', sv, raw, pos)
 
         return self._do_main()

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/formats/json/stream/parse.py

@@ -1,9 +1,12 @@
+import dataclasses as dc
 import typing as ta
 
 from .... import lang
 from ....funcs.genmachine import GenMachine
+from .errors import JsonStreamError
 from .lex import SCALAR_VALUE_TYPES
 from .lex import VALUE_TOKEN_KINDS
+from .lex import Position
 from .lex import ScalarValue
 from .lex import Token
 
@@ -79,6 +82,13 @@ def yield_parser_events(obj: ta.Any) -> ta.Generator[JsonStreamParserEvent, None
 ##
 
 
+@dc.dataclass()
+class JsonStreamParseError(JsonStreamError):
+    message: str
+
+    pos: Position | None = None
+
+
 class JsonStreamObject(list):
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}({super().__repr__()})'
@@ -100,29 +110,29 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         if tt == 'KEY':
             self._stack.pop()
             if not self._stack:
-                raise self.StateError
+                raise JsonStreamParseError('Unexpected key')
 
             tt2 = self._stack[-1]
             if tt2 == 'OBJECT':
                 return ((v,), self._do_after_pair())
 
             else:
-                raise self.StateError
+                raise JsonStreamParseError('Unexpected key')
 
         elif tt == 'ARRAY':
             return ((v,), self._do_after_element())
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError(f'Unexpected value: {v!r}')
 
     #
 
-    def _do_value(self):
+    def _do_value(self, *, must_be_present: bool = False):
         try:
             tok = yield None
         except GeneratorExit:
             if self._stack:
-                raise self.StateError from None
+                raise JsonStreamParseError('Expected value') from None
             else:
                 raise
 
@@ -141,13 +151,16 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             yield y
             return r
 
+        elif must_be_present:
+            raise JsonStreamParseError('Expected value', tok.pos)
+
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()
             yield y
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected value', tok.pos)
 
     #
 
@@ -157,19 +170,19 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
 
     def _emit_end_object(self):
         if not self._stack:
-            raise self.StateError
+            raise JsonStreamParseError('Unexpected end object')
 
         tt = self._stack.pop()
         if tt != 'OBJECT':
-            raise self.StateError
+            raise JsonStreamParseError('Unexpected end object')
 
         return self._emit_event(EndObject)
 
-    def _do_object_body(self):
+    def _do_object_body(self, *, must_be_present: bool = False):
         try:
             tok = yield None
         except GeneratorExit:
-            raise self.StateError from None
+            raise JsonStreamParseError('Expected object body') from None
 
         if tok.kind == 'STRING':
             k = tok.value
@@ -177,30 +190,33 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             try:
                 tok = yield None
             except GeneratorExit:
-                raise self.StateError from None
+                raise JsonStreamParseError('Expected key') from None
             if tok.kind != 'COLON':
-                raise self.StateError
+                raise JsonStreamParseError('Expected colon', tok.pos)
 
             yield (Key(k),)
             self._stack.append('KEY')
             return self._do_value()
 
+        elif must_be_present:
+            raise JsonStreamParseError('Expected value', tok.pos)
+
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
             yield y
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected value', tok.pos)
 
     def _do_after_pair(self):
         try:
             tok = yield None
         except GeneratorExit:
-            raise self.StateError from None
+            raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_object_body()
+            return self._do_object_body(must_be_present=True)
 
         elif tok.kind == 'RBRACE':
             y, r = self._emit_end_object()
@@ -208,7 +224,7 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected continuation', tok.pos)
 
     #
 
@@ -218,11 +234,11 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
 
     def _emit_end_array(self):
         if not self._stack:
-            raise self.StateError
+            raise JsonStreamParseError('Expected end array')
 
         tt = self._stack.pop()
         if tt != 'ARRAY':
-            raise self.StateError
+            raise JsonStreamParseError('Unexpected end array')
 
         return self._emit_event(EndArray)
 
@@ -230,10 +246,10 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
         try:
             tok = yield None
         except GeneratorExit:
-            raise self.StateError from None
+            raise JsonStreamParseError('Expected continuation') from None
 
         if tok.kind == 'COMMA':
-            return self._do_value()
+            return self._do_value(must_be_present=True)
 
         elif tok.kind == 'RBRACKET':
             y, r = self._emit_end_array()
@@ -241,4 +257,4 @@ class JsonStreamParser(GenMachine[Token, JsonStreamParserEvent]):
             return r
 
         else:
-            raise self.StateError
+            raise JsonStreamParseError('Expected continuation', tok.pos)

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/funcs/genmachine.py

@@ -67,8 +67,7 @@ class GenMachine(ta.Generic[I, O]):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is None:
-            self.close()
+        self.close()
 
     #
 

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/io/compress/bz2.py

@@ -24,6 +24,7 @@ class IncrementalBz2Compressor:
 
         self._compresslevel = compresslevel
 
+    @lang.autostart
     def __call__(self) -> IncrementalCompressor:
         return CompressorIncrementalAdapter(
             functools.partial(

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/io/compress/gzip.py

@@ -42,7 +42,7 @@ import typing as ta
 from ... import cached
 from ... import check
 from ... import lang
-from ..generators import PrependableBytesGeneratorReader
+from ..generators.readers import PrependableBytesGeneratorReader
 from .types import IncrementalCompressor
 from .types import IncrementalDecompressor
 
@@ -123,10 +123,12 @@ class IncrementalGzipCompressor:
         if fname:
             check.none((yield fname + b'\000'))
 
+    @lang.autostart
     def __call__(self) -> IncrementalCompressor:
         crc = _zero_crc()
         size = 0
         offset = 0  # Current file offset for seek(), tell(), etc
+        wrote_header = False
 
         compress = zlib.compressobj(
             self._compresslevel,
@@ -136,10 +138,13 @@ class IncrementalGzipCompressor:
             0,
         )
 
-        yield from self._write_gzip_header()
-
         while True:
             data: ta.Any = check.isinstance((yield None), bytes)
+
+            if not wrote_header:
+                yield from self._write_gzip_header()
+                wrote_header = True
+
             if not data:
                 break
 

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/io/compress/lzma.py

@@ -17,6 +17,7 @@ class IncrementalLzmaCompressor:
     def __init__(self) -> None:
         super().__init__()
 
+    @lang.autostart
     def __call__(self) -> IncrementalCompressor:
         return CompressorIncrementalAdapter(
             lzma.LZMACompressor,  # type: ignore

omlish-0.0.0.dev139/omlish/io/generators/readers.py

@@ -0,0 +1,183 @@
+"""
+TODO:
+ - BufferedBytesGeneratorReader
+ - docstrings
+ - memoryviews
+"""
+import abc
+import typing as ta
+
+from ... import check
+
+
+T = ta.TypeVar('T')
+I = ta.TypeVar('I')
+R = ta.TypeVar('R')
+AnyT = ta.TypeVar('AnyT', bound=ta.Any)
+
+
+ReaderGenerator: ta.TypeAlias = ta.Generator[int | None, I, R]
+ExactReaderGenerator: ta.TypeAlias = ta.Generator[int, I, R]
+
+BytesReaderGenerator: ta.TypeAlias = ReaderGenerator[bytes, R]
+BytesExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[bytes, R]
+
+StrReaderGenerator: ta.TypeAlias = ReaderGenerator[str, R]
+StrExactReaderGenerator: ta.TypeAlias = ExactReaderGenerator[str, R]
+
+
+##
+
+
+class _BytesJoiner:
+    def _join(self, lst: list[bytes]) -> bytes:
+        return b''.join(lst)
+
+
+class _StrJoiner:
+    def _join(self, lst: list[str]) -> str:
+        return ''.join(lst)
+
+
+##
+
+
+class GeneratorReader(abc.ABC, ta.Generic[T]):
+    @abc.abstractmethod
+    def read(self, sz: int | None) -> ta.Generator[int | None, T, T]:
+        raise NotImplementedError
+
+    def read_exact(self, sz: int) -> ta.Generator[int | None, T, T]:
+        d: ta.Any = yield from self.read(sz)
+        if len(d) != sz:
+            raise EOFError(f'GeneratorReader got {len(d)}, expected {sz}')
+        return d
+
+
+##
+
+
+class PrependableGeneratorReader(GeneratorReader[AnyT]):
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._queue: list[tuple[AnyT, int]] = []
+
+    @abc.abstractmethod
+    def _join(self, lst: list[AnyT]) -> AnyT:
+        raise NotImplementedError
+
+    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+        if not self._queue:
+            d: AnyT = check.not_none((yield sz))
+            return d
+
+        if sz is None:
+            return self._queue.pop(0)[0]
+
+        lst: list[AnyT] = []
+        rem = sz
+        while rem > 0 and self._queue:
+            c, p = self._queue[0]
+
+            if len(c) - p > rem:
+                lst.append(c[p:p + rem])
+                self._queue[0] = (c, p + rem)
+                return self._join(lst)
+
+            lst.append(c[p:])
+            rem -= len(c) - p
+            self._queue.pop(0)
+
+        if rem:
+            d = check.not_none((yield rem))
+            if d:
+                lst.append(d)  # type: ignore[unreachable]
+
+        if len(lst) == 1:
+            return lst[0]
+        else:
+            return self._join(lst)
+
+    def prepend(self, d: AnyT, p: int | None = None) -> None:
+        if d:
+            self._queue.insert(0, (d, p or 0))
+
+
+class PrependableBytesGeneratorReader(
+    _BytesJoiner,
+    PrependableGeneratorReader[bytes],
+):
+    pass
+
+
+class PrependableStrGeneratorReader(
+    _StrJoiner,
+    PrependableGeneratorReader[str],
+):
+    pass
+
+
+prependable_bytes_generator_reader = PrependableBytesGeneratorReader
+prependable_str_generator_reader = PrependableStrGeneratorReader
+
+
+##
+
+
+class BufferedGeneratorReader(PrependableGeneratorReader[AnyT], abc.ABC):
+    DEFAULT_BUFFER_SIZE = 4 * 0x1000
+
+    def __init__(
+            self,
+            buffer_size: int = DEFAULT_BUFFER_SIZE,
+    ) -> None:
+        check.arg(buffer_size > 0)
+
+        super().__init__()
+
+        self._buffer_size = buffer_size
+
+    def read(self, sz: int | None) -> ta.Generator[int | None, AnyT, AnyT]:
+        g = super().read(sz)
+        i: ta.Any = None
+        while True:
+            try:
+                q = g.send(i)
+            except StopIteration as e:
+                return e.value
+
+            check.state(not self._queue)
+
+            if q is None:
+                i = check.not_none((yield None))
+                continue
+
+            r = max(q, self._buffer_size)
+            d: AnyT = check.not_none((yield r))
+            if len(d) < q:
+                i = d
+                continue
+
+            i = d[:q]
+            self.prepend(d, q)
+
+
+class BufferedBytesGeneratorReader(
+    _BytesJoiner,
+    BufferedGeneratorReader[bytes],
+    PrependableGeneratorReader[bytes],
+):
+    pass
+
+
+class BufferedStrGeneratorReader(
+    _StrJoiner,
+    BufferedGeneratorReader[str],
+    PrependableGeneratorReader[str],
+):
+    pass
+
+
+buffered_bytes_generator_reader = BufferedBytesGeneratorReader
+buffered_str_generator_reader = BufferedStrGeneratorReader

omlish-0.0.0.dev139/omlish/io/generators/stepped.py

@@ -0,0 +1,104 @@
+import typing as ta
+
+from ... import lang
+
+
+T = ta.TypeVar('T')
+I = ta.TypeVar('I')
+O = ta.TypeVar('O')
+OF = ta.TypeVar('OF')
+OT = ta.TypeVar('OT')
+R = ta.TypeVar('R')
+
+
+SteppedGenerator: ta.TypeAlias = ta.Generator[O | None, I | None, R]
+
+
+##
+
+
+@lang.autostart
+def flatmap_stepped_generator(
+        fn: ta.Callable[[list[OF]], OT],
+        g: SteppedGenerator[OF, I, R],
+        *,
+        terminate: ta.Callable[[OF], bool] | None = None,
+) -> ta.Generator[OT, I, lang.Maybe[R]]:
+    """
+    Given a 'stepped generator' - a generator which accepts input items and yields zero or more non-None values in
+    response until it signals it's ready for the next input by yielding None - and a function taking a list, returns a
+    1:1 generator which accepts input, builds a list of yielded generator output, calls the given function with that
+    list, and yields the result.
+
+    An optional terminate function may be provided which will cause this function to return early if it returns true for
+    an encountered yielded value. The encountered value causing termination will be included in the list sent to the
+    given fn.
+
+    Returns a Maybe of either the given generator's return value or empty if the terminator was encountered.
+    """
+
+    l: list[OF]
+    i: I | None = yield  # type: ignore
+    while True:
+        l = []
+
+        while True:
+            try:
+                o = g.send(i)
+            except StopIteration as e:
+                if l:
+                    yield fn(l)
+                return lang.just(e.value)
+
+            i = None
+
+            if o is None:
+                break
+
+            l.append(o)
+
+            if terminate is not None and terminate(o):
+                yield fn(l)
+                return lang.empty()
+
+        i = yield fn(l)
+
+
+##
+
+
+def _join_bytes(l: ta.Sequence[bytes]) -> bytes:
+    if not l:
+        return b''
+    elif len(l) == 1:
+        return l[0]
+    else:
+        return b''.join(l)
+
+
+def _join_str(l: ta.Sequence[str]) -> str:
+    if not l:
+        return ''
+    elif len(l) == 1:
+        return l[0]
+    else:
+        return ''.join(l)
+
+
+def _is_empty(o: T) -> bool:
+    return len(o) < 1  # type: ignore
+
+
+##
+
+
+def joined_bytes_stepped_generator(
+        g: ta.Generator[bytes | None, bytes | None, R],
+) -> ta.Generator[bytes, bytes, R]:
+    return flatmap_stepped_generator(_join_bytes, g, terminate=_is_empty)
+
+
+def joined_str_stepped_generator(
+        g: ta.Generator[str | None, str | None, R],
+) -> ta.Generator[str, str, R]:
+    return flatmap_stepped_generator(_join_str, g, terminate=_is_empty)

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/lang/__init__.py

@@ -124,7 +124,10 @@ from .generators import (  # noqa
     CoroutineGenerator,
     Generator,
     GeneratorLike,
+    GeneratorMappedIterator,
+    autostart,
     corogen,
+    genmap,
     nextgen,
 )
 

{omlish-0.0.0.dev137 → omlish-0.0.0.dev139}/omlish/lang/functions.py

@@ -82,7 +82,6 @@ def identity(obj: T) -> T:
 
 
 class constant(ta.Generic[T]):  # noqa
-
     def __init__(self, obj: T) -> None:
         super().__init__()
 
@@ -116,7 +115,6 @@ class VoidError(Exception):
 
 
 class Void:
-
     def __new__(cls, *args: ta.Any, **kwargs: ta.Any) -> None:  # type: ignore  # noqa
         raise VoidError