omlish 0.0.0.dev472__py3-none-any.whl → 0.0.0.dev474__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev472'
-__revision__ = '78c62ff9afe49d9f0ae12b27ff595b2079e9c082'
+__version__ = '0.0.0.dev474'
+__revision__ = '5a7b02bc5884a6c8e41f2e2afd5aaf7f495cc28e'
 
 
 #
@@ -41,7 +41,7 @@ class Project(ProjectBase):
 
             'greenlet ~= 3.2',
 
-            'trio ~= 0.31',
+            'trio ~= 0.32',
             'trio-asyncio ~= 0.15',
         ],
 

omlish/collections/__init__.py

@@ -8,7 +8,9 @@ with _lang.auto_proxy_init(globals()):
 
     from .attrregistry import (  # noqa
         AttrRegistry,
+
         AttrRegistryCache,
+        SimpleAttrRegistryCache,
     )
 
     from .bimap import (  # noqa
@@ -17,6 +19,8 @@ with _lang.auto_proxy_init(globals()):
         make_bi_map,
     )
 
+    from . import cache  # noqa
+
     from .coerce import (  # noqa
         abs_set,
         abs_set_of,

omlish/collections/attrregistry.py

@@ -1,3 +1,7 @@
+"""
+TODO:
+ - lock?
+"""
 import dataclasses as dc
 import typing as ta
 import weakref
@@ -50,13 +54,32 @@ class AttrRegistry(ta.Generic[K, V]):
     def add_invalidate_callback(self, callback: ta.Callable[[], None]) -> None:
         self._invalidate_callbacks.append(callback)
 
+    @ta.overload
     def register(self, obj: K, val: V) -> None:
-        check.not_in(obj, self._objs)
+        ...
 
-        self._objs[obj] = val
+    @ta.overload
+    def register(self, val: V) -> ta.Callable[[T], T]:
+        ...
 
-        for iv in self._invalidate_callbacks:
-            iv()
+    def register(self, *args):
+        def inner(obj, val):
+            check.not_in(obj, self._objs)
+
+            self._objs[obj] = val
+
+            for iv in self._invalidate_callbacks:
+                iv()
+
+            return obj
+
+        if len(args) == 1:
+            return lambda obj: inner(obj, args[0])
+        elif len(args) == 2:
+            inner = inner(*args)
+            return None
+        else:
+            raise TypeError(args)
 
     def _lookup(self, obj: ta.Any) -> lang.Maybe[V]:
         if not self._identity:
@@ -180,3 +203,8 @@ class AttrRegistryCache(ta.Generic[K, V, T]):
         out = self._prepare(instance_cls, collected)
         self._cache[weakref.ref(instance_cls, self._cache_remove)] = out
         return out
+
+
+class SimpleAttrRegistryCache(AttrRegistryCache[K, V, dict[str, tuple[K, V]]], ta.Generic[K, V]):
+    def __init__(self, registry: AttrRegistry[K, V]) -> None:
+        super().__init__(registry, lambda _, dct: dct)

omlish/diag/{lslocks.py → cmds/lslocks.py}

@@ -7,12 +7,12 @@ import dataclasses as dc
 import json
 import typing as ta
 
-from ..lite.check import check
-from ..lite.marshal import OBJ_MARSHALER_FIELD_KEY
-from ..lite.marshal import unmarshal_obj
-from ..subprocesses.run import SubprocessRun
-from ..subprocesses.run import SubprocessRunnable
-from ..subprocesses.run import SubprocessRunOutput
+from ...lite.check import check
+from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from ...lite.marshal import unmarshal_obj
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunnable
+from ...subprocesses.run import SubprocessRunOutput
 
 
 ##

omlish/diag/{lsof.py → cmds/lsof.py}

@@ -7,12 +7,12 @@ import dataclasses as dc
 import enum
 import typing as ta
 
-from ..lite.check import check
-from ..lite.dataclasses import dataclass_repr_omit_falsey
-from ..lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
-from ..subprocesses.run import SubprocessRun
-from ..subprocesses.run import SubprocessRunnable
-from ..subprocesses.run import SubprocessRunOutput
+from ...lite.check import check
+from ...lite.dataclasses import dataclass_repr_omit_falsey
+from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunnable
+from ...subprocesses.run import SubprocessRunOutput
 
 
 ##

omlish/diag/{ps.py → cmds/ps.py}

@@ -4,12 +4,12 @@ import dataclasses as dc
 import os
 import typing as ta
 
-from ..lite.check import check
-from ..lite.timeouts import Timeout
-from ..subprocesses.run import SubprocessRun
-from ..subprocesses.run import SubprocessRunnable
-from ..subprocesses.run import SubprocessRunOutput
-from ..subprocesses.sync import subprocesses
+from ...lite.check import check
+from ...lite.timeouts import Timeout
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunnable
+from ...subprocesses.run import SubprocessRunOutput
+from ...subprocesses.sync import subprocesses
 
 
 ##

omlish/dispatch/__init__.py

@@ -1,12 +1,18 @@
-from .dispatch import (  # noqa
-    Dispatcher,
-)
-
-from .functions import (  # noqa
-    function,
-)
-
-from .methods import (  # noqa
-    install_method,
-    method,
-)
+from .. import lang as _lang
+
+
+with _lang.auto_proxy_init(globals()):
+    ##
+
+    from .dispatch import (  # noqa
+        Dispatcher,
+    )
+
+    from .functions import (  # noqa
+        function,
+    )
+
+    from .methods import (  # noqa
+        install_method,
+        method,
+    )

omlish/formats/json/stream/__init__.py

@@ -1,3 +1,16 @@
+"""
+A generator powered, configurable, mostly fully streaming JSON parser.
+
+Regarding the 'streamyness' of the subsystems:
+ - Lexing only buffers for string and number literals.
+ - Parsing maintains only a stack that scales by nesting depth.
+ - Building values will obviously hold everything under the topmost object it's building until it's finished.
+
+It's reasonably optimized, but performance is not a primary or even secondary goal: its goal is flexibility. If speed
+matters use a native library.
+"""
+
+
 from .building import (  # noqa
     JsonValueBuilder,
 )

omlish/http/clients/asyncs.py

@@ -3,9 +3,9 @@
 import abc
 import contextlib
 import dataclasses as dc
-import io
 import typing as ta
 
+from ...io.readers import AsyncBufferedBytesReader
 from ...lite.abstract import Abstract
 from ...lite.dataclasses import dataclass_shallow_asdict
 from .base import BaseHttpClient
@@ -27,25 +27,19 @@ AsyncHttpClientT = ta.TypeVar('AsyncHttpClientT', bound='AsyncHttpClient')
 @ta.final
 @dc.dataclass(frozen=True)  # kw_only=True
 class AsyncStreamHttpResponse(BaseHttpResponse):
-    class Stream(ta.Protocol):
-        def read1(self, /, n: int = -1) -> ta.Awaitable[bytes]: ...
+    _stream: ta.Optional[AsyncBufferedBytesReader] = None
 
-    @ta.final
-    class _NullStream:
-        def read1(self, /, n: int = -1) -> ta.Awaitable[bytes]:
-            raise TypeError
-
-    stream: Stream = _NullStream()
+    @property
+    def stream(self) -> 'AsyncBufferedBytesReader':
+        if (st := self._stream) is None:
+            raise TypeError('No data')
+        return st
 
     @property
     def has_data(self) -> bool:
-        return not isinstance(self.stream, AsyncStreamHttpResponse._NullStream)
+        return self._stream is not None
 
-    async def read_all(self) -> bytes:
-        buf = io.BytesIO()
-        while (b := await self.stream.read1()):
-            buf.write(b)
-        return buf.getvalue()
+    #
 
     _closer: ta.Optional[ta.Callable[[], ta.Awaitable[None]]] = None
 
@@ -96,8 +90,8 @@ async def async_read_http_client_response(resp: BaseHttpResponse) -> HttpRespons
 
     elif isinstance(resp, AsyncStreamHttpResponse):
         return HttpResponse(**{
-            **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('stream', '_closer')},
-            **({'data': await resp.read_all()} if resp.has_data else {}),
+            **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('_stream', '_closer')},
+            **({'data': await resp.stream.readall()} if resp.has_data else {}),
         })
 
     else:

omlish/http/clients/coro/sync.py

@@ -5,6 +5,7 @@ import socket
 import typing as ta
 import urllib.parse
 
+from ....io.buffers import ReadableListBuffer
 from ....lite.check import check
 from ...coro.client.connection import CoroHttpClientConnection
 from ...coro.client.response import CoroHttpClientResponse
@@ -92,7 +93,7 @@ class CoroHttpClient(HttpClient):
                     headers=HttpHeaders(resp._state.headers.items()),  # noqa
                     request=self._req,
                     underlying=self,
-                    stream=self,
+                    _stream=ReadableListBuffer().new_buffered_reader(self),
                     _closer=self.close,
                 )
 
@@ -156,7 +157,7 @@ class CoroHttpClient(HttpClient):
             else:
                 raise TypeError(o)
 
-        def read1(self, /, n: int = -1) -> bytes:
+        def read1(self, n: int = -1, /) -> bytes:
             return self._run_coro(check.not_none(self._resp).read(n if n >= 0 else None))
 
         def close(self) -> None:

omlish/http/clients/default.py

@@ -83,7 +83,7 @@ def client() -> HttpClient:
 
 
 @contextlib.contextmanager
-def manage_client(client: HttpClient | None) -> ta.Generator[HttpClient]:  # noqa
+def manage_client(client: HttpClient | None = None) -> ta.Generator[HttpClient]:  # noqa
     if client is not None:
         yield client
 
@@ -167,7 +167,7 @@ def async_client() -> AsyncHttpClient:
 
 
 @contextlib.asynccontextmanager
-async def manage_async_client(client: AsyncHttpClient | None) -> ta.AsyncGenerator[AsyncHttpClient]:  # noqa
+async def manage_async_client(client: AsyncHttpClient | None = None) -> ta.AsyncGenerator[AsyncHttpClient]:  # noqa
     if client is not None:
         yield client
 

omlish/http/clients/executor.py

@@ -30,9 +30,15 @@ class ExecutorAsyncHttpClient(AsyncHttpClient):
         owner: 'ExecutorAsyncHttpClient'
         resp: StreamHttpResponse
 
-        async def read1(self, /, n: int = -1) -> bytes:
+        async def read1(self, n: int = -1, /) -> bytes:
             return await self.owner._run_in_executor(self.resp.stream.read1, n)  # noqa
 
+        async def read(self, n: int = -1, /) -> bytes:
+            return await self.owner._run_in_executor(self.resp.stream.read, n)  # noqa
+
+        async def readall(self) -> bytes:
+            return await self.owner._run_in_executor(self.resp.stream.readall)  # noqa
+
         async def close(self) -> None:
             return await self.owner._run_in_executor(self.resp.close)  # noqa
 
@@ -44,7 +50,7 @@ class ExecutorAsyncHttpClient(AsyncHttpClient):
             request=req,
             underlying=resp,
             **(dict(  # type: ignore
-                stream=(adapter := self._StreamAdapter(self, resp)),
+                _stream=(adapter := self._StreamAdapter(self, resp)),
                 _closer=adapter.close,
             ) if resp.has_data else {}),
         )

omlish/http/clients/httpx.py

@@ -32,27 +32,12 @@ class HttpxHttpClient(HttpClient):
     @dc.dataclass(frozen=True)
     class _StreamAdapter:
         it: ta.Iterator[bytes]
-        buf: ReadableListBuffer = dc.field(default_factory=ReadableListBuffer)
-
-        def read1(self, /, n: int = -1) -> bytes:
-            if n < 0:
-                if (b := self.buf.read(n)) is not None:
-                    return b
-                try:
-                    return next(self.it)
-                except StopIteration:
-                    return b''
-
-            else:
-                while len(self.buf) < n:
-                    try:
-                        b = next(self.it)
-                    except StopIteration:
-                        b = b''
-                    if not b:
-                        return self.buf.read() or b''
-                    self.buf.feed(b)
-                return self.buf.read(n) or b''
+
+        def read1(self, n: int = -1, /) -> bytes:
+            try:
+                return next(self.it)
+            except StopIteration:
+                return b''
 
     def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         try:
@@ -76,7 +61,7 @@ class HttpxHttpClient(HttpClient):
                 headers=HttpHeaders(resp.headers.raw),
                 request=req,
                 underlying=resp,
-                stream=self._StreamAdapter(resp.iter_bytes()),
+                _stream=ReadableListBuffer().new_buffered_reader(self._StreamAdapter(resp.iter_bytes())),
                 _closer=resp_close,  # type: ignore
             )
 
@@ -84,7 +69,7 @@ class HttpxHttpClient(HttpClient):
             resp_close()
             raise HttpClientError from e
 
-        except Exception:
+        except BaseException:
             resp_close()
             raise
 
@@ -96,27 +81,12 @@ class HttpxAsyncHttpClient(AsyncHttpClient):
     @dc.dataclass(frozen=True)
     class _StreamAdapter:
         it: ta.AsyncIterator[bytes]
-        buf: ReadableListBuffer = dc.field(default_factory=ReadableListBuffer)
-
-        async def read1(self, /, n: int = -1) -> bytes:
-            if n < 0:
-                if (b := self.buf.read(n)) is not None:
-                    return b
-                try:
-                    return await anext(self.it)
-                except StopAsyncIteration:
-                    return b''
-
-            else:
-                while len(self.buf) < n:
-                    try:
-                        b = await anext(self.it)
-                    except StopAsyncIteration:
-                        b = b''
-                    if not b:
-                        return self.buf.read() or b''
-                    self.buf.feed(b)
-                return self.buf.read(n) or b''
+
+        async def read1(self, n: int = -1, /) -> bytes:
+            try:
+                return await anext(self.it)
+            except StopAsyncIteration:
+                return b''
 
     async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
         es = contextlib.AsyncExitStack()
@@ -132,12 +102,25 @@ class HttpxAsyncHttpClient(AsyncHttpClient):
                 timeout=req.timeout_s,
             ))
 
+            it = resp.aiter_bytes()
+
+            # FIXME:
+            #  this has a tendency to raise `RuntimeError: async generator ignored GeneratorExit` when all of the
+            #  following conditions are met:
+            #   - stopped iterating midway through
+            #   - shutting down the event loop
+            #   - debugging under pycharm / pydevd
+            #   - running under asyncio
+            #  it does not seem to happen unless all of these conditions are met. see:
+            #   https://gist.github.com/wrmsr/a0578ee5d5371b53804cfb56aeb84cdf .
+            es.push_async_callback(it.aclose)  # type: ignore[attr-defined]
+
             return AsyncStreamHttpResponse(
                 status=resp.status_code,
                 headers=HttpHeaders(resp.headers.raw),
                 request=req,
                 underlying=resp,
-                stream=self._StreamAdapter(resp.aiter_bytes()),
+                _stream=ReadableListBuffer().new_async_buffered_reader(self._StreamAdapter(it)),
                 _closer=es.aclose,
             )
 
@@ -145,6 +128,6 @@ class HttpxAsyncHttpClient(AsyncHttpClient):
             await es.aclose()
             raise HttpClientError from e
 
-        except Exception:
+        except BaseException:
             await es.aclose()
             raise

omlish/http/clients/sync.py

@@ -3,9 +3,9 @@
 import abc
 import contextlib
 import dataclasses as dc
-import io
 import typing as ta
 
+from ...io.readers import BufferedBytesReader
 from ...lite.abstract import Abstract
 from ...lite.dataclasses import dataclass_shallow_asdict
 from .base import BaseHttpClient
@@ -27,25 +27,19 @@ HttpClientT = ta.TypeVar('HttpClientT', bound='HttpClient')
 @ta.final
 @dc.dataclass(frozen=True)  # kw_only=True
 class StreamHttpResponse(BaseHttpResponse):
-    class Stream(ta.Protocol):
-        def read1(self, /, n: int = -1) -> bytes: ...
+    _stream: ta.Optional[BufferedBytesReader] = None
 
-    @ta.final
-    class _NullStream:
-        def read1(self, /, n: int = -1) -> bytes:
-            raise TypeError
-
-    stream: Stream = _NullStream()
+    @property
+    def stream(self) -> 'BufferedBytesReader':
+        if (st := self._stream) is None:
+            raise TypeError('No data')
+        return st
 
     @property
     def has_data(self) -> bool:
-        return not isinstance(self.stream, StreamHttpResponse._NullStream)
+        return self._stream is not None
 
-    def read_all(self) -> bytes:
-        buf = io.BytesIO()
-        while (b := self.stream.read1()):
-            buf.write(b)
-        return buf.getvalue()
+    #
 
     _closer: ta.Optional[ta.Callable[[], None]] = None
 
@@ -94,8 +88,8 @@ def read_http_client_response(resp: BaseHttpResponse) -> HttpResponse:
 
     elif isinstance(resp, StreamHttpResponse):
         return HttpResponse(**{
-            **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('stream', '_closer')},
-            **({'data': resp.read_all()} if resp.has_data else {}),
+            **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('_stream', '_closer')},
+            **({'data': resp.stream.readall()} if resp.has_data else {}),
         })
 
     else:

omlish/http/clients/syncasync.py

@@ -23,9 +23,15 @@ class SyncAsyncHttpClient(AsyncHttpClient):
     class _StreamAdapter:
         ul: StreamHttpResponse
 
-        async def read1(self, /, n: int = -1) -> bytes:
+        async def read1(self, n: int = -1, /) -> bytes:
             return self.ul.stream.read1(n)
 
+        async def read(self, n: int = -1, /) -> bytes:
+            return self.ul.stream.read(n)
+
+        async def readall(self) -> bytes:
+            return self.ul.stream.readall()
+
         async def close(self) -> None:
             self.ul.close()
 
@@ -37,7 +43,7 @@ class SyncAsyncHttpClient(AsyncHttpClient):
             request=req,
             underlying=resp,
             **(dict(  # type: ignore
-                stream=(adapter := self._StreamAdapter(resp)),
+                _stream=(adapter := self._StreamAdapter(resp)),
                 _closer=adapter.close,
             ) if resp.has_data else {}),
         )

omlish/http/clients/urllib.py

@@ -5,6 +5,7 @@ import typing as ta
 import urllib.error
 import urllib.request
 
+from ...io.buffers import ReadableListBuffer
 from ..headers import HttpHeaders
 from .base import DEFAULT_ENCODING
 from .base import HttpClientContext
@@ -72,7 +73,7 @@ class UrllibHttpClient(HttpClient):
                 headers=HttpHeaders(resp.headers.items()),
                 request=req,
                 underlying=resp,
-                stream=resp,
+                _stream=ReadableListBuffer().new_buffered_reader(resp),
                 _closer=resp.close,
             )