omlish 0.0.0.dev276__tar.gz → 0.0.0.dev277__tar.gz

{omlish-0.0.0.dev276/omlish.egg-info → omlish-0.0.0.dev277}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omlish
-Version: 0.0.0.dev276
+Version: 0.0.0.dev277
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev276'
-__revision__ = 'b59ea0791ca796acd261e3bd7d25125492a027e4'
+__version__ = '0.0.0.dev277'
+__revision__ = 'b8087c9a0ca9d46a8ce3bfc58ba6bef8d3370196'
 
 
 #

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/http/all.py

@@ -1,13 +1,18 @@
 from . import consts  # noqa
 
 from .clients import (  # noqa
+    BaseHttpResponse,
     HttpClient,
     HttpClientError,
     HttpRequest,
     HttpResponse,
     HttpxHttpClient,
+    StreamHttpResponse,
     UrllibHttpClient,
     client,
+    close_response,
+    closing_response,
+    read_response,
     request,
 )
 

omlish-0.0.0.dev277/omlish/http/clients.py

@@ -0,0 +1,388 @@
+"""
+TODO:
+ - stream
+  - chunk size - httpx interface is awful, punch through?
+ - httpx catch
+ - return non-200 HttpResponses
+ - async
+"""
+import abc
+import contextlib
+import functools
+import http.client
+import typing as ta
+import urllib.error
+import urllib.request
+
+from .. import cached
+from .. import dataclasses as dc
+from .. import lang
+from .headers import CanHttpHeaders
+from .headers import HttpHeaders
+
+
+if ta.TYPE_CHECKING:
+    import httpx
+else:
+    httpx = lang.proxy_import('httpx')
+
+
+BaseHttpResponseT = ta.TypeVar('BaseHttpResponseT', bound='BaseHttpResponse')
+
+
+##
+
+
+DEFAULT_ENCODING = 'utf-8'
+
+
+def is_success_status(status: int) -> bool:
+    return 200 <= status < 300
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class HttpRequest(lang.Final):
+    url: str
+    method: str | None = None  # noqa
+
+    _: dc.KW_ONLY
+
+    headers: CanHttpHeaders | None = dc.xfield(None, repr=dc.truthy_repr)
+    data: bytes | str | None = dc.xfield(None, repr_fn=lambda v: '...' if v is not None else None)
+
+    timeout_s: float | None = None
+
+    #
+
+    @property
+    def method_or_default(self) -> str:
+        if self.method is not None:
+            return self.method
+        if self.data is not None:
+            return 'POST'
+        return 'GET'
+
+    @cached.property
+    def headers_(self) -> HttpHeaders | None:
+        return HttpHeaders(self.headers) if self.headers is not None else None
+
+
+#
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class BaseHttpResponse(lang.Abstract):
+    status: int
+
+    headers: HttpHeaders | None = dc.xfield(None, repr=dc.truthy_repr)
+
+    request: HttpRequest
+    underlying: ta.Any = dc.field(default=None, repr=False)
+
+    @property
+    def is_success(self) -> bool:
+        return is_success_status(self.status)
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class HttpResponse(BaseHttpResponse, lang.Final):
+    data: bytes | None = dc.xfield(None, repr_fn=lambda v: '...' if v is not None else None)
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class StreamHttpResponse(BaseHttpResponse, lang.Final):
+    class Stream(ta.Protocol):
+        def read(self, /, n: int = -1) -> bytes: ...
+
+    stream: Stream
+
+    _closer: ta.Callable[[], None] | None = dc.field(default=None, repr=False)
+
+    def __enter__(self) -> ta.Self:
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def close(self) -> None:
+        if (c := self._closer) is not None:
+            c()
+
+
+def close_response(resp: BaseHttpResponse) -> None:
+    if isinstance(resp, HttpResponse):
+        pass
+
+    elif isinstance(resp, StreamHttpResponse):
+        resp.close()
+
+    else:
+        raise TypeError(resp)
+
+
+@contextlib.contextmanager
+def closing_response(resp: BaseHttpResponseT) -> ta.Iterator[BaseHttpResponseT]:
+    if isinstance(resp, HttpResponse):
+        yield resp  # type: ignore
+        return
+
+    elif isinstance(resp, StreamHttpResponse):
+        with contextlib.closing(resp):
+            yield resp  # type: ignore
+
+    else:
+        raise TypeError(resp)
+
+
+def read_response(resp: BaseHttpResponse) -> HttpResponse:
+    if isinstance(resp, HttpResponse):
+        return resp
+
+    elif isinstance(resp, StreamHttpResponse):
+        data = resp.stream.read()
+        return HttpResponse(**{
+            **{k: v for k, v in dc.shallow_asdict(resp).items() if k not in ('stream', '_closer')},
+            'data': data,
+        })
+
+    else:
+        raise TypeError(resp)
+
+
+#
+
+
+class HttpClientError(Exception):
+    @property
+    def cause(self) -> BaseException | None:
+        return self.__cause__
+
+
+@dc.dataclass(frozen=True)
+class HttpStatusError(HttpClientError):
+    response: HttpResponse
+
+
+#
+
+
+class HttpClient(lang.Abstract):
+    def __enter__(self) -> ta.Self:
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def request(
+            self,
+            req: HttpRequest,
+            *,
+            check: bool = False,
+    ) -> HttpResponse:
+        with closing_response(self.stream_request(
+            req,
+            check=check,
+        )) as resp:
+            return read_response(resp)
+
+    def stream_request(
+            self,
+            req: HttpRequest,
+            *,
+            check: bool = False,
+    ) -> StreamHttpResponse:
+        resp = self._stream_request(req)
+
+        try:
+            if check and not resp.is_success:
+                if isinstance(resp.underlying, Exception):
+                    cause = resp.underlying
+                else:
+                    cause = None
+                raise HttpStatusError(read_response(resp)) from cause  # noqa
+
+        except Exception:
+            close_response(resp)
+            raise
+
+        return resp
+
+    @abc.abstractmethod
+    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+        raise NotImplementedError
+
+
+##
+
+
+class UrllibHttpClient(HttpClient):
+    def _build_request(self, req: HttpRequest) -> urllib.request.Request:
+        d: ta.Any
+        if (d := req.data) is not None:
+            if isinstance(d, str):
+                d = d.encode(DEFAULT_ENCODING)
+
+        # urllib headers are dumb dicts [1], and keys *must* be strings or it will automatically add problematic default
+        # headers because it doesn't see string keys in its header dict [2]. frustratingly it has no problem accepting
+        # bytes values though [3].
+        # [1]: https://github.com/python/cpython/blob/232b303e4ca47892f544294bf42e31dc34f0ec72/Lib/urllib/request.py#L319-L325  # noqa
+        # [2]: https://github.com/python/cpython/blob/232b303e4ca47892f544294bf42e31dc34f0ec72/Lib/urllib/request.py#L1276-L1279  # noqa
+        # [3]: https://github.com/python/cpython/blob/232b303e4ca47892f544294bf42e31dc34f0ec72/Lib/http/client.py#L1300-L1301  # noqa
+        h: dict[str, str] = {}
+        if hs := req.headers_:
+            for k, v in hs.strict_dct.items():
+                h[k.decode('ascii')] = v.decode('ascii')
+
+        return urllib.request.Request(  # noqa
+            req.url,
+            method=req.method_or_default,
+            headers=h,
+            data=d,
+        )
+
+    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+        try:
+            resp = urllib.request.urlopen(  # noqa
+                self._build_request(req),
+                timeout=req.timeout_s,
+            )
+
+        except urllib.error.HTTPError as e:
+            try:
+                return StreamHttpResponse(
+                    status=e.code,
+                    headers=HttpHeaders(e.headers.items()),
+                    request=req,
+                    underlying=e,
+                    stream=e,
+                    _closer=e.close,
+                )
+
+            except Exception:
+                e.close()
+                raise
+
+        except (urllib.error.URLError, http.client.HTTPException) as e:
+            raise HttpClientError from e
+
+        try:
+            return StreamHttpResponse(
+                status=resp.status,
+                headers=HttpHeaders(resp.headers.items()),
+                request=req,
+                underlying=resp,
+                stream=resp,
+                _closer=resp.close,
+            )
+
+        except Exception:  # noqa
+            resp.close()
+            raise
+
+
+##
+
+
+class HttpxHttpClient(HttpClient):
+    @dc.dataclass(frozen=True)
+    class _StreamAdapter:
+        it: ta.Iterator[bytes]
+
+        def read(self, /, n: int = -1) -> bytes:
+            if n < 0:
+                return b''.join(self.it)
+            else:
+                try:
+                    return next(self.it)
+                except StopIteration:
+                    return b''
+
+    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+        try:
+            resp_cm = httpx.stream(
+                method=req.method_or_default,
+                url=req.url,
+                headers=req.headers_ or None,  # type: ignore
+                content=req.data,
+                timeout=req.timeout_s,
+            )
+
+        except httpx.HTTPError as e:
+            raise HttpClientError from e
+
+        resp_close = functools.partial(resp_cm.__exit__, None, None, None)
+
+        try:
+            resp = resp_cm.__enter__()
+            return StreamHttpResponse(
+                status=resp.status_code,
+                headers=HttpHeaders(resp.headers.raw),
+                request=req,
+                underlying=resp,
+                stream=self._StreamAdapter(resp.iter_bytes()),
+                _closer=resp_close,  # type: ignore
+            )
+
+        except httpx.HTTPError as e:
+            resp_close()
+            raise HttpClientError from e
+
+        except Exception:
+            resp_close()
+            raise
+
+
+##
+
+
+def _default_client() -> HttpClient:
+    return UrllibHttpClient()
+
+
+def client() -> HttpClient:
+    return _default_client()
+
+
+def request(
+        url: str,
+        method: str | None = None,
+        *,
+        headers: CanHttpHeaders | None = None,
+        data: bytes | str | None = None,
+
+        timeout_s: float | None = None,
+
+        check: bool = False,
+
+        client: HttpClient | None = None,  # noqa
+
+        **kwargs: ta.Any,
+) -> HttpResponse:
+    req = HttpRequest(
+        url,
+        method=method,
+
+        headers=headers,
+        data=data,
+
+        timeout_s=timeout_s,
+
+        **kwargs,
+    )
+
+    def do(cli: HttpClient) -> HttpResponse:
+        return cli.request(
+            req,
+
+            check=check,
+        )
+
+    if client is not None:
+        return do(client)
+
+    else:
+        with _default_client() as cli:
+            return do(cli)

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/http/sse.py

@@ -12,6 +12,9 @@ from .. import dataclasses as dc
 from .. import lang
 
 
+##
+
+
 class SseDecoderOutput(lang.Abstract):
     pass
 

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/io/buffers.py

@@ -1,4 +1,4 @@
-# ruff: noqa: UP007
+# ruff: noqa: UP006 UP007
 # @omlish-lite
 import io
 import typing as ta
@@ -7,9 +7,16 @@ from ..lite.check import check
 from ..lite.strings import attr_repr
 
 
+##
+
+
 class DelimitingBuffer:
     """
     https://github.com/python-trio/trio/issues/796 :|
+
+    FIXME: when given overlapping delimiters like [b'\r', b'\r\n'], *should* refuse to output a line ending in '\r'
+      without knowing it will not be followed by '\n'. does not currently do this - currently only picks longest
+      delimiter present in the buffer. does this need a prefix-trie? is this borderline parsing?
     """
 
     #
@@ -31,19 +38,37 @@ class DelimitingBuffer:
 
     def __init__(
             self,
-            delimiters: ta.Iterable[int] = DEFAULT_DELIMITERS,
+            delimiters: ta.Iterable[ta.Union[int, bytes]] = DEFAULT_DELIMITERS,
             *,
             keep_ends: bool = False,
             max_size: ta.Optional[int] = None,
     ) -> None:
         super().__init__()
 
-        self._delimiters = frozenset(check.isinstance(d, int) for d in delimiters)
+        ds: ta.Set[bytes] = set()
+        for d in delimiters:
+            if isinstance(d, int):
+                d = bytes([d])
+            ds.add(check.isinstance(d, bytes))
+
+        self._delimiters: ta.FrozenSet[bytes] = frozenset(ds)
         self._keep_ends = keep_ends
         self._max_size = max_size
 
         self._buf: ta.Optional[io.BytesIO] = io.BytesIO()
 
+        ddl = {}
+        dl = sorted(self._delimiters, key=lambda d: -len(d))
+        for i, d in enumerate(dl):
+            for j, d2 in enumerate(dl):
+                if len(d2) < len(d):
+                    break
+                if i == j or not d2.startswith(d):
+                    continue
+                ddl[d] = len(d2)
+                break
+        self._delimiter_disambiguation_lens: ta.Dict[bytes, int] = ddl
+
     #
 
     @property
@@ -60,13 +85,30 @@ class DelimitingBuffer:
             raise self.ClosedError(self)
         return buf.getvalue()
 
-    def _find_delim(self, data: ta.Union[bytes, bytearray], i: int) -> ta.Optional[int]:
-        r = None  # type: int | None
+    def _find_delim(self, data: ta.Union[bytes, bytearray], i: int) -> ta.Optional[ta.Tuple[int, int]]:
+        rp = None  # type: int | None
+        rl = None  # type: int | None
+        rd = None  # type: bytes | None
+
         for d in self._delimiters:
-            if (p := data.find(d, i)) >= 0:
-                if r is None or p < r:
-                    r = p
-        return r
+            if (p := data.find(d, i)) < 0:
+                continue
+
+            dl = len(d)
+
+            if rp is None or p < rp:
+                rp, rl, rd = p, dl, d
+            elif rp == p and rl < dl:  # type: ignore
+                rl, rd = dl, d  # noqa
+
+        if rp is None:
+            return None
+
+        # FIXME:
+        # if (ddl := self._delimiter_disambiguation_lens.get(rd)) is not None:
+        #     raise NotImplementedError
+
+        return rp, rl  # type: ignore
 
     def _append_and_reset(self, chunk: bytes) -> bytes:
         buf = check.not_none(self._buf)
@@ -97,10 +139,11 @@ class DelimitingBuffer:
         l = len(data)
         i = 0
         while i < l:
-            if (p := self._find_delim(data, i)) is None:
+            if (pt := self._find_delim(data, i)) is None:
                 break
 
-            n = p + 1
+            p, pl = pt
+            n = p + pl
             if self._keep_ends:
                 p = n
 
@@ -128,6 +171,9 @@ class DelimitingBuffer:
             i = p
 
 
+##
+
+
 class ReadableListBuffer:
     # FIXME: merge with PrependableGeneratorReader
 
@@ -199,6 +245,9 @@ class ReadableListBuffer:
         return r if isinstance(r, bytes) else None
 
 
+##
+
+
 class IncrementalWriteBuffer:
     def __init__(
             self,

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/typedvalues/__init__.py

@@ -16,6 +16,10 @@ from .holder import (  # noqa
     TypedValueHolder,
 )
 
+from .reflect import (  # noqa
+    reflect_typed_values_impls,
+)
+
 from .values import (  # noqa
     TypedValue,
 

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/typedvalues/collection.py

@@ -129,6 +129,13 @@ class TypedValues(
 
     #
 
+    def check_all_isinstance(self, ty: type | tuple[type, ...]) -> ta.Self:
+        for tv in self._tup:
+            check.isinstance(tv, ty)
+        return self
+
+    #
+
     _hash: int
 
     def __hash__(self) -> int:

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277}/omlish/typedvalues/marshal.py

@@ -1,5 +1,3 @@
-import typing as ta
-
 from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
@@ -8,6 +6,7 @@ from omlish import reflect as rfl
 from omlish.funcs import match as mfs
 
 from .collection import TypedValues
+from .reflect import reflect_typed_values_impls
 from .values import ScalarTypedValue
 from .values import TypedValue
 
@@ -83,50 +82,35 @@ def _build_typed_values_impls(rty: rfl.Type) -> msh.Impls:
     gty = check.isinstance(rty, rfl.Generic)
     check.is_(gty.cls, TypedValues)
 
-    opt_cls_set: set[type[TypedValue]] = set()
-
-    todo = [check.single(gty.args)]
-    seen = set()
-    while todo:
-        cur = todo.pop()
-        if cur in seen:
-            continue
-        seen.add(cur)
-
-        if isinstance(cur, rfl.Union):
-            todo.extend(cur.args)
-        elif isinstance(cur, ta.TypeVar):
-            todo.append(rfl.get_type_var_bound(cur))
-        else:
-            opt_cls_set.add(check.issubclass(check.isinstance(cur, type), TypedValue))
+    tv_cls_set = reflect_typed_values_impls(check.single(gty.args))
 
-    opt_impls: list[msh.Impl] = []
-    for opt_cls in opt_cls_set:
-        opt_impls.extend(_build_typed_value_poly(opt_cls).impls)
+    tv_impls: list[msh.Impl] = []
+    for tv_cls in tv_cls_set:
+        tv_impls.extend(_build_typed_value_poly(tv_cls).impls)
 
-    return msh.Impls(opt_impls)
+    return msh.Impls(tv_impls)
 
 
 class TypedValuesMarshalerFactory(msh.MarshalerFactoryMatchClass):
     @mfs.simple(lambda _, ctx, rty: isinstance(rty, rfl.Generic) and rty.cls is TypedValues)
     def _build(self, ctx: msh.MarshalContext, rty: rfl.Type) -> msh.Marshaler:
-        opt_m = msh.make_polymorphism_marshaler(
+        tv_m = msh.make_polymorphism_marshaler(
             msh.Impls(_build_typed_values_impls(rty)),
             msh.WrapperTypeTagging(),
             ctx,
         )
-        return msh.IterableMarshaler(opt_m)
+        return msh.IterableMarshaler(tv_m)
 
 
 class TypedValuesUnmarshalerFactory(msh.UnmarshalerFactoryMatchClass):
     @mfs.simple(lambda _, ctx, rty: isinstance(rty, rfl.Generic) and rty.cls is TypedValues)
     def _build(self, ctx: msh.UnmarshalContext, rty: rfl.Type) -> msh.Unmarshaler:
-        opt_u = msh.make_polymorphism_unmarshaler(
+        tv_u = msh.make_polymorphism_unmarshaler(
             msh.Impls(_build_typed_values_impls(rty)),
             msh.WrapperTypeTagging(),
             ctx,
         )
-        return msh.IterableUnmarshaler(lambda it: TypedValues(*it), opt_u)  # noqa
+        return msh.IterableUnmarshaler(lambda it: TypedValues(*it), tv_u)  # noqa
 
 
 ##

omlish-0.0.0.dev277/omlish/typedvalues/reflect.py

@@ -0,0 +1,30 @@
+import typing as ta
+
+from omlish import check
+from omlish import reflect as rfl
+
+from .values import TypedValue
+
+
+##
+
+
+def reflect_typed_values_impls(rty: rfl.Type) -> set[type[TypedValue]]:
+    tv_cls_set: set[type[TypedValue]] = set()
+
+    todo = [rty]
+    seen = set()
+    while todo:
+        cur = todo.pop()
+        if cur in seen:
+            continue
+        seen.add(cur)
+
+        if isinstance(cur, rfl.Union):
+            todo.extend(cur.args)
+        elif isinstance(cur, ta.TypeVar):
+            todo.append(rfl.get_type_var_bound(cur))
+        else:
+            tv_cls_set.add(check.issubclass(check.isinstance(cur, type), TypedValue))
+
+    return tv_cls_set

{omlish-0.0.0.dev276 → omlish-0.0.0.dev277/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omlish
-Version: 0.0.0.dev276
+Version: 0.0.0.dev277
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause