omlish 0.0.0.dev468__py3-none-any.whl → 0.0.0.dev470__py3-none-any.whl

omlish/http/clients/default.py

@@ -1,9 +1,11 @@
 import abc
+import contextlib
 import typing as ta
 
 from ... import lang
 from ..headers import CanHttpHeaders
 from .asyncs import AsyncHttpClient
+from .base import HttpClientContext
 from .base import HttpRequest
 from .base import HttpResponse
 from .sync import HttpClient
@@ -32,12 +34,13 @@ class _DefaultRequester(lang.Abstract, ta.Generic[C, R]):
 
             timeout_s: float | None = None,
 
+            context: HttpClientContext | None = None,
             check: bool = False,
             client: C | None = None,  # noqa
 
             **kwargs: ta.Any,
     ) -> R:
-        req = HttpRequest(
+        request = HttpRequest(  # noqa
             url,
             method=method,
 
@@ -50,7 +53,8 @@ class _DefaultRequester(lang.Abstract, ta.Generic[C, R]):
         )
 
         return self._do(
-            req,
+            request,
+            context=context,
             check=check,
             client=client,
         )
@@ -58,8 +62,9 @@ class _DefaultRequester(lang.Abstract, ta.Generic[C, R]):
     @abc.abstractmethod
     def _do(
             self,
-            req: HttpRequest,
+            request: HttpRequest,  # noqa
             *,
+            context: HttpClientContext | None = None,
             check: bool = False,
             client: C | None = None,  # noqa
     ) -> R:
@@ -77,30 +82,74 @@ def client() -> HttpClient:
     return _default_client()
 
 
+@contextlib.contextmanager
+def manage_client(client: HttpClient | None) -> ta.Generator[HttpClient]:  # noqa
+    if client is not None:
+        yield client
+
+    else:
+        with _default_client() as client:  # noqa
+            yield client
+
+
 #
 
 
-class _SyncDefaultRequester(_DefaultRequester[HttpClient, HttpResponse]):
+class _BaseSyncDefaultRequester(_DefaultRequester[HttpClient, R], lang.Abstract, ta.Generic[R]):
     def _do(
             self,
-            req: HttpRequest,
+            request: HttpRequest,  # noqa
             *,
+            context: HttpClientContext | None = None,
             check: bool = False,
             client: HttpClient | None = None,  # noqa
-    ) -> HttpResponse:
-        def do(cli: HttpClient) -> HttpResponse:  # noqa
-            return cli.request(
-                req,
+    ) -> R:
+        if context is None:
+            context = HttpClientContext()
 
+        if client is not None:
+            return self._do_(
+                client,
+                context,
+                request,
                 check=check,
             )
 
-        if client is not None:
-            return do(client)
-
         else:
-            with _default_client() as cli:
-                return do(cli)
+            with _default_client() as client:  # noqa
+                return self._do_(
+                    client,
+                    context,
+                    request,
+                    check=check,
+                )
+
+    @abc.abstractmethod
+    def _do_(
+            self,
+            client: HttpClient,  # noqa
+            context: HttpClientContext,
+            request: HttpRequest,  # noqa
+            *,
+            check: bool = False,  # noqa
+    ) -> R:
+        raise NotImplementedError
+
+
+class _SyncDefaultRequester(_BaseSyncDefaultRequester[HttpResponse]):
+    def _do_(
+            self,
+            client: HttpClient,  # noqa
+            context: HttpClientContext,
+            request: HttpRequest,  # noqa
+            *,
+            check: bool = False,  # noqa
+    ) -> HttpResponse:
+        return client.request(
+            request,
+            context=context,
+            check=check,
+        )
 
 
 request = _SyncDefaultRequester()
@@ -117,30 +166,74 @@ def async_client() -> AsyncHttpClient:
     return _default_async_client()
 
 
+@contextlib.asynccontextmanager
+async def manage_async_client(client: AsyncHttpClient | None) -> ta.AsyncGenerator[AsyncHttpClient]:  # noqa
+    if client is not None:
+        yield client
+
+    else:
+        async with _default_async_client() as client:  # noqa
+            yield client
+
+
 #
 
 
-class _AsyncDefaultRequester(_DefaultRequester[AsyncHttpClient, ta.Awaitable[HttpResponse]]):
+class _BaseAsyncDefaultRequester(_DefaultRequester[AsyncHttpClient, ta.Awaitable[R]], lang.Abstract, ta.Generic[R]):
     async def _do(
             self,
-            req: HttpRequest,
+            request: HttpRequest,  # noqa
             *,
+            context: HttpClientContext | None = None,
             check: bool = False,
             client: AsyncHttpClient | None = None,  # noqa
-    ) -> HttpResponse:
-        async def do(cli: AsyncHttpClient) -> HttpResponse:  # noqa
-            return await cli.request(
-                req,
+    ) -> R:
+        if context is None:
+            context = HttpClientContext()
 
+        if client is not None:
+            return await self._do_(
+                client,
+                context,
+                request,
                 check=check,
             )
 
-        if client is not None:
-            return await do(client)
-
         else:
-            async with _default_async_client() as cli:
-                return await do(cli)
+            async with _default_async_client() as client:  # noqa
+                return await self._do_(
+                    client,
+                    context,
+                    request,
+                    check=check,
+                )
+
+    @abc.abstractmethod
+    def _do_(
+            self,
+            client: AsyncHttpClient,  # noqa
+            context: HttpClientContext,
+            request: HttpRequest,  # noqa
+            *,
+            check: bool = False,  # noqa
+    ) -> ta.Awaitable[R]:
+        raise NotImplementedError
+
+
+class _AsyncDefaultRequester(_BaseAsyncDefaultRequester[HttpResponse]):
+    async def _do_(
+            self,
+            client: AsyncHttpClient,  # noqa
+            context: HttpClientContext,
+            request: HttpRequest,  # noqa
+            *,
+            check: bool = False,
+    ) -> HttpResponse:  # noqa
+        return await client.request(
+            request,
+            context=context,
+            check=check,
+        )
 
 
 async_request = _AsyncDefaultRequester()

omlish/http/clients/executor.py

@@ -0,0 +1,50 @@
+# ruff: noqa: UP043 UP045
+# @omlish-lite
+import dataclasses as dc
+import typing as ta
+
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import HttpClientContext
+from .base import HttpRequest
+from .sync import HttpClient
+from .sync import StreamHttpResponse
+
+
+##
+
+
+class ExecutorAsyncHttpClient(AsyncHttpClient):
+    def __init__(
+            self,
+            run_in_executor: ta.Callable[..., ta.Awaitable],
+            client: HttpClient,
+    ) -> None:
+        super().__init__()
+
+        self._run_in_executor = run_in_executor
+        self._client = client
+
+    @dc.dataclass(frozen=True)
+    class _StreamAdapter:
+        owner: 'ExecutorAsyncHttpClient'
+        resp: StreamHttpResponse
+
+        async def read1(self, /, n: int = -1) -> bytes:
+            return await self.owner._run_in_executor(self.resp.stream.read1, n)  # noqa
+
+        async def close(self) -> None:
+            return await self.owner._run_in_executor(self.resp.close)  # noqa
+
+    async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
+        resp: StreamHttpResponse = await self._run_in_executor(lambda: self._client.stream_request(req, context=ctx))
+        return AsyncStreamHttpResponse(
+            status=resp.status,
+            headers=resp.headers,
+            request=req,
+            underlying=resp,
+            **(dict(  # type: ignore
+                stream=(adapter := self._StreamAdapter(self, resp)),
+                _closer=adapter.close,
+            ) if resp.has_data else {}),
+        )

omlish/http/clients/httpx.py

@@ -4,14 +4,15 @@ TODO:
 """
 import contextlib
 import functools
-import io
 import typing as ta
 
 from ... import dataclasses as dc
 from ... import lang
+from ...io.buffers import ReadableListBuffer
 from ..headers import HttpHeaders
 from .asyncs import AsyncHttpClient
 from .asyncs import AsyncStreamHttpResponse
+from .base import HttpClientContext
 from .base import HttpClientError
 from .base import HttpRequest
 from .sync import HttpClient
@@ -31,18 +32,29 @@ class HttpxHttpClient(HttpClient):
     @dc.dataclass(frozen=True)
     class _StreamAdapter:
         it: ta.Iterator[bytes]
+        buf: ReadableListBuffer = dc.field(default_factory=ReadableListBuffer)
 
-        def read(self, /, n: int = -1) -> bytes:
-            # FIXME: lol n
+        def read1(self, /, n: int = -1) -> bytes:
             if n < 0:
-                return b''.join(self.it)
-            else:
+                if (b := self.buf.read(n)) is not None:
+                    return b
                 try:
                     return next(self.it)
                 except StopIteration:
                     return b''
 
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+            else:
+                while len(self.buf) < n:
+                    try:
+                        b = next(self.it)
+                    except StopIteration:
+                        b = b''
+                    if not b:
+                        return self.buf.read() or b''
+                    self.buf.feed(b)
+                return self.buf.read(n) or b''
+
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         try:
             resp_cm = httpx.stream(
                 method=req.method_or_default,
@@ -84,21 +96,29 @@ class HttpxAsyncHttpClient(AsyncHttpClient):
     @dc.dataclass(frozen=True)
     class _StreamAdapter:
         it: ta.AsyncIterator[bytes]
+        buf: ReadableListBuffer = dc.field(default_factory=ReadableListBuffer)
 
-        async def read(self, /, n: int = -1) -> bytes:
-            # FIXME: lol n
+        async def read1(self, /, n: int = -1) -> bytes:
             if n < 0:
-                buf = io.BytesIO()
-                async for chunk in self.it:
-                    buf.write(chunk)
-                return buf.getvalue()
-            else:
+                if (b := self.buf.read(n)) is not None:
+                    return b
                 try:
                     return await anext(self.it)
-                except StopIteration:
+                except StopAsyncIteration:
                     return b''
 
-    async def _stream_request(self, req: HttpRequest) -> AsyncStreamHttpResponse:
+            else:
+                while len(self.buf) < n:
+                    try:
+                        b = await anext(self.it)
+                    except StopAsyncIteration:
+                        b = b''
+                    if not b:
+                        return self.buf.read() or b''
+                    self.buf.feed(b)
+                return self.buf.read(n) or b''
+
+    async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
         es = contextlib.AsyncExitStack()
 
         try:

omlish/http/clients/middleware.py

@@ -0,0 +1,181 @@
+# ruff: noqa: UP007 UP043 UP045
+# @omlish-lite
+"""
+TODO:
+ - redirect
+  - referrer header?
+  - non-forwarded headers, host check, etc lol
+ - 'check' kw becomes StatusCheckingMiddleware?
+"""
+import dataclasses as dc
+import typing as ta
+import urllib.parse
+
+from ...lite.abstract import Abstract
+from ...lite.check import check
+from ..urls import parsed_url_replace
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import BaseHttpClient
+from .base import BaseHttpResponse
+from .base import HttpClientContext
+from .base import HttpClientError
+from .base import HttpRequest
+from .sync import HttpClient
+from .sync import StreamHttpResponse
+from .sync import close_http_client_response
+
+
+BaseHttpClientT = ta.TypeVar('BaseHttpClientT', bound=BaseHttpClient)
+
+
+##
+
+
+class HttpClientMiddleware(Abstract):
+    def process_request(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+    ) -> HttpRequest:
+        return req
+
+    def process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:
+        return resp
+
+
+class AbstractMiddlewareHttpClient(Abstract, ta.Generic[BaseHttpClientT]):
+    def __init__(
+            self,
+            client: BaseHttpClientT,
+            middlewares: ta.Iterable[HttpClientMiddleware],
+    ) -> None:
+        super().__init__()
+
+        self._client = client
+        self._middlewares = list(middlewares)
+
+    def _process_request(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+    ) -> HttpRequest:
+        for mw in self._middlewares:
+            req = mw.process_request(ctx, req)
+        return req
+
+    def _process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:
+        for mw in self._middlewares:
+            nxt = mw.process_response(ctx, req, resp)
+            if isinstance(nxt, HttpRequest):
+                return nxt
+            else:
+                resp = nxt
+        return resp
+
+
+#
+
+
+class MiddlewareHttpClient(AbstractMiddlewareHttpClient[HttpClient], HttpClient):
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
+        while True:
+            req = self._process_request(ctx, req)
+
+            resp = self._client.stream_request(req, context=ctx)
+
+            try:
+                out = self._process_response(ctx, req, resp)
+
+                if isinstance(out, HttpRequest):
+                    close_http_client_response(resp)
+                    req = out
+                    continue
+
+                elif isinstance(out, StreamHttpResponse):
+                    return out
+
+                else:
+                    raise TypeError(out)  # noqa
+
+            except Exception:
+                close_http_client_response(resp)
+                raise
+
+        raise RuntimeError
+
+
+class MiddlewareAsyncHttpClient(AbstractMiddlewareHttpClient[AsyncHttpClient], AsyncHttpClient):
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> ta.Awaitable[AsyncStreamHttpResponse]:
+        return self._client.stream_request(self._process_request(ctx, req))
+
+
+##
+
+
+class TooManyRedirectsHttpClientError(HttpClientError):
+    pass
+
+
+class RedirectHandlingHttpClientMiddleware(HttpClientMiddleware):
+    DEFAULT_MAX_REDIRECTS: ta.ClassVar[int] = 5
+
+    def __init__(
+            self,
+            *,
+            max_redirects: ta.Optional[int] = None,
+    ) -> None:
+        super().__init__()
+
+        if max_redirects is None:
+            max_redirects = self.DEFAULT_MAX_REDIRECTS
+        self._max_redirects = max_redirects
+
+    @dc.dataclass()
+    class _State:
+        num_redirects: int = 0
+
+    def _get_state(self, ctx: HttpClientContext) -> _State:
+        try:
+            return ctx._dct[self._State]  # noqa
+        except KeyError:
+            ret = ctx._dct[self._State] = self._State()  # noqa
+            return ret
+
+    def process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:  # noqa
+        if resp.status == 302:
+            st = self._get_state(ctx)
+            if st.num_redirects >= self._max_redirects:
+                raise TooManyRedirectsHttpClientError
+            st.num_redirects += 1
+
+            rd_url = check.not_none(resp.headers).single_str_dct['location']
+
+            rd_purl = urllib.parse.urlparse(rd_url)
+            if not rd_purl.netloc:
+                rq_purl = urllib.parse.urlparse(req.url)
+                rd_purl = parsed_url_replace(
+                    rd_purl,
+                    scheme=rq_purl.scheme,
+                    netloc=rq_purl.netloc,
+                )
+                rd_url = urllib.parse.urlunparse(rd_purl)
+
+            return dc.replace(req, url=rd_url)
+
+        return resp

omlish/http/clients/sync.py

@@ -3,13 +3,15 @@
 import abc
 import contextlib
 import dataclasses as dc
+import io
 import typing as ta
 
 from ...lite.abstract import Abstract
-from ...lite.dataclasses import dataclass_maybe_post_init
 from ...lite.dataclasses import dataclass_shallow_asdict
+from .base import BaseHttpClient
 from .base import BaseHttpResponse
 from .base import BaseHttpResponseT
+from .base import HttpClientContext
 from .base import HttpRequest
 from .base import HttpResponse
 from .base import HttpStatusError
@@ -26,21 +28,26 @@ HttpClientT = ta.TypeVar('HttpClientT', bound='HttpClient')
 @dc.dataclass(frozen=True)  # kw_only=True
 class StreamHttpResponse(BaseHttpResponse):
     class Stream(ta.Protocol):
-        def read(self, /, n: int = -1) -> bytes: ...
+        def read1(self, /, n: int = -1) -> bytes: ...
 
     @ta.final
     class _NullStream:
-        def read(self, /, n: int = -1) -> bytes:
+        def read1(self, /, n: int = -1) -> bytes:
             raise TypeError
 
     stream: Stream = _NullStream()
 
-    _closer: ta.Optional[ta.Callable[[], None]] = None
+    @property
+    def has_data(self) -> bool:
+        return not isinstance(self.stream, StreamHttpResponse._NullStream)
+
+    def read_all(self) -> bytes:
+        buf = io.BytesIO()
+        while (b := self.stream.read1()):
+            buf.write(b)
+        return buf.getvalue()
 
-    def __post_init__(self) -> None:
-        dataclass_maybe_post_init(super())
-        if isinstance(self.stream, StreamHttpResponse._NullStream):
-            raise TypeError(self.stream)
+    _closer: ta.Optional[ta.Callable[[], None]] = None
 
     def __enter__(self: StreamHttpResponseT) -> StreamHttpResponseT:
         return self
@@ -50,13 +57,13 @@ class StreamHttpResponse(BaseHttpResponse):
 
     def close(self) -> None:
         if (c := self._closer) is not None:
-            c()
+            c()  # noqa
 
 
 #
 
 
-def close_response(resp: BaseHttpResponse) -> None:
+def close_http_client_response(resp: BaseHttpResponse) -> None:
     if isinstance(resp, HttpResponse):
         pass
 
@@ -68,7 +75,7 @@ def close_response(resp: BaseHttpResponse) -> None:
 
 
 @contextlib.contextmanager
-def closing_response(resp: BaseHttpResponseT) -> ta.Iterator[BaseHttpResponseT]:
+def closing_http_client_response(resp: BaseHttpResponseT) -> ta.Iterator[BaseHttpResponseT]:
     if isinstance(resp, HttpResponse):
         yield resp
         return
@@ -81,15 +88,14 @@ def closing_response(resp: BaseHttpResponseT) -> ta.Iterator[BaseHttpResponseT]:
         raise TypeError(resp)
 
 
-def read_response(resp: BaseHttpResponse) -> HttpResponse:
+def read_http_client_response(resp: BaseHttpResponse) -> HttpResponse:
     if isinstance(resp, HttpResponse):
         return resp
 
     elif isinstance(resp, StreamHttpResponse):
-        data = resp.stream.read()
         return HttpResponse(**{
             **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('stream', '_closer')},
-            'data': data,
+            **({'data': resp.read_all()} if resp.has_data else {}),
         })
 
     else:
@@ -99,7 +105,7 @@ def read_response(resp: BaseHttpResponse) -> HttpResponse:
 ##
 
 
-class HttpClient(Abstract):
+class HttpClient(BaseHttpClient, Abstract):
     def __enter__(self: HttpClientT) -> HttpClientT:
         return self
 
@@ -110,21 +116,27 @@ class HttpClient(Abstract):
             self,
             req: HttpRequest,
             *,
+            context: ta.Optional[HttpClientContext] = None,
             check: bool = False,
     ) -> HttpResponse:
-        with closing_response(self.stream_request(
+        with closing_http_client_response(self.stream_request(
                 req,
+                context=context,
                 check=check,
         )) as resp:
-            return read_response(resp)
+            return read_http_client_response(resp)
 
     def stream_request(
             self,
             req: HttpRequest,
             *,
+            context: ta.Optional[HttpClientContext] = None,
             check: bool = False,
     ) -> StreamHttpResponse:
-        resp = self._stream_request(req)
+        if context is None:
+            context = HttpClientContext()
+
+        resp = self._stream_request(context, req)
 
         try:
             if check and not resp.is_success:
@@ -132,14 +144,14 @@ class HttpClient(Abstract):
                     cause = resp.underlying
                 else:
                     cause = None
-                raise HttpStatusError(read_response(resp)) from cause  # noqa
+                raise HttpStatusError(read_http_client_response(resp)) from cause  # noqa
 
         except Exception:
-            close_response(resp)
+            close_http_client_response(resp)
             raise
 
         return resp
 
     @abc.abstractmethod
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         raise NotImplementedError