omlish 0.0.0.dev423__py3-none-any.whl → 0.0.0.dev484__py3-none-any.whl

omlish/http/clients/executor.py

@@ -0,0 +1,56 @@
+# ruff: noqa: UP043 UP045
+# @omlish-lite
+import dataclasses as dc
+import typing as ta
+
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import HttpClientContext
+from .base import HttpRequest
+from .sync import HttpClient
+from .sync import StreamHttpResponse
+
+
+##
+
+
+class ExecutorAsyncHttpClient(AsyncHttpClient):
+    def __init__(
+            self,
+            run_in_executor: ta.Callable[..., ta.Awaitable],
+            client: HttpClient,
+    ) -> None:
+        super().__init__()
+
+        self._run_in_executor = run_in_executor
+        self._client = client
+
+    @dc.dataclass(frozen=True)
+    class _StreamAdapter:
+        owner: 'ExecutorAsyncHttpClient'
+        resp: StreamHttpResponse
+
+        async def read1(self, n: int = -1, /) -> bytes:
+            return await self.owner._run_in_executor(self.resp.stream.read1, n)  # noqa
+
+        async def read(self, n: int = -1, /) -> bytes:
+            return await self.owner._run_in_executor(self.resp.stream.read, n)  # noqa
+
+        async def readall(self) -> bytes:
+            return await self.owner._run_in_executor(self.resp.stream.readall)  # noqa
+
+        async def close(self) -> None:
+            return await self.owner._run_in_executor(self.resp.close)  # noqa
+
+    async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
+        resp: StreamHttpResponse = await self._run_in_executor(lambda: self._client.stream_request(req, context=ctx))
+        return AsyncStreamHttpResponse(
+            status=resp.status,
+            headers=resp.headers,
+            request=req,
+            underlying=resp,
+            **(dict(  # type: ignore
+                _stream=(adapter := self._StreamAdapter(self, resp)),
+                _closer=adapter.close,
+            ) if resp.has_data else {}),
+        )

omlish/http/clients/httpx.py

@@ -1,13 +1,22 @@
+"""
+TODO:
+ - standardize following redirects
+"""
+import contextlib
 import functools
 import typing as ta
 
 from ... import dataclasses as dc
 from ... import lang
+from ...io.buffers import ReadableListBuffer
 from ..headers import HttpHeaders
-from .base import HttpClient
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import HttpClientContext
 from .base import HttpClientError
 from .base import HttpRequest
-from .base import StreamHttpResponse
+from .sync import HttpClient
+from .sync import StreamHttpResponse
 
 
 if ta.TYPE_CHECKING:
@@ -24,16 +33,13 @@ class HttpxHttpClient(HttpClient):
     class _StreamAdapter:
         it: ta.Iterator[bytes]
 
-        def read(self, /, n: int = -1) -> bytes:
-            if n < 0:
-                return b''.join(self.it)
-            else:
-                try:
-                    return next(self.it)
-                except StopIteration:
-                    return b''
+        def read1(self, n: int = -1, /) -> bytes:
+            try:
+                return next(self.it)
+            except StopIteration:
+                return b''
 
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         try:
             resp_cm = httpx.stream(
                 method=req.method_or_default,
@@ -55,7 +61,7 @@ class HttpxHttpClient(HttpClient):
                 headers=HttpHeaders(resp.headers.raw),
                 request=req,
                 underlying=resp,
-                stream=self._StreamAdapter(resp.iter_bytes()),
+                _stream=ReadableListBuffer().new_buffered_reader(self._StreamAdapter(resp.iter_bytes())),
                 _closer=resp_close,  # type: ignore
             )
 
@@ -63,6 +69,65 @@ class HttpxHttpClient(HttpClient):
             resp_close()
             raise HttpClientError from e
 
-        except Exception:
+        except BaseException:
             resp_close()
             raise
+
+
+##
+
+
+class HttpxAsyncHttpClient(AsyncHttpClient):
+    @dc.dataclass(frozen=True)
+    class _StreamAdapter:
+        it: ta.AsyncIterator[bytes]
+
+        async def read1(self, n: int = -1, /) -> bytes:
+            try:
+                return await anext(self.it)
+            except StopAsyncIteration:
+                return b''
+
+    async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
+        es = contextlib.AsyncExitStack()
+
+        try:
+            client = await es.enter_async_context(httpx.AsyncClient())
+
+            resp = await es.enter_async_context(client.stream(
+                method=req.method_or_default,
+                url=req.url,
+                headers=req.headers_ or None,  # type: ignore
+                content=req.data,
+                timeout=req.timeout_s,
+            ))
+
+            it = resp.aiter_bytes()
+
+            # FIXME:
+            #  this has a tendency to raise `RuntimeError: async generator ignored GeneratorExit` when all of the
+            #  following conditions are met:
+            #   - stopped iterating midway through
+            #   - shutting down the event loop
+            #   - debugging under pycharm / pydevd
+            #   - running under asyncio
+            #  it does not seem to happen unless all of these conditions are met. see:
+            #   https://gist.github.com/wrmsr/a0578ee5d5371b53804cfb56aeb84cdf .
+            es.push_async_callback(it.aclose)  # type: ignore[attr-defined]
+
+            return AsyncStreamHttpResponse(
+                status=resp.status_code,
+                headers=HttpHeaders(resp.headers.raw),
+                request=req,
+                underlying=resp,
+                _stream=ReadableListBuffer().new_async_buffered_reader(self._StreamAdapter(it)),
+                _closer=es.aclose,
+            )
+
+        except httpx.HTTPError as e:
+            await es.aclose()
+            raise HttpClientError from e
+
+        except BaseException:
+            await es.aclose()
+            raise

omlish/http/clients/middleware.py

@@ -0,0 +1,181 @@
+# ruff: noqa: UP007 UP043 UP045
+# @omlish-lite
+"""
+TODO:
+ - redirect
+  - referrer header?
+  - non-forwarded headers, host check, etc lol
+ - 'check' kw becomes StatusCheckingMiddleware?
+"""
+import dataclasses as dc
+import typing as ta
+import urllib.parse
+
+from ...lite.abstract import Abstract
+from ...lite.check import check
+from ..urls import parsed_url_replace
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import BaseHttpClient
+from .base import BaseHttpResponse
+from .base import HttpClientContext
+from .base import HttpClientError
+from .base import HttpRequest
+from .sync import HttpClient
+from .sync import StreamHttpResponse
+from .sync import close_http_client_response
+
+
+BaseHttpClientT = ta.TypeVar('BaseHttpClientT', bound=BaseHttpClient)
+
+
+##
+
+
+class HttpClientMiddleware(Abstract):
+    def process_request(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+    ) -> HttpRequest:
+        return req
+
+    def process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:
+        return resp
+
+
+class AbstractMiddlewareHttpClient(Abstract, ta.Generic[BaseHttpClientT]):
+    def __init__(
+            self,
+            client: BaseHttpClientT,
+            middlewares: ta.Iterable[HttpClientMiddleware],
+    ) -> None:
+        super().__init__()
+
+        self._client = client
+        self._middlewares = list(middlewares)
+
+    def _process_request(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+    ) -> HttpRequest:
+        for mw in self._middlewares:
+            req = mw.process_request(ctx, req)
+        return req
+
+    def _process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:
+        for mw in self._middlewares:
+            nxt = mw.process_response(ctx, req, resp)
+            if isinstance(nxt, HttpRequest):
+                return nxt
+            else:
+                resp = nxt
+        return resp
+
+
+#
+
+
+class MiddlewareHttpClient(AbstractMiddlewareHttpClient[HttpClient], HttpClient):
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
+        while True:
+            req = self._process_request(ctx, req)
+
+            resp = self._client.stream_request(req, context=ctx)
+
+            try:
+                out = self._process_response(ctx, req, resp)
+
+                if isinstance(out, HttpRequest):
+                    close_http_client_response(resp)
+                    req = out
+                    continue
+
+                elif isinstance(out, StreamHttpResponse):
+                    return out
+
+                else:
+                    raise TypeError(out)  # noqa
+
+            except Exception:
+                close_http_client_response(resp)
+                raise
+
+        raise RuntimeError
+
+
+class MiddlewareAsyncHttpClient(AbstractMiddlewareHttpClient[AsyncHttpClient], AsyncHttpClient):
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> ta.Awaitable[AsyncStreamHttpResponse]:
+        return self._client.stream_request(self._process_request(ctx, req))
+
+
+##
+
+
+class TooManyRedirectsHttpClientError(HttpClientError):
+    pass
+
+
+class RedirectHandlingHttpClientMiddleware(HttpClientMiddleware):
+    DEFAULT_MAX_REDIRECTS: ta.ClassVar[int] = 5
+
+    def __init__(
+            self,
+            *,
+            max_redirects: ta.Optional[int] = None,
+    ) -> None:
+        super().__init__()
+
+        if max_redirects is None:
+            max_redirects = self.DEFAULT_MAX_REDIRECTS
+        self._max_redirects = max_redirects
+
+    @dc.dataclass()
+    class _State:
+        num_redirects: int = 0
+
+    def _get_state(self, ctx: HttpClientContext) -> _State:
+        try:
+            return ctx._dct[self._State]  # noqa
+        except KeyError:
+            ret = ctx._dct[self._State] = self._State()  # noqa
+            return ret
+
+    def process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:  # noqa
+        if resp.status == 302:
+            st = self._get_state(ctx)
+            if st.num_redirects >= self._max_redirects:
+                raise TooManyRedirectsHttpClientError
+            st.num_redirects += 1
+
+            rd_url = check.not_none(resp.headers).single_str_dct['location']
+
+            rd_purl = urllib.parse.urlparse(rd_url)
+            if not rd_purl.netloc:
+                rq_purl = urllib.parse.urlparse(req.url)
+                rd_purl = parsed_url_replace(
+                    rd_purl,
+                    scheme=rq_purl.scheme,
+                    netloc=rq_purl.netloc,
+                )
+                rd_url = urllib.parse.urlunparse(rd_purl)
+
+            return dc.replace(req, url=rd_url)
+
+        return resp

omlish/http/clients/sync.py

@@ -0,0 +1,151 @@
+# ruff: noqa: UP045
+# @omlish-lite
+import abc
+import contextlib
+import dataclasses as dc
+import typing as ta
+
+from ...io.readers import BufferedBytesReader
+from ...lite.abstract import Abstract
+from ...lite.dataclasses import dataclass_shallow_asdict
+from .base import BaseHttpClient
+from .base import BaseHttpResponse
+from .base import BaseHttpResponseT
+from .base import HttpClientContext
+from .base import HttpRequest
+from .base import HttpResponse
+from .base import HttpStatusError
+
+
+StreamHttpResponseT = ta.TypeVar('StreamHttpResponseT', bound='StreamHttpResponse')
+HttpClientT = ta.TypeVar('HttpClientT', bound='HttpClient')
+
+
+##
+
+
+@ta.final
+@dc.dataclass(frozen=True)  # kw_only=True
+class StreamHttpResponse(BaseHttpResponse):
+    _stream: ta.Optional[BufferedBytesReader] = None
+
+    @property
+    def stream(self) -> 'BufferedBytesReader':
+        if (st := self._stream) is None:
+            raise TypeError('No data')
+        return st
+
+    @property
+    def has_data(self) -> bool:
+        return self._stream is not None
+
+    #
+
+    _closer: ta.Optional[ta.Callable[[], None]] = None
+
+    def __enter__(self: StreamHttpResponseT) -> StreamHttpResponseT:
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def close(self) -> None:
+        if (c := self._closer) is not None:
+            c()  # noqa
+
+
+#
+
+
+def close_http_client_response(resp: BaseHttpResponse) -> None:
+    if isinstance(resp, HttpResponse):
+        pass
+
+    elif isinstance(resp, StreamHttpResponse):
+        resp.close()
+
+    else:
+        raise TypeError(resp)
+
+
+@contextlib.contextmanager
+def closing_http_client_response(resp: BaseHttpResponseT) -> ta.Iterator[BaseHttpResponseT]:
+    if isinstance(resp, HttpResponse):
+        yield resp
+        return
+
+    elif isinstance(resp, StreamHttpResponse):
+        with contextlib.closing(resp):
+            yield resp
+
+    else:
+        raise TypeError(resp)
+
+
+def read_http_client_response(resp: BaseHttpResponse) -> HttpResponse:
+    if isinstance(resp, HttpResponse):
+        return resp
+
+    elif isinstance(resp, StreamHttpResponse):
+        return HttpResponse(**{
+            **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('_stream', '_closer')},
+            **({'data': resp.stream.readall()} if resp.has_data else {}),
+        })
+
+    else:
+        raise TypeError(resp)
+
+
+##
+
+
+class HttpClient(BaseHttpClient, Abstract):
+    def __enter__(self: HttpClientT) -> HttpClientT:
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def request(
+            self,
+            req: HttpRequest,
+            *,
+            context: ta.Optional[HttpClientContext] = None,
+            check: bool = False,
+    ) -> HttpResponse:
+        with closing_http_client_response(self.stream_request(
+                req,
+                context=context,
+                check=check,
+        )) as resp:
+            return read_http_client_response(resp)
+
+    def stream_request(
+            self,
+            req: HttpRequest,
+            *,
+            context: ta.Optional[HttpClientContext] = None,
+            check: bool = False,
+    ) -> StreamHttpResponse:
+        if context is None:
+            context = HttpClientContext()
+
+        resp = self._stream_request(context, req)
+
+        try:
+            if check and not resp.is_success:
+                if isinstance(resp.underlying, Exception):
+                    cause = resp.underlying
+                else:
+                    cause = None
+                raise HttpStatusError(read_http_client_response(resp)) from cause  # noqa
+
+        except Exception:
+            close_http_client_response(resp)
+            raise
+
+        return resp
+
+    @abc.abstractmethod
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
+        raise NotImplementedError

omlish/http/clients/syncasync.py

@@ -0,0 +1,49 @@
+# ruff: noqa: UP043 UP045
+# @omlish-lite
+import dataclasses as dc
+
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import HttpClientContext
+from .base import HttpRequest
+from .sync import HttpClient
+from .sync import StreamHttpResponse
+
+
+##
+
+
+class SyncAsyncHttpClient(AsyncHttpClient):
+    def __init__(self, client: HttpClient) -> None:
+        super().__init__()
+
+        self._client = client
+
+    @dc.dataclass(frozen=True)
+    class _StreamAdapter:
+        ul: StreamHttpResponse
+
+        async def read1(self, n: int = -1, /) -> bytes:
+            return self.ul.stream.read1(n)
+
+        async def read(self, n: int = -1, /) -> bytes:
+            return self.ul.stream.read(n)
+
+        async def readall(self) -> bytes:
+            return self.ul.stream.readall()
+
+        async def close(self) -> None:
+            self.ul.close()
+
+    async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
+        resp = self._client.stream_request(req, context=ctx)
+        return AsyncStreamHttpResponse(
+            status=resp.status,
+            headers=resp.headers,
+            request=req,
+            underlying=resp,
+            **(dict(  # type: ignore
+                _stream=(adapter := self._StreamAdapter(resp)),
+                _closer=adapter.close,
+            ) if resp.has_data else {}),
+        )

omlish/http/clients/urllib.py

@@ -1,14 +1,18 @@
+# ruff: noqa: UP043 UP045
+# @omlish-lite
 import http.client
 import typing as ta
 import urllib.error
 import urllib.request
 
+from ...io.buffers import ReadableListBuffer
 from ..headers import HttpHeaders
 from .base import DEFAULT_ENCODING
-from .base import HttpClient
+from .base import HttpClientContext
 from .base import HttpClientError
 from .base import HttpRequest
-from .base import StreamHttpResponse
+from .sync import HttpClient
+from .sync import StreamHttpResponse
 
 
 ##
@@ -39,7 +43,7 @@ class UrllibHttpClient(HttpClient):
             data=d,
         )
 
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         try:
             resp = urllib.request.urlopen(  # noqa
                 self._build_request(req),
@@ -53,7 +57,6 @@ class UrllibHttpClient(HttpClient):
                     headers=HttpHeaders(e.headers.items()),
                     request=req,
                     underlying=e,
-                    stream=e,  # noqa
                     _closer=e.close,
                 )
 
@@ -70,7 +73,7 @@ class UrllibHttpClient(HttpClient):
                 headers=HttpHeaders(resp.headers.items()),
                 request=req,
                 underlying=resp,
-                stream=resp,
+                _stream=ReadableListBuffer().new_buffered_reader(resp),
                 _closer=resp.close,
             )