omlish 0.0.0.dev468__py3-none-any.whl → 0.0.0.dev469__py3-none-any.whl

omlish/http/clients/httpx.py

@@ -4,14 +4,15 @@ TODO:
 """
 import contextlib
 import functools
-import io
 import typing as ta
 
 from ... import dataclasses as dc
 from ... import lang
+from ...io.buffers import ReadableListBuffer
 from ..headers import HttpHeaders
 from .asyncs import AsyncHttpClient
 from .asyncs import AsyncStreamHttpResponse
+from .base import HttpClientContext
 from .base import HttpClientError
 from .base import HttpRequest
 from .sync import HttpClient
@@ -31,18 +32,29 @@ class HttpxHttpClient(HttpClient):
     @dc.dataclass(frozen=True)
     class _StreamAdapter:
         it: ta.Iterator[bytes]
+        buf: ReadableListBuffer = dc.field(default_factory=ReadableListBuffer)
 
-        def read(self, /, n: int = -1) -> bytes:
-            # FIXME: lol n
+        def read1(self, /, n: int = -1) -> bytes:
             if n < 0:
-                return b''.join(self.it)
-            else:
+                if (b := self.buf.read(n)) is not None:
+                    return b
                 try:
                     return next(self.it)
                 except StopIteration:
                     return b''
 
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+            else:
+                while len(self.buf) < n:
+                    try:
+                        b = next(self.it)
+                    except StopIteration:
+                        b = b''
+                    if not b:
+                        return self.buf.read() or b''
+                    self.buf.feed(b)
+                return self.buf.read(n) or b''
+
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         try:
             resp_cm = httpx.stream(
                 method=req.method_or_default,
@@ -84,21 +96,29 @@ class HttpxAsyncHttpClient(AsyncHttpClient):
     @dc.dataclass(frozen=True)
     class _StreamAdapter:
         it: ta.AsyncIterator[bytes]
+        buf: ReadableListBuffer = dc.field(default_factory=ReadableListBuffer)
 
-        async def read(self, /, n: int = -1) -> bytes:
-            # FIXME: lol n
+        async def read1(self, /, n: int = -1) -> bytes:
             if n < 0:
-                buf = io.BytesIO()
-                async for chunk in self.it:
-                    buf.write(chunk)
-                return buf.getvalue()
-            else:
+                if (b := self.buf.read(n)) is not None:
+                    return b
                 try:
                     return await anext(self.it)
-                except StopIteration:
+                except StopAsyncIteration:
                     return b''
 
-    async def _stream_request(self, req: HttpRequest) -> AsyncStreamHttpResponse:
+            else:
+                while len(self.buf) < n:
+                    try:
+                        b = await anext(self.it)
+                    except StopAsyncIteration:
+                        b = b''
+                    if not b:
+                        return self.buf.read() or b''
+                    self.buf.feed(b)
+                return self.buf.read(n) or b''
+
+    async def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> AsyncStreamHttpResponse:
         es = contextlib.AsyncExitStack()
 
         try:

omlish/http/clients/middleware.py

@@ -0,0 +1,178 @@
+# ruff: noqa: UP007 UP043 UP045
+# @omlish-lite
+"""
+TODO:
+ - redirect
+  - referrer header?
+  - non-forwarded headers, host check, etc lol
+ - 'check' kw becomes StatusCheckingMiddleware?
+"""
+import dataclasses as dc
+import typing as ta
+import urllib.parse
+
+from ...lite.abstract import Abstract
+from ...lite.check import check
+from ..urls import parsed_url_replace
+from .asyncs import AsyncHttpClient
+from .asyncs import AsyncStreamHttpResponse
+from .base import BaseHttpClient
+from .base import BaseHttpResponse
+from .base import HttpClientContext
+from .base import HttpClientError
+from .base import HttpRequest
+from .sync import HttpClient
+from .sync import StreamHttpResponse
+from .sync import close_response
+
+
+BaseHttpClientT = ta.TypeVar('BaseHttpClientT', bound=BaseHttpClient)
+
+
+##
+
+
+class HttpClientMiddleware(Abstract):
+    def process_request(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+    ) -> HttpRequest:
+        return req
+
+    def process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:
+        return resp
+
+
+class AbstractMiddlewareHttpClient(Abstract, ta.Generic[BaseHttpClientT]):
+    def __init__(
+            self,
+            client: BaseHttpClientT,
+            middlewares: ta.Iterable[HttpClientMiddleware],
+    ) -> None:
+        super().__init__()
+
+        self._client = client
+        self._middlewares = list(middlewares)
+
+    def _process_request(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+    ) -> HttpRequest:
+        for mw in self._middlewares:
+            req = mw.process_request(ctx, req)
+        return req
+
+    def _process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:
+        for mw in self._middlewares:
+            nxt = mw.process_response(ctx, req, resp)
+            if isinstance(nxt, HttpRequest):
+                return nxt
+            else:
+                resp = nxt
+        return resp
+
+
+class MiddlewareHttpClient(AbstractMiddlewareHttpClient[HttpClient], HttpClient):
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
+        while True:
+            req = self._process_request(ctx, req)
+
+            resp = self._client.stream_request(req, context=ctx)
+
+            try:
+                out = self._process_response(ctx, req, resp)
+
+                if isinstance(out, HttpRequest):
+                    close_response(resp)
+                    req = out
+                    continue
+
+                elif isinstance(out, StreamHttpResponse):
+                    return out
+
+                else:
+                    raise TypeError(out)  # noqa
+
+            except Exception:
+                close_response(resp)
+                raise
+
+        raise RuntimeError
+
+
+class MiddlewareAsyncHttpClient(AbstractMiddlewareHttpClient[AsyncHttpClient], AsyncHttpClient):
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> ta.Awaitable[AsyncStreamHttpResponse]:
+        return self._client.stream_request(self._process_request(ctx, req))
+
+
+##
+
+
+class TooManyRedirectsHttpClientError(HttpClientError):
+    pass
+
+
+class RedirectHandlingHttpClientMiddleware(HttpClientMiddleware):
+    DEFAULT_MAX_REDIRECTS: ta.ClassVar[int] = 5
+
+    def __init__(
+            self,
+            *,
+            max_redirects: ta.Optional[int] = None,
+    ) -> None:
+        super().__init__()
+
+        if max_redirects is None:
+            max_redirects = self.DEFAULT_MAX_REDIRECTS
+        self._max_redirects = max_redirects
+
+    @dc.dataclass()
+    class _State:
+        num_redirects: int = 0
+
+    def _get_state(self, ctx: HttpClientContext) -> _State:
+        try:
+            return ctx._dct[self._State]  # noqa
+        except KeyError:
+            ret = ctx._dct[self._State] = self._State()  # noqa
+            return ret
+
+    def process_response(
+            self,
+            ctx: HttpClientContext,
+            req: HttpRequest,
+            resp: BaseHttpResponse,
+    ) -> ta.Union[BaseHttpResponse, HttpRequest]:  # noqa
+        if resp.status == 302:
+            st = self._get_state(ctx)
+            if st.num_redirects >= self._max_redirects:
+                raise TooManyRedirectsHttpClientError
+            st.num_redirects += 1
+
+            rd_url = check.not_none(resp.headers).single_str_dct['location']
+
+            rd_purl = urllib.parse.urlparse(rd_url)
+            if not rd_purl.netloc:
+                rq_purl = urllib.parse.urlparse(req.url)
+                rd_purl = parsed_url_replace(
+                    rd_purl,
+                    scheme=rq_purl.scheme,
+                    netloc=rq_purl.netloc,
+                )
+                rd_url = urllib.parse.urlunparse(rd_purl)
+
+            return dc.replace(req, url=rd_url)
+
+        return resp

omlish/http/clients/sync.py

@@ -3,13 +3,15 @@
 import abc
 import contextlib
 import dataclasses as dc
+import io
 import typing as ta
 
 from ...lite.abstract import Abstract
-from ...lite.dataclasses import dataclass_maybe_post_init
 from ...lite.dataclasses import dataclass_shallow_asdict
+from .base import BaseHttpClient
 from .base import BaseHttpResponse
 from .base import BaseHttpResponseT
+from .base import HttpClientContext
 from .base import HttpRequest
 from .base import HttpResponse
 from .base import HttpStatusError
@@ -26,21 +28,26 @@ HttpClientT = ta.TypeVar('HttpClientT', bound='HttpClient')
 @dc.dataclass(frozen=True)  # kw_only=True
 class StreamHttpResponse(BaseHttpResponse):
     class Stream(ta.Protocol):
-        def read(self, /, n: int = -1) -> bytes: ...
+        def read1(self, /, n: int = -1) -> bytes: ...
 
     @ta.final
     class _NullStream:
-        def read(self, /, n: int = -1) -> bytes:
+        def read1(self, /, n: int = -1) -> bytes:
             raise TypeError
 
     stream: Stream = _NullStream()
 
-    _closer: ta.Optional[ta.Callable[[], None]] = None
+    @property
+    def has_data(self) -> bool:
+        return not isinstance(self.stream, StreamHttpResponse._NullStream)
+
+    def read_all(self) -> bytes:
+        buf = io.BytesIO()
+        while (b := self.stream.read1()):
+            buf.write(b)
+        return buf.getvalue()
 
-    def __post_init__(self) -> None:
-        dataclass_maybe_post_init(super())
-        if isinstance(self.stream, StreamHttpResponse._NullStream):
-            raise TypeError(self.stream)
+    _closer: ta.Optional[ta.Callable[[], None]] = None
 
     def __enter__(self: StreamHttpResponseT) -> StreamHttpResponseT:
         return self
@@ -86,10 +93,9 @@ def read_response(resp: BaseHttpResponse) -> HttpResponse:
         return resp
 
     elif isinstance(resp, StreamHttpResponse):
-        data = resp.stream.read()
         return HttpResponse(**{
             **{k: v for k, v in dataclass_shallow_asdict(resp).items() if k not in ('stream', '_closer')},
-            'data': data,
+            **({'data': resp.read_all()} if resp.has_data else {}),
         })
 
     else:
@@ -99,7 +105,7 @@ def read_response(resp: BaseHttpResponse) -> HttpResponse:
 ##
 
 
-class HttpClient(Abstract):
+class HttpClient(BaseHttpClient, Abstract):
     def __enter__(self: HttpClientT) -> HttpClientT:
         return self
 
@@ -110,10 +116,12 @@ class HttpClient(Abstract):
             self,
             req: HttpRequest,
             *,
+            context: ta.Optional[HttpClientContext] = None,
             check: bool = False,
     ) -> HttpResponse:
         with closing_response(self.stream_request(
                 req,
+                context=context,
                 check=check,
         )) as resp:
             return read_response(resp)
@@ -122,9 +130,13 @@ class HttpClient(Abstract):
             self,
             req: HttpRequest,
             *,
+            context: ta.Optional[HttpClientContext] = None,
             check: bool = False,
     ) -> StreamHttpResponse:
-        resp = self._stream_request(req)
+        if context is None:
+            context = HttpClientContext()
+
+        resp = self._stream_request(context, req)
 
         try:
             if check and not resp.is_success:
@@ -141,5 +153,5 @@ class HttpClient(Abstract):
         return resp
 
     @abc.abstractmethod
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         raise NotImplementedError

omlish/http/clients/urllib.py

@@ -1,3 +1,5 @@
+# ruff: noqa: UP043 UP045
+# @omlish-lite
 import http.client
 import typing as ta
 import urllib.error
@@ -5,6 +7,7 @@ import urllib.request
 
 from ..headers import HttpHeaders
 from .base import DEFAULT_ENCODING
+from .base import HttpClientContext
 from .base import HttpClientError
 from .base import HttpRequest
 from .sync import HttpClient
@@ -39,7 +42,7 @@ class UrllibHttpClient(HttpClient):
             data=d,
         )
 
-    def _stream_request(self, req: HttpRequest) -> StreamHttpResponse:
+    def _stream_request(self, ctx: HttpClientContext, req: HttpRequest) -> StreamHttpResponse:
         try:
             resp = urllib.request.urlopen(  # noqa
                 self._build_request(req),
@@ -53,7 +56,6 @@ class UrllibHttpClient(HttpClient):
                     headers=HttpHeaders(e.headers.items()),
                     request=req,
                     underlying=e,
-                    stream=e,  # noqa
                     _closer=e.close,
                 )
 

omlish/http/coro/client/connection.py

@@ -116,10 +116,10 @@ class CoroHttpClientConnection:
     _http_version = 11
     _http_version_str = 'HTTP/1.1'
 
-    http_port: ta.ClassVar[int] = 80
-    https_port: ta.ClassVar[int] = 443
+    HTTP_PORT: ta.ClassVar[int] = 80
+    HTTPS_PORT: ta.ClassVar[int] = 443
 
-    default_port = http_port
+    DEFAULT_PORT: ta.ClassVar[int] = HTTP_PORT
 
     class _NOT_SET:  # noqa
         def __new__(cls, *args, **kwargs):  # noqa
@@ -139,6 +139,7 @@ class CoroHttpClientConnection:
             source_address: ta.Optional[str] = None,
             block_size: int = 8192,
             auto_open: bool = True,
+            default_port: ta.Optional[int] = None,
     ) -> None:
         super().__init__()
 
@@ -146,6 +147,9 @@ class CoroHttpClientConnection:
         self._source_address = source_address
         self._block_size = block_size
         self._auto_open = auto_open
+        if default_port is None:
+            default_port = self.DEFAULT_PORT
+        self._default_port = default_port
 
         self._connected = False
         self._buffer: ta.List[bytes] = []
@@ -162,6 +166,10 @@ class CoroHttpClientConnection:
 
         CoroHttpClientValidation.validate_host(self._host)
 
+    @property
+    def http_version(self) -> int:
+        return self._http_version
+
     #
 
     def _get_hostport(self, host: str, port: ta.Optional[int]) -> ta.Tuple[str, int]:
@@ -173,12 +181,12 @@ class CoroHttpClientConnection:
                     port = int(host[i + 1:])
                 except ValueError:
                     if host[i + 1:] == '':  # http://foo.com:/ == http://foo.com/
-                        port = self.default_port
+                        port = self._default_port
                     else:
                         raise CoroHttpClientErrors.InvalidUrlError(f"non-numeric port: '{host[i + 1:]}'") from None
                 host = host[:i]
             else:
-                port = self.default_port
+                port = self._default_port
 
         if host and host[0] == '[' and host[-1] == ']':
             host = host[1:-1]
@@ -286,6 +294,7 @@ class CoroHttpClientConnection:
                 source_address=self._source_address,
                 **(dict(timeout=self._timeout) if self._timeout is not self._NOT_SET else {}),
             ),
+            server_hostname=self._tunnel_host if self._tunnel_host else self._host,
         )))
 
         self._connected = True
@@ -526,7 +535,7 @@ class CoroHttpClientConnection:
                     if ':' in host:
                         host_enc = self._strip_ipv6_iface(host_enc)
 
-                    if port == self.default_port:
+                    if port == self._default_port:
                         self.put_header('Host', host_enc)
                     else:
                         self.put_header('Host', f"{host_enc.decode('ascii')}:{port}")

omlish/http/coro/io.py

@@ -37,6 +37,8 @@ class CoroHttpIo:
         args: ta.Tuple[ta.Any, ...]
         kwargs: ta.Optional[ta.Dict[str, ta.Any]] = None
 
+        server_hostname: ta.Optional[str] = None
+
     #
 
     class CloseIo(Io):

omlish/http/urls.py

@@ -0,0 +1,67 @@
+# ruff: noqa: UP006 UP007 UP045
+# @omlish-lite
+import re
+import typing as ta
+import urllib.parse
+
+from ..lite.cached import cached_nullary
+
+
+##
+
+
+@cached_nullary
+def _url_split_host_pat() -> re.Pattern:
+    return re.compile('//([^/#?]*)(.*)', re.DOTALL)
+
+
+def url_split_host(url: str) -> ta.Tuple[ta.Optional[str], str]:
+    """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
+
+    # https://github.com/python/cpython/blob/364ae607d8035db8ba92486ebebd8225446c1a90/Lib/urllib/parse.py#L1143
+    if not (m := _url_split_host_pat().match(url)):
+        return None, url
+
+    host_port, path = m.groups()
+    if path and path[0] != '/':
+        path = '/' + path
+    return host_port, path
+
+
+##
+
+
+def unparse_url_request_path(url: ta.Union[str, urllib.parse.ParseResult]) -> str:
+    if isinstance(url, urllib.parse.ParseResult):
+        ups = url
+    else:
+        ups = urllib.parse.urlparse(url)
+
+    return urllib.parse.urlunparse((
+        '',
+        '',
+        ups.path,
+        ups.params,
+        ups.query,
+        ups.fragment,
+    ))
+
+
+def parsed_url_replace(
+        url: urllib.parse.ParseResult,
+        *,
+        scheme: ta.Optional[str] = None,
+        netloc: ta.Optional[str] = None,
+        path: ta.Optional[str] = None,
+        params: ta.Optional[str] = None,
+        query: ta.Optional[str] = None,
+        fragment: ta.Optional[str] = None,
+) -> urllib.parse.ParseResult:
+    return urllib.parse.ParseResult(
+        scheme if scheme is not None else url.scheme,
+        netloc if netloc is not None else url.netloc,
+        path if path is not None else url.path,
+        params if params is not None else url.params,
+        query if query is not None else url.query,
+        fragment if fragment is not None else url.fragment,
+    )

omlish/io/buffers.py

@@ -208,6 +208,9 @@ class ReadableListBuffer:
 
     def read(self, n: ta.Optional[int] = None) -> ta.Optional[bytes]:
         if n is None:
+            if not self._lst:
+                return b''
+
             o = b''.join(self._lst)
             self._lst = []
             return o

omlish/lang/__init__.py

@@ -283,6 +283,7 @@ with _auto_proxy_init(globals(), update_exports=True):
 
         new_function,
         new_function_kwargs,
+        copy_function,
     )
 
     from .generators import (  # noqa
@@ -420,6 +421,8 @@ with _auto_proxy_init(globals(), update_exports=True):
     )
 
     from .params import (  # noqa
+        CanParamSpec,
+
         Param,
 
         VarParam,

omlish/lang/functions.py

@@ -250,16 +250,17 @@ def new_function(
         # a tuple that supplies the bindings for free variables
         closure: tuple | None = None,
 
-        # # a dictionary that specifies the default keyword argument values
-        # kwdefaults: dict | None = None,
+        # a dictionary that specifies the default keyword argument values
+        kwdefaults: dict | None = None,
 ) -> types.FunctionType:
+    # https://github.com/python/cpython/blob/9c8eade20c6c6cc6f31dffb5e42472391d63bbf4/Objects/funcobject.c#L909
     return types.FunctionType(
         code=code,
         globals=globals,
         name=name,
         argdefs=argdefs,
         closure=closure,
-        # kwdefaults=kwdefaults,
+        kwdefaults=kwdefaults,
     )
 
 
@@ -270,5 +271,9 @@ def new_function_kwargs(f: types.FunctionType) -> dict[str, ta.Any]:
         name=f.__name__,
         argdefs=f.__defaults__,
         closure=f.__closure__,
-        # kwdefaults=f.__kwdefaults__,
+        kwdefaults=f.__kwdefaults__,
     )
+
+
+def copy_function(f: types.FunctionType) -> types.FunctionType:
+    return new_function(**new_function_kwargs(f))

omlish/lang/params.py

@@ -1,6 +1,7 @@
 """
 TODO:
  - check validity
+ - signature vs getfullargspec - diff unwrapping + 'self' handling
 """
 import dataclasses as dc
 import enum
@@ -16,6 +17,13 @@ from .classes.restrict import Sealed
 T = ta.TypeVar('T')
 
 
+CanParamSpec: ta.TypeAlias = ta.Union[
+    'ParamSpec',
+    inspect.Signature,
+    ta.Callable,
+]
+
+
 ##
 
 
@@ -101,6 +109,15 @@ class ParamSpec(ta.Sequence[Param], Final):
 
     #
 
+    @classmethod
+    def of(cls, obj: CanParamSpec) -> 'ParamSpec':
+        if isinstance(obj, ParamSpec):
+            return obj
+        elif isinstance(obj, inspect.Signature):
+            return cls.of_signature(obj)
+        else:
+            return cls.inspect(obj)
+
     @classmethod
     def of_signature(
             cls,