hishel 1.0.0.dev1__py3-none-any.whl → 1.0.0.dev3__py3-none-any.whl

hishel/_sync_cache.py

@@ -4,7 +4,7 @@ import hashlib
 import logging
 import time
 from dataclasses import replace
-from typing import Iterator, Awaitable, Callable
+from typing import Iterable, Iterator, Awaitable, Callable
 
 from typing_extensions import assert_never
 
@@ -24,8 +24,9 @@ from hishel import (
     StoreAndUse,
     create_idle_state,
 )
-from hishel._core._spec import InvalidatePairs, vary_headers_match
-from hishel._core.models import CompletePair
+from hishel._core._spec import InvalidateEntries, vary_headers_match
+from hishel._core.models import Entry, ResponseMetadata
+from hishel._utils import make_sync_iterator
 
 logger = logging.getLogger("hishel.integrations.clients")
 
@@ -42,12 +43,12 @@ class SyncCacheProxy:
 
     def __init__(
         self,
-        send_request: Callable[[Request], Response],
+        request_sender: Callable[[Request], Response],
         storage: SyncBaseStorage | None = None,
         cache_options: CacheOptions | None = None,
         ignore_specification: bool = False,
     ) -> None:
-        self.send_request = send_request
+        self.send_request = request_sender
         self.storage = storage if storage is not None else SyncSqliteStorage()
         self.cache_options = cache_options if cache_options is not None else CacheOptions()
         self.ignore_specification = ignore_specification
@@ -59,26 +60,30 @@ class SyncCacheProxy:
 
     def _get_key_for_request(self, request: Request) -> str:
         if request.metadata.get("hishel_body_key"):
-            assert isinstance(request.stream, Iterator)
+            assert isinstance(request.stream, (Iterator, Iterable))
             collected = b"".join([chunk for chunk in request.stream])
             hash_ = hashlib.sha256(collected).hexdigest()
+            request.stream = make_sync_iterator([collected])
             return f"{str(request.url)}-{hash_}"
-        return str(request.url)
+        return hashlib.sha256(str(request.url).encode("utf-8")).hexdigest()
 
-    def _maybe_refresh_pair_ttl(self, pair: CompletePair) -> None:
+    def _maybe_refresh_pair_ttl(self, pair: Entry) -> None:
         if pair.request.metadata.get("hishel_refresh_ttl_on_access"):
-            self.storage.update_pair(
+            self.storage.update_entry(
                 pair.id,
-                lambda complete_pair: replace(complete_pair, meta=replace(complete_pair.meta, created_at=time.time())),
+                lambda complete_pair: replace(
+                    complete_pair,
+                    meta=replace(complete_pair.meta, created_at=time.time()),
+                ),
             )
 
     def _handle_request_ignoring_spec(self, request: Request) -> Response:
         logger.debug("Trying to get cached response ignoring specification")
-        pairs = self.storage.get_pairs(self._get_key_for_request(request))
+        entries = self.storage.get_entries(self._get_key_for_request(request))
 
-        logger.debug(f"Found {len(pairs)} cached pairs for the request")
+        logger.debug(f"Found {len(entries)} cached entries for the request")
 
-        for pair in pairs:
+        for pair in entries:
             if (
                 str(pair.request.url) == str(request.url)
                 and pair.request.method == request.method
@@ -90,20 +95,26 @@ class SyncCacheProxy:
                 logger.debug(
                     "Found matching cached response for the request",
                 )
-                pair.response.metadata["hishel_from_cache"] = True  # type: ignore
+                response_meta = ResponseMetadata(
+                    hishel_spec_ignored=True,
+                    hishel_from_cache=True,
+                    hishel_created_at=pair.meta.created_at,
+                    hishel_revalidated=False,
+                    hishel_stored=False,
+                )
+                pair.response.metadata.update(response_meta)  # type: ignore
                 self._maybe_refresh_pair_ttl(pair)
                 return pair.response
 
-        incomplete_pair = self.storage.create_pair(
-            request,
-        )
-        response = self.send_request(incomplete_pair.request)
+        response = self.send_request(request)
 
         logger.debug("Storing response in cache ignoring specification")
-        complete_pair = self.storage.add_response(
-            incomplete_pair.id, response, self._get_key_for_request(request)
+        entry = self.storage.create_entry(
+            request,
+            response,
+            self._get_key_for_request(request),
         )
-        return complete_pair.response
+        return entry.response
 
     def _handle_request_respecting_spec(self, request: Request) -> Response:
         state: AnyState = create_idle_state("client", self.cache_options)
@@ -125,25 +136,26 @@ class SyncCacheProxy:
                 return state.pair.response
             elif isinstance(state, NeedToBeUpdated):
                 state = self._handle_update(state)
-            elif isinstance(state, InvalidatePairs):
-                state = self._handle_invalidate_pairs(state)
+            elif isinstance(state, InvalidateEntries):
+                state = self._handle_invalidate_entries(state)
             else:
                 assert_never(state)
 
         raise RuntimeError("Unreachable")
 
     def _handle_idle_state(self, state: IdleClient, request: Request) -> AnyState:
-        stored_pairs = self.storage.get_pairs(self._get_key_for_request(request))
-        return state.next(request, stored_pairs)
+        stored_entries = self.storage.get_entries(self._get_key_for_request(request))
+        return state.next(request, stored_entries)
 
     def _handle_cache_miss(self, state: CacheMiss) -> AnyState:
-        incomplete_pair = self.storage.create_pair(state.request)
-        response = self.send_request(incomplete_pair.request)
-        return state.next(response, incomplete_pair.id)
+        response = self.send_request(state.request)
+        return state.next(response)
 
     def _handle_store_and_use(self, state: StoreAndUse, request: Request) -> Response:
-        complete_pair = self.storage.add_response(
-            state.pair_id, state.response, self._get_key_for_request(request)
+        complete_pair = self.storage.create_entry(
+            request,
+            state.response,
+            self._get_key_for_request(request),
         )
         return complete_pair.response
 
@@ -152,16 +164,17 @@ class SyncCacheProxy:
         return state.next(revalidation_response)
 
     def _handle_update(self, state: NeedToBeUpdated) -> AnyState:
-        for pair in state.updating_pairs:
-            self.storage.update_pair(
-                pair.id,
+        for entry in state.updating_entries:
+            self.storage.update_entry(
+                entry.id,
                 lambda complete_pair: replace(
-                    complete_pair, response=replace(pair.response, headers=pair.response.headers)
+                    complete_pair,
+                    response=replace(entry.response, headers=entry.response.headers),
                 ),
             )
         return state.next()
 
-    def _handle_invalidate_pairs(self, state: InvalidatePairs) -> AnyState:
-        for pair_id in state.pair_ids:
-            self.storage.remove(pair_id)
+    def _handle_invalidate_entries(self, state: InvalidateEntries) -> AnyState:
+        for entry_id in state.entry_ids:
+            self.storage.remove_entry(entry_id)
         return state.next()

hishel/_sync_httpx.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import ssl
+import typing as t
+from typing import (
+    Iterable,
+    Iterator,
+    Iterator,
+    Union,
+    cast,
+    overload,
+)
+
+from httpx import RequestNotRead
+
+from hishel import SyncCacheProxy, Headers, Request, Response
+from hishel._core._spec import (
+    CacheOptions,
+)
+from hishel._core._storages._sync_base import SyncBaseStorage
+from hishel._core.models import RequestMetadata, extract_metadata_from_headers
+from hishel._utils import (
+    filter_mapping,
+    make_sync_iterator,
+)
+
+try:
+    import httpx
+except ImportError as e:
+    raise ImportError(
+        "httpx is required to use hishel.httpx module. "
+        "Please install hishel with the 'httpx' extra, "
+        "e.g., 'pip install hishel[httpx]'."
+    ) from e
+
+SOCKET_OPTION = t.Union[
+    t.Tuple[int, int, int],
+    t.Tuple[int, int, t.Union[bytes, bytearray]],
+    t.Tuple[int, int, None, int],
+]
+
+# 128 KB
+CHUNK_SIZE = 131072
+
+
+@overload
+def _internal_to_httpx(
+    value: Request,
+) -> httpx.Request: ...
+@overload
+def _internal_to_httpx(
+    value: Response,
+) -> httpx.Response: ...
+def _internal_to_httpx(
+    value: Union[Request, Response],
+) -> Union[httpx.Request, httpx.Response]:
+    """
+    Convert internal Request/Response to httpx.Request/httpx.Response.
+    """
+    if isinstance(value, Request):
+        return httpx.Request(
+            method=value.method,
+            url=value.url,
+            headers=value.headers,
+            stream=_IteratorStream(value.iter_stream()),
+            extensions=value.metadata,
+        )
+    elif isinstance(value, Response):
+        return httpx.Response(
+            status_code=value.status_code,
+            headers=value.headers,
+            stream=_IteratorStream(value.iter_stream()),
+            extensions=value.metadata,
+        )
+
+
+@overload
+def _httpx_to_internal(
+    value: httpx.Request,
+) -> Request: ...
+@overload
+def _httpx_to_internal(
+    value: httpx.Response,
+) -> Response: ...
+def _httpx_to_internal(
+    value: Union[httpx.Request, httpx.Response],
+) -> Union[Request, Response]:
+    """
+    Convert httpx.Request/httpx.Response to internal Request/Response.
+    """
+    headers = Headers(
+        filter_mapping(
+            Headers({key: value for key, value in value.headers.items()}),
+            ["Transfer-Encoding"],
+        )
+    )
+    if isinstance(value, httpx.Request):
+        extension_metadata = RequestMetadata(
+            hishel_refresh_ttl_on_access=value.extensions.get("hishel_refresh_ttl_on_access"),
+            hishel_ttl=value.extensions.get("hishel_ttl"),
+            hishel_spec_ignore=value.extensions.get("hishel_spec_ignore"),
+            hishel_body_key=value.extensions.get("hishel_body_key"),
+        )
+        headers_metadata = extract_metadata_from_headers(value.headers)
+
+        for key, val in extension_metadata.items():
+            if key in value.extensions:
+                headers_metadata[key] = val  # type: ignore
+
+        try:
+            stream = make_sync_iterator([value.content])
+        except RequestNotRead:
+            stream = cast(Iterator[bytes], value.stream)
+
+        return Request(
+            method=value.method,
+            url=str(value.url),
+            headers=headers,
+            stream=stream,
+            metadata=headers_metadata,
+        )
+    elif isinstance(value, httpx.Response):
+        if value.is_stream_consumed and "content-encoding" in value.headers:
+            raise RuntimeError("Can't get the raw stream of a response with `Content-Encoding` header.")
+        stream = (
+            make_sync_iterator([value.content]) if value.is_stream_consumed else value.iter_raw(chunk_size=CHUNK_SIZE)
+        )
+
+        return Response(
+            status_code=value.status_code,
+            headers=headers,
+            stream=stream,
+            metadata={},
+        )
+
+
+class _IteratorStream(httpx.SyncByteStream, httpx.AsyncByteStream):
+    def __init__(self, iterator: Iterator[bytes] | Iterator[bytes]) -> None:
+        self.iterator = iterator
+
+    def __iter__(self) -> Iterator[bytes]:
+        assert isinstance(self.iterator, (Iterator, Iterable))
+        for chunk in self.iterator:
+            yield chunk
+
+
+class SyncCacheTransport(httpx.BaseTransport):
+    def __init__(
+        self,
+        next_transport: httpx.BaseTransport,
+        storage: SyncBaseStorage | None = None,
+        cache_options: CacheOptions | None = None,
+        ignore_specification: bool = False,
+    ) -> None:
+        self.next_transport = next_transport
+        self._cache_proxy: SyncCacheProxy = SyncCacheProxy(
+            request_sender=self.request_sender,
+            storage=storage,
+            cache_options=cache_options,
+            ignore_specification=ignore_specification,
+        )
+        self.storage = self._cache_proxy.storage
+
+    def handle_request(
+        self,
+        request: httpx.Request,
+    ) -> httpx.Response:
+        internal_request = _httpx_to_internal(request)
+        internal_response = self._cache_proxy.handle_request(internal_request)
+        response = _internal_to_httpx(internal_response)
+        return response
+
+    def close(self) -> None:
+        self.next_transport.close()
+        self.storage.close()
+        super().close()
+
+    def request_sender(self, request: Request) -> Response:
+        httpx_request = _internal_to_httpx(request)
+        httpx_response = self.next_transport.handle_request(httpx_request)
+        return _httpx_to_internal(httpx_response)
+
+
+class SyncCacheClient(httpx.Client):
+    def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
+        self.storage: SyncBaseStorage | None = kwargs.pop("storage", None)
+        self.cache_options: CacheOptions | None = kwargs.pop("cache_options", None)
+        self.ignore_specification: bool = kwargs.pop("ignore_specification", False)
+        super().__init__(*args, **kwargs)
+
+    def _init_transport(
+        self,
+        verify: ssl.SSLContext | str | bool = True,
+        cert: t.Union[str, t.Tuple[str, str], t.Tuple[str, str, str], None] = None,
+        trust_env: bool = True,
+        http1: bool = True,
+        http2: bool = False,
+        limits: httpx.Limits = httpx.Limits(max_connections=100, max_keepalive_connections=20),
+        transport: httpx.BaseTransport | None = None,
+        **kwargs: t.Any,
+    ) -> httpx.BaseTransport:
+        if transport is not None:
+            return transport
+
+        return SyncCacheTransport(
+            next_transport=httpx.HTTPTransport(
+                verify=verify,
+                cert=cert,
+                trust_env=trust_env,
+                http1=http1,
+                http2=http2,
+                limits=limits,
+            ),
+            storage=self.storage,
+            cache_options=self.cache_options,
+            ignore_specification=False,
+        )
+
+    def _init_proxy_transport(
+        self,
+        proxy: httpx.Proxy,
+        verify: ssl.SSLContext | str | bool = True,
+        cert: t.Union[str, t.Tuple[str, str], t.Tuple[str, str, str], None] = None,
+        trust_env: bool = True,
+        http1: bool = True,
+        http2: bool = False,
+        limits: httpx.Limits = httpx.Limits(max_connections=100, max_keepalive_connections=20),
+        **kwargs: t.Any,
+    ) -> httpx.BaseTransport:
+        return SyncCacheTransport(
+            next_transport=httpx.HTTPTransport(
+                verify=verify,
+                cert=cert,
+                trust_env=trust_env,
+                http1=http1,
+                http2=http2,
+                limits=limits,
+                proxy=proxy,
+            ),
+            storage=self.storage,
+            cache_options=self.cache_options,
+            ignore_specification=self.ignore_specification,
+        )

hishel/_utils.py

@@ -1,107 +1,17 @@
 from __future__ import annotations
 
 import calendar
-import hashlib
 import time
 import typing as tp
-from email.utils import parsedate_tz
-from typing import AsyncIterator, Generator, Iterable, Iterator, TypeVar
-
-import httpcore
-import httpx
+from email.utils import formatdate, parsedate_tz
+from pathlib import Path
+from typing import AsyncIterator, Iterable, Iterator
 
 HEADERS_ENCODING = "iso-8859-1"
 
 T = tp.TypeVar("T")
 
 
-class BaseClock:
-    def now(self) -> int:
-        raise NotImplementedError()
-
-
-class Clock(BaseClock):
-    def now(self) -> int:
-        return int(time.time())
-
-
-def normalized_url(url: tp.Union[httpcore.URL, str, bytes]) -> str:
-    if isinstance(url, str):  # pragma: no cover
-        return url
-
-    if isinstance(url, bytes):  # pragma: no cover
-        return url.decode("ascii")
-
-    if isinstance(url, httpcore.URL):
-        port = f":{url.port}" if url.port is not None else ""
-        return f"{url.scheme.decode('ascii')}://{url.host.decode('ascii')}{port}{url.target.decode('ascii')}"
-    assert False, "Invalid type for `normalized_url`"  # pragma: no cover
-
-
-def get_safe_url(url: httpcore.URL) -> str:
-    httpx_url = httpx.URL(bytes(url).decode("ascii"))
-
-    schema = httpx_url.scheme
-    host = httpx_url.host
-    path = httpx_url.path
-
-    return f"{schema}://{host}{path}"
-
-
-def generate_key(request: httpcore.Request, body: bytes = b"") -> str:
-    encoded_url = normalized_url(request.url).encode("ascii")
-
-    key_parts = [request.method, encoded_url, body]
-
-    # FIPs mode disables blake2 algorithm, use sha256 instead when not found.
-    blake2b_hasher = None
-    sha256_hasher = hashlib.sha256(usedforsecurity=False)
-    try:
-        blake2b_hasher = hashlib.blake2b(digest_size=16, usedforsecurity=False)
-    except (ValueError, TypeError, AttributeError):
-        pass
-
-    hexdigest: str
-    if blake2b_hasher:
-        for part in key_parts:
-            blake2b_hasher.update(part)
-
-        hexdigest = blake2b_hasher.hexdigest()
-    else:
-        for part in key_parts:
-            sha256_hasher.update(part)
-
-        hexdigest = sha256_hasher.hexdigest()
-    return hexdigest
-
-
-def extract_header_values(
-    headers: tp.List[tp.Tuple[bytes, bytes]],
-    header_key: tp.Union[bytes, str],
-    single: bool = False,
-) -> tp.List[bytes]:
-    if isinstance(header_key, str):
-        header_key = header_key.encode(HEADERS_ENCODING)
-    extracted_headers = []
-    for key, value in headers:
-        if key.lower() == header_key.lower():
-            extracted_headers.append(value)
-            if single:
-                break
-    return extracted_headers
-
-
-def extract_header_values_decoded(
-    headers: tp.List[tp.Tuple[bytes, bytes]], header_key: bytes, single: bool = False
-) -> tp.List[str]:
-    values = extract_header_values(headers=headers, header_key=header_key, single=single)
-    return [value.decode(HEADERS_ENCODING) for value in values]
-
-
-def header_presents(headers: tp.List[tp.Tuple[bytes, bytes]], header_key: bytes) -> bool:
-    return bool(extract_header_values(headers, header_key, single=True))
-
-
 def parse_date(date: str) -> tp.Optional[int]:
     expires = parsedate_tz(date)
     if expires is None:
@@ -114,10 +24,6 @@ def sleep(seconds: tp.Union[int, float]) -> None:
     time.sleep(seconds)
 
 
-def float_seconds_to_int_milliseconds(seconds: float) -> int:
-    return int(seconds * 1000)
-
-
 def partition(iterable: tp.Iterable[T], predicate: tp.Callable[[T], bool]) -> tp.Tuple[tp.List[T], tp.List[T]]:
     """
     Partition an iterable into two lists: one for matching items and one for non-matching items.
@@ -145,11 +51,35 @@ def partition(iterable: tp.Iterable[T], predicate: tp.Callable[[T], bool]) -> tp
     return matching, non_matching
 
 
-async def make_async_iterator(iterable: Iterable[bytes]) -> AsyncIterator[bytes]:
+async def make_async_iterator(
+    iterable: Iterable[bytes],
+) -> AsyncIterator[bytes]:
     for item in iterable:
         yield item
 
 
+def filter_mapping(mapping: tp.Mapping[str, T], keys_to_exclude: tp.Iterable[str]) -> tp.Dict[str, T]:
+    """
+        Filter out specified keys from a string-keyed mapping using case-insensitive comparison.
+
+        Args:
+            mapping: The input mapping with string keys to filter.
+            keys_to_exclude: An iterable of string keys to exclude (case-insensitive).
+
+        Returns:
+            A new dictionary with the specified keys excluded.
+
+        Example:
+    ```python
+            original = {'a': 1, 'B': 2, 'c': 3}
+            filtered = filter_mapping(original, ['b'])
+            # filtered will be {'a': 1, 'c': 3}
+    ```
+    """
+    exclude_set = {k.lower() for k in keys_to_exclude}
+    return {k: v for k, v in mapping.items() if k.lower() not in exclude_set}
+
+
 def make_sync_iterator(iterable: Iterable[bytes]) -> Iterator[bytes]:
     for item in iterable:
         yield item
@@ -177,42 +107,23 @@ def snake_to_header(text: str) -> str:
     return "X-" + "-".join(word.capitalize() for word in text.split("_"))
 
 
-_T = TypeVar("_T")
-
-
-class GeneratorWithReturnValue:
-    def __init__(
-        self, gen: Generator[None, bytes | None, bytes], stream: AsyncIterator[bytes] | Iterator[bytes]
-    ) -> None:
-        self.gen = gen
-        self.stream = stream
-        self.value: bytes | None = None
-
-    def __iter__(self) -> Iterator[bytes]:
-        return self
-
-    def __next__(self) -> bytes:
-        assert isinstance(self.stream, Iterator)
-
-        try:
-            chunk = next(self.stream)
-            self.gen.send(chunk)
-        except StopIteration as exc:
-            self.gen.send(None)
-            self.value = exc.value
-            raise
-        return chunk
-
-    def __aiter__(self) -> AsyncIterator[bytes]:
-        return self
-
-    async def __anext__(self) -> bytes:
-        assert isinstance(self.stream, AsyncIterator)
-        try:
-            chunk = await self.stream.__anext__()
-            self.gen.send(chunk)
-        except StopIteration as exc:
-            self.gen.send(None)
-            self.value = exc.value
-            raise
-        return chunk
+def ensure_cache_dict(base_path: str | None = None) -> Path:
+    _base_path = Path(base_path) if base_path is not None else Path(".cache/hishel")
+    _gitignore_file = _base_path / ".gitignore"
+
+    _base_path.mkdir(parents=True, exist_ok=True)
+
+    if not _gitignore_file.is_file():
+        with open(_gitignore_file, "w", encoding="utf-8") as f:
+            f.write("# Automatically created by Hishel\n*")
+    return _base_path
+
+
+def generate_http_date() -> str:
+    """
+    Generate a Date header value for HTTP responses.
+    Returns date in RFC 1123 format (required by HTTP/1.1).
+
+    Example output: 'Sun, 26 Oct 2025 12:34:56 GMT'
+    """
+    return formatdate(timeval=None, localtime=False, usegmt=True)