hishel 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

hishel/_utils.py

@@ -1,15 +1,25 @@
+from __future__ import annotations
+
 import calendar
 import hashlib
+import json
+import sqlite3
 import time
 import typing as tp
+from datetime import date
 from email.utils import parsedate_tz
+from typing import Any, AsyncIterator, Generator, Iterable, Iterator, TypeVar
 
 import anyio
+import anysqlite
 import httpcore
 import httpx
+from anyio import from_thread, to_thread
 
 HEADERS_ENCODING = "iso-8859-1"
 
+T = tp.TypeVar("T")
+
 
 class BaseClock:
     def now(self) -> int:
@@ -116,3 +126,333 @@ def sleep(seconds: tp.Union[int, float]) -> None:
 
 def float_seconds_to_int_milliseconds(seconds: float) -> int:
     return int(seconds * 1000)
+
+
+def partition(iterable: tp.Iterable[T], predicate: tp.Callable[[T], bool]) -> tp.Tuple[tp.List[T], tp.List[T]]:
+    """
+    Partition an iterable into two lists: one for matching items and one for non-matching items.
+
+    Args:
+        iterable (tp.Iterable[T]): The input iterable to partition.
+        predicate (tp.Callable[[T], bool]): A function that evaluates each item in the iterable.
+
+    Returns:
+        tp.Tuple[tp.List[T], tp.List[T]]: A tuple containing two lists: the first for matching items,
+        and the second for non-matching items.
+    Example:
+        ```
+        iterable = [1, 2, 3, 4, 5]
+        is_even = lambda x: x % 2 == 0
+        evens, odds = partition(iterable, is_even)
+        ```
+    """
+    matching, non_matching = [], []
+    for item in iterable:
+        if predicate(item):
+            matching.append(item)
+        else:
+            non_matching.append(item)
+    return matching, non_matching
+
+
+def async_iterator_to_sync(iterator: AsyncIterator[bytes]) -> Iterator[bytes]:
+    """
+    Convert an asynchronous byte iterator to a synchronous one.
+    This function takes an asynchronous iterator that yields bytes and converts it into
+    a synchronous iterator.
+
+    Args:
+        iterator (AsyncIterator[bytes]): The asynchronous byte iterator to be converted.
+    Returns:
+        Iterator[bytes]: A synchronous iterator that yields the same byte chunks as the input iterator.
+    Example:
+        ```python
+        async_iter = some_async_byte_stream()
+        sync_iter = async_iterator_to_sync(async_iter)
+        for chunk in sync_iter:
+            process_bytes(chunk)
+        ```
+    """
+
+    while True:
+        try:
+            chunk = from_thread.run(iterator.__anext__)
+        except StopAsyncIteration:
+            break
+        yield chunk
+
+
+def _call_next(iterator: Iterator[bytes]) -> bytes:
+    try:
+        return iterator.__next__()
+    except StopIteration:
+        raise StopAsyncIteration
+
+
+async def sync_iterator_to_async(iterator: Iterator[bytes]) -> AsyncIterator[bytes]:
+    """
+    Converts a synchronous bytes iterator to an asynchronous one.
+    This function takes a synchronous iterator that yields bytes and converts it into an
+    asynchronous iterator, allowing it to be used in async contexts without blocking.
+    Args:
+        iterator (Iterator[bytes]): A synchronous iterator yielding bytes objects.
+    Returns:
+        AsyncIterator[bytes]: An asynchronous iterator yielding the same bytes objects.
+    Example:
+        ```
+        sync_iter = iter([b'data1', b'data2'])
+        async for chunk in sync_iterator_to_async(sync_iter):
+            await process_chunk(chunk)
+        ```
+    """
+
+    while True:
+        try:
+            chunk = await to_thread.run_sync(_call_next, iterator)
+        except StopAsyncIteration:
+            break
+
+        yield chunk
+
+
+async def make_async_iterator(iterable: Iterable[bytes]) -> AsyncIterator[bytes]:
+    for item in iterable:
+        yield item
+
+
+def make_sync_iterator(iterable: Iterable[bytes]) -> Iterator[bytes]:
+    for item in iterable:
+        yield item
+
+
+def snake_to_header(text: str) -> str:
+    """
+    Convert snake_case string to Header-Case format.
+
+    Args:
+        text: Snake case string (e.g., "hishel_ttl")
+
+    Returns:
+        Header case string (e.g., "X-Hishel-Ttl")
+
+    Examples:
+        >>> snake_to_header("hishel_ttl")
+        'X-Hishel-Ttl'
+        >>> snake_to_header("cache_control")
+        'X-Cache-Control'
+        >>> snake_to_header("content_type")
+        'X-Content-Type'
+    """
+    # Split by underscore, capitalize each word, join with dash, add X- prefix
+    return "X-" + "-".join(word.capitalize() for word in text.split("_"))
+
+
+_T = TypeVar("_T")
+
+
+class GeneratorWithReturnValue:
+    def __init__(
+        self, gen: Generator[None, bytes | None, bytes], stream: AsyncIterator[bytes] | Iterator[bytes]
+    ) -> None:
+        self.gen = gen
+        self.stream = stream
+        self.value: bytes | None = None
+
+    def __iter__(self) -> Iterator[bytes]:
+        return self
+
+    def __next__(self) -> bytes:
+        assert isinstance(self.stream, Iterator)
+
+        try:
+            chunk = next(self.stream)
+            self.gen.send(chunk)
+        except StopIteration as exc:
+            self.gen.send(None)
+            self.value = exc.value
+            raise
+        return chunk
+
+    def __aiter__(self) -> AsyncIterator[bytes]:
+        return self
+
+    async def __anext__(self) -> bytes:
+        assert isinstance(self.stream, AsyncIterator)
+        try:
+            chunk = await self.stream.__anext__()
+            self.gen.send(chunk)
+        except StopIteration as exc:
+            self.gen.send(None)
+            self.value = exc.value
+            raise
+        return chunk
+
+
+def print_sqlite_state(conn: sqlite3.Connection) -> str:
+    """
+    Print all tables and their rows in a pretty format suitable for inline snapshots.
+
+    Args:
+        conn: SQLite database connection
+
+    Returns:
+        Formatted string representation of the database state
+    """
+    cursor = conn.cursor()
+
+    # Get all table names
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+    tables = [row[0] for row in cursor.fetchall()]
+
+    output_lines = []
+    output_lines.append("=" * 80)
+    output_lines.append("DATABASE SNAPSHOT")
+    output_lines.append("=" * 80)
+
+    for table_name in tables:
+        # Get column information
+        cursor.execute(f"PRAGMA table_info({table_name})")
+        columns = cursor.fetchall()
+        column_names = [col[1] for col in columns]
+        column_types = {col[1]: col[2] for col in columns}
+
+        # Get all rows
+        cursor.execute(f"SELECT * FROM {table_name}")
+        rows = cursor.fetchall()
+
+        output_lines.append("")
+        output_lines.append(f"TABLE: {table_name}")
+        output_lines.append("-" * 80)
+        output_lines.append(f"Rows: {len(rows)}")
+        output_lines.append("")
+
+        if not rows:
+            output_lines.append("  (empty)")
+            continue
+
+        # Format each row
+        for idx, row in enumerate(rows, 1):
+            output_lines.append(f"  Row {idx}:")
+
+            for col_name, value in zip(column_names, row):
+                col_type = column_types[col_name]
+                formatted_value = format_value(value, col_name, col_type)
+                output_lines.append(f"    {col_name:15} = {formatted_value}")
+
+            if idx < len(rows):
+                output_lines.append("")
+
+    output_lines.append("")
+    output_lines.append("=" * 80)
+
+    result = "\n".join(output_lines)
+    return result
+
+
+async def aprint_sqlite_state(conn: anysqlite.Connection) -> str:
+    """
+    Print all tables and their rows in a pretty format suitable for inline snapshots.
+
+    Args:
+        conn: SQLite database connection
+
+    Returns:
+        Formatted string representation of the database state
+    """
+    cursor = await conn.cursor()
+
+    # Get all table names
+    await cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+    tables = [row[0] for row in await cursor.fetchall()]
+
+    output_lines = []
+    output_lines.append("=" * 80)
+    output_lines.append("DATABASE SNAPSHOT")
+    output_lines.append("=" * 80)
+
+    for table_name in tables:
+        # Get column information
+        await cursor.execute(f"PRAGMA table_info({table_name})")
+        columns = await cursor.fetchall()
+        column_names = [col[1] for col in columns]
+        column_types = {col[1]: col[2] for col in columns}
+
+        # Get all rows
+        await cursor.execute(f"SELECT * FROM {table_name}")
+        rows = await cursor.fetchall()
+
+        output_lines.append("")
+        output_lines.append(f"TABLE: {table_name}")
+        output_lines.append("-" * 80)
+        output_lines.append(f"Rows: {len(rows)}")
+        output_lines.append("")
+
+        if not rows:
+            output_lines.append("  (empty)")
+            continue
+
+        # Format each row
+        for idx, row in enumerate(rows, 1):
+            output_lines.append(f"  Row {idx}:")
+
+            for col_name, value in zip(column_names, row):
+                col_type = column_types[col_name]
+                formatted_value = format_value(value, col_name, col_type)
+                output_lines.append(f"    {col_name:15} = {formatted_value}")
+
+            if idx < len(rows):
+                output_lines.append("")
+
+    output_lines.append("")
+    output_lines.append("=" * 80)
+
+    result = "\n".join(output_lines)
+    return result
+
+
+def format_value(value: Any, col_name: str, col_type: str) -> str:
+    """Format a value for display based on its type and column name."""
+
+    if value is None:
+        return "NULL"
+
+    # Handle BLOB columns
+    if col_type.upper() == "BLOB":
+        if isinstance(value, bytes):
+            # Try to decode as UTF-8 string first
+            try:
+                decoded = value.decode("utf-8")
+                # Check if it looks like JSON
+                if decoded.strip().startswith("{") or decoded.strip().startswith("["):
+                    try:
+                        parsed = json.loads(decoded)
+                        return f"(JSON) {json.dumps(parsed, indent=2)}"
+                    except json.JSONDecodeError:
+                        pass
+                # Show string if it's printable
+                if all(32 <= ord(c) <= 126 or c in "\n\r\t" for c in decoded):
+                    return f"(str) '{decoded}'"
+            except UnicodeDecodeError:
+                pass
+
+            # Show hex representation for binary data
+            hex_str = value.hex()
+            if len(hex_str) > 64:
+                return f"(bytes) 0x{hex_str[:60]}... ({len(value)} bytes)"
+            return f"(bytes) 0x{hex_str} ({len(value)} bytes)"
+        return repr(value)
+
+    # Handle timestamps - ONLY show date, not the raw timestamp
+    if col_name.endswith("_at") and isinstance(value, (int, float)):
+        try:
+            dt = date.fromtimestamp(value)
+            return dt.isoformat()  # Changed: removed the timestamp prefix
+        except (ValueError, OSError):
+            return str(value)
+
+    # Handle TEXT columns
+    if col_type.upper() == "TEXT":
+        return f"'{value}'"
+
+    # Handle other types
+    return str(value)

hishel/beta/__init__.py

@@ -0,0 +1,59 @@
+from hishel.beta._core._async._storages._sqlite import AsyncSqliteStorage
+from hishel.beta._core._base._storages._base import (
+    AsyncBaseStorage as AsyncBaseStorage,
+    SyncBaseStorage as SyncBaseStorage,
+)
+from hishel.beta._core._headers import Headers as Headers
+from hishel.beta._core._spec import (
+    AnyState as AnyState,
+    CacheMiss as CacheMiss,
+    CacheOptions as CacheOptions,
+    CouldNotBeStored as CouldNotBeStored,
+    FromCache as FromCache,
+    IdleClient as IdleClient,
+    NeedRevalidation as NeedRevalidation,
+    NeedToBeUpdated as NeedToBeUpdated,
+    State as State,
+    StoreAndUse as StoreAndUse,
+    create_idle_state as create_idle_state,
+)
+from hishel.beta._core._sync._storages._sqlite import SyncSqliteStorage
+from hishel.beta._core.models import (
+    CompletePair as CompletePair,
+    IncompletePair as IncompletePair,
+    Pair as Pair,
+    PairMeta as PairMeta,
+    Request as Request,
+    Response,
+)
+
+__all__ = (
+    # New API
+    ## States
+    "AnyState",
+    "IdleClient",
+    "CacheMiss",
+    "FromCache",
+    "NeedRevalidation",
+    "AnyState",
+    "CacheOptions",
+    "NeedToBeUpdated",
+    "State",
+    "StoreAndUse",
+    "CouldNotBeStored",
+    "create_idle_state",
+    ## Models
+    "Request",
+    "Response",
+    "Pair",
+    "IncompletePair",
+    "CompletePair",
+    "PairMeta",
+    ## Headers
+    "Headers",
+    ## Storages
+    "SyncBaseStorage",
+    "AsyncBaseStorage",
+    "SyncSqliteStorage",
+    "AsyncSqliteStorage",
+)

hishel/beta/_async_cache.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+import time
+from dataclasses import replace
+from typing import AsyncIterator, Awaitable, Callable
+
+from typing_extensions import assert_never
+
+from hishel.beta import (
+    AnyState,
+    AsyncBaseStorage,
+    AsyncSqliteStorage,
+    CacheMiss,
+    CacheOptions,
+    CouldNotBeStored,
+    FromCache,
+    IdleClient,
+    NeedRevalidation,
+    NeedToBeUpdated,
+    Request,
+    Response,
+    StoreAndUse,
+    create_idle_state,
+)
+from hishel.beta._core._spec import InvalidatePairs, vary_headers_match
+from hishel.beta._core.models import CompletePair
+
+logger = logging.getLogger("hishel.integrations.clients")
+
+
+class AsyncCacheProxy:
+    """
+    A proxy for HTTP caching in clients.
+
+    This class is independent of any specific HTTP library and works only with internal models.
+    It delegates request execution to a user-provided callable, making it compatible with any
+    HTTP client. Caching behavior can be configured to either fully respect HTTP
+    caching rules or bypass them entirely.
+    """
+
+    def __init__(
+        self,
+        send_request: Callable[[Request], Awaitable[Response]],
+        storage: AsyncBaseStorage | None = None,
+        cache_options: CacheOptions | None = None,
+        ignore_specification: bool = False,
+    ) -> None:
+        self.send_request = send_request
+        self.storage = storage if storage is not None else AsyncSqliteStorage()
+        self.cache_options = cache_options if cache_options is not None else CacheOptions()
+        self.ignore_specification = ignore_specification
+
+    async def handle_request(self, request: Request) -> Response:
+        if self.ignore_specification or request.metadata.get("hishel_spec_ignore"):
+            return await self._handle_request_ignoring_spec(request)
+        return await self._handle_request_respecting_spec(request)
+
+    async def _get_key_for_request(self, request: Request) -> str:
+        if request.metadata.get("hishel_body_key"):
+            assert isinstance(request.stream, AsyncIterator)
+            collected = b"".join([chunk async for chunk in request.stream])
+            hash_ = hashlib.sha256(collected).hexdigest()
+            return f"{str(request.url)}-{hash_}"
+        return str(request.url)
+
+    async def _maybe_refresh_pair_ttl(self, pair: CompletePair) -> None:
+        if pair.request.metadata.get("hishel_refresh_ttl_on_access"):
+            await self.storage.update_pair(
+                pair.id,
+                lambda complete_pair: replace(complete_pair, meta=replace(complete_pair.meta, created_at=time.time())),
+            )
+
+    async def _handle_request_ignoring_spec(self, request: Request) -> Response:
+        logger.debug("Trying to get cached response ignoring specification")
+        pairs = await self.storage.get_pairs(await self._get_key_for_request(request))
+
+        logger.debug(f"Found {len(pairs)} cached pairs for the request")
+
+        for pair in pairs:
+            if (
+                str(pair.request.url) == str(request.url)
+                and pair.request.method == request.method
+                and vary_headers_match(
+                    request,
+                    pair,
+                )
+            ):
+                logger.debug(
+                    "Found matching cached response for the request",
+                )
+                pair.response.metadata["hishel_from_cache"] = True  # type: ignore
+                await self._maybe_refresh_pair_ttl(pair)
+                return pair.response
+
+        incomplete_pair = await self.storage.create_pair(
+            request,
+        )
+        response = await self.send_request(incomplete_pair.request)
+
+        logger.debug("Storing response in cache ignoring specification")
+        complete_pair = await self.storage.add_response(
+            incomplete_pair.id, response, await self._get_key_for_request(request)
+        )
+        return complete_pair.response
+
+    async def _handle_request_respecting_spec(self, request: Request) -> Response:
+        state: AnyState = create_idle_state("client", self.cache_options)
+
+        while state:
+            logger.debug(f"Handling state: {state.__class__.__name__}")
+            if isinstance(state, IdleClient):
+                state = await self._handle_idle_state(state, request)
+            elif isinstance(state, CacheMiss):
+                state = await self._handle_cache_miss(state)
+            elif isinstance(state, StoreAndUse):
+                return await self._handle_store_and_use(state, request)
+            elif isinstance(state, CouldNotBeStored):
+                return state.response
+            elif isinstance(state, NeedRevalidation):
+                state = await self._handle_revalidation(state)
+            elif isinstance(state, FromCache):
+                await self._maybe_refresh_pair_ttl(state.pair)
+                return state.pair.response
+            elif isinstance(state, NeedToBeUpdated):
+                state = await self._handle_update(state)
+            elif isinstance(state, InvalidatePairs):
+                state = await self._handle_invalidate_pairs(state)
+            else:
+                assert_never(state)
+
+        raise RuntimeError("Unreachable")
+
+    async def _handle_idle_state(self, state: IdleClient, request: Request) -> AnyState:
+        stored_pairs = await self.storage.get_pairs(await self._get_key_for_request(request))
+        return state.next(request, stored_pairs)
+
+    async def _handle_cache_miss(self, state: CacheMiss) -> AnyState:
+        incomplete_pair = await self.storage.create_pair(state.request)
+        response = await self.send_request(incomplete_pair.request)
+        return state.next(response, incomplete_pair.id)
+
+    async def _handle_store_and_use(self, state: StoreAndUse, request: Request) -> Response:
+        complete_pair = await self.storage.add_response(
+            state.pair_id, state.response, await self._get_key_for_request(request)
+        )
+        return complete_pair.response
+
+    async def _handle_revalidation(self, state: NeedRevalidation) -> AnyState:
+        revalidation_response = await self.send_request(state.request)
+        return state.next(revalidation_response)
+
+    async def _handle_update(self, state: NeedToBeUpdated) -> AnyState:
+        for pair in state.updating_pairs:
+            await self.storage.update_pair(
+                pair.id,
+                lambda complete_pair: replace(
+                    complete_pair, response=replace(pair.response, headers=pair.response.headers)
+                ),
+            )
+        return state.next()
+
+    async def _handle_invalidate_pairs(self, state: InvalidatePairs) -> AnyState:
+        for pair_id in state.pair_ids:
+            await self.storage.remove(pair_id)
+        return state.next()