datacurrent 1.0.0__py3-none-any.whl

datacurrent/reliability/retry.py

@@ -0,0 +1,134 @@
+"""Retry policies with exponential backoff and jitter."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+from ..exceptions import ConfigurationError
+
+T = TypeVar("T")
+
+__all__ = ["RetryPolicy", "is_retryable", "retry_async"]
+
+
+def is_retryable(predicate: Any, error: BaseException) -> bool:
+    """Return whether ``error`` matches a retry predicate.
+
+    ``predicate`` may be an exception class, a tuple of classes, or a callable
+    taking the exception and returning a bool.
+    """
+    if predicate is None:
+        return True
+    if isinstance(predicate, tuple):
+        return isinstance(error, predicate)
+    if isinstance(predicate, type) and issubclass(predicate, BaseException):
+        return isinstance(error, predicate)
+    if callable(predicate):
+        return bool(predicate(error))
+    raise ConfigurationError(
+        f"retry predicate must be an exception class, tuple or callable, got {predicate!r}"
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class RetryPolicy:
+    """Declarative retry configuration.
+
+    Args:
+        max_attempts: Total attempts including the initial one (>= 1).
+        base_delay: Delay before the first retry, in seconds.
+        max_delay: Upper bound for a single delay, in seconds.
+        multiplier: Exponential growth factor (>= 1).
+        jitter: Randomization fraction in ``[0, 1]``; 0 disables jitter.
+        retry_on: Exception class, tuple of classes or predicate.
+    """
+
+    max_attempts: int = 3
+    base_delay: float = 0.1
+    max_delay: float = 10.0
+    multiplier: float = 2.0
+    jitter: float = 0.1
+    retry_on: Any = Exception
+
+    def __post_init__(self) -> None:
+        if isinstance(self.max_attempts, bool) or not isinstance(self.max_attempts, int):
+            raise ConfigurationError("max_attempts must be an integer")
+        if self.max_attempts < 1:
+            raise ConfigurationError("max_attempts must be >= 1")
+        for name in ("base_delay", "max_delay", "multiplier", "jitter"):
+            value = getattr(self, name)
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                raise ConfigurationError(f"{name} must be a number")
+            if value < 0:
+                raise ConfigurationError(f"{name} must be >= 0")
+        if self.multiplier < 1:
+            raise ConfigurationError("multiplier must be >= 1")
+        if self.jitter > 1:
+            raise ConfigurationError("jitter must be between 0 and 1")
+
+    def should_retry(self, error: BaseException) -> bool:
+        return is_retryable(self.retry_on, error)
+
+    def delay_for(self, attempt: int, *, rng: random.Random | None = None) -> float:
+        """Delay before retry number ``attempt`` (1 = first retry)."""
+        if attempt < 1:
+            raise ConfigurationError("attempt must be >= 1")
+        if self.base_delay == 0:
+            return 0.0
+        exponent = attempt - 1
+        delay = min(float(self.max_delay), float(self.base_delay) * (self.multiplier**exponent))
+        if self.jitter:
+            source = rng.random() if rng is not None else random.random()
+            spread = delay * self.jitter
+            delay = delay - spread + (2 * spread * source)
+        return max(0.0, delay)
+
+    @classmethod
+    def coerce(cls, value: RetryPolicy | int | Mapping[str, Any] | None) -> RetryPolicy:
+        """Build a policy from a policy, an attempt count or a mapping."""
+        if isinstance(value, cls):
+            return value
+        if value is None:
+            return cls()
+        if isinstance(value, bool):
+            raise ConfigurationError("retry policy must not be a bool")
+        if isinstance(value, int):
+            return cls(max_attempts=value)
+        if isinstance(value, Mapping):
+            return cls(**dict(value))
+        raise ConfigurationError(
+            f"cannot interpret {value!r} as a RetryPolicy; pass RetryPolicy(...), "
+            "an integer attempt count, or a mapping of policy fields"
+        )
+
+
+async def retry_async(
+    func: Callable[[], Awaitable[T]],
+    policy: RetryPolicy | int | Mapping[str, Any] | None = None,
+    *,
+    on: Any = None,
+) -> T:
+    """Call ``func`` until it succeeds or the policy is exhausted.
+
+    The final error is re-raised unchanged, so callers keep the original
+    traceback and exception type.
+    """
+    resolved = RetryPolicy.coerce(policy)
+    predicate = resolved.retry_on if on is None else on
+    attempt = 1
+    while True:
+        try:
+            return await func()
+        except asyncio.CancelledError:
+            raise
+        except Exception as error:
+            if attempt >= resolved.max_attempts or not is_retryable(predicate, error):
+                raise
+            delay = resolved.delay_for(attempt)
+            attempt += 1
+            if delay:
+                await asyncio.sleep(delay)

datacurrent/reliability/timeout.py

@@ -0,0 +1,39 @@
+"""Timeout helpers that raise DataCurrent's timeout error type."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable
+from typing import Any, TypeVar
+
+from ..exceptions import ConfigurationError, StreamTimeoutError
+
+T = TypeVar("T")
+
+__all__ = ["validate_timeout", "with_timeout"]
+
+
+def validate_timeout(seconds: Any, *, name: str = "timeout") -> float:
+    if isinstance(seconds, bool) or not isinstance(seconds, (int, float)):
+        raise ConfigurationError(f"{name} must be a number, got {type(seconds).__name__}")
+    if seconds <= 0:
+        raise ConfigurationError(f"{name} must be > 0, got {seconds}")
+    return float(seconds)
+
+
+async def with_timeout(
+    awaitable: Awaitable[T],
+    seconds: float,
+    *,
+    what: str = "operation",
+) -> T:
+    """Await ``awaitable`` for at most ``seconds``.
+
+    Raises:
+        StreamTimeoutError: The deadline elapsed.
+    """
+    timeout = validate_timeout(seconds)
+    try:
+        return await asyncio.wait_for(awaitable, timeout=timeout)
+    except TimeoutError:
+        raise StreamTimeoutError(f"{what} exceeded {timeout:g}s") from None

datacurrent/transports/__init__.py

@@ -0,0 +1,31 @@
+"""Transports: in-memory fan-out, SSE, HTTP streaming and WebSocket clients."""
+
+from __future__ import annotations
+
+from .base import StreamTransport
+from .http import HTTPStream
+from .memory import Channel
+from .sse import (
+    SSEEvent,
+    SSELineDecoder,
+    SSEParser,
+    SSEStream,
+    format_heartbeat,
+    format_sse,
+    parse_sse,
+)
+from .websocket import WebSocketStream
+
+__all__ = [
+    "Channel",
+    "HTTPStream",
+    "SSEEvent",
+    "SSELineDecoder",
+    "SSEParser",
+    "SSEStream",
+    "StreamTransport",
+    "WebSocketStream",
+    "format_heartbeat",
+    "format_sse",
+    "parse_sse",
+]

datacurrent/transports/_deps.py

@@ -0,0 +1,23 @@
+"""Lazy imports for optional transport dependencies."""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+
+from ..exceptions import OptionalDependencyError
+
+__all__ = ["require_module"]
+
+
+def require_module(
+    name: str,
+    *,
+    extra: str | None = None,
+    purpose: str,
+) -> ModuleType:
+    """Import a dependency or raise a helpful install error."""
+    try:
+        return importlib.import_module(name)
+    except ImportError as error:  # pragma: no cover - depends on environment
+        raise OptionalDependencyError(name.split(".")[0], extra=extra, purpose=purpose) from error

datacurrent/transports/base.py

@@ -0,0 +1,80 @@
+"""Extension point for custom transports.
+
+A transport is just an object that produces messages and can be closed.  Build
+one when your protocol is not covered by the SSE, HTTP or WebSocket adapters::
+
+    class MyTransport(StreamTransport[dict]):
+        async def open(self): ...
+        async def close(self): ...
+        async def messages(self): yield {"...": "..."}
+
+    stream = MyTransport().as_stream()
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator
+from typing import Any, Generic, TypeVar
+
+from ..core.stream import Stream
+from ..exceptions import StreamClosedError
+
+T = TypeVar("T")
+
+__all__ = ["StreamTransport"]
+
+
+class StreamTransport(ABC, Generic[T]):
+    """Base class for transports that expose an async stream of messages."""
+
+    def __init__(self, *, name: str | None = None) -> None:
+        self.name = name or type(self).__name__
+        self._opened = False
+        self._closed = False
+
+    @property
+    def is_open(self) -> bool:
+        return self._opened and not self._closed
+
+    @abstractmethod
+    async def open(self) -> None:
+        """Establish the connection / start producing."""
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release every resource.  Must be idempotent."""
+
+    @abstractmethod
+    def messages(self) -> AsyncIterator[T]:
+        """Yield messages until the transport is closed."""
+
+    async def __aenter__(self) -> StreamTransport[T]:
+        await self.open()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        await self.close()
+
+    def as_stream(self, **stream_kwargs: Any) -> Stream[T]:
+        """Expose the transport as a single-consumer stream.
+
+        Closing the stream closes the transport, so ``break`` cannot leak a
+        connection.
+        """
+        transport = self
+
+        async def factory() -> AsyncIterator[T]:
+            if not transport.is_open:
+                await transport.open()
+            try:
+                async for message in transport.messages():
+                    yield message
+            finally:
+                await transport.close()
+
+        return Stream(factory, replayable=False, **stream_kwargs)
+
+    async def _ensure_open(self) -> None:
+        if self._closed:
+            raise StreamClosedError(f"transport {self.name!r} is closed")

datacurrent/transports/http.py

@@ -0,0 +1,211 @@
+"""HTTP streaming over ``httpx``.
+
+Use :class:`HTTPStream` to consume slow endpoints, streaming downloads or
+newline-delimited JSON without buffering the whole response in memory.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Mapping
+from typing import Any
+
+from ..config import DEFAULT_TIMEOUT
+from ..core.stream import Stream
+from ..exceptions import ConfigurationError, MessageTooLargeError, StreamClosedError
+from ._deps import require_module
+
+__all__ = ["HTTPStream"]
+
+
+class HTTPStream:
+    """An HTTP response consumed incrementally.
+
+    Example::
+
+        async with HTTPStream("https://example.com/feed.ndjson") as response:
+            async for line in response.iter_lines():
+                print(line)
+
+    The object must be opened with ``async with``; closing it releases the
+    response and the owned HTTP client.
+    """
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        method: str = "GET",
+        headers: Mapping[str, str] | None = None,
+        params: Mapping[str, Any] | None = None,
+        json: Any = None,
+        data: Any = None,
+        content: Any = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        chunk_size: int = 64 * 1024,
+        client: Any = None,
+        max_bytes: int | None = None,
+        raise_for_status: bool = True,
+        follow_redirects: bool = True,
+    ) -> None:
+        if not isinstance(url, str) or not url.lower().startswith(("http://", "https://")):
+            raise ConfigurationError("HTTPStream requires an http:// or https:// URL")
+        if isinstance(chunk_size, bool) or not isinstance(chunk_size, int) or chunk_size < 1:
+            raise ConfigurationError("chunk_size must be an integer >= 1")
+        if max_bytes is not None and max_bytes < 1:
+            raise ConfigurationError("max_bytes must be >= 1 when provided")
+        self._url = url
+        self._method = method.upper()
+        self._headers = dict(headers or {})
+        self._params = dict(params) if params else None
+        self._json = json
+        self._data = data
+        self._content = content
+        self._timeout = float(timeout)
+        self._chunk_size = chunk_size
+        self._client = client
+        self._owned_client: Any = None
+        self._max_bytes = max_bytes
+        self._raise_for_status = raise_for_status
+        self._follow_redirects = follow_redirects
+        self._response: Any = None
+        self._closed = False
+
+    # ------------------------------------------------------------------ #
+    # Lifecycle
+    # ------------------------------------------------------------------ #
+    @classmethod
+    def get(cls, url: str, **kwargs: Any) -> HTTPStream:
+        """Convenience constructor for GET requests."""
+        kwargs.setdefault("method", "GET")
+        return cls(url, **kwargs)
+
+    @property
+    def url(self) -> str:
+        return self._url
+
+    @property
+    def status_code(self) -> int:
+        return int(self._require_response().status_code)
+
+    @property
+    def response(self) -> Any:
+        return self._require_response()
+
+    async def __aenter__(self) -> HTTPStream:
+        httpx = require_module("httpx", purpose="HTTPStream")
+        if self._client is not None:
+            client = self._client
+        else:
+            client = httpx.AsyncClient(
+                follow_redirects=self._follow_redirects,
+                timeout=self._timeout,
+            )
+            self._owned_client = client
+        self._client = client
+        request = client.build_request(
+            self._method,
+            self._url,
+            headers=self._headers,
+            params=self._params,
+            json=self._json,
+            data=self._data,
+            content=self._content,
+            timeout=self._timeout,
+        )
+        self._response = await client.send(request, stream=True)
+        if self._raise_for_status and self._response.is_error:
+            await self._response.aread()
+            try:
+                self._response.raise_for_status()
+            finally:
+                await self._close_response()
+                await self._close_owned_client()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        await self.aclose()
+
+    async def _close_response(self) -> None:
+        response, self._response = self._response, None
+        if response is not None:
+            await response.aclose()
+
+    async def _close_owned_client(self) -> None:
+        client, self._owned_client = self._owned_client, None
+        if client is not None:
+            await client.aclose()
+
+    async def aclose(self) -> None:
+        """Close the response and the owned client.  Idempotent."""
+        if self._closed:
+            return
+        self._closed = True
+        await self._close_response()
+        await self._close_owned_client()
+
+    def __aiter__(self) -> AsyncIterator[bytes]:
+        return self.iter_bytes()
+
+    def _require_response(self) -> Any:
+        if self._response is None:
+            raise StreamClosedError(
+                "HTTPStream is not open; use 'async with HTTPStream(...) as response'"
+            )
+        return self._response
+
+    # ------------------------------------------------------------------ #
+    # Body iteration
+    # ------------------------------------------------------------------ #
+    async def iter_bytes(self, *, chunk_size: int | None = None) -> AsyncIterator[bytes]:
+        """Yield decoded response chunks as they arrive."""
+        response = self._require_response()
+        size = self._chunk_size if chunk_size is None else chunk_size
+        async for chunk in response.aiter_bytes(size):
+            yield chunk
+
+    async def iter_text(self, encoding: str = "utf-8") -> AsyncIterator[str]:
+        """Yield text chunks decoded with ``encoding``."""
+        async for chunk in self.iter_bytes():
+            yield chunk.decode(encoding, errors="replace")
+
+    async def iter_lines(self) -> AsyncIterator[str]:
+        """Yield response lines without their terminators."""
+        response = self._require_response()
+        async for line in response.aiter_lines():
+            yield line
+
+    async def read_all(self, *, max_bytes: int | None = None) -> bytes:
+        """Read the whole body, honouring the configured byte limit."""
+        limit = self._max_bytes if max_bytes is None else max_bytes
+        chunks: list[bytes] = []
+        total = 0
+        async for chunk in self.iter_bytes():
+            total += len(chunk)
+            if limit is not None and total > limit:
+                raise MessageTooLargeError(total, limit)
+            chunks.append(chunk)
+        return b"".join(chunks)
+
+    def as_stream(self, *, config: Any = None) -> Stream[bytes]:
+        """Expose the body as a stream of chunks.
+
+        The stream owns the response and closes it when iteration ends.
+        """
+        http = self
+
+        async def factory() -> AsyncIterator[bytes]:
+            if http._response is None:
+                await http.__aenter__()
+            try:
+                async for chunk in http.iter_bytes():
+                    yield chunk
+            finally:
+                await http.aclose()
+
+        return Stream(factory, config=config, replayable=False)
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return (
+            f"HTTPStream(method={self._method!r}, url={self._url!r}, "
+            f"open={self._response is not None})"
+        )

datacurrent/transports/memory.py

@@ -0,0 +1,171 @@
+"""In-memory fan-out with explicit delivery semantics.
+
+Use :class:`Channel` when several consumers must each receive every message.
+A ``Stream`` is single-consumer; a channel gives every subscriber its own bounded
+queue.  Delivery is *at most once* and lives only in this process: a subscriber
+that is not connected when a message is published will not see it.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Generic, TypeVar
+
+from ..buffering.backpressure import BufferPolicy
+from ..buffering.queue import BoundedQueue
+from ..config import OverflowPolicy, StreamConfig
+from ..core.stream import Stream
+from ..exceptions import QueueClosedError, StreamClosedError
+from ..observability.metrics import StreamStats
+
+T = TypeVar("T")
+
+__all__ = ["Channel"]
+
+
+class Channel(Generic[T]):
+    """A bounded broadcast channel.
+
+    Args:
+        buffer_size: Per-subscriber queue capacity.
+        overflow: Policy used when a subscriber queue is full.  The default
+            ``BLOCK`` applies backpressure to publishers instead of losing data.
+        name: Optional name used in reprs and errors.
+
+    Example::
+
+        channel = Channel(buffer_size=32)
+        first = channel.subscribe()
+        second = channel.subscribe()
+
+        await channel.publish({"n": 1})   # both subscribers receive it
+
+        await channel.aclose()
+    """
+
+    def __init__(
+        self,
+        *,
+        buffer_size: int = 256,
+        overflow: OverflowPolicy | str = OverflowPolicy.BLOCK,
+        name: str | None = None,
+    ) -> None:
+        self._policy = BufferPolicy(size=buffer_size, overflow=OverflowPolicy.coerce(overflow))
+        self._name = name
+        self._subscribers: set[BoundedQueue[T]] = set()
+        self._closed = False
+        self._published = 0
+        self._delivered = 0
+        self._dropped = 0
+
+    @property
+    def name(self) -> str | None:
+        return self._name
+
+    @property
+    def subscriber_count(self) -> int:
+        return len(self._subscribers)
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    def subscribe(
+        self,
+        *,
+        buffer_size: int | None = None,
+        overflow: OverflowPolicy | str | None = None,
+        name: str | None = None,
+    ) -> Stream[T]:
+        """Create a new single-consumer stream of published messages."""
+        if self._closed:
+            raise StreamClosedError(f"channel {self._name!r} is closed")
+        policy = BufferPolicy(
+            size=self._policy.size if buffer_size is None else buffer_size,
+            overflow=OverflowPolicy.coerce(self._policy.overflow if overflow is None else overflow),
+        )
+
+        def record_drop() -> None:
+            self._dropped += 1
+
+        queue: BoundedQueue[T] = policy.build(name=self._name or "channel", on_drop=record_drop)
+        self._subscribers.add(queue)
+        stats = StreamStats(name=name or self._name or "channel")
+        config = StreamConfig(
+            name=name or self._name,
+            buffer_size=policy.size,
+            overflow=policy.overflow,
+            observe=True,
+        )
+
+        channel = self
+
+        async def factory() -> Any:
+            try:
+                async for item in queue:
+                    stats.items_out += 1
+                    yield item
+            finally:
+                channel._subscribers.discard(queue)
+                await queue.aclose()
+
+        stream: Stream[T] = Stream(factory, config=config, replayable=False)
+        stream._stats = stats
+        return stream
+
+    async def publish(self, item: T) -> int:
+        """Deliver ``item`` to every active subscriber.
+
+        Returns the number of subscriber queues that accepted the item.  A
+        subscriber whose queue is closed is removed automatically.
+        """
+        if self._closed:
+            raise StreamClosedError(f"channel {self._name!r} is closed")
+        self._published += 1
+        delivered = 0
+        for queue in list(self._subscribers):
+            try:
+                if await queue.put(item):
+                    delivered += 1
+            except QueueClosedError:
+                self._subscribers.discard(queue)
+        self._delivered += delivered
+        return delivered
+
+    async def publish_many(self, items: Any) -> int:
+        """Publish every item in ``items``; returns the number of deliveries."""
+        total = 0
+        for item in items:
+            total += await self.publish(item)
+        return total
+
+    async def aclose(self) -> None:
+        """Close the channel and every subscriber queue."""
+        if self._closed:
+            return
+        self._closed = True
+        subscribers = list(self._subscribers)
+        self._subscribers.clear()
+        for queue in subscribers:
+            await queue.aclose()
+
+    async def __aenter__(self) -> Channel[T]:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        await self.aclose()
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            "name": self._name,
+            "published": self._published,
+            "delivered": self._delivered,
+            "dropped": self._dropped,
+            "subscribers": len(self._subscribers),
+            "closed": self._closed,
+        }
+
+    def __repr__(self) -> str:
+        return (
+            f"Channel(name={self._name!r}, subscribers={len(self._subscribers)}, "
+            f"buffer_size={self._policy.size}, overflow={self._policy.overflow.value!r})"
+        )