datacurrent 1.0.0__py3-none-any.whl

datacurrent/exceptions.py

@@ -0,0 +1,191 @@
+"""Exception hierarchy used across DataCurrent.
+
+Every exception raised deliberately by the library derives from
+:class:`DataCurrentError`.  Applications can therefore catch one base class
+without accidentally swallowing programming errors such as ``TypeError`` or
+control-flow exceptions such as ``asyncio.CancelledError``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = [
+    "BufferOverflowError",
+    "ConfigurationError",
+    "ConnectionFailedError",
+    "DataCurrentError",
+    "EmptyStreamError",
+    "EventBusError",
+    "MessageTooLargeError",
+    "NonReplayableStreamError",
+    "OptionalDependencyError",
+    "QueueClosedError",
+    "QueueEmptyError",
+    "RetryError",
+    "SSEProtocolError",
+    "StreamClosedError",
+    "StreamConsumedError",
+    "StreamError",
+    "StreamProtocolError",
+    "StreamTimeoutError",
+    "SubscriptionClosedError",
+    "TransportError",
+    "WebSocketClosedError",
+]
+
+
+class DataCurrentError(Exception):
+    """Base class for every error raised by DataCurrent."""
+
+
+class ConfigurationError(DataCurrentError, ValueError):
+    """The caller supplied an invalid configuration value."""
+
+
+class OptionalDependencyError(DataCurrentError, ImportError):
+    """An optional integration was used without its dependency installed."""
+
+    def __init__(
+        self,
+        package: str,
+        *,
+        extra: str | None = None,
+        purpose: str = "This feature",
+    ) -> None:
+        self.package = package
+        self.extra = extra
+        hint = f'pip install "datacurrent[{extra}]"' if extra else f"pip install {package}"
+        super().__init__(
+            f"{purpose} requires the optional dependency {package!r}. Install it with: {hint}"
+        )
+
+
+# --------------------------------------------------------------------------- #
+# Streams
+# --------------------------------------------------------------------------- #
+class StreamError(DataCurrentError):
+    """Base class for errors related to stream consumption."""
+
+
+class StreamConsumedError(StreamError):
+    """A single-consumer stream was iterated more than once."""
+
+
+class NonReplayableStreamError(StreamError):
+    """An operation required restarting a stream that cannot be replayed."""
+
+
+class EmptyStreamError(StreamError):
+    """A terminal operation required at least one item but the stream was empty."""
+
+
+class StreamTimeoutError(StreamError, TimeoutError):
+    """No item arrived within the configured timeout."""
+
+
+class StreamClosedError(StreamError):
+    """The stream or its underlying queue has been closed."""
+
+
+# --------------------------------------------------------------------------- #
+# Buffering
+# --------------------------------------------------------------------------- #
+class BufferOverflowError(StreamError, BufferError):
+    """A bounded buffer was full and the configured policy is ``error``.
+
+    Attributes:
+        maxsize: Capacity of the buffer that overflowed.
+        policy: Overflow policy that produced the error.
+        dropped: Number of items dropped in total by that buffer.
+    """
+
+    def __init__(
+        self,
+        maxsize: int,
+        *,
+        policy: Any = "error",
+        dropped: int = 0,
+    ) -> None:
+        self.maxsize = maxsize
+        self.policy = getattr(policy, "value", policy)
+        self.dropped = dropped
+        super().__init__(
+            f"buffer is full (maxsize={maxsize}, policy={self.policy!r}, "
+            f"dropped={dropped}); slow down the producer or enlarge the buffer"
+        )
+
+
+class QueueClosedError(DataCurrentError):
+    """An operation was attempted on a closed queue."""
+
+
+class QueueEmptyError(DataCurrentError):
+    """A non-blocking read was attempted on an empty queue."""
+
+
+# --------------------------------------------------------------------------- #
+# Transports
+# --------------------------------------------------------------------------- #
+class TransportError(DataCurrentError):
+    """Base class for networking and transport failures."""
+
+
+class ConnectionFailedError(TransportError):
+    """A connection could not be established or was lost."""
+
+
+class StreamProtocolError(TransportError):
+    """The peer sent data that violates the expected protocol."""
+
+
+class SSEProtocolError(StreamProtocolError):
+    """A Server-Sent Events response was malformed or had the wrong content type."""
+
+
+class MessageTooLargeError(TransportError):
+    """A received message exceeded the configured limit."""
+
+    def __init__(self, size: int, limit: int) -> None:
+        self.size = size
+        self.limit = limit
+        super().__init__(f"message of {size} bytes exceeds the limit of {limit} bytes")
+
+
+class WebSocketClosedError(TransportError):
+    """The WebSocket connection was closed before reading finished."""
+
+    def __init__(self, code: int | None = None, reason: str | None = None) -> None:
+        self.code = code
+        self.reason = reason
+        detail = []
+        if code is not None:
+            detail.append(f"code={code}")
+        if reason:
+            detail.append(f"reason={reason}")
+        suffix = f" ({', '.join(detail)})" if detail else ""
+        super().__init__(f"websocket connection closed{suffix}")
+
+
+# --------------------------------------------------------------------------- #
+# Reliability
+# --------------------------------------------------------------------------- #
+class RetryError(DataCurrentError):
+    """All retry attempts were exhausted."""
+
+    def __init__(self, attempts: int, last_error: BaseException | None = None) -> None:
+        self.attempts = attempts
+        self.last_error = last_error
+        suffix = f": {last_error!r}" if last_error is not None else ""
+        super().__init__(f"giving up after {attempts} attempt(s){suffix}")
+
+
+# --------------------------------------------------------------------------- #
+# Events
+# --------------------------------------------------------------------------- #
+class EventBusError(DataCurrentError):
+    """Base class for event bus failures."""
+
+
+class SubscriptionClosedError(EventBusError):
+    """The subscription is no longer able to receive events."""

datacurrent/integrations/__init__.py

@@ -0,0 +1,18 @@
+"""Framework integrations.
+
+The helpers here import FastAPI/Starlette lazily, so ``import datacurrent`` and
+this package work in applications that do not use a web framework.
+"""
+
+from __future__ import annotations
+
+from .fastapi import sse_response, websocket_messages, websocket_stream
+from .starlette import StreamLifespan, streaming_response
+
+__all__ = [
+    "StreamLifespan",
+    "sse_response",
+    "streaming_response",
+    "websocket_messages",
+    "websocket_stream",
+]

datacurrent/integrations/fastapi.py

@@ -0,0 +1,76 @@
+"""FastAPI helpers: SSE responses and WebSocket message streams.
+
+FastAPI is optional and imported lazily inside the functions.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterable, AsyncIterator, Mapping
+from typing import Any
+
+from ..core._util import aclose
+from ..core.stream import Stream
+from ..transports.sse import SSEEvent, format_sse
+
+__all__ = ["sse_response", "websocket_messages", "websocket_stream"]
+
+
+def sse_response(
+    events: AsyncIterable[SSEEvent] | Stream[SSEEvent],
+    *,
+    headers: Mapping[str, str] | None = None,
+    status_code: int = 200,
+) -> Any:
+    """Return a ``StreamingResponse`` that encodes events as ``text/event-stream``."""
+    from starlette.responses import StreamingResponse
+
+    default_headers = {
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "X-Accel-Buffering": "no",
+    }
+    merged = {**default_headers, **dict(headers or {})}
+
+    async def body() -> AsyncIterator[bytes]:
+        source: AsyncIterable[SSEEvent] = events
+        iterator = source.__aiter__()
+        try:
+            async for event in iterator:
+                yield format_sse(event).encode("utf-8")
+        finally:
+            await aclose(iterator)
+
+    return StreamingResponse(
+        body(),
+        media_type="text/event-stream",
+        headers=merged,
+        status_code=status_code,
+    )
+
+
+async def websocket_messages(websocket: Any) -> AsyncIterator[Any]:
+    """Yield text and binary messages from a Starlette/FastAPI WebSocket."""
+    while True:
+        message = await websocket.receive()
+        message_type = message.get("type")
+        if message_type == "websocket.disconnect":
+            return
+        if "text" in message and message["text"] is not None:
+            yield message["text"]
+        elif "bytes" in message and message["bytes"] is not None:
+            yield message["bytes"]
+
+
+def websocket_stream(websocket: Any, *, config: Any = None) -> Stream[Any]:
+    """Expose an incoming WebSocket as a single-consumer stream."""
+
+    async def factory() -> AsyncIterator[Any]:
+        try:
+            async for message in websocket_messages(websocket):
+                yield message
+        finally:
+            closer = getattr(websocket, "close", None)
+            if closer is not None:
+                await aclose(websocket)
+
+    return Stream(factory, config=config, replayable=False)

datacurrent/integrations/starlette.py

@@ -0,0 +1,75 @@
+"""Starlette helpers (also used by the FastAPI integration).
+
+Starlette is optional: importing this module never imports Starlette itself.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterable, Mapping
+from typing import Any
+
+from ..core._util import aclose
+from ..core.stream import Stream
+
+__all__ = ["StreamLifespan", "streaming_response"]
+
+
+def streaming_response(
+    stream: Stream[Any] | AsyncIterable[Any],
+    *,
+    media_type: str = "text/plain; charset=utf-8",
+    headers: Mapping[str, str] | None = None,
+    status_code: int = 200,
+) -> Any:
+    """Return a Starlette ``StreamingResponse`` backed by a stream.
+
+    Bytes are forwarded untouched and strings are encoded as UTF-8.  The stream
+    is closed when the response finishes or the client disconnects.
+    """
+    from starlette.responses import StreamingResponse
+
+    source = stream.__aiter__() if isinstance(stream, Stream) else stream
+
+    async def body() -> Any:
+        iterator = source.__aiter__()
+        try:
+            async for chunk in iterator:
+                yield chunk.encode("utf-8") if isinstance(chunk, str) else chunk
+        finally:
+            await aclose(iterator)
+
+    return StreamingResponse(
+        body(),
+        media_type=media_type,
+        headers=dict(headers or {}),
+        status_code=status_code,
+    )
+
+
+class StreamLifespan:
+    """Run async closables during application startup/shutdown.
+
+    Example::
+
+        lifespan = StreamLifespan(bus, channel)
+
+        app = FastAPI(lifespan=lifespan.as_context_manager())
+    """
+
+    def __init__(self, *resources: Any) -> None:
+        self._resources = list(resources)
+
+    async def __aenter__(self) -> StreamLifespan:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        resources, self._resources = self._resources, []
+        for resource in reversed(resources):
+            await aclose(resource)
+
+    def as_context_manager(self) -> Any:
+        """Return the lifespan as an async context manager for ASGI frameworks."""
+        return self

datacurrent/observability/__init__.py

@@ -0,0 +1,17 @@
+"""Observability: metrics, statistics and safe logging."""
+
+from __future__ import annotations
+
+from .logging import get_logger, redact_headers, redact_secret, redact_url
+from .metrics import Counter, Metrics, StreamStats, snapshot_stats
+
+__all__ = [
+    "Counter",
+    "Metrics",
+    "StreamStats",
+    "get_logger",
+    "redact_headers",
+    "redact_secret",
+    "redact_url",
+    "snapshot_stats",
+]

datacurrent/observability/logging.py

@@ -0,0 +1,128 @@
+"""Logging helpers.
+
+DataCurrent attaches a :class:`logging.NullHandler` so importing the library
+never changes application logging.  URLs and headers are redacted before they
+are handed to the logging module: tokens in query strings, user info in URLs,
+``Authorization`` and cookie headers never reach log records.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import parse_qsl, urlsplit
+
+__all__ = [
+    "LOGGER_NAME",
+    "get_logger",
+    "redact_headers",
+    "redact_secret",
+    "redact_url",
+]
+
+LOGGER_NAME = "datacurrent"
+
+_SENSITIVE_FRAGMENTS = (
+    "token",
+    "secret",
+    "password",
+    "passwd",
+    "api_key",
+    "apikey",
+    "access_key",
+    "private_key",
+    "credential",
+    "signature",
+    "session",
+    "authorization",
+    "auth",
+    "code",
+    "key",
+)
+_SENSITIVE_HEADERS = {
+    "authorization",
+    "proxy-authorization",
+    "cookie",
+    "set-cookie",
+    "x-api-key",
+    "x-auth-token",
+    "api-key",
+    "sec-websocket-key",
+    "sec-websocket-protocol",
+    "x-amz-security-token",
+}
+
+_root_logger = logging.getLogger(LOGGER_NAME)
+if not any(isinstance(handler, logging.NullHandler) for handler in _root_logger.handlers):
+    _root_logger.addHandler(logging.NullHandler())
+
+
+def get_logger(name: str | None = None) -> logging.Logger:
+    """Return a namespaced logger. Never configure the logging system here."""
+    if not name:
+        return _root_logger
+    if name.startswith(LOGGER_NAME):
+        return logging.getLogger(name)
+    return logging.getLogger(f"{LOGGER_NAME}.{name}")
+
+
+def _is_sensitive(name: str) -> bool:
+    lowered = name.lower()
+    return any(fragment in lowered for fragment in _SENSITIVE_FRAGMENTS)
+
+
+def redact_url(url: str) -> str:
+    """Return ``url`` with user info and sensitive query values replaced by ``***``.
+
+    The structure (scheme, host, port, path, non-sensitive parameters) is kept so
+    the value is still useful in logs.
+    """
+    if not isinstance(url, str):  # defensive: never log an object's repr
+        return "<redacted-url>"
+    try:
+        parts = urlsplit(url)
+    except ValueError:
+        return "<invalid-url>"
+
+    netloc = parts.netloc
+    if "@" in netloc:
+        host = netloc.rsplit("@", 1)[1]
+        netloc = f"***@{host}"
+
+    query = parts.query
+    if query:
+        rendered: list[str] = []
+        try:
+            pairs = parse_qsl(query, keep_blank_values=True)
+        except ValueError:
+            pairs = []
+        if pairs:
+            for key, value in pairs:
+                rendered.append(f"{key}=***" if _is_sensitive(key) else f"{key}={value}")
+            query = "&".join(rendered)
+        else:
+            query = "***"
+
+    return parts._replace(netloc=netloc, query=query).geturl()
+
+
+def redact_headers(headers: Mapping[str, Any] | None) -> dict[str, str]:
+    """Return a copy of ``headers`` with sensitive values replaced by ``***``."""
+    if not headers:
+        return {}
+    redacted: dict[str, str] = {}
+    for key, value in headers.items():
+        name = str(key)
+        redacted[name] = "***" if name.lower() in _SENSITIVE_HEADERS else str(value)
+    return redacted
+
+
+def redact_secret(value: Any) -> str:
+    """Render a secret for logs without revealing its contents."""
+    if value is None:
+        return "<none>"
+    text = str(value)
+    if not text:
+        return "<empty>"
+    return f"*** ({len(text)} chars)"

datacurrent/observability/metrics.py

@@ -0,0 +1,127 @@
+"""Small, dependency-free metrics primitives.
+
+DataCurrent never ships metrics to a hosted service on its own.  You expose the
+snapshot to whatever system you already use (Prometheus, OpenTelemetry, logs).
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Any
+
+__all__ = ["Counter", "Metrics", "StreamStats", "snapshot_stats"]
+
+
+@dataclass(slots=True)
+class StreamStats:
+    """Mutable counters for a single stream instance."""
+
+    name: str | None = None
+    items_out: int = 0
+    errors: int = 0
+    retries: int = 0
+    dropped: int = 0
+
+    def snapshot(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "items_out": self.items_out,
+            "errors": self.errors,
+            "retries": self.retries,
+            "dropped": self.dropped,
+        }
+
+
+def snapshot_stats(stats: StreamStats | None) -> dict[str, Any] | None:
+    return None if stats is None else stats.snapshot()
+
+
+class Counter:
+    """A monotonically increasing counter."""
+
+    __slots__ = ("_value",)
+
+    def __init__(self) -> None:
+        self._value = 0
+
+    def inc(self, amount: int = 1) -> int:
+        self._value += amount
+        return self._value
+
+    @property
+    def value(self) -> int:
+        return self._value
+
+    def reset(self) -> None:
+        self._value = 0
+
+    def __int__(self) -> int:
+        return self._value
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return f"Counter(value={self._value})"
+
+
+@dataclass(slots=True)
+class Metrics:
+    """A tiny registry of counters and gauges.
+
+    Keys are plain strings; tags are folded into the key so the output can be
+    logged as-is or exported to a metrics backend.
+
+    Example::
+
+        metrics = Metrics()
+        metrics.inc("stream.items", tags={"stream": "orders"})
+        metrics.snapshot()["stream.items{stream=orders}"] == 1
+    """
+
+    prefix: str = "datacurrent"
+    counters: dict[str, int] = field(default_factory=dict)
+    gauges: dict[str, float] = field(default_factory=dict)
+
+    @staticmethod
+    def _key(name: str, tags: dict[str, Any] | None) -> str:
+        if not tags:
+            return name
+        rendered = ",".join(f"{key}={value}" for key, value in sorted(tags.items()))
+        return f"{name}{{{rendered}}}"
+
+    def inc(self, name: str, amount: int = 1, **tags: Any) -> int:
+        key = self._key(name, tags or None)
+        self.counters[key] = self.counters.get(key, 0) + amount
+        return self.counters[key]
+
+    def set(self, name: str, value: float, **tags: Any) -> float:
+        key = self._key(name, tags or None)
+        self.gauges[key] = float(value)
+        return self.gauges[key]
+
+    def get(self, name: str, default: int = 0, **tags: Any) -> int:
+        return self.counters.get(self._key(name, tags or None), default)
+
+    @contextmanager
+    def timer(self, name: str, **tags: Any) -> Iterator[None]:
+        started = time.perf_counter()
+        try:
+            yield
+        finally:
+            self.observe_duration(name, time.perf_counter() - started, **tags)
+
+    def observe_duration(self, name: str, seconds: float, **tags: Any) -> None:
+        key = self._key(f"{name}.milliseconds", tags or None)
+        self.gauges[key] = seconds * 1000.0
+
+    def snapshot(self) -> dict[str, Any]:
+        return {
+            "prefix": self.prefix,
+            "counters": dict(self.counters),
+            "gauges": dict(self.gauges),
+        }
+
+    def reset(self) -> None:
+        self.counters.clear()
+        self.gauges.clear()

datacurrent/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561: DataCurrent ships inline type information.

datacurrent/reliability/__init__.py

@@ -0,0 +1,14 @@
+"""Reliability: retries, backoff and timeouts."""
+
+from __future__ import annotations
+
+from .retry import RetryPolicy, is_retryable, retry_async
+from .timeout import validate_timeout, with_timeout
+
+__all__ = [
+    "RetryPolicy",
+    "is_retryable",
+    "retry_async",
+    "validate_timeout",
+    "with_timeout",
+]