datacurrent 1.0.0__py3-none-any.whl

datacurrent/__init__.py

@@ -0,0 +1,157 @@
+"""DataCurrent — Python streaming, made simple.
+
+Async streams, composable operators, real backpressure and modular real-time
+transports for Python applications.
+
+    from datacurrent import Stream
+
+    stream = Stream.from_iterable([1, 2, 3]).map(lambda n: n * 2)
+    async for value in stream:
+        print(value)
+"""
+
+from __future__ import annotations
+
+from ._version import VERSION, __version__
+from .buffering import BoundedQueue, BufferPolicy, BufferStats
+from .concurrency import ConcurrencyLimiter, TaskTracker, gather_limited
+from .config import (
+    DEFAULT_BUFFER_SIZE,
+    DEFAULT_CONCURRENCY,
+    DEFAULT_MAX_MESSAGE_SIZE,
+    DEFAULT_TIMEOUT,
+    OverflowPolicy,
+    StreamConfig,
+)
+from .core import (
+    Stream,
+    StreamIterator,
+    concat_sources,
+    consume,
+    drain_into,
+    merge_sources,
+    pipeline,
+)
+from .events import Event, EventBus, Subscription, matches
+from .exceptions import (
+    BufferOverflowError,
+    ConfigurationError,
+    ConnectionFailedError,
+    DataCurrentError,
+    EmptyStreamError,
+    EventBusError,
+    MessageTooLargeError,
+    NonReplayableStreamError,
+    OptionalDependencyError,
+    QueueClosedError,
+    QueueEmptyError,
+    RetryError,
+    SSEProtocolError,
+    StreamClosedError,
+    StreamConsumedError,
+    StreamError,
+    StreamProtocolError,
+    StreamTimeoutError,
+    SubscriptionClosedError,
+    TransportError,
+    WebSocketClosedError,
+)
+from .observability import (
+    Counter,
+    Metrics,
+    StreamStats,
+    get_logger,
+    redact_headers,
+    redact_secret,
+    redact_url,
+)
+from .reliability import RetryPolicy, retry_async, with_timeout
+from .transports import (
+    Channel,
+    HTTPStream,
+    SSEEvent,
+    SSEStream,
+    StreamTransport,
+    WebSocketStream,
+    format_heartbeat,
+    format_sse,
+    parse_sse,
+)
+
+__all__ = [
+    # Constants
+    "DEFAULT_BUFFER_SIZE",
+    "DEFAULT_CONCURRENCY",
+    "DEFAULT_MAX_MESSAGE_SIZE",
+    "DEFAULT_TIMEOUT",
+    "VERSION",
+    # Buffering
+    "BoundedQueue",
+    "BufferOverflowError",
+    "BufferPolicy",
+    "BufferStats",
+    # Transports
+    "Channel",
+    # Concurrency
+    "ConcurrencyLimiter",
+    "ConfigurationError",
+    "ConnectionFailedError",
+    "Counter",
+    # Exceptions
+    "DataCurrentError",
+    "EmptyStreamError",
+    # Events
+    "Event",
+    "EventBus",
+    "EventBusError",
+    "HTTPStream",
+    "MessageTooLargeError",
+    # Observability
+    "Metrics",
+    "NonReplayableStreamError",
+    "OptionalDependencyError",
+    "OverflowPolicy",
+    "QueueClosedError",
+    "QueueEmptyError",
+    "RetryError",
+    # Reliability
+    "RetryPolicy",
+    "SSEEvent",
+    "SSEProtocolError",
+    "SSEStream",
+    # Core engine
+    "Stream",
+    "StreamClosedError",
+    "StreamConfig",
+    "StreamConsumedError",
+    "StreamError",
+    "StreamIterator",
+    "StreamProtocolError",
+    "StreamStats",
+    "StreamTimeoutError",
+    "StreamTransport",
+    "Subscription",
+    "SubscriptionClosedError",
+    "TaskTracker",
+    "TransportError",
+    "WebSocketClosedError",
+    "WebSocketStream",
+    # Version
+    "__version__",
+    "concat_sources",
+    "consume",
+    "drain_into",
+    "format_heartbeat",
+    "format_sse",
+    "gather_limited",
+    "get_logger",
+    "matches",
+    "merge_sources",
+    "parse_sse",
+    "pipeline",
+    "redact_headers",
+    "redact_secret",
+    "redact_url",
+    "retry_async",
+    "with_timeout",
+]

datacurrent/_version.py

@@ -0,0 +1,6 @@
+"""Single source of truth for the DataCurrent version."""
+
+from __future__ import annotations
+
+__version__ = "1.0.0"
+VERSION = tuple(int(part) for part in __version__.split("."))

datacurrent/buffering/__init__.py

@@ -0,0 +1,8 @@
+"""Buffering: bounded queues, overflow policies and backpressure."""
+
+from __future__ import annotations
+
+from .backpressure import BufferPolicy
+from .queue import BoundedQueue, BufferStats
+
+__all__ = ["BoundedQueue", "BufferPolicy", "BufferStats"]

datacurrent/buffering/backpressure.py

@@ -0,0 +1,47 @@
+"""Normalized buffer settings shared by streams, channels and the event bus."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from ..config import DEFAULT_BUFFER_SIZE, OverflowPolicy, StreamConfig
+from ..exceptions import ConfigurationError
+from .queue import BoundedQueue
+
+__all__ = ["BufferPolicy"]
+
+
+@dataclass(frozen=True, slots=True)
+class BufferPolicy:
+    """A size + overflow policy pair that can build :class:`BoundedQueue` objects."""
+
+    size: int = DEFAULT_BUFFER_SIZE
+    overflow: OverflowPolicy = OverflowPolicy.BLOCK
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "overflow", OverflowPolicy.coerce(self.overflow))
+        if isinstance(self.size, bool) or not isinstance(self.size, int):
+            raise ConfigurationError(
+                f"buffer size must be an integer, got {type(self.size).__name__}"
+            )
+        if self.size < 1:
+            raise ConfigurationError(f"buffer size must be >= 1, got {self.size}")
+
+    def build(self, *, name: str | None = None, on_drop: Any = None) -> BoundedQueue[Any]:
+        """Create a queue configured by this policy."""
+        return BoundedQueue(self.size, overflow=self.overflow, name=name, on_drop=on_drop)
+
+    @classmethod
+    def from_config(
+        cls,
+        config: StreamConfig,
+        *,
+        size: int | None = None,
+        overflow: OverflowPolicy | str | None = None,
+    ) -> BufferPolicy:
+        """Derive a policy from a stream config, allowing per-call overrides."""
+        return cls(
+            size=config.buffer_size if size is None else size,
+            overflow=OverflowPolicy.coerce(config.overflow if overflow is None else overflow),
+        )

datacurrent/buffering/queue.py

@@ -0,0 +1,304 @@
+"""Bounded asynchronous queues with explicit overflow policies.
+
+The queue is the backpressure primitive used by ``Stream.buffer()``,
+``Stream.merge()``, :class:`~datacurrent.Channel` and the event bus.  It never
+silently drops data: the default policy blocks the producer.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections import deque
+from collections.abc import AsyncIterator, Callable
+from dataclasses import dataclass
+from typing import Any, Generic, TypeVar
+
+from ..config import DEFAULT_BUFFER_SIZE, OverflowPolicy
+from ..exceptions import (
+    BufferOverflowError,
+    ConfigurationError,
+    QueueClosedError,
+    QueueEmptyError,
+)
+from ..observability.logging import get_logger
+
+T = TypeVar("T")
+
+_EMPTY = object()
+_log = get_logger("buffering")
+
+__all__ = ["BoundedQueue", "BufferStats"]
+
+
+@dataclass(frozen=True, slots=True)
+class BufferStats:
+    """Point-in-time snapshot of a queue."""
+
+    size: int
+    maxsize: int
+    enqueued: int
+    dequeued: int
+    dropped: int
+    overflow: int
+    high_watermark: int
+    closed: bool
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "size": self.size,
+            "maxsize": self.maxsize,
+            "enqueued": self.enqueued,
+            "dequeued": self.dequeued,
+            "dropped": self.dropped,
+            "overflow": self.overflow,
+            "high_watermark": self.high_watermark,
+            "closed": self.closed,
+        }
+
+
+class BoundedQueue(Generic[T]):
+    """An asyncio queue with a bounded size and a configurable overflow policy.
+
+    Args:
+        maxsize: Maximum number of buffered items (must be >= 1).
+        overflow: Policy applied when the buffer is full.  ``BLOCK`` applies
+            backpressure, ``DROP_NEWEST`` / ``DROP_OLDEST`` discard one item per
+            overflow, and ``ERROR`` raises :class:`BufferOverflowError`.
+        name: Optional name used in logging and error messages.
+        on_drop: Called once per dropped item (used for stream statistics).
+
+    The queue is intended for use from a single event loop.  It is not thread
+    safe and does not perform cross-loop wakeups.
+    """
+
+    __slots__ = (
+        "_closed",
+        "_cond",
+        "_dequeued",
+        "_dropped",
+        "_enqueued",
+        "_error",
+        "_high_watermark",
+        "_items",
+        "_maxsize",
+        "_name",
+        "_on_drop",
+        "_overflow",
+        "_overflow_count",
+    )
+
+    def __init__(
+        self,
+        maxsize: int = DEFAULT_BUFFER_SIZE,
+        *,
+        overflow: OverflowPolicy | str = OverflowPolicy.BLOCK,
+        name: str | None = None,
+        on_drop: Callable[[], None] | None = None,
+    ) -> None:
+        if isinstance(maxsize, bool) or not isinstance(maxsize, int):
+            raise ConfigurationError(f"maxsize must be an integer, got {type(maxsize).__name__}")
+        if maxsize < 1:
+            raise ConfigurationError(f"maxsize must be >= 1, got {maxsize}")
+        self._items: deque[T] = deque()
+        self._maxsize = maxsize
+        self._overflow = OverflowPolicy.coerce(overflow)
+        self._cond = asyncio.Condition()
+        self._closed = False
+        self._error: BaseException | None = None
+        self._name = name
+        self._on_drop = on_drop
+        self._enqueued = 0
+        self._dequeued = 0
+        self._dropped = 0
+        self._overflow_count = 0
+        self._high_watermark = 0
+
+    # ------------------------------------------------------------------ #
+    # Introspection
+    # ------------------------------------------------------------------ #
+    @property
+    def name(self) -> str | None:
+        return self._name
+
+    @property
+    def maxsize(self) -> int:
+        return self._maxsize
+
+    @property
+    def overflow(self) -> OverflowPolicy:
+        return self._overflow
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    @property
+    def error(self) -> BaseException | None:
+        """The failure that terminated the queue, if any."""
+        return self._error
+
+    def qsize(self) -> int:
+        return len(self._items)
+
+    def empty(self) -> bool:
+        return not self._items
+
+    def full(self) -> bool:
+        return len(self._items) >= self._maxsize
+
+    def stats(self) -> BufferStats:
+        return BufferStats(
+            size=len(self._items),
+            maxsize=self._maxsize,
+            enqueued=self._enqueued,
+            dequeued=self._dequeued,
+            dropped=self._dropped,
+            overflow=self._overflow_count,
+            high_watermark=self._high_watermark,
+            closed=self._closed,
+        )
+
+    def _label(self) -> str:
+        return f" {self._name!r}" if self._name else ""
+
+    def _record_drop(self) -> None:
+        self._dropped += 1
+        self._overflow_count += 1
+        if self._on_drop is not None:
+            self._on_drop()
+
+    # ------------------------------------------------------------------ #
+    # Writing
+    # ------------------------------------------------------------------ #
+    async def put(self, item: T) -> bool:
+        """Insert ``item``.
+
+        Returns:
+            ``True`` if the item was stored, ``False`` if it was dropped by a
+            drop policy.
+
+        Raises:
+            QueueClosedError: The queue has been closed.
+            BufferOverflowError: The queue is full and the policy is ``ERROR``.
+        """
+        async with self._cond:
+            while True:
+                if self._closed:
+                    raise QueueClosedError(f"queue{self._label()} is closed")
+                if len(self._items) < self._maxsize:
+                    break
+                policy = self._overflow
+                if policy is OverflowPolicy.DROP_NEWEST:
+                    self._record_drop()
+                    self._cond.notify_all()
+                    return False
+                if policy is OverflowPolicy.DROP_OLDEST:
+                    self._items.popleft()
+                    self._record_drop()
+                    break
+                if policy is OverflowPolicy.ERROR:
+                    self._overflow_count += 1
+                    raise BufferOverflowError(self._maxsize, policy=policy, dropped=self._dropped)
+                await self._cond.wait()
+
+            self._items.append(item)
+            self._enqueued += 1
+            if len(self._items) > self._high_watermark:
+                self._high_watermark = len(self._items)
+            self._cond.notify_all()
+            return True
+
+    async def put_many(self, items: Any) -> int:
+        """Insert every item from an iterable, returning the number stored."""
+        stored = 0
+        for item in items:
+            if await self.put(item):
+                stored += 1
+        return stored
+
+    # ------------------------------------------------------------------ #
+    # Reading
+    # ------------------------------------------------------------------ #
+    async def _take(self, default: Any) -> Any:
+        async with self._cond:
+            while not self._items:
+                if self._error is not None:
+                    raise self._error
+                if self._closed:
+                    return default
+                await self._cond.wait()
+            item = self._items.popleft()
+            self._dequeued += 1
+            self._cond.notify_all()
+            return item
+
+    async def get(self) -> T:
+        """Return the oldest item, waiting if necessary.
+
+        Raises:
+            QueueClosedError: The queue is closed and empty.
+        """
+        item = await self._take(_EMPTY)
+        if item is _EMPTY:
+            raise QueueClosedError(f"queue{self._label()} is closed and empty")
+        return item
+
+    async def get_nowait(self) -> T:
+        """Return the oldest item without waiting.
+
+        Raises:
+            QueueEmptyError: No item is buffered.
+        """
+        async with self._cond:
+            if not self._items:
+                if self._error is not None:
+                    raise self._error
+                raise QueueEmptyError(f"queue{self._label()} is empty")
+            item = self._items.popleft()
+            self._dequeued += 1
+            self._cond.notify_all()
+            return item
+
+    def __aiter__(self) -> AsyncIterator[T]:
+        return self._iterate()
+
+    async def _iterate(self) -> AsyncIterator[T]:
+        while True:
+            item = await self._take(_EMPTY)
+            if item is _EMPTY:
+                return
+            yield item
+
+    # ------------------------------------------------------------------ #
+    # Lifecycle
+    # ------------------------------------------------------------------ #
+    async def aclose(self) -> None:
+        """Close the queue.  Buffered items can still be drained."""
+        async with self._cond:
+            if self._closed:
+                return
+            self._closed = True
+            self._cond.notify_all()
+
+    async def fail(self, error: BaseException) -> None:
+        """Store ``error`` and close the queue; consumers re-raise it when drained."""
+        if not isinstance(error, BaseException):
+            raise TypeError("error must be an exception instance")
+        async with self._cond:
+            if self._error is None:
+                self._error = error
+            self._closed = True
+            self._cond.notify_all()
+        _log.debug("queue%s failed: %r", self._label(), error)
+
+    async def __aenter__(self) -> BoundedQueue[T]:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        await self.aclose()
+
+    def __repr__(self) -> str:
+        return (
+            f"BoundedQueue(size={len(self._items)}, maxsize={self._maxsize}, "
+            f"overflow={self._overflow.value!r}, closed={self._closed})"
+        )

datacurrent/concurrency/__init__.py

@@ -0,0 +1,8 @@
+"""Concurrency: bounded limits and structured background tasks."""
+
+from __future__ import annotations
+
+from .limits import ConcurrencyLimiter, gather_limited
+from .tasks import TaskTracker
+
+__all__ = ["ConcurrencyLimiter", "TaskTracker", "gather_limited"]

datacurrent/concurrency/limits.py

@@ -0,0 +1,106 @@
+"""Limits and helpers for bounded concurrency."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Iterable
+from typing import Any, TypeVar, cast
+
+from ..core._util import maybe_await
+from ..exceptions import ConfigurationError
+
+T = TypeVar("T")
+
+__all__ = ["ConcurrencyLimiter", "gather_limited"]
+
+
+class ConcurrencyLimiter:
+    """A semaphore with introspection.
+
+    ``limit`` workers may run at once; additional callers wait instead of
+    spawning unbounded tasks.
+    """
+
+    __slots__ = ("_active", "_limit", "_peak", "_semaphore", "_waiting")
+
+    def __init__(self, limit: int) -> None:
+        if isinstance(limit, bool) or not isinstance(limit, int):
+            raise ConfigurationError(
+                f"concurrency limit must be an integer, got {type(limit).__name__}"
+            )
+        if limit < 1:
+            raise ConfigurationError(f"concurrency limit must be >= 1, got {limit}")
+        self._limit = limit
+        self._semaphore = asyncio.Semaphore(limit)
+        self._active = 0
+        self._peak = 0
+        self._waiting = 0
+
+    @property
+    def limit(self) -> int:
+        return self._limit
+
+    @property
+    def active(self) -> int:
+        return self._active
+
+    @property
+    def waiting(self) -> int:
+        return self._waiting
+
+    @property
+    def peak(self) -> int:
+        return self._peak
+
+    async def acquire(self) -> None:
+        self._waiting += 1
+        try:
+            await self._semaphore.acquire()
+        finally:
+            self._waiting -= 1
+        self._active += 1
+        self._peak = max(self._peak, self._active)
+
+    def release(self) -> None:
+        if self._active > 0:
+            self._active -= 1
+        self._semaphore.release()
+
+    async def __aenter__(self) -> ConcurrencyLimiter:
+        await self.acquire()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        self.release()
+
+    async def run(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+        async with self:
+            return await maybe_await(func(*args, **kwargs))
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return (
+            f"ConcurrencyLimiter(limit={self._limit}, active={self._active}, "
+            f"waiting={self._waiting}, peak={self._peak})"
+        )
+
+
+async def gather_limited(
+    limit: int,
+    awaitables: Iterable[Awaitable[T]],
+    *,
+    return_exceptions: bool = False,
+) -> list[T]:
+    """Await ``awaitables`` with at most ``limit`` running at the same time."""
+    limiter = ConcurrencyLimiter(limit)
+
+    async def run(awaitable: Awaitable[T]) -> T:
+        async with limiter:
+            return await awaitable
+
+    return cast(
+        "list[T]",
+        await asyncio.gather(
+            *(run(awaitable) for awaitable in awaitables),
+            return_exceptions=return_exceptions,
+        ),
+    )