datacurrent 1.0.0__py3-none-any.whl

datacurrent/concurrency/tasks.py

@@ -0,0 +1,86 @@
+"""Structured task management: no unbounded background tasks, no leaks."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Coroutine
+from typing import Any, TypeVar
+
+from ..core.lifecycle import cancel_tasks
+from ..exceptions import StreamClosedError
+from ..observability.logging import get_logger
+
+T = TypeVar("T")
+
+__all__ = ["TaskTracker"]
+
+_log = get_logger("concurrency")
+
+
+class TaskTracker:
+    """Track background tasks so they can always be cancelled deterministically.
+
+    DataCurrent uses this for buffering producers, ``merge()`` pumps and event
+    bus dispatchers.  Closing the tracker cancels every task and waits for it,
+    which is what keeps cancellation from leaking work into a stopped loop.
+    """
+
+    __slots__ = ("_closed", "_name", "_tasks")
+
+    def __init__(self, *, name: str = "datacurrent") -> None:
+        self._tasks: set[asyncio.Task[Any]] = set()
+        self._name = name
+        self._closed = False
+
+    @property
+    def active(self) -> int:
+        return len(self._tasks)
+
+    @property
+    def tasks(self) -> tuple[asyncio.Task[Any], ...]:
+        return tuple(self._tasks)
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    def spawn(
+        self,
+        coro: Awaitable[T] | Coroutine[Any, Any, T],
+        *,
+        name: str | None = None,
+    ) -> asyncio.Task[T]:
+        """Schedule ``coro`` and keep a strong reference until it finishes."""
+        if self._closed:
+            if asyncio.iscoroutine(coro):
+                coro.close()
+            raise StreamClosedError(f"task tracker {self._name!r} is closed")
+        task = asyncio.ensure_future(coro)
+        task.set_name(name or f"{self._name}-{len(self._tasks) + 1}")
+        self._tasks.add(task)
+        task.add_done_callback(self._on_done)
+        return task
+
+    def _on_done(self, task: asyncio.Task[Any]) -> None:
+        self._tasks.discard(task)
+        if task.cancelled():
+            return
+        error = task.exception()
+        if error is not None:
+            _log.warning("%s: background task failed: %r", self._name, error)
+
+    async def aclose(self, *, timeout: float | None = 5.0) -> None:  # noqa: ASYNC109
+        """Cancel and await every tracked task."""
+        self._closed = True
+        tasks = list(self._tasks)
+        self._tasks.clear()
+        await cancel_tasks(tasks, timeout=timeout)
+
+    async def __aenter__(self) -> TaskTracker:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        await self.aclose()
+
+    def __repr__(self) -> str:  # pragma: no cover - trivial
+        return f"TaskTracker(name={self._name!r}, active={len(self._tasks)}, closed={self._closed})"

datacurrent/config.py

@@ -0,0 +1,86 @@
+"""Configuration objects shared by streams, buffers, channels and the event bus."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any, Final
+
+from .exceptions import ConfigurationError
+
+DEFAULT_BUFFER_SIZE: Final[int] = 1024
+DEFAULT_CONCURRENCY: Final[int] = 1
+DEFAULT_MAX_MESSAGE_SIZE: Final[int] = 8 * 1024 * 1024
+DEFAULT_TIMEOUT: Final[float] = 30.0
+
+__all__ = [
+    "DEFAULT_BUFFER_SIZE",
+    "DEFAULT_CONCURRENCY",
+    "DEFAULT_MAX_MESSAGE_SIZE",
+    "DEFAULT_TIMEOUT",
+    "OverflowPolicy",
+    "StreamConfig",
+]
+
+
+class OverflowPolicy(StrEnum):
+    """What a bounded buffer does when it is full.
+
+    ``BLOCK`` (the default) applies real backpressure: the producer waits.
+    Drops are only ever performed when explicitly configured.
+    """
+
+    BLOCK = "block"
+    DROP_NEWEST = "drop_newest"
+    DROP_OLDEST = "drop_oldest"
+    ERROR = "error"
+
+    @classmethod
+    def coerce(cls, value: Any) -> OverflowPolicy:
+        """Normalize ``value`` (enum or string) into an :class:`OverflowPolicy`."""
+        if isinstance(value, cls):
+            return value
+        if isinstance(value, str):
+            normalized = value.strip().lower().replace("-", "_")
+            try:
+                return cls(normalized)
+            except ValueError:
+                pass
+        allowed = ", ".join(policy.value for policy in cls)
+        raise ConfigurationError(f"unknown overflow policy {value!r}; expected one of: {allowed}")
+
+
+def _require_int(name: str, value: Any, minimum: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ConfigurationError(f"{name} must be an integer, got {type(value).__name__}")
+    if value < minimum:
+        raise ConfigurationError(f"{name} must be >= {minimum}, got {value}")
+    return value
+
+
+@dataclass(frozen=True, slots=True)
+class StreamConfig:
+    """Immutable configuration attached to a stream.
+
+    Attributes:
+        name: Optional human readable name used in logs and reprs.
+        buffer_size: Capacity used by buffering operators such as ``buffer()``.
+        overflow: Policy applied when a buffer is full.
+        concurrency: Default worker count for operators that support it.
+        observe: Collect per-stream counters (items out, errors, drops, retries).
+    """
+
+    name: str | None = None
+    buffer_size: int = DEFAULT_BUFFER_SIZE
+    overflow: OverflowPolicy = OverflowPolicy.BLOCK
+    concurrency: int = DEFAULT_CONCURRENCY
+    observe: bool = False
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "overflow", OverflowPolicy.coerce(self.overflow))
+        _require_int("buffer_size", self.buffer_size, 1)
+        _require_int("concurrency", self.concurrency, 1)
+        if self.name is not None and not isinstance(self.name, str):
+            raise ConfigurationError(
+                f"name must be a string or None, got {type(self.name).__name__}"
+            )

datacurrent/core/__init__.py

@@ -0,0 +1,17 @@
+"""Internal implementation modules for the DataCurrent core engine."""
+
+from __future__ import annotations
+
+from .pipeline import concat_sources, merge_sources, pipeline
+from .sink import consume, drain_into
+from .stream import Stream, StreamIterator
+
+__all__ = [
+    "Stream",
+    "StreamIterator",
+    "concat_sources",
+    "consume",
+    "drain_into",
+    "merge_sources",
+    "pipeline",
+]

datacurrent/core/_util.py

@@ -0,0 +1,42 @@
+"""Internal async helpers shared by the stream engine and operators."""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import AsyncIterator, Iterable
+from typing import Any, TypeVar
+
+from .lifecycle import aclose
+
+T = TypeVar("T")
+
+__all__ = ["aclose", "as_aiterator", "from_sync_iterable", "maybe_await"]
+
+
+async def maybe_await(value: Any) -> Any:
+    """Await ``value`` when it is awaitable, otherwise return it unchanged.
+
+    This is what allows every operator callback to be either a plain function
+    or a coroutine function.
+    """
+    if inspect.isawaitable(value):
+        return await value
+    return value
+
+
+async def from_sync_iterable(iterable: Iterable[T]) -> AsyncIterator[T]:
+    """Adapt a synchronous iterable to an async iterator without a thread."""
+    for item in iterable:
+        yield item
+
+
+def as_aiterator(source: Any) -> AsyncIterator[Any]:
+    """Return an async iterator for an async iterable, iterable or stream."""
+    aiter = getattr(source, "__aiter__", None)
+    if aiter is not None:
+        return aiter()
+    if hasattr(source, "__iter__"):
+        return from_sync_iterable(source)
+    raise TypeError(
+        f"expected an async iterable, an iterable, or a Stream, got {type(source).__name__}"
+    )

datacurrent/core/lifecycle.py

@@ -0,0 +1,81 @@
+"""Deterministic cleanup helpers for async resources and tasks."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Iterable
+from typing import Any
+
+__all__ = ["aclose", "aclose_many", "cancel_task", "cancel_tasks"]
+
+
+async def aclose(obj: Any, *, suppress: bool = True) -> None:
+    """Close an object that may expose ``aclose()`` or ``close()``.
+
+    Used in ``finally`` blocks of every operator so early exits (``break``,
+    cancellation, exceptions) never leak generators, responses or connections.
+    """
+    if obj is None:
+        return
+    close = getattr(obj, "aclose", None)
+    if close is None:
+        close = getattr(obj, "close", None)
+    if close is None:
+        return
+    try:
+        result = close()
+        if inspect.isawaitable(result):
+            await result
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        if not suppress:
+            raise
+
+
+async def aclose_many(objs: Iterable[Any], *, suppress: bool = True) -> None:
+    for obj in objs:
+        await aclose(obj, suppress=suppress)
+
+
+async def cancel_task(
+    task: asyncio.Task[Any] | None,
+    *,
+    timeout: float | None = None,  # noqa: ASYNC109 - cleanup deadline
+) -> None:
+    """Cancel a single task and wait for it to finish."""
+    if task is None:
+        return
+    if not task.done():
+        task.cancel()
+    try:
+        await asyncio.wait_for(asyncio.shield(task), timeout=timeout)
+    except (asyncio.CancelledError, TimeoutError):
+        if not task.done():
+            # Give the loop a chance to run the cancellation callbacks.
+            await asyncio.sleep(0)
+    except Exception:
+        # Exceptions are the caller's concern; cleanup must still succeed.
+        pass
+
+
+async def cancel_tasks(
+    tasks: Iterable[asyncio.Task[Any]],
+    *,
+    timeout: float | None = None,  # noqa: ASYNC109 - cleanup deadline
+) -> None:
+    """Cancel every task in ``tasks`` and wait for their completion."""
+    pending = [task for task in tasks if not task.done()]
+    for task in pending:
+        task.cancel()
+    if not pending:
+        return
+    done, still_pending = await asyncio.wait(pending, timeout=timeout)
+    for task in done:
+        if task.cancelled():
+            continue
+        # Retrieve exceptions so asyncio does not emit "never retrieved" warnings.
+        task.exception()
+    for task in still_pending:
+        task.cancel()