starfish-queuing 3.0.0a5__py3-none-any.whl

starfish_queuing/__init__.py

@@ -0,0 +1,42 @@
+"""``starfish-queuing`` — change-event queuing extension.
+
+Public surface: the ``AbstractQueue`` transport base, the in-process
+``MemoryQueue`` and callback ``CustomQueue`` backends, the ``QueueMessage``
+shape, the per-collection ``QueueConfig``, and ``create_queuing_server_plugin``
+— a ``ServerPlugin`` whose ``after_write`` hook publishes a message after each
+successful push.
+
+``NatsQueue``/``NatsQueueOptions`` live in ``starfish_queuing.nats`` (require the
+``nats`` optional extra) and are imported on demand.
+"""
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.memory import MemoryQueue, CustomQueue
+from starfish_queuing.message import QueueMessage
+from starfish_queuing.config import QueueConfig
+
+
+def __getattr__(name: str):
+    """Lazy import of ``create_queuing_server_plugin`` (keeps the
+    ``starfish_protocol.plugins`` import off the hot path for backend-only
+    users) and of ``NatsQueue``/``NatsQueueOptions`` (keeps ``nats-py`` an
+    optional dependency)."""
+    if name == "create_queuing_server_plugin":
+        from starfish_queuing.plugin import create_queuing_server_plugin as _p
+        return _p
+    if name in ("NatsQueue", "NatsQueueOptions"):
+        from starfish_queuing import nats as _nats
+        return getattr(_nats, name)
+    raise AttributeError(f"module 'starfish_queuing' has no attribute {name!r}")
+
+
+__all__ = [
+    "AbstractQueue",
+    "MemoryQueue",
+    "CustomQueue",
+    "QueueMessage",
+    "QueueConfig",
+    "create_queuing_server_plugin",
+    "NatsQueue",
+    "NatsQueueOptions",
+]

starfish_queuing/base.py

@@ -0,0 +1,28 @@
+"""Abstract base class for queue backends."""
+
+from abc import ABC, abstractmethod
+
+
+class AbstractQueue(ABC):
+    """Publish-only queue interface for emitting data-change events.
+
+    Implementations must provide :meth:`publish`.  The :meth:`connect` and
+    :meth:`close` lifecycle hooks have default no-ops — override them only
+    when the backend requires explicit connection management (e.g. NATS).
+    """
+
+    async def connect(self) -> None:
+        """Establish a connection to the queue backend.
+
+        Override for network-backed implementations.  The default is a no-op.
+        """
+
+    @abstractmethod
+    async def publish(self, subject: str, payload: bytes) -> None:
+        raise NotImplementedError("publish must be implemented")
+
+    async def close(self) -> None:
+        """Tear down the connection.  Safe to call multiple times.
+
+        Override for network-backed implementations.  The default is a no-op.
+        """

starfish_queuing/config.py

@@ -0,0 +1,24 @@
+"""Per-collection queue configuration (owned by the queuing plugin)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class QueueConfig:
+    """Per-collection queue publishing configuration.
+
+    Apps pass a ``{collection_name: QueueConfig}`` map to
+    :func:`create_queuing_server_plugin`. Collections absent from that map
+    publish nothing.
+    """
+
+    topic: str | None = None
+    """Subject/topic to publish to. Defaults to the collection name."""
+
+    include_params: bool = False
+    """Include the resolved route path parameters in the published message."""
+
+    include_body: bool = False
+    """Include the pushed ``data`` object in the message (JSON collections only)."""

starfish_queuing/memory.py

@@ -0,0 +1,51 @@
+"""In-memory and callback-based queue implementations."""
+
+import inspect
+from typing import Any, Awaitable, Callable
+
+from starfish_queuing.base import AbstractQueue
+
+
+class MemoryQueue(AbstractQueue):
+    """In-memory queue that records published messages for testing.
+
+    Every call to :meth:`publish` appends a ``(subject, payload)`` tuple
+    to :attr:`messages`::
+
+        queue = MemoryQueue()
+        await queue.publish("posts", b'{"collection":"posts"}')
+        assert len(queue.messages) == 1
+    """
+
+    def __init__(self) -> None:
+        self.messages: list[tuple[str, bytes]] = []
+
+    async def publish(self, subject: str, payload: bytes) -> None:
+        self.messages.append((subject, payload))
+
+
+async def _call(fn: Callable[..., Any], *args: Any) -> Any:
+    """Invoke *fn* with *args*, awaiting the result if it is a coroutine."""
+    result = fn(*args)
+    if inspect.isawaitable(result):
+        return await result
+    return result
+
+
+PublishFn = Callable[[str, bytes], None | Awaitable[None]]
+
+
+class CustomQueue(AbstractQueue):
+    """Queue backed by a user-supplied callback function.
+
+    The callback may be synchronous or ``async``::
+
+        queue = CustomQueue(on_publish=lambda subject, payload: print(subject))
+    """
+
+    def __init__(self, *, on_publish: PublishFn | None = None) -> None:
+        self._on_publish = on_publish
+
+    async def publish(self, subject: str, payload: bytes) -> None:
+        if self._on_publish is not None:
+            await _call(self._on_publish, subject, payload)

starfish_queuing/message.py

@@ -0,0 +1,25 @@
+"""Typed shape of the message published to the queue after a successful push."""
+
+from typing import Any, Required, TypedDict
+
+
+class QueueMessage(TypedDict, total=False):
+    """Message published to the queue after a successful document push.
+
+    Required fields (always present):
+        collection: Collection name.
+        hash:       SHA-256 hex hash of the stored document.
+        timestamp:  Milliseconds since epoch when the push completed.
+
+    Optional fields (present only when the corresponding QueueConfig flag is set):
+        params: Resolved URL path parameters — present when ``include_params=True``.
+        body:   Push request data field — present when ``include_body=True``
+                (JSON collections only). Contains the document as sent by the client,
+                before server-side sanitization.
+    """
+
+    collection: Required[str]
+    hash: Required[str]
+    timestamp: Required[int]
+    params: dict[str, str]
+    body: dict[str, Any]

starfish_queuing/nats.py

@@ -0,0 +1,66 @@
+"""NATS queue implementation."""
+
+import asyncio
+import logging
+from dataclasses import dataclass
+
+import nats
+from nats.aio.client import Client as NatsClient
+
+from starfish_queuing.base import AbstractQueue
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class NatsQueueOptions:
+    """Connection options for :class:`NatsQueue`."""
+
+    servers: str | list[str] = "nats://localhost:4222"
+    """NATS server URL(s)."""
+
+    name: str | None = None
+    """Optional client name (visible in NATS monitoring)."""
+
+
+class NatsQueue(AbstractQueue):
+    """Publish messages to a NATS server.
+
+    Connects lazily on the first :meth:`publish` call if :meth:`connect` has
+    not been called explicitly.  Use the explicit lifecycle for best control::
+
+        queue = NatsQueue(NatsQueueOptions(servers="nats://localhost:4222"))
+        await queue.connect()
+        # … use queue …
+        await queue.close()
+    """
+
+    def __init__(self, opts: NatsQueueOptions | None = None) -> None:
+        self._opts = opts or NatsQueueOptions()
+        self._nc: NatsClient | None = None
+        self._lock = asyncio.Lock()
+
+    async def connect(self) -> None:
+        async with self._lock:
+            if self._nc is not None and self._nc.is_connected:
+                return
+            self._nc = await nats.connect(
+                servers=self._opts.servers,
+                name=self._opts.name,
+            )
+
+    async def _ensure_connected(self) -> NatsClient:
+        if self._nc is None or not self._nc.is_connected:
+            await self.connect()
+        assert self._nc is not None  # noqa: S101
+        return self._nc
+
+    async def publish(self, subject: str, payload: bytes) -> None:
+        nc = await self._ensure_connected()
+        await nc.publish(subject, payload)
+
+    async def close(self) -> None:
+        async with self._lock:
+            if self._nc is not None and self._nc.is_connected:
+                await self._nc.close()
+                self._nc = None

starfish_queuing/plugin.py

@@ -0,0 +1,49 @@
+"""Server plugin for the queuing extension (Python mirror).
+
+Implements the ``after_write`` write-path hook from the ``ServerPlugin``
+contract: after a successful push the server hands the plugin a
+:class:`WriteEvent`; for any collection present in the plugin's ``collections``
+map it builds a :class:`QueueMessage` and publishes it to the configured
+:class:`AbstractQueue`. ``shutdown`` closes the queue during graceful shutdown.
+
+The ``ServerPlugin``/``WriteEvent`` types live in ``starfish-protocol`` (the
+shared contract layer), so this package needs no dependency on
+``starfish-server`` — applications wire both packages at the top level.
+"""
+
+from __future__ import annotations
+
+from typing import Mapping
+
+from starfish_protocol.plugins import ServerPlugin, WriteEvent
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.config import QueueConfig
+from starfish_queuing.publish import publish_change_event
+
+
+def create_queuing_server_plugin(
+    *,
+    queue: AbstractQueue,
+    collections: Mapping[str, QueueConfig],
+) -> ServerPlugin:
+    """Build a :class:`ServerPlugin` that publishes a change event to *queue*
+    after every successful push to a configured collection."""
+
+    async def _after_write(event: WriteEvent) -> None:
+        cfg = collections.get(event.collection)
+        if cfg is None:
+            return
+        await publish_change_event(queue, cfg, event)
+
+    async def _shutdown() -> None:
+        await queue.close()
+
+    return ServerPlugin(
+        name="starfish-queuing",
+        after_write=_after_write,
+        shutdown=_shutdown,
+    )
+
+
+__all__ = ["create_queuing_server_plugin"]

starfish_queuing/publish.py

@@ -0,0 +1,49 @@
+"""Build and publish a queue message from a :class:`WriteEvent`."""
+
+from __future__ import annotations
+
+import json
+import logging
+
+from starfish_protocol.plugins import WriteEvent
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.config import QueueConfig
+from starfish_queuing.message import QueueMessage
+
+logger = logging.getLogger(__name__)
+
+
+async def publish_change_event(
+    queue: AbstractQueue,
+    config: QueueConfig,
+    event: WriteEvent,
+) -> None:
+    """Publish a change event for *event* using *config*.
+
+    Errors are logged but never propagate — a queue outage must not break
+    client writes.
+    """
+    try:
+        subject = config.topic or event.collection
+        msg: QueueMessage = {
+            "collection": event.collection,
+            "hash": event.hash,
+            "timestamp": event.timestamp,
+        }
+        if config.include_params and event.params:
+            msg["params"] = dict(event.params)
+        if config.include_body:
+            if event.body is not None:
+                msg["body"] = dict(event.body)
+            else:
+                logger.warning(
+                    "include_body enabled for %s but request data is not a plain "
+                    "object; body omitted from queue message",
+                    event.collection,
+                )
+        await queue.publish(subject, json.dumps(msg).encode())
+    except Exception:
+        logger.warning(
+            "Failed to publish queue event for %s", event.collection, exc_info=True,
+        )

starfish_queuing-3.0.0a5.dist-info/METADATA

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: starfish-queuing
+Version: 3.0.0a5
+Summary: Starfish change-event queuing extension (post-push publish hook, MemoryQueue/CustomQueue/NatsQueue backends, per-collection config)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Provides-Extra: nats
+Requires-Dist: nats-py>=2.0; extra == "nats"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: respx>=0.23.1; extra == "dev"
+Requires-Dist: nats-py>=2.0; extra == "dev"
+Requires-Dist: starfish-server; extra == "dev"

starfish_queuing-3.0.0a5.dist-info/RECORD

@@ -0,0 +1,12 @@
+starfish_queuing/__init__.py,sha256=QbSdMhPenO3wrQdw44uLaWGTxT0wOlent6R8_JgKDe4,1560
+starfish_queuing/base.py,sha256=ZdZnh7vrD0uEr3yFeORZAjZ8FuN0oiUZWjsFXndZ8uo,947
+starfish_queuing/config.py,sha256=oPvrzfJUo5veew-9EoUVhcWG4wSdVxVfqxpOgjAsA7A,724
+starfish_queuing/memory.py,sha256=YaR1vuzTCEHw5lILEwQ_H8KcucaiyngT6K9p0tLsicg,1536
+starfish_queuing/message.py,sha256=HZxqB2miIeOfmEDmm3KI_NSK00oX6BhpuWnjYzhjygQ,980
+starfish_queuing/nats.py,sha256=laGwcFChtnXp3n2xtodOLH87IDXfsZ7aMMBjypaQEqo,1985
+starfish_queuing/plugin.py,sha256=QsRod4uwzsAq7IrWq3L8INppj1DNIN2AI2oWFWRfsbg,1643
+starfish_queuing/publish.py,sha256=aI8gfC85oWnNmnXxYgd37ebBCM2Cuk_WAYZHOcnapR0,1538
+starfish_queuing-3.0.0a5.dist-info/METADATA,sha256=NwzUk6SqRs2HnOaW99B8Pq5QO1TIS8-WBCynqVMObCg,577
+starfish_queuing-3.0.0a5.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+starfish_queuing-3.0.0a5.dist-info/top_level.txt,sha256=ZSiigH0fBtHQ0DzxOy8SUtWeMy0cHCACbmmGwrcunSM,17
+starfish_queuing-3.0.0a5.dist-info/RECORD,,

starfish_queuing-3.0.0a5.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

starfish_queuing-3.0.0a5.dist-info/top_level.txt

@@ -0,0 +1 @@
+starfish_queuing