starfish-queuing 3.0.0a5__tar.gz

starfish_queuing-3.0.0a5/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: starfish-queuing
+Version: 3.0.0a5
+Summary: Starfish change-event queuing extension (post-push publish hook, MemoryQueue/CustomQueue/NatsQueue backends, per-collection config)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Provides-Extra: nats
+Requires-Dist: nats-py>=2.0; extra == "nats"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: respx>=0.23.1; extra == "dev"
+Requires-Dist: nats-py>=2.0; extra == "dev"
+Requires-Dist: starfish-server; extra == "dev"

starfish_queuing-3.0.0a5/README.md

@@ -0,0 +1,47 @@
+# starfish-queuing
+
+Change-event queuing extension for Starfish (Python). After a successful push,
+the server hands each registered plugin a `WriteEvent`; this plugin publishes a
+`QueueMessage` to a configured transport (`MemoryQueue`, `CustomQueue`,
+`NatsQueue`, or your own `AbstractQueue`).
+
+## Install
+
+```sh
+pip install starfish-server starfish-queuing
+# with NATS support:
+pip install "starfish-queuing[nats]"
+```
+
+## Usage
+
+```python
+from starfish_server import create_sync_router, SyncRouterOptions
+from starfish_queuing import create_queuing_server_plugin, MemoryQueue, QueueConfig
+
+queue = MemoryQueue()
+
+plugin = create_queuing_server_plugin(
+    queue=queue,
+    collections={
+        "events": QueueConfig(topic="events", include_params=True, include_body=True),
+    },
+)
+
+router = create_sync_router(
+    SyncRouterOptions(
+        config=config,
+        store=store,
+        # …
+        plugins=[plugin],
+    )
+)
+```
+
+The plugin publishes only for collections present in its `collections` map.
+`topic` defaults to the collection name — an unset *or empty-string* topic falls
+back to it (an empty broker subject is a footgun). `shutdown()` closes the queue
+when the server's graceful-shutdown handler is given the plugin list.
+
+See `docs/ts/queuing/` for the full guide (the TypeScript and Python APIs mirror
+each other).

starfish_queuing-3.0.0a5/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "starfish-queuing"
+version = "3.0.0a5"
+description = "Starfish change-event queuing extension (post-push publish hook, MemoryQueue/CustomQueue/NatsQueue backends, per-collection config)"
+requires-python = ">=3.11"
+dependencies = [
+  "starfish-protocol",
+]
+
+[project.optional-dependencies]
+nats = ["nats-py>=2.0"]
+dev = [
+  "pytest>=7.0",
+  "pytest-asyncio>=0.21",
+  "respx>=0.23.1",
+  "nats-py>=2.0",
+  "starfish-server",
+]
+
+[tool.uv.sources]
+starfish-protocol = { path = "../protocol", editable = true }
+starfish-server = { path = "../server", editable = true }
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

starfish_queuing-3.0.0a5/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

starfish_queuing-3.0.0a5/starfish_queuing/__init__.py

@@ -0,0 +1,42 @@
+"""``starfish-queuing`` — change-event queuing extension.
+
+Public surface: the ``AbstractQueue`` transport base, the in-process
+``MemoryQueue`` and callback ``CustomQueue`` backends, the ``QueueMessage``
+shape, the per-collection ``QueueConfig``, and ``create_queuing_server_plugin``
+— a ``ServerPlugin`` whose ``after_write`` hook publishes a message after each
+successful push.
+
+``NatsQueue``/``NatsQueueOptions`` live in ``starfish_queuing.nats`` (require the
+``nats`` optional extra) and are imported on demand.
+"""
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.memory import MemoryQueue, CustomQueue
+from starfish_queuing.message import QueueMessage
+from starfish_queuing.config import QueueConfig
+
+
+def __getattr__(name: str):
+    """Lazy import of ``create_queuing_server_plugin`` (keeps the
+    ``starfish_protocol.plugins`` import off the hot path for backend-only
+    users) and of ``NatsQueue``/``NatsQueueOptions`` (keeps ``nats-py`` an
+    optional dependency)."""
+    if name == "create_queuing_server_plugin":
+        from starfish_queuing.plugin import create_queuing_server_plugin as _p
+        return _p
+    if name in ("NatsQueue", "NatsQueueOptions"):
+        from starfish_queuing import nats as _nats
+        return getattr(_nats, name)
+    raise AttributeError(f"module 'starfish_queuing' has no attribute {name!r}")
+
+
+__all__ = [
+    "AbstractQueue",
+    "MemoryQueue",
+    "CustomQueue",
+    "QueueMessage",
+    "QueueConfig",
+    "create_queuing_server_plugin",
+    "NatsQueue",
+    "NatsQueueOptions",
+]

starfish_queuing-3.0.0a5/starfish_queuing/base.py

@@ -0,0 +1,28 @@
+"""Abstract base class for queue backends."""
+
+from abc import ABC, abstractmethod
+
+
+class AbstractQueue(ABC):
+    """Publish-only queue interface for emitting data-change events.
+
+    Implementations must provide :meth:`publish`.  The :meth:`connect` and
+    :meth:`close` lifecycle hooks have default no-ops — override them only
+    when the backend requires explicit connection management (e.g. NATS).
+    """
+
+    async def connect(self) -> None:
+        """Establish a connection to the queue backend.
+
+        Override for network-backed implementations.  The default is a no-op.
+        """
+
+    @abstractmethod
+    async def publish(self, subject: str, payload: bytes) -> None:
+        raise NotImplementedError("publish must be implemented")
+
+    async def close(self) -> None:
+        """Tear down the connection.  Safe to call multiple times.
+
+        Override for network-backed implementations.  The default is a no-op.
+        """

starfish_queuing-3.0.0a5/starfish_queuing/config.py

@@ -0,0 +1,24 @@
+"""Per-collection queue configuration (owned by the queuing plugin)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class QueueConfig:
+    """Per-collection queue publishing configuration.
+
+    Apps pass a ``{collection_name: QueueConfig}`` map to
+    :func:`create_queuing_server_plugin`. Collections absent from that map
+    publish nothing.
+    """
+
+    topic: str | None = None
+    """Subject/topic to publish to. Defaults to the collection name."""
+
+    include_params: bool = False
+    """Include the resolved route path parameters in the published message."""
+
+    include_body: bool = False
+    """Include the pushed ``data`` object in the message (JSON collections only)."""

starfish_queuing-3.0.0a5/starfish_queuing/memory.py

@@ -0,0 +1,51 @@
+"""In-memory and callback-based queue implementations."""
+
+import inspect
+from typing import Any, Awaitable, Callable
+
+from starfish_queuing.base import AbstractQueue
+
+
+class MemoryQueue(AbstractQueue):
+    """In-memory queue that records published messages for testing.
+
+    Every call to :meth:`publish` appends a ``(subject, payload)`` tuple
+    to :attr:`messages`::
+
+        queue = MemoryQueue()
+        await queue.publish("posts", b'{"collection":"posts"}')
+        assert len(queue.messages) == 1
+    """
+
+    def __init__(self) -> None:
+        self.messages: list[tuple[str, bytes]] = []
+
+    async def publish(self, subject: str, payload: bytes) -> None:
+        self.messages.append((subject, payload))
+
+
+async def _call(fn: Callable[..., Any], *args: Any) -> Any:
+    """Invoke *fn* with *args*, awaiting the result if it is a coroutine."""
+    result = fn(*args)
+    if inspect.isawaitable(result):
+        return await result
+    return result
+
+
+PublishFn = Callable[[str, bytes], None | Awaitable[None]]
+
+
+class CustomQueue(AbstractQueue):
+    """Queue backed by a user-supplied callback function.
+
+    The callback may be synchronous or ``async``::
+
+        queue = CustomQueue(on_publish=lambda subject, payload: print(subject))
+    """
+
+    def __init__(self, *, on_publish: PublishFn | None = None) -> None:
+        self._on_publish = on_publish
+
+    async def publish(self, subject: str, payload: bytes) -> None:
+        if self._on_publish is not None:
+            await _call(self._on_publish, subject, payload)

starfish_queuing-3.0.0a5/starfish_queuing/message.py

@@ -0,0 +1,25 @@
+"""Typed shape of the message published to the queue after a successful push."""
+
+from typing import Any, Required, TypedDict
+
+
+class QueueMessage(TypedDict, total=False):
+    """Message published to the queue after a successful document push.
+
+    Required fields (always present):
+        collection: Collection name.
+        hash:       SHA-256 hex hash of the stored document.
+        timestamp:  Milliseconds since epoch when the push completed.
+
+    Optional fields (present only when the corresponding QueueConfig flag is set):
+        params: Resolved URL path parameters — present when ``include_params=True``.
+        body:   Push request data field — present when ``include_body=True``
+                (JSON collections only). Contains the document as sent by the client,
+                before server-side sanitization.
+    """
+
+    collection: Required[str]
+    hash: Required[str]
+    timestamp: Required[int]
+    params: dict[str, str]
+    body: dict[str, Any]

starfish_queuing-3.0.0a5/starfish_queuing/nats.py

@@ -0,0 +1,66 @@
+"""NATS queue implementation."""
+
+import asyncio
+import logging
+from dataclasses import dataclass
+
+import nats
+from nats.aio.client import Client as NatsClient
+
+from starfish_queuing.base import AbstractQueue
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class NatsQueueOptions:
+    """Connection options for :class:`NatsQueue`."""
+
+    servers: str | list[str] = "nats://localhost:4222"
+    """NATS server URL(s)."""
+
+    name: str | None = None
+    """Optional client name (visible in NATS monitoring)."""
+
+
+class NatsQueue(AbstractQueue):
+    """Publish messages to a NATS server.
+
+    Connects lazily on the first :meth:`publish` call if :meth:`connect` has
+    not been called explicitly.  Use the explicit lifecycle for best control::
+
+        queue = NatsQueue(NatsQueueOptions(servers="nats://localhost:4222"))
+        await queue.connect()
+        # … use queue …
+        await queue.close()
+    """
+
+    def __init__(self, opts: NatsQueueOptions | None = None) -> None:
+        self._opts = opts or NatsQueueOptions()
+        self._nc: NatsClient | None = None
+        self._lock = asyncio.Lock()
+
+    async def connect(self) -> None:
+        async with self._lock:
+            if self._nc is not None and self._nc.is_connected:
+                return
+            self._nc = await nats.connect(
+                servers=self._opts.servers,
+                name=self._opts.name,
+            )
+
+    async def _ensure_connected(self) -> NatsClient:
+        if self._nc is None or not self._nc.is_connected:
+            await self.connect()
+        assert self._nc is not None  # noqa: S101
+        return self._nc
+
+    async def publish(self, subject: str, payload: bytes) -> None:
+        nc = await self._ensure_connected()
+        await nc.publish(subject, payload)
+
+    async def close(self) -> None:
+        async with self._lock:
+            if self._nc is not None and self._nc.is_connected:
+                await self._nc.close()
+                self._nc = None

starfish_queuing-3.0.0a5/starfish_queuing/plugin.py

@@ -0,0 +1,49 @@
+"""Server plugin for the queuing extension (Python mirror).
+
+Implements the ``after_write`` write-path hook from the ``ServerPlugin``
+contract: after a successful push the server hands the plugin a
+:class:`WriteEvent`; for any collection present in the plugin's ``collections``
+map it builds a :class:`QueueMessage` and publishes it to the configured
+:class:`AbstractQueue`. ``shutdown`` closes the queue during graceful shutdown.
+
+The ``ServerPlugin``/``WriteEvent`` types live in ``starfish-protocol`` (the
+shared contract layer), so this package needs no dependency on
+``starfish-server`` — applications wire both packages at the top level.
+"""
+
+from __future__ import annotations
+
+from typing import Mapping
+
+from starfish_protocol.plugins import ServerPlugin, WriteEvent
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.config import QueueConfig
+from starfish_queuing.publish import publish_change_event
+
+
+def create_queuing_server_plugin(
+    *,
+    queue: AbstractQueue,
+    collections: Mapping[str, QueueConfig],
+) -> ServerPlugin:
+    """Build a :class:`ServerPlugin` that publishes a change event to *queue*
+    after every successful push to a configured collection."""
+
+    async def _after_write(event: WriteEvent) -> None:
+        cfg = collections.get(event.collection)
+        if cfg is None:
+            return
+        await publish_change_event(queue, cfg, event)
+
+    async def _shutdown() -> None:
+        await queue.close()
+
+    return ServerPlugin(
+        name="starfish-queuing",
+        after_write=_after_write,
+        shutdown=_shutdown,
+    )
+
+
+__all__ = ["create_queuing_server_plugin"]

starfish_queuing-3.0.0a5/starfish_queuing/publish.py

@@ -0,0 +1,49 @@
+"""Build and publish a queue message from a :class:`WriteEvent`."""
+
+from __future__ import annotations
+
+import json
+import logging
+
+from starfish_protocol.plugins import WriteEvent
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.config import QueueConfig
+from starfish_queuing.message import QueueMessage
+
+logger = logging.getLogger(__name__)
+
+
+async def publish_change_event(
+    queue: AbstractQueue,
+    config: QueueConfig,
+    event: WriteEvent,
+) -> None:
+    """Publish a change event for *event* using *config*.
+
+    Errors are logged but never propagate — a queue outage must not break
+    client writes.
+    """
+    try:
+        subject = config.topic or event.collection
+        msg: QueueMessage = {
+            "collection": event.collection,
+            "hash": event.hash,
+            "timestamp": event.timestamp,
+        }
+        if config.include_params and event.params:
+            msg["params"] = dict(event.params)
+        if config.include_body:
+            if event.body is not None:
+                msg["body"] = dict(event.body)
+            else:
+                logger.warning(
+                    "include_body enabled for %s but request data is not a plain "
+                    "object; body omitted from queue message",
+                    event.collection,
+                )
+        await queue.publish(subject, json.dumps(msg).encode())
+    except Exception:
+        logger.warning(
+            "Failed to publish queue event for %s", event.collection, exc_info=True,
+        )

starfish_queuing-3.0.0a5/starfish_queuing.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: starfish-queuing
+Version: 3.0.0a5
+Summary: Starfish change-event queuing extension (post-push publish hook, MemoryQueue/CustomQueue/NatsQueue backends, per-collection config)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Provides-Extra: nats
+Requires-Dist: nats-py>=2.0; extra == "nats"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: respx>=0.23.1; extra == "dev"
+Requires-Dist: nats-py>=2.0; extra == "dev"
+Requires-Dist: starfish-server; extra == "dev"

starfish_queuing-3.0.0a5/starfish_queuing.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+README.md
+pyproject.toml
+starfish_queuing/__init__.py
+starfish_queuing/base.py
+starfish_queuing/config.py
+starfish_queuing/memory.py
+starfish_queuing/message.py
+starfish_queuing/nats.py
+starfish_queuing/plugin.py
+starfish_queuing/publish.py
+starfish_queuing.egg-info/PKG-INFO
+starfish_queuing.egg-info/SOURCES.txt
+starfish_queuing.egg-info/dependency_links.txt
+starfish_queuing.egg-info/requires.txt
+starfish_queuing.egg-info/top_level.txt
+tests/test_append_only_no_persist.py
+tests/test_lifecycle_close.py
+tests/test_memory.py
+tests/test_plugin.py

starfish_queuing-3.0.0a5/starfish_queuing.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

starfish_queuing-3.0.0a5/starfish_queuing.egg-info/requires.txt

@@ -0,0 +1,11 @@
+starfish-protocol
+
+[dev]
+pytest>=7.0
+pytest-asyncio>=0.21
+respx>=0.23.1
+nats-py>=2.0
+starfish-server
+
+[nats]
+nats-py>=2.0

starfish_queuing-3.0.0a5/starfish_queuing.egg-info/top_level.txt

@@ -0,0 +1 @@
+starfish_queuing

starfish_queuing-3.0.0a5/tests/test_append_only_no_persist.py

@@ -0,0 +1,172 @@
+"""Tests for appendOnly+persist=false collection (replaces queueOnly), driven via the queuing plugin."""
+
+import json
+
+import pytest
+from fastapi import FastAPI, Request
+from httpx import AsyncClient, ASGITransport
+
+from starfish_server.config.schema import (
+    SyncConfig,
+    CollectionConfig,
+    AppendOnlyConfig,
+)
+from starfish_server.config.validate import validate_config
+from starfish_server.router.route_builder import (
+    create_sync_router,
+    SyncRouterOptions,
+    AuthResult,
+)
+from starfish_queuing import MemoryQueue, QueueConfig, create_queuing_server_plugin
+
+from tests.helpers import MemoryObjectStore
+
+
+def _make_col(**overrides) -> CollectionConfig:
+    defaults = dict(
+        name="events",
+        storagePath="events/{eventId}",
+        readRoles=["public"],
+        writeRoles=["admin"],
+        encryption="none",
+        maxBodyBytes=65536,
+    )
+    defaults.update(overrides)
+    return CollectionConfig(**defaults)
+
+
+def _build_app(col: CollectionConfig, queue_collections: dict[str, QueueConfig] | None = None):
+    store = MemoryObjectStore()
+    q = MemoryQueue()
+    config = SyncConfig(version=1, collections=[col])
+
+    async def role_resolver(request: Request) -> AuthResult:
+        return AuthResult(identity="user-1", roles=["admin"])
+
+    plugin = create_queuing_server_plugin(queue=q, collections=queue_collections or {})
+    router = create_sync_router(
+        SyncRouterOptions(
+            store=store, config=config, role_resolver=role_resolver, plugins=[plugin],
+        ),
+    )
+    app = FastAPI()
+    app.include_router(router)
+    return app, store, q
+
+
+async def _push(client: AsyncClient, path: str = "/push/events/evt-1", base_hash=None):
+    return await client.post(
+        path,
+        json={"data": {"type": "click"}, "baseHash": base_hash},
+        headers={"content-type": "application/json"},
+    )
+
+
+@pytest.mark.asyncio
+async def test_push_returns_hash_and_timestamp():
+    app, _, _ = _build_app(_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False)))
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await _push(client)
+    assert resp.status_code == 200
+    body = resp.json()
+    assert isinstance(body["hash"], str)
+    assert len(body["hash"]) == 64  # SHA-256 hex
+    assert isinstance(body["timestamp"], int)
+
+
+@pytest.mark.asyncio
+async def test_does_not_write_to_storage():
+    app, store, _ = _build_app(_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False)))
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+    stored = await store.get_string("events/evt-1")
+    assert stored is None
+
+
+@pytest.mark.asyncio
+async def test_pull_returns_empty_data():
+    """Nothing stored → pull returns empty."""
+    app, _, _ = _build_app(_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False)))
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+        resp = await client.get("/pull/events/evt-1")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["data"] == {}
+    assert body["hash"] == ""
+
+
+@pytest.mark.asyncio
+async def test_accepts_any_base_hash():
+    """No conflict detection — any baseHash is accepted."""
+    app, _, _ = _build_app(_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False)))
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+        resp = await _push(client, base_hash="arbitrary-wrong-hash")
+    assert resp.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_consistent_hash_for_same_data():
+    app, _, _ = _build_app(_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False)))
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp1 = await _push(client, "/push/events/evt-1")
+        resp2 = await _push(client, "/push/events/evt-2")
+    assert resp1.json()["hash"] == resp2.json()["hash"]
+
+
+@pytest.mark.asyncio
+async def test_publishes_queue_event():
+    col = _make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False))
+    app, _, q = _build_app(col, {"events": QueueConfig(topic="events.created")})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await _push(client)
+    push_body = resp.json()
+
+    assert len(q.messages) == 1
+    subject, payload = q.messages[0]
+    msg = json.loads(payload)
+    assert subject == "events.created"
+    assert msg["collection"] == "events"
+    assert msg["hash"] == push_body["hash"]
+    assert msg["timestamp"] == push_body["timestamp"]
+
+
+@pytest.mark.asyncio
+async def test_push_accepted_without_queue_configured():
+    """appendOnly+persist=false without a configured queue: ephemeral (no storage, no event)."""
+    col = _make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False))
+    app, _, q = _build_app(col)  # no queue_collections
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await _push(client)
+    assert resp.status_code == 200
+    assert len(q.messages) == 0
+
+
+@pytest.mark.asyncio
+async def test_still_validates_missing_data_field():
+    app, _, _ = _build_app(_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False)))
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await client.post(
+            "/push/events/evt-1",
+            json={"baseHash": None},
+            headers={"content-type": "application/json"},
+        )
+    assert resp.status_code == 400
+
+
+def test_valid_append_only_no_persist_collection():
+    errors = validate_config(SyncConfig(version=1, collections=[_make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False))]))
+    assert errors == []
+
+
+def test_append_only_binary_collection_rejected():
+    col = _make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False), allowedMimeTypes=["image/png"])
+    errors = validate_config(SyncConfig(version=1, collections=[col]))
+    assert any("appendOnly cannot be used with binary collections" in e for e in errors)
+
+
+def test_append_only_pullonly_rejected():
+    col = _make_col(appendOnly=AppendOnlyConfig(type="by_timestamp", persist=False), pull_only=True)
+    errors = validate_config(SyncConfig(version=1, collections=[col]))
+    assert any("appendOnly cannot be used with pullOnly" in e for e in errors)

starfish_queuing-3.0.0a5/tests/test_lifecycle_close.py

@@ -0,0 +1,40 @@
+"""The queuing plugin's shutdown hook closes the queue during graceful shutdown."""
+
+import pytest
+from unittest.mock import AsyncMock
+
+from starfish_server.lifecycle import GracefulShutdown, GracefulShutdownOptions
+from starfish_queuing import create_queuing_server_plugin
+from starfish_queuing.base import AbstractQueue
+
+
+class _SpyQueue(AbstractQueue):
+    def __init__(self) -> None:
+        self.close_calls = 0
+
+    async def publish(self, subject: str, payload: bytes) -> None:  # pragma: no cover
+        pass
+
+    async def close(self) -> None:
+        self.close_calls += 1
+
+
+@pytest.mark.asyncio
+async def test_graceful_shutdown_invokes_plugin_shutdown_closing_queue():
+    queue = _SpyQueue()
+    plugin = create_queuing_server_plugin(queue=queue, collections={})
+
+    gs = GracefulShutdown(GracefulShutdownOptions(plugins=[plugin], signals=[]))
+    await gs.shutdown()
+
+    assert queue.close_calls == 1
+
+
+@pytest.mark.asyncio
+async def test_plugin_shutdown_hook_calls_queue_close():
+    queue = AsyncMock(spec=AbstractQueue)
+    plugin = create_queuing_server_plugin(queue=queue, collections={})
+
+    assert plugin.shutdown is not None
+    await plugin.shutdown()
+    queue.close.assert_awaited_once()

starfish_queuing-3.0.0a5/tests/test_memory.py

@@ -0,0 +1,56 @@
+"""Tests for in-memory and callback-based queue implementations."""
+
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.memory import MemoryQueue, CustomQueue
+
+
+async def test_memory_queue_records_messages():
+    q = MemoryQueue()
+    await q.publish("posts", b'{"collection":"posts"}')
+    await q.publish("settings", b'{"collection":"settings"}')
+
+    assert len(q.messages) == 2
+    assert q.messages[0] == ("posts", b'{"collection":"posts"}')
+    assert q.messages[1] == ("settings", b'{"collection":"settings"}')
+
+
+async def test_memory_queue_starts_empty():
+    q = MemoryQueue()
+    assert q.messages == []
+
+
+async def test_custom_queue_sync_callback():
+    received: list[tuple[str, bytes]] = []
+
+    def on_publish(subject: str, payload: bytes) -> None:
+        received.append((subject, payload))
+
+    q = CustomQueue(on_publish=on_publish)
+    await q.publish("topic", b"data")
+
+    assert received == [("topic", b"data")]
+
+
+async def test_custom_queue_async_callback():
+    received: list[tuple[str, bytes]] = []
+
+    async def on_publish(subject: str, payload: bytes) -> None:
+        received.append((subject, payload))
+
+    q = CustomQueue(on_publish=on_publish)
+    await q.publish("topic", b"data")
+
+    assert received == [("topic", b"data")]
+
+
+async def test_custom_queue_no_callback_is_noop():
+    q = CustomQueue()
+    await q.publish("topic", b"data")  # should not raise
+
+
+async def test_abstract_queue_connect_close_are_noops():
+    """Default connect() and close() do nothing — safe to call."""
+    q = MemoryQueue()
+    await q.connect()
+    await q.close()
+    assert isinstance(q, AbstractQueue)

starfish_queuing-3.0.0a5/tests/test_plugin.py

@@ -0,0 +1,311 @@
+"""Integration tests — the queuing plugin publishes events on push through the FastAPI router."""
+
+import json
+
+import pytest
+from fastapi import FastAPI, Request
+from httpx import AsyncClient, ASGITransport
+
+from starfish_server.config.schema import SyncConfig, CollectionConfig
+from starfish_server.router.route_builder import (
+    create_sync_router,
+    SyncRouterOptions,
+    AuthResult,
+)
+from starfish_queuing import (
+    MemoryQueue,
+    QueueConfig,
+    create_queuing_server_plugin,
+)
+from starfish_queuing.base import AbstractQueue
+from starfish_queuing.publish import publish_change_event
+from starfish_protocol.plugins import WriteEvent
+
+from tests.helpers import MemoryObjectStore
+
+
+def _build_app(
+    collections: list[CollectionConfig],
+    queue_collections: dict[str, QueueConfig] | None = None,
+    queue: AbstractQueue | None = None,
+) -> tuple[FastAPI, MemoryObjectStore, AbstractQueue]:
+    store = MemoryObjectStore()
+    q = queue or MemoryQueue()
+    config = SyncConfig(version=1, collections=collections)
+
+    async def role_resolver(request: Request) -> AuthResult:
+        return AuthResult(identity="user-1", roles=["admin", "self"])
+
+    plugin = create_queuing_server_plugin(queue=q, collections=queue_collections or {})
+    router = create_sync_router(
+        SyncRouterOptions(
+            store=store, config=config, role_resolver=role_resolver, plugins=[plugin],
+        ),
+    )
+    app = FastAPI()
+    app.include_router(router)
+    return app, store, q
+
+
+def _col(name: str = "posts", storage_path: str = "posts/{postId}", **overrides) -> CollectionConfig:
+    return CollectionConfig(
+        name=name,
+        storagePath=storage_path,
+        readRoles=["public"],
+        writeRoles=["admin"],
+        encryption="none",
+        maxBodyBytes=65536,
+        **overrides,
+    )
+
+
+async def _push(client: AsyncClient, path: str = "/push/posts/abc") -> dict:
+    resp = await client.post(
+        path,
+        json={"data": {"title": "Hello"}, "baseHash": None},
+        headers={"content-type": "application/json"},
+    )
+    assert resp.status_code == 200
+    return resp.json()
+
+
+@pytest.mark.asyncio
+async def test_push_publishes_queue_event():
+    app, store, q = _build_app([_col()], {"posts": QueueConfig()})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        push_body = await _push(client)
+
+    assert len(q.messages) == 1
+    subject, payload = q.messages[0]
+    msg = json.loads(payload)
+    assert subject == "posts"  # default topic = collection name
+    assert msg["collection"] == "posts"
+    assert msg["hash"] == push_body["hash"]
+    assert msg["timestamp"] == push_body["timestamp"]
+    assert "params" not in msg
+
+
+@pytest.mark.asyncio
+async def test_push_collection_not_configured_no_event():
+    app, store, q = _build_app([_col()], {})  # "posts" not in plugin map
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+
+    assert len(q.messages) == 0
+
+
+@pytest.mark.asyncio
+async def test_custom_topic():
+    app, store, q = _build_app([_col()], {"posts": QueueConfig(topic="custom.topic")})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+
+    assert q.messages[0][0] == "custom.topic"
+
+
+@pytest.mark.asyncio
+async def test_include_params():
+    app, store, q = _build_app([_col()], {"posts": QueueConfig(include_params=True)})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/posts/my-post-id")
+
+    msg = json.loads(q.messages[0][1])
+    assert msg["params"] == {"postId": "my-post-id"}
+
+
+@pytest.mark.asyncio
+async def test_no_include_params_by_default():
+    app, store, q = _build_app([_col()], {"posts": QueueConfig()})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+
+    msg = json.loads(q.messages[0][1])
+    assert "params" not in msg
+
+
+@pytest.mark.asyncio
+async def test_no_event_on_conflict():
+    """Queue event should NOT be published when push returns 409 (hash mismatch)."""
+    app, store, q = _build_app([_col()], {"posts": QueueConfig()})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # First push succeeds
+        await _push(client)
+        q.messages.clear()
+
+        # Second push with wrong baseHash → 409
+        resp = await client.post(
+            "/push/posts/abc",
+            json={"data": {"title": "Updated"}, "baseHash": "wrong-hash"},
+            headers={"content-type": "application/json"},
+        )
+        assert resp.status_code == 409
+
+    assert len(q.messages) == 0
+
+
+@pytest.mark.asyncio
+async def test_include_body():
+    app, store, q = _build_app([_col()], {"posts": QueueConfig(include_body=True)})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/posts/my-post-id")
+
+    msg = json.loads(q.messages[0][1])
+    assert msg["body"] == {"title": "Hello"}
+
+
+@pytest.mark.asyncio
+async def test_no_include_body_by_default():
+    app, store, q = _build_app([_col()], {"posts": QueueConfig()})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+
+    msg = json.loads(q.messages[0][1])
+    assert "body" not in msg
+
+
+@pytest.mark.asyncio
+async def test_include_body_and_params_together():
+    app, store, q = _build_app(
+        [_col()], {"posts": QueueConfig(include_body=True, include_params=True)},
+    )
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/posts/post-42")
+
+    msg = json.loads(q.messages[0][1])
+    assert msg["body"] == {"title": "Hello"}
+    assert msg["params"] == {"postId": "post-42"}
+
+
+@pytest.mark.asyncio
+async def test_binary_collection_include_body_never_emits_body():
+    col = _col(
+        name="avatar",
+        storage_path="users/{userId}/avatar",
+        allowedMimeTypes=["image/png"],
+    )
+    app, store, q = _build_app([col], {"avatar": QueueConfig(include_body=True)})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await client.post(
+            "/push/users/user-1/avatar",
+            content=b"\x89PNG",
+            headers={"content-type": "image/png"},
+        )
+        assert resp.status_code == 200
+
+    assert len(q.messages) == 1
+    msg = json.loads(q.messages[0][1])
+    assert "body" not in msg
+    assert msg["collection"] == "avatar"
+
+
+@pytest.mark.asyncio
+async def test_queue_failure_does_not_break_push():
+    """A queue publish error must not propagate to the client."""
+
+    class FailingQueue(AbstractQueue):
+        async def publish(self, subject: str, payload: bytes) -> None:
+            raise RuntimeError("NATS connection lost")
+
+    app, store, q = _build_app([_col()], {"posts": QueueConfig()}, queue=FailingQueue())
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        push_body = await _push(client)
+
+    assert "hash" in push_body
+    assert "timestamp" in push_body
+
+
+@pytest.mark.asyncio
+async def test_bundle_collection_include_body_emits_body():
+    col = _col(
+        name="prefs",
+        storage_path="users/{userId}/data",
+        bundle="userdata",
+    )
+    app, store, q = _build_app([col], {"prefs": QueueConfig(include_body=True)})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await client.post(
+            "/push/users/user-1/data/prefs",
+            json={"data": {"theme": "dark"}, "baseHash": None},
+            headers={"content-type": "application/json"},
+        )
+        assert resp.status_code == 200
+
+    assert len(q.messages) == 1
+    msg = json.loads(q.messages[0][1])
+    assert msg["collection"] == "prefs"
+    assert msg["body"] == {"theme": "dark"}
+
+
+@pytest.mark.asyncio
+async def test_empty_topic_falls_back_to_collection_name():
+    # Cross-language divergence on a reachable, user-settable config field.
+    # Python uses `config.topic or event.collection` (publish.py), which coalesces
+    # an empty-string topic to the collection name — the safe behaviour, since an
+    # empty broker subject is a footgun. TS uses `cfg.topic ?? event.collection`,
+    # which keeps "" verbatim and publishes to subject "". This test pins the
+    # convergent behaviour; the TS side is pinned as it.fails in plugin.test.ts.
+    app, store, q = _build_app([_col()], {"posts": QueueConfig(topic="")})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client)
+
+    assert q.messages[0][0] == "posts"
+
+
+@pytest.mark.asyncio
+async def test_omits_params_when_storage_path_has_no_path_params():
+    # include_params gate is `if config.include_params and event.params:`, so an
+    # empty params map (no `{…}` placeholders in storage_path) publishes no params.
+    col = _col(name="config", storage_path="global/config")
+    app, store, q = _build_app([col], {"config": QueueConfig(include_params=True)})
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await client.post(
+            "/push/global/config",
+            json={"data": {"x": 1}, "baseHash": None},
+            headers={"content-type": "application/json"},
+        )
+        assert resp.status_code == 200
+
+    assert len(q.messages) == 1
+    msg = json.loads(q.messages[0][1])
+    assert "params" not in msg
+
+
+@pytest.mark.asyncio
+async def test_preserves_unicode_in_topic_and_body():
+    # Path segments are charset-restricted (non-ASCII rejected at the door), so
+    # unicode is probed on the reachable surfaces: config topic and JSON body.
+    app, store, q = _build_app(
+        [_col()], {"posts": QueueConfig(include_body=True, topic="更新.notify")},
+    )
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await client.post(
+            "/push/posts/post-1",
+            json={"data": {"note": "Ñoño 🎉", "ключ": "значение"}, "baseHash": None},
+            headers={"content-type": "application/json"},
+        )
+        assert resp.status_code == 200
+
+    assert len(q.messages) == 1
+    subject, payload = q.messages[0]
+    assert subject == "更新.notify"
+    msg = json.loads(payload)
+    assert msg["body"]["note"] == "Ñoño 🎉"
+    assert msg["body"]["ключ"] == "значение"
+
+
+@pytest.mark.asyncio
+async def test_omits_a_none_body_handed_directly():
+    # The Python gate is `if event.body is not None`, so a None body is OMITTED.
+    # The server NEVER emits body=None as distinct from absent — route_builder.py
+    # sets WriteEvent.body only when the pushed data is a dict, otherwise leaves it
+    # None — so this is the absent case. TS's gate is `if (event.body !== undefined)`,
+    # which INCLUDES an explicit null body; the difference is benign because neither
+    # path is reached for a real document. Pinned (and flagged) so it's locked if
+    # WriteEvent population ever changes. See plugin.test.ts for the TS side.
+    q = MemoryQueue()
+    event = WriteEvent(collection="posts", hash="h", timestamp=1, params={}, body=None)
+    await publish_change_event(q, QueueConfig(include_body=True), event)
+
+    assert len(q.messages) == 1
+    msg = json.loads(q.messages[0][1])
+    assert "body" not in msg