starfish-projection 3.0.0a17__tar.gz

starfish_projection-3.0.0a17/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: starfish-projection
+Version: 3.0.0a17
+Summary: Starfish incremental-list extension (post-push projection hook: fold each write into a single list document — append, update-in-place, or remove)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Requires-Dist: starfish-server
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: httpx>=0.25.0; extra == "dev"
+Requires-Dist: fastapi>=0.100; extra == "dev"

starfish_projection-3.0.0a17/README.md

@@ -0,0 +1,83 @@
+# starfish-projection
+
+Incremental-list extension for Starfish (Python). After a successful push, the
+server hands each registered plugin a `WriteEvent`; this plugin runs an
+app-supplied **pure** mapping for each watched source collection and folds the
+result into a single **list document** — appending a new entry, updating one in
+place, or removing it. The client then pulls one document to read the whole list.
+
+## Install
+
+```sh
+pip install starfish-server starfish-projection
+```
+
+## Usage
+
+```python
+from starfish_server import create_sync_router, SyncRouterOptions
+from starfish_projection import (
+    Projection, ProjectionSet, ProjectionRemove, create_projection_server_plugin,
+)
+
+def project(e):
+    meta = e.body or {}
+    if meta.get("deleted") is True:
+        return ProjectionRemove(id=e.params["id"])
+    return ProjectionSet(id=e.params["id"], value={"name": meta.get("name")})
+
+plugin = create_projection_server_plugin(
+    store=store,
+    projections=[
+        Projection(
+            source="products",
+            # One list document per tenant keeps each list bounded.
+            target=lambda e: f"catalog/{e.params['tenant']}",
+            project=project,
+        )
+    ],
+)
+
+router = create_sync_router(
+    SyncRouterOptions(config=config, store=store, plugins=[plugin]),
+)
+```
+
+`project(event)` returns one of:
+
+- `ProjectionSet(id, value)` — **upsert**: append a new entry `{id, value}` to the
+  target list, or replace an existing entry's `value` in place (keeping its
+  position),
+- `ProjectionRemove(id)` — **remove** the entry with this `id` (a no-op if absent).
+  There is no delete route on the server, so a removal is signalled by a normal
+  write whose body your mapping recognises as a deletion (a tombstone),
+- `None` — **ignore** the event.
+
+`project` may be sync or async. The list is stored as
+`{"items": [{"id", "value"}, …]}` in insertion order — `id` is held alongside
+`value`, never merged into it. `target` is a fixed storage key or a function of
+the event; return `None` from the function to ignore the event, or a per-bucket
+key to **shard** a large view into many small lists.
+
+The list is written in-process against the store, never over HTTP, so the target
+collection can be declared `pull_only=True` — clients read it, but only the
+projection writes it.
+
+### Concurrency & scale
+
+Many source writes can target the same list at once, so each apply is a
+compare-and-set loop: the plugin re-pulls the list, folds the entry in, and
+pushes with the pulled hash; on a hash mismatch (a concurrent write landed first)
+it re-pulls and re-applies rather than clobbering. No update is lost.
+
+Every write rewrites and re-hashes the **whole** list document under one per-key
+lock, and in-process pushes bypass the HTTP `max_body_bytes` limit, so an
+unbounded list can grow without limit. Keep lists bounded — **shard** with a
+`target` function (one list per tenant/bucket), and optionally set `max_items` as
+a safety cap (once full, further appends are logged and dropped; updates and
+removes still apply). `max_retries` (default 8) bounds the CAS loop.
+
+Projection failures are logged and never break the originating client write.
+
+See `docs/ts/projection/` for the full guide (the TypeScript and Python APIs
+mirror each other).

starfish_projection-3.0.0a17/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "starfish-projection"
+version = "3.0.0a17"
+description = "Starfish incremental-list extension (post-push projection hook: fold each write into a single list document — append, update-in-place, or remove)"
+requires-python = ">=3.11"
+dependencies = [
+  "starfish-protocol",
+  "starfish-server",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=7.0",
+  "pytest-asyncio>=0.21",
+  "httpx>=0.25.0",
+  "fastapi>=0.100",
+]
+
+[tool.uv.sources]
+starfish-protocol = { path = "../protocol", editable = true }
+starfish-server = { path = "../server", editable = true }
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

starfish_projection-3.0.0a17/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

starfish_projection-3.0.0a17/starfish_projection/__init__.py

@@ -0,0 +1,37 @@
+"""``starfish-projection`` — incremental-list extension.
+
+Public surface: the :class:`Projection` list spec and its outcome types
+(:class:`ProjectionSet`, :class:`ProjectionRemove`), and
+``create_projection_server_plugin`` — a ``ServerPlugin`` whose ``after_write``
+hook folds each source write into a single target list document (append /
+update-in-place / remove). Clients pull that one document to read the whole list.
+Pair the target collection with ``pull_only=True`` so only the projection writes
+it (clients read it only).
+"""
+
+from starfish_projection.config import (
+    Projection,
+    ProjectionOp,
+    ProjectionRemove,
+    ProjectionSet,
+    ProjectionTarget,
+)
+
+
+def __getattr__(name: str):
+    """Lazy import of ``create_projection_server_plugin`` (keeps the
+    ``starfish_server`` import off the hot path for import-only users)."""
+    if name == "create_projection_server_plugin":
+        from starfish_projection.plugin import create_projection_server_plugin as _p
+        return _p
+    raise AttributeError(f"module 'starfish_projection' has no attribute {name!r}")
+
+
+__all__ = [
+    "Projection",
+    "ProjectionSet",
+    "ProjectionRemove",
+    "ProjectionOp",
+    "ProjectionTarget",
+    "create_projection_server_plugin",
+]

starfish_projection-3.0.0a17/starfish_projection/config.py

@@ -0,0 +1,63 @@
+"""Projection (incremental-list) configuration types."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Awaitable, Callable, Sequence
+
+from starfish_protocol.plugins import WriteEvent
+
+
+@dataclass
+class ProjectionSet:
+    """UPSERT outcome: append the entry ``{id, value}`` to the target list, or —
+    if an entry with this ``id`` already exists — replace its ``value`` in place
+    (keeping its position)."""
+
+    id: str
+    value: dict
+
+
+@dataclass
+class ProjectionRemove:
+    """REMOVE outcome: drop the entry with this ``id`` from the target list (a
+    no-op if absent). The server has no delete route, so a removal is signalled by
+    a normal write whose body your mapping recognises as a deletion (a
+    tombstone)."""
+
+    id: str
+
+
+# A projection function returns an upsert, a remove, or ``None`` (ignore).
+ProjectionOp = ProjectionSet | ProjectionRemove | None
+
+ProjectFn = Callable[[WriteEvent], "ProjectionOp | Awaitable[ProjectionOp]"]
+
+# Where a projection writes its list: a fixed storage key, or a function of the
+# event returning a key (route the entry into that list) or ``None`` (ignore).
+ProjectionTarget = str | Callable[[WriteEvent], "str | None"]
+
+
+@dataclass
+class Projection:
+    """A single projection list.
+
+    On every write to one of ``source`` collections, ``project`` derives an entry
+    op which the plugin folds into the target list document
+    (:class:`ProjectionSet` → append / update-in-place, :class:`ProjectionRemove`
+    → remove, ``None`` → ignore). The plugin owns the read-modify-write against
+    the store — the app only supplies the pure mapping.
+
+    ``project`` MUST be a pure function of the event: it receives the
+    :class:`WriteEvent` (carrying ``collection``, ``params``, optional ``body``,
+    ``hash``, ``timestamp``, ``identity``). The server populates
+    ``WriteEvent.body`` for JSON pushes; ``params`` is always present.
+
+    ``target`` is a fixed storage key or a function of the event; return ``None``
+    from the function to ignore the event, or a per-bucket key to shard a large
+    view into many small lists (e.g. one per tenant).
+    """
+
+    source: str | Sequence[str]
+    target: ProjectionTarget
+    project: ProjectFn

starfish_projection-3.0.0a17/starfish_projection/plugin.py

@@ -0,0 +1,157 @@
+"""Server plugin for the projection (incremental-list) extension (Python mirror).
+
+Implements the ``after_write`` write-path hook from the ``ServerPlugin``
+contract: after a successful push the server hands the plugin a
+:class:`WriteEvent`; for any projection whose ``source`` includes the event's
+collection, the plugin runs the app-supplied pure ``project(event)`` mapping and
+folds its outcome into a single target *list document* — appending a new entry,
+replacing an existing one in place (:class:`ProjectionSet`), or removing it
+(:class:`ProjectionRemove`); ``None`` ignores the event. The app supplies only
+the mapping; the plugin owns all store IO. The client then pulls one document to
+read the whole list, rather than enumerating a directory of per-entry documents.
+
+The list is written in-process, directly against the object store — never over
+HTTP — so the target collection can be configured ``pull_only=True`` to reject
+every *client* write while still being populated here. That ``pull_only`` + this
+plugin is how a target list becomes "owned by the indexer": clients read it, but
+only the projection writes it.
+
+Concurrency: many source writes can target the same list at once, so each apply
+is a CAS loop — pull the current list, fold the entry in, then ``push`` with the
+pulled ``base_hash``. ``push`` rejects on a stale hash (optimistic concurrency),
+so on conflict we re-pull and re-apply onto fresh state rather than clobbering a
+concurrent write. The pull MUST happen inside the loop so each retry sees the
+latest list. Failures are logged, never raised — ``after_write`` must not break
+the originating client write (same contract as starfish-queuing).
+
+Scale: every write rewrites and re-hashes the whole list document under one
+per-key lock, and in-process pushes bypass the HTTP ``max_body_bytes`` limit, so
+a single list can grow unbounded server-side. Keep lists bounded — shard via a
+``target`` function (one list per tenant/bucket) and/or set ``max_items``.
+"""
+
+from __future__ import annotations
+
+import inspect
+import logging
+from typing import Sequence
+
+from starfish_protocol.plugins import ServerPlugin, WriteEvent
+from starfish_server.protocol.pull import pull
+from starfish_server.protocol.push import push
+from starfish_server.protocol.types import PushConflict
+from starfish_server.storage.base import AbstractObjectStore, StoreContext
+
+from starfish_projection.config import Projection, ProjectionOp, ProjectionRemove
+
+_log = logging.getLogger(__name__)
+
+DEFAULT_MAX_RETRIES = 8
+
+
+def _source_set(source: str | Sequence[str]) -> frozenset[str]:
+    return frozenset([source] if isinstance(source, str) else source)
+
+
+async def _apply_op(
+    store: AbstractObjectStore,
+    target_key: str,
+    op: ProjectionOp,
+    max_retries: int,
+    max_items: int | None,
+    source_collection: str,
+) -> None:
+    """Fold a single entry op into the target list document under a CAS retry loop."""
+    # A projection-owned write runs in-process with the plugin's authority, not a
+    # client's — no per-document role gating.
+    ctx = StoreContext(
+        collection=source_collection,
+        params={},
+        identity=None,
+        roles=(),
+        action="push",
+    )
+
+    for _ in range(max_retries):
+        # Re-pull every iteration so each retry folds onto the latest list.
+        current = await pull(store, target_key, context=ctx)
+        base_hash = current.hash or None
+        stored = current.data.get("items") if isinstance(current.data, dict) else None
+        items = list(stored) if isinstance(stored, list) else []
+        idx = next((i for i, it in enumerate(items) if it.get("id") == op.id), -1)
+
+        if isinstance(op, ProjectionRemove):
+            if idx == -1:
+                return  # already absent — nothing to write
+            items.pop(idx)
+        elif idx == -1:
+            if max_items is not None and len(items) >= max_items:
+                _log.warning(
+                    "projection list %r at max_items=%d; dropping append of id %r",
+                    target_key,
+                    max_items,
+                    op.id,
+                )
+                return
+            items.append({"id": op.id, "value": op.value})
+        else:
+            # Update in place: keep the entry's position, full-replace its value.
+            items[idx] = {"id": op.id, "value": op.value}
+
+        result = await push(store, target_key, {"items": items}, base_hash, context=ctx)
+        if not isinstance(result, PushConflict):
+            return  # PushSuccess — done
+        # hash_mismatch: a concurrent write changed the list; loop to re-pull/re-apply.
+
+    _log.warning(
+        "projection list %r exhausted %d CAS retries; dropped op for id %r",
+        target_key,
+        max_retries,
+        op.id,
+    )
+
+
+def create_projection_server_plugin(
+    *,
+    store: AbstractObjectStore,
+    projections: list[Projection],
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    max_items: int | None = None,
+) -> ServerPlugin:
+    """Build a :class:`ServerPlugin` that maintains one or more projection lists:
+    after a successful push to a watched ``source`` collection, it derives an
+    entry op via the app's ``project`` function and folds it into the target list
+    document in *store*.
+
+    ``max_retries`` (default 8) bounds the CAS loop; on exhaustion the op is
+    logged and dropped. ``max_items`` optionally caps each list — once full,
+    further appends are logged and dropped (existing entries are never evicted);
+    prefer sharding via a ``target`` function for large views.
+    """
+
+    compiled = [(_source_set(p.source), p.target, p.project) for p in projections]
+
+    async def _after_write(event: WriteEvent) -> None:
+        for sources, target, project in compiled:
+            if event.collection not in sources:
+                continue
+            try:
+                # Resolve the target list before running the mapping; None = ignore.
+                target_key = target(event) if callable(target) else target
+                if target_key is None:
+                    continue
+                op = project(event)
+                if inspect.isawaitable(op):
+                    op = await op
+                if op is None:
+                    continue
+                await _apply_op(
+                    store, target_key, op, max_retries, max_items, event.collection
+                )
+            except Exception as exc:  # noqa: BLE001 — must not break client writes
+                _log.warning("projection for %r failed: %s", event.collection, exc)
+
+    return ServerPlugin(name="starfish-projection", after_write=_after_write)
+
+
+__all__ = ["create_projection_server_plugin"]

starfish_projection-3.0.0a17/starfish_projection.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: starfish-projection
+Version: 3.0.0a17
+Summary: Starfish incremental-list extension (post-push projection hook: fold each write into a single list document — append, update-in-place, or remove)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Requires-Dist: starfish-server
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: httpx>=0.25.0; extra == "dev"
+Requires-Dist: fastapi>=0.100; extra == "dev"

starfish_projection-3.0.0a17/starfish_projection.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+starfish_projection/__init__.py
+starfish_projection/config.py
+starfish_projection/plugin.py
+starfish_projection.egg-info/PKG-INFO
+starfish_projection.egg-info/SOURCES.txt
+starfish_projection.egg-info/dependency_links.txt
+starfish_projection.egg-info/requires.txt
+starfish_projection.egg-info/top_level.txt
+tests/test_plugin.py

starfish_projection-3.0.0a17/starfish_projection.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

starfish_projection-3.0.0a17/starfish_projection.egg-info/requires.txt

@@ -0,0 +1,8 @@
+starfish-protocol
+starfish-server
+
+[dev]
+pytest>=7.0
+pytest-asyncio>=0.21
+httpx>=0.25.0
+fastapi>=0.100

starfish_projection-3.0.0a17/starfish_projection.egg-info/top_level.txt

@@ -0,0 +1 @@
+starfish_projection

starfish_projection-3.0.0a17/tests/test_plugin.py

@@ -0,0 +1,304 @@
+"""Integration tests — the projection plugin maintains an incremental list on push."""
+
+import json
+
+import pytest
+from fastapi import FastAPI, Request
+from httpx import AsyncClient, ASGITransport
+
+from starfish_server.config.schema import SyncConfig, CollectionConfig
+from starfish_server.router.route_builder import (
+    create_sync_router,
+    SyncRouterOptions,
+    AuthResult,
+)
+from starfish_projection import (
+    Projection,
+    ProjectionRemove,
+    ProjectionSet,
+    create_projection_server_plugin,
+)
+from starfish_protocol.plugins import WriteEvent
+
+from tests.helpers import MemoryObjectStore, OneShotConflictStore
+
+
+def _build_app(
+    collections: list[CollectionConfig],
+    projections: list[Projection],
+    *,
+    store=None,
+    max_items: int | None = None,
+    max_retries: int = 8,
+):
+    store = store or MemoryObjectStore()
+    config = SyncConfig(version=1, collections=collections)
+
+    async def role_resolver(request: Request) -> AuthResult:
+        return AuthResult(identity="user-1", roles=["self"])
+
+    plugin = create_projection_server_plugin(
+        store=store, projections=projections, max_retries=max_retries, max_items=max_items
+    )
+    router = create_sync_router(
+        SyncRouterOptions(
+            store=store, config=config, role_resolver=role_resolver, plugins=[plugin],
+        ),
+    )
+    app = FastAPI()
+    app.include_router(router)
+    return app, store
+
+
+def _col(name: str, storage_path: str, **overrides) -> CollectionConfig:
+    return CollectionConfig(
+        name=name,
+        storagePath=storage_path,
+        readRoles=["self"],
+        writeRoles=["self"],
+        encryption="none",
+        maxBodyBytes=1_000_000,
+        **overrides,
+    )
+
+
+def _source_and_list() -> list[CollectionConfig]:
+    return [
+        _col("products", "products/{id}"),
+        _col("catalog", "catalog", pullOnly=True),
+    ]
+
+
+async def _push(client: AsyncClient, path: str, data: dict) -> None:
+    # Read the current hash first so an update to an existing key passes the
+    # optimistic-concurrency check (a second push with baseHash:None would 409).
+    pull_path = path.replace("/push/", "/pull/")
+    cur = await client.get(pull_path)
+    base_hash = (cur.json().get("hash") or None) if cur.status_code == 200 else None
+    resp = await client.post(
+        path,
+        json={"data": data, "baseHash": base_hash},
+        headers={"content-type": "application/json"},
+    )
+    assert resp.status_code == 200
+
+
+async def _read_list(store, key: str) -> list[dict]:
+    raw = await store.get_string(key)
+    if raw is None:
+        return []
+    return json.loads(raw)["data"]["items"]
+
+
+def _ids(items: list[dict]) -> list[str]:
+    return [i["id"] for i in items]
+
+
+# Mirror each product as a {id, value:{name}} entry in `catalog`, treating
+# {deleted: True} as a removal.
+def _catalog_project(e: WriteEvent):
+    if (e.body or {}).get("deleted") is True:
+        return ProjectionRemove(id=e.params["id"])
+    return ProjectionSet(id=e.params["id"], value={"name": (e.body or {}).get("name", "")})
+
+
+def _catalog_projection() -> Projection:
+    return Projection(source="products", target="catalog", project=_catalog_project)
+
+
+@pytest.mark.asyncio
+async def test_appends_entries_and_serves_whole_list():
+    app, store = _build_app(_source_and_list(), [_catalog_projection()])
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/a", {"name": "Alpha"})
+        await _push(client, "/push/products/b", {"name": "Beta"})
+        body = (await client.get("/pull/catalog")).json()
+
+    expected = [
+        {"id": "a", "value": {"name": "Alpha"}},
+        {"id": "b", "value": {"name": "Beta"}},
+    ]
+    assert await _read_list(store, "catalog") == expected
+    # The client reads the whole list in a single GET of the one list document.
+    assert body["data"]["items"] == expected
+
+
+@pytest.mark.asyncio
+async def test_updates_entry_in_place_keeping_position():
+    app, store = _build_app(_source_and_list(), [_catalog_projection()])
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/a", {"name": "Alpha"})
+        await _push(client, "/push/products/b", {"name": "Beta"})
+        await _push(client, "/push/products/a", {"name": "Alpha v2"})
+
+    items = await _read_list(store, "catalog")
+    assert _ids(items) == ["a", "b"]  # position preserved
+    assert items[0]["value"] == {"name": "Alpha v2"}  # value fully replaced
+
+
+@pytest.mark.asyncio
+async def test_removes_entry_on_tombstone_and_list_survives_when_emptied():
+    app, store = _build_app(_source_and_list(), [_catalog_projection()])
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/a", {"name": "Alpha"})
+        await _push(client, "/push/products/b", {"name": "Beta"})
+        await _push(client, "/push/products/a", {"deleted": True})
+        assert _ids(await _read_list(store, "catalog")) == ["b"]
+
+        # Removing an absent id is a no-op.
+        await _push(client, "/push/products/zzz", {"deleted": True})
+        assert _ids(await _read_list(store, "catalog")) == ["b"]
+
+        # Emptying the list leaves an empty list document, not a 404.
+        await _push(client, "/push/products/b", {"deleted": True})
+        assert await _read_list(store, "catalog") == []
+        assert (await client.get("/pull/catalog")).status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_ignores_when_project_returns_none():
+    app, store = _build_app(
+        _source_and_list(),
+        [Projection(source="products", target="catalog", project=lambda e: None)],
+    )
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/a", {"name": "Alpha"})
+    assert await store.get_string("catalog") is None
+
+
+@pytest.mark.asyncio
+async def test_target_function_shards_across_multiple_sources():
+    app, store = _build_app(
+        collections=[
+            _col("products", "products/{tenant}/{id}"),
+            _col("services", "services/{tenant}/{id}"),
+        ],
+        projections=[
+            Projection(
+                source=["products", "services"],
+                target=lambda e: (f"catalog/{e.params['tenant']}" if e.params.get("tenant") else None),
+                project=lambda e: ProjectionSet(
+                    id=e.params["id"],
+                    value={"kind": e.collection, "name": (e.body or {}).get("name", "")},
+                ),
+            )
+        ],
+    )
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/t1/p1", {"name": "P1"})
+        await _push(client, "/push/services/t1/s1", {"name": "S1"})
+        await _push(client, "/push/products/t2/p2", {"name": "P2"})
+
+    assert await _read_list(store, "catalog/t1") == [
+        {"id": "p1", "value": {"kind": "products", "name": "P1"}},
+        {"id": "s1", "value": {"kind": "services", "name": "S1"}},
+    ]
+    assert _ids(await _read_list(store, "catalog/t2")) == ["p2"]
+
+
+@pytest.mark.asyncio
+async def test_client_fetches_all_shards_via_list_endpoint():
+    # Shard a product catalog by category (the documented manual-sharding pattern).
+    app, _ = _build_app(
+        collections=[
+            _col("products", "products/{id}"),
+            _col("catalog", "catalog/{category}", pullOnly=True, listable=True),
+        ],
+        projections=[
+            Projection(
+                source="products",
+                target=lambda e: (f"catalog/{e.body['category']}" if (e.body or {}).get("category") else None),
+                project=lambda e: ProjectionSet(id=e.params["id"], value={"name": (e.body or {}).get("name", "")}),
+            )
+        ],
+    )
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/p1", {"name": "Novel", "category": "books"})
+        await _push(client, "/push/products/p2", {"name": "Phone", "category": "electronics"})
+        await _push(client, "/push/products/p3", {"name": "Comic", "category": "books"})
+
+        # Discover shards via the list endpoint, then pull each and concatenate.
+        shards = (await client.get("/list/catalog")).json()["items"]
+        all_items: list[dict] = []
+        for cat in shards:
+            body = (await client.get(f"/pull/catalog/{cat}")).json()
+            all_items.extend(body["data"]["items"])
+
+    assert sorted(shards) == ["books", "electronics"]
+    assert sorted(i["id"] for i in all_items) == ["p1", "p2", "p3"]
+
+
+@pytest.mark.asyncio
+async def test_concurrent_writes_do_not_lose_updates():
+    store = OneShotConflictStore()
+    app, _ = _build_app(_source_and_list(), [_catalog_projection()], store=store)
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        # Seed the list with one entry.
+        await _push(client, "/push/products/a", {"name": "Alpha"})
+
+        # Arm a competing write that adds entry "c", then push "b". The plugin's
+        # first push hash-mismatches, re-pulls (now seeing "a" + "c") and re-applies
+        # "b" on top — losing neither.
+        store.arm(
+            "catalog",
+            json.dumps(
+                {
+                    "v": 1,
+                    "data": {
+                        "items": [
+                            {"id": "a", "value": {"name": "Alpha"}},
+                            {"id": "c", "value": {"name": "Concurrent"}},
+                        ]
+                    },
+                    "ts": 1,
+                    "hash": "f" * 64,
+                }
+            ),
+        )
+        await _push(client, "/push/products/b", {"name": "Beta"})
+
+    assert _ids(await _read_list(store, "catalog")) == ["a", "c", "b"]
+
+
+@pytest.mark.asyncio
+async def test_max_items_caps_the_list():
+    app, store = _build_app(_source_and_list(), [_catalog_projection()], max_items=2)
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/a", {"name": "A"})
+        await _push(client, "/push/products/b", {"name": "B"})
+        await _push(client, "/push/products/c", {"name": "C"})  # exceeds cap → dropped
+    assert _ids(await _read_list(store, "catalog")) == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_pullonly_list_rejects_client_writes():
+    app, _ = _build_app(_source_and_list(), [_catalog_projection()])
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        await _push(client, "/push/products/a", {"name": "Alpha"})
+        assert (await client.get("/pull/catalog")).status_code == 200
+        # pullOnly → no push route registered for the list.
+        resp = await client.post(
+            "/push/catalog",
+            json={"data": {"tampered": True}, "baseHash": None},
+            headers={"content-type": "application/json"},
+        )
+    assert resp.status_code >= 400
+
+
+@pytest.mark.asyncio
+async def test_projection_failure_does_not_break_client_write():
+    def boom(e: WriteEvent):
+        raise RuntimeError("boom")
+
+    app, _ = _build_app(
+        collections=[_col("products", "products/{id}")],
+        projections=[Projection(source="products", target="catalog", project=boom)],
+    )
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        resp = await client.post(
+            "/push/products/a",
+            json={"data": {"name": "x"}, "baseHash": None},
+            headers={"content-type": "application/json"},
+        )
+    assert resp.status_code == 200
+    assert len(resp.json()["hash"]) == 64