starfish-outbox 3.0.0a20__tar.gz

starfish_outbox-3.0.0a20/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: starfish-outbox
+Version: 3.0.0a20
+Summary: Starfish durable offline write-queue — dedup-by-id, claim, retry, reconnect-drain, persisted per identity, framework-free
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+
+# starfish-outbox
+
+A durable, per-identity **offline write-queue** — the client-side complement to the
+server-side `queuing` extension.
+
+Generic over the queued item: you own what an item is, supply its dedup `id`, and a
+`send` that performs the real write. The queue handles persistence (write-through to a
+`LocalCache`), dedup-by-id, single-shot claim (no double-send), attempt counting with
+auto-retry-then-fail, crash-safe recovery of stuck `sending` entries, and subscriptions.
+`drain_outbox` is connectivity-agnostic — you trigger it on whatever reconnect signal you have.
+
+## Install
+
+```sh
+pip install starfish-outbox
+```
+
+## Usage
+
+```python
+from starfish_outbox import OutboxQueue, drain_outbox
+
+queue: OutboxQueue = OutboxQueue(local_cache)  # local_cache: async get_item / set_item / remove_item
+await queue.hydrate(f"outbox.{user_id}")
+
+# Enqueue a write while offline (the id is yours to thread into the eventual write):
+await queue.enqueue(write_id, {"path": path, "body": body})
+
+# Drain on reconnect:
+async def send(entry):
+    await client.push(entry.item["path"], entry.item["body"], base_hash_for(entry.item["path"]))
+
+await drain_outbox(queue, send, max_attempts=5)
+```
+
+Mutating methods (`enqueue` / `claim` / `remove` / `retry` / …) are `async` because each
+writes through to the cache; `get` / `pending` / `subscribe` are synchronous. JSON-compatible
+on disk with the TypeScript `@drakkar.software/starfish-outbox`.

starfish_outbox-3.0.0a20/README.md

@@ -0,0 +1,38 @@
+# starfish-outbox
+
+A durable, per-identity **offline write-queue** — the client-side complement to the
+server-side `queuing` extension.
+
+Generic over the queued item: you own what an item is, supply its dedup `id`, and a
+`send` that performs the real write. The queue handles persistence (write-through to a
+`LocalCache`), dedup-by-id, single-shot claim (no double-send), attempt counting with
+auto-retry-then-fail, crash-safe recovery of stuck `sending` entries, and subscriptions.
+`drain_outbox` is connectivity-agnostic — you trigger it on whatever reconnect signal you have.
+
+## Install
+
+```sh
+pip install starfish-outbox
+```
+
+## Usage
+
+```python
+from starfish_outbox import OutboxQueue, drain_outbox
+
+queue: OutboxQueue = OutboxQueue(local_cache)  # local_cache: async get_item / set_item / remove_item
+await queue.hydrate(f"outbox.{user_id}")
+
+# Enqueue a write while offline (the id is yours to thread into the eventual write):
+await queue.enqueue(write_id, {"path": path, "body": body})
+
+# Drain on reconnect:
+async def send(entry):
+    await client.push(entry.item["path"], entry.item["body"], base_hash_for(entry.item["path"]))
+
+await drain_outbox(queue, send, max_attempts=5)
+```
+
+Mutating methods (`enqueue` / `claim` / `remove` / `retry` / …) are `async` because each
+writes through to the cache; `get` / `pending` / `subscribe` are synchronous. JSON-compatible
+on disk with the TypeScript `@drakkar.software/starfish-outbox`.

starfish_outbox-3.0.0a20/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "starfish-outbox"
+version = "3.0.0a20"
+description = "Starfish durable offline write-queue — dedup-by-id, claim, retry, reconnect-drain, persisted per identity, framework-free"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = []
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=7.0",
+  "pytest-asyncio>=0.21",
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

starfish_outbox-3.0.0a20/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

starfish_outbox-3.0.0a20/starfish_outbox/__init__.py

@@ -0,0 +1,29 @@
+"""``starfish-outbox`` — a durable, per-identity offline **write-queue** (the
+client-side complement to the server-side ``queuing`` extension).
+
+Generic over the queued item: the caller owns
+what an item is, supplies its dedup ``id``, and a ``send`` that performs the real
+write. The queue handles persistence (write-through to a ``LocalCache``),
+dedup-by-id, single-shot claim, attempt counting with auto-retry-then-fail,
+crash-safe ``sending`` recovery, and subscriptions. :func:`drain_outbox` is
+connectivity-agnostic — the caller triggers it on whatever reconnect signal it has.
+"""
+
+from starfish_outbox.queue import (
+    LocalCache,
+    OutboxEntry,
+    OutboxQueue,
+    OutboxStatus,
+    reset_sending_to_queued,
+)
+from starfish_outbox.drain import DrainResult, drain_outbox
+
+__all__ = [
+    "OutboxQueue",
+    "OutboxEntry",
+    "OutboxStatus",
+    "LocalCache",
+    "reset_sending_to_queued",
+    "drain_outbox",
+    "DrainResult",
+]

starfish_outbox-3.0.0a20/starfish_outbox/drain.py

@@ -0,0 +1,47 @@
+"""Drain driver for an :class:`OutboxQueue`. Connectivity-agnostic by design: the
+caller decides *when* to drain (typically on a reconnect signal), keeping the queue
+free of any platform/connectivity dependency."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Awaitable, Callable, TypeVar
+
+from .queue import OutboxEntry, OutboxQueue
+
+T = TypeVar("T")
+
+
+@dataclass
+class DrainResult:
+    sent: int
+    failed: int
+
+
+async def drain_outbox(
+    queue: OutboxQueue,
+    send: Callable[[OutboxEntry], Awaitable[None]],
+    *,
+    max_attempts: int = 5,
+) -> DrainResult:
+    """One drain pass: for each ``queued`` entry (oldest first), claim it, run
+    ``send``, then ``remove`` on success or ``record_failure`` on raise. ``claim``
+    is single-shot, so concurrent drains never double-send. ``failed`` entries are
+    skipped (awaiting a manual ``retry``)."""
+    sent = 0
+    failed = 0
+    queued = [e for e in queue.get() if e.status == "queued"]
+    for entry in queued:
+        if not await queue.claim(entry.id):
+            continue
+        try:
+            await send(entry)
+            await queue.remove(entry.id)
+            sent += 1
+        except Exception:
+            before = next((e for e in queue.get() if e.id == entry.id), None)
+            await queue.record_failure(entry.id, max_attempts)
+            after = next((e for e in queue.get() if e.id == entry.id), None)
+            if after is not None and after.status == "failed" and (before is None or before.status != "failed"):
+                failed += 1
+    return DrainResult(sent=sent, failed=failed)

starfish_outbox-3.0.0a20/starfish_outbox/queue.py

@@ -0,0 +1,167 @@
+"""A durable, per-identity offline **write-queue** — the client-side complement to
+the server-side ``queuing`` extension. It queues opaque items and the caller owns *what*
+a queued item is and *how* it is sent.
+
+Invariants:
+
+- **Dedup by id.** ``enqueue(id, …)`` ignores an id already queued.
+- **Removed only on confirmed success** (see :func:`drain_outbox`).
+- **Persisted per identity.** Write-through to a :class:`LocalCache` under the key
+  bound by :meth:`OutboxQueue.hydrate` — so mutating methods are ``async``.
+- **Crash-safe claim.** A stuck ``sending`` entry resets to ``queued`` on the next
+  ``hydrate``; ``claim`` is single-shot so concurrent drains never double-send.
+
+JSON-compatible on disk with the TypeScript ``@drakkar.software/starfish-outbox``
+(entry keys ``id`` / ``item`` / ``status`` / ``attempts`` / ``enqueuedAt``).
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass
+from typing import Any, Callable, Generic, Optional, Protocol, TypeVar
+
+T = TypeVar("T")
+
+OutboxStatus = str  # "queued" | "sending" | "failed"
+
+
+class LocalCache(Protocol):
+    """Minimal async key/value cache (localStorage / AsyncStorage wrapper)."""
+
+    async def get_item(self, key: str) -> Optional[str]: ...
+    async def set_item(self, key: str, value: str) -> None: ...
+    async def remove_item(self, key: str) -> None: ...
+
+
+@dataclass
+class OutboxEntry(Generic[T]):
+    """One queued write: the caller's opaque ``item`` plus queue bookkeeping."""
+
+    id: str
+    item: T
+    status: OutboxStatus
+    attempts: int
+    enqueued_at: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "item": self.item,
+            "status": self.status,
+            "attempts": self.attempts,
+            "enqueuedAt": self.enqueued_at,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "OutboxEntry[Any]":
+        return cls(
+            id=d["id"],
+            item=d["item"],
+            status=d.get("status", "queued"),
+            attempts=int(d.get("attempts", 0)),
+            enqueued_at=int(d.get("enqueuedAt", 0)),
+        )
+
+
+def reset_sending_to_queued(items: list[OutboxEntry[T]]) -> list[OutboxEntry[T]]:
+    """Reset entries stuck ``sending`` (claimed but never resolved) to ``queued``."""
+    for it in items:
+        if it.status == "sending":
+            it.status = "queued"
+    return items
+
+
+class OutboxQueue(Generic[T]):
+    """See module docstring. Mutators are ``async`` because each writes through to
+    the cache; ``get`` / ``pending`` / ``subscribe`` are synchronous."""
+
+    def __init__(self, cache: LocalCache) -> None:
+        self._cache = cache
+        self._items: list[OutboxEntry[T]] = []
+        self._cache_key: Optional[str] = None
+        self._listeners: set[Callable[[], None]] = set()
+
+    def get(self) -> list[OutboxEntry[T]]:
+        return self._items
+
+    def subscribe(self, listener: Callable[[], None]) -> Callable[[], None]:
+        self._listeners.add(listener)
+        return lambda: self._listeners.discard(listener)
+
+    def _notify(self) -> None:
+        for listener in list(self._listeners):
+            listener()
+
+    async def _commit(self, items: list[OutboxEntry[T]]) -> None:
+        self._items = items
+        if self._cache_key is not None:
+            try:
+                await self._cache.set_item(self._cache_key, json.dumps([e.to_dict() for e in items]))
+            except Exception:
+                pass
+        self._notify()
+
+    async def hydrate(self, cache_key: str) -> None:
+        self._cache_key = cache_key
+        loaded: list[OutboxEntry[T]] = []
+        try:
+            raw = await self._cache.get_item(cache_key)
+            if raw:
+                parsed = json.loads(raw)
+                if isinstance(parsed, list):
+                    loaded = [OutboxEntry.from_dict(d) for d in parsed]
+        except Exception:
+            loaded = []
+        self._items = reset_sending_to_queued(loaded)
+        self._notify()
+
+    def clear(self) -> None:
+        self._cache_key = None
+        self._items = []
+        self._notify()
+
+    async def enqueue(self, id: str, item: T, ts: Optional[int] = None) -> None:
+        if any(i.id == id for i in self._items):
+            return  # dedup by id
+        entry = OutboxEntry(id=id, item=item, status="queued", attempts=0, enqueued_at=ts or int(time.time() * 1000))
+        await self._commit([*self._items, entry])
+
+    async def claim(self, id: str) -> bool:
+        it = next((i for i in self._items if i.id == id), None)
+        if it is None or it.status == "sending":
+            return False
+        await self._commit([
+            (OutboxEntry(i.id, i.item, "sending", i.attempts, i.enqueued_at) if i.id == id else i)
+            for i in self._items
+        ])
+        return True
+
+    async def remove(self, id: str) -> None:
+        await self._commit([i for i in self._items if i.id != id])
+
+    async def record_failure(self, id: str, max_attempts: int) -> None:
+        def updated(i: OutboxEntry[T]) -> OutboxEntry[T]:
+            if i.id != id:
+                return i
+            attempts = i.attempts + 1
+            status = "failed" if attempts >= max_attempts else "queued"
+            return OutboxEntry(i.id, i.item, status, attempts, i.enqueued_at)
+
+        await self._commit([updated(i) for i in self._items])
+
+    async def mark_failed(self, id: str) -> None:
+        await self._commit([
+            (OutboxEntry(i.id, i.item, "failed", i.attempts + 1, i.enqueued_at) if i.id == id else i)
+            for i in self._items
+        ])
+
+    async def retry(self, id: str) -> None:
+        await self._commit([
+            (OutboxEntry(i.id, i.item, "queued", i.attempts, i.enqueued_at) if i.id == id else i)
+            for i in self._items
+        ])
+
+    def pending(self, predicate: Optional[Callable[[T], bool]] = None) -> list[OutboxEntry[T]]:
+        return [i for i in self._items if (predicate(i.item) if predicate else True)]

starfish_outbox-3.0.0a20/starfish_outbox.egg-info/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: starfish-outbox
+Version: 3.0.0a20
+Summary: Starfish durable offline write-queue — dedup-by-id, claim, retry, reconnect-drain, persisted per identity, framework-free
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+
+# starfish-outbox
+
+A durable, per-identity **offline write-queue** — the client-side complement to the
+server-side `queuing` extension.
+
+Generic over the queued item: you own what an item is, supply its dedup `id`, and a
+`send` that performs the real write. The queue handles persistence (write-through to a
+`LocalCache`), dedup-by-id, single-shot claim (no double-send), attempt counting with
+auto-retry-then-fail, crash-safe recovery of stuck `sending` entries, and subscriptions.
+`drain_outbox` is connectivity-agnostic — you trigger it on whatever reconnect signal you have.
+
+## Install
+
+```sh
+pip install starfish-outbox
+```
+
+## Usage
+
+```python
+from starfish_outbox import OutboxQueue, drain_outbox
+
+queue: OutboxQueue = OutboxQueue(local_cache)  # local_cache: async get_item / set_item / remove_item
+await queue.hydrate(f"outbox.{user_id}")
+
+# Enqueue a write while offline (the id is yours to thread into the eventual write):
+await queue.enqueue(write_id, {"path": path, "body": body})
+
+# Drain on reconnect:
+async def send(entry):
+    await client.push(entry.item["path"], entry.item["body"], base_hash_for(entry.item["path"]))
+
+await drain_outbox(queue, send, max_attempts=5)
+```
+
+Mutating methods (`enqueue` / `claim` / `remove` / `retry` / …) are `async` because each
+writes through to the cache; `get` / `pending` / `subscribe` are synchronous. JSON-compatible
+on disk with the TypeScript `@drakkar.software/starfish-outbox`.

starfish_outbox-3.0.0a20/starfish_outbox.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+starfish_outbox/__init__.py
+starfish_outbox/drain.py
+starfish_outbox/queue.py
+starfish_outbox.egg-info/PKG-INFO
+starfish_outbox.egg-info/SOURCES.txt
+starfish_outbox.egg-info/dependency_links.txt
+starfish_outbox.egg-info/requires.txt
+starfish_outbox.egg-info/top_level.txt
+tests/test_outbox.py

starfish_outbox-3.0.0a20/starfish_outbox.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

starfish_outbox-3.0.0a20/starfish_outbox.egg-info/requires.txt

@@ -0,0 +1,4 @@
+
+[dev]
+pytest>=7.0
+pytest-asyncio>=0.21

starfish_outbox-3.0.0a20/starfish_outbox.egg-info/top_level.txt

@@ -0,0 +1 @@
+starfish_outbox

starfish_outbox-3.0.0a20/tests/test_outbox.py

@@ -0,0 +1,122 @@
+"""Durable write-queue: dedup, persistence + crash-safe hydrate, single-shot claim,
+drain success/failure with auto-retry-then-fail, predicate filtering."""
+
+from __future__ import annotations
+
+import json
+from typing import Optional
+
+from starfish_outbox import OutboxQueue, drain_outbox
+
+
+class MemCache:
+    def __init__(self) -> None:
+        self.map: dict[str, str] = {}
+
+    async def get_item(self, key: str) -> Optional[str]:
+        return self.map.get(key)
+
+    async def set_item(self, key: str, value: str) -> None:
+        self.map[key] = value
+
+    async def remove_item(self, key: str) -> None:
+        self.map.pop(key, None)
+
+
+async def test_enqueue_dedup_and_persist() -> None:
+    cache = MemCache()
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    await q.enqueue("m1", {"room": "r1", "text": "hi"})
+    await q.enqueue("m1", {"room": "r1", "text": "dup"})  # ignored
+    await q.enqueue("m2", {"room": "r2", "text": "yo"})
+    assert [e.id for e in q.get()] == ["m1", "m2"]
+    persisted = json.loads(cache.map["outbox.me"])
+    assert len(persisted) == 2
+    assert persisted[0]["item"]["text"] == "hi"
+    assert "enqueuedAt" in persisted[0]
+
+
+async def test_reset_sending_on_hydrate() -> None:
+    cache = MemCache()
+    cache.map["outbox.me"] = json.dumps(
+        [{"id": "m1", "item": {"text": "x"}, "status": "sending", "attempts": 1, "enqueuedAt": 1}]
+    )
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    assert q.get()[0].status == "queued"
+
+
+async def test_claim_single_shot() -> None:
+    cache = MemCache()
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    await q.enqueue("m1", {"text": "x"})
+    assert await q.claim("m1") is True
+    assert await q.claim("m1") is False
+
+
+async def test_drain_sends_oldest_first_and_removes() -> None:
+    cache = MemCache()
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    await q.enqueue("m1", {"text": "a"})
+    await q.enqueue("m2", {"text": "b"})
+    order: list[str] = []
+
+    async def send(entry):
+        order.append(entry.item["text"])
+
+    res = await drain_outbox(q, send)
+    assert order == ["a", "b"]
+    assert (res.sent, res.failed) == (2, 0)
+    assert q.get() == []
+
+
+async def test_auto_retry_then_failed() -> None:
+    cache = MemCache()
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    await q.enqueue("m1", {"text": "x"})
+
+    async def send(_entry):
+        raise RuntimeError("offline")
+
+    r1 = await drain_outbox(q, send, max_attempts=2)
+    assert (r1.sent, r1.failed) == (0, 0)
+    assert q.get()[0].status == "queued" and q.get()[0].attempts == 1
+    r2 = await drain_outbox(q, send, max_attempts=2)
+    assert (r2.sent, r2.failed) == (0, 1)
+    assert q.get()[0].status == "failed"
+
+
+async def test_failed_skipped_until_retry() -> None:
+    cache = MemCache()
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    await q.enqueue("m1", {"text": "x"})
+
+    async def fail(_e):
+        raise RuntimeError("x")
+
+    await drain_outbox(q, fail, max_attempts=1)
+    assert q.get()[0].status == "failed"
+
+    async def ok(_e):
+        return None
+
+    r1 = await drain_outbox(q, ok)
+    assert r1.sent == 0
+    await q.retry("m1")
+    r2 = await drain_outbox(q, ok)
+    assert r2.sent == 1
+
+
+async def test_pending_predicate() -> None:
+    cache = MemCache()
+    q: OutboxQueue = OutboxQueue(cache)
+    await q.hydrate("outbox.me")
+    await q.enqueue("m1", {"room": "r1"})
+    await q.enqueue("m2", {"room": "r2"})
+    await q.enqueue("m3", {"room": "r1"})
+    assert [e.id for e in q.pending(lambda m: m["room"] == "r1")] == ["m1", "m3"]