dispatch-kit 0.1.0__tar.gz → 0.2.0__tar.gz

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dispatch-kit
-Version: 0.1.0
+Version: 0.2.0
 Summary: Pure, fail-closed cost-gating for expensive remote/external work: a hard $ budget cap, backend routing (local->cloud->SDK), and opt-in audited API egress.
 Author-email: Aryan Falahatpisheh <aryanfalahat@gmail.com>
 License: MIT

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dispatch-kit"
-version = "0.1.0"
+version = "0.2.0"
 description = "Pure, fail-closed cost-gating for expensive remote/external work: a hard $ budget cap, backend routing (local->cloud->SDK), and opt-in audited API egress."
 authors = [{ name = "Aryan Falahatpisheh", email = "aryanfalahat@gmail.com" }]
 readme = "README.md"
@@ -77,6 +77,8 @@ source-roots = ["src"]
 max-line-length = 100
 
 [tool.pylint.design]
-# Pure value-object domain: frozen data records with few/no methods are expected.
+# Pure value-object domain: frozen data records with few/no methods — and, for a full
+# reproducibility record (Provenance), many fields — are expected, not a design smell.
 min-public-methods = 0
 max-args = 6
+max-attributes = 15

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/src/dispatch_kit/__init__.py

@@ -30,6 +30,7 @@ from .dispatch import (
     DispatchError,
     JobStore,
     Lease,
+    RetriableError,
     Transport,
     WorkerExecutor,
     is_lease_stale,
@@ -42,7 +43,10 @@ from .egress import (
     SecretRef,
     log_egress,
 )
+from .engine import drain
 from .estimate import CostEstimate, HostCapabilities, vram_fits
+from .faults import Contained
+from .provenance import Determinism, GpuContext, Provenance
 from .routing import (
     BackendCapabilities,
     BackendKind,
@@ -62,16 +66,21 @@ __all__ = [
     "BudgetCap",
     "BudgetState",
     "BudgetWindow",
+    "Contained",
     "CostEstimate",
     "CostRates",
+    "Determinism",
     "DispatchError",
     "EnvLookup",
     "ExternalEndpoint",
+    "GpuContext",
     "HostCapabilities",
     "JobStore",
     "Lease",
     "NoEligibleBackendError",
     "NodeIdentity",
+    "Provenance",
+    "RetriableError",
     "Routable",
     "SecretMissingError",
     "SecretRef",
@@ -79,6 +88,7 @@ __all__ = [
     "Transport",
     "WorkerExecutor",
     "admits",
+    "drain",
     "estimate_cost",
     "is_lease_stale",
     "log_egress",

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/src/dispatch_kit/dispatch.py

@@ -2,8 +2,9 @@
 
 Both a PULL worker (it polls: claim a job, run it, complete it) and a PUSH orchestrator (it posts a
 job to a worker and waits) need the SAME guarantees: a job is CLAIMED atomically so it runs exactly
-once, a stale result is REJECTED, and a worker that dies mid-job has its lease RECOVERED. This
-module is the shared CONTRACT — the pure lease rules + the store/transport/worker protocols. The
+once, a stale result is REJECTED, a clean failure is RECORDED (terminally, or retriably so it is
+re-queued up to a cap), and a worker that dies mid-job has its lease RECOVERED. This module is the
+shared CONTRACT — the pure lease rules + the store/transport/worker protocols. The
 store (SQLite vs SQLAlchemy), the transport (pull vs push), and the payload (a transcribe request vs
 a tool invocation) are per-app ADAPTERS, so two apps converge on one model WITHOUT a shared DB.
 
@@ -22,6 +23,15 @@ class DispatchError(RuntimeError):
     """A dispatch invariant was violated (a stale complete, or a job recovered past its cap)."""
 
 
+class RetriableError(Exception):
+    """An executor raises this to mark a failure TRANSIENT, so the engine re-queues the job for
+    another attempt (bumping the attempt count, then poisoning past the cap) instead of failing it
+    terminally. Use it for an infrastructure hiccup a retry can fix — a timeout, an OOM, a rate
+    limit — NOT for a bad input or a logic error, where retrying only wastes attempts. Any OTHER
+    exception from the executor is treated as terminal.
+    """
+
+
 @dataclass(frozen=True, slots=True)
 class Lease:
     """A claim on a job: when it was leased and how many times it has been recovered.
@@ -56,15 +66,17 @@ def should_give_up(attempts: int, max_attempts: int) -> bool:
 
 
 class JobStore(Protocol):
-    """The authoritative job store — the ONE place a job's claim/complete is decided, ATOMICALLY.
+    """The authoritative job store — the ONE place claim/complete/fail is decided, ATOMICALLY.
 
     ``claim`` must be atomic (one transaction / compare-and-set): two concurrent workers can never
     both claim the same job — that single claim is the run-exactly-once guarantee. ``complete`` must
     REJECT a result for a job not currently leased/running (a stale resubmit after the lease was
-    recovered), returning ``False`` so a re-run cannot clobber a fresh result. ``recover_stale``
-    re-leases jobs whose lease outlived the TTL (see :func:`is_lease_stale`), bumping the attempt
-    count and failing a job past ``max_attempts`` (see :func:`should_give_up`). The payload type is
-    the app's own (a transcribe request, a tool invocation, ...).
+    recovered), returning ``False`` so a re-run cannot clobber a fresh result. ``fail`` records a
+    clean failure: terminal (a bad input), or retriable — re-queued for another attempt, bumping
+    attempts and poisoned past the cap (like a recovered crash). ``recover_stale`` re-leases jobs
+    whose lease outlived the TTL (see :func:`is_lease_stale`), bumping the attempt count and
+    failing a job past ``max_attempts`` (see :func:`should_give_up`). The payload type is the
+    app's own (a transcribe request, a tool invocation, ...).
     """
 
     def claim(self, lanes: Sequence[str]) -> tuple[str, Any] | None:
@@ -73,6 +85,16 @@ class JobStore(Protocol):
     def complete(self, job_id: str, result: Any) -> bool:
         """Apply a result IFF the job is leased/running; ``False`` if stale (done/recovered)."""
 
+    def fail(self, job_id: str, error: str, *, retriable: bool, max_attempts: int) -> bool:
+        """Record a clean failure of a leased job; ``False`` if it was not leased/running.
+
+        ``retriable=False`` fails it terminally now — a bad input or logic error a retry cannot
+        fix. ``retriable=True`` returns it to the queue for another attempt, bumping the attempt
+        count; once it has been attempted ``max_attempts`` times it is failed terminally instead
+        (the poison rule of :func:`should_give_up`), so a persistently-transient job can never
+        loop forever.
+        """
+
     def recover_stale(self, *, now: float, ttl_seconds: float, max_attempts: int) -> Sequence[str]:
         """Re-lease jobs whose lease is stale (return their ids); fail those past the cap."""
 

dispatch_kit-0.2.0/src/dispatch_kit/engine.py

@@ -0,0 +1,63 @@
+"""The run-exactly-once engine — drain a :class:`~dispatch_kit.dispatch.JobStore`.
+
+Pure coordination over the dispatch ports: :func:`drain` owns the claim -> run -> record skeleton
+AND the failure classification, so every app gets robust draining without re-implementing it. The
+JobStore (persistence + atomic claim + retry timing) and the ``execute`` callable (the actual work)
+are the app's adapters — pull a transcribe request, run a tool invocation, whatever; the engine only
+coordinates.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from .dispatch import JobStore, RetriableError
+from .faults import Contained
+
+
+def drain(
+    store: JobStore,
+    execute: Callable[[Any], Any],
+    lanes: Sequence[str],
+    *,
+    max_attempts: int,
+) -> int:
+    """Claim and run every currently-runnable job in ``lanes``; return how many were handled.
+
+    Each job is isolated (:class:`~dispatch_kit.faults.Contained`): a failure is recorded —
+    terminal, or retriable if ``execute`` raised :class:`RetriableError` — and the drain CONTINUES,
+    so one bad job never stalls the queue. Loops until ``claim`` returns ``None``, so call it on a
+    poll interval. Whether a retriable re-queue is retried this pass or later (a backoff) is the
+    store's policy, not the engine's.
+    """
+    handled = 0
+    while (claimed := store.claim(lanes)) is not None:
+        job_id, payload = claimed
+        _record(store, execute, job_id, payload, max_attempts=max_attempts)
+        handled += 1
+    return handled
+
+
+def _record(
+    store: JobStore,
+    execute: Callable[[Any], Any],
+    job_id: str,
+    payload: Any,
+    *,
+    max_attempts: int,
+) -> None:
+    """Run one claimed job and record its outcome: complete, or fail (terminal / retriable)."""
+    result: Any = None
+    with Contained() as box:
+        result = execute(payload)
+    error = box.error
+    if error is None:
+        store.complete(job_id, result)
+    else:
+        store.fail(
+            job_id,
+            str(error),
+            retriable=isinstance(error, RetriableError),
+            max_attempts=max_attempts,
+        )

dispatch_kit-0.2.0/src/dispatch_kit/faults.py

@@ -0,0 +1,43 @@
+"""Fault isolation — contain a unit of work's failure instead of letting it kill the loop.
+
+A drain loop / worker / handler must not die because one job raised. :class:`Contained` records an
+ordinary exception via the context-manager protocol, so there is exactly ONE audited place where
+arbitrary failures stop (no scattered ``except Exception``). A ``BaseException`` that is not an
+``Exception`` (``KeyboardInterrupt`` / ``SystemExit``) still propagates, so shutdown works.
+"""
+
+from __future__ import annotations
+
+from types import TracebackType
+
+
+class Contained:
+    """Context manager that records, rather than raises, an ordinary exception.
+
+    Read ``.error`` after the block to handle what happened::
+
+        with Contained() as box:
+            result = risky()
+        if box.error is not None:
+            ...  # handle the failure (``result`` is unset)
+
+    It contains failures through ``__exit__`` (returning ``True`` swallows) rather than
+    ``except Exception``, so there is exactly one audited boundary where arbitrary failures stop.
+    """
+
+    def __init__(self) -> None:
+        self.error: Exception | None = None
+
+    def __enter__(self) -> Contained:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> bool:
+        if exc is None or not isinstance(exc, Exception):
+            return False  # clean exit, or a BaseException we must let propagate
+        self.error = exc
+        return True  # contained: swallow the exception

dispatch_kit-0.2.0/src/dispatch_kit/provenance.py

@@ -0,0 +1,86 @@
+"""Provenance — the reproducibility record stamped on a completed job.
+
+Generic across apps (a tool invocation, an ML transcription, any dispatched compute): every field
+that makes a run reproducible is mandatory unless genuinely inapplicable (no GPU on a CPU tool, no
+seed on a deterministic one), and those exceptions are explicit ``None``, never silent omissions —
+a job that cannot state its provenance cannot commit. Invalid provenance is unrepresentable: a
+seed/determinism contradiction is rejected at construction.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+
+from .routing import BackendKind, NodeIdentity
+
+__all__ = ["Determinism", "GpuContext", "Provenance"]
+
+
+class Determinism(enum.StrEnum):
+    """How reproducible a tool's output is — recorded so non-determinism is declared, not hidden."""
+
+    DETERMINISTIC = "deterministic"
+    """Same inputs always give bit-identical outputs."""
+    SEEDED = "seeded"
+    """Reproducible given the recorded seed."""
+    NONDETERMINISTIC = "nondeterministic"
+    """Output varies run-to-run (e.g. unseeded sampling); flagged for ensemble handling."""
+
+
+@dataclass(frozen=True, slots=True)
+class GpuContext:
+    """The GPU a job ran on; ``None`` at the Provenance level means a CPU-only job."""
+
+    model: str
+    vram_gb: float
+
+    def __post_init__(self) -> None:
+        if self.vram_gb <= 0:
+            raise ValueError(f"GPU vram_gb must be positive; got {self.vram_gb}")
+
+
+@dataclass(frozen=True, slots=True)
+class Provenance:
+    """The full reproducibility record stamped onto a completed job.
+
+    Hashes (weights sha256, params, input) and reference-data versions let a re-run be checked for
+    drift; ``determinism`` plus ``seed`` say whether an identical result is even expected. A
+    ``NONDETERMINISTIC`` job with a seed, or a ``SEEDED`` job without one, is a contradiction and is
+    rejected at construction — invalid provenance is unrepresentable.
+    """
+
+    tool_id: str
+    tool_version: str
+    weights_id: str | None
+    weights_sha256: str | None
+    params_hash: str
+    input_hash: str
+    reference_data_versions: dict[str, str]
+    container_digest: str | None
+    seed: int | None
+    determinism: Determinism
+    runtime_seconds: float
+    gpu: GpuContext | None = None
+    backend: BackendKind = BackendKind.LOCAL
+    """Where compute ran. Defaults to LOCAL: a job with no remote dispatch ran here."""
+    node: NodeIdentity | None = None
+    """The pinned identity of the node that ran the job; ``None`` only when the host is unknown."""
+    extra: dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.tool_id or not self.tool_version:
+            raise ValueError("provenance requires a tool_id and tool_version")
+        if not self.params_hash or not self.input_hash:
+            raise ValueError("provenance requires params_hash and input_hash")
+        if self.runtime_seconds < 0:
+            raise ValueError(f"runtime_seconds must be non-negative; got {self.runtime_seconds}")
+        self._validate_seed_determinism()
+
+    def _validate_seed_determinism(self) -> None:
+        if self.determinism is Determinism.SEEDED and self.seed is None:
+            raise ValueError("a SEEDED job must record its seed")
+        if self.determinism is Determinism.NONDETERMINISTIC and self.seed is not None:
+            raise ValueError(
+                "a NONDETERMINISTIC job must not carry a seed (a seed implies reproducibility)"
+            )

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/src/dispatch_kit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dispatch-kit
-Version: 0.1.0
+Version: 0.2.0
 Summary: Pure, fail-closed cost-gating for expensive remote/external work: a hard $ budget cap, backend routing (local->cloud->SDK), and opt-in audited API egress.
 Author-email: Aryan Falahatpisheh <aryanfalahat@gmail.com>
 License: MIT

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/src/dispatch_kit.egg-info/SOURCES.txt

@@ -6,7 +6,10 @@ src/dispatch_kit/approval.py
 src/dispatch_kit/budget.py
 src/dispatch_kit/dispatch.py
 src/dispatch_kit/egress.py
+src/dispatch_kit/engine.py
 src/dispatch_kit/estimate.py
+src/dispatch_kit/faults.py
+src/dispatch_kit/provenance.py
 src/dispatch_kit/py.typed
 src/dispatch_kit/routing.py
 src/dispatch_kit.egg-info/PKG-INFO
@@ -18,5 +21,8 @@ tests/test_approval.py
 tests/test_budget.py
 tests/test_dispatch.py
 tests/test_egress.py
+tests/test_engine.py
 tests/test_estimate.py
+tests/test_faults.py
+tests/test_provenance.py
 tests/test_routing.py

{dispatch_kit-0.1.0 → dispatch_kit-0.2.0}/tests/test_dispatch.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from dispatch_kit import Lease, is_lease_stale, should_give_up
+from dispatch_kit import Lease, RetriableError, is_lease_stale, should_give_up
 
 
 def test_lease_is_stale_only_past_the_ttl() -> None:
@@ -23,3 +23,13 @@ def test_lease_carries_its_attempt_count() -> None:
     assert fresh.attempts == 0
     recovered = Lease(job_id="j1", leased_at=200.0, attempts=1)
     assert recovered.attempts == 1
+
+
+def test_retriable_error_marks_a_failure_for_retry() -> None:
+    # An executor raises RetriableError to opt a transient failure into re-queue (vs terminal);
+    # it is a plain Exception the engine catches like any other, routing it to fail(retriable=True).
+    assert issubclass(RetriableError, Exception)
+    try:
+        raise RetriableError("transient")
+    except RetriableError as exc:
+        assert str(exc) == "transient"

dispatch_kit-0.2.0/tests/test_engine.py

@@ -0,0 +1,103 @@
+"""The drain engine over a fake in-memory JobStore: complete, terminal/retriable fail, isolation.
+
+The fake store implements just enough of the contract to drive the engine. Its ``fail(retriable)``
+re-queues immediately (no backoff), so a retriable job reaches the poison cap within one ``drain``
+pass — a real store adds a backoff so a transient failure isn't burned to poison instantly.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from dispatch_kit import RetriableError, drain
+from dispatch_kit.dispatch import should_give_up
+
+
+class _FakeStore:
+    def __init__(self, jobs: dict[str, dict[str, Any]]) -> None:
+        self.jobs = jobs  # id -> {payload, lane, status, attempts}
+        self.results: dict[str, Any] = {}
+        self.errors: dict[str, str] = {}
+
+    def claim(self, lanes: Sequence[str]) -> tuple[str, Any] | None:
+        for job_id, job in self.jobs.items():
+            if job["status"] == "pending" and job["lane"] in lanes:
+                job["status"] = "working"
+                return job_id, job["payload"]
+        return None
+
+    def complete(self, job_id: str, result: Any) -> bool:
+        job = self.jobs.get(job_id)
+        if job is None or job["status"] != "working":
+            return False
+        job["status"] = "done"
+        self.results[job_id] = result
+        return True
+
+    def fail(self, job_id: str, error: str, *, retriable: bool, max_attempts: int) -> bool:
+        job = self.jobs.get(job_id)
+        if job is None or job["status"] != "working":
+            return False
+        if retriable:
+            job["attempts"] += 1
+            if should_give_up(job["attempts"], max_attempts):
+                job["status"] = "failed"
+                self.errors[job_id] = error
+            else:
+                job["status"] = "pending"  # re-queue
+        else:
+            job["status"] = "failed"
+            self.errors[job_id] = error
+        return True
+
+    def recover_stale(self, *, now: float, ttl_seconds: float, max_attempts: int) -> Sequence[str]:
+        return []
+
+
+def _job(payload: str, *, lane: str = "ml") -> dict[str, Any]:
+    return {"payload": payload, "lane": lane, "status": "pending", "attempts": 0}
+
+
+def test_drain_runs_every_pending_job_and_completes_it() -> None:
+    store = _FakeStore({"j1": _job("a"), "j2": _job("b")})
+    handled = drain(store, str.upper, ["ml"], max_attempts=3)
+    assert handled == 2
+    assert store.results == {"j1": "A", "j2": "B"}
+    assert all(job["status"] == "done" for job in store.jobs.values())
+
+
+def test_terminal_failure_is_recorded_and_draining_continues() -> None:
+    def execute(payload: str) -> str:
+        if payload == "bad":
+            raise ValueError("nope")
+        return payload
+
+    store = _FakeStore({"j1": _job("bad"), "j2": _job("ok")})
+    drain(store, execute, ["ml"], max_attempts=3)
+    assert store.jobs["j1"]["status"] == "failed"
+    assert "nope" in store.errors["j1"]
+    assert store.results == {"j2": "ok"}  # the good job still ran despite the bad one
+
+
+def test_retriable_failure_requeues_then_poisons_at_the_cap() -> None:
+    def execute(_payload: str) -> str:
+        raise RetriableError("transient")
+
+    store = _FakeStore({"j1": _job("x")})
+    drain(store, execute, ["ml"], max_attempts=2)
+    assert store.jobs["j1"]["status"] == "failed"  # poisoned after the cap
+    assert store.jobs["j1"]["attempts"] == 2
+    assert "transient" in store.errors["j1"]
+
+
+def test_drain_only_touches_its_lanes() -> None:
+    store = _FakeStore({"j1": _job("a", lane="ml"), "j2": _job("b", lane="sync")})
+    handled = drain(store, str.upper, ["ml"], max_attempts=3)
+    assert handled == 1
+    assert store.jobs["j1"]["status"] == "done"
+    assert store.jobs["j2"]["status"] == "pending"  # the other lane is untouched
+
+
+def test_drain_returns_zero_when_nothing_runnable() -> None:
+    assert drain(_FakeStore({}), str.upper, ["ml"], max_attempts=3) == 0

dispatch_kit-0.2.0/tests/test_faults.py

@@ -0,0 +1,26 @@
+"""Contained records an ordinary failure but lets a BaseException through."""
+
+from __future__ import annotations
+
+import pytest
+
+from dispatch_kit import Contained
+
+
+def test_contained_records_an_ordinary_exception() -> None:
+    with Contained() as box:
+        raise ValueError("boom")
+    assert isinstance(box.error, ValueError)
+    assert str(box.error) == "boom"
+
+
+def test_contained_is_clean_when_nothing_raises() -> None:
+    with Contained() as box:
+        pass
+    assert box.error is None
+
+
+def test_contained_lets_base_exceptions_propagate() -> None:
+    # KeyboardInterrupt / SystemExit must NOT be swallowed — shutdown + cancellation still work.
+    with pytest.raises(KeyboardInterrupt), Contained():
+        raise KeyboardInterrupt

dispatch_kit-0.2.0/tests/test_provenance.py

@@ -0,0 +1,61 @@
+"""Provenance is a reproducibility record whose invalid states are unrepresentable."""
+
+from __future__ import annotations
+
+import pytest
+
+from dispatch_kit import BackendKind, Determinism, GpuContext, Provenance
+
+
+def _prov(
+    *,
+    tool_id: str = "tool",
+    params_hash: str = "p",
+    runtime_seconds: float = 1.0,
+    determinism: Determinism = Determinism.DETERMINISTIC,
+    seed: int | None = None,
+) -> Provenance:
+    return Provenance(
+        tool_id=tool_id,
+        tool_version="1.0",
+        weights_id=None,
+        weights_sha256=None,
+        params_hash=params_hash,
+        input_hash="i",
+        reference_data_versions={},
+        container_digest=None,
+        seed=seed,
+        determinism=determinism,
+        runtime_seconds=runtime_seconds,
+    )
+
+
+def test_a_valid_deterministic_provenance_constructs() -> None:
+    prov = _prov()
+    assert prov.backend is BackendKind.LOCAL  # ran here unless a remote dispatch says otherwise
+    assert prov.gpu is None  # CPU-only by default
+
+
+def test_missing_tool_or_hashes_are_rejected() -> None:
+    with pytest.raises(ValueError, match="tool_id"):
+        _prov(tool_id="")
+    with pytest.raises(ValueError, match="params_hash"):
+        _prov(params_hash="")
+
+
+def test_negative_runtime_is_rejected() -> None:
+    with pytest.raises(ValueError, match="runtime_seconds"):
+        _prov(runtime_seconds=-1.0)
+
+
+def test_seed_determinism_contradictions_are_rejected() -> None:
+    with pytest.raises(ValueError, match="must record its seed"):
+        _prov(determinism=Determinism.SEEDED, seed=None)
+    with pytest.raises(ValueError, match="must not carry a seed"):
+        _prov(determinism=Determinism.NONDETERMINISTIC, seed=42)
+
+
+def test_gpu_vram_must_be_positive() -> None:
+    with pytest.raises(ValueError, match="vram_gb"):
+        GpuContext(model="L4", vram_gb=0.0)
+    assert GpuContext(model="L4", vram_gb=24.0).vram_gb == 24.0