dispatch-kit 0.1.0__tar.gz → 0.3.0__tar.gz

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dispatch-kit
-Version: 0.1.0
+Version: 0.3.0
 Summary: Pure, fail-closed cost-gating for expensive remote/external work: a hard $ budget cap, backend routing (local->cloud->SDK), and opt-in audited API egress.
 Author-email: Aryan Falahatpisheh <aryanfalahat@gmail.com>
 License: MIT

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dispatch-kit"
-version = "0.1.0"
+version = "0.3.0"
 description = "Pure, fail-closed cost-gating for expensive remote/external work: a hard $ budget cap, backend routing (local->cloud->SDK), and opt-in audited API egress."
 authors = [{ name = "Aryan Falahatpisheh", email = "aryanfalahat@gmail.com" }]
 readme = "README.md"
@@ -77,6 +77,8 @@ source-roots = ["src"]
 max-line-length = 100
 
 [tool.pylint.design]
-# Pure value-object domain: frozen data records with few/no methods are expected.
+# Pure value-object domain: frozen data records with few/no methods — and, for a full
+# reproducibility record (Provenance), many fields — are expected, not a design smell.
 min-public-methods = 0
 max-args = 6
+max-attributes = 15

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/src/dispatch_kit/__init__.py

@@ -30,6 +30,7 @@ from .dispatch import (
     DispatchError,
     JobStore,
     Lease,
+    RetriableError,
     Transport,
     WorkerExecutor,
     is_lease_stale,
@@ -42,7 +43,11 @@ from .egress import (
     SecretRef,
     log_egress,
 )
+from .engine import drain
 from .estimate import CostEstimate, HostCapabilities, vram_fits
+from .faults import Contained
+from .observe import Observe, ObserveConfig
+from .provenance import Determinism, GpuContext, Provenance
 from .routing import (
     BackendCapabilities,
     BackendKind,
@@ -62,16 +67,23 @@ __all__ = [
     "BudgetCap",
     "BudgetState",
     "BudgetWindow",
+    "Contained",
     "CostEstimate",
     "CostRates",
+    "Determinism",
     "DispatchError",
     "EnvLookup",
     "ExternalEndpoint",
+    "GpuContext",
     "HostCapabilities",
     "JobStore",
     "Lease",
     "NoEligibleBackendError",
     "NodeIdentity",
+    "Observe",
+    "ObserveConfig",
+    "Provenance",
+    "RetriableError",
     "Routable",
     "SecretMissingError",
     "SecretRef",
@@ -79,6 +91,7 @@ __all__ = [
     "Transport",
     "WorkerExecutor",
     "admits",
+    "drain",
     "estimate_cost",
     "is_lease_stale",
     "log_egress",

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/src/dispatch_kit/dispatch.py

@@ -2,8 +2,9 @@
 
 Both a PULL worker (it polls: claim a job, run it, complete it) and a PUSH orchestrator (it posts a
 job to a worker and waits) need the SAME guarantees: a job is CLAIMED atomically so it runs exactly
-once, a stale result is REJECTED, and a worker that dies mid-job has its lease RECOVERED. This
-module is the shared CONTRACT — the pure lease rules + the store/transport/worker protocols. The
+once, a stale result is REJECTED, a clean failure is RECORDED (terminally, or retriably so it is
+re-queued up to a cap), and a worker that dies mid-job has its lease RECOVERED. This module is the
+shared CONTRACT — the pure lease rules + the store/transport/worker protocols. The
 store (SQLite vs SQLAlchemy), the transport (pull vs push), and the payload (a transcribe request vs
 a tool invocation) are per-app ADAPTERS, so two apps converge on one model WITHOUT a shared DB.
 
@@ -22,6 +23,15 @@ class DispatchError(RuntimeError):
     """A dispatch invariant was violated (a stale complete, or a job recovered past its cap)."""
 
 
+class RetriableError(Exception):
+    """An executor raises this to mark a failure TRANSIENT, so the engine re-queues the job for
+    another attempt (bumping the attempt count, then poisoning past the cap) instead of failing it
+    terminally. Use it for an infrastructure hiccup a retry can fix — a timeout, an OOM, a rate
+    limit — NOT for a bad input or a logic error, where retrying only wastes attempts. Any OTHER
+    exception from the executor is treated as terminal.
+    """
+
+
 @dataclass(frozen=True, slots=True)
 class Lease:
     """A claim on a job: when it was leased and how many times it has been recovered.
@@ -56,15 +66,17 @@ def should_give_up(attempts: int, max_attempts: int) -> bool:
 
 
 class JobStore(Protocol):
-    """The authoritative job store — the ONE place a job's claim/complete is decided, ATOMICALLY.
+    """The authoritative job store — the ONE place claim/complete/fail is decided, ATOMICALLY.
 
     ``claim`` must be atomic (one transaction / compare-and-set): two concurrent workers can never
     both claim the same job — that single claim is the run-exactly-once guarantee. ``complete`` must
     REJECT a result for a job not currently leased/running (a stale resubmit after the lease was
-    recovered), returning ``False`` so a re-run cannot clobber a fresh result. ``recover_stale``
-    re-leases jobs whose lease outlived the TTL (see :func:`is_lease_stale`), bumping the attempt
-    count and failing a job past ``max_attempts`` (see :func:`should_give_up`). The payload type is
-    the app's own (a transcribe request, a tool invocation, ...).
+    recovered), returning ``False`` so a re-run cannot clobber a fresh result. ``fail`` records a
+    clean failure: terminal (a bad input), or retriable — re-queued for another attempt, bumping
+    attempts and poisoned past the cap (like a recovered crash). ``recover_stale`` re-leases jobs
+    whose lease outlived the TTL (see :func:`is_lease_stale`), bumping the attempt count and
+    failing a job past ``max_attempts`` (see :func:`should_give_up`). The payload type is the
+    app's own (a transcribe request, a tool invocation, ...).
     """
 
     def claim(self, lanes: Sequence[str]) -> tuple[str, Any] | None:
@@ -73,6 +85,16 @@ class JobStore(Protocol):
     def complete(self, job_id: str, result: Any) -> bool:
         """Apply a result IFF the job is leased/running; ``False`` if stale (done/recovered)."""
 
+    def fail(self, job_id: str, error: str, *, retriable: bool, max_attempts: int) -> bool:
+        """Record a clean failure of a leased job; ``False`` if it was not leased/running.
+
+        ``retriable=False`` fails it terminally now — a bad input or logic error a retry cannot
+        fix. ``retriable=True`` returns it to the queue for another attempt, bumping the attempt
+        count; once it has been attempted ``max_attempts`` times it is failed terminally instead
+        (the poison rule of :func:`should_give_up`), so a persistently-transient job can never
+        loop forever.
+        """
+
     def recover_stale(self, *, now: float, ttl_seconds: float, max_attempts: int) -> Sequence[str]:
         """Re-lease jobs whose lease is stale (return their ids); fail those past the cap."""
 

dispatch_kit-0.3.0/src/dispatch_kit/engine.py

@@ -0,0 +1,63 @@
+"""The run-exactly-once engine — drain a :class:`~dispatch_kit.dispatch.JobStore`.
+
+Pure coordination over the dispatch ports: :func:`drain` owns the claim -> run -> record skeleton
+AND the failure classification, so every app gets robust draining without re-implementing it. The
+JobStore (persistence + atomic claim + retry timing) and the ``execute`` callable (the actual work)
+are the app's adapters — pull a transcribe request, run a tool invocation, whatever; the engine only
+coordinates.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from .dispatch import JobStore, RetriableError
+from .faults import Contained
+
+
+def drain(
+    store: JobStore,
+    execute: Callable[[Any], Any],
+    lanes: Sequence[str],
+    *,
+    max_attempts: int,
+) -> int:
+    """Claim and run every currently-runnable job in ``lanes``; return how many were handled.
+
+    Each job is isolated (:class:`~dispatch_kit.faults.Contained`): a failure is recorded —
+    terminal, or retriable if ``execute`` raised :class:`RetriableError` — and the drain CONTINUES,
+    so one bad job never stalls the queue. Loops until ``claim`` returns ``None``, so call it on a
+    poll interval. Whether a retriable re-queue is retried this pass or later (a backoff) is the
+    store's policy, not the engine's.
+    """
+    handled = 0
+    while (claimed := store.claim(lanes)) is not None:
+        job_id, payload = claimed
+        _record(store, execute, job_id, payload, max_attempts=max_attempts)
+        handled += 1
+    return handled
+
+
+def _record(
+    store: JobStore,
+    execute: Callable[[Any], Any],
+    job_id: str,
+    payload: Any,
+    *,
+    max_attempts: int,
+) -> None:
+    """Run one claimed job and record its outcome: complete, or fail (terminal / retriable)."""
+    result: Any = None
+    with Contained() as box:
+        result = execute(payload)
+    error = box.error
+    if error is None:
+        store.complete(job_id, result)
+    else:
+        store.fail(
+            job_id,
+            str(error),
+            retriable=isinstance(error, RetriableError),
+            max_attempts=max_attempts,
+        )

dispatch_kit-0.3.0/src/dispatch_kit/faults.py

@@ -0,0 +1,43 @@
+"""Fault isolation — contain a unit of work's failure instead of letting it kill the loop.
+
+A drain loop / worker / handler must not die because one job raised. :class:`Contained` records an
+ordinary exception via the context-manager protocol, so there is exactly ONE audited place where
+arbitrary failures stop (no scattered ``except Exception``). A ``BaseException`` that is not an
+``Exception`` (``KeyboardInterrupt`` / ``SystemExit``) still propagates, so shutdown works.
+"""
+
+from __future__ import annotations
+
+from types import TracebackType
+
+
+class Contained:
+    """Context manager that records, rather than raises, an ordinary exception.
+
+    Read ``.error`` after the block to handle what happened::
+
+        with Contained() as box:
+            result = risky()
+        if box.error is not None:
+            ...  # handle the failure (``result`` is unset)
+
+    It contains failures through ``__exit__`` (returning ``True`` swallows) rather than
+    ``except Exception``, so there is exactly one audited boundary where arbitrary failures stop.
+    """
+
+    def __init__(self) -> None:
+        self.error: Exception | None = None
+
+    def __enter__(self) -> Contained:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> bool:
+        if exc is None or not isinstance(exc, Exception):
+            return False  # clean exit, or a BaseException we must let propagate
+        self.error = exc
+        return True  # contained: swallow the exception

dispatch_kit-0.3.0/src/dispatch_kit/observe.py

@@ -0,0 +1,183 @@
+"""Optional observability facade — OpenTelemetry traces + Sentry errors + structured JSON logs.
+
+Opt-in, OFF by default: nothing leaves the box unless the operator sets an env var. Construct one
+:class:`Observe` per service with a service name + an env-var prefix; until a DSN/endpoint is set,
+every call is a zero-cost no-op, so instrumentation can live permanently at the call sites. The
+export SDKs are loaded via :func:`importlib.import_module` (a runtime call, so this module — and the
+whole package — stays import-clean and dependency-free without them); the facade degrades to no-ops
+when they are absent.
+
+This owns the parts every service shares: traces, error reporting, JSON log formatting. A service
+that also exports *metrics* builds its own meter, reusing identity via :meth:`Observe.resource`.
+"""
+
+from __future__ import annotations
+
+import importlib
+import json
+import logging
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ObserveConfig:
+    """Per-service observability settings.
+
+    ``env_prefix`` namespaces the opt-in vars: ``<PREFIX>_SENTRY_DSN`` (falling back to the standard
+    ``SENTRY_DSN``) enables Sentry, ``<PREFIX>_LOG_JSON`` turns on JSON logs; OTLP traces follow the
+    standard ``OTEL_EXPORTER_OTLP_ENDPOINT``. ``pip_hint`` is shown when an export is requested but
+    the optional SDK extra is not installed. ``tracer_name`` names the tracer/logger.
+    """
+
+    service_name: str
+    env_prefix: str
+    pip_hint: str
+    tracer_name: str
+
+
+class _JsonFormatter(logging.Formatter):
+    """One-line JSON per record (ts/level/logger/msg + service/command) so logs are parseable
+    and shippable — searchable across containers, not just grep on one box."""
+
+    def __init__(self, service: str, command: str | None) -> None:
+        super().__init__()
+        self._base: dict[str, str] = {"service": service}
+        if command:
+            self._base["command"] = command
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, object] = {
+            "ts": self.formatTime(record),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+            **self._base,
+        }
+        if record.exc_info:
+            payload["exc"] = self.formatException(record.exc_info)
+        return json.dumps(payload)
+
+
+class Observe:
+    """A per-service observability facade: traces (OTel), errors (Sentry), JSON logs.
+
+    Construct with an :class:`ObserveConfig`, call :meth:`setup` once at process start, then use
+    :meth:`span` / :meth:`capture_exception` freely — both no-op when export is off. The
+    ``build_sentry`` / ``build_tracer`` methods are the seams tests patch.
+    """
+
+    def __init__(self, config: ObserveConfig) -> None:
+        self._cfg = config
+        self._log = logging.getLogger(config.tracer_name)
+        self._sentry: Any = None
+        self._tracer: Any = None
+
+    def setup(self, *, command: str | None = None) -> None:
+        """Wire whichever exporters the environment opts into. Idempotent — resets first, so it can
+        be called once per process or re-run in tests."""
+        self.reset()
+        self._init_logging(command)
+        self._init_sentry()
+        self._init_tracer(command)
+
+    def reset(self) -> None:
+        """Deactivate every exporter (before a re-:meth:`setup` and in tests)."""
+        self._sentry = None
+        self._tracer = None
+
+    def is_active(self) -> dict[str, bool]:
+        """Which exporters are live — for a health check and asserted by tests."""
+        return {"sentry": self._sentry is not None, "otel": self._tracer is not None}
+
+    @contextmanager
+    def span(self, name: str, **attributes: object) -> Iterator[Any]:
+        """Trace the wrapped block as one OTel span (with ``attributes``). A no-op context yielding
+        ``None`` when tracing is off, so call sites need no guard."""
+        tracer = self._tracer
+        if tracer is None:
+            yield None
+            return
+        with tracer.start_as_current_span(name) as active:
+            for key, value in attributes.items():
+                active.set_attribute(key, value)
+            yield active
+
+    def capture_exception(self, exc: BaseException, **tags: object) -> None:
+        """Report a caught exception to Sentry with ``tags`` for grouping. No-op when off — the
+        caller still records its own failure normally."""
+        sentry = self._sentry
+        if sentry is None:
+            return
+        with sentry.new_scope() as scope:
+            for key, value in tags.items():
+                scope.set_tag(key, str(value))
+            sentry.capture_exception(exc)
+
+    def otlp_endpoint(self) -> str | None:
+        """The OTLP endpoint if set — a service building its own meter checks this."""
+        return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") or None
+
+    def resource(self, **extra: Any) -> Any:
+        """An OTel ``Resource`` carrying ``service.name`` + ``extra`` — so a service's own meter
+        shares this service's identity. Requires the opentelemetry SDK."""
+        resources = importlib.import_module("opentelemetry.sdk.resources")
+        return resources.Resource.create({"service.name": self._cfg.service_name, **extra})
+
+    def _init_logging(self, command: str | None) -> None:
+        if not os.environ.get(f"{self._cfg.env_prefix}_LOG_JSON"):
+            return
+        fmt = _JsonFormatter(self._cfg.service_name, command)
+        root = logging.getLogger()
+        if not root.handlers:
+            root.addHandler(logging.StreamHandler())
+        for handler in root.handlers:
+            handler.setFormatter(fmt)
+
+    def _init_sentry(self) -> None:
+        dsn = os.environ.get(f"{self._cfg.env_prefix}_SENTRY_DSN") or os.environ.get("SENTRY_DSN")
+        if not dsn:
+            return
+        try:
+            self._sentry = self.build_sentry(dsn)
+        except ImportError:
+            self._log.warning("SENTRY_DSN set but sentry-sdk is missing — %s", self._cfg.pip_hint)
+            return
+        self._log.info("Sentry error reporting enabled")
+
+    def _init_tracer(self, command: str | None) -> None:
+        endpoint = self.otlp_endpoint()
+        if not endpoint:
+            return
+        try:
+            self._tracer = self.build_tracer(command)
+        except ImportError:
+            self._log.warning(
+                "OTEL_EXPORTER_OTLP_ENDPOINT set but opentelemetry is missing — %s",
+                self._cfg.pip_hint,
+            )
+            return
+        self._log.info("OpenTelemetry export enabled -> %s", endpoint)
+
+    def build_sentry(self, dsn: str) -> Any:
+        """Initialise the real Sentry SDK (errors only; OTel owns traces). The dynamic import is the
+        seam tests patch + the ImportError surface when the extra is absent."""
+        sentry_sdk = importlib.import_module("sentry_sdk")
+        sentry_sdk.init(dsn=dsn, send_default_pii=False, traces_sample_rate=0.0)
+        return sentry_sdk
+
+    def build_tracer(self, command: str | None) -> Any:
+        """Register an OTLP-exporting tracer provider over this service's resource; return its
+        tracer. Endpoint/headers are read from the standard ``OTEL_EXPORTER_OTLP_*`` env."""
+        trace = importlib.import_module("opentelemetry.trace")
+        otlp = importlib.import_module("opentelemetry.exporter.otlp.proto.http.trace_exporter")
+        sdk_trace = importlib.import_module("opentelemetry.sdk.trace")
+        sdk_export = importlib.import_module("opentelemetry.sdk.trace.export")
+        extra = {f"{self._cfg.tracer_name}.command": command} if command else {}
+        provider = sdk_trace.TracerProvider(resource=self.resource(**extra))
+        provider.add_span_processor(sdk_export.BatchSpanProcessor(otlp.OTLPSpanExporter()))
+        trace.set_tracer_provider(provider)
+        return trace.get_tracer(self._cfg.tracer_name)

dispatch_kit-0.3.0/src/dispatch_kit/provenance.py

@@ -0,0 +1,86 @@
+"""Provenance — the reproducibility record stamped on a completed job.
+
+Generic across apps (a tool invocation, an ML transcription, any dispatched compute): every field
+that makes a run reproducible is mandatory unless genuinely inapplicable (no GPU on a CPU tool, no
+seed on a deterministic one), and those exceptions are explicit ``None``, never silent omissions —
+a job that cannot state its provenance cannot commit. Invalid provenance is unrepresentable: a
+seed/determinism contradiction is rejected at construction.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+
+from .routing import BackendKind, NodeIdentity
+
+__all__ = ["Determinism", "GpuContext", "Provenance"]
+
+
+class Determinism(enum.StrEnum):
+    """How reproducible a tool's output is — recorded so non-determinism is declared, not hidden."""
+
+    DETERMINISTIC = "deterministic"
+    """Same inputs always give bit-identical outputs."""
+    SEEDED = "seeded"
+    """Reproducible given the recorded seed."""
+    NONDETERMINISTIC = "nondeterministic"
+    """Output varies run-to-run (e.g. unseeded sampling); flagged for ensemble handling."""
+
+
+@dataclass(frozen=True, slots=True)
+class GpuContext:
+    """The GPU a job ran on; ``None`` at the Provenance level means a CPU-only job."""
+
+    model: str
+    vram_gb: float
+
+    def __post_init__(self) -> None:
+        if self.vram_gb <= 0:
+            raise ValueError(f"GPU vram_gb must be positive; got {self.vram_gb}")
+
+
+@dataclass(frozen=True, slots=True)
+class Provenance:
+    """The full reproducibility record stamped onto a completed job.
+
+    Hashes (weights sha256, params, input) and reference-data versions let a re-run be checked for
+    drift; ``determinism`` plus ``seed`` say whether an identical result is even expected. A
+    ``NONDETERMINISTIC`` job with a seed, or a ``SEEDED`` job without one, is a contradiction and is
+    rejected at construction — invalid provenance is unrepresentable.
+    """
+
+    tool_id: str
+    tool_version: str
+    weights_id: str | None
+    weights_sha256: str | None
+    params_hash: str
+    input_hash: str
+    reference_data_versions: dict[str, str]
+    container_digest: str | None
+    seed: int | None
+    determinism: Determinism
+    runtime_seconds: float
+    gpu: GpuContext | None = None
+    backend: BackendKind = BackendKind.LOCAL
+    """Where compute ran. Defaults to LOCAL: a job with no remote dispatch ran here."""
+    node: NodeIdentity | None = None
+    """The pinned identity of the node that ran the job; ``None`` only when the host is unknown."""
+    extra: dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.tool_id or not self.tool_version:
+            raise ValueError("provenance requires a tool_id and tool_version")
+        if not self.params_hash or not self.input_hash:
+            raise ValueError("provenance requires params_hash and input_hash")
+        if self.runtime_seconds < 0:
+            raise ValueError(f"runtime_seconds must be non-negative; got {self.runtime_seconds}")
+        self._validate_seed_determinism()
+
+    def _validate_seed_determinism(self) -> None:
+        if self.determinism is Determinism.SEEDED and self.seed is None:
+            raise ValueError("a SEEDED job must record its seed")
+        if self.determinism is Determinism.NONDETERMINISTIC and self.seed is not None:
+            raise ValueError(
+                "a NONDETERMINISTIC job must not carry a seed (a seed implies reproducibility)"
+            )

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/src/dispatch_kit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dispatch-kit
-Version: 0.1.0
+Version: 0.3.0
 Summary: Pure, fail-closed cost-gating for expensive remote/external work: a hard $ budget cap, backend routing (local->cloud->SDK), and opt-in audited API egress.
 Author-email: Aryan Falahatpisheh <aryanfalahat@gmail.com>
 License: MIT

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/src/dispatch_kit.egg-info/SOURCES.txt

@@ -6,7 +6,11 @@ src/dispatch_kit/approval.py
 src/dispatch_kit/budget.py
 src/dispatch_kit/dispatch.py
 src/dispatch_kit/egress.py
+src/dispatch_kit/engine.py
 src/dispatch_kit/estimate.py
+src/dispatch_kit/faults.py
+src/dispatch_kit/observe.py
+src/dispatch_kit/provenance.py
 src/dispatch_kit/py.typed
 src/dispatch_kit/routing.py
 src/dispatch_kit.egg-info/PKG-INFO
@@ -18,5 +22,9 @@ tests/test_approval.py
 tests/test_budget.py
 tests/test_dispatch.py
 tests/test_egress.py
+tests/test_engine.py
 tests/test_estimate.py
+tests/test_faults.py
+tests/test_observe.py
+tests/test_provenance.py
 tests/test_routing.py

{dispatch_kit-0.1.0 → dispatch_kit-0.3.0}/tests/test_dispatch.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from dispatch_kit import Lease, is_lease_stale, should_give_up
+from dispatch_kit import Lease, RetriableError, is_lease_stale, should_give_up
 
 
 def test_lease_is_stale_only_past_the_ttl() -> None:
@@ -23,3 +23,13 @@ def test_lease_carries_its_attempt_count() -> None:
     assert fresh.attempts == 0
     recovered = Lease(job_id="j1", leased_at=200.0, attempts=1)
     assert recovered.attempts == 1
+
+
+def test_retriable_error_marks_a_failure_for_retry() -> None:
+    # An executor raises RetriableError to opt a transient failure into re-queue (vs terminal);
+    # it is a plain Exception the engine catches like any other, routing it to fail(retriable=True).
+    assert issubclass(RetriableError, Exception)
+    try:
+        raise RetriableError("transient")
+    except RetriableError as exc:
+        assert str(exc) == "transient"

dispatch_kit-0.3.0/tests/test_engine.py

@@ -0,0 +1,103 @@
+"""The drain engine over a fake in-memory JobStore: complete, terminal/retriable fail, isolation.
+
+The fake store implements just enough of the contract to drive the engine. Its ``fail(retriable)``
+re-queues immediately (no backoff), so a retriable job reaches the poison cap within one ``drain``
+pass — a real store adds a backoff so a transient failure isn't burned to poison instantly.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from dispatch_kit import RetriableError, drain
+from dispatch_kit.dispatch import should_give_up
+
+
+class _FakeStore:
+    def __init__(self, jobs: dict[str, dict[str, Any]]) -> None:
+        self.jobs = jobs  # id -> {payload, lane, status, attempts}
+        self.results: dict[str, Any] = {}
+        self.errors: dict[str, str] = {}
+
+    def claim(self, lanes: Sequence[str]) -> tuple[str, Any] | None:
+        for job_id, job in self.jobs.items():
+            if job["status"] == "pending" and job["lane"] in lanes:
+                job["status"] = "working"
+                return job_id, job["payload"]
+        return None
+
+    def complete(self, job_id: str, result: Any) -> bool:
+        job = self.jobs.get(job_id)
+        if job is None or job["status"] != "working":
+            return False
+        job["status"] = "done"
+        self.results[job_id] = result
+        return True
+
+    def fail(self, job_id: str, error: str, *, retriable: bool, max_attempts: int) -> bool:
+        job = self.jobs.get(job_id)
+        if job is None or job["status"] != "working":
+            return False
+        if retriable:
+            job["attempts"] += 1
+            if should_give_up(job["attempts"], max_attempts):
+                job["status"] = "failed"
+                self.errors[job_id] = error
+            else:
+                job["status"] = "pending"  # re-queue
+        else:
+            job["status"] = "failed"
+            self.errors[job_id] = error
+        return True
+
+    def recover_stale(self, *, now: float, ttl_seconds: float, max_attempts: int) -> Sequence[str]:
+        return []
+
+
+def _job(payload: str, *, lane: str = "ml") -> dict[str, Any]:
+    return {"payload": payload, "lane": lane, "status": "pending", "attempts": 0}
+
+
+def test_drain_runs_every_pending_job_and_completes_it() -> None:
+    store = _FakeStore({"j1": _job("a"), "j2": _job("b")})
+    handled = drain(store, str.upper, ["ml"], max_attempts=3)
+    assert handled == 2
+    assert store.results == {"j1": "A", "j2": "B"}
+    assert all(job["status"] == "done" for job in store.jobs.values())
+
+
+def test_terminal_failure_is_recorded_and_draining_continues() -> None:
+    def execute(payload: str) -> str:
+        if payload == "bad":
+            raise ValueError("nope")
+        return payload
+
+    store = _FakeStore({"j1": _job("bad"), "j2": _job("ok")})
+    drain(store, execute, ["ml"], max_attempts=3)
+    assert store.jobs["j1"]["status"] == "failed"
+    assert "nope" in store.errors["j1"]
+    assert store.results == {"j2": "ok"}  # the good job still ran despite the bad one
+
+
+def test_retriable_failure_requeues_then_poisons_at_the_cap() -> None:
+    def execute(_payload: str) -> str:
+        raise RetriableError("transient")
+
+    store = _FakeStore({"j1": _job("x")})
+    drain(store, execute, ["ml"], max_attempts=2)
+    assert store.jobs["j1"]["status"] == "failed"  # poisoned after the cap
+    assert store.jobs["j1"]["attempts"] == 2
+    assert "transient" in store.errors["j1"]
+
+
+def test_drain_only_touches_its_lanes() -> None:
+    store = _FakeStore({"j1": _job("a", lane="ml"), "j2": _job("b", lane="sync")})
+    handled = drain(store, str.upper, ["ml"], max_attempts=3)
+    assert handled == 1
+    assert store.jobs["j1"]["status"] == "done"
+    assert store.jobs["j2"]["status"] == "pending"  # the other lane is untouched
+
+
+def test_drain_returns_zero_when_nothing_runnable() -> None:
+    assert drain(_FakeStore({}), str.upper, ["ml"], max_attempts=3) == 0

dispatch_kit-0.3.0/tests/test_faults.py

@@ -0,0 +1,26 @@
+"""Contained records an ordinary failure but lets a BaseException through."""
+
+from __future__ import annotations
+
+import pytest
+
+from dispatch_kit import Contained
+
+
+def test_contained_records_an_ordinary_exception() -> None:
+    with Contained() as box:
+        raise ValueError("boom")
+    assert isinstance(box.error, ValueError)
+    assert str(box.error) == "boom"
+
+
+def test_contained_is_clean_when_nothing_raises() -> None:
+    with Contained() as box:
+        pass
+    assert box.error is None
+
+
+def test_contained_lets_base_exceptions_propagate() -> None:
+    # KeyboardInterrupt / SystemExit must NOT be swallowed — shutdown + cancellation still work.
+    with pytest.raises(KeyboardInterrupt), Contained():
+        raise KeyboardInterrupt

dispatch_kit-0.3.0/tests/test_observe.py

@@ -0,0 +1,151 @@
+"""The observability facade is OFF by default and a safe no-op until an env var opts in.
+
+The real SDKs are patched via the ``build_sentry`` / ``build_tracer`` seams, so these tests need
+neither the export extras nor a network endpoint.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+
+import pytest
+
+from dispatch_kit.observe import Observe, ObserveConfig, _JsonFormatter
+
+_CFG = ObserveConfig(
+    service_name="svc",
+    env_prefix="SVC",
+    pip_hint="pip install 'svc[observe]'",
+    tracer_name="svc",
+)
+
+
+@pytest.fixture
+def obs(monkeypatch: pytest.MonkeyPatch) -> Observe:
+    """A fresh facade with every opt-in env var cleared."""
+    for var in ("SVC_SENTRY_DSN", "SENTRY_DSN", "OTEL_EXPORTER_OTLP_ENDPOINT", "SVC_LOG_JSON"):
+        monkeypatch.delenv(var, raising=False)
+    return Observe(_CFG)
+
+
+class _FakeScope:
+    def __init__(self) -> None:
+        self.tags: dict[str, str] = {}
+
+    def __enter__(self) -> _FakeScope:
+        return self
+
+    def __exit__(self, *_exc: object) -> bool:
+        return False
+
+    def set_tag(self, key: str, value: str) -> None:
+        self.tags[key] = value
+
+
+class _FakeSentry:
+    def __init__(self) -> None:
+        self.captured: list[tuple[BaseException, dict[str, str]]] = []
+        self._scope = _FakeScope()
+
+    def new_scope(self) -> _FakeScope:
+        self._scope = _FakeScope()
+        return self._scope
+
+    def capture_exception(self, exc: BaseException) -> None:
+        self.captured.append((exc, dict(self._scope.tags)))
+
+
+class _FakeSpan:
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.attrs: dict[str, object] = {}
+
+    def __enter__(self) -> _FakeSpan:
+        return self
+
+    def __exit__(self, *_exc: object) -> bool:
+        return False
+
+    def set_attribute(self, key: str, value: object) -> None:
+        self.attrs[key] = value
+
+
+class _FakeTracer:
+    def __init__(self) -> None:
+        self.spans: list[_FakeSpan] = []
+
+    def start_as_current_span(self, name: str) -> _FakeSpan:
+        span = _FakeSpan(name)
+        self.spans.append(span)
+        return span
+
+
+def test_inactive_by_default_is_a_safe_noop(obs: Observe) -> None:
+    obs.setup()
+    assert obs.is_active() == {"sentry": False, "otel": False}
+    with obs.span("op", tag="t") as active:
+        assert active is None
+    obs.capture_exception(ValueError("x"), op="t")  # must not raise
+
+
+def test_sentry_opt_in_forwards_exception_with_tags(
+    obs: Observe, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    fake = _FakeSentry()
+    monkeypatch.setenv("SVC_SENTRY_DSN", "https://k@example.test/1")
+    monkeypatch.setattr(obs, "build_sentry", lambda _dsn: fake)
+    obs.setup()
+    assert obs.is_active()["sentry"] is True
+    err = ValueError("boom")
+    obs.capture_exception(err, op="x", id="r1")
+    assert fake.captured == [(err, {"op": "x", "id": "r1"})]
+
+
+def test_otel_opt_in_records_spans(obs: Observe, monkeypatch: pytest.MonkeyPatch) -> None:
+    tracer = _FakeTracer()
+    monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
+    monkeypatch.setattr(obs, "build_tracer", lambda _cmd: tracer)
+    obs.setup(command="work")
+    assert obs.is_active()["otel"] is True
+    with obs.span("job", op="x") as active:
+        assert active is not None
+    assert tracer.spans[0].name == "job"
+    assert tracer.spans[0].attrs == {"op": "x"}
+
+
+def test_missing_extra_logs_and_degrades(
+    obs: Observe, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    def _no_sdk(_dsn: str) -> object:
+        raise ImportError("no sentry_sdk")
+
+    monkeypatch.setenv("SVC_SENTRY_DSN", "https://k@example.test/1")
+    monkeypatch.setattr(obs, "build_sentry", _no_sdk)
+    with caplog.at_level(logging.WARNING):
+        obs.setup()
+    assert obs.is_active()["sentry"] is False
+    assert any("sentry-sdk" in rec.message for rec in caplog.records)
+
+
+def test_setup_is_idempotent_and_resets(obs: Observe, monkeypatch: pytest.MonkeyPatch) -> None:
+    fake = _FakeSentry()
+    monkeypatch.setenv("SVC_SENTRY_DSN", "https://k@example.test/1")
+    monkeypatch.setattr(obs, "build_sentry", lambda _dsn: fake)
+    obs.setup()
+    assert obs.is_active()["sentry"] is True
+    monkeypatch.delenv("SVC_SENTRY_DSN")
+    obs.setup()
+    assert obs.is_active()["sentry"] is False
+
+
+def test_json_formatter_emits_structured_record() -> None:
+    formatter = _JsonFormatter("svc", "work")
+    record = logging.LogRecord("svc.x", logging.INFO, "f.py", 1, "hello %s", ("world",), None)
+    payload = json.loads(formatter.format(record))
+    assert payload["service"] == "svc"
+    assert payload["command"] == "work"
+    assert payload["level"] == "INFO"
+    assert payload["logger"] == "svc.x"
+    assert payload["msg"] == "hello world"
+    assert "ts" in payload

dispatch_kit-0.3.0/tests/test_provenance.py

@@ -0,0 +1,61 @@
+"""Provenance is a reproducibility record whose invalid states are unrepresentable."""
+
+from __future__ import annotations
+
+import pytest
+
+from dispatch_kit import BackendKind, Determinism, GpuContext, Provenance
+
+
+def _prov(
+    *,
+    tool_id: str = "tool",
+    params_hash: str = "p",
+    runtime_seconds: float = 1.0,
+    determinism: Determinism = Determinism.DETERMINISTIC,
+    seed: int | None = None,
+) -> Provenance:
+    return Provenance(
+        tool_id=tool_id,
+        tool_version="1.0",
+        weights_id=None,
+        weights_sha256=None,
+        params_hash=params_hash,
+        input_hash="i",
+        reference_data_versions={},
+        container_digest=None,
+        seed=seed,
+        determinism=determinism,
+        runtime_seconds=runtime_seconds,
+    )
+
+
+def test_a_valid_deterministic_provenance_constructs() -> None:
+    prov = _prov()
+    assert prov.backend is BackendKind.LOCAL  # ran here unless a remote dispatch says otherwise
+    assert prov.gpu is None  # CPU-only by default
+
+
+def test_missing_tool_or_hashes_are_rejected() -> None:
+    with pytest.raises(ValueError, match="tool_id"):
+        _prov(tool_id="")
+    with pytest.raises(ValueError, match="params_hash"):
+        _prov(params_hash="")
+
+
+def test_negative_runtime_is_rejected() -> None:
+    with pytest.raises(ValueError, match="runtime_seconds"):
+        _prov(runtime_seconds=-1.0)
+
+
+def test_seed_determinism_contradictions_are_rejected() -> None:
+    with pytest.raises(ValueError, match="must record its seed"):
+        _prov(determinism=Determinism.SEEDED, seed=None)
+    with pytest.raises(ValueError, match="must not carry a seed"):
+        _prov(determinism=Determinism.NONDETERMINISTIC, seed=42)
+
+
+def test_gpu_vram_must_be_positive() -> None:
+    with pytest.raises(ValueError, match="vram_gb"):
+        GpuContext(model="L4", vram_gb=0.0)
+    assert GpuContext(model="L4", vram_gb=24.0).vram_gb == 24.0