obzerve 0.1.0__py3-none-any.whl

obzerve/__init__.py

@@ -0,0 +1,7 @@
+"""Obzerve Python SDK: trace AI-agent workflows with OpenTelemetry."""
+
+from obzerve.attributes import SpanKind
+from obzerve.delivery import DeliveryStats
+from obzerve.observer import Observer, ObzerveSpan, __version__
+
+__all__ = ["DeliveryStats", "Observer", "ObzerveSpan", "SpanKind", "__version__"]

obzerve/_version.py

@@ -0,0 +1,3 @@
+"""Obzerve SDK version (single source of truth for the package metadata)."""
+
+__version__ = "0.1.0"

obzerve/attributes.py

@@ -0,0 +1,61 @@
+"""Span attribute keys shared by the SDK and the Obzerve server.
+
+AI-specific keys follow the OpenTelemetry GenAI semantic conventions where one
+exists; Obzerve-specific keys live under the ``obzerve.`` namespace.
+"""
+
+# Obzerve
+SPAN_KIND = "obzerve.span.kind"
+RETRY_COUNT = "obzerve.retry.count"
+RETRY_ATTEMPT = "obzerve.retry.attempt"
+INPUT = "obzerve.input"
+OUTPUT = "obzerve.output"
+INPUT_TRUNCATED = "obzerve.input.truncated"
+OUTPUT_TRUNCATED = "obzerve.output.truncated"
+INPUT_REDACTED = "obzerve.input.redacted"
+SDK_VERSION = "obzerve.sdk.version"
+
+# Replay: recording (set on replayable operations)
+REPLAY_KEY = "obzerve.replay.key"  # "<name>#<occurrence>" within the trace
+REPLAY_INPUT_HASH = "obzerve.replay.input_hash"
+REPLAY_COMPLETE = "obzerve.replay.complete"  # the recorded output round-trips exactly
+REPLAY_INCOMPLETE_REASON = "obzerve.replay.incomplete_reason"
+REPLAY_SIDE_EFFECT = "obzerve.replay.side_effect"
+REPLAY_ERROR_CLASS = "obzerve.replay.error_class"  # "module:QualName" of a raised exception
+# Replay: replaying
+REPLAY_SOURCE = "obzerve.replay.source"  # "live" or "recorded"
+REPLAY_INPUT_MATCHED = "obzerve.replay.input_matched"
+REPLAY_OF = "obzerve.replay.of"  # on the root span: the trace being replayed
+REPLAY_MODE = "obzerve.replay.mode"
+REPLAY_LABEL = "obzerve.replay.label"
+
+# Versioned configuration and evaluation (root span)
+PROMPT_NAME = "obzerve.prompt.name"
+PROMPT_VERSION = "obzerve.prompt.version"
+EVAL_RUN_ID = "obzerve.eval.run_id"
+EVAL_CASE_ID = "obzerve.eval.case_id"
+
+# OpenTelemetry GenAI semantic conventions
+GEN_AI_PROVIDER = "gen_ai.provider.name"
+GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
+GEN_AI_INPUT_TOKENS = "gen_ai.usage.input_tokens"
+GEN_AI_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
+# Input tokens served from the provider's prompt cache (a subset of input tokens).
+GEN_AI_CACHED_INPUT_TOKENS = "gen_ai.usage.cache_read.input_tokens"
+
+# OpenTelemetry general conventions
+ERROR_TYPE = "error.type"
+SERVICE_NAME = "service.name"
+
+
+class SpanKind:
+    """Logical operation types understood by the dashboard."""
+
+    AGENT = "agent"
+    LLM = "llm"
+    TOOL = "tool"
+    RETRIEVAL = "retrieval"
+    CHAIN = "chain"
+    HANDOFF = "handoff"
+    FUNCTION = "function"

obzerve/cli.py

@@ -0,0 +1,188 @@
+"""``obzerve`` command line: turn traces into regression cases, compare eval runs,
+and gate releases in CI (exit code 1 when the gate fails).
+
+Reads ``OBZERVE_ENDPOINT`` and ``OBZERVE_API_KEY`` from the environment.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from obzerve.client import ObzerveAPIError, ObzerveClient
+
+
+def _client(args: argparse.Namespace) -> ObzerveClient:
+    return ObzerveClient(args.endpoint, args.api_key)
+
+
+def _pct(value: float | None) -> str:
+    return "–" if value is None else f"{value:+.1f}%"
+
+
+def _money(value: float | None) -> str:
+    return "–" if value is None else f"${value:.6f}"
+
+
+def cmd_case_add(args: argparse.Namespace) -> int:
+    assertions: list[dict] = []
+    if args.no_error:
+        assertions.append({"type": "no_error"})
+    if args.contains_all:
+        assertions.append({"type": "contains_all", "values": args.contains_all})
+    if args.contains_any:
+        assertions.append({"type": "contains_any", "values": args.contains_any})
+    if args.not_contains:
+        assertions.append({"type": "not_contains", "values": args.not_contains})
+    if args.regex:
+        assertions.append({"type": "regex", "pattern": args.regex})
+    if args.max_latency_ms:
+        assertions.append({"type": "max_latency_ms", "value": args.max_latency_ms})
+    for raw in args.assertion or []:
+        assertions.append(json.loads(raw))
+    if not assertions:
+        print("add at least one assertion (e.g. --no-error)", file=sys.stderr)
+        return 2
+    case = _client(args).create_case(
+        args.dataset, args.trace_id, args.name, assertions, args.notes, args.mode
+    )
+    print(
+        f"case {case['id']} added to {args.dataset!r} ({case['replayability']}, "
+        f"{case['recordings']} recorded operations, replayed {case['replay_policy']})"
+    )
+    for reason in case["replay_reasons"]:
+        print(f"  note: {reason}")
+    return 0
+
+
+def _print_comparison(cmp: dict) -> None:
+    b, c = cmp["baseline"], cmp["candidate"]
+    rows = [
+        ("label", b["label"], c["label"]),
+        ("prompt", _prompt(b), _prompt(c)),
+        (
+            "pass rate",
+            f"{b['pass_rate']:.0%} ({b['passed']}/{b['cases']})",
+            f"{c['pass_rate']:.0%} ({c['passed']}/{c['cases']})",
+        ),
+        (
+            "est. cost / case",
+            _money(b["estimated_cost_per_case_usd"]),
+            _money(c["estimated_cost_per_case_usd"]),
+        ),
+        (
+            "output tokens / case",
+            _num(b["output_tokens_per_case"]),
+            _num(c["output_tokens_per_case"]),
+        ),
+        ("p95 latency", _ms(b["latency_p95_ms"]), _ms(c["latency_p95_ms"])),
+    ]
+    width = max(len(r[1]) for r in rows) + 2
+    print(f"{'':22}{'baseline':{width}}candidate")
+    for name, x, y in rows:
+        print(f"{name:22}{x:{width}}{y}")
+    print(
+        f"\nregressions: {cmp['regressions']}   fixes: {cmp['fixes']}   "
+        f"cost/case {_pct(cmp['cost_per_case_delta_pct'])}   "
+        f"p95 {_pct(cmp['latency_p95_delta_pct'])}"
+    )
+    if not cmp["comparable"]:
+        print("NOT COMPARABLE: " + "; ".join(cmp["reasons"]))
+    for case in cmp["cases"]:
+        if case["change"] in ("regression", "fix", "still_failing"):
+            why = f" — {'; '.join(case['failed_assertions'])}" if case["failed_assertions"] else ""
+            print(f"  {case['change']:13} {case['case_name']}{why}")
+
+
+def _prompt(run: dict) -> str:
+    p = run.get("prompt")
+    return f"{p['name']} v{p['version']}" if p else "–"
+
+
+def _num(v: float | None) -> str:
+    return "–" if v is None else f"{v:.0f}"
+
+
+def _ms(v: float | None) -> str:
+    return "–" if v is None else f"{v:.0f} ms"
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+    _print_comparison(_client(args).compare_runs(args.baseline, args.candidate))
+    return 0
+
+
+def cmd_gate(args: argparse.Namespace) -> int:
+    result = _client(args).gate(
+        args.baseline,
+        args.candidate,
+        max_regressions=args.max_regressions,
+        min_pass_rate=args.min_pass_rate,
+        max_pass_rate_drop=args.max_pass_rate_drop,
+        max_cost_increase_pct=args.max_cost_increase_pct,
+        max_p95_latency_increase_pct=args.max_p95_latency_increase_pct,
+    )
+    _print_comparison(result["comparison"])
+    print()
+    for check in result["checks"]:
+        print(f"  [{'PASS' if check['passed'] else 'FAIL'}] {check['name']}: {check['detail']}")
+    print("\nGATE " + ("PASSED" if result["passed"] else "FAILED"))
+    return 0 if result["passed"] else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="obzerve", description=__doc__.split("\n\n")[0])
+    parser.add_argument(
+        "--endpoint", default=os.environ.get("OBZERVE_ENDPOINT", "http://localhost:8000")
+    )
+    parser.add_argument("--api-key", default=os.environ.get("OBZERVE_API_KEY"))
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    case = sub.add_parser("case", help="Regression cases").add_subparsers(dest="sub", required=True)
+    add = case.add_parser("add", help="Turn a trace into a regression case")
+    add.add_argument("trace_id")
+    add.add_argument("--dataset", required=True)
+    add.add_argument("--name", required=True)
+    add.add_argument("--notes")
+    add.add_argument(
+        "--mode",
+        choices=["recorded", "llm-live", "live"],
+        default="llm-live",
+        help="recorded: re-run the recorded failure exactly; llm-live: model live (default)",
+    )
+    add.add_argument("--no-error", action="store_true", help="the run must not raise")
+    add.add_argument("--contains-all", nargs="+", metavar="TERM")
+    add.add_argument("--contains-any", nargs="+", metavar="TERM")
+    add.add_argument("--not-contains", nargs="+", metavar="TERM")
+    add.add_argument("--regex", help="output must match (case-insensitive)")
+    add.add_argument("--max-latency-ms", type=float)
+    add.add_argument("--assertion", action="append", metavar="JSON", help="raw assertion spec")
+    add.set_defaults(fn=cmd_case_add)
+
+    ev = sub.add_parser("eval", help="Eval runs").add_subparsers(dest="sub", required=True)
+    compare = ev.add_parser("compare", help="Compare a candidate run with a baseline run")
+    compare.add_argument("baseline")
+    compare.add_argument("candidate")
+    compare.set_defaults(fn=cmd_compare)
+    gate = ev.add_parser("gate", help="Fail (exit 1) if the candidate does not meet the criteria")
+    gate.add_argument("baseline")
+    gate.add_argument("candidate")
+    gate.add_argument("--max-regressions", type=int, default=0)
+    gate.add_argument("--min-pass-rate", type=float)
+    gate.add_argument("--max-pass-rate-drop", type=float)
+    gate.add_argument("--max-cost-increase-pct", type=float)
+    gate.add_argument("--max-p95-latency-increase-pct", type=float)
+    gate.set_defaults(fn=cmd_gate)
+
+    args = parser.parse_args(argv)
+    try:
+        return args.fn(args)
+    except ObzerveAPIError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

obzerve/client.py

@@ -0,0 +1,98 @@
+"""HTTP client for Obzerve's API: replay bundles, prompt versions, eval datasets and runs."""
+
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import quote
+
+import requests
+
+from obzerve.replay import ReplayBundle
+
+
+class ObzerveAPIError(Exception):
+    def __init__(self, status: int, detail: Any):
+        super().__init__(f"HTTP {status}: {detail}")
+        self.status = status
+        self.detail = detail
+
+
+class ObzerveClient:
+    def __init__(
+        self,
+        endpoint: str = "http://localhost:8000",
+        api_key: str | None = None,
+        timeout_s: float = 30.0,
+    ):
+        self.endpoint = endpoint.rstrip("/")
+        self.timeout_s = timeout_s
+        self._session = requests.Session()
+        if api_key:
+            self._session.headers["Authorization"] = f"Bearer {api_key}"
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> Any:
+        resp = self._session.request(
+            method, f"{self.endpoint}{path}", timeout=self.timeout_s, **kwargs
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("detail", resp.text)
+            except ValueError:
+                detail = resp.text
+            raise ObzerveAPIError(resp.status_code, detail)
+        return resp.json() if resp.content else None
+
+    # Replay
+    def replay_bundle(self, trace_id: str) -> ReplayBundle:
+        return ReplayBundle.from_json(self._request("GET", f"/api/v1/traces/{trace_id}/replay"))
+
+    def compare_traces(self, a: str, b: str) -> dict[str, Any]:
+        return self._request("GET", "/api/v1/trace-comparison", params={"a": a, "b": b})
+
+    # Prompt versions
+    def register_prompt(self, name: str, config: dict[str, Any]) -> dict[str, Any]:
+        """Register a prompt/model configuration; identical content returns the same version."""
+        return self._request(
+            "POST", f"/api/v1/prompts/{quote(name, safe='')}/versions", json={"config": config}
+        )
+
+    # Datasets and cases
+    def dataset(
+        self, name: str, *, version: int | None = None, include_bundles: bool = False
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"include_bundles": str(include_bundles).lower()}
+        if version is not None:
+            params["version"] = version
+        return self._request("GET", f"/api/v1/datasets/{quote(name, safe='')}", params=params)
+
+    def create_case(
+        self,
+        dataset: str,
+        trace_id: str,
+        name: str,
+        assertions: list[dict[str, Any]],
+        notes: str | None = None,
+        replay_policy: str = "llm-live",
+    ) -> dict[str, Any]:
+        return self._request(
+            "POST",
+            f"/api/v1/datasets/{quote(dataset, safe='')}/cases",
+            json={
+                "trace_id": trace_id,
+                "name": name,
+                "assertions": assertions,
+                "notes": notes,
+                "replay_policy": replay_policy,
+            },
+        )
+
+    # Eval runs
+    def submit_run(self, run: dict[str, Any]) -> dict[str, Any]:
+        return self._request("POST", "/api/v1/eval-runs", json=run)
+
+    def compare_runs(self, baseline: str, candidate: str) -> dict[str, Any]:
+        return self._request("GET", f"/api/v1/eval-runs/compare/{baseline}/{candidate}")
+
+    def gate(self, baseline: str, candidate: str, **criteria: Any) -> dict[str, Any]:
+        params = {k: v for k, v in criteria.items() if v is not None}
+        return self._request("GET", f"/api/v1/eval-runs/gate/{baseline}/{candidate}", params=params)

obzerve/delivery.py

@@ -0,0 +1,139 @@
+"""Bounded, accounted span delivery.
+
+Wraps OpenTelemetry's ``BatchSpanProcessor`` and exporter so that every ended
+span is counted as exactly one of: exported, failed (the backend rejected it
+or was unreachable after the exporter's retries), dropped (the buffer was
+full), or still pending. The buffer bound is enforced here, before the batch
+processor's own queue, so drops are counted precisely.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from opentelemetry.context import Context
+from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
+
+logger = logging.getLogger("obzerve")
+
+_WARN_INTERVAL_S = 60.0
+
+
+@dataclass(frozen=True)
+class DeliveryStats:
+    """Telemetry delivery counters since the Observer was created."""
+
+    spans_ended: int
+    spans_exported: int
+    spans_failed: int
+    spans_dropped: int
+    spans_pending: int
+
+
+class _Counters:
+    def __init__(self) -> None:
+        self.lock = threading.Lock()
+        self.ended = 0
+        self.exported = 0
+        self.failed = 0
+        self.dropped = 0
+        self.pending = 0
+        self._last_warning = 0.0
+
+    def warn(self, message: str, *args: object) -> None:
+        """Log at most once per interval, so a dead backend cannot flood the agent's logs."""
+        now = time.monotonic()
+        if now - self._last_warning >= _WARN_INTERVAL_S:
+            self._last_warning = now
+            logger.warning(message, *args)
+
+
+class AccountingExporter(SpanExporter):
+    def __init__(self, inner: SpanExporter, counters: _Counters):
+        self._inner = inner
+        self._counters = counters
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        try:
+            result = self._inner.export(spans)
+        except Exception:  # an exporter bug must not kill the export thread
+            logger.exception("Obzerve exporter raised")
+            result = SpanExportResult.FAILURE
+        c = self._counters
+        with c.lock:
+            c.pending -= len(spans)
+            if result is SpanExportResult.SUCCESS:
+                c.exported += len(spans)
+            else:
+                c.failed += len(spans)
+                failed = c.failed
+        if result is not SpanExportResult.SUCCESS:
+            c.warn("Obzerve could not deliver %d spans (%d failed in total)", len(spans), failed)
+        return result
+
+    def shutdown(self) -> None:
+        self._inner.shutdown()
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        return self._inner.force_flush(timeout_millis)
+
+
+class AccountingProcessor(SpanProcessor):
+    """Admits a span to the batch processor only while fewer than
+    ``max_pending`` spans are buffered or being exported."""
+
+    def __init__(
+        self,
+        exporter: SpanExporter,
+        *,
+        max_queue_size: int,
+        max_export_batch_size: int,
+        schedule_delay_ms: int,
+        export_timeout_s: float,
+    ):
+        self.counters = _Counters()
+        self._max_pending = max_queue_size
+        # The inner queue holds at most ``max_pending`` spans, so it never drops on its own.
+        self._batch = BatchSpanProcessor(
+            AccountingExporter(exporter, self.counters),
+            max_queue_size=max_queue_size,
+            max_export_batch_size=min(max_export_batch_size, max_queue_size),
+            schedule_delay_millis=schedule_delay_ms,
+            export_timeout_millis=export_timeout_s * 1000,
+        )
+
+    def on_start(self, span: Span, parent_context: Context | None = None) -> None:
+        self._batch.on_start(span, parent_context)
+
+    def on_end(self, span: ReadableSpan) -> None:
+        if not span.context.trace_flags.sampled:
+            return
+        c = self.counters
+        with c.lock:
+            c.ended += 1
+            admitted = c.pending < self._max_pending
+            if admitted:
+                c.pending += 1
+            else:
+                c.dropped += 1
+                dropped = c.dropped
+        if admitted:
+            self._batch.on_end(span)
+        else:
+            c.warn("Obzerve telemetry buffer full; dropping spans (%d dropped in total)", dropped)
+
+    def stats(self) -> DeliveryStats:
+        c = self.counters
+        with c.lock:
+            return DeliveryStats(c.ended, c.exported, c.failed, c.dropped, c.pending)
+
+    def shutdown(self) -> None:
+        self._batch.shutdown()
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        return self._batch.force_flush(timeout_millis)

obzerve/evals.py

@@ -0,0 +1,156 @@
+"""Evaluation runs: replay every case of a dataset against a candidate and
+check its assertions.
+
+Each case carries the recorded input and dependencies of the trace it was
+made from, and its own replay policy: ``recorded`` to re-run a recorded failure
+exactly, or ``llm-live`` to test the model/prompt against recorded tools and
+retrieval. ``run_eval`` replays each case, applies its deterministic
+assertions, and submits the results with everything needed to judge whether
+two runs are comparable: dataset version, evaluator version, prompt version,
+replay policy, and configuration.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+from obzerve import attributes as attrs
+from obzerve.client import ObzerveClient
+from obzerve.observer import Observer
+from obzerve.replay import POLICIES, ReplayBundle, ReplayPolicy
+
+# Bump whenever an assertion's semantics change: runs with different evaluator
+# versions are not compared.
+EVALUATOR_VERSION = "obzerve-assertions/1"
+
+
+@dataclass
+class Outcome:
+    output: str | None
+    error: str | None
+    latency_ms: float
+
+
+def output_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return value
+    try:
+        return json.dumps(value, ensure_ascii=False, default=repr)
+    except (TypeError, ValueError):
+        return repr(value)
+
+
+def evaluate(assertion: dict[str, Any], outcome: Outcome) -> tuple[bool, str]:
+    """Apply one assertion; returns ``(passed, explanation)``."""
+    kind = assertion["type"]
+    text = (outcome.output or "").lower()
+    if kind == "no_error":
+        return outcome.error is None, "completed without error" if outcome.error is None else (
+            f"raised {outcome.error}"
+        )
+    if kind == "max_latency_ms":
+        limit = assertion["value"]
+        return outcome.latency_ms <= limit, f"took {outcome.latency_ms:.0f} ms (limit {limit:g})"
+    if outcome.error is not None:
+        return False, f"no output: raised {outcome.error}"
+    if kind == "contains_all":
+        missing = [v for v in assertion["values"] if v.lower() not in text]
+        return not missing, "contains all terms" if not missing else f"missing {missing}"
+    if kind == "contains_any":
+        found = [v for v in assertion["values"] if v.lower() in text]
+        need = assertion.get("min_matches", 1)
+        return len(found) >= need, f"found {len(found)} of required {need}: {found}"
+    if kind == "not_contains":
+        present = [v for v in assertion["values"] if v.lower() in text]
+        return not present, "none present" if not present else f"contains {present}"
+    if kind == "regex":
+        matched = re.search(assertion["pattern"], outcome.output or "", re.IGNORECASE) is not None
+        want = assertion.get("must_match", True)
+        return matched == want, f"pattern {'matched' if matched else 'did not match'}"
+    if kind == "max_output_chars":
+        n, limit = len(outcome.output or ""), assertion["value"]
+        return n <= limit, f"{n} chars (limit {limit})"
+    return False, f"unknown assertion type {kind!r} for {EVALUATOR_VERSION}"
+
+
+def run_eval(
+    observer: Observer,
+    client: ObzerveClient,
+    dataset: str,
+    target: Callable[[Any], Any],
+    *,
+    label: str,
+    policy: ReplayPolicy | None = None,
+    prompt_version: dict[str, Any] | None = None,
+    config: dict[str, Any] | None = None,
+    on_case: Callable[[dict[str, Any], dict[str, Any]], None] | None = None,
+) -> dict[str, Any]:
+    """Run every active case of ``dataset`` through ``target(root_input)`` and
+    submit the results. ``policy`` overrides every case's own replay policy
+    (default: per case). ``prompt_version`` is the dict returned by
+    ``client.register_prompt``. Returns the stored run (with its summary)."""
+    data = client.dataset(dataset, include_bundles=True)
+    if not data["cases"]:
+        raise ValueError(f"dataset {dataset!r} has no active cases")
+    started = datetime.now(timezone.utc)
+    run_id = str(uuid.uuid4())
+    replay_attrs: dict[str, Any] = {attrs.EVAL_RUN_ID: run_id}
+    if prompt_version:
+        replay_attrs[attrs.PROMPT_NAME] = prompt_version["name"]
+        replay_attrs[attrs.PROMPT_VERSION] = str(prompt_version["version"])
+
+    results = []
+    for case in data["cases"]:
+        bundle = ReplayBundle.from_json(case["bundle"])
+        case_attrs = {**replay_attrs, attrs.EVAL_CASE_ID: case["id"]}
+        t0 = time.perf_counter()
+        output = error = None
+        case_policy = policy or POLICIES[case.get("replay_policy", "llm-live")]
+        with observer.replay(bundle, case_policy, label=label, attributes=case_attrs) as session:
+            try:
+                output = output_text(target(bundle.root_input))
+            except Exception as exc:  # a failing case is a result, not a crash
+                error = f"{type(exc).__name__}: {exc}"
+        outcome = Outcome(output, error, (time.perf_counter() - t0) * 1000)
+        verdicts = []
+        for assertion in case["assertions"]:
+            passed, detail = evaluate(assertion, outcome)
+            verdicts.append({"type": assertion["type"], "passed": passed, "detail": detail})
+        result = {
+            "case_id": case["id"],
+            "trace_id": session.root_trace_id,
+            "passed": all(v["passed"] for v in verdicts),
+            "assertions": verdicts,
+            "output": (output or "")[:200_000] or None,
+            "error": error,
+            "latency_ms": outcome.latency_ms,
+        }
+        results.append(result)
+        if on_case:
+            on_case(case, result)
+
+    observer.flush()
+    return client.submit_run(
+        {
+            "id": run_id,
+            "dataset": dataset,
+            "dataset_version": data["version"],
+            "evaluator_version": EVALUATOR_VERSION,
+            "label": label,
+            "replay_policy": policy.name if policy else "per-case",
+            "prompt_version_id": prompt_version["id"] if prompt_version else None,
+            "config": config or {},
+            "started_at": started.isoformat(),
+            "finished_at": datetime.now(timezone.utc).isoformat(),
+            "results": results,
+        }
+    )