tracefork 0.1.0__py3-none-any.whl

tracefork/transport.py

@@ -0,0 +1,137 @@
+"""Recording/replay httpx transports — sync and async, streaming SSE capable.
+
+Record mode: forward to the inner transport, buffer the full response body
+(works for both streaming SSE and non-streaming JSON — httpx buffers both
+identically via .read()/.aread()), append to the tape, return the response.
+
+Replay mode: for each request, sha256-assert its *body* matches the tape record,
+then serve the recorded bytes back. A replay transport has no inner transport;
+any unrecorded request is a hard error. The matched surface is the request body;
+request headers (e.g. ``anthropic-beta`` / ``anthropic-version``) are out of scope
+for the bit-exactness claim — see the README's determinism-boundary note.
+"""
+
+from __future__ import annotations
+
+import httpx
+
+from .nondet import DivergenceError
+from .tape import Tape, sha256_hex
+
+
+class TraceforkTransport(httpx.BaseTransport):
+    """Sync recording/replay transport."""
+
+    def __init__(
+        self,
+        mode: str,
+        tape: Tape,
+        inner: httpx.BaseTransport | None = None,
+    ) -> None:
+        assert mode in ("record", "replay")
+        if mode == "record" and inner is None:
+            raise ValueError("record mode requires an inner transport")
+        self.mode = mode
+        self.tape = tape
+        self.inner = inner
+        self._i = 0
+        self.matched = 0
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        body = request.content
+
+        if self.mode == "record":
+            inner_resp = self.inner.handle_request(request)  # type: ignore[union-attr]
+            resp_body = inner_resp.read()
+            self.tape.append_exchange(body, resp_body)
+            return httpx.Response(
+                inner_resp.status_code,
+                headers={
+                    "content-type": inner_resp.headers.get("content-type", "application/json")
+                },
+                content=resp_body,
+                request=request,
+            )
+
+        # replay
+        if self._i >= len(self.tape.exchanges):
+            raise DivergenceError(
+                f"replay made unrecorded request #{self._i} "
+                f"(tape has {len(self.tape.exchanges)} exchanges)"
+            )
+        rec_req, rec_resp = self.tape.exchange(self._i)
+        if sha256_hex(rec_req) != sha256_hex(body):
+            raise DivergenceError(
+                f"request #{self._i} diverged from tape "
+                f"(recorded {sha256_hex(rec_req)[:12]}, replay {sha256_hex(body)[:12]})"
+            )
+        self._i += 1
+        self.matched += 1
+        return httpx.Response(
+            200,
+            headers={"content-type": "application/json"},
+            content=rec_resp,
+            request=request,
+        )
+
+    def fully_consumed(self) -> bool:
+        return self.mode == "replay" and self._i == len(self.tape.exchanges)
+
+
+class AsyncTraceforkTransport(httpx.AsyncBaseTransport):
+    """Async recording/replay transport — identical logic to sync variant."""
+
+    def __init__(
+        self,
+        mode: str,
+        tape: Tape,
+        inner: httpx.AsyncBaseTransport | None = None,
+    ) -> None:
+        assert mode in ("record", "replay")
+        if mode == "record" and inner is None:
+            raise ValueError("record mode requires an inner transport")
+        self.mode = mode
+        self.tape = tape
+        self.inner = inner
+        self._i = 0
+        self.matched = 0
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        body = request.content
+
+        if self.mode == "record":
+            inner_resp = await self.inner.handle_async_request(request)  # type: ignore[union-attr]
+            resp_body = await inner_resp.aread()
+            self.tape.append_exchange(body, resp_body)
+            return httpx.Response(
+                inner_resp.status_code,
+                headers={
+                    "content-type": inner_resp.headers.get("content-type", "application/json")
+                },
+                content=resp_body,
+                request=request,
+            )
+
+        # replay
+        if self._i >= len(self.tape.exchanges):
+            raise DivergenceError(
+                f"replay made unrecorded request #{self._i} "
+                f"(tape has {len(self.tape.exchanges)} exchanges)"
+            )
+        rec_req, rec_resp = self.tape.exchange(self._i)
+        if sha256_hex(rec_req) != sha256_hex(body):
+            raise DivergenceError(
+                f"request #{self._i} diverged from tape "
+                f"(recorded {sha256_hex(rec_req)[:12]}, replay {sha256_hex(body)[:12]})"
+            )
+        self._i += 1
+        self.matched += 1
+        return httpx.Response(
+            200,
+            headers={"content-type": "application/json"},
+            content=rec_resp,
+            request=request,
+        )
+
+    def fully_consumed(self) -> bool:
+        return self.mode == "replay" and self._i == len(self.tape.exchanges)

tracefork/validate.py

@@ -0,0 +1,177 @@
+"""Self-validation: run the blame engine on fault-injected runs with known
+ground truth and measure how often it fingers the right step.
+
+Fully offline and $0. Each run:
+  1. record a clean two-step tape with a synthetic agent;
+  2. inject a known fault into step 0 (the "root cause");
+  3. run the blame engine — forking re-runs the synthetic agent, which echoes
+     each response into its next request, so the fault marker reaches the
+     fault-aware tail and flips the outcome;
+  4. score a hit when blame ranks the fault step #1 (top-1 precision).
+
+A negative control runs blame with a no-op perturbation and asserts the
+flip-rate stays near zero — otherwise a high "precision" would be meaningless.
+
+The synthetic agent is the same callable during recording and every fork, so
+the fork prefix replays bit-for-bit (the determinism contract blame relies on).
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+
+import anthropic
+import httpx
+
+from .blame import BlameEngine, StringMatchOracle
+from .faults import FAULT_MARKER_BYTES, FaultClass, FaultInjector
+from .synthetic import FaultAwareFakeLLM, ScriptedFakeLLM
+from .tape import Tape
+from .transport import TraceforkTransport
+from .wire import make_text_response, make_tool_use_response
+
+SUCCESS_RESP = make_text_response("SUCCESS — confirmed")
+FAIL_RESP = make_text_response("FAIL — cancelled")
+TOOL_RESP = make_tool_use_response("check_availability", {"seats": 3, "destination": "Tokyo"})
+
+
+def _serialize_response(msg) -> str:
+    """Flatten an Anthropic message's content to a deterministic string, so the
+    agent can echo it (markers and all) into its next request."""
+    parts: list[str] = []
+    for block in msg.content:
+        t = getattr(block, "type", None)
+        if t == "text":
+            parts.append(block.text)
+        elif t == "tool_use":
+            parts.append(f"{block.name} {json.dumps(block.input, sort_keys=True)}")
+    return " | ".join(parts) or "(empty)"
+
+
+def synthetic_agent(client: anthropic.Anthropic) -> str:
+    """Two-turn agent: ask, then confirm — echoing turn 1's response into the
+    turn-2 request so an injected fault propagates to the outcome."""
+    r1 = client.messages.create(
+        model="claude-sonnet-4-6",
+        max_tokens=100,
+        messages=[{"role": "user", "content": "book a flight to Tokyo"}],
+    )
+    echoed = _serialize_response(r1)
+    r2 = client.messages.create(
+        model="claude-sonnet-4-6",
+        max_tokens=100,
+        messages=[
+            {"role": "user", "content": "book a flight to Tokyo"},
+            {"role": "assistant", "content": echoed},
+            {"role": "user", "content": "confirm"},
+        ],
+    )
+    return _serialize_response(r2)
+
+
+def _record_clean_tape() -> Tape:
+    fake = ScriptedFakeLLM([TOOL_RESP, SUCCESS_RESP])
+    tape = Tape(agent_name="synthetic_booking_agent")
+    transport = TraceforkTransport("record", tape, fake)
+    client = anthropic.Anthropic(
+        api_key="sk-ant-fake",
+        http_client=httpx.Client(transport=transport),
+        max_retries=0,
+    )
+    synthetic_agent(client)
+    return tape
+
+
+@dataclass
+class ValidationReport:
+    fault_class: FaultClass
+    n_runs: int
+    top1_correct: int
+    top1_precision: float
+    negative_control_max_flip: float = 0.0
+
+
+class ValidationRunner:
+    """Runs offline fault-injection validation for a single fault class."""
+
+    def __init__(self, fault_class: FaultClass, *, k: int = 3, n_runs: int = 5) -> None:
+        self._fault_class = fault_class
+        self._k = k
+        self._n_runs = n_runs
+
+    def run(self) -> ValidationReport:
+        oracle = StringMatchOracle(success_re=r"SUCCESS", failure_re=r"FAIL")
+        fault_step = 0
+        top1_correct = 0
+        max_flip_control = 0.0
+
+        for _run in range(self._n_runs):
+            tape = _record_clean_tape()
+            mutated_resp = FaultInjector.inject(tape, fault_step, self._fault_class)
+
+            # Scope note: this is a positive-vs-inert control — the faulted step gets a
+            # flip-capable tail, every other step an inert one. It proves the engine ranks
+            # a genuinely outcome-flipping step first (test_blame.py injects the flip at the
+            # *final* step to show it isn't hardwired to step 0), not that it discriminates
+            # among multiple competing causes on a long tape. See README → Validation scope.
+            def perturb_factory(step_idx: int, _mutated=mutated_resp, _fault=fault_step):
+                if step_idx == _fault:
+                    # Inject the fault; the tail flips when it sees the marker.
+                    return _mutated, FaultAwareFakeLLM(
+                        normal_responses=[SUCCESS_RESP] * 10,
+                        fault_responses=[FAIL_RESP] * 10,
+                        fault_marker=FAULT_MARKER_BYTES,
+                    )
+                # Other steps: a benign perturbation that should not flip.
+                return SUCCESS_RESP, ScriptedFakeLLM([SUCCESS_RESP] * 10)
+
+            report = BlameEngine.rank(
+                tape,
+                synthetic_agent,
+                oracle,
+                perturb_factory=perturb_factory,
+                k=self._k,
+                budget_usd=100.0,
+            )
+            top = report.top()
+            if top is not None and top.step_index == fault_step:
+                top1_correct += 1
+
+            # Negative control: no real perturbation anywhere → expect no flips.
+            def null_perturb_factory(step_idx: int):
+                return SUCCESS_RESP, ScriptedFakeLLM([SUCCESS_RESP] * 10)
+
+            ctrl = BlameEngine.rank(
+                tape,
+                synthetic_agent,
+                oracle,
+                perturb_factory=null_perturb_factory,
+                k=self._k,
+                budget_usd=100.0,
+            )
+            for r in ctrl.results:
+                max_flip_control = max(max_flip_control, r.flip_rate)
+
+        precision = top1_correct / self._n_runs if self._n_runs > 0 else 0.0
+        return ValidationReport(
+            fault_class=self._fault_class,
+            n_runs=self._n_runs,
+            top1_correct=top1_correct,
+            top1_precision=precision,
+            negative_control_max_flip=max_flip_control,
+        )
+
+
+def run_all_fault_classes(k: int = 3, n_runs: int = 5) -> dict:
+    """Run validation for all five fault classes; return a report dict."""
+    results = {}
+    for fc in FaultClass:
+        report = ValidationRunner(fc, k=k, n_runs=n_runs).run()
+        results[fc.value] = {
+            "top1_precision": report.top1_precision,
+            "top1_correct": report.top1_correct,
+            "n_runs": report.n_runs,
+            "negative_control_max_flip": report.negative_control_max_flip,
+        }
+    return results

tracefork/web/report.html

@@ -0,0 +1,209 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>tracefork — time-travel debugger</title>
+<style>
+  * { box-sizing: border-box; margin: 0; padding: 0; }
+  :root {
+    --bg: #0d1117; --surface: #161b22; --border: #30363d;
+    --text: #c9d1d9; --muted: #6e7681; --green: #3fb950;
+    --blue: #58a6ff; --orange: #f0883e; --purple: #d2a8ff;
+    --red: #f85149; --yellow: #e3b341;
+  }
+  body { background: var(--bg); color: var(--text); font-family: ui-monospace, monospace; font-size: 13px; }
+  header { padding: 12px 20px; border-bottom: 1px solid var(--border); display: flex; align-items: center; gap: 12px; }
+  header h1 { font-size: 15px; color: var(--green); }
+  header .run-meta { color: var(--muted); font-size: 11px; }
+  .layout { display: grid; grid-template-columns: 280px 1fr 320px; height: calc(100vh - 45px); }
+  .panel { border-right: 1px solid var(--border); overflow-y: auto; }
+  .panel:last-child { border-right: none; }
+  .panel-header { padding: 10px 14px; background: var(--surface); border-bottom: 1px solid var(--border);
+                  font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.05em; }
+
+  /* Panel 1 — Timeline */
+  .exchange-item { padding: 10px 14px; border-bottom: 1px solid var(--border); cursor: pointer; }
+  .exchange-item:hover, .exchange-item.active { background: var(--surface); }
+  .exchange-item .step { font-size: 10px; color: var(--muted); }
+  .exchange-item .role { font-size: 12px; margin: 3px 0; }
+  .role-user { color: var(--blue); }
+  .role-assistant { color: var(--purple); }
+  .role-tool { color: var(--orange); }
+  .exchange-item .preview { font-size: 11px; color: var(--muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+  .blame-badge { float: right; font-size: 10px; padding: 2px 6px; border-radius: 10px; background: #1a3a1a; color: var(--green); }
+  .blame-badge.high { background: #3a1a1a; color: var(--red); }
+
+  /* Panel 2 — Detail */
+  .detail-empty { padding: 40px; color: var(--muted); text-align: center; }
+  .detail-section { padding: 14px; border-bottom: 1px solid var(--border); }
+  .detail-section h3 { font-size: 11px; color: var(--muted); margin-bottom: 8px; text-transform: uppercase; }
+  pre { background: var(--surface); padding: 10px; border-radius: 4px; overflow-x: auto; font-size: 11px; line-height: 1.6; }
+  .key { color: var(--blue); }
+  .string { color: var(--green); }
+  .number { color: var(--yellow); }
+
+  /* Panel 3 — Blame */
+  .blame-empty { padding: 40px; color: var(--muted); text-align: center; }
+  .blame-row { padding: 8px 14px; border-bottom: 1px solid var(--border); display: flex; gap: 8px; align-items: center; }
+  .blame-rank { width: 20px; color: var(--muted); font-size: 11px; }
+  .blame-info { flex: 1; }
+  .blame-step { font-size: 12px; }
+  .blame-ci { font-size: 10px; color: var(--muted); }
+  .blame-bar-wrap { width: 80px; }
+  .blame-bar { height: 6px; background: var(--green); border-radius: 3px; }
+  .blame-bar.decisive { background: var(--red); }
+  .blame-rate { font-size: 11px; text-align: right; min-width: 50px; }
+  .blame-rate.decisive { color: var(--red); font-weight: bold; }
+  .blame-header-row { padding: 6px 14px; background: var(--surface); border-bottom: 1px solid var(--border);
+                       display: flex; gap: 8px; font-size: 10px; color: var(--muted); }
+
+  .loading { padding: 40px; text-align: center; color: var(--muted); }
+  .error { padding: 20px; color: var(--red); }
+  .tag { display: inline-block; padding: 1px 6px; border-radius: 3px; font-size: 10px; margin-left: 6px; }
+  .tag-live { background: #1a3a1a; color: var(--green); }
+  .tag-static { background: #1a2a3a; color: var(--blue); }
+</style>
+</head>
+<body>
+<header>
+  <h1>tracefork</h1>
+  <span class="run-meta" id="run-meta">loading…</span>
+  <span class="tag" id="mode-tag"></span>
+</header>
+<div class="layout">
+  <div class="panel" id="timeline-panel">
+    <div class="panel-header">Timeline</div>
+    <div id="timeline-content"><div class="loading">loading…</div></div>
+  </div>
+  <div class="panel" id="detail-panel">
+    <div class="panel-header">Exchange Detail</div>
+    <div id="detail-content"><div class="detail-empty">← select an exchange</div></div>
+  </div>
+  <div class="panel" id="blame-panel">
+    <div class="panel-header">Blame</div>
+    <div id="blame-content"><div class="blame-empty">run tracefork blame to populate</div></div>
+  </div>
+</div>
+
+<script>
+// ── data source ───────────────────────────────────────────────────────────
+let DATA = null;
+
+async function loadData() {
+  if (window.__TRACEFORK_DATA__) {
+    // static mode: data injected at report-generation time
+    document.getElementById('mode-tag').textContent = 'static';
+    document.getElementById('mode-tag').className = 'tag tag-static';
+    return window.__TRACEFORK_DATA__;
+  }
+  if (window.__TRACEFORK_SERVER_URL__ !== undefined) {
+    // live mode: fetch from the serving origin (empty base → same-origin, any port)
+    document.getElementById('mode-tag').textContent = 'live';
+    document.getElementById('mode-tag').className = 'tag tag-live';
+    const url = window.__TRACEFORK_SERVER_URL__;
+    const runId = new URLSearchParams(location.search).get('run_id');
+    const resp = await fetch(`${url}/api/run/${runId}`);
+    if (!resp.ok) throw new Error(`server ${resp.status}: run not found`);
+    return resp.json();
+  }
+  throw new Error('No data source configured');
+}
+
+// ── render ─────────────────────────────────────────────────────────────────
+function renderTimeline(data) {
+  const blame = data.blame || {};
+  const html = data.exchanges.map((ex, i) => {
+    const flipRate = blame[i] ? blame[i].flip_rate : null;
+    const isDecisive = flipRate !== null && flipRate >= 0.7;
+    const badgeHtml = flipRate !== null
+      ? `<span class="blame-badge${isDecisive ? ' high' : ''}">${Math.round(flipRate*100)}%</span>`
+      : '';
+    const roleClass = ex.role === 'user' ? 'role-user' : ex.role === 'assistant' ? 'role-assistant' : 'role-tool';
+    return `<div class="exchange-item" data-i="${i}" onclick="selectExchange(${i})">
+      ${badgeHtml}
+      <div class="step">exchange ${i}</div>
+      <div class="role ${roleClass}">${escape(ex.role || 'unknown')}</div>
+      <div class="preview">${escape(ex.preview || '')}</div>
+    </div>`;
+  }).join('');
+  document.getElementById('timeline-content').innerHTML = html;
+}
+
+function renderBlame(data) {
+  const blame = data.blame;
+  if (!blame || Object.keys(blame).length === 0) return;
+  const entries = Object.entries(blame).sort((a, b) => b[1].flip_rate - a[1].flip_rate);
+  const headerHtml = `<div class="blame-header-row">
+    <span style="width:20px">rank</span>
+    <span style="flex:1">step</span>
+    <span style="width:80px">bar</span>
+    <span style="min-width:50px;text-align:right">flip</span>
+  </div>`;
+  const rowsHtml = entries.map(([step, info], rank) => {
+    const rate = info.flip_rate;
+    const isDecisive = rate >= 0.7;
+    const pct = Math.round(rate * 100);
+    return `<div class="blame-row">
+      <span class="blame-rank">${rank+1}</span>
+      <div class="blame-info">
+        <div class="blame-step">step-${step}</div>
+        <div class="blame-ci">95% CI [${info.ci_lo.toFixed(2)}, ${info.ci_hi.toFixed(2)}]</div>
+      </div>
+      <div class="blame-bar-wrap"><div class="blame-bar${isDecisive ? ' decisive' : ''}" style="width:${pct}%"></div></div>
+      <div class="blame-rate${isDecisive ? ' decisive' : ''}">${pct}%</div>
+    </div>`;
+  }).join('');
+  document.getElementById('blame-content').innerHTML = headerHtml + rowsHtml;
+}
+
+let _selected = -1;
+function selectExchange(i) {
+  if (_selected >= 0) {
+    document.querySelector(`[data-i="${_selected}"]`)?.classList.remove('active');
+  }
+  _selected = i;
+  document.querySelector(`[data-i="${i}"]`)?.classList.add('active');
+  const ex = DATA.exchanges[i];
+  const reqJson = syntaxHighlight(JSON.stringify(ex.request || {}, null, 2));
+  const respJson = syntaxHighlight(JSON.stringify(ex.response_preview || {}, null, 2));
+  document.getElementById('detail-content').innerHTML = `
+    <div class="detail-section">
+      <h3>Request (exchange ${i})</h3>
+      <pre>${reqJson}</pre>
+    </div>
+    <div class="detail-section">
+      <h3>Response</h3>
+      <pre>${respJson}</pre>
+    </div>`;
+}
+
+function syntaxHighlight(json) {
+  return json
+    .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+    .replace(/("(\\u[\da-fA-F]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g,
+      m => {
+        let cls = 'number';
+        if (/^"/.test(m)) cls = /:$/.test(m) ? 'key' : 'string';
+        return `<span class="${cls}">${m}</span>`;
+      });
+}
+
+function escape(s) { return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;'); }
+
+// ── boot ───────────────────────────────────────────────────────────────────
+(async () => {
+  try {
+    DATA = await loadData();
+    document.getElementById('run-meta').textContent =
+      `${DATA.agent_name || 'unknown agent'} · ${DATA.exchanges.length} exchanges · ${DATA.created_at || ''}`;
+    renderTimeline(DATA);
+    renderBlame(DATA);
+    if (DATA.exchanges.length > 0) selectExchange(0);
+  } catch (e) {
+    document.getElementById('timeline-content').innerHTML = `<div class="error">${e.message}</div>`;
+  }
+})();
+</script>
+</body>
+</html>

tracefork/wire.py

@@ -0,0 +1,76 @@
+"""Anthropic wire-format response builders.
+
+Real Anthropic Messages-API JSON, used in three places:
+  - the offline test fakes (`tests/fakes.py` re-exports these),
+  - the blame engine's perturbation responses,
+  - the fault-injection validation suite.
+
+Keeping them in the package (not in tests/) means production code never
+imports from the test tree.
+"""
+
+from __future__ import annotations
+
+import json
+
+from .tape import sha256_hex
+
+
+def make_text_response(
+    text: str,
+    *,
+    model: str = "claude-sonnet-4-6",
+    input_tokens: int = 100,
+    output_tokens: int = 20,
+) -> bytes:
+    """Return Anthropic wire-format JSON bytes for a final text response."""
+    rid = "msg_" + sha256_hex((text + model).encode())[:20]
+    return json.dumps(
+        {
+            "id": rid,
+            "type": "message",
+            "role": "assistant",
+            "model": model,
+            "content": [{"type": "text", "text": text}],
+            "stop_reason": "end_turn",
+            "stop_sequence": None,
+            "usage": {"input_tokens": input_tokens, "output_tokens": output_tokens},
+        }
+    ).encode()
+
+
+def make_tool_use_response(
+    tool_name: str,
+    tool_input: dict,
+    *,
+    model: str = "claude-sonnet-4-6",
+    preamble: str = "",
+    input_tokens: int = 100,
+    output_tokens: int = 30,
+) -> bytes:
+    """Return Anthropic wire-format JSON bytes for a tool_use response."""
+    content: list[dict] = []
+    if preamble:
+        content.append({"type": "text", "text": preamble})
+    toolu_id = "toolu_" + sha256_hex((tool_name + json.dumps(tool_input)).encode())[:18]
+    content.append(
+        {
+            "type": "tool_use",
+            "id": toolu_id,
+            "name": tool_name,
+            "input": tool_input,
+        }
+    )
+    rid = "msg_" + sha256_hex((tool_name + model).encode())[:20]
+    return json.dumps(
+        {
+            "id": rid,
+            "type": "message",
+            "role": "assistant",
+            "model": model,
+            "content": content,
+            "stop_reason": "tool_use",
+            "stop_sequence": None,
+            "usage": {"input_tokens": input_tokens, "output_tokens": output_tokens},
+        }
+    ).encode()