tracefork 0.1.0__py3-none-any.whl

tracefork_spike/spike.py

@@ -0,0 +1,125 @@
+"""Spike 0 orchestration: record -> persist -> load -> replay -> verify.
+
+Answers exactly one question: can we record a tool-using Anthropic-SDK agent run and
+replay it bit-exact, with proof, for $0 and no network — within a declared
+determinism boundary? The boundary here: a single-process, synchronous agent whose
+only nondeterminism sources are clock and id generation, both routed through the
+NondetSource seam.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+
+from .agent import make_client, run_agent
+from .fake_llm import FakeAnthropicTransport
+from .nondet import DivergenceError, DriftingNondet, RecordingNondet, ReplayNondet
+from .tape import Tape
+from .transport import TraceforkTransport
+
+
+def record_replay_verify(tape_path: str | None = None) -> dict:
+    """Run the full spike and return a structured result dict (used by the CLI and tests)."""
+    cleanup = False
+    if tape_path is None:
+        fd, tape_path = tempfile.mkstemp(suffix=".tape.sqlite")
+        os.close(fd)
+        cleanup = True
+
+    try:
+        # 1. RECORD — real agent + SDK, fake (offline) endpoint, genuine nondeterminism.
+        rec_tape = Tape()
+        rec_nondet = RecordingNondet()
+        rec_transport = TraceforkTransport("record", rec_tape, inner=FakeAnthropicTransport())
+        rec_result = run_agent(make_client(rec_transport), rec_nondet)
+        rec_tape.draws = rec_nondet.draws
+        record_fingerprint = rec_tape.digest()
+
+        # 2. PERSIST + RELOAD — prove the content-addressed tape round-trips through disk.
+        rec_tape.save(tape_path)
+        loaded = Tape.load(tape_path)
+        assert loaded.digest() == record_fingerprint, "tape changed across save/load"
+
+        # 3. REPLAY — no network: replay transport has no inner; nondeterminism virtualized.
+        rep_nondet = ReplayNondet(loaded.draws)
+        rep_transport = TraceforkTransport("replay", loaded)  # inner=None -> any real call errors
+        rep_result = run_agent(make_client(rep_transport), rep_nondet)
+        replay_fingerprint = loaded.digest()
+
+        # 4. VERIFY — observable output identical, every request hash matched,
+        #    all recorded draws consumed, no leftover exchanges.
+        checks = {
+            "output_identical": rep_result == rec_result,
+            "fingerprint_match": replay_fingerprint == record_fingerprint,
+            "all_request_hashes_matched": rep_transport.matched == len(loaded.exchanges),
+            "all_exchanges_consumed": rep_transport.fully_consumed(),
+            "all_draws_consumed": rep_nondet.fully_consumed(),
+        }
+
+        # 5. NEGATIVE CONTROL — replay with drifting (fresh) nondeterminism MUST diverge.
+        drift_detected = False
+        drift_at: str | None = None
+        try:
+            run_agent(make_client(TraceforkTransport("replay", loaded)), DriftingNondet())
+        except DivergenceError as e:
+            drift_detected = True
+            drift_at = str(e)
+        checks["negative_control_detected_drift"] = drift_detected
+
+        return {
+            "exchanges": len(loaded.exchanges),
+            "draws": len(loaded.draws),
+            "request_hashes_matched": rep_transport.matched,
+            "record_fingerprint": record_fingerprint,
+            "replay_fingerprint": replay_fingerprint,
+            "final_text": rec_result["final_text"],
+            "checks": checks,
+            "drift_at": drift_at,
+            "passed": all(checks.values()),
+        }
+    finally:
+        if cleanup and os.path.exists(tape_path):
+            os.remove(tape_path)
+
+
+def _fmt(result: dict) -> str:
+    c = result["checks"]
+    ok = "PASS" if result["passed"] else "FAIL"
+    lines = [
+        "",
+        "  tracefork — Spike 0: bit-exact record/replay",
+        "  " + "-" * 52,
+        f"  recorded exchanges ........ {result['exchanges']}",
+        f"  nondeterminism draws ...... {result['draws']}  (clock + id, virtualized)",
+        f"  request hashes matched .... {result['request_hashes_matched']}/{result['exchanges']}",
+        f"  tape fingerprint .......... {result['record_fingerprint'][:24]}…",
+        f"  replay fingerprint ........ {result['replay_fingerprint'][:24]}…",
+        "  network calls / spend ..... 0 / $0.00",
+        f"  agent final answer ........ {result['final_text']!r}",
+        "",
+        f"    [{'x' if c['output_identical'] else ' '}] "
+        "replayed trajectory byte-identical to recorded",
+        f"    [{'x' if c['fingerprint_match'] else ' '}] "
+        "tape fingerprint matches after save/load round-trip",
+        f"    [{'x' if c['all_request_hashes_matched'] else ' '}] "
+        "every replayed request hash matched the tape",
+        f"    [{'x' if c['all_draws_consumed'] else ' '}] "
+        "every recorded nondeterminism draw consumed",
+        f"    [{'x' if c['negative_control_detected_drift'] else ' '}] "
+        "negative control: drift was DETECTED, not silently passed",
+        "",
+        f"  RESULT: {ok}",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    result = record_replay_verify()
+    print(_fmt(result))
+    return 0 if result["passed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tracefork_spike/tape.py

@@ -0,0 +1,79 @@
+"""Content-addressed, persistable tape.
+
+A tape is the recorded artifact of one agent run: the ordered HTTP exchanges
+(request body + response body) plus the ordered nondeterminism draws. Response and
+request bodies are stored content-addressed (keyed by sha256), so identical bytes are
+stored once; an ordered event log preserves sequence. `digest()` is a hash chain over
+the whole tape — the single fingerprint reported in the receipt.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+from dataclasses import dataclass, field
+
+
+def sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+@dataclass
+class Tape:
+    exchanges: list[tuple[bytes, bytes]] = field(default_factory=list)
+    draws: list[tuple[str, str]] = field(default_factory=list)
+
+    def append_exchange(self, request_body: bytes, response_body: bytes) -> None:
+        self.exchanges.append((request_body, response_body))
+
+    def exchange(self, i: int) -> tuple[bytes, bytes]:
+        return self.exchanges[i]
+
+    def digest(self) -> str:
+        """sha256 hash chain over draws then exchanges — the tape fingerprint."""
+        h = hashlib.sha256()
+        for kind, value in self.draws:
+            h.update(b"D:" + kind.encode() + b":" + value.encode() + b"\n")
+        for req, resp in self.exchanges:
+            h.update(b"X:" + sha256_hex(req).encode() + b":" + sha256_hex(resp).encode() + b"\n")
+        return h.hexdigest()
+
+    # --- persistence: content-addressed blobs + ordered event log ---------------
+
+    def save(self, path: str) -> None:
+        con = sqlite3.connect(path)
+        try:
+            con.executescript(
+                """
+                DROP TABLE IF EXISTS blobs;
+                DROP TABLE IF EXISTS events;
+                CREATE TABLE blobs (hash TEXT PRIMARY KEY, data BLOB NOT NULL);
+                CREATE TABLE events (seq INTEGER PRIMARY KEY AUTOINCREMENT,
+                                     kind TEXT NOT NULL, a TEXT NOT NULL, b TEXT NOT NULL);
+                """
+            )
+            for kind, value in self.draws:
+                con.execute("INSERT INTO events (kind, a, b) VALUES ('draw', ?, ?)", (kind, value))
+            for req, resp in self.exchanges:
+                rh, sh = sha256_hex(req), sha256_hex(resp)
+                con.execute("INSERT OR IGNORE INTO blobs (hash, data) VALUES (?, ?)", (rh, req))
+                con.execute("INSERT OR IGNORE INTO blobs (hash, data) VALUES (?, ?)", (sh, resp))
+                con.execute("INSERT INTO events (kind, a, b) VALUES ('exchange', ?, ?)", (rh, sh))
+            con.commit()
+        finally:
+            con.close()
+
+    @classmethod
+    def load(cls, path: str) -> Tape:
+        con = sqlite3.connect(path)
+        try:
+            blobs = dict(con.execute("SELECT hash, data FROM blobs").fetchall())
+            tape = cls()
+            for kind, a, b in con.execute("SELECT kind, a, b FROM events ORDER BY seq").fetchall():
+                if kind == "draw":
+                    tape.draws.append((a, b))
+                elif kind == "exchange":
+                    tape.exchanges.append((bytes(blobs[a]), bytes(blobs[b])))
+            return tape
+        finally:
+            con.close()

tracefork_spike/transport.py

@@ -0,0 +1,68 @@
+"""The recording/replay httpx transport — the model-I/O capture seam.
+
+Record mode: forward each request to an inner transport, capture the request body
+and the full response body into the tape, and return the response unchanged.
+
+Replay mode: ignore the network entirely. For each request, pop the next recorded
+exchange, assert the request body is byte-identical to what was recorded (this is the
+divergence detector), and serve the recorded response bytes back. A replay transport
+has no inner transport, so any unexpected/extra request is a hard error rather than a
+silent network call.
+"""
+
+from __future__ import annotations
+
+import httpx
+
+from .nondet import DivergenceError
+from .tape import Tape, sha256_hex
+
+
+class TraceforkTransport(httpx.BaseTransport):
+    def __init__(self, mode: str, tape: Tape, inner: httpx.BaseTransport | None = None) -> None:
+        assert mode in ("record", "replay")
+        if mode == "record" and inner is None:
+            raise ValueError("record mode requires an inner transport")
+        self.mode = mode
+        self.tape = tape
+        self.inner = inner
+        self._i = 0
+        self.matched = 0  # number of replay request-hashes that matched the tape
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        body = request.content
+
+        if self.mode == "record":
+            inner_resp = self.inner.handle_request(request)  # type: ignore[union-attr]
+            resp_body = inner_resp.read()
+            self.tape.append_exchange(body, resp_body)
+            return httpx.Response(
+                inner_resp.status_code,
+                headers={"content-type": "application/json"},
+                content=resp_body,
+                request=request,
+            )
+
+        # replay
+        if self._i >= len(self.tape.exchanges):
+            raise DivergenceError(
+                f"replay made an unrecorded request #{self._i} "
+                f"(tape has {len(self.tape.exchanges)} exchanges)"
+            )
+        rec_req, rec_resp = self.tape.exchange(self._i)
+        if sha256_hex(rec_req) != sha256_hex(body):
+            raise DivergenceError(
+                f"request #{self._i} diverged from the tape "
+                f"(recorded {sha256_hex(rec_req)[:12]}, replay {sha256_hex(body)[:12]})"
+            )
+        self._i += 1
+        self.matched += 1
+        return httpx.Response(
+            200,
+            headers={"content-type": "application/json"},
+            content=rec_resp,
+            request=request,
+        )
+
+    def fully_consumed(self) -> bool:
+        return self.mode == "replay" and self._i == len(self.tape.exchanges)