tracefork 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. {tracefork-0.1.0 → tracefork-0.2.0}/.github/workflows/ci.yml +2 -2
  2. {tracefork-0.1.0 → tracefork-0.2.0}/.gitignore +5 -0
  3. tracefork-0.2.0/CHANGELOG.md +124 -0
  4. tracefork-0.2.0/CLAUDE.md +236 -0
  5. tracefork-0.2.0/PKG-INFO +589 -0
  6. tracefork-0.2.0/README.md +529 -0
  7. tracefork-0.2.0/experiments/replay_fixtures/manifest.json +14 -0
  8. tracefork-0.2.0/experiments/replay_fixtures/single_turn.tape.sqlite +0 -0
  9. tracefork-0.2.0/experiments/replay_fixtures/two_turn.tape.sqlite +0 -0
  10. tracefork-0.2.0/pyproject.toml +169 -0
  11. tracefork-0.2.0/scripts/e2e.sh +47 -0
  12. tracefork-0.2.0/scripts/gen_replay_fixtures.py +91 -0
  13. tracefork-0.2.0/src/tracefork/__init__.py +134 -0
  14. tracefork-0.2.0/src/tracefork/adapters/__init__.py +118 -0
  15. tracefork-0.2.0/src/tracefork/adapters/autogen.py +313 -0
  16. tracefork-0.2.0/src/tracefork/adapters/base.py +377 -0
  17. tracefork-0.2.0/src/tracefork/adapters/crewai.py +359 -0
  18. tracefork-0.2.0/src/tracefork/adapters/langchain.py +716 -0
  19. tracefork-0.2.0/src/tracefork/adapters/openai_agents.py +400 -0
  20. tracefork-0.2.0/src/tracefork/bedrock_transport.py +297 -0
  21. tracefork-0.2.0/src/tracefork/bench.py +152 -0
  22. tracefork-0.2.0/src/tracefork/blame.py +917 -0
  23. tracefork-0.2.0/src/tracefork/boundary_guard.py +153 -0
  24. tracefork-0.2.0/src/tracefork/cli.py +766 -0
  25. tracefork-0.2.0/src/tracefork/competing_faults.py +260 -0
  26. tracefork-0.2.0/src/tracefork/config.py +123 -0
  27. tracefork-0.2.0/src/tracefork/constants.py +70 -0
  28. tracefork-0.2.0/src/tracefork/data/pricing.json +38 -0
  29. tracefork-0.2.0/src/tracefork/divergence.py +198 -0
  30. tracefork-0.2.0/src/tracefork/eventstream.py +149 -0
  31. tracefork-0.2.0/src/tracefork/faults.py +168 -0
  32. tracefork-0.2.0/src/tracefork/fixtures.py +53 -0
  33. tracefork-0.2.0/src/tracefork/fork.py +370 -0
  34. tracefork-0.2.0/src/tracefork/interop.py +472 -0
  35. tracefork-0.2.0/src/tracefork/judge.py +390 -0
  36. tracefork-0.2.0/src/tracefork/matcher.py +281 -0
  37. tracefork-0.2.0/src/tracefork/mcp_client.py +98 -0
  38. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork/nondet.py +25 -2
  39. tracefork-0.2.0/src/tracefork/observability.py +195 -0
  40. tracefork-0.2.0/src/tracefork/plugins.py +120 -0
  41. tracefork-0.2.0/src/tracefork/pricing.py +108 -0
  42. tracefork-0.2.0/src/tracefork/providers/__init__.py +41 -0
  43. tracefork-0.2.0/src/tracefork/providers/anthropic.py +210 -0
  44. tracefork-0.2.0/src/tracefork/providers/base.py +173 -0
  45. tracefork-0.2.0/src/tracefork/providers/bedrock.py +216 -0
  46. tracefork-0.2.0/src/tracefork/providers/gemini.py +232 -0
  47. tracefork-0.2.0/src/tracefork/providers/openai.py +266 -0
  48. tracefork-0.2.0/src/tracefork/proxy.py +280 -0
  49. tracefork-0.2.0/src/tracefork/record_mode.py +75 -0
  50. tracefork-0.2.0/src/tracefork/recorder.py +270 -0
  51. tracefork-0.2.0/src/tracefork/redact.py +368 -0
  52. tracefork-0.2.0/src/tracefork/replay.py +273 -0
  53. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork/report.py +17 -11
  54. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork/store.py +70 -18
  55. tracefork-0.2.0/src/tracefork/synthetic.py +249 -0
  56. tracefork-0.2.0/src/tracefork/tape.py +472 -0
  57. tracefork-0.2.0/src/tracefork/tools.py +328 -0
  58. tracefork-0.2.0/src/tracefork/transport.py +302 -0
  59. tracefork-0.2.0/src/tracefork/wire.py +51 -0
  60. {tracefork-0.1.0 → tracefork-0.2.0}/tests/fakes.py +2 -0
  61. tracefork-0.2.0/tests/fixtures/legacy_tape_v1.blob +1 -0
  62. tracefork-0.2.0/tests/test_adapters_autogen.py +221 -0
  63. tracefork-0.2.0/tests/test_adapters_base.py +232 -0
  64. tracefork-0.2.0/tests/test_adapters_crewai.py +222 -0
  65. tracefork-0.2.0/tests/test_adapters_langchain.py +371 -0
  66. tracefork-0.2.0/tests/test_adapters_openai_agents.py +252 -0
  67. tracefork-0.2.0/tests/test_bedrock_eventstream.py +98 -0
  68. tracefork-0.2.0/tests/test_bedrock_provider.py +163 -0
  69. tracefork-0.2.0/tests/test_bedrock_transport.py +268 -0
  70. tracefork-0.2.0/tests/test_bench.py +62 -0
  71. tracefork-0.2.0/tests/test_blame.py +571 -0
  72. tracefork-0.2.0/tests/test_boundary_guard.py +189 -0
  73. tracefork-0.2.0/tests/test_cli.py +313 -0
  74. tracefork-0.2.0/tests/test_cli_smoke.py +360 -0
  75. tracefork-0.2.0/tests/test_competing_faults.py +184 -0
  76. tracefork-0.2.0/tests/test_concurrency.py +324 -0
  77. tracefork-0.2.0/tests/test_config.py +140 -0
  78. tracefork-0.2.0/tests/test_divergence.py +194 -0
  79. tracefork-0.2.0/tests/test_e2e.py +709 -0
  80. tracefork-0.2.0/tests/test_fork.py +313 -0
  81. tracefork-0.2.0/tests/test_gemini_provider.py +170 -0
  82. tracefork-0.2.0/tests/test_interop.py +305 -0
  83. tracefork-0.2.0/tests/test_judge.py +465 -0
  84. tracefork-0.2.0/tests/test_matcher.py +275 -0
  85. tracefork-0.2.0/tests/test_mcp_client.py +57 -0
  86. tracefork-0.2.0/tests/test_nondet.py +157 -0
  87. tracefork-0.2.0/tests/test_observability.py +123 -0
  88. tracefork-0.2.0/tests/test_openai_provider.py +192 -0
  89. tracefork-0.2.0/tests/test_plugins.py +223 -0
  90. tracefork-0.2.0/tests/test_pricing.py +162 -0
  91. tracefork-0.2.0/tests/test_provider_faults.py +131 -0
  92. tracefork-0.2.0/tests/test_providers.py +310 -0
  93. tracefork-0.2.0/tests/test_proxy.py +221 -0
  94. tracefork-0.2.0/tests/test_record_mode.py +53 -0
  95. tracefork-0.2.0/tests/test_redact.py +434 -0
  96. tracefork-0.2.0/tests/test_replay.py +259 -0
  97. tracefork-0.2.0/tests/test_report.py +237 -0
  98. tracefork-0.2.0/tests/test_storage.py +248 -0
  99. tracefork-0.2.0/tests/test_tools.py +333 -0
  100. tracefork-0.2.0/uv.lock +4471 -0
  101. {tracefork-0.1.0 → tracefork-0.2.0}/web/report.html +101 -2
  102. tracefork-0.1.0/CHANGELOG.md +0 -45
  103. tracefork-0.1.0/CLAUDE.md +0 -112
  104. tracefork-0.1.0/PKG-INFO +0 -235
  105. tracefork-0.1.0/README.md +0 -198
  106. tracefork-0.1.0/pyproject.toml +0 -94
  107. tracefork-0.1.0/src/tracefork/__init__.py +0 -6
  108. tracefork-0.1.0/src/tracefork/blame.py +0 -296
  109. tracefork-0.1.0/src/tracefork/cli.py +0 -367
  110. tracefork-0.1.0/src/tracefork/constants.py +0 -24
  111. tracefork-0.1.0/src/tracefork/faults.py +0 -129
  112. tracefork-0.1.0/src/tracefork/fork.py +0 -173
  113. tracefork-0.1.0/src/tracefork/recorder.py +0 -140
  114. tracefork-0.1.0/src/tracefork/replay.py +0 -119
  115. tracefork-0.1.0/src/tracefork/synthetic.py +0 -104
  116. tracefork-0.1.0/src/tracefork/tape.py +0 -135
  117. tracefork-0.1.0/src/tracefork/transport.py +0 -137
  118. tracefork-0.1.0/src/tracefork/wire.py +0 -76
  119. tracefork-0.1.0/tests/test_blame.py +0 -175
  120. tracefork-0.1.0/tests/test_cli.py +0 -81
  121. tracefork-0.1.0/tests/test_fork.py +0 -118
  122. tracefork-0.1.0/tests/test_replay.py +0 -124
  123. tracefork-0.1.0/tests/test_report.py +0 -116
  124. tracefork-0.1.0/uv.lock +0 -874
  125. {tracefork-0.1.0 → tracefork-0.2.0}/.editorconfig +0 -0
  126. {tracefork-0.1.0 → tracefork-0.2.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  127. {tracefork-0.1.0 → tracefork-0.2.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  128. {tracefork-0.1.0 → tracefork-0.2.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  129. {tracefork-0.1.0 → tracefork-0.2.0}/.github/dependabot.yml +0 -0
  130. {tracefork-0.1.0 → tracefork-0.2.0}/.github/pull_request_template.md +0 -0
  131. {tracefork-0.1.0 → tracefork-0.2.0}/.github/workflows/release.yml +0 -0
  132. {tracefork-0.1.0 → tracefork-0.2.0}/.pre-commit-config.yaml +0 -0
  133. {tracefork-0.1.0 → tracefork-0.2.0}/CODE_OF_CONDUCT.md +0 -0
  134. {tracefork-0.1.0 → tracefork-0.2.0}/CONTRIBUTING.md +0 -0
  135. {tracefork-0.1.0 → tracefork-0.2.0}/LICENSE +0 -0
  136. {tracefork-0.1.0 → tracefork-0.2.0}/SECURITY.md +0 -0
  137. {tracefork-0.1.0 → tracefork-0.2.0}/SPIKE0.md +0 -0
  138. {tracefork-0.1.0 → tracefork-0.2.0}/docs/demo.png +0 -0
  139. {tracefork-0.1.0 → tracefork-0.2.0}/examples/demo_report.py +0 -0
  140. {tracefork-0.1.0 → tracefork-0.2.0}/experiments/validation_report_committed.json +0 -0
  141. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork/py.typed +0 -0
  142. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork/server.py +0 -0
  143. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork/validate.py +0 -0
  144. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/__init__.py +0 -0
  145. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/__main__.py +0 -0
  146. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/agent.py +0 -0
  147. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/fake_llm.py +0 -0
  148. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/nondet.py +0 -0
  149. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/spike.py +0 -0
  150. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/tape.py +0 -0
  151. {tracefork-0.1.0 → tracefork-0.2.0}/src/tracefork_spike/transport.py +0 -0
  152. {tracefork-0.1.0 → tracefork-0.2.0}/tests/__init__.py +0 -0
  153. {tracefork-0.1.0 → tracefork-0.2.0}/tests/test_faults.py +0 -0
  154. {tracefork-0.1.0 → tracefork-0.2.0}/tests/test_recorder.py +0 -0
  155. {tracefork-0.1.0 → tracefork-0.2.0}/tests/test_spike0.py +0 -0
  156. {tracefork-0.1.0 → tracefork-0.2.0}/tests/test_tape.py +0 -0
  157. {tracefork-0.1.0 → tracefork-0.2.0}/tests/test_transport.py +0 -0
@@ -13,9 +13,9 @@ jobs:
13
13
  matrix:
14
14
  python-version: ["3.12", "3.13"]
15
15
  steps:
16
- - uses: actions/checkout@v4
16
+ - uses: actions/checkout@v7
17
17
  - name: Install uv
18
- uses: astral-sh/setup-uv@v3
18
+ uses: astral-sh/setup-uv@v7
19
19
  with:
20
20
  python-version: ${{ matrix.python-version }}
21
21
  - name: Install dependencies
@@ -9,11 +9,16 @@ __pycache__/
9
9
  *.tape.sqlite
10
10
  /tmp_tapes/
11
11
 
12
+ # Exception: the committed replay-fixture corpus (replay --check regression
13
+ # gate) — these tapes are checked in, not regenerated per-run.
14
+ !/experiments/replay_fixtures/*.tape.sqlite
15
+
12
16
  # Runtime artifacts (CLI output, regenerated on demand)
13
17
  /store.db
14
18
  /report.html
15
19
  /blame_*.json
16
20
  /validation_report.json
21
+ /bench_report.json
17
22
  /examples/demo_report.html
18
23
 
19
24
  # Build, coverage, and tool caches (regenerated on demand)
@@ -0,0 +1,124 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.2.0] - 2026-07-02
11
+
12
+ Generic, multi-provider, production-hardening release. The bit-exact, $0,
13
+ hash-verified replay substrate from 0.1.0 is the contract and is unchanged —
14
+ everything below is additive around it: all engine internals stay byte-stable,
15
+ `digest()` is format-stable, and every existing tape still loads and replays.
16
+
17
+ ### Added
18
+
19
+ - **Provider abstraction seam** (`providers/`): a `ProviderAdapter` protocol + registry
20
+ and a `gen_ai.*`-style `NormalizedResponse` view. Anthropic is now a *registered*
21
+ adapter, not a hardcoded assumption; raw request/response **bytes** stay the immutable
22
+ replay + hash contract. (#7)
23
+ - **More provider backends** — **OpenAI** and **Google Gemini** (httpx-based, captured
24
+ through the existing transport seam, with provider-generic fault injection), and **AWS
25
+ Bedrock** (a separate botocore `before-send` seam, a stdlib `vnd.amazon.eventstream`
26
+ frame codec, and SigV4-canonical request matching so a fresh signature/timestamp is not
27
+ a false divergence). (#12, #21)
28
+ - **Pluggable canonicalization / divergence matcher** (`matcher.py`): identity +
29
+ canonicalizing matchers so volatile material (Gemini `?key=`, Bedrock `x-amz-date`,
30
+ rotating auth) hashes equal while a real request-body change still diverges. (#9)
31
+ - **Plugin & extension architecture** (`plugins.py`): entry-point registries for
32
+ providers, matchers, storage backends, transports, oracles, and adapters — each gated by
33
+ the same explicit allowlist. (#11)
34
+ - **MCP + native tool-call record/replay** (`tools.py`, `mcp_client.py`): a JSON-RPC tee so
35
+ tool exchanges share the tape with LLM exchanges and replay bit-exact. (#14)
36
+ - **Framework adapters** (`adapters/`, opt-in extras): a minimal
37
+ `bind()`/`on_step()`/`teardown()` protocol + a `StepDAG` normalizer, with
38
+ **LangChain/LangGraph** (including a tape-backed LangGraph checkpointer) plus **OpenAI
39
+ Agents SDK**, **CrewAI**, and **AutoGen** adapters — each binding at the framework's real
40
+ model-call chokepoint (never a second capture path) and each guarded so `import
41
+ tracefork` and the full offline suite run with none installed. (#16, #23)
42
+ - **OTel GenAI / OpenInference interop** (`interop.py`, `tracefork export`/`ingest`): export
43
+ a tape (+ blame report) as OTel GenAI spans or an OpenInference dataset, and ingest either
44
+ back into a tape's step structure for **blame-by-re-execution** (explicitly *not* $0
45
+ bit-exact replay — an ingested tape diverges on `replay`/`fork` by design). Plain JSON, no
46
+ `opentelemetry-sdk` required. Plus an **opt-in `observability` extra** (structlog + OTel
47
+ self-instrumentation of record/replay/fork/blame), off by default. (#15)
48
+ - **Deterministic asyncio concurrency replay** (`transport.py`): records the completion
49
+ order of concurrent fan-out (`gather`/`TaskGroup`) and re-imposes it on replay, so
50
+ concurrent agents replay bit-exact — not just single-call-at-a-time ones. (#17)
51
+ - **Causal blame depth**: coalition / temporal-Shapley attribution with necessity +
52
+ sufficiency, and a long-tape competing-fault benchmark (`tracefork bench`) with Wilson
53
+ CIs. The one temporal-order Shapley limitation is documented, not hidden. (#18, #24)
54
+ - **Oracle rigor** (`judge.py`, opt-in): an LLM-judge oracle with position-swap averaging +
55
+ a self-judge guard, gold-set calibration (FPR/FNR/Cohen's kappa), and Rogan-Gladen
56
+ flip-rate debiasing — all offline-testable via an injected judge function. (#19)
57
+ - **Divergence diagnostics & debug UX** (`divergence.py`, web report): a structured
58
+ divergence diff, a report rewind panel, blame trust flags, and real-vs-tolerated
59
+ divergence messaging. (#20)
60
+ - **Localhost record/replay proxy** (`tracefork proxy`): a base-URL MITM-style proxy (binds
61
+ 127.0.0.1) for non-Python / non-httpx clients (curl, Node, Go), reusing the tape +
62
+ matcher. Record/replay only — outside the in-process `NondetSource` determinism
63
+ boundary. (#22)
64
+ - **Hardening seams**: an opt-in record-time `BoundaryGuard` (thread/subprocess spawn,
65
+ direct `random`/`time` bypasses), `random_float()` virtualization through `NondetSource`,
66
+ a `replay --check` fixture-corpus regression gate, and opt-in secret/PII redaction on
67
+ record. (#10, #13)
68
+ - **CLI**: new `bench`, `export`, `ingest`, and `proxy` commands (joining
69
+ `replay`/`verify`/`fork`/`report`/`serve`/`blame`/`validate`), plus a full end-to-end
70
+ integration suite and a single `scripts/e2e.sh` receipt. (#25)
71
+
72
+ ### Changed
73
+
74
+ - **Versioned tape envelope** (`tape.py`): a `TAPE_MAGIC` + uint16 version header over a
75
+ zstd + content-addressed binary container (no base64, no pickle), with a read-time
76
+ upcaster chain (v1→v4). The version header is **not** part of `digest()`, so the hash
77
+ chain is byte-stable across format versions and legacy tapes still load and replay. (#6)
78
+ - **Blame statistics**: three-valued trials (flip / no-flip / undefined) and pluggable CI
79
+ methods. (#8)
80
+ - **SQLite persistence hardened**: WAL, `synchronous=NORMAL`, `busy_timeout`,
81
+ `foreign_keys=ON`, and writers take `BEGIN IMMEDIATE`. (#6)
82
+
83
+ ### Fixed
84
+
85
+ - Wilson CI boundaries snap to analytically-exact 0/1 to avoid ~1e-17 platform float-dust
86
+ that differed across CI runners. (#8)
87
+
88
+ ## [0.1.0] - 2026-07-02
89
+
90
+ ### Added
91
+
92
+ - **Record/replay** at the Anthropic SDK's httpx transport boundary
93
+ (`TraceforkTransport` / `AsyncTraceforkTransport`), streaming-SSE capable, with
94
+ bit-exact replay proven by sha256-checking every replayed request body against the
95
+ recorded tape — and drift detection that fails loudly on divergence rather than
96
+ silently falling back to the network.
97
+ - **Content-addressed tape format** (`Tape`) — sha256 blobs plus an ordered event log,
98
+ JSON + base64 (never pickle), persistable to SQLite, with a hash-chain `digest()`
99
+ fingerprint.
100
+ - **Nondeterminism virtualization** (`NondetSource`) — the only path through which an
101
+ agent reads time and ids, with `RecordingNondet`, `ReplayNondet`, and a `DriftingNondet`
102
+ negative control that proves the divergence detector actually detects divergence.
103
+ - **Three-phase fork engine** (`ForkEngine`, `ForkTransport`) — prefix-replay ($0),
104
+ mutation-injection (swap a response), and tail-record (the recorded counterfactual
105
+ continuation), re-running the same agent that produced the original tape.
106
+ - **Causal blame engine** (`BlameEngine`) — forks each step `k` times, re-runs the agent,
107
+ grades outcomes via an `Oracle`, and ranks steps by flip-rate with Wilson score
108
+ confidence intervals; a `BudgetGovernor` estimates dollar cost from the pricing table
109
+ and refuses to exceed a caller-supplied budget before making any real API calls.
110
+ - **Fault-injection self-validation suite** (`faults.py`, `validate.py`) — five fault
111
+ classes with markers embedded in valid Anthropic JSON, scored end-to-end offline
112
+ against a synthetic fault-aware agent: **1.00 top-1 precision** across all five classes,
113
+ with an enforced negative-control threshold so the proof isn't vacuous.
114
+ - **Single-file web report/UI** (`report.py`, `server.py`, `web/report.html`) — a
115
+ dependency-free, three-panel HTML report (timeline, exchange detail, blame ranking)
116
+ either rendered statically or served live via FastAPI (`serve`, 127.0.0.1, no CORS).
117
+ - **CLI** (`cli.py`, Typer) — `replay`, `verify`, `fork`, `blame`, `report`, `serve`,
118
+ `validate`.
119
+ - `src/tracefork_spike/` — the original Spike 0 that de-risked bit-exact, no-key replay
120
+ within the declared determinism boundary.
121
+
122
+ [Unreleased]: https://github.com/pratik916/tracefork/compare/v0.2.0...HEAD
123
+ [0.2.0]: https://github.com/pratik916/tracefork/compare/v0.1.0...v0.2.0
124
+ [0.1.0]: https://github.com/pratik916/tracefork/releases/tag/v0.1.0
@@ -0,0 +1,236 @@
1
+ # CLAUDE.md
2
+
3
+ This file guides Claude Code when working in the `tracefork` repository.
4
+
5
+ ## What this is
6
+
7
+ `tracefork` is a time-travel debugger for AI agents: record an agent run to a
8
+ content-addressed **tape**, replay it **bit-exact for $0** (hash-verified), fork any
9
+ step, and measure causal blame with confidence intervals — the instrument itself
10
+ validated against runs with injected, known root-cause faults.
11
+
12
+ **Current state: v1 built.** All five product pillars work offline and are tested
13
+ (653 tests, $0): streaming-capable record/replay with drift detection, the three-phase
14
+ fork engine, the causal blame engine with Wilson CIs and a budget governor, the
15
+ single-file web report/UI, and the fault-injection self-validation suite (5 fault
16
+ classes at 1.00 top-1 precision, plus a longer competing-fault fixture that measures
17
+ whether the coalition/temporal-Shapley engine discriminates *several* simultaneously
18
+ planted causes — `tracefork bench`; see README → Validation scope for exactly what each
19
+ number does and doesn't claim). `src/tracefork_spike/` keeps the original Spike 0 that
20
+ de-risked the load-bearing assumption (bit-exact, no-key replay within a declared
21
+ determinism boundary). Design/feature list: `../ideas/2026-06-11-tracefork-features.md`;
22
+ spike finding: `SPIKE0.md`.
23
+
24
+ ## Commands
25
+
26
+ Python is **3.12 via uv**. The tests, the spike, `validate`, the demo, and
27
+ record/replay/fork are offline and $0 — **no `ANTHROPIC_API_KEY`, no network**. Only
28
+ `blame` against a *real* run hits the live API (budget-capped). Always prefix `uv run`.
29
+
30
+ ```bash
31
+ uv sync --extra dev # install (anthropic, zstandard, typer, fastapi, uvicorn + pytest)
32
+ uv run pytest -q # full offline suite (653 tests)
33
+ uv run pytest tests/test_faults.py::test_validation_runner_fingers_fault_step -q # one test
34
+ uv run tracefork validate # self-validation: blame vs injected, known faults
35
+ uv run tracefork validate --check # regression-gate vs experiments/validation_report_committed.json
36
+ uv run tracefork bench # long-tape competing-fault discrimination benchmark
37
+ uv run python examples/demo_report.py # write examples/demo_report.html (the README screenshot)
38
+ uv run python -m tracefork_spike # the original Spike 0 bit-exact replay receipt
39
+ uv run tracefork --help # replay, verify, fork, report, serve, blame, validate, bench, proxy
40
+ uv run tracefork replay --check experiments/replay_fixtures # replay-as-regression gate
41
+ bash scripts/e2e.sh # single-receipt gate: sync, lint, type-check, tests+coverage,
42
+ # validate --check, replay --check, bench, build+twine, one PASS banner
43
+ ```
44
+
45
+ ## Architecture (the parts that span files)
46
+
47
+ The spine is a **record/replay seam at the Anthropic SDK's httpx boundary**, plus a
48
+ **nondeterminism-virtualization seam** the agent reads time/ids through. Bit-exactness
49
+ is the contract between them.
50
+
51
+ The product lives in `src/tracefork/`:
52
+
53
+ - `nondet.py` — `NondetSource` is the *only* way the agent gets time/ids/random draws
54
+ (`now_iso`/`new_uuid_hex`/`random_float`). `RecordingNondet` draws real values and logs
55
+ them; `ReplayNondet` serves them back in order; `DriftingNondet` is the negative control
56
+ (fresh values → forced divergence). `random_float()` logs the exact `float.hex()` string
57
+ (lossless round-trip, no float-formatting dust) and, like `now_iso()`, is additive/opt-in
58
+ — an agent must be handed the active `NondetSource` explicitly; nothing patches `random`
59
+ globally the way `Recorder` patches `uuid.uuid4`. `find_divergence()` unwraps a
60
+ `DivergenceError` from the `APIConnectionError` the SDK wraps transport exceptions in —
61
+ **keep this; without it a real divergence looks like a network blip.**
62
+ - `boundary_guard.py` — `BoundaryGuard`, an **opt-in** (default off) record-mode guard:
63
+ hard-errors on `threading.Thread.start`/`subprocess.Popen` (crossing the single-process
64
+ boundary) and direct `random.random`/`time.monotonic`/`time.sleep` (bypassing
65
+ `NondetSource`) instead of letting the tape fail replay later, mysteriously. Deliberately
66
+ does **not** guard `datetime.datetime.now()` (same immutable-C-type reason `recorder.py`
67
+ doesn't patch it) or `time.time()` (httpx's cookie-jar machinery calls it on every
68
+ response — guarding it would false-positive on every exchange). Pre-warms the Anthropic
69
+ SDK's `platform_headers()` `lru_cache` on `__enter__` so its one internal
70
+ `subprocess.Popen` call (uncached platform detection) doesn't trip the guard on the first
71
+ real API call. Wired into `Recorder`/`AsyncRecorder` via `boundary_guard=` (tri-state:
72
+ explicit wins over `TraceforkConfig.boundary_guard`, both default `False`).
73
+ - `transport.py` — `TraceforkTransport` (sync) + `AsyncTraceforkTransport` (async) are the
74
+ capture seam, streaming-SSE capable (buffer via `.read()`/`.aread()`). Record mode tees
75
+ request+response bytes into the tape; replay mode serves recorded bytes and
76
+ sha256-asserts each request body matches (the divergence detector). A replay transport
77
+ has **no inner transport**, so any unrecorded request is a hard error. **Async
78
+ concurrency:** the async transport records the completion order of concurrent fan-out
79
+ (`asyncio.gather`/`TaskGroup`) — appending exchanges at completion and logging each
80
+ fully-overlapping batch to `tape.async_batches` — and on replay **correlates each request
81
+ to its recorded exchange by fingerprint (not positional arrival) and releases responses in
82
+ the recorded completion order** via an ordered gate, so a fan-out agent replays bit-exact.
83
+ A strictly-sequential async run never waits at the gate → byte-identical to before; the
84
+ sync transport is untouched (stays positional). `chaos_release_order(tape, seed)` derives a
85
+ seeded, physically-possible reordering of a recorded schedule (chaos-mode replay) for
86
+ race/ordering-bug analysis; **do NOT** add awaits/ordering to the sequential or sync path.
87
+ - `tape.py` — `Tape` is content-addressed (sha256 blobs) + an ordered event log,
88
+ persistable to SQLite, with a hash-chain `digest()` fingerprint. `to_bytes`/`from_bytes`
89
+ emit a **versioned envelope** (`TAPE_MAGIC` + uint16 version, then a zstd + content-
90
+ addressed binary container — no base64); `from_bytes` dispatches on the version through a
91
+ read-time upcaster chain and still loads legacy header-less JSON blobs as v1. The version
92
+ header is envelope metadata, **not** part of `digest()`, so the hash chain is byte-stable
93
+ across format versions. **v4** adds the `async_batches` concurrency-batch log (recorded
94
+ completion order of fully-overlapping fan-out); like `boundary`/`agent_name` it is
95
+ persisted but **never** fed into `digest()` (the completion order is already fingerprinted
96
+ by the `exchanges` list ordering), so every existing and every sequential/sync tape's
97
+ digest is byte-identical and v1/v2/v3 tapes upcast to an empty batch log. It's a JSON
98
+ header + zstd blobs, **not pickle** — no
99
+ arbitrary-code-execution risk. `open_sqlite()` is the one hardened connection factory
100
+ (WAL, `synchronous=NORMAL`, `busy_timeout`, `foreign_keys=ON`); writers take `BEGIN
101
+ IMMEDIATE`. `store.py` reuses it and serializes its write fan-out with a lock.
102
+ - `recorder.py` — `Recorder` context manager wraps a real `anthropic.Anthropic` at its
103
+ `_client._transport` seam (via `client.copy(http_client=...)`, so base_url / auth_token /
104
+ default headers are preserved). Patches `uuid.uuid4` globally; **does not** patch
105
+ `datetime.datetime` (immutable C type in 3.12+, and a subclass breaks the SDK's pydantic
106
+ schema builder) — agents needing deterministic clocks/random read `NondetSource` directly.
107
+ Optionally wraps the recording window in a `BoundaryGuard` (see `boundary_guard.py`).
108
+ - `fork.py` — `ForkTransport` runs three phases: prefix-replay ($0, request asserted to
109
+ match the parent), mutation-injection (same request, swapped response), tail-record (the
110
+ counterfactual continuation). `Branch` carries `prefix_replayed`/`tail_recorded` counts.
111
+ `ForkEngine.fork()` re-runs the **same** agent that produced the tape.
112
+ - `store.py` — `TapeStore`, SQLite persistence for tapes + the branch DAG.
113
+ - `blame.py` — `BlameEngine.rank()` forks each step `k` times, re-runs the agent, grades
114
+ via an `Oracle`, counts flips vs. the parent outcome; `wilson_ci()` for intervals;
115
+ `BudgetGovernor` estimates tail-call cost from `constants.PRICING_TABLE` before spend and
116
+ `rank()` raises `BudgetExceededError` if the estimate exceeds `budget_usd`.
117
+ - `judge.py` — OPT-IN, additive on top of `blame.py`'s `Oracle` protocol (never imported by the
118
+ default $0 path): `LLMJudgeOracle` is a binary-rubric judge with few-shot examples, a
119
+ configurable ("cross-family") judge model with a self-judge guard, position-swap averaging
120
+ (grades twice with the candidate output moved in the prompt; disagreement or low average
121
+ confidence abstains via `None`) — testable OFFLINE via an injected `judge_fn` (prompt -> raw
122
+ text), never a real API call in tests. `calibrate_oracle()` measures any `Oracle`'s FPR/FNR/
123
+ Cohen's kappa against a labeled gold set (`kappa_alert` below 0.6). `rogan_gladen_correct()` /
124
+ `debias_flip_rate()` debias an observed flip-rate for judge FPR/FNR (Rogan-Gladen 1978) and
125
+ widen a step's CI via delta-method propagation of BOTH k-sampling noise and finite-gold-set
126
+ judge noise. Pure math, reuses `blame.z_from_confidence`; registers `"llm_judge"` into
127
+ `blame.ORACLE_REGISTRY` at import time (importing the module is itself the opt-in).
128
+ - `faults.py` / `validate.py` — 5 fault classes (valid JSON, marker **inside** a content
129
+ field) + the self-validation runner; a synthetic agent echoes each response forward so an
130
+ injected fault propagates to a fault-aware tail. `run_all_fault_classes()` scores top-1.
131
+ **Scope (don't overstate):** the fixture is a positive-vs-inert control on a short tape —
132
+ it proves the engine is genuinely causal (not a fixed-slot artifact), not that it
133
+ discriminates among competing causes on long tapes. See README → Validation scope.
134
+ - `competing_faults.py` / `bench.py` — the longer-tape ANSWER to that scope note: a
135
+ 7-exchange tape with several causally-DISTINCT faults planted at once (a root,
136
+ a downstream echo, and a two-part necessary-not-sufficient AND-conjunction), scored
137
+ against `blame.py`'s coalition/temporal-Shapley engine (`shapley_rank`) via `tracefork
138
+ bench`. 8/9 planted cases resolve correctly; the 9th is a documented, NOT hidden,
139
+ limitation of single-ordering temporal Shapley (it under-credits the earlier half of a
140
+ symmetric conjunction) — see `competing_faults.py`'s module docstring and README →
141
+ Validation scope. Cites, but does not reproduce, the published Who&When (ICML 2025)
142
+ ~14.2% log-based step-attribution anchor as context only — no external dataset is ever
143
+ downloaded (offline/$0 invariant applies here too). Zero-diff over the engines: both
144
+ modules only call `blame.py`'s existing public API.
145
+ - `report.py` / `server.py` / `web/report.html` — the single-file, dependency-free
146
+ three-panel UI; `report.py` injects tape JSON (HTML-escaped against `</script>`
147
+ breakout), `server.py` is FastAPI same-origin (no CORS, binds 127.0.0.1).
148
+ - `wire.py` / `synthetic.py` — Anthropic wire-format builders and the offline
149
+ Scripted/FaultAware fake transports, in the **package** so production never imports from
150
+ `tests/`; `tests/fakes.py` re-exports them.
151
+ - `replay.py` — `ReplayVerifier` (per-tape) and `run_fixture_corpus_check()`, which extends
152
+ the `validate --check` idea to plain replay: gates a committed tape corpus
153
+ (`experiments/replay_fixtures/` + its `manifest.json`) by asserting both bit-exact replay
154
+ and a `digest()` match per fixture — `tracefork replay --check <dir>`. `fixtures.py` holds
155
+ the tiny deterministic agents the corpus is built from (kept out of `validate.py` so the
156
+ corpus doesn't couple to fault-testing concerns); `scripts/gen_replay_fixtures.py`
157
+ (re)generates the corpus offline.
158
+ - `proxy.py` — `RecordProxy`/`ReplayProxy` (wired into FastAPI apps via
159
+ `build_record_app`/`build_replay_app`) are a **localhost base-URL record/replay proxy**
160
+ for clients the in-process httpx seam can't reach (curl, Node, Go, non-wrapped Python):
161
+ point the client's `base_url` at `http://127.0.0.1:<port>` instead of the provider.
162
+ Record forwards to a real (or, in tests, injected-fake) upstream and tees request+response
163
+ bytes into a `Tape`, streaming SSE chunk-by-chunk while forwarding; replay serves recorded
164
+ bytes with **no upstream**, matching each request to its recorded exchange by
165
+ `matcher.py`'s existing `RequestMatcher` fingerprint (an unrecorded request, or a real
166
+ body change, is a hard HTTP 502). Reuses `tape.py`/`matcher.py` unchanged — no in-process
167
+ `NondetSource` exists on this path, so a proxy-recorded tape (`Tape.boundary =
168
+ constants.PROXY_BOUNDARY`) sits outside the full single-process determinism boundary; see
169
+ the module docstring and README → Localhost record/replay proxy.
170
+ - `adapters/` — opt-in framework adapters (`bind()` routes a framework's LLM client
171
+ through the *existing* `TraceforkTransport`/`NondetSource`; `on_step()` maps a
172
+ framework callback/event to a neutral `Step`/`StepDAG` overlay — observer-only,
173
+ never a second capture path). `base.py` owns the protocol/registry;
174
+ `langchain.py` (`ChatOpenAI`/`ChatAnthropic` + a tape-backed LangGraph
175
+ checkpointer), `openai_agents.py` (defensive client-attribute injection +
176
+ `agents.set_default_openai_client()` + a `TracingProcessor`),
177
+ `crewai.py` (LiteLLM's `client_session`/`aclient_session` — CrewAI's actual httpx
178
+ chokepoint — + a `crewai_event_bus` listener), and `autogen.py` (defensive
179
+ client-attribute injection + an `InterventionHandler` message-level seam) are the
180
+ concrete adapters, each its own optional extra with every framework import
181
+ guarded, so `import tracefork` and the whole offline suite work with none of
182
+ them installed. Where a framework's exact internal attribute names/event
183
+ shapes aren't a documented stable API, injection is defensive (a short
184
+ candidate list, never one hard-coded name) — see each module's docstring.
185
+ - `cli.py` — Typer entry point for all eleven commands.
186
+
187
+ `src/tracefork_spike/` holds the original Spike 0 (`fake_llm.py`, `agent.py`, `spike.py`):
188
+ record → save → load → replay → verify + negative control, with its own tests.
189
+
190
+ Most test files prove ONE module (or one seam) in isolation. Two are deliberately
191
+ cross-module, added once every feature bead had merged: `tests/test_e2e.py` chains
192
+ record → `TapeStore` save/load → replay → fork → blame → validate through the SAME
193
+ tape at every stage (not fresh fixtures per stage), plus the negative control,
194
+ asyncio-concurrency determinism, and every cross-feature path (MCP/tool exchanges
195
+ sharing a tape with LLM exchanges, redaction, OTel/OpenInference export→ingest,
196
+ plugin-registry resolution, `BoundaryGuard`, `divergence.py` diagnostics, the
197
+ base-URL proxy, `bench`, and the Bedrock/OpenAI/Gemini provider seams with their
198
+ documented scope boundaries called out explicitly, not papered over) — all
199
+ offline/$0. `tests/test_cli_smoke.py` invokes every one of the eleven CLI
200
+ subcommands and asserts its real exit code; `serve`/`proxy record`/`proxy replay`
201
+ call `uvicorn.run()` directly, so those are driven by monkeypatching `uvicorn.run`
202
+ to a no-op (proving the CLI's own wiring without binding a socket) plus a
203
+ `TestClient`/ASGI-transport hit against the underlying FastAPI app for actual
204
+ serving behavior. `scripts/e2e.sh` runs the whole gate — sync, lint, format,
205
+ mypy, tests+coverage, `validate --check`, `replay --check`, `bench`, build+twine
206
+ — as one script with a single PASS/FAIL verdict. Both test files are additive
207
+ only: zero-diff over `transport.py`/`tape.py`/`fork.py`/`blame.py`/`matcher.py`.
208
+
209
+ ## Invariants / conventions
210
+
211
+ - **Offline and $0 is non-negotiable** for the whole test suite, the spike, `validate`,
212
+ and the demo — no key, no network. The synthetic transports (`synthetic.py`) are the
213
+ seam; add to them rather than reaching for the real API. (`blame` on a real run is the
214
+ one budget-capped exception.)
215
+ - **The agent must read time/ids/random only through `NondetSource`** — any direct
216
+ `datetime.now()` / `uuid` / `random` breaks the determinism boundary and the
217
+ bit-exactness claim. `BoundaryGuard` (opt-in, default off) turns a subset of these
218
+ violations (thread/subprocess spawn, direct `random`/`time.monotonic`/`time.sleep`) into
219
+ a loud record-time error instead of a mysterious later replay failure — see
220
+ `boundary_guard.py` for exactly which calls it can and can't intercept.
221
+ - **The verifier proves, not asserts** — every request body is hash-checked against the
222
+ tape; the negative control must keep failing (drift detected) or the proof is vacuous.
223
+ - **Declared determinism boundary (v1):** single-process (sync **or** asyncio), clock +
224
+ id nondeterminism captured through `NondetSource`, **plus concurrency-graph determinism**
225
+ — the completion order of concurrent asyncio fan-out is recorded and re-imposed on replay
226
+ (see `transport.py`), so `gather`/`TaskGroup` agents replay bit-exact, not just
227
+ single-call-at-a-time ones. Threads/subprocess are out of scope; fork and blame
228
+ additionally assume the agent rebuilds its prefix deterministically (the property replay
229
+ proves) — see `SPIKE0.md`.
230
+ - **No `Co-Authored-By: Claude` trailer** on commits in this repo (public portfolio repo,
231
+ sole-author attribution).
232
+ - **Model IDs / pricing / SDK usage:** consult the `claude-api` skill before writing or
233
+ editing any Anthropic integration code rather than relying on memory.
234
+ - `docs/superpowers/`, `.beads/`, `planning/` are gitignored local scaffolding (but
235
+ `docs/demo.png` is committed). Runtime artifacts (`store.db`, `report.html`,
236
+ `blame_*.json`, `validation_report.json`, `examples/demo_report.html`) are gitignored.