tracefork 0.1.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. {tracefork-0.1.0 → tracefork-0.2.1}/.github/workflows/ci.yml +2 -2
  2. {tracefork-0.1.0 → tracefork-0.2.1}/.gitignore +5 -0
  3. tracefork-0.2.1/CHANGELOG.md +163 -0
  4. tracefork-0.2.1/CLAUDE.md +241 -0
  5. tracefork-0.2.1/PKG-INFO +636 -0
  6. tracefork-0.2.1/README.md +566 -0
  7. tracefork-0.2.1/docs/packaging-decisions.md +170 -0
  8. tracefork-0.2.1/experiments/replay_fixtures/manifest.json +14 -0
  9. tracefork-0.2.1/experiments/replay_fixtures/single_turn.tape.sqlite +0 -0
  10. tracefork-0.2.1/experiments/replay_fixtures/two_turn.tape.sqlite +0 -0
  11. tracefork-0.2.1/pyproject.toml +197 -0
  12. tracefork-0.2.1/scripts/e2e.sh +47 -0
  13. tracefork-0.2.1/scripts/gen_replay_fixtures.py +91 -0
  14. tracefork-0.2.1/src/tracefork/__init__.py +146 -0
  15. tracefork-0.2.1/src/tracefork/adapters/__init__.py +134 -0
  16. tracefork-0.2.1/src/tracefork/adapters/adk.py +465 -0
  17. tracefork-0.2.1/src/tracefork/adapters/autogen.py +321 -0
  18. tracefork-0.2.1/src/tracefork/adapters/base.py +377 -0
  19. tracefork-0.2.1/src/tracefork/adapters/crewai.py +367 -0
  20. tracefork-0.2.1/src/tracefork/adapters/langchain.py +732 -0
  21. tracefork-0.2.1/src/tracefork/adapters/openai_agents.py +408 -0
  22. tracefork-0.2.1/src/tracefork/bedrock_transport.py +297 -0
  23. tracefork-0.2.1/src/tracefork/bench.py +152 -0
  24. tracefork-0.2.1/src/tracefork/blame.py +917 -0
  25. tracefork-0.2.1/src/tracefork/boundary_guard.py +153 -0
  26. tracefork-0.2.1/src/tracefork/cli.py +766 -0
  27. tracefork-0.2.1/src/tracefork/competing_faults.py +260 -0
  28. tracefork-0.2.1/src/tracefork/config.py +123 -0
  29. tracefork-0.2.1/src/tracefork/constants.py +70 -0
  30. tracefork-0.2.1/src/tracefork/data/pricing.json +38 -0
  31. tracefork-0.2.1/src/tracefork/divergence.py +198 -0
  32. tracefork-0.2.1/src/tracefork/eventstream.py +149 -0
  33. tracefork-0.2.1/src/tracefork/faults.py +168 -0
  34. tracefork-0.2.1/src/tracefork/fixtures.py +53 -0
  35. tracefork-0.2.1/src/tracefork/fork.py +370 -0
  36. tracefork-0.2.1/src/tracefork/interop.py +472 -0
  37. tracefork-0.2.1/src/tracefork/judge.py +390 -0
  38. tracefork-0.2.1/src/tracefork/matcher.py +281 -0
  39. tracefork-0.2.1/src/tracefork/mcp_client.py +98 -0
  40. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork/nondet.py +25 -2
  41. tracefork-0.2.1/src/tracefork/observability.py +195 -0
  42. tracefork-0.2.1/src/tracefork/plugins.py +120 -0
  43. tracefork-0.2.1/src/tracefork/pricing.py +108 -0
  44. tracefork-0.2.1/src/tracefork/providers/__init__.py +41 -0
  45. tracefork-0.2.1/src/tracefork/providers/anthropic.py +210 -0
  46. tracefork-0.2.1/src/tracefork/providers/base.py +173 -0
  47. tracefork-0.2.1/src/tracefork/providers/bedrock.py +216 -0
  48. tracefork-0.2.1/src/tracefork/providers/gemini.py +232 -0
  49. tracefork-0.2.1/src/tracefork/providers/openai.py +266 -0
  50. tracefork-0.2.1/src/tracefork/proxy.py +280 -0
  51. tracefork-0.2.1/src/tracefork/record_mode.py +75 -0
  52. tracefork-0.2.1/src/tracefork/recorder.py +270 -0
  53. tracefork-0.2.1/src/tracefork/redact.py +368 -0
  54. tracefork-0.2.1/src/tracefork/replay.py +273 -0
  55. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork/report.py +17 -11
  56. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork/store.py +70 -18
  57. tracefork-0.2.1/src/tracefork/synthetic.py +249 -0
  58. tracefork-0.2.1/src/tracefork/tape.py +472 -0
  59. tracefork-0.2.1/src/tracefork/tools.py +328 -0
  60. tracefork-0.2.1/src/tracefork/transport.py +302 -0
  61. tracefork-0.2.1/src/tracefork/wire.py +51 -0
  62. {tracefork-0.1.0 → tracefork-0.2.1}/tests/fakes.py +2 -0
  63. tracefork-0.2.1/tests/fixtures/legacy_tape_v1.blob +1 -0
  64. tracefork-0.2.1/tests/test_adapters_adk.py +305 -0
  65. tracefork-0.2.1/tests/test_adapters_autogen.py +222 -0
  66. tracefork-0.2.1/tests/test_adapters_base.py +232 -0
  67. tracefork-0.2.1/tests/test_adapters_crewai.py +223 -0
  68. tracefork-0.2.1/tests/test_adapters_langchain.py +373 -0
  69. tracefork-0.2.1/tests/test_adapters_openai_agents.py +253 -0
  70. tracefork-0.2.1/tests/test_bedrock_eventstream.py +98 -0
  71. tracefork-0.2.1/tests/test_bedrock_provider.py +163 -0
  72. tracefork-0.2.1/tests/test_bedrock_transport.py +268 -0
  73. tracefork-0.2.1/tests/test_bench.py +62 -0
  74. tracefork-0.2.1/tests/test_blame.py +571 -0
  75. tracefork-0.2.1/tests/test_boundary_guard.py +189 -0
  76. tracefork-0.2.1/tests/test_cli.py +313 -0
  77. tracefork-0.2.1/tests/test_cli_smoke.py +360 -0
  78. tracefork-0.2.1/tests/test_competing_faults.py +184 -0
  79. tracefork-0.2.1/tests/test_concurrency.py +324 -0
  80. tracefork-0.2.1/tests/test_config.py +140 -0
  81. tracefork-0.2.1/tests/test_divergence.py +194 -0
  82. tracefork-0.2.1/tests/test_e2e.py +709 -0
  83. tracefork-0.2.1/tests/test_fork.py +313 -0
  84. tracefork-0.2.1/tests/test_gemini_provider.py +170 -0
  85. tracefork-0.2.1/tests/test_interop.py +305 -0
  86. tracefork-0.2.1/tests/test_judge.py +465 -0
  87. tracefork-0.2.1/tests/test_matcher.py +275 -0
  88. tracefork-0.2.1/tests/test_mcp_client.py +57 -0
  89. tracefork-0.2.1/tests/test_nondet.py +157 -0
  90. tracefork-0.2.1/tests/test_observability.py +123 -0
  91. tracefork-0.2.1/tests/test_openai_provider.py +192 -0
  92. tracefork-0.2.1/tests/test_plugins.py +223 -0
  93. tracefork-0.2.1/tests/test_pricing.py +162 -0
  94. tracefork-0.2.1/tests/test_provider_faults.py +131 -0
  95. tracefork-0.2.1/tests/test_providers.py +310 -0
  96. tracefork-0.2.1/tests/test_proxy.py +221 -0
  97. tracefork-0.2.1/tests/test_record_mode.py +53 -0
  98. tracefork-0.2.1/tests/test_redact.py +434 -0
  99. tracefork-0.2.1/tests/test_replay.py +259 -0
  100. tracefork-0.2.1/tests/test_report.py +237 -0
  101. tracefork-0.2.1/tests/test_storage.py +248 -0
  102. tracefork-0.2.1/tests/test_tools.py +333 -0
  103. tracefork-0.2.1/uv.lock +4606 -0
  104. {tracefork-0.1.0 → tracefork-0.2.1}/web/report.html +101 -2
  105. tracefork-0.1.0/CHANGELOG.md +0 -45
  106. tracefork-0.1.0/CLAUDE.md +0 -112
  107. tracefork-0.1.0/PKG-INFO +0 -235
  108. tracefork-0.1.0/README.md +0 -198
  109. tracefork-0.1.0/pyproject.toml +0 -94
  110. tracefork-0.1.0/src/tracefork/__init__.py +0 -6
  111. tracefork-0.1.0/src/tracefork/blame.py +0 -296
  112. tracefork-0.1.0/src/tracefork/cli.py +0 -367
  113. tracefork-0.1.0/src/tracefork/constants.py +0 -24
  114. tracefork-0.1.0/src/tracefork/faults.py +0 -129
  115. tracefork-0.1.0/src/tracefork/fork.py +0 -173
  116. tracefork-0.1.0/src/tracefork/recorder.py +0 -140
  117. tracefork-0.1.0/src/tracefork/replay.py +0 -119
  118. tracefork-0.1.0/src/tracefork/synthetic.py +0 -104
  119. tracefork-0.1.0/src/tracefork/tape.py +0 -135
  120. tracefork-0.1.0/src/tracefork/transport.py +0 -137
  121. tracefork-0.1.0/src/tracefork/wire.py +0 -76
  122. tracefork-0.1.0/tests/test_blame.py +0 -175
  123. tracefork-0.1.0/tests/test_cli.py +0 -81
  124. tracefork-0.1.0/tests/test_fork.py +0 -118
  125. tracefork-0.1.0/tests/test_replay.py +0 -124
  126. tracefork-0.1.0/tests/test_report.py +0 -116
  127. tracefork-0.1.0/uv.lock +0 -874
  128. {tracefork-0.1.0 → tracefork-0.2.1}/.editorconfig +0 -0
  129. {tracefork-0.1.0 → tracefork-0.2.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  130. {tracefork-0.1.0 → tracefork-0.2.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  131. {tracefork-0.1.0 → tracefork-0.2.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  132. {tracefork-0.1.0 → tracefork-0.2.1}/.github/dependabot.yml +0 -0
  133. {tracefork-0.1.0 → tracefork-0.2.1}/.github/pull_request_template.md +0 -0
  134. {tracefork-0.1.0 → tracefork-0.2.1}/.github/workflows/release.yml +0 -0
  135. {tracefork-0.1.0 → tracefork-0.2.1}/.pre-commit-config.yaml +0 -0
  136. {tracefork-0.1.0 → tracefork-0.2.1}/CODE_OF_CONDUCT.md +0 -0
  137. {tracefork-0.1.0 → tracefork-0.2.1}/CONTRIBUTING.md +0 -0
  138. {tracefork-0.1.0 → tracefork-0.2.1}/LICENSE +0 -0
  139. {tracefork-0.1.0 → tracefork-0.2.1}/SECURITY.md +0 -0
  140. {tracefork-0.1.0 → tracefork-0.2.1}/SPIKE0.md +0 -0
  141. {tracefork-0.1.0 → tracefork-0.2.1}/docs/demo.png +0 -0
  142. {tracefork-0.1.0 → tracefork-0.2.1}/examples/demo_report.py +0 -0
  143. {tracefork-0.1.0 → tracefork-0.2.1}/experiments/validation_report_committed.json +0 -0
  144. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork/py.typed +0 -0
  145. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork/server.py +0 -0
  146. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork/validate.py +0 -0
  147. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/__init__.py +0 -0
  148. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/__main__.py +0 -0
  149. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/agent.py +0 -0
  150. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/fake_llm.py +0 -0
  151. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/nondet.py +0 -0
  152. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/spike.py +0 -0
  153. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/tape.py +0 -0
  154. {tracefork-0.1.0 → tracefork-0.2.1}/src/tracefork_spike/transport.py +0 -0
  155. {tracefork-0.1.0 → tracefork-0.2.1}/tests/__init__.py +0 -0
  156. {tracefork-0.1.0 → tracefork-0.2.1}/tests/test_faults.py +0 -0
  157. {tracefork-0.1.0 → tracefork-0.2.1}/tests/test_recorder.py +0 -0
  158. {tracefork-0.1.0 → tracefork-0.2.1}/tests/test_spike0.py +0 -0
  159. {tracefork-0.1.0 → tracefork-0.2.1}/tests/test_tape.py +0 -0
  160. {tracefork-0.1.0 → tracefork-0.2.1}/tests/test_transport.py +0 -0
@@ -13,9 +13,9 @@ jobs:
13
13
  matrix:
14
14
  python-version: ["3.12", "3.13"]
15
15
  steps:
16
- - uses: actions/checkout@v4
16
+ - uses: actions/checkout@v7
17
17
  - name: Install uv
18
- uses: astral-sh/setup-uv@v3
18
+ uses: astral-sh/setup-uv@v7
19
19
  with:
20
20
  python-version: ${{ matrix.python-version }}
21
21
  - name: Install dependencies
@@ -9,11 +9,16 @@ __pycache__/
9
9
  *.tape.sqlite
10
10
  /tmp_tapes/
11
11
 
12
+ # Exception: the committed replay-fixture corpus (replay --check regression
13
+ # gate) — these tapes are checked in, not regenerated per-run.
14
+ !/experiments/replay_fixtures/*.tape.sqlite
15
+
12
16
  # Runtime artifacts (CLI output, regenerated on demand)
13
17
  /store.db
14
18
  /report.html
15
19
  /blame_*.json
16
20
  /validation_report.json
21
+ /bench_report.json
17
22
  /examples/demo_report.html
18
23
 
19
24
  # Build, coverage, and tool caches (regenerated on demand)
@@ -0,0 +1,163 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.2.1] - 2026-07-03
11
+
12
+ ### Added
13
+
14
+ - **Google ADK adapter** (`adapters/adk.py`, opt-in `adk` extra) — `bind()` routes
15
+ an ADK `LlmAgent`/`Gemini`'s underlying `google-genai` client through the
16
+ existing `TraceforkTransport` via a short candidate-path search for the
17
+ `google.genai` `BaseApiClient`'s private `_httpx_client`/`_async_httpx_client`
18
+ attributes (the target itself, a `genai.Client`, an ADK `Gemini` model wrapper,
19
+ or an `LlmAgent` whose `.model` already holds one) — the same Gemini
20
+ `generateContent` wire format `providers/gemini.py` already parses. Step
21
+ visibility is a real `BasePlugin` (`make_plugin()`) over ADK's documented
22
+ agent/model/tool before/after callback boundaries, registered once on the
23
+ `Runner` rather than threaded through every agent — observer-only, never a
24
+ second capture path.
25
+ - **Curated `all` extra** (`pip install 'tracefork[all]'`) — a self-referential
26
+ convenience bundle over `providers` + `bedrock` + `mcp` + `observability` (the
27
+ internally-consistent, stable-wire family). Deliberately excludes the five
28
+ independently-capped, fast-moving framework stacks (`frameworks`,
29
+ `openai-agents`, `crewai`, `autogen`, `adk`) so one future cap collision on any
30
+ single framework can't `ResolutionImpossible` the whole `all` install.
31
+
32
+ ### Changed
33
+
34
+ - **Relaxed the `frameworks` extra's version caps to floors** — dropped the
35
+ speculative `<2` upper bounds on `langchain-core`/`langchain-openai`/
36
+ `langchain-anthropic`/`langgraph`. LangChain 1.0 GA commits to no breaking
37
+ changes until 2.0, so a `<2` cap was speculative and, more importantly, useless
38
+ against the real observed failure mode: intra-1.x patch regressions ship
39
+ *inside* the allowed range regardless of the cap. The remaining internal-
40
+ coupling caps (`openai-agents`, `crewai`, `autogen`, `adk` — each an adapter
41
+ injecting into private/undocumented framework internals) are kept, each now
42
+ with an inline "hint, not a guard" rationale and a revisit TODO.
43
+ - **Adapter import guards now chain the root cause** (`raise ImportError(HINT)
44
+ from exc`) in `adapters/{langchain,openai_agents,crewai,autogen,adk}.py`'s
45
+ `require_*()` functions, so an installed-but-broken dependency's real
46
+ `ImportError` is preserved instead of being masked as "not installed".
47
+
48
+ ## [0.2.0] - 2026-07-02
49
+
50
+ Generic, multi-provider, production-hardening release. The bit-exact, $0,
51
+ hash-verified replay substrate from 0.1.0 is the contract and is unchanged —
52
+ everything below is additive around it: all engine internals stay byte-stable,
53
+ `digest()` is format-stable, and every existing tape still loads and replays.
54
+
55
+ ### Added
56
+
57
+ - **Provider abstraction seam** (`providers/`): a `ProviderAdapter` protocol + registry
58
+ and a `gen_ai.*`-style `NormalizedResponse` view. Anthropic is now a *registered*
59
+ adapter, not a hardcoded assumption; raw request/response **bytes** stay the immutable
60
+ replay + hash contract. (#7)
61
+ - **More provider backends** — **OpenAI** and **Google Gemini** (httpx-based, captured
62
+ through the existing transport seam, with provider-generic fault injection), and **AWS
63
+ Bedrock** (a separate botocore `before-send` seam, a stdlib `vnd.amazon.eventstream`
64
+ frame codec, and SigV4-canonical request matching so a fresh signature/timestamp is not
65
+ a false divergence). (#12, #21)
66
+ - **Pluggable canonicalization / divergence matcher** (`matcher.py`): identity +
67
+ canonicalizing matchers so volatile material (Gemini `?key=`, Bedrock `x-amz-date`,
68
+ rotating auth) hashes equal while a real request-body change still diverges. (#9)
69
+ - **Plugin & extension architecture** (`plugins.py`): entry-point registries for
70
+ providers, matchers, storage backends, transports, oracles, and adapters — each gated by
71
+ the same explicit allowlist. (#11)
72
+ - **MCP + native tool-call record/replay** (`tools.py`, `mcp_client.py`): a JSON-RPC tee so
73
+ tool exchanges share the tape with LLM exchanges and replay bit-exact. (#14)
74
+ - **Framework adapters** (`adapters/`, opt-in extras): a minimal
75
+ `bind()`/`on_step()`/`teardown()` protocol + a `StepDAG` normalizer, with
76
+ **LangChain/LangGraph** (including a tape-backed LangGraph checkpointer) plus **OpenAI
77
+ Agents SDK**, **CrewAI**, and **AutoGen** adapters — each binding at the framework's real
78
+ model-call chokepoint (never a second capture path) and each guarded so `import
79
+ tracefork` and the full offline suite run with none installed. (#16, #23)
80
+ - **OTel GenAI / OpenInference interop** (`interop.py`, `tracefork export`/`ingest`): export
81
+ a tape (+ blame report) as OTel GenAI spans or an OpenInference dataset, and ingest either
82
+ back into a tape's step structure for **blame-by-re-execution** (explicitly *not* $0
83
+ bit-exact replay — an ingested tape diverges on `replay`/`fork` by design). Plain JSON, no
84
+ `opentelemetry-sdk` required. Plus an **opt-in `observability` extra** (structlog + OTel
85
+ self-instrumentation of record/replay/fork/blame), off by default. (#15)
86
+ - **Deterministic asyncio concurrency replay** (`transport.py`): records the completion
87
+ order of concurrent fan-out (`gather`/`TaskGroup`) and re-imposes it on replay, so
88
+ concurrent agents replay bit-exact — not just single-call-at-a-time ones. (#17)
89
+ - **Causal blame depth**: coalition / temporal-Shapley attribution with necessity +
90
+ sufficiency, and a long-tape competing-fault benchmark (`tracefork bench`) with Wilson
91
+ CIs. The one temporal-order Shapley limitation is documented, not hidden. (#18, #24)
92
+ - **Oracle rigor** (`judge.py`, opt-in): an LLM-judge oracle with position-swap averaging +
93
+ a self-judge guard, gold-set calibration (FPR/FNR/Cohen's kappa), and Rogan-Gladen
94
+ flip-rate debiasing — all offline-testable via an injected judge function. (#19)
95
+ - **Divergence diagnostics & debug UX** (`divergence.py`, web report): a structured
96
+ divergence diff, a report rewind panel, blame trust flags, and real-vs-tolerated
97
+ divergence messaging. (#20)
98
+ - **Localhost record/replay proxy** (`tracefork proxy`): a base-URL MITM-style proxy (binds
99
+ 127.0.0.1) for non-Python / non-httpx clients (curl, Node, Go), reusing the tape +
100
+ matcher. Record/replay only — outside the in-process `NondetSource` determinism
101
+ boundary. (#22)
102
+ - **Hardening seams**: an opt-in record-time `BoundaryGuard` (thread/subprocess spawn,
103
+ direct `random`/`time` bypasses), `random_float()` virtualization through `NondetSource`,
104
+ a `replay --check` fixture-corpus regression gate, and opt-in secret/PII redaction on
105
+ record. (#10, #13)
106
+ - **CLI**: new `bench`, `export`, `ingest`, and `proxy` commands (joining
107
+ `replay`/`verify`/`fork`/`report`/`serve`/`blame`/`validate`), plus a full end-to-end
108
+ integration suite and a single `scripts/e2e.sh` receipt. (#25)
109
+
110
+ ### Changed
111
+
112
+ - **Versioned tape envelope** (`tape.py`): a `TAPE_MAGIC` + uint16 version header over a
113
+ zstd + content-addressed binary container (no base64, no pickle), with a read-time
114
+ upcaster chain (v1→v4). The version header is **not** part of `digest()`, so the hash
115
+ chain is byte-stable across format versions and legacy tapes still load and replay. (#6)
116
+ - **Blame statistics**: three-valued trials (flip / no-flip / undefined) and pluggable CI
117
+ methods. (#8)
118
+ - **SQLite persistence hardened**: WAL, `synchronous=NORMAL`, `busy_timeout`,
119
+ `foreign_keys=ON`, and writers take `BEGIN IMMEDIATE`. (#6)
120
+
121
+ ### Fixed
122
+
123
+ - Wilson CI boundaries snap to analytically-exact 0/1 to avoid ~1e-17 platform float-dust
124
+ that differed across CI runners. (#8)
125
+
126
+ ## [0.1.0] - 2026-07-02
127
+
128
+ ### Added
129
+
130
+ - **Record/replay** at the Anthropic SDK's httpx transport boundary
131
+ (`TraceforkTransport` / `AsyncTraceforkTransport`), streaming-SSE capable, with
132
+ bit-exact replay proven by sha256-checking every replayed request body against the
133
+ recorded tape — and drift detection that fails loudly on divergence rather than
134
+ silently falling back to the network.
135
+ - **Content-addressed tape format** (`Tape`) — sha256 blobs plus an ordered event log,
136
+ JSON + base64 (never pickle), persistable to SQLite, with a hash-chain `digest()`
137
+ fingerprint.
138
+ - **Nondeterminism virtualization** (`NondetSource`) — the only path through which an
139
+ agent reads time and ids, with `RecordingNondet`, `ReplayNondet`, and a `DriftingNondet`
140
+ negative control that proves the divergence detector actually detects divergence.
141
+ - **Three-phase fork engine** (`ForkEngine`, `ForkTransport`) — prefix-replay ($0),
142
+ mutation-injection (swap a response), and tail-record (the recorded counterfactual
143
+ continuation), re-running the same agent that produced the original tape.
144
+ - **Causal blame engine** (`BlameEngine`) — forks each step `k` times, re-runs the agent,
145
+ grades outcomes via an `Oracle`, and ranks steps by flip-rate with Wilson score
146
+ confidence intervals; a `BudgetGovernor` estimates dollar cost from the pricing table
147
+ and refuses to exceed a caller-supplied budget before making any real API calls.
148
+ - **Fault-injection self-validation suite** (`faults.py`, `validate.py`) — five fault
149
+ classes with markers embedded in valid Anthropic JSON, scored end-to-end offline
150
+ against a synthetic fault-aware agent: **1.00 top-1 precision** across all five classes,
151
+ with an enforced negative-control threshold so the proof isn't vacuous.
152
+ - **Single-file web report/UI** (`report.py`, `server.py`, `web/report.html`) — a
153
+ dependency-free, three-panel HTML report (timeline, exchange detail, blame ranking)
154
+ either rendered statically or served live via FastAPI (`serve`, 127.0.0.1, no CORS).
155
+ - **CLI** (`cli.py`, Typer) — `replay`, `verify`, `fork`, `blame`, `report`, `serve`,
156
+ `validate`.
157
+ - `src/tracefork_spike/` — the original Spike 0 that de-risked bit-exact, no-key replay
158
+ within the declared determinism boundary.
159
+
160
+ [Unreleased]: https://github.com/pratik916/tracefork/compare/v0.2.1...HEAD
161
+ [0.2.1]: https://github.com/pratik916/tracefork/compare/v0.2.0...v0.2.1
162
+ [0.2.0]: https://github.com/pratik916/tracefork/compare/v0.1.0...v0.2.0
163
+ [0.1.0]: https://github.com/pratik916/tracefork/releases/tag/v0.1.0
@@ -0,0 +1,241 @@
1
+ # CLAUDE.md
2
+
3
+ This file guides Claude Code when working in the `tracefork` repository.
4
+
5
+ ## What this is
6
+
7
+ `tracefork` is a time-travel debugger for AI agents: record an agent run to a
8
+ content-addressed **tape**, replay it **bit-exact for $0** (hash-verified), fork any
9
+ step, and measure causal blame with confidence intervals — the instrument itself
10
+ validated against runs with injected, known root-cause faults.
11
+
12
+ **Current state: v1 built.** All five product pillars work offline and are tested
13
+ (672 tests, $0): streaming-capable record/replay with drift detection, the three-phase
14
+ fork engine, the causal blame engine with Wilson CIs and a budget governor, the
15
+ single-file web report/UI, and the fault-injection self-validation suite (5 fault
16
+ classes at 1.00 top-1 precision, plus a longer competing-fault fixture that measures
17
+ whether the coalition/temporal-Shapley engine discriminates *several* simultaneously
18
+ planted causes — `tracefork bench`; see README → Validation scope for exactly what each
19
+ number does and doesn't claim). `src/tracefork_spike/` keeps the original Spike 0 that
20
+ de-risked the load-bearing assumption (bit-exact, no-key replay within a declared
21
+ determinism boundary). Design/feature list: `../ideas/2026-06-11-tracefork-features.md`;
22
+ spike finding: `SPIKE0.md`.
23
+
24
+ ## Commands
25
+
26
+ Python is **3.12 via uv**. The tests, the spike, `validate`, the demo, and
27
+ record/replay/fork are offline and $0 — **no `ANTHROPIC_API_KEY`, no network**. Only
28
+ `blame` against a *real* run hits the live API (budget-capped). Always prefix `uv run`.
29
+
30
+ ```bash
31
+ uv sync --extra dev # install (anthropic, zstandard, typer, fastapi, uvicorn + pytest)
32
+ uv run pytest -q # full offline suite (672 tests)
33
+ uv run pytest tests/test_faults.py::test_validation_runner_fingers_fault_step -q # one test
34
+ uv run tracefork validate # self-validation: blame vs injected, known faults
35
+ uv run tracefork validate --check # regression-gate vs experiments/validation_report_committed.json
36
+ uv run tracefork bench # long-tape competing-fault discrimination benchmark
37
+ uv run python examples/demo_report.py # write examples/demo_report.html (the README screenshot)
38
+ uv run python -m tracefork_spike # the original Spike 0 bit-exact replay receipt
39
+ uv run tracefork --help # replay, verify, fork, report, serve, blame, validate, bench, proxy
40
+ uv run tracefork replay --check experiments/replay_fixtures # replay-as-regression gate
41
+ bash scripts/e2e.sh # single-receipt gate: sync, lint, type-check, tests+coverage,
42
+ # validate --check, replay --check, bench, build+twine, one PASS banner
43
+ ```
44
+
45
+ ## Architecture (the parts that span files)
46
+
47
+ The spine is a **record/replay seam at the Anthropic SDK's httpx boundary**, plus a
48
+ **nondeterminism-virtualization seam** the agent reads time/ids through. Bit-exactness
49
+ is the contract between them.
50
+
51
+ The product lives in `src/tracefork/`:
52
+
53
+ - `nondet.py` — `NondetSource` is the *only* way the agent gets time/ids/random draws
54
+ (`now_iso`/`new_uuid_hex`/`random_float`). `RecordingNondet` draws real values and logs
55
+ them; `ReplayNondet` serves them back in order; `DriftingNondet` is the negative control
56
+ (fresh values → forced divergence). `random_float()` logs the exact `float.hex()` string
57
+ (lossless round-trip, no float-formatting dust) and, like `now_iso()`, is additive/opt-in
58
+ — an agent must be handed the active `NondetSource` explicitly; nothing patches `random`
59
+ globally the way `Recorder` patches `uuid.uuid4`. `find_divergence()` unwraps a
60
+ `DivergenceError` from the `APIConnectionError` the SDK wraps transport exceptions in —
61
+ **keep this; without it a real divergence looks like a network blip.**
62
+ - `boundary_guard.py` — `BoundaryGuard`, an **opt-in** (default off) record-mode guard:
63
+ hard-errors on `threading.Thread.start`/`subprocess.Popen` (crossing the single-process
64
+ boundary) and direct `random.random`/`time.monotonic`/`time.sleep` (bypassing
65
+ `NondetSource`) instead of letting the tape fail replay later, mysteriously. Deliberately
66
+ does **not** guard `datetime.datetime.now()` (same immutable-C-type reason `recorder.py`
67
+ doesn't patch it) or `time.time()` (httpx's cookie-jar machinery calls it on every
68
+ response — guarding it would false-positive on every exchange). Pre-warms the Anthropic
69
+ SDK's `platform_headers()` `lru_cache` on `__enter__` so its one internal
70
+ `subprocess.Popen` call (uncached platform detection) doesn't trip the guard on the first
71
+ real API call. Wired into `Recorder`/`AsyncRecorder` via `boundary_guard=` (tri-state:
72
+ explicit wins over `TraceforkConfig.boundary_guard`, both default `False`).
73
+ - `transport.py` — `TraceforkTransport` (sync) + `AsyncTraceforkTransport` (async) are the
74
+ capture seam, streaming-SSE capable (buffer via `.read()`/`.aread()`). Record mode tees
75
+ request+response bytes into the tape; replay mode serves recorded bytes and
76
+ sha256-asserts each request body matches (the divergence detector). A replay transport
77
+ has **no inner transport**, so any unrecorded request is a hard error. **Async
78
+ concurrency:** the async transport records the completion order of concurrent fan-out
79
+ (`asyncio.gather`/`TaskGroup`) — appending exchanges at completion and logging each
80
+ fully-overlapping batch to `tape.async_batches` — and on replay **correlates each request
81
+ to its recorded exchange by fingerprint (not positional arrival) and releases responses in
82
+ the recorded completion order** via an ordered gate, so a fan-out agent replays bit-exact.
83
+ A strictly-sequential async run never waits at the gate → byte-identical to before; the
84
+ sync transport is untouched (stays positional). `chaos_release_order(tape, seed)` derives a
85
+ seeded, physically-possible reordering of a recorded schedule (chaos-mode replay) for
86
+ race/ordering-bug analysis; **do NOT** add awaits/ordering to the sequential or sync path.
87
+ - `tape.py` — `Tape` is content-addressed (sha256 blobs) + an ordered event log,
88
+ persistable to SQLite, with a hash-chain `digest()` fingerprint. `to_bytes`/`from_bytes`
89
+ emit a **versioned envelope** (`TAPE_MAGIC` + uint16 version, then a zstd + content-
90
+ addressed binary container — no base64); `from_bytes` dispatches on the version through a
91
+ read-time upcaster chain and still loads legacy header-less JSON blobs as v1. The version
92
+ header is envelope metadata, **not** part of `digest()`, so the hash chain is byte-stable
93
+ across format versions. **v4** adds the `async_batches` concurrency-batch log (recorded
94
+ completion order of fully-overlapping fan-out); like `boundary`/`agent_name` it is
95
+ persisted but **never** fed into `digest()` (the completion order is already fingerprinted
96
+ by the `exchanges` list ordering), so every existing and every sequential/sync tape's
97
+ digest is byte-identical and v1/v2/v3 tapes upcast to an empty batch log. It's a JSON
98
+ header + zstd blobs, **not pickle** — no
99
+ arbitrary-code-execution risk. `open_sqlite()` is the one hardened connection factory
100
+ (WAL, `synchronous=NORMAL`, `busy_timeout`, `foreign_keys=ON`); writers take `BEGIN
101
+ IMMEDIATE`. `store.py` reuses it and serializes its write fan-out with a lock.
102
+ - `recorder.py` — `Recorder` context manager wraps a real `anthropic.Anthropic` at its
103
+ `_client._transport` seam (via `client.copy(http_client=...)`, so base_url / auth_token /
104
+ default headers are preserved). Patches `uuid.uuid4` globally; **does not** patch
105
+ `datetime.datetime` (immutable C type in 3.12+, and a subclass breaks the SDK's pydantic
106
+ schema builder) — agents needing deterministic clocks/random read `NondetSource` directly.
107
+ Optionally wraps the recording window in a `BoundaryGuard` (see `boundary_guard.py`).
108
+ - `fork.py` — `ForkTransport` runs three phases: prefix-replay ($0, request asserted to
109
+ match the parent), mutation-injection (same request, swapped response), tail-record (the
110
+ counterfactual continuation). `Branch` carries `prefix_replayed`/`tail_recorded` counts.
111
+ `ForkEngine.fork()` re-runs the **same** agent that produced the tape.
112
+ - `store.py` — `TapeStore`, SQLite persistence for tapes + the branch DAG.
113
+ - `blame.py` — `BlameEngine.rank()` forks each step `k` times, re-runs the agent, grades
114
+ via an `Oracle`, counts flips vs. the parent outcome; `wilson_ci()` for intervals;
115
+ `BudgetGovernor` estimates tail-call cost from `constants.PRICING_TABLE` before spend and
116
+ `rank()` raises `BudgetExceededError` if the estimate exceeds `budget_usd`.
117
+ - `judge.py` — OPT-IN, additive on top of `blame.py`'s `Oracle` protocol (never imported by the
118
+ default $0 path): `LLMJudgeOracle` is a binary-rubric judge with few-shot examples, a
119
+ configurable ("cross-family") judge model with a self-judge guard, position-swap averaging
120
+ (grades twice with the candidate output moved in the prompt; disagreement or low average
121
+ confidence abstains via `None`) — testable OFFLINE via an injected `judge_fn` (prompt -> raw
122
+ text), never a real API call in tests. `calibrate_oracle()` measures any `Oracle`'s FPR/FNR/
123
+ Cohen's kappa against a labeled gold set (`kappa_alert` below 0.6). `rogan_gladen_correct()` /
124
+ `debias_flip_rate()` debias an observed flip-rate for judge FPR/FNR (Rogan-Gladen 1978) and
125
+ widen a step's CI via delta-method propagation of BOTH k-sampling noise and finite-gold-set
126
+ judge noise. Pure math, reuses `blame.z_from_confidence`; registers `"llm_judge"` into
127
+ `blame.ORACLE_REGISTRY` at import time (importing the module is itself the opt-in).
128
+ - `faults.py` / `validate.py` — 5 fault classes (valid JSON, marker **inside** a content
129
+ field) + the self-validation runner; a synthetic agent echoes each response forward so an
130
+ injected fault propagates to a fault-aware tail. `run_all_fault_classes()` scores top-1.
131
+ **Scope (don't overstate):** the fixture is a positive-vs-inert control on a short tape —
132
+ it proves the engine is genuinely causal (not a fixed-slot artifact), not that it
133
+ discriminates among competing causes on long tapes. See README → Validation scope.
134
+ - `competing_faults.py` / `bench.py` — the longer-tape ANSWER to that scope note: a
135
+ 7-exchange tape with several causally-DISTINCT faults planted at once (a root,
136
+ a downstream echo, and a two-part necessary-not-sufficient AND-conjunction), scored
137
+ against `blame.py`'s coalition/temporal-Shapley engine (`shapley_rank`) via `tracefork
138
+ bench`. 8/9 planted cases resolve correctly; the 9th is a documented, NOT hidden,
139
+ limitation of single-ordering temporal Shapley (it under-credits the earlier half of a
140
+ symmetric conjunction) — see `competing_faults.py`'s module docstring and README →
141
+ Validation scope. Cites, but does not reproduce, the published Who&When (ICML 2025)
142
+ ~14.2% log-based step-attribution anchor as context only — no external dataset is ever
143
+ downloaded (offline/$0 invariant applies here too). Zero-diff over the engines: both
144
+ modules only call `blame.py`'s existing public API.
145
+ - `report.py` / `server.py` / `web/report.html` — the single-file, dependency-free
146
+ three-panel UI; `report.py` injects tape JSON (HTML-escaped against `</script>`
147
+ breakout), `server.py` is FastAPI same-origin (no CORS, binds 127.0.0.1).
148
+ - `wire.py` / `synthetic.py` — Anthropic wire-format builders and the offline
149
+ Scripted/FaultAware fake transports, in the **package** so production never imports from
150
+ `tests/`; `tests/fakes.py` re-exports them.
151
+ - `replay.py` — `ReplayVerifier` (per-tape) and `run_fixture_corpus_check()`, which extends
152
+ the `validate --check` idea to plain replay: gates a committed tape corpus
153
+ (`experiments/replay_fixtures/` + its `manifest.json`) by asserting both bit-exact replay
154
+ and a `digest()` match per fixture — `tracefork replay --check <dir>`. `fixtures.py` holds
155
+ the tiny deterministic agents the corpus is built from (kept out of `validate.py` so the
156
+ corpus doesn't couple to fault-testing concerns); `scripts/gen_replay_fixtures.py`
157
+ (re)generates the corpus offline.
158
+ - `proxy.py` — `RecordProxy`/`ReplayProxy` (wired into FastAPI apps via
159
+ `build_record_app`/`build_replay_app`) are a **localhost base-URL record/replay proxy**
160
+ for clients the in-process httpx seam can't reach (curl, Node, Go, non-wrapped Python):
161
+ point the client's `base_url` at `http://127.0.0.1:<port>` instead of the provider.
162
+ Record forwards to a real (or, in tests, injected-fake) upstream and tees request+response
163
+ bytes into a `Tape`, streaming SSE chunk-by-chunk while forwarding; replay serves recorded
164
+ bytes with **no upstream**, matching each request to its recorded exchange by
165
+ `matcher.py`'s existing `RequestMatcher` fingerprint (an unrecorded request, or a real
166
+ body change, is a hard HTTP 502). Reuses `tape.py`/`matcher.py` unchanged — no in-process
167
+ `NondetSource` exists on this path, so a proxy-recorded tape (`Tape.boundary =
168
+ constants.PROXY_BOUNDARY`) sits outside the full single-process determinism boundary; see
169
+ the module docstring and README → Localhost record/replay proxy.
170
+ - `adapters/` — opt-in framework adapters (`bind()` routes a framework's LLM client
171
+ through the *existing* `TraceforkTransport`/`NondetSource`; `on_step()` maps a
172
+ framework callback/event to a neutral `Step`/`StepDAG` overlay — observer-only,
173
+ never a second capture path). `base.py` owns the protocol/registry;
174
+ `langchain.py` (`ChatOpenAI`/`ChatAnthropic` + a tape-backed LangGraph
175
+ checkpointer), `openai_agents.py` (defensive client-attribute injection +
176
+ `agents.set_default_openai_client()` + a `TracingProcessor`),
177
+ `crewai.py` (LiteLLM's `client_session`/`aclient_session` — CrewAI's actual httpx
178
+ chokepoint — + a `crewai_event_bus` listener), `autogen.py` (defensive
179
+ client-attribute injection + an `InterventionHandler` message-level seam), and
180
+ `adk.py` (Google ADK: candidate-path injection into the `google-genai`
181
+ `BaseApiClient`'s private `_httpx_client`/`_async_httpx_client` — reached
182
+ through the target itself, a `genai.Client`, an ADK `Gemini` model wrapper, or
183
+ an `LlmAgent` — + a `BasePlugin` registered once on the `Runner` for
184
+ agent/model/tool before/after boundaries) are the concrete adapters, each its
185
+ own optional extra with every framework import guarded, so `import tracefork`
186
+ and the whole offline suite work with none of them installed. Where a
187
+ framework's exact internal attribute names/event shapes aren't a documented
188
+ stable API, injection is defensive (a short candidate list, never one
189
+ hard-coded name) — see each module's docstring.
190
+ - `cli.py` — Typer entry point for all eleven commands.
191
+
192
+ `src/tracefork_spike/` holds the original Spike 0 (`fake_llm.py`, `agent.py`, `spike.py`):
193
+ record → save → load → replay → verify + negative control, with its own tests.
194
+
195
+ Most test files prove ONE module (or one seam) in isolation. Two are deliberately
196
+ cross-module, added once every feature bead had merged: `tests/test_e2e.py` chains
197
+ record → `TapeStore` save/load → replay → fork → blame → validate through the SAME
198
+ tape at every stage (not fresh fixtures per stage), plus the negative control,
199
+ asyncio-concurrency determinism, and every cross-feature path (MCP/tool exchanges
200
+ sharing a tape with LLM exchanges, redaction, OTel/OpenInference export→ingest,
201
+ plugin-registry resolution, `BoundaryGuard`, `divergence.py` diagnostics, the
202
+ base-URL proxy, `bench`, and the Bedrock/OpenAI/Gemini provider seams with their
203
+ documented scope boundaries called out explicitly, not papered over) — all
204
+ offline/$0. `tests/test_cli_smoke.py` invokes every one of the eleven CLI
205
+ subcommands and asserts its real exit code; `serve`/`proxy record`/`proxy replay`
206
+ call `uvicorn.run()` directly, so those are driven by monkeypatching `uvicorn.run`
207
+ to a no-op (proving the CLI's own wiring without binding a socket) plus a
208
+ `TestClient`/ASGI-transport hit against the underlying FastAPI app for actual
209
+ serving behavior. `scripts/e2e.sh` runs the whole gate — sync, lint, format,
210
+ mypy, tests+coverage, `validate --check`, `replay --check`, `bench`, build+twine
211
+ — as one script with a single PASS/FAIL verdict. Both test files are additive
212
+ only: zero-diff over `transport.py`/`tape.py`/`fork.py`/`blame.py`/`matcher.py`.
213
+
214
+ ## Invariants / conventions
215
+
216
+ - **Offline and $0 is non-negotiable** for the whole test suite, the spike, `validate`,
217
+ and the demo — no key, no network. The synthetic transports (`synthetic.py`) are the
218
+ seam; add to them rather than reaching for the real API. (`blame` on a real run is the
219
+ one budget-capped exception.)
220
+ - **The agent must read time/ids/random only through `NondetSource`** — any direct
221
+ `datetime.now()` / `uuid` / `random` breaks the determinism boundary and the
222
+ bit-exactness claim. `BoundaryGuard` (opt-in, default off) turns a subset of these
223
+ violations (thread/subprocess spawn, direct `random`/`time.monotonic`/`time.sleep`) into
224
+ a loud record-time error instead of a mysterious later replay failure — see
225
+ `boundary_guard.py` for exactly which calls it can and can't intercept.
226
+ - **The verifier proves, not asserts** — every request body is hash-checked against the
227
+ tape; the negative control must keep failing (drift detected) or the proof is vacuous.
228
+ - **Declared determinism boundary (v1):** single-process (sync **or** asyncio), clock +
229
+ id nondeterminism captured through `NondetSource`, **plus concurrency-graph determinism**
230
+ — the completion order of concurrent asyncio fan-out is recorded and re-imposed on replay
231
+ (see `transport.py`), so `gather`/`TaskGroup` agents replay bit-exact, not just
232
+ single-call-at-a-time ones. Threads/subprocess are out of scope; fork and blame
233
+ additionally assume the agent rebuilds its prefix deterministically (the property replay
234
+ proves) — see `SPIKE0.md`.
235
+ - **No `Co-Authored-By: Claude` trailer** on commits in this repo (public portfolio repo,
236
+ sole-author attribution).
237
+ - **Model IDs / pricing / SDK usage:** consult the `claude-api` skill before writing or
238
+ editing any Anthropic integration code rather than relying on memory.
239
+ - `docs/superpowers/`, `.beads/`, `planning/` are gitignored local scaffolding (but
240
+ `docs/demo.png` is committed). Runtime artifacts (`store.db`, `report.html`,
241
+ `blame_*.json`, `validation_report.json`, `examples/demo_report.html`) are gitignored.