crosstrace 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. crosstrace-0.1.0/LICENSE +21 -0
  2. crosstrace-0.1.0/PKG-INFO +60 -0
  3. crosstrace-0.1.0/PYPI.md +28 -0
  4. crosstrace-0.1.0/README.md +225 -0
  5. crosstrace-0.1.0/crosstrace/__init__.py +6 -0
  6. crosstrace-0.1.0/crosstrace/__main__.py +5 -0
  7. crosstrace-0.1.0/crosstrace/cli.py +309 -0
  8. crosstrace-0.1.0/crosstrace/config.py +251 -0
  9. crosstrace-0.1.0/crosstrace/fixes.py +140 -0
  10. crosstrace-0.1.0/crosstrace/masks.py +61 -0
  11. crosstrace-0.1.0/crosstrace/mcp/__init__.py +2 -0
  12. crosstrace-0.1.0/crosstrace/mcp/http.py +189 -0
  13. crosstrace-0.1.0/crosstrace/mcp/ledger.py +128 -0
  14. crosstrace-0.1.0/crosstrace/mcp/verdict.py +57 -0
  15. crosstrace-0.1.0/crosstrace/mcp/wrap.py +236 -0
  16. crosstrace-0.1.0/crosstrace/proxy/__init__.py +7 -0
  17. crosstrace-0.1.0/crosstrace/proxy/faults.py +538 -0
  18. crosstrace-0.1.0/crosstrace/proxy/http1.py +148 -0
  19. crosstrace-0.1.0/crosstrace/proxy/providers.py +78 -0
  20. crosstrace-0.1.0/crosstrace/proxy/recording.py +117 -0
  21. crosstrace-0.1.0/crosstrace/proxy/server.py +318 -0
  22. crosstrace-0.1.0/crosstrace/report.py +203 -0
  23. crosstrace-0.1.0/crosstrace/runner.py +396 -0
  24. crosstrace-0.1.0/crosstrace/verdict.py +485 -0
  25. crosstrace-0.1.0/crosstrace.egg-info/PKG-INFO +60 -0
  26. crosstrace-0.1.0/crosstrace.egg-info/SOURCES.txt +63 -0
  27. crosstrace-0.1.0/crosstrace.egg-info/dependency_links.txt +1 -0
  28. crosstrace-0.1.0/crosstrace.egg-info/entry_points.txt +2 -0
  29. crosstrace-0.1.0/crosstrace.egg-info/requires.txt +11 -0
  30. crosstrace-0.1.0/crosstrace.egg-info/top_level.txt +1 -0
  31. crosstrace-0.1.0/pyproject.toml +46 -0
  32. crosstrace-0.1.0/setup.cfg +4 -0
  33. crosstrace-0.1.0/tests/test_anthropic_llm.py +102 -0
  34. crosstrace-0.1.0/tests/test_assign.py +100 -0
  35. crosstrace-0.1.0/tests/test_async_client.py +30 -0
  36. crosstrace-0.1.0/tests/test_cebpf.py +81 -0
  37. crosstrace-0.1.0/tests/test_cli.py +291 -0
  38. crosstrace-0.1.0/tests/test_combined.py +240 -0
  39. crosstrace-0.1.0/tests/test_combined_pooled.py +87 -0
  40. crosstrace-0.1.0/tests/test_combo_faults.py +24 -0
  41. crosstrace-0.1.0/tests/test_connect_end_modes.py +82 -0
  42. crosstrace-0.1.0/tests/test_continuation.py +264 -0
  43. crosstrace-0.1.0/tests/test_correlator.py +219 -0
  44. crosstrace-0.1.0/tests/test_diagnosers.py +68 -0
  45. crosstrace-0.1.0/tests/test_faultlab.py +66 -0
  46. crosstrace-0.1.0/tests/test_faults.py +352 -0
  47. crosstrace-0.1.0/tests/test_join_perf.py +16 -0
  48. crosstrace-0.1.0/tests/test_kernel.py +93 -0
  49. crosstrace-0.1.0/tests/test_langgraph_agent.py +87 -0
  50. crosstrace-0.1.0/tests/test_launcher.py +56 -0
  51. crosstrace-0.1.0/tests/test_local_port.py +79 -0
  52. crosstrace-0.1.0/tests/test_masks.py +52 -0
  53. crosstrace-0.1.0/tests/test_mcp.py +345 -0
  54. crosstrace-0.1.0/tests/test_mcp_e2e.py +100 -0
  55. crosstrace-0.1.0/tests/test_mcp_http.py +160 -0
  56. crosstrace-0.1.0/tests/test_metrics.py +57 -0
  57. crosstrace-0.1.0/tests/test_obi_vs_join.py +84 -0
  58. crosstrace-0.1.0/tests/test_probes_s8.py +166 -0
  59. crosstrace-0.1.0/tests/test_proxy.py +250 -0
  60. crosstrace-0.1.0/tests/test_real_path.py +139 -0
  61. crosstrace-0.1.0/tests/test_scripts_compile.py +19 -0
  62. crosstrace-0.1.0/tests/test_smoke.py +61 -0
  63. crosstrace-0.1.0/tests/test_stage6.py +133 -0
  64. crosstrace-0.1.0/tests/test_v5.py +128 -0
  65. crosstrace-0.1.0/tests/test_verdicts.py +349 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Mayank Kumar Singh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,60 @@
1
+ Metadata-Version: 2.4
2
+ Name: crosstrace
3
+ Version: 0.1.0
4
+ Summary: Find the failures your AI agent hides: replay real runs with byte-level provider faults and get a verdict for each
5
+ Author: Mayank Kumar Singh
6
+ License: MIT
7
+ Project-URL: Homepage, https://crosstrace-site.vercel.app/
8
+ Project-URL: Repository, https://github.com/mac146/crosstrace
9
+ Project-URL: Issues, https://github.com/mac146/crosstrace/issues
10
+ Project-URL: Documentation, https://github.com/mac146/crosstrace/blob/main/docs/cli.md
11
+ Keywords: ai-agents,testing,fault-injection,resilience,llm,mcp,ci
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Software Development :: Testing
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: httpx>=0.27
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest; extra == "dev"
26
+ Provides-Extra: mcp
27
+ Requires-Dist: mcp>=2; extra == "mcp"
28
+ Requires-Dist: openai; extra == "mcp"
29
+ Provides-Extra: research
30
+ Requires-Dist: opentelemetry-sdk==1.45.2; extra == "research"
31
+ Dynamic: license-file
32
+
33
+ # CrossTrace
34
+
35
+ **A crash test for AI agents.** CrossTrace records real runs of your agent, replays them offline with no API cost, and breaks one model call
36
+ at a time the way providers and networks really break: a stream cut short, a stalled connection, a reset, a rate limit, an overloaded API, a
37
+ reply cut at the token limit. For every failure it tells you what your agent did next, and the fix for the SDK it detected. It runs in CI.
38
+
39
+ Your agent needs no code changes. It only has to read its provider base URL from the environment (the OpenAI and Anthropic SDKs, LangChain,
40
+ LangGraph, LiteLLM, pydantic-ai, the OpenAI Agents SDK and the Vercel AI SDK all do).
41
+
42
+ ```bash
43
+ pip install crosstrace
44
+ crosstrace init --command "python agent.py {input}"
45
+ crosstrace record # runs your agent for real, once per input (uses your API key)
46
+ crosstrace test # replays with faults, offline; exit code 1 if your agent hides a failure
47
+ ```
48
+
49
+ What it found in a stock LangGraph agent: a stream cut inside a tool call made it run the tool on the truncated arguments (the model asked for
50
+ order 42, the agent looked up order 4, no error); a stalled stream froze it; a cut answer was printed as the whole answer.
51
+
52
+ It also tests MCP tool calls over stdio: when a tool succeeded but its response was lost, a retry can repeat a write (a second email, a second
53
+ form submit). CrossTrace counts writes after recovery and reports `DUPLICATE`.
54
+
55
+ Early (v0.1), open source (MIT).
56
+
57
+ - Source, full README and examples: https://github.com/mac146/crosstrace
58
+ - Short version with an animated walkthrough: https://crosstrace-site.vercel.app/
59
+ - Command reference: https://github.com/mac146/crosstrace/blob/main/docs/cli.md
60
+ - Issues: https://github.com/mac146/crosstrace/issues
@@ -0,0 +1,28 @@
1
+ # CrossTrace
2
+
3
+ **A crash test for AI agents.** CrossTrace records real runs of your agent, replays them offline with no API cost, and breaks one model call
4
+ at a time the way providers and networks really break: a stream cut short, a stalled connection, a reset, a rate limit, an overloaded API, a
5
+ reply cut at the token limit. For every failure it tells you what your agent did next, and the fix for the SDK it detected. It runs in CI.
6
+
7
+ Your agent needs no code changes. It only has to read its provider base URL from the environment (the OpenAI and Anthropic SDKs, LangChain,
8
+ LangGraph, LiteLLM, pydantic-ai, the OpenAI Agents SDK and the Vercel AI SDK all do).
9
+
10
+ ```bash
11
+ pip install crosstrace
12
+ crosstrace init --command "python agent.py {input}"
13
+ crosstrace record # runs your agent for real, once per input (uses your API key)
14
+ crosstrace test # replays with faults, offline; exit code 1 if your agent hides a failure
15
+ ```
16
+
17
+ What it found in a stock LangGraph agent: a stream cut inside a tool call made it run the tool on the truncated arguments (the model asked for
18
+ order 42, the agent looked up order 4, no error); a stalled stream froze it; a cut answer was printed as the whole answer.
19
+
20
+ It also tests MCP tool calls over stdio: when a tool succeeded but its response was lost, a retry can repeat a write (a second email, a second
21
+ form submit). CrossTrace counts writes after recovery and reports `DUPLICATE`.
22
+
23
+ Early (v0.1), open source (MIT).
24
+
25
+ - Source, full README and examples: https://github.com/mac146/crosstrace
26
+ - Short version with an animated walkthrough: https://crosstrace-site.vercel.app/
27
+ - Command reference: https://github.com/mac146/crosstrace/blob/main/docs/cli.md
28
+ - Issues: https://github.com/mac146/crosstrace/issues
@@ -0,0 +1,225 @@
1
+ # CrossTrace
2
+
3
+ **A crash test for AI agents.** CrossTrace records real runs of your agent, replays them with no API cost, and breaks one model call at a
4
+ time the way providers and networks really break: a stream cut short, a stalled connection, a reset, a 429, an overloaded API. For every
5
+ failure it tells you what your agent did next. It runs in CI on every pull request.
6
+
7
+ Your agent needs no code changes and no SDK. It runs as it is, with its real SDK and framework code paths.
8
+
9
+ **[crosstrace-site.vercel.app](https://crosstrace-site.vercel.app/)**: the short version, with an animated walkthrough.
10
+
11
+ ## What it finds in a stock LangGraph agent
12
+
13
+ A support agent written the way most LangGraph tutorials write one (`create_react_agent`, streaming `ChatOpenAI`, default settings,
14
+ three tools; [source](examples/real_agents/langgraph_support/agent.py)),
15
+ recorded once against a real model on OpenRouter, then tested offline:
16
+
17
+ ```
18
+ ✗ tool_args_cut SILENT 3/3
19
+ e.g. path 0 exchange 0: ran tool get_order with arguments {"order_id": 4} from a response that never
20
+ finished, instead of retrying exchange 0 (no retry)
21
+ ✗ stream_stall HANG 5/5
22
+ e.g. path 0 exchange 0: no exit within 60 s (no retry)
23
+ ✗ stream_cut_clean PARTIAL 2/5, ENDED_EARLY 3/5
24
+ e.g. path 0 exchange 0: printed nothing and exited 0 after exchange 0 failed
25
+ ✓ server_error RECOVERED 5/5
26
+ ✓ overloaded!persistent SURFACED 5/5
27
+ ✓ rate_limit RECOVERED 5/5
28
+ ✓ reset_before_response RECOVERED 5/5
29
+ ✓ context_length SURFACED 5/5
30
+ ! connection churn 1.0 new connections per model call
31
+ ```
32
+
33
+ - **The stream broke inside a tool call, and the agent ran the tool anyway.** The model asked for order 42. LangChain completed the
34
+ half-received JSON, and the agent looked up **order 4**, with no error anywhere. The same happened with `get_order({})` and a tracking
35
+ number cut to `"BD1234"`.
36
+ - **A stalled stream froze the agent.** `ChatOpenAI` has no read timeout by default; the openai SDK waits up to 10 minutes per attempt.
37
+ Setting `ChatOpenAI(timeout=30)` turns the hang into an error (SURFACED 5/5), but not into a recovery: the SDK's retries stop once
38
+ response headers have arrived, so a mid-stream timeout is never retried.
39
+ - **A cut final answer was printed as the answer.** "Your order 42" and "I'm sorry your mug arrived broken, Asha." with exit code 0, or
40
+ nothing at all with exit code 0.
41
+ - **What it handled well:** HTTP errors before the stream starts. The SDK's retries cover those.
42
+
43
+ Each finding comes with the fix for the SDK it detected, and the exact command that reruns that one trial.
44
+
45
+ These are not exotic faults. A clean cut without the final event is what a gateway produces when its upstream dies but it closes the
46
+ response politely. The same behaviour shows up across SDKs: [what the OpenAI, Anthropic and Vercel SDKs do under each fault](docs/sdk-fault-table.md).
47
+
48
+ ## Other agents checked
49
+
50
+ The same faults against other popular frameworks, each recorded once on a cheap real model and then replayed offline (recordings not published).
51
+ Every row is what CrossTrace reported after its verdicts were corrected against what the framework really did.
52
+
53
+ | Framework | Cut stream or cut tool call | Stalled or silent response | Other |
54
+ |---|---|---|---|
55
+ | LangGraph (`create_react_agent`) | **ran the tool on the cut arguments**, printed a cut answer as complete | hangs (no read timeout) | errors before the stream recovered |
56
+ | OpenAI Agents SDK (Responses API) | raised an error (it waits for the completed event) | hangs; `set_default_openai_client(AsyncOpenAI(timeout=...))` turns it into an error | errors before the stream recovered |
57
+ | pydantic-ai | refused the cut tool call and told the model (one case: arguments were already complete and it ran the tool); printed a cut answer as complete, also for a Responses `incomplete` status | hangs; `model_settings={"timeout": ...}` turns it into an error | empty reply re-prompted |
58
+ | CrewAI (native openai client) | printed a cut answer as complete | hangs; `LLM(timeout=...)` recovers | context-length error summarised and continued; a persistent outage retried 7+ times in about 3 s |
59
+
60
+ The timeout fixes were run, not assumed (with 10 seconds). In the three streamed agents (LangGraph, the Agents SDK, pydantic-ai) a stall in the middle of a stream is raised, not retried.
61
+
62
+ ## Try it (no API key)
63
+
64
+ [examples/broken_agents](examples/broken_agents) is a small agent on the official openai SDK whose failure handling can be switched
65
+ between good and deliberately broken. Its recording (made against a local mock provider) is in the repo, so the test runs offline:
66
+
67
+ ```bash
68
+ git clone https://github.com/mac146/crosstrace && cd crosstrace
69
+ pip install . openai
70
+ cd examples/broken_agents
71
+ crosstrace test # the naive agent: hidden failures, exit code 1
72
+ sed -i 's/"naive"/"good"/' crosstrace.toml && crosstrace test # the same recording, a fixed agent: exit code 0
73
+ ```
74
+
75
+ To reproduce the LangGraph results above, record the agent once with an OpenAI-compatible key (an OpenRouter key works as is, a few
76
+ cents), then test:
77
+
78
+ ```bash
79
+ pip install -r examples/real_agents/langgraph_support/requirements.txt
80
+ cd examples/real_agents/langgraph_support
81
+ OPENAI_API_KEY=... crosstrace record && crosstrace test
82
+ ```
83
+
84
+ A real model answers differently from run to run, so your recordings, and the exchanges a fault lands on, will differ from ours; the
85
+ behaviour under each fault is what to compare.
86
+
87
+ ## Use it on your agent
88
+
89
+ ```bash
90
+ pip install git+https://github.com/mac146/crosstrace # not on PyPI yet
91
+ crosstrace init --command "python agent.py {input}" # writes crosstrace.toml: your inputs, timeouts, faults
92
+ crosstrace record # runs your agent for real, once per input (uses your API key)
93
+ crosstrace test # replays with faults; exit code 1 if your agent hides a failure
94
+ ```
95
+
96
+ Your agent must read its provider base URL from the environment, which the OpenAI and Anthropic SDKs, LangChain, LangGraph, LiteLLM,
97
+ pydantic-ai and the Vercel AI SDK all do ([checked for each](docs/framework-matrix.md)). If yours uses its own variable, map it in
98
+ `crosstrace.toml`: `env = { MY_BASE_URL = "{proxy_url}/v1" }`.
99
+
100
+ Commit `crosstrace.toml` and `.crosstrace/recordings/`, and CI tests every pull request without API keys:
101
+
102
+ ```yaml
103
+ # .github/workflows/crosstrace.yml
104
+ on: [pull_request]
105
+ jobs:
106
+ resilience:
107
+ runs-on: ubuntu-latest
108
+ steps:
109
+ - uses: actions/checkout@v4
110
+ - uses: mac146/crosstrace@main
111
+ with:
112
+ install: pip install -r requirements.txt
113
+ ```
114
+
115
+ The report goes to the job summary. A nightly run tests every fault at every model call: [examples/github/crosstrace.yml](examples/github/crosstrace.yml).
116
+
117
+ ## How it decides
118
+
119
+ CrossTrace sits between your agent and the provider as a local proxy. It breaks one model call per trial, at the byte level, so your
120
+ real SDK sees exactly what it would see in production. It then watches the agent's **next move**: did it retry that call, stop with an
121
+ error, or carry on as if nothing happened?
122
+
123
+ | verdict | meaning | |
124
+ |---|---|---|
125
+ | `RECOVERED` | retried (or fell back) and finished with the healthy answer | ok |
126
+ | `SURFACED` | stopped with an error | ok |
127
+ | `SILENT` | carried on with a missing or partial answer in its context | problem |
128
+ | `PARTIAL` | printed the cut answer as if it were complete | problem |
129
+ | `HANG` | never finished | problem |
130
+ | `RETRY_STORM`, `IGNORED_RETRY_AFTER`, `WRONG_RETRY` | retried too often, too soon, or what can't succeed | problem |
131
+ | `ENDED_EARLY`, `RECOVERED_DIFFERENT` | exited 0 without the failed step, or recovered with a different answer | review |
132
+ | `REPROMPTED` | answered the failure with a different request instead of retrying the same call: told the model its reply failed, had its tool arguments refused and the model told, or summarised after a context-length error (the text is shown, so you can judge it) | review |
133
+ | `DIVERGED` | after a reply that arrived but was defective, sent a request that matches nothing recorded, and nothing shows it acted on the reply or carried on as before. The run is then continued and settled by how it ends: `RECOVERED` if it finishes, `SURFACED` if it stops with an error, `RETRY_STORM` if it never stops. Only if the replay cannot answer a later request does it stay a review | review |
134
+
135
+ **What can fail your build, and what cannot.** A verdict fails the build only when it rests on something CrossTrace measured: a hang, too many
136
+ or too fast retries, a retried request that can never succeed, a write that ran twice, an answer printed cut, or an agent that acted on a
137
+ defective reply or carried on exactly as if nothing had happened. When an agent changes course in a way CrossTrace can only guess at, the
138
+ result is a **review** with the facts shown (what it sent, how many messages), never a failure. Every result in the JSON report carries
139
+ `observed`: the attempts, the status of the fault, and the agent's next move.
140
+
141
+ **The run continues past the fault.** When an agent answers a failure with a request the recording does not contain (it re-prompts the
142
+ model, summarises, rewrites its request), CrossTrace answers it with the next healthy reply of the recorded plan instead of an error, so the
143
+ run reaches its real end and the verdict comes from how it ended. A persistent fault (`"length_cut!persistent"`) keeps hitting the agent's
144
+ recovery attempts too, which tests whether its recovery has a bound: an agent that keeps asking after 8 different requests is a `RETRY_STORM`.
145
+ No model and no key are needed, so it stays deterministic and offline. It cannot tell whether a re-prompted agent would have received a good
146
+ answer from a real model; it only shows that the agent stops. To check that too, add `--live`: those requests then go to the real provider
147
+ with your own key (`OPENAI_API_KEY` or `ANTHROPIC_API_KEY`, read from the environment, never stored or logged), at most 6 calls per trial
148
+ (`--live-max-calls`), so a re-prompt is answered by a real model and the run ends on its merits. It spends money, so it is opt-in and meant for a
149
+ handful of trials (`--fault`, `--path`, `--at`), not a whole run. Turn continuation off with `--no-continue` or `[test] continue = false`.
150
+
151
+ No LLM judge and no comparison of free-text answers: the verdict comes from which model call the agent made next, so it is the same
152
+ every time. Faults are injected at every model call of every recorded path (nightly) or a fixed sample (pull requests), so an agent can't
153
+ pass by getting lucky on one run. Before any fault, each recording must replay exactly; if your prompts change on every run, CrossTrace
154
+ names the field to mask instead of reporting a false pass.
155
+
156
+ Details: [docs/cli.md](docs/cli.md). Fault list: `crosstrace faults`.
157
+
158
+ ## Status and limits
159
+
160
+ Early (v0.1). Tested end to end on the example agents here and the SDK matrix above; not yet on anyone else's production agent. If you
161
+ try it on yours, an issue with what it found (or got wrong) is the most useful thing you can send.
162
+
163
+ - Model calls: OpenAI chat completions and Responses, Anthropic Messages. Tool calls: MCP over stdio and streamable HTTP, see above.
164
+ - Python 3.11+. Your agent can be in any language; it just has to be runnable as a command.
165
+ - The healthy run must be repeatable enough to replay. Random values in prompts need a mask; heavy run-to-run variation in what your
166
+ agent does needs several recorded runs.
167
+
168
+ ## Tool calls (MCP): a tool that ran but whose reply was lost
169
+
170
+ A model call that fails is annoying. A tool call that **succeeded but whose response was lost** can do damage: the email was sent, the
171
+ card was charged, the form was submitted, the reply never reached the agent, and a retry does it a second time.
172
+
173
+ CrossTrace tests this for MCP servers over stdio and over streamable HTTP. For a stdio server, one change in your agent's MCP config:
174
+ launch it as `crosstrace mcp-wrap -- <its usual command>`. For a remote server, list its URL in `[mcp.http]` and have the agent read its
175
+ server address from `CROSSTRACE_MCP_URL_<NAME>`. The wrapper (or the local stand-in for the URL) records the tool calls in
176
+ `crosstrace record`, and in `crosstrace test` it **replays them without starting or calling the real server**, so a test cannot cause a
177
+ side effect. An agent with several servers names each one (`mcp-wrap --name github -- ...`, or the key in `[mcp.http]`).
178
+
179
+ - **`lost_response`:** one tool call is made (and counted), and its reply is replaced by a transport error. Then we watch what the agent does.
180
+ - **Write counting:** every call to a tool that changes something is logged. A write that ran more often than in the healthy run is a
181
+ **`DUPLICATE`**, in **every** trial, not only this one. A run that recovers and prints the right answer can still have written twice, so
182
+ recovering is not enough to pass. The same check catches an agent that recovers from a *model* failure by starting the task over.
183
+ - **Which tools are writes:** your `[mcp]` lists first, then what the server says about itself (a tool it declares `readOnlyHint` is a read),
184
+ then a guess from the name (`send`, `submit`, `pay`, `delete`, `click`, ...; `get`, `list`, `status` are reads).
185
+ List the ones it gets wrong in `[mcp] writes` / `reads`. For browser tools, click, type, fill_form, file_upload, press_key (Enter submits a
186
+ form), evaluate and run_code are writes; navigate, snapshot and screenshot are reads. `select_option` and `handle_dialog` are ambiguous and
187
+ count as reads: list them under `writes` if a choice or a confirmation can change something.
188
+
189
+ ```
190
+ ✗ lost_response DUPLICATE 1/1
191
+ e.g. path 0 tool call 0: refund({"amount":129.99,"customer_id":"42",...}) ran 2 time(s), 1 in the healthy run;
192
+ the agent recovered and finished, but the write was repeated
193
+ ```
194
+
195
+ Runnable without any key: [examples/mcp_agents](examples/mcp_agents) (`pip install mcp openai`). Also checked by hand against the official
196
+ `@playwright/mcp` 0.0.85 (25 tools): a scripted agent, a real Chrome and a local order form. Recording placed one real order; replays and
197
+ trials placed none. Losing the response to the click made the retrying agent **DUPLICATE** and the checking agent stop. That agent was
198
+ scripted, so it tests the tool-call handling, not how well a model drives a browser. **What this does not do yet:**
199
+
200
+ - Streamable HTTP is covered for what an agent sends as a POST (answered as JSON or an event stream). A server that pushes messages on its
201
+ own GET stream is not: that stream gets a 405, which the protocol allows. Checked against the official Python SDK's server and client only.
202
+ A lost response over HTTP is a 504 by default; with `how = "drop"` the connection is closed, which the official client turns into a crash
203
+ of the whole session, so the agent cannot retry at all.
204
+ - Several servers are handled by name. Each name must be used in both `record` and `test`; two unnamed stdio servers are not told apart.
205
+ - Your MCP client must start the wrapper with `CROSSTRACE_MCP` in its environment. The official Python SDK starts servers with a reduced
206
+ environment, so pass it on (the example does).
207
+ - Replay cannot answer a state check. An agent that asks a read tool "did it go through?" before retrying gets the recorded answer, or
208
+ "not recorded", so it shows as stopping with an error, not as recovering safely.
209
+ - A retry counts as a repeat unless the server declares the tool idempotent (`idempotentHint`) or the call carries an idempotency key as a
210
+ top-level argument (`idempotency_key`, `request_id`, `client_token` and similar). CrossTrace takes the key at its word: it cannot tell
211
+ whether the server honours it. The official Playwright server declares `readOnlyHint` for 6 of its 25 tools and no `idempotentHint`.
212
+ - By default the repeat is counted in replay, from a log of the tool calls. `crosstrace test --real-tools` runs the real servers in the
213
+ `lost_response` trials (the model is still replayed), so the retry really happens twice and the report quotes what the server answered.
214
+ It is opt-in because it does the work for real: point it at a test account or a sandbox. Once they are, set `[mcp] real = true` and every
215
+ run, CI included, shows the double write for real (`--no-real-tools` turns it off for one run). Model-fault trials always stay in replay. If the
216
+ real server answers differently from the recording, the agent's next model request may match nothing recorded.
217
+ - The lost response is a transport error on a live session, not a dead server process.
218
+
219
+ ## Background
220
+
221
+ CrossTrace started as research on tracing AI agents across the application and the kernel (eBPF): 21 stages, predictions written
222
+ before each run. Mining real GitHub issues showed that most agent failures in the wild are failure-*handling* bugs in SDKs and frameworks,
223
+ not slow networks, which is why the product tests failure handling. The research is in [docs/research.md](docs/research.md).
224
+
225
+ MIT licensed, see [LICENSE](LICENSE).
@@ -0,0 +1,6 @@
1
+ """CrossTrace: find the failures AI agents hide, before production (see BUILD_PLAN.md, private).
2
+
3
+ The product package. The research prototype (agent/, collectors/, correlator/, diagnosers/, faultlab/, eval/) stays where it is and
4
+ is imported where it adds evidence.
5
+ """
6
+ __version__ = "0.1.0"
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from crosstrace.cli import main
4
+
5
+ sys.exit(main())