crosstrace 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crosstrace-0.1.0/LICENSE +21 -0
- crosstrace-0.1.0/PKG-INFO +60 -0
- crosstrace-0.1.0/PYPI.md +28 -0
- crosstrace-0.1.0/README.md +225 -0
- crosstrace-0.1.0/crosstrace/__init__.py +6 -0
- crosstrace-0.1.0/crosstrace/__main__.py +5 -0
- crosstrace-0.1.0/crosstrace/cli.py +309 -0
- crosstrace-0.1.0/crosstrace/config.py +251 -0
- crosstrace-0.1.0/crosstrace/fixes.py +140 -0
- crosstrace-0.1.0/crosstrace/masks.py +61 -0
- crosstrace-0.1.0/crosstrace/mcp/__init__.py +2 -0
- crosstrace-0.1.0/crosstrace/mcp/http.py +189 -0
- crosstrace-0.1.0/crosstrace/mcp/ledger.py +128 -0
- crosstrace-0.1.0/crosstrace/mcp/verdict.py +57 -0
- crosstrace-0.1.0/crosstrace/mcp/wrap.py +236 -0
- crosstrace-0.1.0/crosstrace/proxy/__init__.py +7 -0
- crosstrace-0.1.0/crosstrace/proxy/faults.py +538 -0
- crosstrace-0.1.0/crosstrace/proxy/http1.py +148 -0
- crosstrace-0.1.0/crosstrace/proxy/providers.py +78 -0
- crosstrace-0.1.0/crosstrace/proxy/recording.py +117 -0
- crosstrace-0.1.0/crosstrace/proxy/server.py +318 -0
- crosstrace-0.1.0/crosstrace/report.py +203 -0
- crosstrace-0.1.0/crosstrace/runner.py +396 -0
- crosstrace-0.1.0/crosstrace/verdict.py +485 -0
- crosstrace-0.1.0/crosstrace.egg-info/PKG-INFO +60 -0
- crosstrace-0.1.0/crosstrace.egg-info/SOURCES.txt +63 -0
- crosstrace-0.1.0/crosstrace.egg-info/dependency_links.txt +1 -0
- crosstrace-0.1.0/crosstrace.egg-info/entry_points.txt +2 -0
- crosstrace-0.1.0/crosstrace.egg-info/requires.txt +11 -0
- crosstrace-0.1.0/crosstrace.egg-info/top_level.txt +1 -0
- crosstrace-0.1.0/pyproject.toml +46 -0
- crosstrace-0.1.0/setup.cfg +4 -0
- crosstrace-0.1.0/tests/test_anthropic_llm.py +102 -0
- crosstrace-0.1.0/tests/test_assign.py +100 -0
- crosstrace-0.1.0/tests/test_async_client.py +30 -0
- crosstrace-0.1.0/tests/test_cebpf.py +81 -0
- crosstrace-0.1.0/tests/test_cli.py +291 -0
- crosstrace-0.1.0/tests/test_combined.py +240 -0
- crosstrace-0.1.0/tests/test_combined_pooled.py +87 -0
- crosstrace-0.1.0/tests/test_combo_faults.py +24 -0
- crosstrace-0.1.0/tests/test_connect_end_modes.py +82 -0
- crosstrace-0.1.0/tests/test_continuation.py +264 -0
- crosstrace-0.1.0/tests/test_correlator.py +219 -0
- crosstrace-0.1.0/tests/test_diagnosers.py +68 -0
- crosstrace-0.1.0/tests/test_faultlab.py +66 -0
- crosstrace-0.1.0/tests/test_faults.py +352 -0
- crosstrace-0.1.0/tests/test_join_perf.py +16 -0
- crosstrace-0.1.0/tests/test_kernel.py +93 -0
- crosstrace-0.1.0/tests/test_langgraph_agent.py +87 -0
- crosstrace-0.1.0/tests/test_launcher.py +56 -0
- crosstrace-0.1.0/tests/test_local_port.py +79 -0
- crosstrace-0.1.0/tests/test_masks.py +52 -0
- crosstrace-0.1.0/tests/test_mcp.py +345 -0
- crosstrace-0.1.0/tests/test_mcp_e2e.py +100 -0
- crosstrace-0.1.0/tests/test_mcp_http.py +160 -0
- crosstrace-0.1.0/tests/test_metrics.py +57 -0
- crosstrace-0.1.0/tests/test_obi_vs_join.py +84 -0
- crosstrace-0.1.0/tests/test_probes_s8.py +166 -0
- crosstrace-0.1.0/tests/test_proxy.py +250 -0
- crosstrace-0.1.0/tests/test_real_path.py +139 -0
- crosstrace-0.1.0/tests/test_scripts_compile.py +19 -0
- crosstrace-0.1.0/tests/test_smoke.py +61 -0
- crosstrace-0.1.0/tests/test_stage6.py +133 -0
- crosstrace-0.1.0/tests/test_v5.py +128 -0
- crosstrace-0.1.0/tests/test_verdicts.py +349 -0
crosstrace-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mayank Kumar Singh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crosstrace
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find the failures your AI agent hides: replay real runs with byte-level provider faults and get a verdict for each
|
|
5
|
+
Author: Mayank Kumar Singh
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://crosstrace-site.vercel.app/
|
|
8
|
+
Project-URL: Repository, https://github.com/mac146/crosstrace
|
|
9
|
+
Project-URL: Issues, https://github.com/mac146/crosstrace/issues
|
|
10
|
+
Project-URL: Documentation, https://github.com/mac146/crosstrace/blob/main/docs/cli.md
|
|
11
|
+
Keywords: ai-agents,testing,fault-injection,resilience,llm,mcp,ci
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Testing
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest; extra == "dev"
|
|
26
|
+
Provides-Extra: mcp
|
|
27
|
+
Requires-Dist: mcp>=2; extra == "mcp"
|
|
28
|
+
Requires-Dist: openai; extra == "mcp"
|
|
29
|
+
Provides-Extra: research
|
|
30
|
+
Requires-Dist: opentelemetry-sdk==1.45.2; extra == "research"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# CrossTrace
|
|
34
|
+
|
|
35
|
+
**A crash test for AI agents.** CrossTrace records real runs of your agent, replays them offline with no API cost, and breaks one model call
|
|
36
|
+
at a time the way providers and networks really break: a stream cut short, a stalled connection, a reset, a rate limit, an overloaded API, a
|
|
37
|
+
reply cut at the token limit. For every failure it tells you what your agent did next, and the fix for the SDK it detected. It runs in CI.
|
|
38
|
+
|
|
39
|
+
Your agent needs no code changes. It only has to read its provider base URL from the environment (the OpenAI and Anthropic SDKs, LangChain,
|
|
40
|
+
LangGraph, LiteLLM, pydantic-ai, the OpenAI Agents SDK and the Vercel AI SDK all do).
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install crosstrace
|
|
44
|
+
crosstrace init --command "python agent.py {input}"
|
|
45
|
+
crosstrace record # runs your agent for real, once per input (uses your API key)
|
|
46
|
+
crosstrace test # replays with faults, offline; exit code 1 if your agent hides a failure
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
What it found in a stock LangGraph agent: a stream cut inside a tool call made it run the tool on the truncated arguments (the model asked for
|
|
50
|
+
order 42, the agent looked up order 4, no error); a stalled stream froze it; a cut answer was printed as the whole answer.
|
|
51
|
+
|
|
52
|
+
It also tests MCP tool calls over stdio: when a tool succeeded but its response was lost, a retry can repeat a write (a second email, a second
|
|
53
|
+
form submit). CrossTrace counts writes after recovery and reports `DUPLICATE`.
|
|
54
|
+
|
|
55
|
+
Early (v0.1), open source (MIT).
|
|
56
|
+
|
|
57
|
+
- Source, full README and examples: https://github.com/mac146/crosstrace
|
|
58
|
+
- Short version with an animated walkthrough: https://crosstrace-site.vercel.app/
|
|
59
|
+
- Command reference: https://github.com/mac146/crosstrace/blob/main/docs/cli.md
|
|
60
|
+
- Issues: https://github.com/mac146/crosstrace/issues
|
crosstrace-0.1.0/PYPI.md
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# CrossTrace
|
|
2
|
+
|
|
3
|
+
**A crash test for AI agents.** CrossTrace records real runs of your agent, replays them offline with no API cost, and breaks one model call
|
|
4
|
+
at a time the way providers and networks really break: a stream cut short, a stalled connection, a reset, a rate limit, an overloaded API, a
|
|
5
|
+
reply cut at the token limit. For every failure it tells you what your agent did next, and the fix for the SDK it detected. It runs in CI.
|
|
6
|
+
|
|
7
|
+
Your agent needs no code changes. It only has to read its provider base URL from the environment (the OpenAI and Anthropic SDKs, LangChain,
|
|
8
|
+
LangGraph, LiteLLM, pydantic-ai, the OpenAI Agents SDK and the Vercel AI SDK all do).
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install crosstrace
|
|
12
|
+
crosstrace init --command "python agent.py {input}"
|
|
13
|
+
crosstrace record # runs your agent for real, once per input (uses your API key)
|
|
14
|
+
crosstrace test # replays with faults, offline; exit code 1 if your agent hides a failure
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
What it found in a stock LangGraph agent: a stream cut inside a tool call made it run the tool on the truncated arguments (the model asked for
|
|
18
|
+
order 42, the agent looked up order 4, no error); a stalled stream froze it; a cut answer was printed as the whole answer.
|
|
19
|
+
|
|
20
|
+
It also tests MCP tool calls over stdio: when a tool succeeded but its response was lost, a retry can repeat a write (a second email, a second
|
|
21
|
+
form submit). CrossTrace counts writes after recovery and reports `DUPLICATE`.
|
|
22
|
+
|
|
23
|
+
Early (v0.1), open source (MIT).
|
|
24
|
+
|
|
25
|
+
- Source, full README and examples: https://github.com/mac146/crosstrace
|
|
26
|
+
- Short version with an animated walkthrough: https://crosstrace-site.vercel.app/
|
|
27
|
+
- Command reference: https://github.com/mac146/crosstrace/blob/main/docs/cli.md
|
|
28
|
+
- Issues: https://github.com/mac146/crosstrace/issues
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# CrossTrace
|
|
2
|
+
|
|
3
|
+
**A crash test for AI agents.** CrossTrace records real runs of your agent, replays them with no API cost, and breaks one model call at a
|
|
4
|
+
time the way providers and networks really break: a stream cut short, a stalled connection, a reset, a 429, an overloaded API. For every
|
|
5
|
+
failure it tells you what your agent did next. It runs in CI on every pull request.
|
|
6
|
+
|
|
7
|
+
Your agent needs no code changes and no SDK. It runs as it is, with its real SDK and framework code paths.
|
|
8
|
+
|
|
9
|
+
**[crosstrace-site.vercel.app](https://crosstrace-site.vercel.app/)**: the short version, with an animated walkthrough.
|
|
10
|
+
|
|
11
|
+
## What it finds in a stock LangGraph agent
|
|
12
|
+
|
|
13
|
+
A support agent written the way most LangGraph tutorials write one (`create_react_agent`, streaming `ChatOpenAI`, default settings,
|
|
14
|
+
three tools; [source](examples/real_agents/langgraph_support/agent.py)),
|
|
15
|
+
recorded once against a real model on OpenRouter, then tested offline:
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
✗ tool_args_cut SILENT 3/3
|
|
19
|
+
e.g. path 0 exchange 0: ran tool get_order with arguments {"order_id": 4} from a response that never
|
|
20
|
+
finished, instead of retrying exchange 0 (no retry)
|
|
21
|
+
✗ stream_stall HANG 5/5
|
|
22
|
+
e.g. path 0 exchange 0: no exit within 60 s (no retry)
|
|
23
|
+
✗ stream_cut_clean PARTIAL 2/5, ENDED_EARLY 3/5
|
|
24
|
+
e.g. path 0 exchange 0: printed nothing and exited 0 after exchange 0 failed
|
|
25
|
+
✓ server_error RECOVERED 5/5
|
|
26
|
+
✓ overloaded!persistent SURFACED 5/5
|
|
27
|
+
✓ rate_limit RECOVERED 5/5
|
|
28
|
+
✓ reset_before_response RECOVERED 5/5
|
|
29
|
+
✓ context_length SURFACED 5/5
|
|
30
|
+
! connection churn 1.0 new connections per model call
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
- **The stream broke inside a tool call, and the agent ran the tool anyway.** The model asked for order 42. LangChain completed the
|
|
34
|
+
half-received JSON, and the agent looked up **order 4**, with no error anywhere. The same happened with `get_order({})` and a tracking
|
|
35
|
+
number cut to `"BD1234"`.
|
|
36
|
+
- **A stalled stream froze the agent.** `ChatOpenAI` has no read timeout by default; the openai SDK waits up to 10 minutes per attempt.
|
|
37
|
+
Setting `ChatOpenAI(timeout=30)` turns the hang into an error (SURFACED 5/5), but not into a recovery: the SDK's retries stop once
|
|
38
|
+
response headers have arrived, so a mid-stream timeout is never retried.
|
|
39
|
+
- **A cut final answer was printed as the answer.** "Your order 42" and "I'm sorry your mug arrived broken, Asha." with exit code 0, or
|
|
40
|
+
nothing at all with exit code 0.
|
|
41
|
+
- **What it handled well:** HTTP errors before the stream starts. The SDK's retries cover those.
|
|
42
|
+
|
|
43
|
+
Each finding comes with the fix for the SDK it detected, and the exact command that reruns that one trial.
|
|
44
|
+
|
|
45
|
+
These are not exotic faults. A clean cut without the final event is what a gateway produces when its upstream dies but it closes the
|
|
46
|
+
response politely. The same behaviour shows up across SDKs: [what the OpenAI, Anthropic and Vercel SDKs do under each fault](docs/sdk-fault-table.md).
|
|
47
|
+
|
|
48
|
+
## Other agents checked
|
|
49
|
+
|
|
50
|
+
The same faults against other popular frameworks, each recorded once on a cheap real model and then replayed offline (recordings not published).
|
|
51
|
+
Every row is what CrossTrace reported after its verdicts were corrected against what the framework really did.
|
|
52
|
+
|
|
53
|
+
| Framework | Cut stream or cut tool call | Stalled or silent response | Other |
|
|
54
|
+
|---|---|---|---|
|
|
55
|
+
| LangGraph (`create_react_agent`) | **ran the tool on the cut arguments**, printed a cut answer as complete | hangs (no read timeout) | errors before the stream recovered |
|
|
56
|
+
| OpenAI Agents SDK (Responses API) | raised an error (it waits for the completed event) | hangs; `set_default_openai_client(AsyncOpenAI(timeout=...))` turns it into an error | errors before the stream recovered |
|
|
57
|
+
| pydantic-ai | refused the cut tool call and told the model (one case: arguments were already complete and it ran the tool); printed a cut answer as complete, also for a Responses `incomplete` status | hangs; `model_settings={"timeout": ...}` turns it into an error | empty reply re-prompted |
|
|
58
|
+
| CrewAI (native openai client) | printed a cut answer as complete | hangs; `LLM(timeout=...)` recovers | context-length error summarised and continued; a persistent outage retried 7+ times in about 3 s |
|
|
59
|
+
|
|
60
|
+
The timeout fixes were run, not assumed (with 10 seconds). In the three streamed agents (LangGraph, the Agents SDK, pydantic-ai) a stall in the middle of a stream is raised, not retried.
|
|
61
|
+
|
|
62
|
+
## Try it (no API key)
|
|
63
|
+
|
|
64
|
+
[examples/broken_agents](examples/broken_agents) is a small agent on the official openai SDK whose failure handling can be switched
|
|
65
|
+
between good and deliberately broken. Its recording (made against a local mock provider) is in the repo, so the test runs offline:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
git clone https://github.com/mac146/crosstrace && cd crosstrace
|
|
69
|
+
pip install . openai
|
|
70
|
+
cd examples/broken_agents
|
|
71
|
+
crosstrace test # the naive agent: hidden failures, exit code 1
|
|
72
|
+
sed -i 's/"naive"/"good"/' crosstrace.toml && crosstrace test # the same recording, a fixed agent: exit code 0
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
To reproduce the LangGraph results above, record the agent once with an OpenAI-compatible key (an OpenRouter key works as is, a few
|
|
76
|
+
cents), then test:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install -r examples/real_agents/langgraph_support/requirements.txt
|
|
80
|
+
cd examples/real_agents/langgraph_support
|
|
81
|
+
OPENAI_API_KEY=... crosstrace record && crosstrace test
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
A real model answers differently from run to run, so your recordings, and the exchanges a fault lands on, will differ from ours; the
|
|
85
|
+
behaviour under each fault is what to compare.
|
|
86
|
+
|
|
87
|
+
## Use it on your agent
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install git+https://github.com/mac146/crosstrace # not on PyPI yet
|
|
91
|
+
crosstrace init --command "python agent.py {input}" # writes crosstrace.toml: your inputs, timeouts, faults
|
|
92
|
+
crosstrace record # runs your agent for real, once per input (uses your API key)
|
|
93
|
+
crosstrace test # replays with faults; exit code 1 if your agent hides a failure
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Your agent must read its provider base URL from the environment, which the OpenAI and Anthropic SDKs, LangChain, LangGraph, LiteLLM,
|
|
97
|
+
pydantic-ai and the Vercel AI SDK all do ([checked for each](docs/framework-matrix.md)). If yours uses its own variable, map it in
|
|
98
|
+
`crosstrace.toml`: `env = { MY_BASE_URL = "{proxy_url}/v1" }`.
|
|
99
|
+
|
|
100
|
+
Commit `crosstrace.toml` and `.crosstrace/recordings/`, and CI tests every pull request without API keys:
|
|
101
|
+
|
|
102
|
+
```yaml
|
|
103
|
+
# .github/workflows/crosstrace.yml
|
|
104
|
+
on: [pull_request]
|
|
105
|
+
jobs:
|
|
106
|
+
resilience:
|
|
107
|
+
runs-on: ubuntu-latest
|
|
108
|
+
steps:
|
|
109
|
+
- uses: actions/checkout@v4
|
|
110
|
+
- uses: mac146/crosstrace@main
|
|
111
|
+
with:
|
|
112
|
+
install: pip install -r requirements.txt
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The report goes to the job summary. A nightly run tests every fault at every model call: [examples/github/crosstrace.yml](examples/github/crosstrace.yml).
|
|
116
|
+
|
|
117
|
+
## How it decides
|
|
118
|
+
|
|
119
|
+
CrossTrace sits between your agent and the provider as a local proxy. It breaks one model call per trial, at the byte level, so your
|
|
120
|
+
real SDK sees exactly what it would see in production. It then watches the agent's **next move**: did it retry that call, stop with an
|
|
121
|
+
error, or carry on as if nothing happened?
|
|
122
|
+
|
|
123
|
+
| verdict | meaning | |
|
|
124
|
+
|---|---|---|
|
|
125
|
+
| `RECOVERED` | retried (or fell back) and finished with the healthy answer | ok |
|
|
126
|
+
| `SURFACED` | stopped with an error | ok |
|
|
127
|
+
| `SILENT` | carried on with a missing or partial answer in its context | problem |
|
|
128
|
+
| `PARTIAL` | printed the cut answer as if it were complete | problem |
|
|
129
|
+
| `HANG` | never finished | problem |
|
|
130
|
+
| `RETRY_STORM`, `IGNORED_RETRY_AFTER`, `WRONG_RETRY` | retried too often, too soon, or what can't succeed | problem |
|
|
131
|
+
| `ENDED_EARLY`, `RECOVERED_DIFFERENT` | exited 0 without the failed step, or recovered with a different answer | review |
|
|
132
|
+
| `REPROMPTED` | answered the failure with a different request instead of retrying the same call: told the model its reply failed, had its tool arguments refused and the model told, or summarised after a context-length error (the text is shown, so you can judge it) | review |
|
|
133
|
+
| `DIVERGED` | after a reply that arrived but was defective, sent a request that matches nothing recorded, and nothing shows it acted on the reply or carried on as before. The run is then continued and settled by how it ends: `RECOVERED` if it finishes, `SURFACED` if it stops with an error, `RETRY_STORM` if it never stops. Only if the replay cannot answer a later request does it stay a review | review |
|
|
134
|
+
|
|
135
|
+
**What can fail your build, and what cannot.** A verdict fails the build only when it rests on something CrossTrace measured: a hang, too many
|
|
136
|
+
or too fast retries, a retried request that can never succeed, a write that ran twice, an answer printed cut, or an agent that acted on a
|
|
137
|
+
defective reply or carried on exactly as if nothing had happened. When an agent changes course in a way CrossTrace can only guess at, the
|
|
138
|
+
result is a **review** with the facts shown (what it sent, how many messages), never a failure. Every result in the JSON report carries
|
|
139
|
+
`observed`: the attempts, the status of the fault, and the agent's next move.
|
|
140
|
+
|
|
141
|
+
**The run continues past the fault.** When an agent answers a failure with a request the recording does not contain (it re-prompts the
|
|
142
|
+
model, summarises, rewrites its request), CrossTrace answers it with the next healthy reply of the recorded plan instead of an error, so the
|
|
143
|
+
run reaches its real end and the verdict comes from how it ended. A persistent fault (`"length_cut!persistent"`) keeps hitting the agent's
|
|
144
|
+
recovery attempts too, which tests whether its recovery has a bound: an agent that keeps asking after 8 different requests is a `RETRY_STORM`.
|
|
145
|
+
No model and no key are needed, so it stays deterministic and offline. It cannot tell whether a re-prompted agent would have received a good
|
|
146
|
+
answer from a real model; it only shows that the agent stops. To check that too, add `--live`: those requests then go to the real provider
|
|
147
|
+
with your own key (`OPENAI_API_KEY` or `ANTHROPIC_API_KEY`, read from the environment, never stored or logged), at most 6 calls per trial
|
|
148
|
+
(`--live-max-calls`), so a re-prompt is answered by a real model and the run ends on its merits. It spends money, so it is opt-in and meant for a
|
|
149
|
+
handful of trials (`--fault`, `--path`, `--at`), not a whole run. Turn continuation off with `--no-continue` or `[test] continue = false`.
|
|
150
|
+
|
|
151
|
+
No LLM judge and no comparison of free-text answers: the verdict comes from which model call the agent made next, so it is the same
|
|
152
|
+
every time. Faults are injected at every model call of every recorded path (nightly) or a fixed sample (pull requests), so an agent can't
|
|
153
|
+
pass by getting lucky on one run. Before any fault, each recording must replay exactly; if your prompts change on every run, CrossTrace
|
|
154
|
+
names the field to mask instead of reporting a false pass.
|
|
155
|
+
|
|
156
|
+
Details: [docs/cli.md](docs/cli.md). Fault list: `crosstrace faults`.
|
|
157
|
+
|
|
158
|
+
## Status and limits
|
|
159
|
+
|
|
160
|
+
Early (v0.1). Tested end to end on the example agents here and the SDK matrix above; not yet on anyone else's production agent. If you
|
|
161
|
+
try it on yours, an issue with what it found (or got wrong) is the most useful thing you can send.
|
|
162
|
+
|
|
163
|
+
- Model calls: OpenAI chat completions and Responses, Anthropic Messages. Tool calls: MCP over stdio and streamable HTTP, see above.
|
|
164
|
+
- Python 3.11+. Your agent can be in any language; it just has to be runnable as a command.
|
|
165
|
+
- The healthy run must be repeatable enough to replay. Random values in prompts need a mask; heavy run-to-run variation in what your
|
|
166
|
+
agent does needs several recorded runs.
|
|
167
|
+
|
|
168
|
+
## Tool calls (MCP): a tool that ran but whose reply was lost
|
|
169
|
+
|
|
170
|
+
A model call that fails is annoying. A tool call that **succeeded but whose response was lost** can do damage: the email was sent, the
|
|
171
|
+
card was charged, the form was submitted, the reply never reached the agent, and a retry does it a second time.
|
|
172
|
+
|
|
173
|
+
CrossTrace tests this for MCP servers over stdio and over streamable HTTP. For a stdio server, one change in your agent's MCP config:
|
|
174
|
+
launch it as `crosstrace mcp-wrap -- <its usual command>`. For a remote server, list its URL in `[mcp.http]` and have the agent read its
|
|
175
|
+
server address from `CROSSTRACE_MCP_URL_<NAME>`. The wrapper (or the local stand-in for the URL) records the tool calls in
|
|
176
|
+
`crosstrace record`, and in `crosstrace test` it **replays them without starting or calling the real server**, so a test cannot cause a
|
|
177
|
+
side effect. An agent with several servers names each one (`mcp-wrap --name github -- ...`, or the key in `[mcp.http]`).
|
|
178
|
+
|
|
179
|
+
- **`lost_response`:** one tool call is made (and counted), and its reply is replaced by a transport error. Then we watch what the agent does.
|
|
180
|
+
- **Write counting:** every call to a tool that changes something is logged. A write that ran more often than in the healthy run is a
|
|
181
|
+
**`DUPLICATE`**, in **every** trial, not only this one. A run that recovers and prints the right answer can still have written twice, so
|
|
182
|
+
recovering is not enough to pass. The same check catches an agent that recovers from a *model* failure by starting the task over.
|
|
183
|
+
- **Which tools are writes:** your `[mcp]` lists first, then what the server says about itself (a tool it declares `readOnlyHint` is a read),
|
|
184
|
+
then a guess from the name (`send`, `submit`, `pay`, `delete`, `click`, ...; `get`, `list`, `status` are reads).
|
|
185
|
+
List the ones it gets wrong in `[mcp] writes` / `reads`. For browser tools, click, type, fill_form, file_upload, press_key (Enter submits a
|
|
186
|
+
form), evaluate and run_code are writes; navigate, snapshot and screenshot are reads. `select_option` and `handle_dialog` are ambiguous and
|
|
187
|
+
count as reads: list them under `writes` if a choice or a confirmation can change something.
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
✗ lost_response DUPLICATE 1/1
|
|
191
|
+
e.g. path 0 tool call 0: refund({"amount":129.99,"customer_id":"42",...}) ran 2 time(s), 1 in the healthy run;
|
|
192
|
+
the agent recovered and finished, but the write was repeated
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Runnable without any key: [examples/mcp_agents](examples/mcp_agents) (`pip install mcp openai`). Also checked by hand against the official
|
|
196
|
+
`@playwright/mcp` 0.0.85 (25 tools): a scripted agent, a real Chrome and a local order form. Recording placed one real order; replays and
|
|
197
|
+
trials placed none. Losing the response to the click made the retrying agent **DUPLICATE** and the checking agent stop. That agent was
|
|
198
|
+
scripted, so it tests the tool-call handling, not how well a model drives a browser. **What this does not do yet:**
|
|
199
|
+
|
|
200
|
+
- Streamable HTTP is covered for what an agent sends as a POST (answered as JSON or an event stream). A server that pushes messages on its
|
|
201
|
+
own GET stream is not: that stream gets a 405, which the protocol allows. Checked against the official Python SDK's server and client only.
|
|
202
|
+
A lost response over HTTP is a 504 by default; with `how = "drop"` the connection is closed, which the official client turns into a crash
|
|
203
|
+
of the whole session, so the agent cannot retry at all.
|
|
204
|
+
- Several servers are handled by name. Each name must be used in both `record` and `test`; two unnamed stdio servers are not told apart.
|
|
205
|
+
- Your MCP client must start the wrapper with `CROSSTRACE_MCP` in its environment. The official Python SDK starts servers with a reduced
|
|
206
|
+
environment, so pass it on (the example does).
|
|
207
|
+
- Replay cannot answer a state check. An agent that asks a read tool "did it go through?" before retrying gets the recorded answer, or
|
|
208
|
+
"not recorded", so it shows as stopping with an error, not as recovering safely.
|
|
209
|
+
- A retry counts as a repeat unless the server declares the tool idempotent (`idempotentHint`) or the call carries an idempotency key as a
|
|
210
|
+
top-level argument (`idempotency_key`, `request_id`, `client_token` and similar). CrossTrace takes the key at its word: it cannot tell
|
|
211
|
+
whether the server honours it. The official Playwright server declares `readOnlyHint` for 6 of its 25 tools and no `idempotentHint`.
|
|
212
|
+
- By default the repeat is counted in replay, from a log of the tool calls. `crosstrace test --real-tools` runs the real servers in the
|
|
213
|
+
`lost_response` trials (the model is still replayed), so the retry really happens twice and the report quotes what the server answered.
|
|
214
|
+
It is opt-in because it does the work for real: point it at a test account or a sandbox. Once they are, set `[mcp] real = true` and every
|
|
215
|
+
run, CI included, shows the double write for real (`--no-real-tools` turns it off for one run). Model-fault trials always stay in replay. If the
|
|
216
|
+
real server answers differently from the recording, the agent's next model request may match nothing recorded.
|
|
217
|
+
- The lost response is a transport error on a live session, not a dead server process.
|
|
218
|
+
|
|
219
|
+
## Background
|
|
220
|
+
|
|
221
|
+
CrossTrace started as research on tracing AI agents across the application and the kernel (eBPF): 21 stages, predictions written
|
|
222
|
+
before each run. Mining real GitHub issues showed that most agent failures in the wild are failure-*handling* bugs in SDKs and frameworks,
|
|
223
|
+
not slow networks, which is why the product tests failure handling. The research is in [docs/research.md](docs/research.md).
|
|
224
|
+
|
|
225
|
+
MIT licensed, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""CrossTrace: find the failures AI agents hide, before production (see BUILD_PLAN.md, private).
|
|
2
|
+
|
|
3
|
+
The product package. The research prototype (agent/, collectors/, correlator/, diagnosers/, faultlab/, eval/) stays where it is and
|
|
4
|
+
is imported where it adds evidence.
|
|
5
|
+
"""
|
|
6
|
+
__version__ = "0.1.0"
|