crosstrace 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crosstrace/__init__.py +6 -0
- crosstrace/__main__.py +5 -0
- crosstrace/cli.py +309 -0
- crosstrace/config.py +251 -0
- crosstrace/fixes.py +140 -0
- crosstrace/masks.py +61 -0
- crosstrace/mcp/__init__.py +2 -0
- crosstrace/mcp/http.py +189 -0
- crosstrace/mcp/ledger.py +128 -0
- crosstrace/mcp/verdict.py +57 -0
- crosstrace/mcp/wrap.py +236 -0
- crosstrace/proxy/__init__.py +7 -0
- crosstrace/proxy/faults.py +538 -0
- crosstrace/proxy/http1.py +148 -0
- crosstrace/proxy/providers.py +78 -0
- crosstrace/proxy/recording.py +117 -0
- crosstrace/proxy/server.py +318 -0
- crosstrace/report.py +203 -0
- crosstrace/runner.py +396 -0
- crosstrace/verdict.py +485 -0
- crosstrace-0.1.0.dist-info/METADATA +60 -0
- crosstrace-0.1.0.dist-info/RECORD +26 -0
- crosstrace-0.1.0.dist-info/WHEEL +5 -0
- crosstrace-0.1.0.dist-info/entry_points.txt +2 -0
- crosstrace-0.1.0.dist-info/licenses/LICENSE +21 -0
- crosstrace-0.1.0.dist-info/top_level.txt +1 -0
crosstrace/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""CrossTrace: find the failures AI agents hide, before production (see BUILD_PLAN.md, private).
|
|
2
|
+
|
|
3
|
+
The product package. The research prototype (agent/, collectors/, correlator/, diagnosers/, faultlab/, eval/) stays where it is and
|
|
4
|
+
is imported where it adds evidence.
|
|
5
|
+
"""
|
|
6
|
+
__version__ = "0.1.0"
|
crosstrace/__main__.py
ADDED
crosstrace/cli.py
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""crosstrace: find the failures your AI agent hides.
|
|
2
|
+
|
|
3
|
+
crosstrace init write crosstrace.toml for your agent
|
|
4
|
+
crosstrace record run your agent for real and record healthy runs (uses your API keys once)
|
|
5
|
+
crosstrace test replay with faults, no API cost; report and exit code for CI
|
|
6
|
+
crosstrace report show the last report again
|
|
7
|
+
crosstrace faults list the faults
|
|
8
|
+
|
|
9
|
+
Exit codes: 0 no problems · 1 problems found · 2 usage or config error · 3 setup problem (no recordings, unstable replay, agent failed) · 4 internal error
|
|
10
|
+
"""
|
|
11
|
+
import argparse
|
|
12
|
+
import glob
|
|
13
|
+
import hashlib
|
|
14
|
+
import inspect
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import shlex
|
|
19
|
+
import sys
|
|
20
|
+
import time
|
|
21
|
+
|
|
22
|
+
from crosstrace import __version__, config as C, report as R, runner
|
|
23
|
+
from crosstrace.mcp.ledger import effective_lists, is_write, tool_calls
|
|
24
|
+
from crosstrace.proxy.faults import CATALOG, make_fault
|
|
25
|
+
from crosstrace.proxy.recording import Recording
|
|
26
|
+
from crosstrace.verdict import PROBLEM
|
|
27
|
+
|
|
28
|
+
EXIT_OK, EXIT_PROBLEMS, EXIT_USAGE, EXIT_SETUP, EXIT_INTERNAL = 0, 1, 2, 3, 4
|
|
29
|
+
NOT_APPLICABLE_WHY = {"tool_args_cut": "no streamed tool calls in the recordings",
|
|
30
|
+
"lost_response": "no MCP tool calls were recorded: launch the MCP server as `crosstrace mcp-wrap -- <command>`",
|
|
31
|
+
"length_cut": "no non-streamed text answers in the recordings",
|
|
32
|
+
"empty_content": "no non-streamed text answers in the recordings",
|
|
33
|
+
"malformed_body": "no non-streamed model calls in the recordings"}
|
|
34
|
+
NOT_APPLICABLE_WHY.update({f: "no streamed model calls in the recordings" for f in
|
|
35
|
+
("stream_cut_clean", "stream_cut_fin", "stream_cut_rst", "stream_stall", "slow_drip", "idle_kill",
|
|
36
|
+
"malformed_event", "stream_error_event")})
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def err(msg):
|
|
40
|
+
print(f"crosstrace: {msg}", file=sys.stderr)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def slug(text):
|
|
44
|
+
s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:40] or "run"
|
|
45
|
+
return f"{s}-{hashlib.sha1(text.encode()).hexdigest()[:6]}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# ---- init -----------------------------------------------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
def cmd_init(a):
|
|
51
|
+
path = os.path.join(os.getcwd(), C.CONFIG_NAME)
|
|
52
|
+
if os.path.exists(path) and not a.force:
|
|
53
|
+
err(f"{C.CONFIG_NAME} already exists (use --force to overwrite)")
|
|
54
|
+
return EXIT_USAGE
|
|
55
|
+
command = shlex.split(a.command) if a.command else ["python", "agent.py", "{input}"]
|
|
56
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
57
|
+
f.write(C.template(command))
|
|
58
|
+
os.makedirs(os.path.join(C.STATE_DIR, "recordings"), exist_ok=True)
|
|
59
|
+
with open(os.path.join(C.STATE_DIR, ".gitignore"), "w", encoding="utf-8") as f:
|
|
60
|
+
f.write("# recordings/ is meant to be committed (CI replays it); reports are local\nreports/\n")
|
|
61
|
+
print(f"wrote {C.CONFIG_NAME}. Next:\n 1. check [agent] command and inputs\n 2. crosstrace record (runs your agent for real once per input)\n"
|
|
62
|
+
f" 3. crosstrace test")
|
|
63
|
+
return EXIT_OK
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ---- record ---------------------------------------------------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
def spec_for(cfg, command, timeout_s=None):
|
|
69
|
+
return runner.AgentSpec(command=command, cwd=cfg.cwd, env=cfg.env, timeout_s=timeout_s or cfg.timeout_s, upstreams=cfg.upstreams,
|
|
70
|
+
masks=cfg.masks, mcp=cfg.mcp)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def cmd_record(a):
|
|
74
|
+
cfg = C.load(a.config)
|
|
75
|
+
os.makedirs(cfg.recordings_dir, exist_ok=True)
|
|
76
|
+
if a.clean:
|
|
77
|
+
for p in glob.glob(os.path.join(cfg.recordings_dir, "*.json")):
|
|
78
|
+
os.remove(p)
|
|
79
|
+
runs = a.runs or cfg.runs
|
|
80
|
+
print(f"recording {len(cfg.inputs)} input(s) x {runs} run(s) against the real providers"
|
|
81
|
+
+ (f" ({', '.join(f'{k}={v}' for k, v in cfg.upstreams.items())})" if cfg.upstreams else "") + "; this uses your API keys")
|
|
82
|
+
status, saved = EXIT_OK, 0
|
|
83
|
+
for inp in cfg.inputs:
|
|
84
|
+
for n in range(runs):
|
|
85
|
+
cmd = cfg.command_for(inp)
|
|
86
|
+
name = f"{slug(inp)}-{n + 1}.json"
|
|
87
|
+
try:
|
|
88
|
+
rec = runner.record(spec_for(cfg, cmd))
|
|
89
|
+
except RuntimeError as e:
|
|
90
|
+
err(f"{name}: {e}")
|
|
91
|
+
status = EXIT_SETUP
|
|
92
|
+
continue
|
|
93
|
+
rec.meta["input"] = inp
|
|
94
|
+
path = os.path.join(cfg.recordings_dir, name)
|
|
95
|
+
rec.save(path)
|
|
96
|
+
saved += 1
|
|
97
|
+
st = runner.stability(spec_for(cfg, cmd), rec, runs=2)
|
|
98
|
+
streamed = sum(e.stream for e in rec.exchanges)
|
|
99
|
+
recorded = (rec.meta.get("mcp") or {}).get("calls", [])
|
|
100
|
+
mcp_calls = tool_calls(recorded)
|
|
101
|
+
w, r, _ = effective_lists(cfg.mcp, recorded)
|
|
102
|
+
writes = sum(is_write(c["params"]["name"], w, r) for c in mcp_calls)
|
|
103
|
+
print(f" {name}: {len(rec.exchanges)} model calls ({streamed} streamed)"
|
|
104
|
+
+ (f", {len(mcp_calls)} MCP tool calls ({writes} writes)" if mcp_calls else "")
|
|
105
|
+
+ f", {rec.meta['seconds']:.1f} s, replay " + ("stable" if st["stable"] else "NOT stable"))
|
|
106
|
+
fails = rec.meta.get("upstream_failures", [])
|
|
107
|
+
if fails:
|
|
108
|
+
print(f" note: the real provider failed {len(fails)} time(s) during this run "
|
|
109
|
+
f"({', '.join(sorted({f['error'] for f in fails}))}) and your agent retried; only the successful attempts were recorded")
|
|
110
|
+
for p in st["problems"]:
|
|
111
|
+
print(f" {p}")
|
|
112
|
+
if not st["stable"]:
|
|
113
|
+
status = EXIT_SETUP
|
|
114
|
+
if saved:
|
|
115
|
+
print(f"{saved} recording(s) in {os.path.relpath(cfg.recordings_dir)}: they contain your prompts and the model's answers; review "
|
|
116
|
+
f"them, then commit them so CI can replay without API keys")
|
|
117
|
+
return status
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# ---- test -----------------------------------------------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
def load_recordings(cfg):
|
|
123
|
+
paths = sorted(glob.glob(os.path.join(cfg.recordings_dir, "*.json")))
|
|
124
|
+
return paths, [Recording.load(p) for p in paths]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def replay_command(cfg, rec):
|
|
128
|
+
"""Replay with the command from the current config (so a changed flag or a fixed agent is what gets tested); the stability check
|
|
129
|
+
tells if the recording no longer matches. Recordings made outside the CLI have no input: use their own command."""
|
|
130
|
+
return cfg.command_for(rec.meta["input"]) if "input" in rec.meta else rec.meta["command"]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
MIN_TRIAL_S = 60.0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def trial_budget(stability):
|
|
137
|
+
"""Seconds a trial may run before it is a HANG: 4x the slowest fault-free replay (the same agent under the same conditions; the
|
|
138
|
+
recorded wall time would include the real provider's latency and failed attempts), at least 60 s, so an agent with a sensible
|
|
139
|
+
30 s read timeout that retries once is never called a HANG."""
|
|
140
|
+
replays = [s["replay_seconds"] for s in stability if s.get("replay_seconds")]
|
|
141
|
+
return max(MIN_TRIAL_S, 4 * max(replays, default=0))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def cmd_test(a):
|
|
145
|
+
cfg = C.load(a.config)
|
|
146
|
+
if a.mode not in cfg.modes:
|
|
147
|
+
err(f"unknown mode {a.mode!r}; modes: {', '.join(sorted(cfg.modes))}")
|
|
148
|
+
return EXIT_USAGE
|
|
149
|
+
mode = cfg.modes[a.mode]
|
|
150
|
+
faults = mode.faults
|
|
151
|
+
if a.fault:
|
|
152
|
+
faults = [C._fault_spec(f, "--fault") for f in a.fault]
|
|
153
|
+
for fs in faults:
|
|
154
|
+
fs.params = {**cfg.fault_params.get(fs.name, {}), **fs.params}
|
|
155
|
+
jobs = a.jobs or mode.jobs
|
|
156
|
+
files, recs = load_recordings(cfg)
|
|
157
|
+
if not recs:
|
|
158
|
+
err(f"no recordings in {os.path.relpath(cfg.recordings_dir)}; run `crosstrace record` first")
|
|
159
|
+
return EXIT_SETUP
|
|
160
|
+
t0 = time.monotonic()
|
|
161
|
+
log = (lambda *x: None) if a.quiet else (lambda m: print(m, file=sys.stderr, flush=True))
|
|
162
|
+
log(f"checking replay of {len(recs)} recording(s)...")
|
|
163
|
+
stab = [runner.stability(spec_for(cfg, replay_command(cfg, r)), r, runs=mode.stability_runs) for r in recs]
|
|
164
|
+
timeout_s = float(cfg.trial_timeout_s or trial_budget(stab))
|
|
165
|
+
stable_paths = {i for i, s in enumerate(stab) if s["stable"]}
|
|
166
|
+
planned = [t for t in runner.plan_trials(recs, faults, mode.max_exchanges, a.seed, a.path, a.at) if t[0] in stable_paths]
|
|
167
|
+
planned_labels = {fs.label for _, fs, _ in planned}
|
|
168
|
+
not_applicable = {fs.label: NOT_APPLICABLE_WHY.get(fs.name, "no recorded model call it applies to")
|
|
169
|
+
for fs in faults if fs.label not in planned_labels and stable_paths}
|
|
170
|
+
log(f"running {len(planned)} trial(s), {jobs} at a time, budget {timeout_s:.0f} s each...")
|
|
171
|
+
done = [0]
|
|
172
|
+
|
|
173
|
+
def progress(r):
|
|
174
|
+
done[0] += 1
|
|
175
|
+
if r.verdict in PROBLEM:
|
|
176
|
+
log(f" [{done[0]}/{len(planned)}] {r.fault} at path {r.path} exchange {r.at}: {r.verdict}")
|
|
177
|
+
|
|
178
|
+
live_keys = None
|
|
179
|
+
if a.live:
|
|
180
|
+
live_keys = {k: os.environ.get(v) for k, v in (("openai", "OPENAI_API_KEY"), ("anthropic", "ANTHROPIC_API_KEY")) if os.environ.get(v)}
|
|
181
|
+
if not live_keys:
|
|
182
|
+
err("--live needs your real key in OPENAI_API_KEY or ANTHROPIC_API_KEY (the proxy sends it to the provider; the agent never sees it)")
|
|
183
|
+
return EXIT_USAGE
|
|
184
|
+
if not (cfg.continuation and not a.no_continue):
|
|
185
|
+
err("--live needs continuation (do not combine it with --no-continue or [test] continue = false)")
|
|
186
|
+
return EXIT_USAGE
|
|
187
|
+
log(f"live continuation: a request that matches nothing recorded goes to the real {'/'.join(sorted(live_keys))} API with your key "
|
|
188
|
+
f"(at most {a.live_max_calls} calls per trial); this spends money")
|
|
189
|
+
real_tools = (a.real_tools or bool(cfg.mcp.get("real"))) and not a.no_real_tools
|
|
190
|
+
if real_tools:
|
|
191
|
+
log("real tools: lost_response trials run the real MCP servers (the model is still replayed), so a retried write really happens twice; "
|
|
192
|
+
"point them at a test account or a sandbox")
|
|
193
|
+
results = runner.run_planned(lambda pi: spec_for(cfg, replay_command(cfg, recs[pi]), timeout_s), recs, planned, timeout_s, jobs, progress,
|
|
194
|
+
continuation=cfg.continuation and not a.no_continue, live_keys=live_keys, live_max_calls=a.live_max_calls,
|
|
195
|
+
real_tools=real_tools)
|
|
196
|
+
rep = R.build(cfg, mode, recs, stab, results, time.monotonic() - t0, a.seed, not_applicable)
|
|
197
|
+
print(R.for_terminal(R.render_text(rep)))
|
|
198
|
+
if not a.no_save:
|
|
199
|
+
R.save(rep, cfg.reports_dir)
|
|
200
|
+
if a.json:
|
|
201
|
+
with open(a.json, "w", encoding="utf-8") as f:
|
|
202
|
+
json.dump(rep, f, indent=1, default=str)
|
|
203
|
+
if a.md:
|
|
204
|
+
with open(a.md, "a", encoding="utf-8") as f: # append: works with $GITHUB_STEP_SUMMARY
|
|
205
|
+
f.write(R.render_markdown(rep))
|
|
206
|
+
return {"ok": EXIT_OK, "problems": EXIT_PROBLEMS, "setup_error": EXIT_SETUP}[rep["status"]]
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# ---- report / faults ------------------------------------------------------------------------------------------------------
|
|
210
|
+
|
|
211
|
+
def cmd_report(a):
|
|
212
|
+
path = a.path
|
|
213
|
+
if not path:
|
|
214
|
+
cfg = C.load(a.config)
|
|
215
|
+
path = os.path.join(cfg.reports_dir, "latest.json")
|
|
216
|
+
if not os.path.exists(path):
|
|
217
|
+
err(f"{path} not found; run `crosstrace test` first")
|
|
218
|
+
return EXIT_SETUP
|
|
219
|
+
with open(path, encoding="utf-8") as f:
|
|
220
|
+
rep = json.load(f)
|
|
221
|
+
if rep.get("schema") != R.SCHEMA:
|
|
222
|
+
err(f"{path}: unsupported report schema {rep.get('schema')!r}")
|
|
223
|
+
return EXIT_USAGE
|
|
224
|
+
print(R.for_terminal(R.render_markdown(rep) if a.md else R.render_text(rep)))
|
|
225
|
+
return EXIT_OK
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def cmd_faults(a):
|
|
229
|
+
for name in CATALOG:
|
|
230
|
+
f = make_fault(name)
|
|
231
|
+
doc = (inspect.getdoc(type(f)) or "").split("\n")[0]
|
|
232
|
+
params = ", ".join(f"{k}={v}" for k, v in f.p.items())
|
|
233
|
+
label = "streams only " if f.stream_only else "MCP tool calls " if f.tool_fault else ""
|
|
234
|
+
print(f"{name:24s} {label}{doc}" + (f" [{params}]" if params else ""))
|
|
235
|
+
print("\nIn crosstrace.toml or --fault: \"name\" heals after one failed attempt; \"name!persistent\" never heals.")
|
|
236
|
+
return EXIT_OK
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# ---- main -----------------------------------------------------------------------------------------------------------------
|
|
240
|
+
|
|
241
|
+
def build_parser():
|
|
242
|
+
ap = argparse.ArgumentParser(prog="crosstrace", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
243
|
+
ap.add_argument("--version", action="version", version=f"crosstrace {__version__}")
|
|
244
|
+
ap.add_argument("--config", default=None, help="path to crosstrace.toml (default: ./crosstrace.toml)")
|
|
245
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
246
|
+
p = sub.add_parser("init", help="write crosstrace.toml")
|
|
247
|
+
p.add_argument("--command", help='how to run your agent once, e.g. "python agent.py {input}"')
|
|
248
|
+
p.add_argument("--force", action="store_true")
|
|
249
|
+
p.set_defaults(fn=cmd_init)
|
|
250
|
+
p = sub.add_parser("record", help="record healthy runs (real API calls)")
|
|
251
|
+
p.add_argument("--runs", type=int, help="runs per input (default: [record] runs)")
|
|
252
|
+
p.add_argument("--clean", action="store_true", help="delete existing recordings first")
|
|
253
|
+
p.set_defaults(fn=cmd_record)
|
|
254
|
+
p = sub.add_parser("test", help="replay with faults and report")
|
|
255
|
+
p.add_argument("--mode", default="pr", help="pr (default, fast) or nightly (all faults, all exchanges)")
|
|
256
|
+
p.add_argument("--fault", action="append", help="only this fault (repeatable); name or name!persistent")
|
|
257
|
+
p.add_argument("--at", type=int, help="only this exchange")
|
|
258
|
+
p.add_argument("--path", type=int, help="only this recorded path")
|
|
259
|
+
p.add_argument("--jobs", type=int, help="trials in parallel")
|
|
260
|
+
p.add_argument("--seed", type=int, default=0, help="sampling seed for pr mode (default 0: same trials every run)")
|
|
261
|
+
p.add_argument("--json", help="also write the JSON report here")
|
|
262
|
+
p.add_argument("--md", help="append the Markdown report here (e.g. $GITHUB_STEP_SUMMARY)")
|
|
263
|
+
p.add_argument("--no-continue", action="store_true", help="do not answer requests that match nothing recorded after a fault (a 502 instead)")
|
|
264
|
+
p.add_argument("--live", action="store_true", help="send requests that match nothing recorded after a fault to the REAL provider with the key in "
|
|
265
|
+
"OPENAI_API_KEY / ANTHROPIC_API_KEY, so an agent's re-prompt is answered by a real model (spends money)")
|
|
266
|
+
p.add_argument("--no-real-tools", action="store_true", help="stay in replay even if [mcp] real = true")
|
|
267
|
+
p.add_argument("--real-tools", action="store_true", help="run the real MCP servers in lost_response trials, so a retried write really "
|
|
268
|
+
"happens twice and the server's answer to it is in the report (use a sandbox or test account)")
|
|
269
|
+
p.add_argument("--live-max-calls", type=int, default=6, help="real calls allowed per trial before the agent is judged as not stopping (default 6)")
|
|
270
|
+
p.add_argument("--no-save", action="store_true", help="do not write .crosstrace/reports/")
|
|
271
|
+
p.add_argument("--quiet", action="store_true")
|
|
272
|
+
p.set_defaults(fn=cmd_test)
|
|
273
|
+
p = sub.add_parser("report", help="show the last report")
|
|
274
|
+
p.add_argument("path", nargs="?")
|
|
275
|
+
p.add_argument("--md", action="store_true")
|
|
276
|
+
p.set_defaults(fn=cmd_report)
|
|
277
|
+
p = sub.add_parser("faults", help="list faults")
|
|
278
|
+
p.set_defaults(fn=cmd_faults)
|
|
279
|
+
p = sub.add_parser("mcp-wrap", help="run an MCP server through CrossTrace (put this in your agent's MCP config)")
|
|
280
|
+
p.add_argument("--name", help="name this server when the agent uses several (the same name is used for record and replay)")
|
|
281
|
+
p.add_argument("server", nargs=argparse.REMAINDER, help="-- <the server's usual command and arguments>")
|
|
282
|
+
p.set_defaults(fn=cmd_mcp_wrap)
|
|
283
|
+
return ap
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def cmd_mcp_wrap(a):
|
|
287
|
+
from crosstrace.mcp import wrap
|
|
288
|
+
return wrap.main((["--name", a.name] if a.name else []) + a.server)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def main(argv=None):
|
|
292
|
+
a = build_parser().parse_args(argv)
|
|
293
|
+
try:
|
|
294
|
+
return a.fn(a)
|
|
295
|
+
except C.ConfigError as e:
|
|
296
|
+
err(str(e))
|
|
297
|
+
return EXIT_USAGE
|
|
298
|
+
except KeyboardInterrupt:
|
|
299
|
+
err("interrupted")
|
|
300
|
+
return 130
|
|
301
|
+
except Exception: # never exit 1 on a crash: CI would read it as "the agent has problems"
|
|
302
|
+
import traceback
|
|
303
|
+
traceback.print_exc()
|
|
304
|
+
err(f"internal error (exit {EXIT_INTERNAL}); please report it with the traceback above")
|
|
305
|
+
return EXIT_INTERNAL
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
if __name__ == "__main__":
|
|
309
|
+
sys.exit(main())
|
crosstrace/config.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""crosstrace.toml: what to run, which inputs, which faults per mode. Every key is validated; unknown keys are errors (typos fail loudly).
|
|
2
|
+
|
|
3
|
+
TOML because Python 3.11+ reads it natively (no dependency) and it is typed.
|
|
4
|
+
"""
|
|
5
|
+
import os
|
|
6
|
+
import shlex
|
|
7
|
+
import tomllib
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
|
|
10
|
+
from crosstrace.proxy.faults import CATALOG, make_fault
|
|
11
|
+
|
|
12
|
+
CONFIG_NAME = "crosstrace.toml"
|
|
13
|
+
STATE_DIR = ".crosstrace"
|
|
14
|
+
|
|
15
|
+
# PR mode: the faults that most often reveal hidden failures, fast and deterministic. Nightly: the whole catalog.
|
|
16
|
+
PR_FAULTS = ["stream_cut_clean", "tool_args_cut", "stream_stall", "length_cut", "malformed_body", "server_error",
|
|
17
|
+
{"name": "overloaded", "heal": "never"}, "rate_limit", "reset_before_response", "context_length", "lost_response"]
|
|
18
|
+
NIGHTLY_EXTRA = ["stream_cut_fin", "stream_cut_rst", "stream_error_event", "malformed_event", "idle_kill", "slow_first_byte", "unavailable",
|
|
19
|
+
"rate_limit_http_date", "auth", "empty_content", {"name": "server_error", "heal": "never"}]
|
|
20
|
+
DEFAULT_MODES = {
|
|
21
|
+
"pr": {"faults": PR_FAULTS, "max_exchanges": 4, "stability_runs": 1, "jobs": 4},
|
|
22
|
+
"nightly": {"faults": PR_FAULTS + NIGHTLY_EXTRA, "max_exchanges": 0, "stability_runs": 2, "jobs": 4},
|
|
23
|
+
}
|
|
24
|
+
VERDICTS_ALLOWED_IN_ALLOW = {"ENDED_EARLY", "RECOVERED_DIFFERENT", "SILENT", "PARTIAL", "HANG", "RETRY_STORM", "IGNORED_RETRY_AFTER",
|
|
25
|
+
"WRONG_RETRY", "DUPLICATE", "REPROMPTED", "DIVERGED"}
|
|
26
|
+
|
|
27
|
+
TEMPLATE = '''# CrossTrace configuration. Docs: crosstrace --help, crosstrace faults
|
|
28
|
+
# Recordings in .crosstrace/recordings/ are meant to be committed (CI replays them without API keys).
|
|
29
|
+
# They contain the prompts and model answers of your healthy runs: add masks below, review before committing.
|
|
30
|
+
|
|
31
|
+
[agent]
|
|
32
|
+
# The command that runs your agent once. "{input}" is replaced by each input below.
|
|
33
|
+
command = {command}
|
|
34
|
+
# Each input is recorded as one path through your agent. More inputs = more paths covered.
|
|
35
|
+
inputs = ["Where is order 42 and when will it arrive?"]
|
|
36
|
+
timeout_s = 120 # a healthy run must finish within this
|
|
37
|
+
# sdk = "openai-python" # the SDK the fix advice is written for; default: read from the User-Agent (wrong or unrecognised? set it here)
|
|
38
|
+
# cwd = "."
|
|
39
|
+
# env = { MY_SETTING = "1" } # "{proxy_url}" is replaced by the proxy, e.g. MY_BASE_URL = "{proxy_url}/v1"
|
|
40
|
+
|
|
41
|
+
[record]
|
|
42
|
+
runs = 1 # healthy recordings per input (different sampled paths)
|
|
43
|
+
# upstreams = { openai = "https://api.openai.com", anthropic = "https://api.anthropic.com" }
|
|
44
|
+
|
|
45
|
+
[replay]
|
|
46
|
+
# Regexes for values that change between runs (timestamps and UUIDs are masked already).
|
|
47
|
+
masks = []
|
|
48
|
+
|
|
49
|
+
[test]
|
|
50
|
+
# Verdicts that should not fail the build (e.g. "ENDED_EARLY" if your agent ends politely on errors).
|
|
51
|
+
allow = []
|
|
52
|
+
# continue = false # default true: after a fault, a request that matches nothing recorded is answered with the next healthy reply, so the
|
|
53
|
+
# # run reaches its real end and the verdict comes from how it ended (not from the shape of a message)
|
|
54
|
+
# trial_timeout_s = 90 # a trial running longer is a HANG. Default: 4x the slowest fault-free replay, at least 60 s
|
|
55
|
+
|
|
56
|
+
# [modes.pr] # override the built-in modes: faults, max_exchanges (0 = all), stability_runs, jobs
|
|
57
|
+
# faults = ["stream_cut_clean", "server_error", { name = "overloaded", heal = "never" }]
|
|
58
|
+
|
|
59
|
+
# [faults.stream_stall] # fault parameters, see: crosstrace faults
|
|
60
|
+
# stall_s = 90
|
|
61
|
+
|
|
62
|
+
# MCP tools: launch your MCP server as ["crosstrace", "mcp-wrap", "--", <its usual command>] (see docs/cli.md).
|
|
63
|
+
# CrossTrace guesses which tools write (send, submit, pay, delete, click, ...); list the ones it gets wrong.
|
|
64
|
+
# [mcp]
|
|
65
|
+
# writes = ["run_report"]
|
|
66
|
+
# reads = ["refund_status"]
|
|
67
|
+
# real = true # only if these servers run against a test account or sandbox: lost_response trials then call them for real
|
|
68
|
+
'''
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class ConfigError(Exception):
|
|
72
|
+
pass
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class FaultSpec:
|
|
77
|
+
name: str
|
|
78
|
+
heal: object = 1 # attempts before it heals; None = never
|
|
79
|
+
params: dict = field(default_factory=dict)
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def label(self):
|
|
83
|
+
return self.name + ("!persistent" if self.heal is None else "")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class Mode:
|
|
88
|
+
name: str
|
|
89
|
+
faults: list
|
|
90
|
+
max_exchanges: int
|
|
91
|
+
stability_runs: int
|
|
92
|
+
jobs: int
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass
|
|
96
|
+
class Config:
|
|
97
|
+
root: str
|
|
98
|
+
command: list
|
|
99
|
+
inputs: list
|
|
100
|
+
timeout_s: float
|
|
101
|
+
cwd: str
|
|
102
|
+
env: dict
|
|
103
|
+
runs: int
|
|
104
|
+
upstreams: dict
|
|
105
|
+
masks: list
|
|
106
|
+
allow: set
|
|
107
|
+
trial_timeout_s: float
|
|
108
|
+
modes: dict
|
|
109
|
+
fault_params: dict
|
|
110
|
+
mcp: dict = field(default_factory=dict) # {"writes": [...], "reads": [...]}: tool names that override the write heuristic
|
|
111
|
+
sdk: str = None # the SDK family the fix advice is written for; None: read from the User-Agent
|
|
112
|
+
continuation: bool = True # answer requests that match nothing after a fault, so the run reaches its real end
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def state_dir(self):
|
|
116
|
+
return os.path.join(self.root, STATE_DIR)
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def recordings_dir(self):
|
|
120
|
+
return os.path.join(self.state_dir, "recordings")
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def reports_dir(self):
|
|
124
|
+
return os.path.join(self.state_dir, "reports")
|
|
125
|
+
|
|
126
|
+
def command_for(self, inp):
|
|
127
|
+
return [part.replace("{input}", inp) for part in self.command]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _check_keys(where, d, allowed):
|
|
131
|
+
unknown = set(d) - set(allowed)
|
|
132
|
+
if unknown:
|
|
133
|
+
raise ConfigError(f"{where}: unknown key(s) {', '.join(sorted(unknown))}; allowed: {', '.join(sorted(allowed))}")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _http_servers(value):
|
|
137
|
+
"""[mcp.http]: {name: url} for remote MCP servers (streamable HTTP)."""
|
|
138
|
+
if value is None:
|
|
139
|
+
return {}
|
|
140
|
+
if not isinstance(value, dict) or not all(isinstance(k, str) and isinstance(v, str) and v.startswith(("http://", "https://"))
|
|
141
|
+
for k, v in value.items()):
|
|
142
|
+
raise ConfigError("[mcp.http]: name = \"https://host/path\" for each remote MCP server")
|
|
143
|
+
return dict(value)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _sdk(value):
|
|
147
|
+
from crosstrace.fixes import TIMEOUT, COMPLETION
|
|
148
|
+
if value is None:
|
|
149
|
+
return None
|
|
150
|
+
known = sorted(set(TIMEOUT) | set(COMPLETION))
|
|
151
|
+
if value not in known:
|
|
152
|
+
raise ConfigError(f"[agent] sdk: {value!r} is not one of {', '.join(known)}")
|
|
153
|
+
return value
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _fault_spec(item, where):
|
|
157
|
+
if isinstance(item, str):
|
|
158
|
+
name, heal = (item[:-len("!persistent")], None) if item.endswith("!persistent") else (item, 1)
|
|
159
|
+
params = {}
|
|
160
|
+
elif isinstance(item, dict):
|
|
161
|
+
_check_keys(where, item, {"name", "heal", "params"})
|
|
162
|
+
name = item.get("name")
|
|
163
|
+
h = item.get("heal", 1)
|
|
164
|
+
heal = None if h in ("never", None) else h
|
|
165
|
+
if heal is not None and (not isinstance(heal, int) or heal < 1):
|
|
166
|
+
raise ConfigError(f"{where}: heal must be a positive number of failed attempts or \"never\"")
|
|
167
|
+
params = item.get("params", {})
|
|
168
|
+
else:
|
|
169
|
+
raise ConfigError(f"{where}: a fault is a name or {{ name = ..., heal = ..., params = {{...}} }}")
|
|
170
|
+
if name not in CATALOG:
|
|
171
|
+
raise ConfigError(f"{where}: unknown fault {name!r}; run `crosstrace faults` for the list")
|
|
172
|
+
return FaultSpec(name, heal, dict(params))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def load(path=None):
|
|
176
|
+
path = path or os.path.join(os.getcwd(), CONFIG_NAME)
|
|
177
|
+
if not os.path.exists(path):
|
|
178
|
+
raise ConfigError(f"{path} not found; run `crosstrace init` first")
|
|
179
|
+
try:
|
|
180
|
+
with open(path, "rb") as f:
|
|
181
|
+
d = tomllib.load(f)
|
|
182
|
+
except tomllib.TOMLDecodeError as e:
|
|
183
|
+
raise ConfigError(f"{path}: {e}")
|
|
184
|
+
return parse(d, os.path.dirname(os.path.abspath(path)))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def parse(d, root):
|
|
188
|
+
_check_keys("crosstrace.toml", d, {"agent", "record", "replay", "test", "modes", "faults", "mcp"})
|
|
189
|
+
mcp = d.get("mcp") or {}
|
|
190
|
+
_check_keys("[mcp]", mcp, {"writes", "reads", "http", "real"})
|
|
191
|
+
for key in ("writes", "reads"):
|
|
192
|
+
if not isinstance(mcp.get(key, []), list) or not all(isinstance(x, str) for x in mcp.get(key, [])):
|
|
193
|
+
raise ConfigError(f"[mcp] {key}: a list of tool names")
|
|
194
|
+
agent = d.get("agent") or {}
|
|
195
|
+
_check_keys("[agent]", agent, {"command", "inputs", "timeout_s", "cwd", "env", "sdk"})
|
|
196
|
+
cmd = agent.get("command")
|
|
197
|
+
if isinstance(cmd, str):
|
|
198
|
+
cmd = shlex.split(cmd)
|
|
199
|
+
if not cmd or not all(isinstance(c, str) for c in cmd):
|
|
200
|
+
raise ConfigError("[agent] command is required: a list like [\"python\", \"agent.py\", \"{input}\"] or a string")
|
|
201
|
+
inputs = agent.get("inputs") or [""]
|
|
202
|
+
if not isinstance(inputs, list) or not all(isinstance(i, str) for i in inputs):
|
|
203
|
+
raise ConfigError("[agent] inputs: a list of strings")
|
|
204
|
+
if len(inputs) > 1 and not any("{input}" in c for c in cmd):
|
|
205
|
+
raise ConfigError("[agent] has several inputs, but its command has no {input} placeholder")
|
|
206
|
+
rec = d.get("record") or {}
|
|
207
|
+
_check_keys("[record]", rec, {"runs", "upstreams"})
|
|
208
|
+
rep = d.get("replay") or {}
|
|
209
|
+
_check_keys("[replay]", rep, {"masks"})
|
|
210
|
+
test = d.get("test") or {}
|
|
211
|
+
_check_keys("[test]", test, {"allow", "trial_timeout_s", "continue"})
|
|
212
|
+
allow = set(test.get("allow", []))
|
|
213
|
+
bad = allow - VERDICTS_ALLOWED_IN_ALLOW
|
|
214
|
+
if bad:
|
|
215
|
+
raise ConfigError(f"[test] allow: unknown verdict(s) {sorted(bad)}; one of {sorted(VERDICTS_ALLOWED_IN_ALLOW)}")
|
|
216
|
+
upstreams = rec.get("upstreams", {})
|
|
217
|
+
_check_keys("[record] upstreams", upstreams, {"openai", "anthropic"})
|
|
218
|
+
|
|
219
|
+
fault_params = d.get("faults") or {}
|
|
220
|
+
for name, params in fault_params.items():
|
|
221
|
+
if name not in CATALOG:
|
|
222
|
+
raise ConfigError(f"[faults.{name}]: unknown fault; run `crosstrace faults` for the list")
|
|
223
|
+
try:
|
|
224
|
+
make_fault(name, **params)
|
|
225
|
+
except ValueError as e:
|
|
226
|
+
raise ConfigError(f"[faults.{name}]: {e}")
|
|
227
|
+
|
|
228
|
+
modes = {}
|
|
229
|
+
user_modes = d.get("modes") or {}
|
|
230
|
+
for mname in set(DEFAULT_MODES) | set(user_modes):
|
|
231
|
+
m = {**DEFAULT_MODES.get(mname, DEFAULT_MODES["nightly"]), **user_modes.get(mname, {})}
|
|
232
|
+
_check_keys(f"[modes.{mname}]", user_modes.get(mname, {}), {"faults", "max_exchanges", "stability_runs", "jobs"})
|
|
233
|
+
faults = [_fault_spec(x, f"[modes.{mname}] faults[{i}]") for i, x in enumerate(m["faults"])]
|
|
234
|
+
for fs in faults:
|
|
235
|
+
merged = {**fault_params.get(fs.name, {}), **fs.params}
|
|
236
|
+
try:
|
|
237
|
+
make_fault(fs.name, **merged)
|
|
238
|
+
except ValueError as e:
|
|
239
|
+
raise ConfigError(f"[modes.{mname}] {fs.name}: {e}")
|
|
240
|
+
fs.params = merged
|
|
241
|
+
modes[mname] = Mode(mname, faults, int(m["max_exchanges"]), max(1, int(m["stability_runs"])), max(1, int(m["jobs"])))
|
|
242
|
+
|
|
243
|
+
return Config(root=root, command=list(cmd), inputs=list(inputs), timeout_s=float(agent.get("timeout_s", 120)),
|
|
244
|
+
cwd=os.path.join(root, agent.get("cwd", ".")), env={k: str(v) for k, v in (agent.get("env") or {}).items()},
|
|
245
|
+
runs=max(1, int(rec.get("runs", 1))), upstreams=dict(upstreams), masks=list(rep.get("masks", [])), allow=allow,
|
|
246
|
+
trial_timeout_s=test.get("trial_timeout_s"), modes=modes, fault_params=fault_params,
|
|
247
|
+
mcp={"writes": list(mcp.get("writes", [])), "reads": list(mcp.get("reads", [])), "http": _http_servers(mcp.get("http")), "real": bool(mcp.get("real", False))}, continuation=bool(test.get("continue", True)), sdk=_sdk(agent.get("sdk")))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def template(command):
|
|
251
|
+
return TEMPLATE.replace("{command}", "[" + ", ".join(f'"{c}"' for c in command) + "]")
|