crosstrace 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
crosstrace/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ """CrossTrace: find the failures AI agents hide, before production (see BUILD_PLAN.md, private).
2
+
3
+ The product package. The research prototype (agent/, collectors/, correlator/, diagnosers/, faultlab/, eval/) stays where it is and
4
+ is imported where it adds evidence.
5
+ """
6
+ __version__ = "0.1.0"
crosstrace/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from crosstrace.cli import main
4
+
5
+ sys.exit(main())
crosstrace/cli.py ADDED
@@ -0,0 +1,309 @@
1
+ """crosstrace: find the failures your AI agent hides.
2
+
3
+ crosstrace init write crosstrace.toml for your agent
4
+ crosstrace record run your agent for real and record healthy runs (uses your API keys once)
5
+ crosstrace test replay with faults, no API cost; report and exit code for CI
6
+ crosstrace report show the last report again
7
+ crosstrace faults list the faults
8
+
9
+ Exit codes: 0 no problems · 1 problems found · 2 usage or config error · 3 setup problem (no recordings, unstable replay, agent failed) · 4 internal error
10
+ """
11
+ import argparse
12
+ import glob
13
+ import hashlib
14
+ import inspect
15
+ import json
16
+ import os
17
+ import re
18
+ import shlex
19
+ import sys
20
+ import time
21
+
22
+ from crosstrace import __version__, config as C, report as R, runner
23
+ from crosstrace.mcp.ledger import effective_lists, is_write, tool_calls
24
+ from crosstrace.proxy.faults import CATALOG, make_fault
25
+ from crosstrace.proxy.recording import Recording
26
+ from crosstrace.verdict import PROBLEM
27
+
28
+ EXIT_OK, EXIT_PROBLEMS, EXIT_USAGE, EXIT_SETUP, EXIT_INTERNAL = 0, 1, 2, 3, 4
29
+ NOT_APPLICABLE_WHY = {"tool_args_cut": "no streamed tool calls in the recordings",
30
+ "lost_response": "no MCP tool calls were recorded: launch the MCP server as `crosstrace mcp-wrap -- <command>`",
31
+ "length_cut": "no non-streamed text answers in the recordings",
32
+ "empty_content": "no non-streamed text answers in the recordings",
33
+ "malformed_body": "no non-streamed model calls in the recordings"}
34
+ NOT_APPLICABLE_WHY.update({f: "no streamed model calls in the recordings" for f in
35
+ ("stream_cut_clean", "stream_cut_fin", "stream_cut_rst", "stream_stall", "slow_drip", "idle_kill",
36
+ "malformed_event", "stream_error_event")})
37
+
38
+
39
+ def err(msg):
40
+ print(f"crosstrace: {msg}", file=sys.stderr)
41
+
42
+
43
+ def slug(text):
44
+ s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:40] or "run"
45
+ return f"{s}-{hashlib.sha1(text.encode()).hexdigest()[:6]}"
46
+
47
+
48
+ # ---- init -----------------------------------------------------------------------------------------------------------------
49
+
50
+ def cmd_init(a):
51
+ path = os.path.join(os.getcwd(), C.CONFIG_NAME)
52
+ if os.path.exists(path) and not a.force:
53
+ err(f"{C.CONFIG_NAME} already exists (use --force to overwrite)")
54
+ return EXIT_USAGE
55
+ command = shlex.split(a.command) if a.command else ["python", "agent.py", "{input}"]
56
+ with open(path, "w", encoding="utf-8") as f:
57
+ f.write(C.template(command))
58
+ os.makedirs(os.path.join(C.STATE_DIR, "recordings"), exist_ok=True)
59
+ with open(os.path.join(C.STATE_DIR, ".gitignore"), "w", encoding="utf-8") as f:
60
+ f.write("# recordings/ is meant to be committed (CI replays it); reports are local\nreports/\n")
61
+ print(f"wrote {C.CONFIG_NAME}. Next:\n 1. check [agent] command and inputs\n 2. crosstrace record (runs your agent for real once per input)\n"
62
+ f" 3. crosstrace test")
63
+ return EXIT_OK
64
+
65
+
66
+ # ---- record ---------------------------------------------------------------------------------------------------------------
67
+
68
+ def spec_for(cfg, command, timeout_s=None):
69
+ return runner.AgentSpec(command=command, cwd=cfg.cwd, env=cfg.env, timeout_s=timeout_s or cfg.timeout_s, upstreams=cfg.upstreams,
70
+ masks=cfg.masks, mcp=cfg.mcp)
71
+
72
+
73
+ def cmd_record(a):
74
+ cfg = C.load(a.config)
75
+ os.makedirs(cfg.recordings_dir, exist_ok=True)
76
+ if a.clean:
77
+ for p in glob.glob(os.path.join(cfg.recordings_dir, "*.json")):
78
+ os.remove(p)
79
+ runs = a.runs or cfg.runs
80
+ print(f"recording {len(cfg.inputs)} input(s) x {runs} run(s) against the real providers"
81
+ + (f" ({', '.join(f'{k}={v}' for k, v in cfg.upstreams.items())})" if cfg.upstreams else "") + "; this uses your API keys")
82
+ status, saved = EXIT_OK, 0
83
+ for inp in cfg.inputs:
84
+ for n in range(runs):
85
+ cmd = cfg.command_for(inp)
86
+ name = f"{slug(inp)}-{n + 1}.json"
87
+ try:
88
+ rec = runner.record(spec_for(cfg, cmd))
89
+ except RuntimeError as e:
90
+ err(f"{name}: {e}")
91
+ status = EXIT_SETUP
92
+ continue
93
+ rec.meta["input"] = inp
94
+ path = os.path.join(cfg.recordings_dir, name)
95
+ rec.save(path)
96
+ saved += 1
97
+ st = runner.stability(spec_for(cfg, cmd), rec, runs=2)
98
+ streamed = sum(e.stream for e in rec.exchanges)
99
+ recorded = (rec.meta.get("mcp") or {}).get("calls", [])
100
+ mcp_calls = tool_calls(recorded)
101
+ w, r, _ = effective_lists(cfg.mcp, recorded)
102
+ writes = sum(is_write(c["params"]["name"], w, r) for c in mcp_calls)
103
+ print(f" {name}: {len(rec.exchanges)} model calls ({streamed} streamed)"
104
+ + (f", {len(mcp_calls)} MCP tool calls ({writes} writes)" if mcp_calls else "")
105
+ + f", {rec.meta['seconds']:.1f} s, replay " + ("stable" if st["stable"] else "NOT stable"))
106
+ fails = rec.meta.get("upstream_failures", [])
107
+ if fails:
108
+ print(f" note: the real provider failed {len(fails)} time(s) during this run "
109
+ f"({', '.join(sorted({f['error'] for f in fails}))}) and your agent retried; only the successful attempts were recorded")
110
+ for p in st["problems"]:
111
+ print(f" {p}")
112
+ if not st["stable"]:
113
+ status = EXIT_SETUP
114
+ if saved:
115
+ print(f"{saved} recording(s) in {os.path.relpath(cfg.recordings_dir)}: they contain your prompts and the model's answers; review "
116
+ f"them, then commit them so CI can replay without API keys")
117
+ return status
118
+
119
+
120
+ # ---- test -----------------------------------------------------------------------------------------------------------------
121
+
122
+ def load_recordings(cfg):
123
+ paths = sorted(glob.glob(os.path.join(cfg.recordings_dir, "*.json")))
124
+ return paths, [Recording.load(p) for p in paths]
125
+
126
+
127
+ def replay_command(cfg, rec):
128
+ """Replay with the command from the current config (so a changed flag or a fixed agent is what gets tested); the stability check
129
+ tells if the recording no longer matches. Recordings made outside the CLI have no input: use their own command."""
130
+ return cfg.command_for(rec.meta["input"]) if "input" in rec.meta else rec.meta["command"]
131
+
132
+
133
+ MIN_TRIAL_S = 60.0
134
+
135
+
136
+ def trial_budget(stability):
137
+ """Seconds a trial may run before it is a HANG: 4x the slowest fault-free replay (the same agent under the same conditions; the
138
+ recorded wall time would include the real provider's latency and failed attempts), at least 60 s, so an agent with a sensible
139
+ 30 s read timeout that retries once is never called a HANG."""
140
+ replays = [s["replay_seconds"] for s in stability if s.get("replay_seconds")]
141
+ return max(MIN_TRIAL_S, 4 * max(replays, default=0))
142
+
143
+
144
+ def cmd_test(a):
145
+ cfg = C.load(a.config)
146
+ if a.mode not in cfg.modes:
147
+ err(f"unknown mode {a.mode!r}; modes: {', '.join(sorted(cfg.modes))}")
148
+ return EXIT_USAGE
149
+ mode = cfg.modes[a.mode]
150
+ faults = mode.faults
151
+ if a.fault:
152
+ faults = [C._fault_spec(f, "--fault") for f in a.fault]
153
+ for fs in faults:
154
+ fs.params = {**cfg.fault_params.get(fs.name, {}), **fs.params}
155
+ jobs = a.jobs or mode.jobs
156
+ files, recs = load_recordings(cfg)
157
+ if not recs:
158
+ err(f"no recordings in {os.path.relpath(cfg.recordings_dir)}; run `crosstrace record` first")
159
+ return EXIT_SETUP
160
+ t0 = time.monotonic()
161
+ log = (lambda *x: None) if a.quiet else (lambda m: print(m, file=sys.stderr, flush=True))
162
+ log(f"checking replay of {len(recs)} recording(s)...")
163
+ stab = [runner.stability(spec_for(cfg, replay_command(cfg, r)), r, runs=mode.stability_runs) for r in recs]
164
+ timeout_s = float(cfg.trial_timeout_s or trial_budget(stab))
165
+ stable_paths = {i for i, s in enumerate(stab) if s["stable"]}
166
+ planned = [t for t in runner.plan_trials(recs, faults, mode.max_exchanges, a.seed, a.path, a.at) if t[0] in stable_paths]
167
+ planned_labels = {fs.label for _, fs, _ in planned}
168
+ not_applicable = {fs.label: NOT_APPLICABLE_WHY.get(fs.name, "no recorded model call it applies to")
169
+ for fs in faults if fs.label not in planned_labels and stable_paths}
170
+ log(f"running {len(planned)} trial(s), {jobs} at a time, budget {timeout_s:.0f} s each...")
171
+ done = [0]
172
+
173
+ def progress(r):
174
+ done[0] += 1
175
+ if r.verdict in PROBLEM:
176
+ log(f" [{done[0]}/{len(planned)}] {r.fault} at path {r.path} exchange {r.at}: {r.verdict}")
177
+
178
+ live_keys = None
179
+ if a.live:
180
+ live_keys = {k: os.environ.get(v) for k, v in (("openai", "OPENAI_API_KEY"), ("anthropic", "ANTHROPIC_API_KEY")) if os.environ.get(v)}
181
+ if not live_keys:
182
+ err("--live needs your real key in OPENAI_API_KEY or ANTHROPIC_API_KEY (the proxy sends it to the provider; the agent never sees it)")
183
+ return EXIT_USAGE
184
+ if not (cfg.continuation and not a.no_continue):
185
+ err("--live needs continuation (do not combine it with --no-continue or [test] continue = false)")
186
+ return EXIT_USAGE
187
+ log(f"live continuation: a request that matches nothing recorded goes to the real {'/'.join(sorted(live_keys))} API with your key "
188
+ f"(at most {a.live_max_calls} calls per trial); this spends money")
189
+ real_tools = (a.real_tools or bool(cfg.mcp.get("real"))) and not a.no_real_tools
190
+ if real_tools:
191
+ log("real tools: lost_response trials run the real MCP servers (the model is still replayed), so a retried write really happens twice; "
192
+ "point them at a test account or a sandbox")
193
+ results = runner.run_planned(lambda pi: spec_for(cfg, replay_command(cfg, recs[pi]), timeout_s), recs, planned, timeout_s, jobs, progress,
194
+ continuation=cfg.continuation and not a.no_continue, live_keys=live_keys, live_max_calls=a.live_max_calls,
195
+ real_tools=real_tools)
196
+ rep = R.build(cfg, mode, recs, stab, results, time.monotonic() - t0, a.seed, not_applicable)
197
+ print(R.for_terminal(R.render_text(rep)))
198
+ if not a.no_save:
199
+ R.save(rep, cfg.reports_dir)
200
+ if a.json:
201
+ with open(a.json, "w", encoding="utf-8") as f:
202
+ json.dump(rep, f, indent=1, default=str)
203
+ if a.md:
204
+ with open(a.md, "a", encoding="utf-8") as f: # append: works with $GITHUB_STEP_SUMMARY
205
+ f.write(R.render_markdown(rep))
206
+ return {"ok": EXIT_OK, "problems": EXIT_PROBLEMS, "setup_error": EXIT_SETUP}[rep["status"]]
207
+
208
+
209
+ # ---- report / faults ------------------------------------------------------------------------------------------------------
210
+
211
+ def cmd_report(a):
212
+ path = a.path
213
+ if not path:
214
+ cfg = C.load(a.config)
215
+ path = os.path.join(cfg.reports_dir, "latest.json")
216
+ if not os.path.exists(path):
217
+ err(f"{path} not found; run `crosstrace test` first")
218
+ return EXIT_SETUP
219
+ with open(path, encoding="utf-8") as f:
220
+ rep = json.load(f)
221
+ if rep.get("schema") != R.SCHEMA:
222
+ err(f"{path}: unsupported report schema {rep.get('schema')!r}")
223
+ return EXIT_USAGE
224
+ print(R.for_terminal(R.render_markdown(rep) if a.md else R.render_text(rep)))
225
+ return EXIT_OK
226
+
227
+
228
+ def cmd_faults(a):
229
+ for name in CATALOG:
230
+ f = make_fault(name)
231
+ doc = (inspect.getdoc(type(f)) or "").split("\n")[0]
232
+ params = ", ".join(f"{k}={v}" for k, v in f.p.items())
233
+ label = "streams only " if f.stream_only else "MCP tool calls " if f.tool_fault else ""
234
+ print(f"{name:24s} {label}{doc}" + (f" [{params}]" if params else ""))
235
+ print("\nIn crosstrace.toml or --fault: \"name\" heals after one failed attempt; \"name!persistent\" never heals.")
236
+ return EXIT_OK
237
+
238
+
239
+ # ---- main -----------------------------------------------------------------------------------------------------------------
240
+
241
+ def build_parser():
242
+ ap = argparse.ArgumentParser(prog="crosstrace", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
243
+ ap.add_argument("--version", action="version", version=f"crosstrace {__version__}")
244
+ ap.add_argument("--config", default=None, help="path to crosstrace.toml (default: ./crosstrace.toml)")
245
+ sub = ap.add_subparsers(dest="cmd", required=True)
246
+ p = sub.add_parser("init", help="write crosstrace.toml")
247
+ p.add_argument("--command", help='how to run your agent once, e.g. "python agent.py {input}"')
248
+ p.add_argument("--force", action="store_true")
249
+ p.set_defaults(fn=cmd_init)
250
+ p = sub.add_parser("record", help="record healthy runs (real API calls)")
251
+ p.add_argument("--runs", type=int, help="runs per input (default: [record] runs)")
252
+ p.add_argument("--clean", action="store_true", help="delete existing recordings first")
253
+ p.set_defaults(fn=cmd_record)
254
+ p = sub.add_parser("test", help="replay with faults and report")
255
+ p.add_argument("--mode", default="pr", help="pr (default, fast) or nightly (all faults, all exchanges)")
256
+ p.add_argument("--fault", action="append", help="only this fault (repeatable); name or name!persistent")
257
+ p.add_argument("--at", type=int, help="only this exchange")
258
+ p.add_argument("--path", type=int, help="only this recorded path")
259
+ p.add_argument("--jobs", type=int, help="trials in parallel")
260
+ p.add_argument("--seed", type=int, default=0, help="sampling seed for pr mode (default 0: same trials every run)")
261
+ p.add_argument("--json", help="also write the JSON report here")
262
+ p.add_argument("--md", help="append the Markdown report here (e.g. $GITHUB_STEP_SUMMARY)")
263
+ p.add_argument("--no-continue", action="store_true", help="do not answer requests that match nothing recorded after a fault (a 502 instead)")
264
+ p.add_argument("--live", action="store_true", help="send requests that match nothing recorded after a fault to the REAL provider with the key in "
265
+ "OPENAI_API_KEY / ANTHROPIC_API_KEY, so an agent's re-prompt is answered by a real model (spends money)")
266
+ p.add_argument("--no-real-tools", action="store_true", help="stay in replay even if [mcp] real = true")
267
+ p.add_argument("--real-tools", action="store_true", help="run the real MCP servers in lost_response trials, so a retried write really "
268
+ "happens twice and the server's answer to it is in the report (use a sandbox or test account)")
269
+ p.add_argument("--live-max-calls", type=int, default=6, help="real calls allowed per trial before the agent is judged as not stopping (default 6)")
270
+ p.add_argument("--no-save", action="store_true", help="do not write .crosstrace/reports/")
271
+ p.add_argument("--quiet", action="store_true")
272
+ p.set_defaults(fn=cmd_test)
273
+ p = sub.add_parser("report", help="show the last report")
274
+ p.add_argument("path", nargs="?")
275
+ p.add_argument("--md", action="store_true")
276
+ p.set_defaults(fn=cmd_report)
277
+ p = sub.add_parser("faults", help="list faults")
278
+ p.set_defaults(fn=cmd_faults)
279
+ p = sub.add_parser("mcp-wrap", help="run an MCP server through CrossTrace (put this in your agent's MCP config)")
280
+ p.add_argument("--name", help="name this server when the agent uses several (the same name is used for record and replay)")
281
+ p.add_argument("server", nargs=argparse.REMAINDER, help="-- <the server's usual command and arguments>")
282
+ p.set_defaults(fn=cmd_mcp_wrap)
283
+ return ap
284
+
285
+
286
+ def cmd_mcp_wrap(a):
287
+ from crosstrace.mcp import wrap
288
+ return wrap.main((["--name", a.name] if a.name else []) + a.server)
289
+
290
+
291
+ def main(argv=None):
292
+ a = build_parser().parse_args(argv)
293
+ try:
294
+ return a.fn(a)
295
+ except C.ConfigError as e:
296
+ err(str(e))
297
+ return EXIT_USAGE
298
+ except KeyboardInterrupt:
299
+ err("interrupted")
300
+ return 130
301
+ except Exception: # never exit 1 on a crash: CI would read it as "the agent has problems"
302
+ import traceback
303
+ traceback.print_exc()
304
+ err(f"internal error (exit {EXIT_INTERNAL}); please report it with the traceback above")
305
+ return EXIT_INTERNAL
306
+
307
+
308
+ if __name__ == "__main__":
309
+ sys.exit(main())
crosstrace/config.py ADDED
@@ -0,0 +1,251 @@
1
+ """crosstrace.toml: what to run, which inputs, which faults per mode. Every key is validated; unknown keys are errors (typos fail loudly).
2
+
3
+ TOML because Python 3.11+ reads it natively (no dependency) and it is typed.
4
+ """
5
+ import os
6
+ import shlex
7
+ import tomllib
8
+ from dataclasses import dataclass, field
9
+
10
+ from crosstrace.proxy.faults import CATALOG, make_fault
11
+
12
+ CONFIG_NAME = "crosstrace.toml"
13
+ STATE_DIR = ".crosstrace"
14
+
15
+ # PR mode: the faults that most often reveal hidden failures, fast and deterministic. Nightly: the whole catalog.
16
+ PR_FAULTS = ["stream_cut_clean", "tool_args_cut", "stream_stall", "length_cut", "malformed_body", "server_error",
17
+ {"name": "overloaded", "heal": "never"}, "rate_limit", "reset_before_response", "context_length", "lost_response"]
18
+ NIGHTLY_EXTRA = ["stream_cut_fin", "stream_cut_rst", "stream_error_event", "malformed_event", "idle_kill", "slow_first_byte", "unavailable",
19
+ "rate_limit_http_date", "auth", "empty_content", {"name": "server_error", "heal": "never"}]
20
+ DEFAULT_MODES = {
21
+ "pr": {"faults": PR_FAULTS, "max_exchanges": 4, "stability_runs": 1, "jobs": 4},
22
+ "nightly": {"faults": PR_FAULTS + NIGHTLY_EXTRA, "max_exchanges": 0, "stability_runs": 2, "jobs": 4},
23
+ }
24
+ VERDICTS_ALLOWED_IN_ALLOW = {"ENDED_EARLY", "RECOVERED_DIFFERENT", "SILENT", "PARTIAL", "HANG", "RETRY_STORM", "IGNORED_RETRY_AFTER",
25
+ "WRONG_RETRY", "DUPLICATE", "REPROMPTED", "DIVERGED"}
26
+
27
+ TEMPLATE = '''# CrossTrace configuration. Docs: crosstrace --help, crosstrace faults
28
+ # Recordings in .crosstrace/recordings/ are meant to be committed (CI replays them without API keys).
29
+ # They contain the prompts and model answers of your healthy runs: add masks below, review before committing.
30
+
31
+ [agent]
32
+ # The command that runs your agent once. "{input}" is replaced by each input below.
33
+ command = {command}
34
+ # Each input is recorded as one path through your agent. More inputs = more paths covered.
35
+ inputs = ["Where is order 42 and when will it arrive?"]
36
+ timeout_s = 120 # a healthy run must finish within this
37
+ # sdk = "openai-python" # the SDK the fix advice is written for; default: read from the User-Agent (wrong or unrecognised? set it here)
38
+ # cwd = "."
39
+ # env = { MY_SETTING = "1" } # "{proxy_url}" is replaced by the proxy, e.g. MY_BASE_URL = "{proxy_url}/v1"
40
+
41
+ [record]
42
+ runs = 1 # healthy recordings per input (different sampled paths)
43
+ # upstreams = { openai = "https://api.openai.com", anthropic = "https://api.anthropic.com" }
44
+
45
+ [replay]
46
+ # Regexes for values that change between runs (timestamps and UUIDs are masked already).
47
+ masks = []
48
+
49
+ [test]
50
+ # Verdicts that should not fail the build (e.g. "ENDED_EARLY" if your agent ends politely on errors).
51
+ allow = []
52
+ # continue = false # default true: after a fault, a request that matches nothing recorded is answered with the next healthy reply, so the
53
+ # # run reaches its real end and the verdict comes from how it ended (not from the shape of a message)
54
+ # trial_timeout_s = 90 # a trial running longer is a HANG. Default: 4x the slowest fault-free replay, at least 60 s
55
+
56
+ # [modes.pr] # override the built-in modes: faults, max_exchanges (0 = all), stability_runs, jobs
57
+ # faults = ["stream_cut_clean", "server_error", { name = "overloaded", heal = "never" }]
58
+
59
+ # [faults.stream_stall] # fault parameters, see: crosstrace faults
60
+ # stall_s = 90
61
+
62
+ # MCP tools: launch your MCP server as ["crosstrace", "mcp-wrap", "--", <its usual command>] (see docs/cli.md).
63
+ # CrossTrace guesses which tools write (send, submit, pay, delete, click, ...); list the ones it gets wrong.
64
+ # [mcp]
65
+ # writes = ["run_report"]
66
+ # reads = ["refund_status"]
67
+ # real = true # only if these servers run against a test account or sandbox: lost_response trials then call them for real
68
+ '''
69
+
70
+
71
+ class ConfigError(Exception):
72
+ pass
73
+
74
+
75
+ @dataclass
76
+ class FaultSpec:
77
+ name: str
78
+ heal: object = 1 # attempts before it heals; None = never
79
+ params: dict = field(default_factory=dict)
80
+
81
+ @property
82
+ def label(self):
83
+ return self.name + ("!persistent" if self.heal is None else "")
84
+
85
+
86
+ @dataclass
87
+ class Mode:
88
+ name: str
89
+ faults: list
90
+ max_exchanges: int
91
+ stability_runs: int
92
+ jobs: int
93
+
94
+
95
+ @dataclass
96
+ class Config:
97
+ root: str
98
+ command: list
99
+ inputs: list
100
+ timeout_s: float
101
+ cwd: str
102
+ env: dict
103
+ runs: int
104
+ upstreams: dict
105
+ masks: list
106
+ allow: set
107
+ trial_timeout_s: float
108
+ modes: dict
109
+ fault_params: dict
110
+ mcp: dict = field(default_factory=dict) # {"writes": [...], "reads": [...]}: tool names that override the write heuristic
111
+ sdk: str = None # the SDK family the fix advice is written for; None: read from the User-Agent
112
+ continuation: bool = True # answer requests that match nothing after a fault, so the run reaches its real end
113
+
114
+ @property
115
+ def state_dir(self):
116
+ return os.path.join(self.root, STATE_DIR)
117
+
118
+ @property
119
+ def recordings_dir(self):
120
+ return os.path.join(self.state_dir, "recordings")
121
+
122
+ @property
123
+ def reports_dir(self):
124
+ return os.path.join(self.state_dir, "reports")
125
+
126
+ def command_for(self, inp):
127
+ return [part.replace("{input}", inp) for part in self.command]
128
+
129
+
130
+ def _check_keys(where, d, allowed):
131
+ unknown = set(d) - set(allowed)
132
+ if unknown:
133
+ raise ConfigError(f"{where}: unknown key(s) {', '.join(sorted(unknown))}; allowed: {', '.join(sorted(allowed))}")
134
+
135
+
136
+ def _http_servers(value):
137
+ """[mcp.http]: {name: url} for remote MCP servers (streamable HTTP)."""
138
+ if value is None:
139
+ return {}
140
+ if not isinstance(value, dict) or not all(isinstance(k, str) and isinstance(v, str) and v.startswith(("http://", "https://"))
141
+ for k, v in value.items()):
142
+ raise ConfigError("[mcp.http]: name = \"https://host/path\" for each remote MCP server")
143
+ return dict(value)
144
+
145
+
146
+ def _sdk(value):
147
+ from crosstrace.fixes import TIMEOUT, COMPLETION
148
+ if value is None:
149
+ return None
150
+ known = sorted(set(TIMEOUT) | set(COMPLETION))
151
+ if value not in known:
152
+ raise ConfigError(f"[agent] sdk: {value!r} is not one of {', '.join(known)}")
153
+ return value
154
+
155
+
156
+ def _fault_spec(item, where):
157
+ if isinstance(item, str):
158
+ name, heal = (item[:-len("!persistent")], None) if item.endswith("!persistent") else (item, 1)
159
+ params = {}
160
+ elif isinstance(item, dict):
161
+ _check_keys(where, item, {"name", "heal", "params"})
162
+ name = item.get("name")
163
+ h = item.get("heal", 1)
164
+ heal = None if h in ("never", None) else h
165
+ if heal is not None and (not isinstance(heal, int) or heal < 1):
166
+ raise ConfigError(f"{where}: heal must be a positive number of failed attempts or \"never\"")
167
+ params = item.get("params", {})
168
+ else:
169
+ raise ConfigError(f"{where}: a fault is a name or {{ name = ..., heal = ..., params = {{...}} }}")
170
+ if name not in CATALOG:
171
+ raise ConfigError(f"{where}: unknown fault {name!r}; run `crosstrace faults` for the list")
172
+ return FaultSpec(name, heal, dict(params))
173
+
174
+
175
+ def load(path=None):
176
+ path = path or os.path.join(os.getcwd(), CONFIG_NAME)
177
+ if not os.path.exists(path):
178
+ raise ConfigError(f"{path} not found; run `crosstrace init` first")
179
+ try:
180
+ with open(path, "rb") as f:
181
+ d = tomllib.load(f)
182
+ except tomllib.TOMLDecodeError as e:
183
+ raise ConfigError(f"{path}: {e}")
184
+ return parse(d, os.path.dirname(os.path.abspath(path)))
185
+
186
+
187
+ def parse(d, root):
188
+ _check_keys("crosstrace.toml", d, {"agent", "record", "replay", "test", "modes", "faults", "mcp"})
189
+ mcp = d.get("mcp") or {}
190
+ _check_keys("[mcp]", mcp, {"writes", "reads", "http", "real"})
191
+ for key in ("writes", "reads"):
192
+ if not isinstance(mcp.get(key, []), list) or not all(isinstance(x, str) for x in mcp.get(key, [])):
193
+ raise ConfigError(f"[mcp] {key}: a list of tool names")
194
+ agent = d.get("agent") or {}
195
+ _check_keys("[agent]", agent, {"command", "inputs", "timeout_s", "cwd", "env", "sdk"})
196
+ cmd = agent.get("command")
197
+ if isinstance(cmd, str):
198
+ cmd = shlex.split(cmd)
199
+ if not cmd or not all(isinstance(c, str) for c in cmd):
200
+ raise ConfigError("[agent] command is required: a list like [\"python\", \"agent.py\", \"{input}\"] or a string")
201
+ inputs = agent.get("inputs") or [""]
202
+ if not isinstance(inputs, list) or not all(isinstance(i, str) for i in inputs):
203
+ raise ConfigError("[agent] inputs: a list of strings")
204
+ if len(inputs) > 1 and not any("{input}" in c for c in cmd):
205
+ raise ConfigError("[agent] has several inputs, but its command has no {input} placeholder")
206
+ rec = d.get("record") or {}
207
+ _check_keys("[record]", rec, {"runs", "upstreams"})
208
+ rep = d.get("replay") or {}
209
+ _check_keys("[replay]", rep, {"masks"})
210
+ test = d.get("test") or {}
211
+ _check_keys("[test]", test, {"allow", "trial_timeout_s", "continue"})
212
+ allow = set(test.get("allow", []))
213
+ bad = allow - VERDICTS_ALLOWED_IN_ALLOW
214
+ if bad:
215
+ raise ConfigError(f"[test] allow: unknown verdict(s) {sorted(bad)}; one of {sorted(VERDICTS_ALLOWED_IN_ALLOW)}")
216
+ upstreams = rec.get("upstreams", {})
217
+ _check_keys("[record] upstreams", upstreams, {"openai", "anthropic"})
218
+
219
+ fault_params = d.get("faults") or {}
220
+ for name, params in fault_params.items():
221
+ if name not in CATALOG:
222
+ raise ConfigError(f"[faults.{name}]: unknown fault; run `crosstrace faults` for the list")
223
+ try:
224
+ make_fault(name, **params)
225
+ except ValueError as e:
226
+ raise ConfigError(f"[faults.{name}]: {e}")
227
+
228
+ modes = {}
229
+ user_modes = d.get("modes") or {}
230
+ for mname in set(DEFAULT_MODES) | set(user_modes):
231
+ m = {**DEFAULT_MODES.get(mname, DEFAULT_MODES["nightly"]), **user_modes.get(mname, {})}
232
+ _check_keys(f"[modes.{mname}]", user_modes.get(mname, {}), {"faults", "max_exchanges", "stability_runs", "jobs"})
233
+ faults = [_fault_spec(x, f"[modes.{mname}] faults[{i}]") for i, x in enumerate(m["faults"])]
234
+ for fs in faults:
235
+ merged = {**fault_params.get(fs.name, {}), **fs.params}
236
+ try:
237
+ make_fault(fs.name, **merged)
238
+ except ValueError as e:
239
+ raise ConfigError(f"[modes.{mname}] {fs.name}: {e}")
240
+ fs.params = merged
241
+ modes[mname] = Mode(mname, faults, int(m["max_exchanges"]), max(1, int(m["stability_runs"])), max(1, int(m["jobs"])))
242
+
243
+ return Config(root=root, command=list(cmd), inputs=list(inputs), timeout_s=float(agent.get("timeout_s", 120)),
244
+ cwd=os.path.join(root, agent.get("cwd", ".")), env={k: str(v) for k, v in (agent.get("env") or {}).items()},
245
+ runs=max(1, int(rec.get("runs", 1))), upstreams=dict(upstreams), masks=list(rep.get("masks", [])), allow=allow,
246
+ trial_timeout_s=test.get("trial_timeout_s"), modes=modes, fault_params=fault_params,
247
+ mcp={"writes": list(mcp.get("writes", [])), "reads": list(mcp.get("reads", [])), "http": _http_servers(mcp.get("http")), "real": bool(mcp.get("real", False))}, continuation=bool(test.get("continue", True)), sdk=_sdk(agent.get("sdk")))
248
+
249
+
250
+ def template(command):
251
+ return TEMPLATE.replace("{command}", "[" + ", ".join(f'"{c}"' for c in command) + "]")