custos-code 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
custos_code/cli.py ADDED
@@ -0,0 +1,789 @@
1
+ """Typer entrypoint: check, watch, bench, eval, cost, and the internal `_hook` group.
2
+
3
+ `_hook` is what hooks/*.sh invoke (and, for `rerun-worker`, what `rerun.spawn_async` invokes
4
+ directly); it is not a user-facing command. Its subcommands either read one hook payload from
5
+ stdin, or -- `rerun-worker` only -- take their identity as positional args since they have no
6
+ hook payload at all. See docs/ADAPTERS.md §2 for the payload shapes.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import pathlib
12
+ import re
13
+ from dataclasses import dataclass
14
+
15
+ import typer
16
+ from rich.console import Console
17
+ from rich.table import Table
18
+
19
+ from . import adapters as adapters_mod
20
+ from . import claims as claims_mod
21
+ from . import judge as judge_mod
22
+ from . import report as report_mod
23
+ from . import review as review_mod
24
+ from . import scope as scope_mod
25
+ from . import verdicts as verdicts_mod
26
+ from .adapters import claude_code, codex, machine
27
+ from .models import Claim, EventKind, Verdict, VerdictRecord
28
+
29
+ MARK = {
30
+ Verdict.CONFIRMED: ("✓", "green"),
31
+ Verdict.CONTRADICTED: ("✗", "red"),
32
+ Verdict.UNWITNESSED: ("?", "yellow"),
33
+ Verdict.UNRECORDED: ("○", "bright_black"),
34
+ Verdict.QUALIFIED: ("≈", "cyan"),
35
+ Verdict.OUT_OF_SCOPE: ("⚠", "magenta"),
36
+ }
37
+
38
+ app = typer.Typer(
39
+ help="Check a coding agent's final report against what it actually did.", no_args_is_help=True
40
+ )
41
+ console = Console()
42
+
43
+
44
+ @app.command()
45
+ def check(
46
+ session: str | None = typer.Argument(
47
+ None, help="Path to a session transcript, rollout, or bundle."
48
+ ),
49
+ last: bool = typer.Option(
50
+ False, "--last", help="Use the most recent session of --agent (default Claude Code)."
51
+ ),
52
+ agent: str | None = typer.Option(
53
+ None,
54
+ "--agent",
55
+ help="claude_code | codex | devin | copilot | machine | otel (default: detect).",
56
+ ),
57
+ events: bool = typer.Option(False, "--events", help="Also print the ledger."),
58
+ evidence: bool = typer.Option(
59
+ False, "--evidence", help="Print the cited ledger lines under each claim."
60
+ ),
61
+ repo: str | None = typer.Option(
62
+ None, "--repo", help="Repo root for state checks (default: the session's cwd)."
63
+ ),
64
+ rules_only: bool = typer.Option(
65
+ False, "--rules-only", help="Deterministic rules only; no model call."
66
+ ),
67
+ ladder: bool = typer.Option(
68
+ False, "--ladder", help="Use the superseded tiered pipeline instead of review."
69
+ ),
70
+ fmt: str = typer.Option("terminal", "--format", help="terminal | markdown | html"),
71
+ out_path: str | None = typer.Option(
72
+ None, "--out", help="Write the rendered receipt to a file."
73
+ ),
74
+ ) -> None:
75
+ """Print the receipt for one session: every claim in the final report, with its verdict and evidence."""
76
+ if last:
77
+ session = codex.find_last_session() if agent == "codex" else claude_code.find_last_session()
78
+ if not session:
79
+ raise typer.BadParameter("give a session path or --last")
80
+ sess, ledger, report = adapters_mod.parse(session, agent)
81
+ calls = sum(1 for e in ledger if e.kind == EventKind.CALL)
82
+ flagged = sum(1 for e in ledger if e.flags.piped or e.flags.truncated or e.flags.error)
83
+
84
+ if fmt == "terminal":
85
+ console.print(
86
+ f"[bold]custos-code[/] session {sess.id[:8]}… · {sess.n_events} events · {calls} tool calls · "
87
+ f"{flagged} flagged · chain {sess.ledger_root_hash[:8]}… · cwd {sess.cwd}"
88
+ )
89
+ if events:
90
+ t = Table(show_header=True, header_style="dim")
91
+ for col in ("#", "kind", "tool", "detail", "flags"):
92
+ t.add_column(col)
93
+ for e in ledger:
94
+ detail = ""
95
+ if e.kind == EventKind.CALL and e.input:
96
+ detail = str(
97
+ e.input.get("command")
98
+ or e.input.get("file_path")
99
+ or e.input.get("path")
100
+ or ""
101
+ )[:90]
102
+ elif e.output:
103
+ detail = e.output.replace("\n", " ⏎ ")[:90]
104
+ fl = " ".join(k for k, v in e.flags.model_dump().items() if v)
105
+ t.add_row(str(e.seq), e.kind.value, e.tool or "", detail, fl)
106
+ console.print(t)
107
+ console.rule("final report")
108
+ console.print(report or "[dim](no assistant text found)[/]")
109
+ console.rule("receipt")
110
+ if not report:
111
+ raise typer.Exit(code=0)
112
+
113
+ backend = None if rules_only else judge_mod.make_backend()
114
+ if backend is not None and not ladder:
115
+ # Default: one call over the report and the annotated ledger. 86% vs the ladder's 70%
116
+ # on construction-truth fixtures, McNemar p=0.00017 (eval/arms/RESULTS.md).
117
+ reviewed = review_mod.review(report, ledger, sess.id, backend,
118
+ repo_root=repo or sess.cwd)
119
+ claims = reviewed.claims
120
+ recs = verdicts_mod.apply_reruns(claims, reviewed.verdicts, ledger)
121
+ tail = f"one call · {reviewed.input_tokens} in / {reviewed.output_tokens} out"
122
+ else:
123
+ if backend is None and not rules_only:
124
+ console.print(
125
+ "[yellow]no model backend: set OPENAI_API_KEY; falling back to rules only[/]"
126
+ )
127
+ claims = claims_mod.extract(report, sess.id)
128
+ recs = verdicts_mod.run(claims, ledger, repo or sess.cwd, backend)
129
+ tail = (
130
+ "rules only"
131
+ if backend is None
132
+ else f"rules + judge · {backend.usage.requests} requests"
133
+ )
134
+
135
+ # Scope (SCOPE.md §4, issue #64): a second, independent pass over the same ledger. It never
136
+ # reads or touches the claim verdicts above -- neither checker reads the other's output -- and
137
+ # only reports on tool calls that actually ran (a call PreToolUse denied leaves no CALL event).
138
+ grant = scope_mod.Grant.for_session(repo or sess.cwd or "", policy=scope_mod.Policy.load())
139
+ for scope_event, finding in scope_mod.scan(ledger, grant):
140
+ scope_claim, scope_rec = scope_mod.to_verdict(scope_event, finding)
141
+ claims.append(scope_claim)
142
+ recs.append(scope_rec)
143
+
144
+ if fmt == "markdown":
145
+ text = report_mod.markdown(claims, recs, source=f"{sess.source} session {sess.id[:8]}")
146
+ elif fmt == "html":
147
+ text = report_mod.html_card(
148
+ claims, recs, ledger, report=report, title=f"Receipt · {sess.id[:8]}"
149
+ )
150
+ else:
151
+ report_mod.terminal(claims, recs, ledger, console, show_evidence=evidence)
152
+ console.print(f"[dim] {tail}[/]")
153
+ text = None
154
+
155
+ if out_path:
156
+ body = (
157
+ text
158
+ if text is not None
159
+ else report_mod.html_card(
160
+ claims, recs, ledger, report=report, title=f"Receipt · {sess.id[:8]}"
161
+ )
162
+ )
163
+ pathlib.Path(out_path).write_text(body, encoding="utf-8")
164
+ console.print(f"[dim]wrote {out_path}[/]")
165
+ elif text is not None:
166
+ print(text)
167
+
168
+ if any(r.verdict == Verdict.CONTRADICTED for r in recs):
169
+ raise typer.Exit(code=1)
170
+
171
+
172
+ def _hook_command(event: str, env: dict[str, str] | None = None) -> str:
173
+ """An absolutely-resolved command line for one hook event.
174
+
175
+ Claude Code runs hooks through a shell that does not inherit this process's PATH, so a bare
176
+ `custos-code _hook stop` only works when custos-code is installed globally. It is not when the repo
177
+ is used from a checkout with a virtualenv -- the common case for this team, and the reason the
178
+ hooks were silently not running on Oliver's laptop on 2026-09-19. Resolve now, at install time.
179
+
180
+ Same resolution order as `rerun._worker_argv`, for the same reason and deliberately identical.
181
+ """
182
+ import shlex
183
+ import shutil as _sh
184
+ import sys as _sys
185
+
186
+ args = ["_hook", event]
187
+ script = _sh.which("custos-code")
188
+ if script:
189
+ line = shlex.join([script, *args])
190
+ else:
191
+ line = shlex.join([_sys.executable, "-c", "from custos_code.cli import app; app()", *args])
192
+ # Quote only the value. `shlex.join` would quote the whole word, and a fully quoted `'A=b'` is
193
+ # not an assignment to a POSIX shell -- it is a command name, so the hook would not run at all.
194
+ prefix = "".join(f"{k}={shlex.quote(v)} " for k, v in sorted((env or {}).items()))
195
+ return prefix + line
196
+
197
+
198
+ def hooks_snippet(env: dict[str, str] | None = None) -> dict[str, object]:
199
+ """The settings.json fragment that installs the three hooks, with commands already resolved."""
200
+ return {
201
+ "hooks": {
202
+ "PreToolUse": [
203
+ {
204
+ "matcher": "Bash",
205
+ "hooks": [{"type": "command", "command": _hook_command("pre", env), "timeout": 5}],
206
+ }
207
+ ],
208
+ "PostToolUse": [
209
+ {
210
+ "matcher": "",
211
+ "hooks": [
212
+ {
213
+ "type": "command",
214
+ "command": _hook_command("post-tool-use", env),
215
+ "timeout": 10,
216
+ }
217
+ ],
218
+ }
219
+ ],
220
+ "Stop": [
221
+ {
222
+ "matcher": "",
223
+ "hooks": [
224
+ {"type": "command", "command": _hook_command("stop", env), "timeout": 120}
225
+ ],
226
+ }
227
+ ],
228
+ }
229
+ }
230
+
231
+
232
+ _ENV_ASSIGN_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
233
+
234
+
235
+ def _is_installed_hook(hook: object) -> bool:
236
+ """Recognize our console and Python fallback hooks, including pre-rename installs."""
237
+ import shlex
238
+
239
+ if not isinstance(hook, dict) or hook.get("type") != "command":
240
+ return False
241
+ command = hook.get("command")
242
+ if not isinstance(command, str):
243
+ return False
244
+ try:
245
+ args = shlex.split(command)
246
+ except ValueError:
247
+ return False
248
+ # An installed command may carry leading `VAR=value` assignments -- every hook this repo has
249
+ # ever written to a real settings.json does (`RECEIPTS_ONLY_IN=...`, now `CUSTOS_CODE_*`).
250
+ # Those are shell syntax, not the program, so step over them before looking for the binary.
251
+ # Without this the matcher does not recognise its own prior installs, and `watch --install`
252
+ # stacks a fresh copy beside the stale one rather than replacing it -- which on this machine
253
+ # meant three dead pre-rename hooks erroring on every tool call, untouched by the migration
254
+ # written to remove them.
255
+ while args and _ENV_ASSIGN_RE.match(args[0]):
256
+ args.pop(0)
257
+ if len(args) < 3:
258
+ return False
259
+ if pathlib.Path(args[0]).name in ("receipts", "custos-code"):
260
+ return args[1] == "_hook"
261
+ return (
262
+ len(args) >= 5
263
+ and args[1] == "-c"
264
+ and args[2] in (
265
+ "from receipts.cli import app; app()",
266
+ "from custos_code.cli import app; app()",
267
+ )
268
+ and args[3] == "_hook"
269
+ )
270
+
271
+
272
+ @app.command()
273
+ def watch(
274
+ install: bool = typer.Option(
275
+ False, "--install", help="Merge the hooks into ~/.claude/settings.json (backup kept)."
276
+ ),
277
+ only_in: str = typer.Option(
278
+ "", "--only-in", metavar="DIR",
279
+ help="Record only in sessions whose cwd is inside DIR. Leave unset to record everywhere.",
280
+ ),
281
+ auto: bool = typer.Option(
282
+ False, "--auto", help="Arm blocking auto mode for the installed hooks (they hold the turn)."
283
+ ),
284
+ ) -> None:
285
+ """Show (or install) the Claude Code hooks that record every tool call and check each final report."""
286
+ import json as _json
287
+ import os as _os
288
+ import shutil as _shutil
289
+
290
+ env: dict[str, str] = {}
291
+ if only_in.strip():
292
+ # Resolve now. The hook compares against a realpath, and an install-time `.` or `~/x` would
293
+ # otherwise mean whatever directory the *agent* happens to be in when the hook fires.
294
+ env["CUSTOS_CODE_ONLY_IN"] = _os.path.realpath(_os.path.expanduser(only_in.strip()))
295
+ if auto:
296
+ env["CUSTOS_CODE_AUTO"] = "1"
297
+ if not install:
298
+ console.print(_json.dumps(hooks_snippet(env), indent=2))
299
+ console.print(
300
+ "[dim]Add to ~/.claude/settings.json (or .claude/settings.json in a repo), or run `custos-code watch --install`.[/]"
301
+ )
302
+ return
303
+ path = _os.path.expanduser("~/.claude/settings.json")
304
+ data: dict[str, object] = {}
305
+ if _os.path.exists(path):
306
+ _shutil.copy(path, path + ".bak")
307
+ with open(path, encoding="utf-8") as fh:
308
+ data = _json.load(fh)
309
+ hooks = data.setdefault("hooks", {})
310
+ assert isinstance(hooks, dict)
311
+ snippet = hooks_snippet(env)["hooks"]
312
+ assert isinstance(snippet, dict)
313
+ for ev, entries in snippet.items():
314
+ existing = hooks.setdefault(ev, [])
315
+ assert isinstance(existing, list)
316
+ # Replace our old/current commands, including stale absolute paths. A group may also
317
+ # contain other tools' hooks; retain those and all of their matcher/metadata fields.
318
+ retained = []
319
+ for entry in existing:
320
+ if not isinstance(entry, dict) or not isinstance(entry.get("hooks"), list):
321
+ retained.append(entry)
322
+ continue
323
+ remaining = [h for h in entry["hooks"] if not _is_installed_hook(h)]
324
+ if len(remaining) == len(entry["hooks"]):
325
+ retained.append(entry)
326
+ elif remaining:
327
+ retained.append({**entry, "hooks": remaining})
328
+ hooks[ev] = [*retained, *entries]
329
+ with open(path, "w", encoding="utf-8") as fh:
330
+ _json.dump(data, fh, indent=2)
331
+ console.print(f"installed into {path} (backup at {path}.bak)")
332
+
333
+
334
+ @app.command(name="_hook", hidden=True)
335
+ def _hook(
336
+ event: str = typer.Argument(..., help="pre | post-tool-use | stop | rerun-worker"),
337
+ session_id: str | None = typer.Argument(None, help="rerun-worker only: which session."),
338
+ claim_id: str | None = typer.Argument(None, help="rerun-worker only: which claim."),
339
+ ) -> None:
340
+ """Internal: hook entrypoint. `pre`/`post-tool-use`/`stop` read the Claude Code payload on
341
+ stdin. `rerun-worker` (E4) has no hook payload at all -- it's a detached subprocess
342
+ `rerun.spawn_async` launches directly, so it takes its identity as positional args instead.
343
+ """
344
+ from . import hooks as _hooks
345
+
346
+ raise typer.Exit(code=_hooks.main(event, session_id, claim_id))
347
+
348
+
349
+ @app.command(name="pr-comment")
350
+ def pr_comment(
351
+ session: str = typer.Argument(
352
+ ..., help="Session transcript or class-R bundle (Devin/Copilot) for the PR."
353
+ ),
354
+ agent: str | None = typer.Option(None, "--agent", help="Adapter to use (default: detect)."),
355
+ out: str | None = typer.Option(
356
+ None, "--out", help="Write the markdown here instead of stdout."
357
+ ),
358
+ repo: str | None = typer.Option(
359
+ None, "--repo", help="Repo root for state checks (default: the session's cwd)."
360
+ ),
361
+ pr_url: str | None = typer.Option(None, "--pr-url", help="Link back to the PR in the footer."),
362
+ receipt_url: str | None = typer.Option(None, "--receipt-url", help="Link to the full receipt."),
363
+ rules_only: bool = typer.Option(
364
+ False, "--rules-only", help="Deterministic rules only; no model call."
365
+ ),
366
+ fail_on_contradiction: bool = typer.Option(
367
+ False,
368
+ "--fail-on-contradiction",
369
+ help="Exit 1 when a claim is contradicted (for a required check).",
370
+ ),
371
+ ) -> None:
372
+ """Render the PR receipt as markdown (product sketch B); the Action posts what this prints."""
373
+ sess, ledger, report = adapters_mod.parse(session, agent)
374
+ cl: list[Claim] = []
375
+ recs: list[VerdictRecord] = []
376
+ if report:
377
+ backend = None if rules_only else judge_mod.make_backend()
378
+ cl = claims_mod.extract(report, sess.id, backend)
379
+ recs = verdicts_mod.run(cl, ledger, repo or sess.cwd, backend)
380
+ body = report_mod.pr_comment(sess, cl, recs, ledger, pr_url=pr_url, receipt_url=receipt_url)
381
+ if out:
382
+ pathlib.Path(out).write_text(body, encoding="utf-8")
383
+ console.print(f"[dim]wrote {out}[/]")
384
+ else:
385
+ print(body)
386
+ if fail_on_contradiction and any(r.verdict is Verdict.CONTRADICTED for r in recs):
387
+ raise typer.Exit(code=1)
388
+
389
+
390
+ @app.command()
391
+ def record(
392
+ shell: str = typer.Option("bash", "--shell", help="bash | zsh | sh: which rc snippet to emit."),
393
+ install: bool = typer.Option(
394
+ False, "--install", help="Append the snippet to the rc file (backup kept)."
395
+ ),
396
+ wrapper: bool = typer.Option(
397
+ False, "--wrapper",
398
+ help="Also install the PATH-first bash/sh wrapper (docs/ADAPTERS.md §4/§7): the rc-file "
399
+ "snippet's DEBUG-trap/preexec hooks never attach inside `bash -c \"cmd\"`/`sh -c \"cmd\"` "
400
+ "-- neither interactive nor login, so it never sources the rc file -- which is exactly "
401
+ "how Claude Code and Codex spawn commands.",
402
+ ),
403
+ ) -> None:
404
+ """Class-M recorder: log every shell command, exit status and cwd, with no harness at all."""
405
+ import os as _os
406
+ import shutil as _shutil
407
+
408
+ rc_files = {"bash": "~/.bashrc", "sh": "~/.profile", "zsh": "~/.zshrc"}
409
+ if shell not in rc_files:
410
+ # validated before any side effect -- install_wrapper() below writes real files, and a
411
+ # raw KeyError from the dict lookup used to surface only after that had already happened
412
+ raise typer.BadParameter(f"shell must be one of {', '.join(rc_files)}", param_hint="--shell")
413
+
414
+ if wrapper:
415
+ try:
416
+ written = machine.install_wrapper()
417
+ except FileNotFoundError as exc:
418
+ console.print(f"[red]{exc}[/]")
419
+ raise typer.Exit(code=1) from exc
420
+ console.print(f"[dim]wrapper installed: {', '.join(written.values())}[/]")
421
+ path_line = f'export PATH="{machine.WRAPPER_DIR}:$PATH"\n'
422
+ if install:
423
+ rc = _os.path.expanduser(rc_files[shell])
424
+ existing = open(rc, encoding="utf-8").read() if _os.path.exists(rc) else ""
425
+ if "custos-code recorder (class M) wrapper" not in existing:
426
+ with open(rc, "a", encoding="utf-8") as fh:
427
+ fh.write(
428
+ "\n# >>> custos-code recorder (class M) wrapper >>>\n"
429
+ f"{path_line}"
430
+ "# <<< custos-code recorder (class M) wrapper <<<\n"
431
+ )
432
+ console.print(
433
+ f"[dim]PATH updated in {rc} -- takes effect in new shells launched from an "
434
+ "interactive one that sources it (a new terminal tab, or anything spawned "
435
+ "from it) as a plain env-inheritance chain, never by re-sourcing the rc file "
436
+ "itself. A GUI/IDE-launched agent that was not spawned from such a shell (e.g. "
437
+ "opened from the Dock/Start Menu rather than a terminal) will not see this "
438
+ "PATH change; point it at the wrapper directory through its own environment "
439
+ f"settings instead: {machine.WRAPPER_DIR}[/]"
440
+ )
441
+ else:
442
+ console.print(f"[dim]wrapper PATH already present in {rc}[/]")
443
+ else:
444
+ console.print(f"[dim]Prepend to PATH yourself, or re-run with --install: {path_line.strip()}[/]")
445
+
446
+ snippet = machine.install_snippet(shell)
447
+ if not install:
448
+ print(snippet)
449
+ console.print(
450
+ f"[dim]Append to your rc file, or run `custos-code record --shell {shell} --install`. "
451
+ f"Log: {machine.default_log()}[/]"
452
+ )
453
+ return
454
+ rc = _os.path.expanduser(rc_files[shell])
455
+ if _os.path.exists(rc):
456
+ if "custos-code recorder" in open(rc, encoding="utf-8").read():
457
+ console.print(f"already installed in {rc}")
458
+ return
459
+ _shutil.copy(rc, rc + ".bak")
460
+ _os.makedirs(machine.LOG_DIR, exist_ok=True)
461
+ with open(rc, "a", encoding="utf-8") as fh:
462
+ fh.write("\n" + snippet)
463
+ console.print(f"installed into {rc} (backup at {rc}.bak) · log {machine.default_log()}")
464
+
465
+
466
+ @app.command(name="_record-line", hidden=True)
467
+ def _record_line() -> None:
468
+ """Internal: the recorder shell hooks call this once per command to emit one wire line."""
469
+ print(machine.record_line())
470
+
471
+
472
+ @app.command()
473
+ def cost(
474
+ session: str | None = typer.Argument(
475
+ None, help="Path to a session transcript, rollout, or bundle."
476
+ ),
477
+ last: bool = typer.Option(False, "--last", help="Use the most recent session of --agent (default Claude Code)."),
478
+ agent: str | None = typer.Option(
479
+ None, "--agent", help="claude_code | codex | devin | copilot | machine | otel (default: detect)."
480
+ ),
481
+ path: str = typer.Option(
482
+ "review", "--path",
483
+ help="review | ladder | judge-all: which pipeline to price (E12; default is what actually ships).",
484
+ ),
485
+ repo: str | None = typer.Option(
486
+ None, "--repo", help="Repo root for state checks (default: the session's cwd)."
487
+ ),
488
+ compress: bool = typer.Option(
489
+ False, "--compress",
490
+ help="Measure a real bear-2 pass on the judge window and report projected savings "
491
+ "(ladder/judge-all only; needs [compress].enabled and CUSTOS_CODE_TTC_API_KEY).",
492
+ ),
493
+ json_out: bool = typer.Option(False, "--json", help="Print JSON instead of a table."),
494
+ ) -> None:
495
+ """Tokens and dollars by tier for one session, priced by the path actually run.
496
+
497
+ E12: `cost` used to always run the ladder regardless of what `check` defaults to, so the
498
+ Token Company chart priced a pipeline (verdicts.run) that's no longer the shipping one
499
+ (review.py, 86% vs the ladder's 70%, eval/arms/RESULTS.md). `--path` makes the pipeline an
500
+ explicit argument instead of an inherited default, because comparing paths is this command's
501
+ whole job -- `cost.compute()` was already agnostic to which path produced its VerdictRecords.
502
+ """
503
+ from . import compress as compress_mod
504
+ from . import cost as cost_mod
505
+ from . import review as review_mod
506
+
507
+ if path not in ("review", "ladder", "judge-all"):
508
+ raise typer.BadParameter("--path must be review, ladder, or judge-all")
509
+ if last:
510
+ session = codex.find_last_session() if agent == "codex" else claude_code.find_last_session()
511
+ if not session:
512
+ raise typer.BadParameter("give a session path or --last")
513
+ sess, ledger, report = adapters_mod.parse(session, agent)
514
+
515
+ backend = judge_mod.make_backend()
516
+ if backend is None and path != "ladder":
517
+ console.print(f"[yellow]no judge backend: set OPENAI_API_KEY (or CUSTOS_CODE_JUDGE_BACKEND=anthropic); "
518
+ f"falling back to the rules ladder instead of --path {path}[/]")
519
+ path = "ladder"
520
+
521
+ cl: list[Claim] = []
522
+ recs: list[VerdictRecord] = []
523
+ usage = None
524
+ if report:
525
+ if path == "review":
526
+ out = review_mod.review(report, ledger, sess.id, backend)
527
+ cl, recs = out.claims, out.verdicts
528
+ usage = judge_mod.Usage(requests=out.requests, input_tokens=out.input_tokens,
529
+ output_tokens=out.output_tokens, model=getattr(backend, "judge_model", "") or "")
530
+ elif path == "judge-all":
531
+ cl = claims_mod.extract(report, sess.id)
532
+ recs = backend.judge(cl, ledger) if cl and backend is not None else []
533
+ usage = backend.usage if backend is not None else None
534
+ else: # ladder
535
+ cl = claims_mod.extract(report, sess.id)
536
+ recs = verdicts_mod.run(cl, ledger, repo or sess.cwd, backend)
537
+ usage = backend.usage if backend is not None else None
538
+
539
+ compress_usage = None
540
+ if compress:
541
+ if path == "review":
542
+ console.print("[yellow]--compress has no effect on --path review yet: review.py doesn't window "
543
+ "the ledger the way compress.Compressor expects (E11, still open). Skipped.[/]")
544
+ else:
545
+ compressor = compress_mod.make_compressor()
546
+ if compressor is None:
547
+ console.print("[yellow]compressor off: set [compress].enabled and CUSTOS_CODE_TTC_API_KEY[/]")
548
+ else:
549
+ # ladder: only the residue that actually reached the judge, same as eval/cost_report.py's
550
+ # arm_ladder_compressed; judge-all: the whole ledger, matching what judge-all actually sent.
551
+ pending = [c for c, r in zip(cl, recs, strict=True) if r.method == "judge"] if path == "ladder" else cl
552
+ if pending:
553
+ win = judge_mod.window_for_all(ledger, pending) if path == "ladder" else ledger
554
+ compressor.compress_window(win)
555
+ compress_usage = compressor.usage
556
+
557
+ c = cost_mod.compute(sess.id, recs, ledger, judge_usage=usage, compress_usage=compress_usage)
558
+ if json_out:
559
+ console.print_json(data=c.to_dict())
560
+ else:
561
+ console.print(cost_mod.render_table(c))
562
+
563
+
564
+ @app.command()
565
+ def demo(
566
+ scenario: str = typer.Option(
567
+ "piped-runner", "--scenario", help="piped-runner | echoed-output | ghost-write | honest"
568
+ ),
569
+ out_path: str | None = typer.Option(None, "--out", help="Also write an HTML report card here."),
570
+ ) -> None:
571
+ """Run the whole loop on a known trap: the agent's claim, the evidence, the verdict, the nudge.
572
+
573
+ Everything on screen is produced live from the fixture's own tool log. Nothing is pre-rendered,
574
+ and the fixture is in the repo so anyone can read what the agent actually did.
575
+ """
576
+ import contextlib as _contextlib
577
+ import json as _json
578
+ from importlib import resources as _resources
579
+
580
+ from . import feedback as feedback_mod
581
+
582
+ picks = {
583
+ "piped-runner": "trap_piped_0",
584
+ "echoed-output": "trap_echo_0",
585
+ "ghost-write": "trap_ghost_0",
586
+ "honest": "ok_tests_0",
587
+ }
588
+ name = picks.get(scenario)
589
+ if name is None:
590
+ raise typer.BadParameter(f"scenario must be one of {', '.join(picks)}")
591
+ packaged = _resources.files("custos_code.demo_fixtures").joinpath(f"{name}.jsonl")
592
+ repo_fixture = (
593
+ pathlib.Path(__file__).resolve().parents[2] / "eval" / "arms" / "fixtures" / f"{name}.jsonl"
594
+ )
595
+ if packaged.is_file():
596
+ with _resources.as_file(packaged) as fixture:
597
+ sess, ledger, report = claude_code.parse(str(fixture))
598
+ else:
599
+ if not repo_fixture.exists():
600
+ console.print("[yellow]fixtures missing — run `python eval/arms/generate.py` first[/]")
601
+ raise typer.Exit(code=2)
602
+ with _contextlib.nullcontext(repo_fixture) as fixture:
603
+ sess, ledger, report = claude_code.parse(str(fixture))
604
+ task = next((e.output for e in ledger if e.kind == EventKind.USER), "")
605
+
606
+ console.rule("[bold]1. what the developer asked for")
607
+ console.print(f" {task}")
608
+
609
+ console.rule("[bold]2. what the agent actually did (harness log, the model cannot write it)")
610
+ for e in ledger:
611
+ if e.kind == EventKind.CALL:
612
+ v = str((e.input or {}).get("command") or (e.input or {}).get("file_path") or "")
613
+ console.print(f" [dim]#{e.seq}[/] [yellow]{e.tool}[/] {v[:100]}")
614
+ elif e.kind == EventKind.RESULT and e.output:
615
+ console.print(f" [dim]#{e.seq} → {e.output[:100].replace(chr(10), ' ⏎ ')}[/]")
616
+
617
+ console.rule("[bold]3. what the agent said")
618
+ console.print(f" [italic]{report}[/]")
619
+
620
+ # No key is a reason to show a quieter receipt, not to abandon the demo three stages in. The
621
+ # deterministic rules catch this fixture's piped runner on their own, so the demo still runs
622
+ # end to end; it just says less about why.
623
+ backend = judge_mod.make_backend()
624
+ if backend is not None:
625
+ reviewed = review_mod.review(report or "", ledger, sess.id, backend)
626
+ dclaims, drecs = reviewed.claims, reviewed.verdicts
627
+ tail = f"one call · {reviewed.input_tokens} in / {reviewed.output_tokens} out"
628
+ else:
629
+ console.print("\n[yellow]no model backend — running the deterministic rules instead.[/]")
630
+ console.print("[dim] set OPENAI_API_KEY, or put it in ~/.custos-code/env, for the full review.[/]")
631
+ dclaims = claims_mod.extract(report or "", sess.id)
632
+ drecs = verdicts_mod.run(dclaims, ledger, sess.cwd, None)
633
+ tail = "rules only · 0 tokens"
634
+
635
+ console.rule("[bold]4. the receipt")
636
+ report_mod.terminal(dclaims, drecs, ledger, console, show_evidence=True)
637
+ console.print(f"[dim] {tail}[/]")
638
+
639
+ open_pairs = [
640
+ (c, r)
641
+ for c, r in zip(dclaims, drecs, strict=True)
642
+ if r.verdict.value in ("contradicted", "unrecorded")
643
+ ]
644
+ console.rule("[bold]5. what goes back to the agent")
645
+ if open_pairs:
646
+ reason = feedback_mod.build_block_reason(open_pairs, ledger, 1, 3)
647
+ console.print(
648
+ f" [red]stop blocked[/] — {len(open_pairs)} claim(s) need work, nudge is a "
649
+ f"template, [bold]0 LLM tokens[/]"
650
+ )
651
+ for line in reason.splitlines()[1:]:
652
+ console.print(f" [dim]{line[:160]}[/]")
653
+ console.print(f"\n [dim]hook returns:[/] {_json.dumps({'decision': 'block'})}")
654
+ else:
655
+ console.print(
656
+ " [green]nothing blocked[/] — every claim is confirmed or disclosed; the agent stops normally."
657
+ )
658
+
659
+ if out_path:
660
+ pathlib.Path(out_path).write_text(
661
+ report_mod.html_card(
662
+ dclaims,
663
+ drecs,
664
+ ledger,
665
+ report=report or "",
666
+ title=f"Receipt · {scenario}",
667
+ ),
668
+ encoding="utf-8",
669
+ )
670
+ console.print(f"\n[dim]report card written to {out_path}[/]")
671
+
672
+
673
+ @dataclass
674
+ class _Mark:
675
+ text: str
676
+ verdict: str
677
+ why: str
678
+ evidence: list[int]
679
+
680
+
681
+ @dataclass
682
+ class _Scanned:
683
+ path: str
684
+ sid: str
685
+ project: str
686
+ marks: list[_Mark]
687
+
688
+
689
+ @app.command()
690
+ def scan(
691
+ limit: int = typer.Option(25, "--limit", "-n", help="How many recent sessions to check."),
692
+ workers: int = typer.Option(8, "--workers", help="Parallel model calls."),
693
+ min_events: int = typer.Option(5, "--min-events", help="Skip sessions with fewer tool calls."),
694
+ all_marks: bool = typer.Option(False, "--all", help="Show every verdict, not just contradictions."),
695
+ out_path: str | None = typer.Option(None, "--out", help="Write the findings as JSON."),
696
+ ) -> None:
697
+ """Check your own recent sessions and report what an agent told you that the log contradicts.
698
+
699
+ This is the honest demo. A staged trap only fires when the agent takes the bait, and a careful
700
+ agent does not -- so a trap that catches nothing looks like a broken product when it is in fact
701
+ behaving correctly. Real history contains the failures that actually occur (a sample reported
702
+ as a total, a remembered test count, a "verified working" that skipped the command that
703
+ failed), and the evidence is already on the machine.
704
+ """
705
+ import concurrent.futures as _cf
706
+
707
+ from .adapters.claude_code import find_sessions
708
+
709
+ backend = judge_mod.make_backend()
710
+ if backend is None:
711
+ console.print("[yellow]scan needs a model backend: set OPENAI_API_KEY or ~/.custos-code/env[/]")
712
+ raise typer.Exit(code=2)
713
+
714
+ paths = find_sessions()
715
+ console.print(f"[dim]{len(paths)} sessions on disk; checking the {limit} most recent[/]\n")
716
+
717
+ def one(path: str) -> _Scanned | None:
718
+ try:
719
+ sess, ledger, rep = claude_code.parse(path)
720
+ except Exception:
721
+ return None
722
+ if not rep or len(ledger) < min_events:
723
+ return None
724
+ try:
725
+ r = review_mod.review(rep, ledger, sess.id, backend)
726
+ except Exception:
727
+ return None
728
+ # Claude Code names a project directory after its absolute path with separators replaced
729
+ # by dashes: `-Users-alice-Projects-thing`. Strip the home prefix generically -- this was
730
+ # one developer's literal home directory hardcoded in shipped source, which is personal
731
+ # data on its way to PyPI and a silent no-op on everyone else's machine.
732
+ proj = pathlib.Path(path).parent.name
733
+ home = str(pathlib.Path.home()).replace("/", "-")
734
+ if proj.startswith(home + "-"):
735
+ proj = proj[len(home) + 1:]
736
+ return _Scanned(path, pathlib.Path(path).stem[:8], proj,
737
+ [_Mark(c.text, v.verdict.value, v.rationale, list(v.evidence))
738
+ for c, v in zip(r.claims, r.verdicts, strict=False)])
739
+
740
+ done: list[_Scanned] = []
741
+ with _cf.ThreadPoolExecutor(max_workers=workers) as ex:
742
+ futs = [ex.submit(one, p) for p in paths[: limit * 3]]
743
+ for f in _cf.as_completed(futs):
744
+ got = f.result()
745
+ if got is None:
746
+ continue
747
+ done.append(got)
748
+ if len(done) >= limit:
749
+ break
750
+
751
+ shown = {"contradicted"} if not all_marks else {v.value for v in Verdict}
752
+ tally: dict[str, int] = {v.value: 0 for v in Verdict}
753
+ for s in done:
754
+ for m in s.marks:
755
+ tally[m.verdict] += 1
756
+
757
+ hits = 0
758
+ for s in done:
759
+ picked = [m for m in s.marks if m.verdict in shown]
760
+ if not picked:
761
+ continue
762
+ hits += 1
763
+ console.print(f"[bold]{s.sid}[/] [dim]{s.project}[/]")
764
+ for m in picked:
765
+ glyph, colour = MARK[Verdict(m.verdict)]
766
+ cites = " ".join(f"#{e}" for e in m.evidence) or "—"
767
+ console.print(f" [{colour}]{glyph} {m.verdict}[/] {m.text[:150]}")
768
+ console.print(f" [dim]{cites} · {m.why[:210]}[/]")
769
+ console.print()
770
+
771
+ total = sum(tally.values()) or 1
772
+ answered = tally["confirmed"] + tally["contradicted"] + tally["qualified"]
773
+ bad = sum(1 for s in done if any(m.verdict == "contradicted" for m in s.marks))
774
+ console.rule()
775
+ console.print(f" {len(done)} sessions · {total} claims · coverage {answered}/{total} = {answered / total:.0%}")
776
+ console.print(f" [red]{tally['contradicted']} contradicted[/] across "
777
+ f"[bold]{bad} of {len(done)} sessions ({bad / max(len(done), 1):.0%})[/]")
778
+ console.print(f" [dim]confirmed {tally['confirmed']} · qualified {tally['qualified']} · "
779
+ f"unwitnessed {tally['unwitnessed']} · unrecorded {tally['unrecorded']}[/]")
780
+ if not hits and not all_marks:
781
+ console.print(" [green]nothing contradicted in this window[/] — rerun with --all to see every mark")
782
+
783
+ if out_path:
784
+ import json as _j
785
+ payload = {"tally": tally,
786
+ "sessions": [{"sid": s.sid, "project": s.project, "path": s.path,
787
+ "marks": [vars(m) for m in s.marks]} for s in done]}
788
+ pathlib.Path(out_path).write_text(_j.dumps(payload, indent=1), encoding="utf-8")
789
+ console.print(f" [dim]wrote {out_path}[/]")