debrief-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
debrief/cli.py ADDED
@@ -0,0 +1,291 @@
1
+ """debrief: review your coding-agent sessions and see what to change.
2
+
3
+ debrief sync ingest local logs, label turns, find patterns, build the report
4
+ debrief sync --project DIR pick a project to analyse (kept in config.toml), then sync
5
+ debrief config --drop DIR stop analysing a project you picked
6
+ debrief sync --dry-run ingest, then say what labeling would cost; no model calls
7
+ debrief open serve the report on localhost and open it in the browser;
8
+ the page can also start a sync, show its progress and stop it
9
+ debrief <stage> [args] run one stage; `debrief <stage> --help` lists its flags
10
+
11
+ Everything is kept under DEBRIEF_HOME (default ~/.local/share/debrief).
12
+ """
13
+
14
+ import argparse
15
+ import contextlib
16
+ import functools
17
+ import hmac
18
+ import http.server
19
+ import importlib
20
+ import io
21
+ import json
22
+ import os
23
+ import secrets
24
+ import signal
25
+ import subprocess
26
+ import sys
27
+ import webbrowser
28
+
29
+ from debrief import config
30
+ from debrief.paths import HOME, SITE_OUT, TOKEN_HEADER
31
+ from debrief.runner import Runner
32
+
33
+ # Stage name -> module. Each module has main(argv).
34
+ STAGES = {
35
+ "ingest": "sources",
36
+ "label": "classify",
37
+ "synthesize": "synthesize",
38
+ "recommend": "recommend",
39
+ "verify": "verify_evidence",
40
+ "export": "export_site",
41
+ "entire": "scrape",
42
+ "entire-load": "normalize",
43
+ "github": "github",
44
+ "prune": "prune",
45
+ "config": "config",
46
+ }
47
+
48
+
49
+ def stage(name: str, argv: list[str]) -> None:
50
+ print(f"\n== {name} {' '.join(argv)}".rstrip(), flush=True)
51
+ importlib.import_module(f"{__package__}.{STAGES[name]}").main(argv)
52
+
53
+
54
+ def sync(argv: list[str]) -> None:
55
+ p = argparse.ArgumentParser(prog="debrief sync", description="Ingest local logs and rebuild the report.")
56
+ p.add_argument("--project", metavar="DIR", action="append", default=[],
57
+ help="Add this project to the ones sync analyses (repeatable; kept in config.toml). "
58
+ "`debrief config --drop DIR` takes one out again")
59
+ p.add_argument("--since", metavar="YYYY-MM-DD", help="Only logs and evidence from this date on")
60
+ p.add_argument("--import", dest="import_dirs", action="append", default=[], metavar="DIR",
61
+ help="Also load a folder of JSONL logs (repeatable)")
62
+ p.add_argument("--include-headless", action="store_true", help="Also load scripted runs (claude -p, SDK jobs)")
63
+ p.add_argument("--llm", choices=["claude-code", "openrouter"],
64
+ help="Use this for labeling and synthesis, whatever config.toml says. "
65
+ "claude-code: `claude -p` on your Claude subscription; openrouter: needs OPENROUTER_API_KEY")
66
+ p.add_argument("--no-github", action="store_true", help="Do not ask GitHub about your sessions' pull requests")
67
+ p.add_argument("--dry-run", action="store_true",
68
+ help="Ingest, then print what labeling would cost; no model calls, no GitHub requests")
69
+ args = p.parse_args(argv)
70
+
71
+ # Ingest reads every log: it is local, and it is what tells you which projects there are.
72
+ ingest = [f"--since={args.since}"] if args.since else []
73
+ ingest += [f"--import={d}" for d in args.import_dirs] + ["--include-headless"] * args.include_headless
74
+ since = [f"--since={args.since}"] if args.since else []
75
+ llm = [f"--llm={args.llm}"] if args.llm else []
76
+
77
+ stage("ingest", ingest)
78
+ if not pick_projects(args.project):
79
+ return
80
+ off = "--no-github" if args.no_github else None if config.github_enabled() else "[github] enabled = false in config.toml"
81
+ if off:
82
+ print(f"\ngithub: skipped, {off}")
83
+ else:
84
+ try:
85
+ stage("github", ["--dry-run"] * args.dry_run)
86
+ except RuntimeError as e: # a failed gh call; the report builds without GitHub data
87
+ print(f"github: skipped, {str(e).splitlines()[0]}")
88
+ if args.dry_run:
89
+ # The label dry run also prints sample prompts; keep only its count and estimate.
90
+ out = io.StringIO()
91
+ with contextlib.redirect_stdout(out):
92
+ stage("label", [*llm, "--dry-run"])
93
+ lines = out.getvalue().splitlines()
94
+ print("\n".join(line for line in lines if line.startswith(("== ", "=== ", "would label"))))
95
+ return
96
+ stage("label", llm)
97
+ stage("synthesize", [*llm, *since])
98
+ stage("recommend", [])
99
+ stage("export", since)
100
+ print("\nDone. Run `debrief open` to read the report.")
101
+
102
+
103
+ def pick_projects(dirs: list[str]) -> bool:
104
+ """Add dirs to the picked projects. False, after printing the projects, when none is picked
105
+ or a dir is not a project: sync then stops before any model call."""
106
+ from debrief.db import connect
107
+ from debrief.export_site import projects_section
108
+
109
+ rows = projects_section(connect(), set(config.projects()))
110
+ known = {r["path"] for r in rows}
111
+ wanted = [os.path.abspath(os.path.expanduser(d)) for d in dirs]
112
+ unknown = [d for d in wanted if d not in known]
113
+ if not unknown:
114
+ for d in wanted:
115
+ config.pick(d)
116
+ if any(p in known for p in config.projects()):
117
+ return True
118
+ print(f"\n{'no sessions ran in ' + ', '.join(unknown) if unknown else 'No project is picked, so nothing was analysed'}. "
119
+ f"Projects found:\n")
120
+ print(f" {'sessions':>8} {'messages':>8} project")
121
+ for r in rows:
122
+ print(f" {r['sessions']:>8} {r['turns']:>8} {r['path']}{' (picked)' if r['picked'] else ''}")
123
+ print("\nPick one with `debrief sync --project DIR` (repeat the flag for more). "
124
+ "Picks are kept in config.toml under [projects]; `debrief config --drop DIR` takes one out.")
125
+ return False
126
+
127
+
128
+ class ReportHandler(http.server.SimpleHTTPRequestHandler):
129
+ """Static files, the settings API and the run API, for requests addressed to this machine only.
130
+
131
+ A web page can point its own hostname at 127.0.0.1 (DNS rebinding) and reach this server
132
+ through the browser; checking Host stops that, since the page cannot fake it.
133
+
134
+ The writes are config.toml, the key in DEBRIEF_HOME/.env, and starting or stopping a sync
135
+ (which takes nothing from the request: what to analyse comes from config). A write must be JSON, which
136
+ a page on another site cannot send without a preflight this server never answers, and must
137
+ come from this origin when the browser names one. It must also carry the token made for this
138
+ run, which only the link `debrief open` prints holds: another user of this machine can reach
139
+ the port but does not have the link.
140
+ """
141
+
142
+ MAX_BODY = 128 * 1024 # room for 200 picked project paths
143
+ token = secrets.token_urlsafe(16)
144
+ jobs = Runner(hide=(token,))
145
+
146
+ def local(self) -> bool:
147
+ ok = self.headers.get("Host") in self.hosts()
148
+ if not ok:
149
+ self.send_error(403, "debrief serves only 127.0.0.1 and localhost")
150
+ return ok
151
+
152
+ def hosts(self) -> tuple[str, str]:
153
+ port = self.server.server_port
154
+ return f"127.0.0.1:{port}", f"localhost:{port}"
155
+
156
+ def send_head(self):
157
+ return super().send_head() if self.local() else None
158
+
159
+ def do_GET(self):
160
+ if self.path not in ("/api/settings", "/api/run"):
161
+ super().do_GET()
162
+ elif self.local():
163
+ self.reply_settings() if self.path == "/api/settings" else self.reply(200, self.jobs.status())
164
+
165
+ def do_POST(self):
166
+ if not self.local():
167
+ return
168
+ self.close_connection = True # a refused body is never read, so the connection cannot be reused
169
+ length = self.headers.get("Content-Length", "0")
170
+ if not hmac.compare_digest(self.headers.get(TOKEN_HEADER, "").encode(), self.token.encode()):
171
+ return self.reply(403, {"error": "to change settings, open the link `debrief open` printed"})
172
+ if self.headers.get("Origin") not in (None, *(f"http://{h}" for h in self.hosts())):
173
+ return self.reply(403, {"error": "wrong origin"})
174
+ if self.headers.get_content_type() != "application/json":
175
+ return self.reply(415, {"error": "send application/json"})
176
+ if not length.isdecimal() or int(length) > self.MAX_BODY:
177
+ return self.reply(413, {"error": f"body must be at most {self.MAX_BODY} bytes"})
178
+ try:
179
+ body = json.loads(self.rfile.read(int(length)) or b"{}")
180
+ if not isinstance(body, dict):
181
+ raise TypeError("expected a JSON object")
182
+ match self.command, self.path:
183
+ case "POST", "/api/settings":
184
+ if not isinstance(body.get("stages"), dict) or not isinstance(body.get("github"), bool):
185
+ raise TypeError("expected {stages: {...}, github: true or false}")
186
+ # The page that does not send the picked projects keeps them.
187
+ config.save({**body["stages"], "github": {"enabled": body["github"]},
188
+ "projects": {"include": body.get("projects", config.projects())}})
189
+ case "POST", "/api/key":
190
+ config.save_key(body.get("key"))
191
+ case "DELETE", "/api/key":
192
+ config.remove_key()
193
+ case "POST", "/api/run":
194
+ if not self.jobs.start():
195
+ return self.reply(409, {"error": "a run or an estimate is already going"})
196
+ return self.reply(200, self.jobs.status())
197
+ case "DELETE", "/api/run":
198
+ self.jobs.cancel()
199
+ return self.reply(200, self.jobs.status())
200
+ case "POST", "/api/estimate":
201
+ # POST, not GET: it ingests, so it writes the database and must carry the token.
202
+ got = self.jobs.estimate()
203
+ if got is None:
204
+ return self.reply(409, {"error": "a run or an estimate is already going"})
205
+ return self.reply(500 if got["exit_code"] else 200, got)
206
+ case _:
207
+ return self.reply(404, {"error": "no such endpoint"})
208
+ except (ValueError, TypeError) as e: # bad JSON is a ValueError
209
+ return self.reply(400, {"error": str(e)})
210
+ self.reply_settings()
211
+
212
+ do_DELETE = do_POST
213
+
214
+ def reply_settings(self) -> None:
215
+ try:
216
+ self.reply(200, config.settings())
217
+ except SystemExit as e: # config.toml was broken by hand
218
+ self.reply(500, {"error": str(e)})
219
+
220
+ def reply(self, status: int, obj: dict) -> None:
221
+ body = json.dumps(obj).encode()
222
+ self.send_response(status)
223
+ self.send_header("Content-Type", "application/json")
224
+ self.send_header("Content-Length", str(len(body)))
225
+ self.send_header("Cache-Control", "no-store")
226
+ self.end_headers()
227
+ self.wfile.write(body)
228
+
229
+ def end_headers(self):
230
+ self.send_header("Cross-Origin-Resource-Policy", "same-origin")
231
+ self.send_header("X-Content-Type-Options", "nosniff")
232
+ super().end_headers()
233
+
234
+
235
+ def serve(argv: list[str]) -> None:
236
+ p = argparse.ArgumentParser(prog="debrief open", description="Serve the report on localhost.")
237
+ p.add_argument("--port", type=int, default=8765)
238
+ p.add_argument("--no-browser", action="store_true", help="Serve without opening a browser tab")
239
+ args = p.parse_args(argv)
240
+ handler = functools.partial(ReportHandler, directory=str(SITE_OUT))
241
+ try:
242
+ # Loopback only: the report holds transcripts.
243
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", args.port), handler)
244
+ except OSError as e:
245
+ sys.exit(f"debrief: cannot listen on port {args.port} ({e.strerror}). Try --port.")
246
+ jobs = ReportHandler.jobs
247
+ jobs.stop_orphan()
248
+ if not (SITE_OUT / "data" / "index.json").exists():
249
+ print("No report yet: reading your local logs so the page has something to show (no model calls) ...", flush=True)
250
+ # In child processes, as every later run is: a stage run here would keep its database
251
+ # connection open for as long as the server lives, and its lock would fail those runs.
252
+ for name in ("ingest", "export"):
253
+ subprocess.run([sys.executable, "-m", f"{__package__}.cli", name], check=True)
254
+ signal.signal(signal.SIGTERM, signal.default_int_handler) # `kill` stops a run too, as Ctrl-C does
255
+ url = f"http://127.0.0.1:{server.server_port}/"
256
+ url += f"?t={ReportHandler.token}" # the page keeps the token for this tab and drops it from the address
257
+ print(f"Serving {SITE_OUT} at {url} (Ctrl-C to stop)", flush=True)
258
+ # The browser is started with its address as a command-line argument, which other users can
259
+ # see in the process list. So it is given a file only this user can read, which sends it on.
260
+ door = HOME / "open.html"
261
+ try:
262
+ if not args.no_browser:
263
+ door.unlink(missing_ok=True)
264
+ with os.fdopen(os.open(door, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as f:
265
+ f.write(f'<!doctype html><meta http-equiv="refresh" content="0; url={url}">')
266
+ webbrowser.open(door.as_uri())
267
+ server.serve_forever()
268
+ except KeyboardInterrupt:
269
+ print()
270
+ finally:
271
+ door.unlink(missing_ok=True)
272
+ jobs.cancel()
273
+
274
+
275
+ def main(argv: list[str] | None = None) -> None:
276
+ argv = sys.argv[1:] if argv is None else argv
277
+ commands = {"sync": sync, "open": serve}
278
+ if not argv or argv[0] in ("-h", "--help"):
279
+ print(__doc__ + f"\nStages: {', '.join(STAGES)}\nData: {HOME}")
280
+ return
281
+ name, rest = argv[0], argv[1:]
282
+ if name in commands:
283
+ commands[name](rest)
284
+ elif name in STAGES:
285
+ importlib.import_module(f"{__package__}.{STAGES[name]}").main(rest)
286
+ else:
287
+ sys.exit(f"debrief: unknown command {name!r}. Try `debrief --help`.")
288
+
289
+
290
+ if __name__ == "__main__":
291
+ main()
@@ -0,0 +1,147 @@
1
+ """Score candidate definitions of one Jev question against labeled examples.
2
+
3
+ Binary (noul) questions report F1, precision and recall; multiclass (choice) questions such
4
+ as held_up report accuracy, macro-F1 and per-class F1.
5
+
6
+ Labels come from a jev-align run (labels.jsonl). Each candidate replaces the question in
7
+ the same multi-question call classify.py makes, and is scored `--repeats` times because
8
+ Jev's answers vary a little between calls.
9
+
10
+ uv run python -m debrief.compare_definitions found_not_filed .jev-align/runs/<run> candidates.py
11
+ uv run python -m debrief.compare_definitions found_not_filed data/jevalign/testset/found_not_filed.jsonl \
12
+ data/jevalign/candidates/baseline.py --baseline
13
+ uv run python -m debrief.compare_definitions held_up data/jevalign/testset/held_up.jsonl \
14
+ data/jevalign/candidates/held_up.py --baseline
15
+
16
+ candidates.py defines CANDIDATES = {"name": noul(...) or choice(...), ...}; "current" is added for you.
17
+ """
18
+
19
+ import argparse
20
+ import json
21
+ import os
22
+ import runpy
23
+ import statistics
24
+ from collections import Counter
25
+ from concurrent.futures import ThreadPoolExecutor
26
+ from pathlib import Path
27
+
28
+ import httpx
29
+
30
+ from debrief import classify, config
31
+
32
+ TAIL_MARKER = "## Human message" # turn states end with the human message; tail states don't
33
+
34
+
35
+ def load_labels(path: Path) -> list[dict]:
36
+ """A jev-align run directory (its labels.jsonl) or a labels file in the same format."""
37
+ file = path / "labels.jsonl" if path.is_dir() else path
38
+ return [json.loads(line) for line in file.read_text().splitlines() if line.strip()]
39
+
40
+
41
+ def questions_for(field: str, definition: dict, state: str) -> dict:
42
+ """The question set classify.py would send for this state, with one question swapped."""
43
+ if field in classify.REVIEW_QUESTIONS:
44
+ return {**classify.REVIEW_QUESTIONS, field: definition}
45
+ if field == "found_not_filed" and TAIL_MARKER not in state:
46
+ return {field: definition}
47
+ return {**classify.TURN_QUESTIONS, field: definition}
48
+
49
+
50
+ def answer(client: httpx.Client, model: str, field: str, definition: dict, state: str) -> float | str:
51
+ """P(true) for a noul question, the chosen key for a choice question."""
52
+ answers = classify.decide(client, model, {"state": state, "questions": questions_for(field, definition, state)})
53
+ a = answers["answers"][field]
54
+ return a["noul"] if a["type"] == "noul" else a["choice"]
55
+
56
+
57
+ def metrics(labels: list[bool], probs: list[float]) -> dict:
58
+ tp = sum(y and p > 0.5 for y, p in zip(labels, probs, strict=True))
59
+ fp = sum(not y and p > 0.5 for y, p in zip(labels, probs, strict=True))
60
+ fn = sum(y and p <= 0.5 for y, p in zip(labels, probs, strict=True))
61
+ precision = tp / (tp + fp) if tp + fp else 0.0
62
+ recall = tp / (tp + fn) if tp + fn else 0.0
63
+ f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
64
+ return {"precision": precision, "recall": recall, "f1": f1, "tp": tp, "fp": fp, "fn": fn}
65
+
66
+
67
+ def choice_metrics(labels: list[str], preds: list[str], classes: list[str]) -> dict:
68
+ """Accuracy, per-class F1 and macro-F1 over the classes that occur in labels or predictions."""
69
+ pairs = list(zip(labels, preds, strict=True))
70
+ f1 = {}
71
+ for c in classes:
72
+ tp = sum(y == c and p == c for y, p in pairs)
73
+ fp = sum(y != c and p == c for y, p in pairs)
74
+ fn = sum(y == c and p != c for y, p in pairs)
75
+ if tp + fp + fn:
76
+ f1[c] = 2 * tp / (2 * tp + fp + fn)
77
+ return {"accuracy": sum(y == p for y, p in pairs) / len(pairs), "macro_f1": statistics.mean(f1.values()),
78
+ "f1": f1, "wrong": [(y, p) for y, p in pairs if y != p]}
79
+
80
+
81
+ def mean_sd(values: list[float]) -> str:
82
+ return f"{statistics.mean(values):.2f}±{statistics.pstdev(values):.2f}"
83
+
84
+
85
+ def report_noul(rows: list[dict], idx: list[int], candidates: dict, by: dict, repeats: int) -> None:
86
+ labels = [rows[i]["label"] is True for i in idx]
87
+ print(f"{len(idx)} items, {sum(labels)} true")
88
+ print(f"{'candidate':<14} {'F1':>11} {'precision':>11} {'recall':>11} TP/FP/FN per repeat")
89
+ for name in candidates:
90
+ runs = [metrics(labels, [by[name][rep][i] for i in idx]) for rep in range(repeats)]
91
+ cells = {k: mean_sd([r[k] for r in runs]) for k in ("f1", "precision", "recall")}
92
+ counts = " ".join(f"{r['tp']}/{r['fp']}/{r['fn']}" for r in runs)
93
+ print(f"{name:<14} {cells['f1']:>11} {cells['precision']:>11} {cells['recall']:>11} {counts}")
94
+
95
+
96
+ def report_choice(rows: list[dict], idx: list[int], candidates: dict, by: dict, repeats: int) -> None:
97
+ labels = [rows[i]["label"] for i in idx]
98
+ classes = list(dict.fromkeys(k for d in candidates.values() for k in d["criteria"]))
99
+ print(f"{len(idx)} items: " + ", ".join(f"{c} {labels.count(c)}" for c in classes))
100
+ print(f"{'candidate':<14} {'accuracy':>11} {'macro-F1':>11} " + " ".join(f"{c[:11]:>11}" for c in classes))
101
+ for name in candidates:
102
+ runs = [choice_metrics(labels, [by[name][rep][i] for i in idx], classes) for rep in range(repeats)]
103
+ per_class = " ".join(f"{statistics.mean(r['f1'].get(c, 0.0) for r in runs):>11.2f}" for c in classes)
104
+ print(f"{name:<14} {mean_sd([r['accuracy'] for r in runs]):>11} "
105
+ f"{mean_sd([r['macro_f1'] for r in runs]):>11} {per_class}")
106
+ confusions = Counter(pair for r in runs for pair in r["wrong"])
107
+ print(f"{'':<14} misses (label->pred, over all repeats): "
108
+ + ", ".join(f"{y}->{p} {n}" for (y, p), n in confusions.most_common()))
109
+
110
+
111
+ def main(argv: list[str] | None = None) -> None:
112
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
113
+ parser.add_argument("field")
114
+ parser.add_argument("labels", type=Path, help="jev-align run directory or labels .jsonl")
115
+ parser.add_argument("candidates", type=Path)
116
+ parser.add_argument("--repeats", type=int, default=3)
117
+ parser.add_argument("--baseline", action="store_true", help="also score the baseline definition")
118
+ args = parser.parse_args(argv)
119
+
120
+ current = {**classify.TURN_QUESTIONS, **classify.REVIEW_QUESTIONS,
121
+ "found_not_filed": classify.FOUND_NOT_FILED}[args.field]
122
+ candidates = {"current": current, **runpy.run_path(str(args.candidates)).get("CANDIDATES", {})}
123
+ if args.baseline:
124
+ baseline = runpy.run_path(str(Path(args.candidates).parent / "baseline.py"))["BASELINE"]
125
+ candidates = {"baseline": baseline[args.field], **candidates}
126
+ rows = load_labels(args.labels)
127
+ config.load_env()
128
+ model = os.environ.get("JEV_MODEL", "typesafe/jev-1.13")
129
+ key = os.environ["OPENROUTER_API_KEY"]
130
+ headers = {"Authorization": f"Bearer {key}"}
131
+
132
+ jobs = [(name, rep, i) for name in candidates for rep in range(args.repeats) for i in range(len(rows))]
133
+ with httpx.Client(headers=headers, timeout=120) as client, ThreadPoolExecutor(classify.WORKERS) as pool:
134
+ results = list(pool.map(
135
+ lambda j: answer(client, model, args.field, candidates[j[0]], rows[j[2]]["inputs"]["state"]), jobs))
136
+ by = {}
137
+ for (name, rep, i), result in zip(jobs, results, strict=True):
138
+ by.setdefault(name, {}).setdefault(rep, {})[i] = result
139
+
140
+ report = report_choice if current["type"] == "choice" else report_noul
141
+ for split in sorted({r["evaluation_split"] for r in rows}, key=["train", "holdout", "test"].index):
142
+ print(f"\n{split}: ", end="")
143
+ report(rows, [i for i, r in enumerate(rows) if r["evaluation_split"] == split], candidates, by, args.repeats)
144
+
145
+
146
+ if __name__ == "__main__":
147
+ main()