debrief-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
debrief/synthesize.py ADDED
@@ -0,0 +1,1052 @@
1
+ """Turn labels into findings: SQL rollups per project and finding kind, then one Claude call per bundle.
2
+
3
+ Only picked projects are read (`[projects] include` in config.toml; see db.session_projects
4
+ for what a project is). Each gets its own pass, so a finding says which repo it is about.
5
+ CROSS_KINDS are about the person, not a repo, and get one pass over all picked projects.
6
+
7
+ Rollups become evidence bundles (bundles/<run_id>/<pass>/<kind>.json); Claude reads each
8
+ bundle and returns findings, which land in the findings table under a new run_id.
9
+ Each bundle holds at most --max-evidence items; its counts cover all evidence.
10
+ --since sets where evidence starts: messages by ts, CI failures by completed_at, review
11
+ comments by created_at, sessions by last activity. The default is all of it.
12
+
13
+ A kind with no evidence is skipped with a note and never sent to the model: ci and review
14
+ need GitHub data, so a database of local sessions skips them.
15
+
16
+ Claude runs through headless Claude Code (`claude -p`, billed to the Claude subscription) by
17
+ default, or through OpenRouter with --llm openrouter.
18
+
19
+ A bundle's hash (model + prompt + bundle) is saved in its file and in bundles/index.json.
20
+ If the latest run for a kind holds findings from the same hash, they are copied into the new
21
+ run instead of calling the model (--force calls it anyway).
22
+
23
+ uv run python -m debrief.synthesize --dry-run
24
+ uv run python -m debrief.synthesize --kind hotspot --min-confidence 0.8
25
+ uv run python -m debrief.synthesize --llm openrouter
26
+ """
27
+
28
+ import argparse
29
+ import hashlib
30
+ import json
31
+ import math
32
+ import os
33
+ import re
34
+ import shutil
35
+ import sqlite3
36
+ import subprocess
37
+ import sys
38
+ import tempfile
39
+ import time
40
+ from collections import Counter
41
+ from collections.abc import Callable
42
+ from concurrent.futures import ThreadPoolExecutor
43
+ from datetime import UTC, datetime
44
+ from functools import partial
45
+ from pathlib import Path
46
+
47
+ import httpx
48
+
49
+ from debrief.config import choose, load_env, projects
50
+ from debrief.db import connect, repo_projects, session_projects
51
+ from debrief.paths import BUNDLE_DIR
52
+ from debrief.relay import is_relayed
53
+
54
+ INDEX_PATH = BUNDLE_DIR / "index.json"
55
+ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
56
+ MODELS_URL = "https://openrouter.ai/api/v1/models"
57
+ CLAUDE_CODE_TIMEOUT = 900 # seconds per call
58
+ CLAUDE_CODE_RETRIES = (30, 120) # backoff before each retry, for rate and usage limits
59
+ MAX_TOKENS = 8000
60
+ KINDS = ["hotspot", "knowledge_gap", "found_not_filed", "correction_pattern", "prompt_practice", "ci", "review"]
61
+ CROSS_KINDS = ["prompt_practice"] # about the person's habits, not a repo: one pass over all picked projects
62
+ CROSS = "all projects" # that pass's name, beside the project folders
63
+ QUOTE_LEN = 400
64
+ PER_SESSION = 5 # items from one session (or PR, or commit) before others get a turn
65
+ HOTSPOT_FILES = 25
66
+
67
+ # turn_labels field -> column name in the turns CTE; each also gets <name>_p, its confidence.
68
+ TURN_FIELDS = {
69
+ "turn_kind": "turn_kind", "correction_type": "correction_type", "resource_pointed_to": "resource_pointed_to",
70
+ "agent_could_have_found_it": "could_have_found", "human_caught_defect": "caught_defect",
71
+ "design_reversal": "reversal", "found_not_filed": "found_not_filed",
72
+ }
73
+
74
+ # A relayed turn (pasted bot output; see relay.py) is not the person correcting the agent:
75
+ # its correction becomes turn_kind 'relayed', its reversal and caught defect drop, and it
76
+ # counts as something the agent could have found.
77
+ RELAYED = {
78
+ "turn_kind": "CASE WHEN r.msg_idx IS NOT NULL AND l.turn_kind = 'correction' THEN 'relayed' ELSE l.turn_kind END",
79
+ "reversal": "CASE WHEN r.msg_idx IS NULL THEN l.reversal END",
80
+ "caught_defect": "CASE WHEN r.msg_idx IS NULL THEN l.caught_defect END",
81
+ "could_have_found": "CASE WHEN r.msg_idx IS NULL THEN l.could_have_found ELSE 'true' END",
82
+ "could_have_found_p": "CASE WHEN r.msg_idx IS NULL THEN l.could_have_found_p "
83
+ "ELSE COALESCE(l.could_have_found_p, :c) END",
84
+ }
85
+
86
+ # One row per labeled human turn, with only the labels that clear :c. Needs mark_relayed().
87
+ TURNS = "labeled AS (\n SELECT session_id, msg_idx,\n" + ",\n".join(
88
+ f" MAX(CASE WHEN field = '{f}' AND confidence >= :c THEN value END) AS {a},\n"
89
+ f" MAX(CASE WHEN field = '{f}' AND confidence >= :c THEN confidence END) AS {a}_p"
90
+ for f, a in TURN_FIELDS.items()) + "\n FROM turn_labels GROUP BY session_id, msg_idx\n),\n" \
91
+ "turns AS (\n SELECT l.session_id, l.msg_idx, r.msg_idx IS NOT NULL AS relayed,\n" + ",\n".join(
92
+ f" {RELAYED.get(c, 'l.' + c)} AS {c}" for a in TURN_FIELDS.values() for c in (a, f"{a}_p")) + \
93
+ "\n FROM labeled l LEFT JOIN relayed r USING (session_id, msg_idx)\n)"
94
+
95
+ # For each labeled message, the idx of the human message before it (-1 if none).
96
+ # The agent activity a label refers to is the messages in (prev_idx, msg_idx).
97
+ WINDOWS = """
98
+ windows AS (
99
+ SELECT t.session_id, t.msg_idx,
100
+ COALESCE((SELECT MAX(p.idx) FROM messages p
101
+ WHERE p.session_id = t.session_id AND p.idx < t.msg_idx
102
+ AND p.role = 'user' AND p.user_kind = 'human'), -1) AS prev_idx
103
+ FROM (SELECT DISTINCT session_id, msg_idx FROM turn_labels) t
104
+ )"""
105
+
106
+
107
+ def mark_relayed(conn: sqlite3.Connection) -> None:
108
+ """Fill the temp table TURNS reads with human turns that paste bot output."""
109
+ conn.execute("CREATE TEMP TABLE IF NOT EXISTS relayed (session_id TEXT, msg_idx INTEGER, "
110
+ "PRIMARY KEY (session_id, msg_idx))")
111
+ conn.execute("DELETE FROM relayed")
112
+ human = conn.execute("SELECT session_id, idx, text FROM messages WHERE role = 'user' AND user_kind = 'human'")
113
+ conn.executemany("INSERT INTO relayed VALUES (?, ?)", [(s, i) for s, i, t in human.fetchall() if is_relayed(t)])
114
+
115
+
116
+ # Tables the rollups read, and the column that says which pass a row belongs to.
117
+ SCOPED = {"sessions": "session_id", "messages": "session_id", "turn_labels": "session_id",
118
+ "tool_calls": "session_id", "gh_checks": "repo", "gh_review_comments": "repo"}
119
+
120
+
121
+ def scope(conn: sqlite3.Connection, sessions: set[str] | None = None, repos: set[str] = frozenset()) -> None:
122
+ """Limit what the rollups read to these sessions and repos; no sessions lifts the limit.
123
+
124
+ TEMP views named after the tables shadow them, so the rollup SQL stays as it is.
125
+ """
126
+ for table in SCOPED:
127
+ conn.execute(f"DROP VIEW IF EXISTS temp.{table}")
128
+ if sessions is None:
129
+ return
130
+ # Session ids and repo names (owner/name) share the table; neither can be mistaken for the other.
131
+ conn.execute("CREATE TEMP TABLE IF NOT EXISTS scope (id TEXT PRIMARY KEY)")
132
+ conn.execute("DELETE FROM scope")
133
+ conn.executemany("INSERT INTO scope VALUES (?)", [(x,) for x in {*sessions, *repos}])
134
+ for table, column in SCOPED.items():
135
+ conn.execute(f"CREATE TEMP VIEW {table} AS SELECT * FROM main.{table} WHERE {column} IN (SELECT id FROM scope)")
136
+
137
+
138
+ def passes(conn: sqlite3.Connection, picked: list[str]) -> list[dict]:
139
+ """One pass per picked project that has sessions (most human turns first), then one over
140
+ all of them for CROSS_KINDS. Nothing picked: no passes."""
141
+ project, repos = session_projects(conn), repo_projects(conn)
142
+ turns: Counter = Counter()
143
+ for sid, n in conn.execute("SELECT session_id, COUNT(*) FROM main.messages "
144
+ "WHERE role = 'user' AND user_kind = 'human' GROUP BY session_id"):
145
+ turns[project.get(sid)] += n
146
+ out = [{"name": p, "project": p, "sessions": {s for s, q in project.items() if q == p},
147
+ "repos": {r for r, q in repos.items() if q == p}, "turns": turns[p],
148
+ "kinds": [k for k in KINDS if k not in CROSS_KINDS]} for p in sorted(set(picked), key=lambda p: -turns[p])]
149
+ out = [p for p in out if p["sessions"]]
150
+ if not out:
151
+ return []
152
+ return [*out, {"name": CROSS, "project": None, "kinds": CROSS_KINDS, "turns": sum(p["turns"] for p in out),
153
+ "sessions": set().union(*(p["sessions"] for p in out)),
154
+ "repos": set().union(*(p["repos"] for p in out))}]
155
+
156
+
157
+ EVENT_REPO = {"check_failure": "SELECT repo FROM main.gh_checks WHERE check_id = ?",
158
+ "review_comment": "SELECT repo FROM main.gh_review_comments WHERE comment_id = ?"}
159
+
160
+
161
+ def ref_project(conn: sqlite3.Connection, project: dict, repos: dict, ref: dict) -> str | None:
162
+ """The project an evidence ref belongs to: its session's, or its repo's for a CI or review event."""
163
+ if "session_id" in ref:
164
+ return project.get(ref["session_id"])
165
+ row = conn.execute(EVENT_REPO[ref["event_type"]], (ref["event_id"],)).fetchone()
166
+ return repos.get(row[0]) if row else None
167
+
168
+
169
+ def short_name(path: str) -> str:
170
+ """The last part of a project folder, for prompts and tables."""
171
+ return Path(path).name or path
172
+
173
+
174
+ def excerpt(text: str | None, n: int = QUOTE_LEN, tail: bool = False) -> str:
175
+ """Collapse whitespace and cut to n chars, keeping the start (or the end)."""
176
+ text = re.sub(r"\s+", " ", text or "").strip()
177
+ if len(text) <= n:
178
+ return text
179
+ return "…" + text[-n:] if tail else text[:n] + "…"
180
+
181
+
182
+ def rows(conn: sqlite3.Connection, sql: str, params: dict) -> list[dict]:
183
+ return [dict(r) for r in conn.execute(sql, params)]
184
+
185
+
186
+ def agent_text(conn: sqlite3.Connection, session_id: str, prev_idx: int, msg_idx: int, n: int = QUOTE_LEN) -> str:
187
+ """Tail of the assistant text in (prev_idx, msg_idx]."""
188
+ texts = [r[0] for r in conn.execute(
189
+ "SELECT text FROM messages WHERE session_id = ? AND idx > ? AND idx <= ? AND role = 'assistant' "
190
+ "AND text != '' ORDER BY idx", (session_id, prev_idx, msg_idx))]
191
+ return excerpt("\n".join(t for t in texts if t), n, tail=True)
192
+
193
+
194
+ def turn_totals(conn: sqlite3.Connection, matched: list[dict], shown: int, since: str) -> dict:
195
+ """Counts over all matching turns, against every human and labeled turn since `since`."""
196
+ human, human_sessions = conn.execute(
197
+ "SELECT COUNT(*), COUNT(DISTINCT session_id) FROM messages WHERE role = 'user' AND user_kind = 'human' "
198
+ "AND COALESCE(ts, '') >= :since", {"since": since}).fetchone()
199
+ labeled, labeled_sessions = conn.execute(
200
+ "SELECT COUNT(*), COUNT(DISTINCT session_id) FROM messages m WHERE role = 'user' AND user_kind = 'human' "
201
+ "AND COALESCE(ts, '') >= :since "
202
+ "AND EXISTS (SELECT 1 FROM turn_labels l WHERE l.session_id = m.session_id AND l.msg_idx = m.idx)",
203
+ {"since": since}).fetchone()
204
+ return {"items": len(matched), "shown": shown, "sessions": len({t["session_id"] for t in matched}),
205
+ "human_turns": human, "human_sessions": human_sessions,
206
+ "labeled_turns": labeled, "labeled_sessions": labeled_sessions}
207
+
208
+
209
+ def pick(groups: list[list[dict]], n: int, key: Callable[[dict], object], per_key: int = PER_SESSION) -> list[dict]:
210
+ """Up to n items, one per group in turn, taking each group's items in order.
211
+
212
+ No key (session, PR, commit) gets more than per_key items until nothing else is left.
213
+ An item listed in several groups is taken once.
214
+ """
215
+ queues = [list(g) for g in groups if g]
216
+ chosen: list[dict] = []
217
+ taken: set = set()
218
+ per: Counter = Counter()
219
+ for limit in (per_key, math.inf):
220
+ progress = True
221
+ while progress and len(chosen) < n:
222
+ progress = False
223
+ for q in queues:
224
+ if len(chosen) >= n:
225
+ break
226
+ q[:] = [x for x in q if ref_key(x) not in taken]
227
+ i = next((i for i, x in enumerate(q) if per[key(x)] < limit), None)
228
+ if i is None:
229
+ continue
230
+ item = q.pop(i)
231
+ chosen.append(item)
232
+ taken.add(ref_key(item))
233
+ per[key(item)] += 1
234
+ progress = True
235
+ return chosen
236
+
237
+
238
+ def by_session(x: dict) -> str:
239
+ return x["session_id"]
240
+
241
+
242
+ def ranked(items: list[dict]) -> list[dict]:
243
+ return sorted(items, key=lambda x: -(x["_p"] or 0))
244
+
245
+
246
+ def clean(item: dict) -> dict:
247
+ return {k: v for k, v in item.items() if not k.startswith("_")}
248
+
249
+
250
+ # A PR is settled once merged or closed (db.settled_prs). CI and review rollups keep only
251
+ # work whose PR is settled. The outcome rollup drops a session only while its PR is open:
252
+ # a session with no linked PR (local logs, or work never pushed) stays in.
253
+ SESSION_PR = "EXISTS (SELECT 1 FROM session_pr_links l WHERE l.session_id = s.session_id)"
254
+ SETTLED_SESSION = """EXISTS (
255
+ SELECT 1 FROM session_pr_links l
256
+ JOIN settled_prs sp ON sp.repo = l.repo AND sp.pr_number = l.pr_number
257
+ WHERE l.session_id = s.session_id)"""
258
+ SETTLED_CHECK = """EXISTS (
259
+ SELECT 1 FROM gh_commit_prs g JOIN settled_prs sp ON sp.repo = g.repo AND sp.pr_number = g.pr_number
260
+ WHERE g.commit_sha = k.commit_sha)"""
261
+
262
+ # Checks that say nothing about the code. The Entire GitHub App's check is an approval gate
263
+ # (only that app has the slug 'entire'). A gate job fails because another job did: its log
264
+ # compares job results, as in `[ "failure" = "success" ]`. Any check name with such a
265
+ # failure is left out, passes included, so the totals don't count it either.
266
+ # shortcut: knows only this shell form of gate job; add forms as they turn up.
267
+ REAL_CHECK = """(COALESCE(k.app, '') != 'entire' AND COALESCE(k.name, '') NOT IN (
268
+ SELECT COALESCE(name, '') FROM gh_checks WHERE conclusion = 'failure' AND (
269
+ failure_log LIKE '%"failure" = "success"%' OR failure_log LIKE '%"cancelled" = "success"%'
270
+ OR failure_log LIKE '%"skipped" = "success"%')))"""
271
+
272
+ # Real CI failures. flaky = 1 when the same check later passed on the same commit.
273
+ # Needs :since, the window start.
274
+ CI_FAILS = f"""
275
+ ci_fails AS (
276
+ SELECT k.*, EXISTS (
277
+ SELECT 1 FROM gh_checks s
278
+ WHERE s.commit_sha = k.commit_sha AND s.name = k.name AND s.conclusion = 'success'
279
+ AND s.completed_at > k.completed_at) AS flaky,
280
+ {SETTLED_CHECK} AS settled
281
+ FROM gh_checks k
282
+ WHERE k.conclusion = 'failure' AND {REAL_CHECK}
283
+ AND COALESCE(k.completed_at, '') >= :since
284
+ )"""
285
+
286
+
287
+ def event_label_sql(event_type: str, field: str, column: str = "value") -> str:
288
+ return (f"(SELECT {column} FROM event_labels WHERE event_type = '{event_type}' AND event_id = {{id}} "
289
+ f"AND field = '{field}' AND confidence >= :c)")
290
+
291
+
292
+ # --- rollups ---------------------------------------------------------------
293
+
294
+ def hotspot(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
295
+ """Files the agent wrote just before corrections, reversals and caught defects; CI failures by check."""
296
+ flagged = rows(conn, f"""
297
+ WITH {TURNS}, {WINDOWS}
298
+ SELECT tu.*, w.prev_idx, m.text,
299
+ (SELECT json_group_array(DISTINCT f.value)
300
+ FROM tool_calls tc JOIN json_each(tc.files) f
301
+ WHERE tc.session_id = w.session_id AND tc.msg_idx > w.prev_idx AND tc.msg_idx < w.msg_idx
302
+ AND tc.writes = 1 AND f.value NOT LIKE '/%') AS files
303
+ FROM turns tu
304
+ JOIN windows w USING (session_id, msg_idx)
305
+ JOIN messages m ON m.session_id = tu.session_id AND m.idx = tu.msg_idx
306
+ WHERE (tu.turn_kind = 'correction' OR tu.reversal = 'true' OR tu.caught_defect = 'true')
307
+ AND COALESCE(m.ts, '') >= :since
308
+ ORDER BY tu.session_id, tu.msg_idx""", {"c": c, "since": since})
309
+ errors = dict(conn.execute("""
310
+ SELECT f.value, COUNT(*) FROM tool_calls tc JOIN json_each(tc.files) f
311
+ JOIN messages m ON m.session_id = tc.session_id AND m.idx = tc.msg_idx
312
+ WHERE tc.is_error = 1 AND COALESCE(m.ts, '') >= :since GROUP BY f.value""", {"since": since}))
313
+ files: dict[str, dict] = {}
314
+ turns = []
315
+ for t in flagged:
316
+ paths = json.loads(t["files"])
317
+ if not paths:
318
+ continue
319
+ # Rank a turn by its strongest flag.
320
+ p = max((t[f"{a}_p"] or 0) for a, hit in (("turn_kind", "correction"), ("reversal", "true"),
321
+ ("caught_defect", "true")) if t[a] == hit)
322
+ turn = {"session_id": t["session_id"], "msg_idx": t["msg_idx"],
323
+ "labels": {k: t[k] for k in ("turn_kind", "correction_type", "reversal", "caught_defect") if t[k]},
324
+ "files_written_before": paths, "human": excerpt(t["text"]), "_p": p, "_prev": t["prev_idx"]}
325
+ turns.append(turn)
326
+ for path in paths:
327
+ f = files.setdefault(path, {"path": path, "sessions": set(), "turns": 0, "corrections": 0, "reversals": 0,
328
+ "caught_defects": 0, "tool_errors": errors.get(path, 0), "_turns": []})
329
+ f["sessions"].add(t["session_id"])
330
+ f["turns"] += 1
331
+ f["corrections"] += t["turn_kind"] == "correction"
332
+ f["reversals"] += t["reversal"] == "true"
333
+ f["caught_defects"] += t["caught_defect"] == "true"
334
+ f["_turns"].append(turn)
335
+ for f in files.values():
336
+ f["sessions"] = len(f["sessions"])
337
+ top = sorted(files.values(), key=lambda f: (-f["sessions"], -f["turns"], -f["tool_errors"]))[:HOTSPOT_FILES]
338
+
339
+ checks = rows(conn, f"""
340
+ WITH {CI_FAILS}
341
+ SELECT name, {event_label_sql('check_failure', 'failure_class').format(id='k.check_id')} AS failure_class,
342
+ COUNT(*) AS failures, SUM(flaky) AS flaky, GROUP_CONCAT(check_id) AS check_ids
343
+ FROM ci_fails k WHERE settled
344
+ GROUP BY name, failure_class ORDER BY failures DESC LIMIT 25""", {"c": c, "since": since})
345
+ # One example per check for the top checks; the ci kind has the detail.
346
+ for i, ch in enumerate(checks):
347
+ first = int(ch.pop("check_ids").split(",")[0])
348
+ if i < n // 5:
349
+ ch["example"] = {"event_type": "check_failure", "event_id": first}
350
+
351
+ # Each top file's turns in turn; the last group holds turns on other files.
352
+ groups = [ranked(f["_turns"]) for f in top] + [ranked(turns)]
353
+ shown = pick(groups, n - sum("example" in ch for ch in checks), by_session)
354
+ shown_keys = {ref_key(t) for t in shown}
355
+ for f in top:
356
+ f["shown_turns"] = [{"session_id": t["session_id"], "msg_idx": t["msg_idx"]}
357
+ for t in f.pop("_turns") if ref_key(t) in shown_keys]
358
+ for t in shown:
359
+ t["agent_before"] = agent_text(conn, t["session_id"], t["_prev"], t["msg_idx"] - 1, 250)
360
+ totals = {**turn_totals(conn, turns, len(shown), since),
361
+ "flagged_turns_without_files": len(flagged) - len(turns), "files": len(files)}
362
+ bundle = {"totals": totals, "files": top, "flagged_turns": [clean(t) for t in shown]}
363
+ if checks: # absent without GitHub data
364
+ bundle["ci_failures_by_check"] = checks
365
+ return bundle
366
+
367
+
368
+ def knowledge_gap(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
369
+ """Turns where the human pointed the agent at a resource or supplied something it could have found.
370
+
371
+ Counts split the session's opening message (context given up front) from later turns
372
+ (the agent needed steering), per resource type.
373
+ """
374
+ turns = rows(conn, f"""
375
+ WITH {TURNS}, {WINDOWS}
376
+ SELECT tu.*, w.prev_idx, w.prev_idx < 0 AS opening, m.text
377
+ FROM turns tu
378
+ JOIN windows w USING (session_id, msg_idx)
379
+ JOIN messages m ON m.session_id = tu.session_id AND m.idx = tu.msg_idx
380
+ WHERE (tu.could_have_found = 'true' OR tu.resource_pointed_to NOT IN ('none', ''))
381
+ AND COALESCE(m.ts, '') >= :since
382
+ ORDER BY tu.session_id, tu.msg_idx""", {"c": c, "since": since})
383
+ groups: dict[str, dict] = {}
384
+ for t in turns:
385
+ t["_p"] = max(t["could_have_found_p"] if t["could_have_found"] == "true" else 0,
386
+ t["resource_pointed_to_p"] if t["resource_pointed_to"] not in (None, "none", "") else 0)
387
+ g = groups.setdefault(t["resource_pointed_to"] or "unlabeled", {
388
+ part: {"turns": 0, "sessions": set(), "agent_could_have_found_it": 0, "_turns": []}
389
+ for part in ("mid_session", "opening")})
390
+ part = g["opening" if t["opening"] else "mid_session"]
391
+ part["turns"] += 1
392
+ part["sessions"].add(t["session_id"])
393
+ part["agent_could_have_found_it"] += t["could_have_found"] == "true"
394
+ part["_turns"].append(t)
395
+ groups = dict(sorted(groups.items(), key=lambda kv: (-kv[1]["mid_session"]["turns"], -kv[1]["opening"]["turns"])))
396
+ # Opening and mid-session turns queue apart, so examples show both.
397
+ parts = [g[part] for part in ("mid_session", "opening") for g in groups.values()]
398
+ shown = {ref_key(t) for t in pick([ranked(p["_turns"]) for p in parts], n, by_session)}
399
+ for g in groups.values():
400
+ examples = []
401
+ for p in (g["mid_session"], g["opening"]):
402
+ p["sessions"] = len(p["sessions"])
403
+ examples += [{
404
+ "session_id": t["session_id"], "msg_idx": t["msg_idx"], "opening": bool(t["opening"]),
405
+ "agent_could_have_found_it": t["could_have_found"], "turn_kind": t["turn_kind"],
406
+ "human": excerpt(t["text"]),
407
+ "agent_before": agent_text(conn, t["session_id"], t["prev_idx"], t["msg_idx"] - 1, 250),
408
+ } for t in p.pop("_turns") if ref_key(t) in shown]
409
+ g["examples"] = examples
410
+ split = {}
411
+ for part, opening in (("mid_session", False), ("opening", True)):
412
+ these = [t for t in turns if bool(t["opening"]) == opening]
413
+ split[part] = {"turns": len(these), "sessions": len({t["session_id"] for t in these})}
414
+ return {"totals": {**turn_totals(conn, turns, len(shown), since), **split}, "by_resource_type": groups}
415
+
416
+
417
+ def found_not_filed(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
418
+ """Every place the agent noticed a problem outside its task and left it unfiled in the session."""
419
+ hits = rows(conn, f"""
420
+ WITH {WINDOWS}
421
+ SELECT l.session_id, l.msg_idx, l.confidence, w.prev_idx, m.role, m.text
422
+ FROM turn_labels l
423
+ JOIN windows w USING (session_id, msg_idx)
424
+ JOIN messages m ON m.session_id = l.session_id AND m.idx = l.msg_idx
425
+ WHERE l.field = 'found_not_filed' AND l.value = 'true' AND l.confidence >= :c AND COALESCE(m.ts, '') >= :since
426
+ ORDER BY l.session_id, l.msg_idx""", {"c": c, "since": since})
427
+ # Repeats of one deferred problem are reworded, so the model merges them, not us.
428
+ for h in hits:
429
+ h["_p"] = h["confidence"]
430
+ shown = sorted(pick([ranked(hits)], n, by_session), key=lambda h: (h["session_id"], h["msg_idx"]))
431
+ grouped: dict[str, list] = {}
432
+ for h in shown:
433
+ # On a human turn the flag covers the agent messages before it; on the
434
+ # per-session row (an assistant message) it covers that message and those before it.
435
+ end = h["msg_idx"] if h["role"] == "assistant" else h["msg_idx"] - 1
436
+ item = {"session_id": h["session_id"], "msg_idx": h["msg_idx"], "confidence": round(h["confidence"], 2),
437
+ "agent_text": agent_text(conn, h["session_id"], h["prev_idx"], end, 800)}
438
+ if h["role"] == "user":
439
+ item["human_reply"] = excerpt(h["text"], 200)
440
+ grouped.setdefault(h["session_id"], []).append(item)
441
+ totals = {"items": len(hits), "shown": len(shown), "sessions": len({h["session_id"] for h in hits}),
442
+ "shown_sessions": len(grouped)}
443
+ return {"totals": totals, "sessions": [{"session_id": k, "hits": v} for k, v in grouped.items()]}
444
+
445
+
446
+ def correction_pattern(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
447
+ """Correction types, how often each occurs, and example turns."""
448
+ turns = rows(conn, f"""
449
+ WITH {TURNS}, {WINDOWS}
450
+ SELECT tu.*, w.prev_idx, m.text
451
+ FROM turns tu
452
+ JOIN windows w USING (session_id, msg_idx)
453
+ JOIN messages m ON m.session_id = tu.session_id AND m.idx = tu.msg_idx
454
+ WHERE tu.turn_kind = 'correction' AND COALESCE(m.ts, '') >= :since
455
+ ORDER BY tu.session_id, tu.msg_idx""", {"c": c, "since": since})
456
+ groups: dict[str, dict] = {}
457
+ for t in turns:
458
+ t["_p"] = t["correction_type_p"] or t["turn_kind_p"]
459
+ g = groups.setdefault(t["correction_type"] or "unlabeled", {"count": 0, "sessions": set(), "_turns": []})
460
+ g["count"] += 1
461
+ g["sessions"].add(t["session_id"])
462
+ g["_turns"].append(t)
463
+ groups = dict(sorted(groups.items(), key=lambda kv: -kv[1]["count"]))
464
+ shown = {ref_key(t) for t in pick([ranked(g["_turns"]) for g in groups.values()], n, by_session)}
465
+ for g in groups.values():
466
+ g["sessions"] = len(g["sessions"])
467
+ g["examples"] = [{
468
+ "session_id": t["session_id"], "msg_idx": t["msg_idx"],
469
+ "reversal": t["reversal"], "caught_defect": t["caught_defect"],
470
+ "human": excerpt(t["text"]),
471
+ "agent_before": agent_text(conn, t["session_id"], t["prev_idx"], t["msg_idx"] - 1, 300),
472
+ } for t in g.pop("_turns") if ref_key(t) in shown]
473
+ return {"totals": turn_totals(conn, turns, len(shown), since), "by_correction_type": groups}
474
+
475
+
476
+ def prompt_practice(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
477
+ """Session-level prompt labels against outcome and per-session correction/reversal rates."""
478
+ sessions = rows(conn, f"""
479
+ WITH {TURNS},
480
+ per_session AS (
481
+ SELECT session_id,
482
+ SUM(turn_kind = 'correction') AS corrections,
483
+ SUM(reversal = 'true') AS reversals,
484
+ SUM(caught_defect = 'true') AS caught_defects
485
+ FROM turns GROUP BY session_id
486
+ ),
487
+ first_human AS (
488
+ SELECT session_id, MIN(idx) AS msg_idx, COUNT(*) AS human_turns
489
+ FROM messages WHERE role = 'user' AND user_kind = 'human' GROUP BY session_id
490
+ )
491
+ SELECT s.session_id, fh.msg_idx, fh.human_turns, s.agent, s.prompt, AVG(sl.confidence) AS _p,
492
+ MAX(CASE WHEN sl.field = 'task_type' THEN sl.value END) AS task_type,
493
+ MAX(CASE WHEN sl.field = 'prompt_style' THEN sl.value END) AS prompt_style,
494
+ MAX(CASE WHEN sl.field = 'prompt_specificity' THEN sl.value END) AS prompt_specificity,
495
+ MAX(CASE WHEN sl.field = 'planned_first' THEN sl.value END) AS planned_first,
496
+ MAX(CASE WHEN sl.field = 'tests_added' THEN sl.value END) AS tests_added,
497
+ MAX(CASE WHEN sl.field = 'outcome' THEN sl.value END) AS outcome,
498
+ COALESCE(p.corrections, 0) AS corrections, COALESCE(p.reversals, 0) AS reversals,
499
+ COALESCE(p.caught_defects, 0) AS caught_defects,
500
+ {SESSION_PR} AS has_pr, {SETTLED_SESSION} AS settled
501
+ FROM sessions s
502
+ JOIN session_labels sl ON sl.session_id = s.session_id AND sl.confidence >= :c
503
+ JOIN first_human fh ON fh.session_id = s.session_id
504
+ LEFT JOIN per_session p ON p.session_id = s.session_id
505
+ WHERE COALESCE(s.last_activity_at, '') >= :since
506
+ GROUP BY s.session_id ORDER BY s.session_id""", {"c": c, "since": since})
507
+ unsettled = sum(s["has_pr"] and not s["settled"] for s in sessions)
508
+ sessions = [s for s in sessions if s["settled"] or not s["has_pr"]]
509
+ by_label: dict[str, dict] = {}
510
+ for field in ("prompt_style", "prompt_specificity", "task_type", "planned_first", "tests_added"):
511
+ for s in sessions:
512
+ if not s[field]:
513
+ continue
514
+ g = by_label.setdefault(field, {}).setdefault(
515
+ s[field], {"sessions": 0, "outcomes": {}, "human_turns": 0, "corrections": 0, "reversals": 0})
516
+ g["sessions"] += 1
517
+ g["outcomes"][s["outcome"] or "unlabeled"] = g["outcomes"].get(s["outcome"] or "unlabeled", 0) + 1
518
+ for k in ("human_turns", "corrections", "reversals"):
519
+ g[k] += s[k]
520
+ for groups in by_label.values():
521
+ for g in groups.values():
522
+ g["corrections_per_session"] = round(g["corrections"] / g["sessions"], 2)
523
+ g["reversals_per_session"] = round(g["reversals"] / g["sessions"], 2)
524
+ # Show sessions from every outcome in turn, most confidently labeled first.
525
+ outcomes: dict[str, list] = {}
526
+ for s in ranked(sessions):
527
+ outcomes.setdefault(s["outcome"] or "unlabeled", []).append(s)
528
+ shown = sorted(pick(list(outcomes.values()), n, by_session), key=lambda s: s["session_id"])
529
+ per_session = [
530
+ {"session_id": s["session_id"], "msg_idx": s["msg_idx"], "agent": s["agent"],
531
+ **{k: s[k] for k in ("task_type", "prompt_style", "prompt_specificity", "planned_first", "tests_added",
532
+ "outcome", "human_turns", "corrections", "reversals", "caught_defects")},
533
+ "pr": "merged_or_closed" if s["has_pr"] else "none", "opening_prompt": excerpt(s["prompt"], 300)}
534
+ for s in shown]
535
+ totals = {"items": len(sessions), "shown": len(shown), "sessions": len(sessions), "excluded_unsettled": unsettled,
536
+ "sessions_without_pr": sum(not s["has_pr"] for s in sessions)}
537
+ return {"totals": totals, "by_label": by_label, "per_session": per_session}
538
+
539
+
540
+ def ci(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
541
+ """Failed CI checks by failure_class, with log snippets, flake flags, and their PRs and sessions."""
542
+ fails = rows(conn, f"""
543
+ WITH {CI_FAILS}
544
+ SELECT k.check_id, k.name, k.commit_sha, k.failure_log, k.flaky, k.settled,
545
+ {event_label_sql('check_failure', 'failure_class').format(id='k.check_id')} AS failure_class,
546
+ {event_label_sql('check_failure', 'failure_class', 'confidence').format(id='k.check_id')} AS _p,
547
+ (SELECT GROUP_CONCAT(repo || '#' || pr_number) FROM gh_commit_prs cp WHERE cp.commit_sha = k.commit_sha) AS prs,
548
+ (SELECT GROUP_CONCAT(DISTINCT session_id) FROM session_commits sc WHERE sc.commit_sha = k.commit_sha) AS sessions
549
+ FROM ci_fails k
550
+ ORDER BY k.check_id""", {"c": c, "since": since})
551
+ unsettled = sum(not f["settled"] for f in fails)
552
+ fails = [f for f in fails if f["settled"]]
553
+ total = conn.execute(f"SELECT COUNT(*) FROM gh_checks k WHERE conclusion IS NOT NULL "
554
+ f"AND {REAL_CHECK} AND COALESCE(completed_at, '') >= :since "
555
+ f"AND {SETTLED_CHECK}", {"since": since}).fetchone()[0]
556
+ groups: dict[str, dict] = {}
557
+ for f in fails:
558
+ f["event_type"], f["event_id"] = "check_failure", f["check_id"]
559
+ g = groups.setdefault(f["failure_class"] or "unlabeled", {"count": 0, "flaky": 0, "checks": {}, "_fails": []})
560
+ g["count"] += 1
561
+ g["flaky"] += f["flaky"]
562
+ g["checks"][f["name"]] = g["checks"].get(f["name"], 0) + 1
563
+ g["_fails"].append(f)
564
+ groups = dict(sorted(groups.items(), key=lambda kv: -kv[1]["count"]))
565
+ shown = {ref_key(f) for f in pick([ranked(g["_fails"]) for g in groups.values()], n, lambda f: f["commit_sha"])}
566
+ for g in groups.values():
567
+ g["examples"] = [{
568
+ "event_type": "check_failure", "event_id": f["check_id"], "check": f["name"],
569
+ "flaky": bool(f["flaky"]), "prs": f["prs"], "sessions": f["sessions"],
570
+ "log": excerpt(f["failure_log"], 500, tail=True),
571
+ } for f in g.pop("_fails") if ref_key(f) in shown]
572
+ totals = {"items": len(fails), "shown": len(shown), "checks": total, "failures": len(fails),
573
+ "excluded_unsettled": unsettled, "flaky": sum(f["flaky"] for f in fails)}
574
+ return {"totals": totals, "by_failure_class": groups}
575
+
576
+
577
+ def review(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
578
+ """Review comments on settled PRs by whether they held up, bot vs human, with thread state."""
579
+ comments = rows(conn, f"""
580
+ SELECT rc.comment_id, rc.repo || '#' || rc.pr_number AS pr, rc.author, rc.is_bot, rc.path, rc.body,
581
+ rc.kind, rc.review_state, rc.is_resolved, rc.is_outdated,
582
+ {event_label_sql('review_comment', 'held_up').format(id='rc.comment_id')} AS held_up,
583
+ {event_label_sql('review_comment', 'held_up', 'confidence').format(id='rc.comment_id')} AS _p,
584
+ {event_label_sql('review_comment', 'comment_type').format(id='rc.comment_id')} AS comment_type,
585
+ EXISTS (SELECT 1 FROM settled_prs sp WHERE sp.repo = rc.repo AND sp.pr_number = rc.pr_number) AS settled
586
+ FROM gh_review_comments rc
587
+ WHERE rc.in_reply_to IS NULL AND COALESCE(rc.created_at, '') >= :since
588
+ ORDER BY rc.comment_id""", {"c": c, "since": since})
589
+ unsettled = sum(not r["settled"] for r in comments)
590
+ comments = [r for r in comments if r["settled"]]
591
+ counts: dict[str, dict] = {}
592
+ groups: dict[str, list] = {}
593
+ for r in comments:
594
+ who = "bot" if r["is_bot"] else "human"
595
+ held, ctype, kind = r["held_up"] or "unlabeled", r["comment_type"] or "unlabeled", r["kind"] or "unknown"
596
+ if kind == "inline":
597
+ # GitHub thread state decides fixed vs open; the label only says valid, wrong or unclear.
598
+ validity = {"valid_and_fixed": "valid", "valid_open": "valid"}.get(held, held)
599
+ status = "resolved" if r["is_resolved"] else "outdated" if r["is_outdated"] else "open"
600
+ outcome = f"{validity}/{status}"
601
+ else:
602
+ outcome = held
603
+ w = counts.setdefault(who, {"comment_type": {}})
604
+ w.setdefault(kind, {})[outcome] = w.setdefault(kind, {}).get(outcome, 0) + 1
605
+ w["comment_type"][ctype] = w["comment_type"].get(ctype, 0) + 1
606
+ groups.setdefault(f"{who}/{kind}/{outcome}", []).append(
607
+ {"event_type": "review_comment", "event_id": r["comment_id"], "pr": r["pr"],
608
+ "author": r["author"], "review_state": r["review_state"],
609
+ "is_resolved": r["is_resolved"], "is_outdated": r["is_outdated"], "path": r["path"],
610
+ "comment_type": ctype, "body": excerpt(r["body"], 300), "_p": r["_p"]})
611
+ groups = dict(sorted(groups.items(), key=lambda kv: -len(kv[1])))
612
+ shown = {ref_key(x) for x in pick([ranked(g) for g in groups.values()], n, lambda x: x["pr"])}
613
+ examples = {k: [clean(x) for x in g if ref_key(x) in shown] for k, g in groups.items()}
614
+ totals = {"items": len(comments), "shown": len(shown), "comments": len(comments),
615
+ "prs": len({r["pr"] for r in comments}), "excluded_unsettled": unsettled}
616
+ return {"totals": totals, "counts": counts, "examples": {k: v for k, v in examples.items() if v}}
617
+
618
+
619
+ ROLLUPS = {
620
+ "hotspot": hotspot, "knowledge_gap": knowledge_gap, "found_not_filed": found_not_filed,
621
+ "correction_pattern": correction_pattern, "prompt_practice": prompt_practice, "ci": ci, "review": review,
622
+ }
623
+
624
+
625
+ # --- evidence refs -------------------------------------------------------------
626
+
627
+ def ref_key(d: dict) -> tuple | None:
628
+ """A hashable key for a session or event ref, or None if d isn't one."""
629
+ try:
630
+ if "session_id" in d and "msg_idx" in d:
631
+ return ("msg", str(d["session_id"]), int(d["msg_idx"]))
632
+ if "event_type" in d and "event_id" in d:
633
+ return ("event", str(d["event_type"]), int(d["event_id"]))
634
+ except (TypeError, ValueError):
635
+ pass
636
+ return None
637
+
638
+
639
+ def ref_of(key: tuple) -> dict:
640
+ kind, a, b = key
641
+ return {"session_id": a, "msg_idx": b} if kind == "msg" else {"event_type": a, "event_id": b}
642
+
643
+
644
+ def bundle_refs(obj) -> set[tuple]:
645
+ """Every ref anywhere in a bundle."""
646
+ if isinstance(obj, list):
647
+ return set().union(*(bundle_refs(x) for x in obj)) if obj else set()
648
+ if isinstance(obj, dict):
649
+ found = {k} if (k := ref_key(obj)) else set()
650
+ return found.union(*(bundle_refs(v) for v in obj.values()))
651
+ return set()
652
+
653
+
654
+ # --- LLM pass ----------------------------------------------------------------------
655
+
656
+ SYSTEM = """You analyze how a developer or a team works with AI coding agents. It may be one person; the
657
+ bundle's refs and counts show how many sessions there are.
658
+ You get an evidence bundle: SQL rollups over labeled agent session transcripts and, when the data has them,
659
+ CI checks and PR review comments. Take the repo, language and tools from the evidence; assume none.
660
+ Labels come from a classifier and can be wrong; quotes are excerpts.
661
+ The bundle shows a capped sample of items. "totals" and the per-group counts cover all evidence, not just the sample:
662
+ use them when you state counts.
663
+
664
+ Write findings for the people who ran these sessions. Rules:
665
+ - Each finding must be specific and actionable: a concrete line to add to the agent's instruction file
666
+ (CLAUDE.md, AGENTS.md or whichever the sessions use), a test to add,
667
+ a script or doc to write, an issue to file, a check to change. Name files, commands and checks when the evidence does.
668
+ - Cite evidence only by copying refs that appear in the bundle: {"session_id": ..., "msg_idx": ...} or
669
+ {"event_type": ..., "event_id": ...}. Never invent or alter a ref.
670
+ - Base each finding on a pattern across several items where possible; say how many items support it.
671
+ - If the evidence is too thin to support a finding, return fewer findings or none. Do not pad. Do not invent.
672
+ - Plain, short sentences.
673
+
674
+ Reply with JSON only:
675
+ {"findings": [{"kind": "<kind>", "title": "<one line>", "detail": "<what the evidence shows, with counts>",
676
+ "suggested_fix": "<the concrete change>", "evidence": [<refs>]}]}"""
677
+
678
+ KIND_HINTS = {
679
+ "hotspot": "Files the agent had just written when a human corrected it, reversed a design or caught a defect; "
680
+ "plus CI failures by check when the data has any. Look for files or areas that need better docs, tests or guardrails.",
681
+ "knowledge_gap": "Turns where the human pointed the agent at something (spec, script, doc, prior PR, code) "
682
+ "or the agent could have found the answer itself. Counts split each resource type into "
683
+ "opening (the session's first human message) and mid_session (later turns); examples carry "
684
+ "the same flag. A pointer in the opening message is context given up front, a good habit, "
685
+ "and not a gap in itself. Mid-session pointers and could-have-found turns are the gaps. "
686
+ "Report opening and mid-session counts separately and never add them together. Look for "
687
+ "knowledge to put where agents will find it.",
688
+ "found_not_filed": "Places where the agent noticed a bug or problem outside its task and deferred it without filing "
689
+ "it during the session. Hits are grouped by session. The agent often repeats the same deferred problem on "
690
+ "later turns: merge repeats into one finding that cites all their refs, and count each "
691
+ "problem once. Some hits are false positives (e.g. a test-result summary): skip any "
692
+ "item whose agent_text does not explicitly defer or leave open a specific problem. Each real "
693
+ "one is a candidate issue: write the issue title and a one-line body in suggested_fix.",
694
+ "correction_pattern": "What kinds of corrections humans make, with examples. Look for repeat mistakes that a "
695
+ "rule, check or test would prevent.",
696
+ "prompt_practice": "Prompt style, specificity and task type against outcome and correction rates. Sessions "
697
+ "whose PR is still open are left out. Each session's pr is merged_or_closed or none; "
698
+ "totals.sessions_without_pr counts the none ones. A session with no PR has no merge to "
699
+ "confirm its outcome, which is then only the classifier's reading of the transcript: say "
700
+ "so when a finding rests on those sessions. Small samples: "
701
+ "report differences only when counts support them, and state the counts.",
702
+ "ci": "Failed CI checks on merged or closed PRs, by failure class with log tails; flaky = the same check later passed on the same "
703
+ "commit (a re-run fixed it). Look for flaky tests to fix, and for failures an agent could catch before pushing "
704
+ "(a local task or script, a pre-push hook, a line in the agent's instruction file).",
705
+ "review": "PR review comments on merged or closed PRs, bot vs human, split by kind (inline thread, review "
706
+ "body, issue comment). Inline outcomes read <validity>/<thread state>: validity (valid | wrong | "
707
+ "unclear) comes from a classifier, thread state (resolved | outdated | open) from GitHub. Other "
708
+ "kinds carry the classifier's held_up label only. Look for review noise to cut and for "
709
+ "comment types that point at gaps agents keep leaving.",
710
+ }
711
+
712
+
713
+ def user_prompt(kind: str, bundle: dict, about: str = "") -> str:
714
+ """about: a line saying which project the bundle covers; empty for the pass over all of them."""
715
+ return (f"Finding kind: {kind}\n{KIND_HINTS[kind]}\n{about}\n"
716
+ f"Evidence bundle:\n{json.dumps(bundle, ensure_ascii=False)}")
717
+
718
+
719
+ def canonical_model(model: str) -> str:
720
+ """One name per model across transports, in OpenRouter form: claude-opus-5-5 -> anthropic/claude-opus-5.5."""
721
+ if model.startswith("claude-"):
722
+ return "anthropic/" + re.sub(r"-(\d{1,2})-(\d{1,2})$", r"-\1.\2", model)
723
+ return model
724
+
725
+
726
+ def prompt_hash(model: str, user: str) -> str:
727
+ # The hash covers the model and prompts, not the transport: Claude Code and OpenRouter
728
+ # send the same prompts to the same model, so a bundle cached from one is reused by the
729
+ # other. A different model (or model version) misses the cache.
730
+ return hashlib.sha256(json.dumps([canonical_model(model), SYSTEM, user], ensure_ascii=False).encode()).hexdigest()
731
+
732
+
733
+ def input_price(model: str) -> float | None:
734
+ """USD per input token from OpenRouter's model list, or None if it can't be fetched."""
735
+ try:
736
+ resp = httpx.get(MODELS_URL, timeout=15)
737
+ resp.raise_for_status()
738
+ model_info = next(m for m in resp.json()["data"] if m["id"] == model)
739
+ return float(model_info["pricing"]["prompt"])
740
+ except (httpx.HTTPError, ValueError, KeyError, TypeError, StopIteration):
741
+ return None
742
+
743
+
744
+ def parse_json(text: str) -> dict:
745
+ """Parse a model reply that should be JSON, tolerating code fences and surrounding prose."""
746
+ text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())
747
+ try:
748
+ return json.loads(text)
749
+ except json.JSONDecodeError:
750
+ start, end = text.find("{"), text.rfind("}")
751
+ return json.loads(text[start:end + 1]) if start >= 0 < end else {}
752
+
753
+
754
+ def findings_of(text: str) -> list[dict]:
755
+ findings = parse_json(text).get("findings", [])
756
+ return findings if isinstance(findings, list) else []
757
+
758
+
759
+ def ask_openrouter(user: str, model: str, key: str, system: str = SYSTEM) -> tuple[list[dict], dict]:
760
+ """One chat completion for one bundle prompt; returns (findings, usage)."""
761
+ resp = httpx.post(
762
+ OPENROUTER_URL,
763
+ headers={"Authorization": f"Bearer {key}"},
764
+ json={"model": model, "max_tokens": MAX_TOKENS, "response_format": {"type": "json_object"},
765
+ "usage": {"include": True},
766
+ "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]},
767
+ timeout=600,
768
+ )
769
+ resp.raise_for_status()
770
+ body = resp.json()
771
+ usage = body.get("usage") or {}
772
+ return findings_of(body["choices"][0]["message"]["content"] or ""), {
773
+ "in": usage.get("prompt_tokens"), "out": usage.get("completion_tokens"), "cost": usage.get("cost") or 0}
774
+
775
+
776
+ class ClaudeCodeError(RuntimeError):
777
+ pass
778
+
779
+
780
+ def claude_code_once(user: str, model: str, cwd: str, system: str = SYSTEM) -> dict:
781
+ """Run `claude -p` as a plain model call and return its JSON result envelope."""
782
+ cmd = [
783
+ "claude", "-p", "--model", model, "--output-format", "json", "--max-turns", "1",
784
+ "--system-prompt", system,
785
+ "--tools", "", # no tools: one model turn, nothing else
786
+ "--setting-sources", "", # skip user/project settings, hooks and CLAUDE.md
787
+ "--strict-mcp-config", # no MCP servers
788
+ "--disable-slash-commands", # no skills
789
+ "--no-session-persistence", # keep these out of `claude --resume`
790
+ ]
791
+ try:
792
+ proc = subprocess.run(cmd, input=user, capture_output=True, text=True, cwd=cwd,
793
+ timeout=CLAUDE_CODE_TIMEOUT, check=False)
794
+ except subprocess.TimeoutExpired as e:
795
+ raise ClaudeCodeError(f"timed out after {CLAUDE_CODE_TIMEOUT}s") from e
796
+ try:
797
+ env = json.loads(proc.stdout)
798
+ except json.JSONDecodeError:
799
+ env = None
800
+ if proc.returncode or not isinstance(env, dict) or env.get("is_error"):
801
+ detail = (env or {}).get("result") or proc.stderr.strip() or proc.stdout.strip()
802
+ # An API error still reports subtype "success", so show the HTTP status instead.
803
+ status = (env or {}).get("api_error_status")
804
+ why = f"HTTP {status}" if status else (env or {}).get("subtype", "no JSON envelope")
805
+ raise ClaudeCodeError(f"exit {proc.returncode}, {why}: {excerpt(detail, 300)}")
806
+ return env
807
+
808
+
809
+ def ask_claude_code(user: str, model: str, system: str = SYSTEM,
810
+ parse: Callable[[str], list[dict]] = findings_of) -> tuple[list[dict], dict]:
811
+ """One headless Claude Code call, retried on failure; returns (parsed items, usage)."""
812
+ # Run from an empty directory so nothing from this project reaches the model.
813
+ with tempfile.TemporaryDirectory(prefix="synth-") as cwd:
814
+ for wait in (*CLAUDE_CODE_RETRIES, None):
815
+ try:
816
+ env = claude_code_once(user, model, cwd, system)
817
+ break
818
+ except ClaudeCodeError as e:
819
+ if wait is None:
820
+ raise
821
+ print(f" claude -p failed ({e}); retrying in {wait}s")
822
+ time.sleep(wait)
823
+ usage = env.get("usage") or {}
824
+ tokens_in = sum(usage.get(k) or 0 for k in
825
+ ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"))
826
+ return parse(env.get("result") or ""), {
827
+ "in": tokens_in, "out": usage.get("output_tokens"), "cost": env.get("total_cost_usd") or 0,
828
+ "seconds": round((env.get("duration_ms") or 0) / 1000)}
829
+
830
+
831
+ def check_evidence(kind: str, findings: list[dict], allowed: set[tuple]) -> list[dict]:
832
+ """Keep only refs present in the bundle; drop findings left with none."""
833
+ kept = []
834
+ for f in findings:
835
+ refs = [ref_key(r) for r in f.get("evidence") or [] if isinstance(r, dict)]
836
+ good = list(dict.fromkeys(r for r in refs if r in allowed))
837
+ if len(good) < len(refs):
838
+ print(f" {kind}: dropped {len(refs) - len(good)} unknown ref(s) from {f.get('title')!r}")
839
+ if not good:
840
+ print(f" {kind}: dropped finding with no valid evidence: {f.get('title')!r}")
841
+ continue
842
+ kept.append({**f, "kind": kind, "evidence": [ref_of(r) for r in good]})
843
+ return kept
844
+
845
+
846
+ # --- cache -----------------------------------------------------------------------------
847
+
848
+ def load_index() -> dict:
849
+ try:
850
+ return json.loads(INDEX_PATH.read_text())
851
+ except (FileNotFoundError, json.JSONDecodeError):
852
+ return {}
853
+
854
+
855
+ def cached_run(conn: sqlite3.Connection, index: dict, digest: str, kind: str, name: str = CROSS) -> dict | None:
856
+ """The index entry whose findings came from this exact prompt, if its run is still the latest for kind.
857
+
858
+ The entry gains "ids", its findings' ids. An entry from before passes existed has no
859
+ "pass" and covers every finding of its kind in its run.
860
+ """
861
+ entry = index.get(digest)
862
+ if not entry or entry["kind"] != kind or entry.get("pass", CROSS) != name:
863
+ return None
864
+ ids = entry["ids"] if "ids" in entry else [r[0] for r in conn.execute(
865
+ "SELECT finding_id FROM findings WHERE run_id = ? AND kind = ?", (entry["run_id"], kind))]
866
+ count = conn.execute(f"SELECT COUNT(*) FROM findings WHERE run_id = ? AND kind = ? "
867
+ f"AND finding_id IN ({','.join('?' * len(ids))})", (entry["run_id"], kind, *ids)).fetchone()[0]
868
+ latest = conn.execute("SELECT MAX(run_id) FROM findings WHERE kind = ?", (kind,)).fetchone()[0]
869
+ if count != entry["findings"] or (latest and latest > entry["run_id"]):
870
+ return None
871
+ return {**entry, "ids": ids}
872
+
873
+
874
+ def save_findings(conn: sqlite3.Connection, run_id: str, kind: str, findings: list[dict], now: str) -> list[int]:
875
+ """Insert findings, each with its "project" (None: it spans projects); returns their ids."""
876
+ ids = [conn.execute(
877
+ "INSERT INTO findings (run_id, kind, title, detail, suggested_fix, evidence, project, created_at) "
878
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
879
+ (run_id, kind, f.get("title"), f.get("detail"), f.get("suggested_fix"),
880
+ json.dumps(f["evidence"]), f.get("project"), now)).lastrowid for f in findings]
881
+ conn.commit()
882
+ return ids
883
+
884
+
885
+ # --- main ------------------------------------------------------------------------------
886
+
887
+ # Kinds built only from GitHub data: the table each needs, and what to call it in the skip note.
888
+ NEEDS = {"ci": ("gh_checks", "CI checks"), "review": ("gh_review_comments", "PR review comments")}
889
+
890
+
891
+ def missing_data(conn: sqlite3.Connection, kind: str) -> str | None:
892
+ """Why kind can't be built from this database, or None if it can."""
893
+ table, what = NEEDS.get(kind, (None, None))
894
+ if table and not conn.execute(f"SELECT 1 FROM {table} LIMIT 1").fetchone():
895
+ return f"no {what} in the database (needs GitHub data)"
896
+ return None
897
+
898
+
899
+ def main(argv: list[str] | None = None) -> None:
900
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
901
+ parser.add_argument("--kind", choices=KINDS, help="Only this finding kind")
902
+ parser.add_argument("--min-confidence", type=float, default=0.7, help="Ignore labels below this confidence")
903
+ parser.add_argument("--max-evidence", type=int, default=40, help="Most evidence items per bundle")
904
+ parser.add_argument("--since", default="", help="Ignore evidence before this ISO date (default: keep all)")
905
+ parser.add_argument("--llm", choices=["claude-code", "openrouter"],
906
+ help="claude-code: `claude -p` on the Claude subscription; openrouter: OpenRouter API. "
907
+ "Default: [synthesize] in config.toml, else claude-code")
908
+ parser.add_argument("--parallel", type=int,
909
+ help="Concurrent model calls (default 2 for claude-code, which shares subscription "
910
+ "limits; one per bundle for openrouter)")
911
+ parser.add_argument("--dry-run", action="store_true", help="Build bundles and print them; no model call")
912
+ parser.add_argument("--force", action="store_true", help="Call the model even when a bundle is cached")
913
+ parser.add_argument("--db", type=Path, help="Database path (default debrief.db)")
914
+ args = parser.parse_args(argv)
915
+ load_env()
916
+
917
+ conn = connect(args.db) if args.db else connect()
918
+ mark_relayed(conn)
919
+ args.llm, model = choose("synthesize", args.llm)
920
+ subscription = args.llm == "claude-code"
921
+ price = None if subscription else input_price(model)
922
+ todo_passes = passes(conn, projects())
923
+ if not todo_passes:
924
+ print("no picked project has sessions: nothing to do. `debrief sync` lists the projects; "
925
+ "`debrief sync --project DIR` picks one.")
926
+ return
927
+ run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
928
+ out = BUNDLE_DIR / run_id
929
+ out.mkdir(parents=True, exist_ok=True)
930
+ index = load_index()
931
+
932
+ # Every dict below is keyed by (pass name, kind).
933
+ bundles, prompts, digests, cached, of_pass = {}, {}, {}, {}, {}
934
+ todo_tokens = 0
935
+ for i, p in enumerate(todo_passes):
936
+ kinds = [k for k in p["kinds"] if args.kind in (None, k)]
937
+ if not kinds:
938
+ continue
939
+ name = p["name"]
940
+ print(f"\n== {short_name(name)}: {len(p['sessions'])} sessions, {p['turns']} human turns")
941
+ about = f"Project: {short_name(name)}\n" if p["project"] else ""
942
+ scope(conn, p["sessions"], p["repos"])
943
+ print(f"{'kind':<20} {'items':>11} {'refs':>5} {'~tokens':>8} {'~cost':>8}")
944
+ for kind in kinds:
945
+ if why := missing_data(conn, kind):
946
+ print(f"{kind:<20} skipped: {why}")
947
+ continue
948
+ key = (name, kind)
949
+ of_pass[key] = p
950
+ bundle = bundles[key] = ROLLUPS[kind](conn, args.min_confidence, args.max_evidence, args.since)
951
+ user = prompts[key] = user_prompt(kind, bundle, about)
952
+ digest = digests[key] = prompt_hash(model, user)
953
+ folder = out / f"{i:02d}-{short_name(name).replace(' ', '-')}"
954
+ folder.mkdir(exist_ok=True)
955
+ (folder / f"{kind}.json").write_text(json.dumps(
956
+ {"kind": kind, "pass": name, "model": model, "hash": digest, "bundle": bundle},
957
+ indent=2, ensure_ascii=False))
958
+ tokens = (len(SYSTEM) + len(user)) // 4
959
+ cost = "sub" if subscription else f"${tokens * price:.3f}" if price is not None else "?"
960
+ note = ""
961
+ if not bundle_refs(bundle):
962
+ note = " skipped: no evidence, nothing to send"
963
+ elif not args.force and (hit := cached_run(conn, index, digest, kind, name)):
964
+ cached[key] = hit
965
+ note = f" cached from {hit['run_id']}"
966
+ else:
967
+ todo_tokens += tokens
968
+ items = f"{bundle['totals']['shown']}/{bundle['totals']['items']}"
969
+ print(f"{kind:<20} {items:>11} {len(bundle_refs(bundle)):>5} {tokens:>8} {cost:>8}{note}")
970
+ scope(conn)
971
+ todo = {k: p for k, p in prompts.items() if bundle_refs(bundles[k]) and k not in cached}
972
+ print(f"\n{len(todo)} model calls, {len(cached)} cached")
973
+ if subscription:
974
+ print(f"~{todo_tokens} input tokens to send to {model} via claude -p (subscription)")
975
+ elif price is None:
976
+ print(f"~{todo_tokens} input tokens to send ({model} price unavailable)")
977
+ else:
978
+ print(f"~{todo_tokens} input tokens to send, ~${todo_tokens * price:.2f} at {model} input price "
979
+ f"(output extra, at most {MAX_TOKENS} tokens per call)")
980
+ print(f"bundles in {out}")
981
+
982
+ if args.dry_run:
983
+ if not bundles:
984
+ return
985
+ key = max(bundles, key=lambda k: len(bundle_refs(bundles[k])))
986
+ print(f"\n--- {short_name(key[0])}: {key[1]} ---\n{json.dumps(bundles[key], indent=2, ensure_ascii=False)[:6000]}")
987
+ return
988
+
989
+ if subscription:
990
+ if not shutil.which("claude"):
991
+ sys.exit("claude (Claude Code) is not on PATH")
992
+ ask = partial(ask_claude_code, model=model)
993
+ else:
994
+ key = os.environ.get("OPENROUTER_API_KEY")
995
+ if not key:
996
+ sys.exit("OPENROUTER_API_KEY is not set")
997
+ ask = partial(ask_openrouter, model=model, key=key)
998
+ now = datetime.now(UTC).isoformat()
999
+ total, cost = 0, 0.0
1000
+ failed = set() # names of passes with a failed call
1001
+
1002
+ def entry(key: tuple, ids: list[int]) -> dict:
1003
+ return {"run_id": run_id, "kind": key[1], "pass": key[0], "findings": len(ids), "ids": ids}
1004
+
1005
+ for key, source in cached.items():
1006
+ marks = ",".join("?" * len(source["ids"]))
1007
+ last = conn.execute("SELECT COALESCE(MAX(finding_id), 0) FROM findings").fetchone()[0]
1008
+ conn.execute(
1009
+ "INSERT INTO findings (run_id, kind, title, detail, suggested_fix, evidence, project, created_at) "
1010
+ "SELECT ?, kind, title, detail, suggested_fix, evidence, project, ? FROM findings "
1011
+ f"WHERE finding_id IN ({marks}) ORDER BY finding_id", (run_id, now, *source["ids"]))
1012
+ conn.commit()
1013
+ ids = [r[0] for r in conn.execute("SELECT finding_id FROM findings WHERE finding_id > ?", (last,))]
1014
+ index[digests[key]] = entry(key, ids)
1015
+ total += len(ids)
1016
+ print(f"{short_name(key[0])}: {key[1]}: {len(ids)} findings copied from {source['run_id']}")
1017
+
1018
+ workers = args.parallel or (2 if subscription else len(todo)) or 1
1019
+ with ThreadPoolExecutor(max_workers=workers) as pool:
1020
+ futures = {k: pool.submit(ask, p) for k, p in todo.items()}
1021
+ for key, fut in futures.items():
1022
+ kind, label = key[1], f"{short_name(key[0])}: {key[1]}"
1023
+ try:
1024
+ raw, usage = fut.result()
1025
+ findings = check_evidence(kind, raw, bundle_refs(bundles[key]))
1026
+ except (httpx.HTTPError, ClaudeCodeError, json.JSONDecodeError, KeyError) as e:
1027
+ print(f"{label}: failed: {e}")
1028
+ failed.add(key[0])
1029
+ continue
1030
+ for f in findings:
1031
+ f["project"] = of_pass[key]["project"]
1032
+ index[digests[key]] = entry(key, save_findings(conn, run_id, kind, findings, now))
1033
+ total += len(findings)
1034
+ cost += usage["cost"]
1035
+ took = f", {usage['seconds']}s" if "seconds" in usage else ""
1036
+ print(f"{label}: {len(findings)} findings, {usage['in']} in / {usage['out']} out tokens, "
1037
+ f"${usage['cost']:.4f}{took}")
1038
+ for f in findings:
1039
+ where = f" {short_name(f['project'])}" if f["project"] else ""
1040
+ print(f"[{kind}{where}] {f.get('title')} ({len(f['evidence'])} refs)\n fix: {f.get('suggested_fix')}")
1041
+ INDEX_PATH.write_text(json.dumps(index, indent=2))
1042
+ if not args.kind: # a whole pass with no failed call: the project is analysed as of this run
1043
+ conn.executemany("INSERT OR REPLACE INTO project_runs VALUES (?, ?, ?)",
1044
+ [(p["project"], run_id, now) for p in todo_passes if p["project"] and p["name"] not in failed])
1045
+ conn.commit()
1046
+ # Claude Code reports what the call would cost at API prices; the subscription pays instead.
1047
+ billed = f"~${cost:.4f} API-equivalent, billed to subscription" if subscription else f"${cost:.4f}"
1048
+ print(f"\n{total} findings saved under run_id {run_id} ({model} via {args.llm}), cost {billed}")
1049
+
1050
+
1051
+ if __name__ == "__main__":
1052
+ main()