debrief-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- debrief/__init__.py +0 -0
- debrief/build_artifact.py +83 -0
- debrief/classify.py +812 -0
- debrief/cli.py +291 -0
- debrief/compare_definitions.py +147 -0
- debrief/compare_llm.py +250 -0
- debrief/config.py +204 -0
- debrief/db.py +341 -0
- debrief/export.py +156 -0
- debrief/export_site.py +661 -0
- debrief/github.py +540 -0
- debrief/normalize.py +635 -0
- debrief/paths.py +17 -0
- debrief/prune.py +117 -0
- debrief/recommend.py +271 -0
- debrief/relay.py +83 -0
- debrief/runner.py +163 -0
- debrief/scrape.py +208 -0
- debrief/site/app.js +1613 -0
- debrief/site/index.html +32 -0
- debrief/site/method.js +117 -0
- debrief/site/style.css +520 -0
- debrief/skills.py +107 -0
- debrief/sources.py +341 -0
- debrief/synthesize.py +1052 -0
- debrief/verify_evidence.py +154 -0
- debrief_cli-0.1.0.dist-info/METADATA +205 -0
- debrief_cli-0.1.0.dist-info/RECORD +31 -0
- debrief_cli-0.1.0.dist-info/WHEEL +4 -0
- debrief_cli-0.1.0.dist-info/entry_points.txt +2 -0
- debrief_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
debrief/synthesize.py
ADDED
|
@@ -0,0 +1,1052 @@
|
|
|
1
|
+
"""Turn labels into findings: SQL rollups per project and finding kind, then one Claude call per bundle.
|
|
2
|
+
|
|
3
|
+
Only picked projects are read (`[projects] include` in config.toml; see db.session_projects
|
|
4
|
+
for what a project is). Each gets its own pass, so a finding says which repo it is about.
|
|
5
|
+
CROSS_KINDS are about the person, not a repo, and get one pass over all picked projects.
|
|
6
|
+
|
|
7
|
+
Rollups become evidence bundles (bundles/<run_id>/<pass>/<kind>.json); Claude reads each
|
|
8
|
+
bundle and returns findings, which land in the findings table under a new run_id.
|
|
9
|
+
Each bundle holds at most --max-evidence items; its counts cover all evidence.
|
|
10
|
+
--since sets where evidence starts: messages by ts, CI failures by completed_at, review
|
|
11
|
+
comments by created_at, sessions by last activity. The default is all of it.
|
|
12
|
+
|
|
13
|
+
A kind with no evidence is skipped with a note and never sent to the model: ci and review
|
|
14
|
+
need GitHub data, so a database of local sessions skips them.
|
|
15
|
+
|
|
16
|
+
Claude runs through headless Claude Code (`claude -p`, billed to the Claude subscription) by
|
|
17
|
+
default, or through OpenRouter with --llm openrouter.
|
|
18
|
+
|
|
19
|
+
A bundle's hash (model + prompt + bundle) is saved in its file and in bundles/index.json.
|
|
20
|
+
If the latest run for a kind holds findings from the same hash, they are copied into the new
|
|
21
|
+
run instead of calling the model (--force calls it anyway).
|
|
22
|
+
|
|
23
|
+
uv run python -m debrief.synthesize --dry-run
|
|
24
|
+
uv run python -m debrief.synthesize --kind hotspot --min-confidence 0.8
|
|
25
|
+
uv run python -m debrief.synthesize --llm openrouter
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
import argparse
|
|
29
|
+
import hashlib
|
|
30
|
+
import json
|
|
31
|
+
import math
|
|
32
|
+
import os
|
|
33
|
+
import re
|
|
34
|
+
import shutil
|
|
35
|
+
import sqlite3
|
|
36
|
+
import subprocess
|
|
37
|
+
import sys
|
|
38
|
+
import tempfile
|
|
39
|
+
import time
|
|
40
|
+
from collections import Counter
|
|
41
|
+
from collections.abc import Callable
|
|
42
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
43
|
+
from datetime import UTC, datetime
|
|
44
|
+
from functools import partial
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
|
|
47
|
+
import httpx
|
|
48
|
+
|
|
49
|
+
from debrief.config import choose, load_env, projects
|
|
50
|
+
from debrief.db import connect, repo_projects, session_projects
|
|
51
|
+
from debrief.paths import BUNDLE_DIR
|
|
52
|
+
from debrief.relay import is_relayed
|
|
53
|
+
|
|
54
|
+
INDEX_PATH = BUNDLE_DIR / "index.json"
|
|
55
|
+
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
|
56
|
+
MODELS_URL = "https://openrouter.ai/api/v1/models"
|
|
57
|
+
CLAUDE_CODE_TIMEOUT = 900 # seconds per call
|
|
58
|
+
CLAUDE_CODE_RETRIES = (30, 120) # backoff before each retry, for rate and usage limits
|
|
59
|
+
MAX_TOKENS = 8000
|
|
60
|
+
KINDS = ["hotspot", "knowledge_gap", "found_not_filed", "correction_pattern", "prompt_practice", "ci", "review"]
|
|
61
|
+
CROSS_KINDS = ["prompt_practice"] # about the person's habits, not a repo: one pass over all picked projects
|
|
62
|
+
CROSS = "all projects" # that pass's name, beside the project folders
|
|
63
|
+
QUOTE_LEN = 400
|
|
64
|
+
PER_SESSION = 5 # items from one session (or PR, or commit) before others get a turn
|
|
65
|
+
HOTSPOT_FILES = 25
|
|
66
|
+
|
|
67
|
+
# turn_labels field -> column name in the turns CTE; each also gets <name>_p, its confidence.
|
|
68
|
+
TURN_FIELDS = {
|
|
69
|
+
"turn_kind": "turn_kind", "correction_type": "correction_type", "resource_pointed_to": "resource_pointed_to",
|
|
70
|
+
"agent_could_have_found_it": "could_have_found", "human_caught_defect": "caught_defect",
|
|
71
|
+
"design_reversal": "reversal", "found_not_filed": "found_not_filed",
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
# A relayed turn (pasted bot output; see relay.py) is not the person correcting the agent:
|
|
75
|
+
# its correction becomes turn_kind 'relayed', its reversal and caught defect drop, and it
|
|
76
|
+
# counts as something the agent could have found.
|
|
77
|
+
RELAYED = {
|
|
78
|
+
"turn_kind": "CASE WHEN r.msg_idx IS NOT NULL AND l.turn_kind = 'correction' THEN 'relayed' ELSE l.turn_kind END",
|
|
79
|
+
"reversal": "CASE WHEN r.msg_idx IS NULL THEN l.reversal END",
|
|
80
|
+
"caught_defect": "CASE WHEN r.msg_idx IS NULL THEN l.caught_defect END",
|
|
81
|
+
"could_have_found": "CASE WHEN r.msg_idx IS NULL THEN l.could_have_found ELSE 'true' END",
|
|
82
|
+
"could_have_found_p": "CASE WHEN r.msg_idx IS NULL THEN l.could_have_found_p "
|
|
83
|
+
"ELSE COALESCE(l.could_have_found_p, :c) END",
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
# One row per labeled human turn, with only the labels that clear :c. Needs mark_relayed().
|
|
87
|
+
TURNS = "labeled AS (\n SELECT session_id, msg_idx,\n" + ",\n".join(
|
|
88
|
+
f" MAX(CASE WHEN field = '{f}' AND confidence >= :c THEN value END) AS {a},\n"
|
|
89
|
+
f" MAX(CASE WHEN field = '{f}' AND confidence >= :c THEN confidence END) AS {a}_p"
|
|
90
|
+
for f, a in TURN_FIELDS.items()) + "\n FROM turn_labels GROUP BY session_id, msg_idx\n),\n" \
|
|
91
|
+
"turns AS (\n SELECT l.session_id, l.msg_idx, r.msg_idx IS NOT NULL AS relayed,\n" + ",\n".join(
|
|
92
|
+
f" {RELAYED.get(c, 'l.' + c)} AS {c}" for a in TURN_FIELDS.values() for c in (a, f"{a}_p")) + \
|
|
93
|
+
"\n FROM labeled l LEFT JOIN relayed r USING (session_id, msg_idx)\n)"
|
|
94
|
+
|
|
95
|
+
# For each labeled message, the idx of the human message before it (-1 if none).
|
|
96
|
+
# The agent activity a label refers to is the messages in (prev_idx, msg_idx).
|
|
97
|
+
WINDOWS = """
|
|
98
|
+
windows AS (
|
|
99
|
+
SELECT t.session_id, t.msg_idx,
|
|
100
|
+
COALESCE((SELECT MAX(p.idx) FROM messages p
|
|
101
|
+
WHERE p.session_id = t.session_id AND p.idx < t.msg_idx
|
|
102
|
+
AND p.role = 'user' AND p.user_kind = 'human'), -1) AS prev_idx
|
|
103
|
+
FROM (SELECT DISTINCT session_id, msg_idx FROM turn_labels) t
|
|
104
|
+
)"""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def mark_relayed(conn: sqlite3.Connection) -> None:
|
|
108
|
+
"""Fill the temp table TURNS reads with human turns that paste bot output."""
|
|
109
|
+
conn.execute("CREATE TEMP TABLE IF NOT EXISTS relayed (session_id TEXT, msg_idx INTEGER, "
|
|
110
|
+
"PRIMARY KEY (session_id, msg_idx))")
|
|
111
|
+
conn.execute("DELETE FROM relayed")
|
|
112
|
+
human = conn.execute("SELECT session_id, idx, text FROM messages WHERE role = 'user' AND user_kind = 'human'")
|
|
113
|
+
conn.executemany("INSERT INTO relayed VALUES (?, ?)", [(s, i) for s, i, t in human.fetchall() if is_relayed(t)])
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Tables the rollups read, and the column that says which pass a row belongs to.
|
|
117
|
+
SCOPED = {"sessions": "session_id", "messages": "session_id", "turn_labels": "session_id",
|
|
118
|
+
"tool_calls": "session_id", "gh_checks": "repo", "gh_review_comments": "repo"}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def scope(conn: sqlite3.Connection, sessions: set[str] | None = None, repos: set[str] = frozenset()) -> None:
|
|
122
|
+
"""Limit what the rollups read to these sessions and repos; no sessions lifts the limit.
|
|
123
|
+
|
|
124
|
+
TEMP views named after the tables shadow them, so the rollup SQL stays as it is.
|
|
125
|
+
"""
|
|
126
|
+
for table in SCOPED:
|
|
127
|
+
conn.execute(f"DROP VIEW IF EXISTS temp.{table}")
|
|
128
|
+
if sessions is None:
|
|
129
|
+
return
|
|
130
|
+
# Session ids and repo names (owner/name) share the table; neither can be mistaken for the other.
|
|
131
|
+
conn.execute("CREATE TEMP TABLE IF NOT EXISTS scope (id TEXT PRIMARY KEY)")
|
|
132
|
+
conn.execute("DELETE FROM scope")
|
|
133
|
+
conn.executemany("INSERT INTO scope VALUES (?)", [(x,) for x in {*sessions, *repos}])
|
|
134
|
+
for table, column in SCOPED.items():
|
|
135
|
+
conn.execute(f"CREATE TEMP VIEW {table} AS SELECT * FROM main.{table} WHERE {column} IN (SELECT id FROM scope)")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def passes(conn: sqlite3.Connection, picked: list[str]) -> list[dict]:
|
|
139
|
+
"""One pass per picked project that has sessions (most human turns first), then one over
|
|
140
|
+
all of them for CROSS_KINDS. Nothing picked: no passes."""
|
|
141
|
+
project, repos = session_projects(conn), repo_projects(conn)
|
|
142
|
+
turns: Counter = Counter()
|
|
143
|
+
for sid, n in conn.execute("SELECT session_id, COUNT(*) FROM main.messages "
|
|
144
|
+
"WHERE role = 'user' AND user_kind = 'human' GROUP BY session_id"):
|
|
145
|
+
turns[project.get(sid)] += n
|
|
146
|
+
out = [{"name": p, "project": p, "sessions": {s for s, q in project.items() if q == p},
|
|
147
|
+
"repos": {r for r, q in repos.items() if q == p}, "turns": turns[p],
|
|
148
|
+
"kinds": [k for k in KINDS if k not in CROSS_KINDS]} for p in sorted(set(picked), key=lambda p: -turns[p])]
|
|
149
|
+
out = [p for p in out if p["sessions"]]
|
|
150
|
+
if not out:
|
|
151
|
+
return []
|
|
152
|
+
return [*out, {"name": CROSS, "project": None, "kinds": CROSS_KINDS, "turns": sum(p["turns"] for p in out),
|
|
153
|
+
"sessions": set().union(*(p["sessions"] for p in out)),
|
|
154
|
+
"repos": set().union(*(p["repos"] for p in out))}]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
EVENT_REPO = {"check_failure": "SELECT repo FROM main.gh_checks WHERE check_id = ?",
|
|
158
|
+
"review_comment": "SELECT repo FROM main.gh_review_comments WHERE comment_id = ?"}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def ref_project(conn: sqlite3.Connection, project: dict, repos: dict, ref: dict) -> str | None:
|
|
162
|
+
"""The project an evidence ref belongs to: its session's, or its repo's for a CI or review event."""
|
|
163
|
+
if "session_id" in ref:
|
|
164
|
+
return project.get(ref["session_id"])
|
|
165
|
+
row = conn.execute(EVENT_REPO[ref["event_type"]], (ref["event_id"],)).fetchone()
|
|
166
|
+
return repos.get(row[0]) if row else None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def short_name(path: str) -> str:
|
|
170
|
+
"""The last part of a project folder, for prompts and tables."""
|
|
171
|
+
return Path(path).name or path
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def excerpt(text: str | None, n: int = QUOTE_LEN, tail: bool = False) -> str:
|
|
175
|
+
"""Collapse whitespace and cut to n chars, keeping the start (or the end)."""
|
|
176
|
+
text = re.sub(r"\s+", " ", text or "").strip()
|
|
177
|
+
if len(text) <= n:
|
|
178
|
+
return text
|
|
179
|
+
return "…" + text[-n:] if tail else text[:n] + "…"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def rows(conn: sqlite3.Connection, sql: str, params: dict) -> list[dict]:
|
|
183
|
+
return [dict(r) for r in conn.execute(sql, params)]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def agent_text(conn: sqlite3.Connection, session_id: str, prev_idx: int, msg_idx: int, n: int = QUOTE_LEN) -> str:
|
|
187
|
+
"""Tail of the assistant text in (prev_idx, msg_idx]."""
|
|
188
|
+
texts = [r[0] for r in conn.execute(
|
|
189
|
+
"SELECT text FROM messages WHERE session_id = ? AND idx > ? AND idx <= ? AND role = 'assistant' "
|
|
190
|
+
"AND text != '' ORDER BY idx", (session_id, prev_idx, msg_idx))]
|
|
191
|
+
return excerpt("\n".join(t for t in texts if t), n, tail=True)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def turn_totals(conn: sqlite3.Connection, matched: list[dict], shown: int, since: str) -> dict:
|
|
195
|
+
"""Counts over all matching turns, against every human and labeled turn since `since`."""
|
|
196
|
+
human, human_sessions = conn.execute(
|
|
197
|
+
"SELECT COUNT(*), COUNT(DISTINCT session_id) FROM messages WHERE role = 'user' AND user_kind = 'human' "
|
|
198
|
+
"AND COALESCE(ts, '') >= :since", {"since": since}).fetchone()
|
|
199
|
+
labeled, labeled_sessions = conn.execute(
|
|
200
|
+
"SELECT COUNT(*), COUNT(DISTINCT session_id) FROM messages m WHERE role = 'user' AND user_kind = 'human' "
|
|
201
|
+
"AND COALESCE(ts, '') >= :since "
|
|
202
|
+
"AND EXISTS (SELECT 1 FROM turn_labels l WHERE l.session_id = m.session_id AND l.msg_idx = m.idx)",
|
|
203
|
+
{"since": since}).fetchone()
|
|
204
|
+
return {"items": len(matched), "shown": shown, "sessions": len({t["session_id"] for t in matched}),
|
|
205
|
+
"human_turns": human, "human_sessions": human_sessions,
|
|
206
|
+
"labeled_turns": labeled, "labeled_sessions": labeled_sessions}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def pick(groups: list[list[dict]], n: int, key: Callable[[dict], object], per_key: int = PER_SESSION) -> list[dict]:
|
|
210
|
+
"""Up to n items, one per group in turn, taking each group's items in order.
|
|
211
|
+
|
|
212
|
+
No key (session, PR, commit) gets more than per_key items until nothing else is left.
|
|
213
|
+
An item listed in several groups is taken once.
|
|
214
|
+
"""
|
|
215
|
+
queues = [list(g) for g in groups if g]
|
|
216
|
+
chosen: list[dict] = []
|
|
217
|
+
taken: set = set()
|
|
218
|
+
per: Counter = Counter()
|
|
219
|
+
for limit in (per_key, math.inf):
|
|
220
|
+
progress = True
|
|
221
|
+
while progress and len(chosen) < n:
|
|
222
|
+
progress = False
|
|
223
|
+
for q in queues:
|
|
224
|
+
if len(chosen) >= n:
|
|
225
|
+
break
|
|
226
|
+
q[:] = [x for x in q if ref_key(x) not in taken]
|
|
227
|
+
i = next((i for i, x in enumerate(q) if per[key(x)] < limit), None)
|
|
228
|
+
if i is None:
|
|
229
|
+
continue
|
|
230
|
+
item = q.pop(i)
|
|
231
|
+
chosen.append(item)
|
|
232
|
+
taken.add(ref_key(item))
|
|
233
|
+
per[key(item)] += 1
|
|
234
|
+
progress = True
|
|
235
|
+
return chosen
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def by_session(x: dict) -> str:
|
|
239
|
+
return x["session_id"]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def ranked(items: list[dict]) -> list[dict]:
|
|
243
|
+
return sorted(items, key=lambda x: -(x["_p"] or 0))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def clean(item: dict) -> dict:
|
|
247
|
+
return {k: v for k, v in item.items() if not k.startswith("_")}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
# A PR is settled once merged or closed (db.settled_prs). CI and review rollups keep only
|
|
251
|
+
# work whose PR is settled. The outcome rollup drops a session only while its PR is open:
|
|
252
|
+
# a session with no linked PR (local logs, or work never pushed) stays in.
|
|
253
|
+
SESSION_PR = "EXISTS (SELECT 1 FROM session_pr_links l WHERE l.session_id = s.session_id)"
|
|
254
|
+
SETTLED_SESSION = """EXISTS (
|
|
255
|
+
SELECT 1 FROM session_pr_links l
|
|
256
|
+
JOIN settled_prs sp ON sp.repo = l.repo AND sp.pr_number = l.pr_number
|
|
257
|
+
WHERE l.session_id = s.session_id)"""
|
|
258
|
+
SETTLED_CHECK = """EXISTS (
|
|
259
|
+
SELECT 1 FROM gh_commit_prs g JOIN settled_prs sp ON sp.repo = g.repo AND sp.pr_number = g.pr_number
|
|
260
|
+
WHERE g.commit_sha = k.commit_sha)"""
|
|
261
|
+
|
|
262
|
+
# Checks that say nothing about the code. The Entire GitHub App's check is an approval gate
|
|
263
|
+
# (only that app has the slug 'entire'). A gate job fails because another job did: its log
|
|
264
|
+
# compares job results, as in `[ "failure" = "success" ]`. Any check name with such a
|
|
265
|
+
# failure is left out, passes included, so the totals don't count it either.
|
|
266
|
+
# shortcut: knows only this shell form of gate job; add forms as they turn up.
|
|
267
|
+
REAL_CHECK = """(COALESCE(k.app, '') != 'entire' AND COALESCE(k.name, '') NOT IN (
|
|
268
|
+
SELECT COALESCE(name, '') FROM gh_checks WHERE conclusion = 'failure' AND (
|
|
269
|
+
failure_log LIKE '%"failure" = "success"%' OR failure_log LIKE '%"cancelled" = "success"%'
|
|
270
|
+
OR failure_log LIKE '%"skipped" = "success"%')))"""
|
|
271
|
+
|
|
272
|
+
# Real CI failures. flaky = 1 when the same check later passed on the same commit.
|
|
273
|
+
# Needs :since, the window start.
|
|
274
|
+
CI_FAILS = f"""
|
|
275
|
+
ci_fails AS (
|
|
276
|
+
SELECT k.*, EXISTS (
|
|
277
|
+
SELECT 1 FROM gh_checks s
|
|
278
|
+
WHERE s.commit_sha = k.commit_sha AND s.name = k.name AND s.conclusion = 'success'
|
|
279
|
+
AND s.completed_at > k.completed_at) AS flaky,
|
|
280
|
+
{SETTLED_CHECK} AS settled
|
|
281
|
+
FROM gh_checks k
|
|
282
|
+
WHERE k.conclusion = 'failure' AND {REAL_CHECK}
|
|
283
|
+
AND COALESCE(k.completed_at, '') >= :since
|
|
284
|
+
)"""
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def event_label_sql(event_type: str, field: str, column: str = "value") -> str:
|
|
288
|
+
return (f"(SELECT {column} FROM event_labels WHERE event_type = '{event_type}' AND event_id = {{id}} "
|
|
289
|
+
f"AND field = '{field}' AND confidence >= :c)")
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
# --- rollups ---------------------------------------------------------------
|
|
293
|
+
|
|
294
|
+
def hotspot(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
295
|
+
"""Files the agent wrote just before corrections, reversals and caught defects; CI failures by check."""
|
|
296
|
+
flagged = rows(conn, f"""
|
|
297
|
+
WITH {TURNS}, {WINDOWS}
|
|
298
|
+
SELECT tu.*, w.prev_idx, m.text,
|
|
299
|
+
(SELECT json_group_array(DISTINCT f.value)
|
|
300
|
+
FROM tool_calls tc JOIN json_each(tc.files) f
|
|
301
|
+
WHERE tc.session_id = w.session_id AND tc.msg_idx > w.prev_idx AND tc.msg_idx < w.msg_idx
|
|
302
|
+
AND tc.writes = 1 AND f.value NOT LIKE '/%') AS files
|
|
303
|
+
FROM turns tu
|
|
304
|
+
JOIN windows w USING (session_id, msg_idx)
|
|
305
|
+
JOIN messages m ON m.session_id = tu.session_id AND m.idx = tu.msg_idx
|
|
306
|
+
WHERE (tu.turn_kind = 'correction' OR tu.reversal = 'true' OR tu.caught_defect = 'true')
|
|
307
|
+
AND COALESCE(m.ts, '') >= :since
|
|
308
|
+
ORDER BY tu.session_id, tu.msg_idx""", {"c": c, "since": since})
|
|
309
|
+
errors = dict(conn.execute("""
|
|
310
|
+
SELECT f.value, COUNT(*) FROM tool_calls tc JOIN json_each(tc.files) f
|
|
311
|
+
JOIN messages m ON m.session_id = tc.session_id AND m.idx = tc.msg_idx
|
|
312
|
+
WHERE tc.is_error = 1 AND COALESCE(m.ts, '') >= :since GROUP BY f.value""", {"since": since}))
|
|
313
|
+
files: dict[str, dict] = {}
|
|
314
|
+
turns = []
|
|
315
|
+
for t in flagged:
|
|
316
|
+
paths = json.loads(t["files"])
|
|
317
|
+
if not paths:
|
|
318
|
+
continue
|
|
319
|
+
# Rank a turn by its strongest flag.
|
|
320
|
+
p = max((t[f"{a}_p"] or 0) for a, hit in (("turn_kind", "correction"), ("reversal", "true"),
|
|
321
|
+
("caught_defect", "true")) if t[a] == hit)
|
|
322
|
+
turn = {"session_id": t["session_id"], "msg_idx": t["msg_idx"],
|
|
323
|
+
"labels": {k: t[k] for k in ("turn_kind", "correction_type", "reversal", "caught_defect") if t[k]},
|
|
324
|
+
"files_written_before": paths, "human": excerpt(t["text"]), "_p": p, "_prev": t["prev_idx"]}
|
|
325
|
+
turns.append(turn)
|
|
326
|
+
for path in paths:
|
|
327
|
+
f = files.setdefault(path, {"path": path, "sessions": set(), "turns": 0, "corrections": 0, "reversals": 0,
|
|
328
|
+
"caught_defects": 0, "tool_errors": errors.get(path, 0), "_turns": []})
|
|
329
|
+
f["sessions"].add(t["session_id"])
|
|
330
|
+
f["turns"] += 1
|
|
331
|
+
f["corrections"] += t["turn_kind"] == "correction"
|
|
332
|
+
f["reversals"] += t["reversal"] == "true"
|
|
333
|
+
f["caught_defects"] += t["caught_defect"] == "true"
|
|
334
|
+
f["_turns"].append(turn)
|
|
335
|
+
for f in files.values():
|
|
336
|
+
f["sessions"] = len(f["sessions"])
|
|
337
|
+
top = sorted(files.values(), key=lambda f: (-f["sessions"], -f["turns"], -f["tool_errors"]))[:HOTSPOT_FILES]
|
|
338
|
+
|
|
339
|
+
checks = rows(conn, f"""
|
|
340
|
+
WITH {CI_FAILS}
|
|
341
|
+
SELECT name, {event_label_sql('check_failure', 'failure_class').format(id='k.check_id')} AS failure_class,
|
|
342
|
+
COUNT(*) AS failures, SUM(flaky) AS flaky, GROUP_CONCAT(check_id) AS check_ids
|
|
343
|
+
FROM ci_fails k WHERE settled
|
|
344
|
+
GROUP BY name, failure_class ORDER BY failures DESC LIMIT 25""", {"c": c, "since": since})
|
|
345
|
+
# One example per check for the top checks; the ci kind has the detail.
|
|
346
|
+
for i, ch in enumerate(checks):
|
|
347
|
+
first = int(ch.pop("check_ids").split(",")[0])
|
|
348
|
+
if i < n // 5:
|
|
349
|
+
ch["example"] = {"event_type": "check_failure", "event_id": first}
|
|
350
|
+
|
|
351
|
+
# Each top file's turns in turn; the last group holds turns on other files.
|
|
352
|
+
groups = [ranked(f["_turns"]) for f in top] + [ranked(turns)]
|
|
353
|
+
shown = pick(groups, n - sum("example" in ch for ch in checks), by_session)
|
|
354
|
+
shown_keys = {ref_key(t) for t in shown}
|
|
355
|
+
for f in top:
|
|
356
|
+
f["shown_turns"] = [{"session_id": t["session_id"], "msg_idx": t["msg_idx"]}
|
|
357
|
+
for t in f.pop("_turns") if ref_key(t) in shown_keys]
|
|
358
|
+
for t in shown:
|
|
359
|
+
t["agent_before"] = agent_text(conn, t["session_id"], t["_prev"], t["msg_idx"] - 1, 250)
|
|
360
|
+
totals = {**turn_totals(conn, turns, len(shown), since),
|
|
361
|
+
"flagged_turns_without_files": len(flagged) - len(turns), "files": len(files)}
|
|
362
|
+
bundle = {"totals": totals, "files": top, "flagged_turns": [clean(t) for t in shown]}
|
|
363
|
+
if checks: # absent without GitHub data
|
|
364
|
+
bundle["ci_failures_by_check"] = checks
|
|
365
|
+
return bundle
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def knowledge_gap(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
369
|
+
"""Turns where the human pointed the agent at a resource or supplied something it could have found.
|
|
370
|
+
|
|
371
|
+
Counts split the session's opening message (context given up front) from later turns
|
|
372
|
+
(the agent needed steering), per resource type.
|
|
373
|
+
"""
|
|
374
|
+
turns = rows(conn, f"""
|
|
375
|
+
WITH {TURNS}, {WINDOWS}
|
|
376
|
+
SELECT tu.*, w.prev_idx, w.prev_idx < 0 AS opening, m.text
|
|
377
|
+
FROM turns tu
|
|
378
|
+
JOIN windows w USING (session_id, msg_idx)
|
|
379
|
+
JOIN messages m ON m.session_id = tu.session_id AND m.idx = tu.msg_idx
|
|
380
|
+
WHERE (tu.could_have_found = 'true' OR tu.resource_pointed_to NOT IN ('none', ''))
|
|
381
|
+
AND COALESCE(m.ts, '') >= :since
|
|
382
|
+
ORDER BY tu.session_id, tu.msg_idx""", {"c": c, "since": since})
|
|
383
|
+
groups: dict[str, dict] = {}
|
|
384
|
+
for t in turns:
|
|
385
|
+
t["_p"] = max(t["could_have_found_p"] if t["could_have_found"] == "true" else 0,
|
|
386
|
+
t["resource_pointed_to_p"] if t["resource_pointed_to"] not in (None, "none", "") else 0)
|
|
387
|
+
g = groups.setdefault(t["resource_pointed_to"] or "unlabeled", {
|
|
388
|
+
part: {"turns": 0, "sessions": set(), "agent_could_have_found_it": 0, "_turns": []}
|
|
389
|
+
for part in ("mid_session", "opening")})
|
|
390
|
+
part = g["opening" if t["opening"] else "mid_session"]
|
|
391
|
+
part["turns"] += 1
|
|
392
|
+
part["sessions"].add(t["session_id"])
|
|
393
|
+
part["agent_could_have_found_it"] += t["could_have_found"] == "true"
|
|
394
|
+
part["_turns"].append(t)
|
|
395
|
+
groups = dict(sorted(groups.items(), key=lambda kv: (-kv[1]["mid_session"]["turns"], -kv[1]["opening"]["turns"])))
|
|
396
|
+
# Opening and mid-session turns queue apart, so examples show both.
|
|
397
|
+
parts = [g[part] for part in ("mid_session", "opening") for g in groups.values()]
|
|
398
|
+
shown = {ref_key(t) for t in pick([ranked(p["_turns"]) for p in parts], n, by_session)}
|
|
399
|
+
for g in groups.values():
|
|
400
|
+
examples = []
|
|
401
|
+
for p in (g["mid_session"], g["opening"]):
|
|
402
|
+
p["sessions"] = len(p["sessions"])
|
|
403
|
+
examples += [{
|
|
404
|
+
"session_id": t["session_id"], "msg_idx": t["msg_idx"], "opening": bool(t["opening"]),
|
|
405
|
+
"agent_could_have_found_it": t["could_have_found"], "turn_kind": t["turn_kind"],
|
|
406
|
+
"human": excerpt(t["text"]),
|
|
407
|
+
"agent_before": agent_text(conn, t["session_id"], t["prev_idx"], t["msg_idx"] - 1, 250),
|
|
408
|
+
} for t in p.pop("_turns") if ref_key(t) in shown]
|
|
409
|
+
g["examples"] = examples
|
|
410
|
+
split = {}
|
|
411
|
+
for part, opening in (("mid_session", False), ("opening", True)):
|
|
412
|
+
these = [t for t in turns if bool(t["opening"]) == opening]
|
|
413
|
+
split[part] = {"turns": len(these), "sessions": len({t["session_id"] for t in these})}
|
|
414
|
+
return {"totals": {**turn_totals(conn, turns, len(shown), since), **split}, "by_resource_type": groups}
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def found_not_filed(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
418
|
+
"""Every place the agent noticed a problem outside its task and left it unfiled in the session."""
|
|
419
|
+
hits = rows(conn, f"""
|
|
420
|
+
WITH {WINDOWS}
|
|
421
|
+
SELECT l.session_id, l.msg_idx, l.confidence, w.prev_idx, m.role, m.text
|
|
422
|
+
FROM turn_labels l
|
|
423
|
+
JOIN windows w USING (session_id, msg_idx)
|
|
424
|
+
JOIN messages m ON m.session_id = l.session_id AND m.idx = l.msg_idx
|
|
425
|
+
WHERE l.field = 'found_not_filed' AND l.value = 'true' AND l.confidence >= :c AND COALESCE(m.ts, '') >= :since
|
|
426
|
+
ORDER BY l.session_id, l.msg_idx""", {"c": c, "since": since})
|
|
427
|
+
# Repeats of one deferred problem are reworded, so the model merges them, not us.
|
|
428
|
+
for h in hits:
|
|
429
|
+
h["_p"] = h["confidence"]
|
|
430
|
+
shown = sorted(pick([ranked(hits)], n, by_session), key=lambda h: (h["session_id"], h["msg_idx"]))
|
|
431
|
+
grouped: dict[str, list] = {}
|
|
432
|
+
for h in shown:
|
|
433
|
+
# On a human turn the flag covers the agent messages before it; on the
|
|
434
|
+
# per-session row (an assistant message) it covers that message and those before it.
|
|
435
|
+
end = h["msg_idx"] if h["role"] == "assistant" else h["msg_idx"] - 1
|
|
436
|
+
item = {"session_id": h["session_id"], "msg_idx": h["msg_idx"], "confidence": round(h["confidence"], 2),
|
|
437
|
+
"agent_text": agent_text(conn, h["session_id"], h["prev_idx"], end, 800)}
|
|
438
|
+
if h["role"] == "user":
|
|
439
|
+
item["human_reply"] = excerpt(h["text"], 200)
|
|
440
|
+
grouped.setdefault(h["session_id"], []).append(item)
|
|
441
|
+
totals = {"items": len(hits), "shown": len(shown), "sessions": len({h["session_id"] for h in hits}),
|
|
442
|
+
"shown_sessions": len(grouped)}
|
|
443
|
+
return {"totals": totals, "sessions": [{"session_id": k, "hits": v} for k, v in grouped.items()]}
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def correction_pattern(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
447
|
+
"""Correction types, how often each occurs, and example turns."""
|
|
448
|
+
turns = rows(conn, f"""
|
|
449
|
+
WITH {TURNS}, {WINDOWS}
|
|
450
|
+
SELECT tu.*, w.prev_idx, m.text
|
|
451
|
+
FROM turns tu
|
|
452
|
+
JOIN windows w USING (session_id, msg_idx)
|
|
453
|
+
JOIN messages m ON m.session_id = tu.session_id AND m.idx = tu.msg_idx
|
|
454
|
+
WHERE tu.turn_kind = 'correction' AND COALESCE(m.ts, '') >= :since
|
|
455
|
+
ORDER BY tu.session_id, tu.msg_idx""", {"c": c, "since": since})
|
|
456
|
+
groups: dict[str, dict] = {}
|
|
457
|
+
for t in turns:
|
|
458
|
+
t["_p"] = t["correction_type_p"] or t["turn_kind_p"]
|
|
459
|
+
g = groups.setdefault(t["correction_type"] or "unlabeled", {"count": 0, "sessions": set(), "_turns": []})
|
|
460
|
+
g["count"] += 1
|
|
461
|
+
g["sessions"].add(t["session_id"])
|
|
462
|
+
g["_turns"].append(t)
|
|
463
|
+
groups = dict(sorted(groups.items(), key=lambda kv: -kv[1]["count"]))
|
|
464
|
+
shown = {ref_key(t) for t in pick([ranked(g["_turns"]) for g in groups.values()], n, by_session)}
|
|
465
|
+
for g in groups.values():
|
|
466
|
+
g["sessions"] = len(g["sessions"])
|
|
467
|
+
g["examples"] = [{
|
|
468
|
+
"session_id": t["session_id"], "msg_idx": t["msg_idx"],
|
|
469
|
+
"reversal": t["reversal"], "caught_defect": t["caught_defect"],
|
|
470
|
+
"human": excerpt(t["text"]),
|
|
471
|
+
"agent_before": agent_text(conn, t["session_id"], t["prev_idx"], t["msg_idx"] - 1, 300),
|
|
472
|
+
} for t in g.pop("_turns") if ref_key(t) in shown]
|
|
473
|
+
return {"totals": turn_totals(conn, turns, len(shown), since), "by_correction_type": groups}
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def prompt_practice(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
477
|
+
"""Session-level prompt labels against outcome and per-session correction/reversal rates."""
|
|
478
|
+
sessions = rows(conn, f"""
|
|
479
|
+
WITH {TURNS},
|
|
480
|
+
per_session AS (
|
|
481
|
+
SELECT session_id,
|
|
482
|
+
SUM(turn_kind = 'correction') AS corrections,
|
|
483
|
+
SUM(reversal = 'true') AS reversals,
|
|
484
|
+
SUM(caught_defect = 'true') AS caught_defects
|
|
485
|
+
FROM turns GROUP BY session_id
|
|
486
|
+
),
|
|
487
|
+
first_human AS (
|
|
488
|
+
SELECT session_id, MIN(idx) AS msg_idx, COUNT(*) AS human_turns
|
|
489
|
+
FROM messages WHERE role = 'user' AND user_kind = 'human' GROUP BY session_id
|
|
490
|
+
)
|
|
491
|
+
SELECT s.session_id, fh.msg_idx, fh.human_turns, s.agent, s.prompt, AVG(sl.confidence) AS _p,
|
|
492
|
+
MAX(CASE WHEN sl.field = 'task_type' THEN sl.value END) AS task_type,
|
|
493
|
+
MAX(CASE WHEN sl.field = 'prompt_style' THEN sl.value END) AS prompt_style,
|
|
494
|
+
MAX(CASE WHEN sl.field = 'prompt_specificity' THEN sl.value END) AS prompt_specificity,
|
|
495
|
+
MAX(CASE WHEN sl.field = 'planned_first' THEN sl.value END) AS planned_first,
|
|
496
|
+
MAX(CASE WHEN sl.field = 'tests_added' THEN sl.value END) AS tests_added,
|
|
497
|
+
MAX(CASE WHEN sl.field = 'outcome' THEN sl.value END) AS outcome,
|
|
498
|
+
COALESCE(p.corrections, 0) AS corrections, COALESCE(p.reversals, 0) AS reversals,
|
|
499
|
+
COALESCE(p.caught_defects, 0) AS caught_defects,
|
|
500
|
+
{SESSION_PR} AS has_pr, {SETTLED_SESSION} AS settled
|
|
501
|
+
FROM sessions s
|
|
502
|
+
JOIN session_labels sl ON sl.session_id = s.session_id AND sl.confidence >= :c
|
|
503
|
+
JOIN first_human fh ON fh.session_id = s.session_id
|
|
504
|
+
LEFT JOIN per_session p ON p.session_id = s.session_id
|
|
505
|
+
WHERE COALESCE(s.last_activity_at, '') >= :since
|
|
506
|
+
GROUP BY s.session_id ORDER BY s.session_id""", {"c": c, "since": since})
|
|
507
|
+
unsettled = sum(s["has_pr"] and not s["settled"] for s in sessions)
|
|
508
|
+
sessions = [s for s in sessions if s["settled"] or not s["has_pr"]]
|
|
509
|
+
by_label: dict[str, dict] = {}
|
|
510
|
+
for field in ("prompt_style", "prompt_specificity", "task_type", "planned_first", "tests_added"):
|
|
511
|
+
for s in sessions:
|
|
512
|
+
if not s[field]:
|
|
513
|
+
continue
|
|
514
|
+
g = by_label.setdefault(field, {}).setdefault(
|
|
515
|
+
s[field], {"sessions": 0, "outcomes": {}, "human_turns": 0, "corrections": 0, "reversals": 0})
|
|
516
|
+
g["sessions"] += 1
|
|
517
|
+
g["outcomes"][s["outcome"] or "unlabeled"] = g["outcomes"].get(s["outcome"] or "unlabeled", 0) + 1
|
|
518
|
+
for k in ("human_turns", "corrections", "reversals"):
|
|
519
|
+
g[k] += s[k]
|
|
520
|
+
for groups in by_label.values():
|
|
521
|
+
for g in groups.values():
|
|
522
|
+
g["corrections_per_session"] = round(g["corrections"] / g["sessions"], 2)
|
|
523
|
+
g["reversals_per_session"] = round(g["reversals"] / g["sessions"], 2)
|
|
524
|
+
# Show sessions from every outcome in turn, most confidently labeled first.
|
|
525
|
+
outcomes: dict[str, list] = {}
|
|
526
|
+
for s in ranked(sessions):
|
|
527
|
+
outcomes.setdefault(s["outcome"] or "unlabeled", []).append(s)
|
|
528
|
+
shown = sorted(pick(list(outcomes.values()), n, by_session), key=lambda s: s["session_id"])
|
|
529
|
+
per_session = [
|
|
530
|
+
{"session_id": s["session_id"], "msg_idx": s["msg_idx"], "agent": s["agent"],
|
|
531
|
+
**{k: s[k] for k in ("task_type", "prompt_style", "prompt_specificity", "planned_first", "tests_added",
|
|
532
|
+
"outcome", "human_turns", "corrections", "reversals", "caught_defects")},
|
|
533
|
+
"pr": "merged_or_closed" if s["has_pr"] else "none", "opening_prompt": excerpt(s["prompt"], 300)}
|
|
534
|
+
for s in shown]
|
|
535
|
+
totals = {"items": len(sessions), "shown": len(shown), "sessions": len(sessions), "excluded_unsettled": unsettled,
|
|
536
|
+
"sessions_without_pr": sum(not s["has_pr"] for s in sessions)}
|
|
537
|
+
return {"totals": totals, "by_label": by_label, "per_session": per_session}
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def ci(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
541
|
+
"""Failed CI checks by failure_class, with log snippets, flake flags, and their PRs and sessions."""
|
|
542
|
+
fails = rows(conn, f"""
|
|
543
|
+
WITH {CI_FAILS}
|
|
544
|
+
SELECT k.check_id, k.name, k.commit_sha, k.failure_log, k.flaky, k.settled,
|
|
545
|
+
{event_label_sql('check_failure', 'failure_class').format(id='k.check_id')} AS failure_class,
|
|
546
|
+
{event_label_sql('check_failure', 'failure_class', 'confidence').format(id='k.check_id')} AS _p,
|
|
547
|
+
(SELECT GROUP_CONCAT(repo || '#' || pr_number) FROM gh_commit_prs cp WHERE cp.commit_sha = k.commit_sha) AS prs,
|
|
548
|
+
(SELECT GROUP_CONCAT(DISTINCT session_id) FROM session_commits sc WHERE sc.commit_sha = k.commit_sha) AS sessions
|
|
549
|
+
FROM ci_fails k
|
|
550
|
+
ORDER BY k.check_id""", {"c": c, "since": since})
|
|
551
|
+
unsettled = sum(not f["settled"] for f in fails)
|
|
552
|
+
fails = [f for f in fails if f["settled"]]
|
|
553
|
+
total = conn.execute(f"SELECT COUNT(*) FROM gh_checks k WHERE conclusion IS NOT NULL "
|
|
554
|
+
f"AND {REAL_CHECK} AND COALESCE(completed_at, '') >= :since "
|
|
555
|
+
f"AND {SETTLED_CHECK}", {"since": since}).fetchone()[0]
|
|
556
|
+
groups: dict[str, dict] = {}
|
|
557
|
+
for f in fails:
|
|
558
|
+
f["event_type"], f["event_id"] = "check_failure", f["check_id"]
|
|
559
|
+
g = groups.setdefault(f["failure_class"] or "unlabeled", {"count": 0, "flaky": 0, "checks": {}, "_fails": []})
|
|
560
|
+
g["count"] += 1
|
|
561
|
+
g["flaky"] += f["flaky"]
|
|
562
|
+
g["checks"][f["name"]] = g["checks"].get(f["name"], 0) + 1
|
|
563
|
+
g["_fails"].append(f)
|
|
564
|
+
groups = dict(sorted(groups.items(), key=lambda kv: -kv[1]["count"]))
|
|
565
|
+
shown = {ref_key(f) for f in pick([ranked(g["_fails"]) for g in groups.values()], n, lambda f: f["commit_sha"])}
|
|
566
|
+
for g in groups.values():
|
|
567
|
+
g["examples"] = [{
|
|
568
|
+
"event_type": "check_failure", "event_id": f["check_id"], "check": f["name"],
|
|
569
|
+
"flaky": bool(f["flaky"]), "prs": f["prs"], "sessions": f["sessions"],
|
|
570
|
+
"log": excerpt(f["failure_log"], 500, tail=True),
|
|
571
|
+
} for f in g.pop("_fails") if ref_key(f) in shown]
|
|
572
|
+
totals = {"items": len(fails), "shown": len(shown), "checks": total, "failures": len(fails),
|
|
573
|
+
"excluded_unsettled": unsettled, "flaky": sum(f["flaky"] for f in fails)}
|
|
574
|
+
return {"totals": totals, "by_failure_class": groups}
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def review(conn: sqlite3.Connection, c: float, n: int, since: str) -> dict:
|
|
578
|
+
"""Review comments on settled PRs by whether they held up, bot vs human, with thread state."""
|
|
579
|
+
comments = rows(conn, f"""
|
|
580
|
+
SELECT rc.comment_id, rc.repo || '#' || rc.pr_number AS pr, rc.author, rc.is_bot, rc.path, rc.body,
|
|
581
|
+
rc.kind, rc.review_state, rc.is_resolved, rc.is_outdated,
|
|
582
|
+
{event_label_sql('review_comment', 'held_up').format(id='rc.comment_id')} AS held_up,
|
|
583
|
+
{event_label_sql('review_comment', 'held_up', 'confidence').format(id='rc.comment_id')} AS _p,
|
|
584
|
+
{event_label_sql('review_comment', 'comment_type').format(id='rc.comment_id')} AS comment_type,
|
|
585
|
+
EXISTS (SELECT 1 FROM settled_prs sp WHERE sp.repo = rc.repo AND sp.pr_number = rc.pr_number) AS settled
|
|
586
|
+
FROM gh_review_comments rc
|
|
587
|
+
WHERE rc.in_reply_to IS NULL AND COALESCE(rc.created_at, '') >= :since
|
|
588
|
+
ORDER BY rc.comment_id""", {"c": c, "since": since})
|
|
589
|
+
unsettled = sum(not r["settled"] for r in comments)
|
|
590
|
+
comments = [r for r in comments if r["settled"]]
|
|
591
|
+
counts: dict[str, dict] = {}
|
|
592
|
+
groups: dict[str, list] = {}
|
|
593
|
+
for r in comments:
|
|
594
|
+
who = "bot" if r["is_bot"] else "human"
|
|
595
|
+
held, ctype, kind = r["held_up"] or "unlabeled", r["comment_type"] or "unlabeled", r["kind"] or "unknown"
|
|
596
|
+
if kind == "inline":
|
|
597
|
+
# GitHub thread state decides fixed vs open; the label only says valid, wrong or unclear.
|
|
598
|
+
validity = {"valid_and_fixed": "valid", "valid_open": "valid"}.get(held, held)
|
|
599
|
+
status = "resolved" if r["is_resolved"] else "outdated" if r["is_outdated"] else "open"
|
|
600
|
+
outcome = f"{validity}/{status}"
|
|
601
|
+
else:
|
|
602
|
+
outcome = held
|
|
603
|
+
w = counts.setdefault(who, {"comment_type": {}})
|
|
604
|
+
w.setdefault(kind, {})[outcome] = w.setdefault(kind, {}).get(outcome, 0) + 1
|
|
605
|
+
w["comment_type"][ctype] = w["comment_type"].get(ctype, 0) + 1
|
|
606
|
+
groups.setdefault(f"{who}/{kind}/{outcome}", []).append(
|
|
607
|
+
{"event_type": "review_comment", "event_id": r["comment_id"], "pr": r["pr"],
|
|
608
|
+
"author": r["author"], "review_state": r["review_state"],
|
|
609
|
+
"is_resolved": r["is_resolved"], "is_outdated": r["is_outdated"], "path": r["path"],
|
|
610
|
+
"comment_type": ctype, "body": excerpt(r["body"], 300), "_p": r["_p"]})
|
|
611
|
+
groups = dict(sorted(groups.items(), key=lambda kv: -len(kv[1])))
|
|
612
|
+
shown = {ref_key(x) for x in pick([ranked(g) for g in groups.values()], n, lambda x: x["pr"])}
|
|
613
|
+
examples = {k: [clean(x) for x in g if ref_key(x) in shown] for k, g in groups.items()}
|
|
614
|
+
totals = {"items": len(comments), "shown": len(shown), "comments": len(comments),
|
|
615
|
+
"prs": len({r["pr"] for r in comments}), "excluded_unsettled": unsettled}
|
|
616
|
+
return {"totals": totals, "counts": counts, "examples": {k: v for k, v in examples.items() if v}}
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
ROLLUPS = {
|
|
620
|
+
"hotspot": hotspot, "knowledge_gap": knowledge_gap, "found_not_filed": found_not_filed,
|
|
621
|
+
"correction_pattern": correction_pattern, "prompt_practice": prompt_practice, "ci": ci, "review": review,
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
# --- evidence refs -------------------------------------------------------------
|
|
626
|
+
|
|
627
|
+
def ref_key(d: dict) -> tuple | None:
|
|
628
|
+
"""A hashable key for a session or event ref, or None if d isn't one."""
|
|
629
|
+
try:
|
|
630
|
+
if "session_id" in d and "msg_idx" in d:
|
|
631
|
+
return ("msg", str(d["session_id"]), int(d["msg_idx"]))
|
|
632
|
+
if "event_type" in d and "event_id" in d:
|
|
633
|
+
return ("event", str(d["event_type"]), int(d["event_id"]))
|
|
634
|
+
except (TypeError, ValueError):
|
|
635
|
+
pass
|
|
636
|
+
return None
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def ref_of(key: tuple) -> dict:
|
|
640
|
+
kind, a, b = key
|
|
641
|
+
return {"session_id": a, "msg_idx": b} if kind == "msg" else {"event_type": a, "event_id": b}
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def bundle_refs(obj) -> set[tuple]:
|
|
645
|
+
"""Every ref anywhere in a bundle."""
|
|
646
|
+
if isinstance(obj, list):
|
|
647
|
+
return set().union(*(bundle_refs(x) for x in obj)) if obj else set()
|
|
648
|
+
if isinstance(obj, dict):
|
|
649
|
+
found = {k} if (k := ref_key(obj)) else set()
|
|
650
|
+
return found.union(*(bundle_refs(v) for v in obj.values()))
|
|
651
|
+
return set()
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
# --- LLM pass ----------------------------------------------------------------------
|
|
655
|
+
|
|
656
|
+
SYSTEM = """You analyze how a developer or a team works with AI coding agents. It may be one person; the
|
|
657
|
+
bundle's refs and counts show how many sessions there are.
|
|
658
|
+
You get an evidence bundle: SQL rollups over labeled agent session transcripts and, when the data has them,
|
|
659
|
+
CI checks and PR review comments. Take the repo, language and tools from the evidence; assume none.
|
|
660
|
+
Labels come from a classifier and can be wrong; quotes are excerpts.
|
|
661
|
+
The bundle shows a capped sample of items. "totals" and the per-group counts cover all evidence, not just the sample:
|
|
662
|
+
use them when you state counts.
|
|
663
|
+
|
|
664
|
+
Write findings for the people who ran these sessions. Rules:
|
|
665
|
+
- Each finding must be specific and actionable: a concrete line to add to the agent's instruction file
|
|
666
|
+
(CLAUDE.md, AGENTS.md or whichever the sessions use), a test to add,
|
|
667
|
+
a script or doc to write, an issue to file, a check to change. Name files, commands and checks when the evidence does.
|
|
668
|
+
- Cite evidence only by copying refs that appear in the bundle: {"session_id": ..., "msg_idx": ...} or
|
|
669
|
+
{"event_type": ..., "event_id": ...}. Never invent or alter a ref.
|
|
670
|
+
- Base each finding on a pattern across several items where possible; say how many items support it.
|
|
671
|
+
- If the evidence is too thin to support a finding, return fewer findings or none. Do not pad. Do not invent.
|
|
672
|
+
- Plain, short sentences.
|
|
673
|
+
|
|
674
|
+
Reply with JSON only:
|
|
675
|
+
{"findings": [{"kind": "<kind>", "title": "<one line>", "detail": "<what the evidence shows, with counts>",
|
|
676
|
+
"suggested_fix": "<the concrete change>", "evidence": [<refs>]}]}"""
|
|
677
|
+
|
|
678
|
+
KIND_HINTS = {
|
|
679
|
+
"hotspot": "Files the agent had just written when a human corrected it, reversed a design or caught a defect; "
|
|
680
|
+
"plus CI failures by check when the data has any. Look for files or areas that need better docs, tests or guardrails.",
|
|
681
|
+
"knowledge_gap": "Turns where the human pointed the agent at something (spec, script, doc, prior PR, code) "
|
|
682
|
+
"or the agent could have found the answer itself. Counts split each resource type into "
|
|
683
|
+
"opening (the session's first human message) and mid_session (later turns); examples carry "
|
|
684
|
+
"the same flag. A pointer in the opening message is context given up front, a good habit, "
|
|
685
|
+
"and not a gap in itself. Mid-session pointers and could-have-found turns are the gaps. "
|
|
686
|
+
"Report opening and mid-session counts separately and never add them together. Look for "
|
|
687
|
+
"knowledge to put where agents will find it.",
|
|
688
|
+
"found_not_filed": "Places where the agent noticed a bug or problem outside its task and deferred it without filing "
|
|
689
|
+
"it during the session. Hits are grouped by session. The agent often repeats the same deferred problem on "
|
|
690
|
+
"later turns: merge repeats into one finding that cites all their refs, and count each "
|
|
691
|
+
"problem once. Some hits are false positives (e.g. a test-result summary): skip any "
|
|
692
|
+
"item whose agent_text does not explicitly defer or leave open a specific problem. Each real "
|
|
693
|
+
"one is a candidate issue: write the issue title and a one-line body in suggested_fix.",
|
|
694
|
+
"correction_pattern": "What kinds of corrections humans make, with examples. Look for repeat mistakes that a "
|
|
695
|
+
"rule, check or test would prevent.",
|
|
696
|
+
"prompt_practice": "Prompt style, specificity and task type against outcome and correction rates. Sessions "
|
|
697
|
+
"whose PR is still open are left out. Each session's pr is merged_or_closed or none; "
|
|
698
|
+
"totals.sessions_without_pr counts the none ones. A session with no PR has no merge to "
|
|
699
|
+
"confirm its outcome, which is then only the classifier's reading of the transcript: say "
|
|
700
|
+
"so when a finding rests on those sessions. Small samples: "
|
|
701
|
+
"report differences only when counts support them, and state the counts.",
|
|
702
|
+
"ci": "Failed CI checks on merged or closed PRs, by failure class with log tails; flaky = the same check later passed on the same "
|
|
703
|
+
"commit (a re-run fixed it). Look for flaky tests to fix, and for failures an agent could catch before pushing "
|
|
704
|
+
"(a local task or script, a pre-push hook, a line in the agent's instruction file).",
|
|
705
|
+
"review": "PR review comments on merged or closed PRs, bot vs human, split by kind (inline thread, review "
|
|
706
|
+
"body, issue comment). Inline outcomes read <validity>/<thread state>: validity (valid | wrong | "
|
|
707
|
+
"unclear) comes from a classifier, thread state (resolved | outdated | open) from GitHub. Other "
|
|
708
|
+
"kinds carry the classifier's held_up label only. Look for review noise to cut and for "
|
|
709
|
+
"comment types that point at gaps agents keep leaving.",
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def user_prompt(kind: str, bundle: dict, about: str = "") -> str:
|
|
714
|
+
"""about: a line saying which project the bundle covers; empty for the pass over all of them."""
|
|
715
|
+
return (f"Finding kind: {kind}\n{KIND_HINTS[kind]}\n{about}\n"
|
|
716
|
+
f"Evidence bundle:\n{json.dumps(bundle, ensure_ascii=False)}")
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def canonical_model(model: str) -> str:
|
|
720
|
+
"""One name per model across transports, in OpenRouter form: claude-opus-5-5 -> anthropic/claude-opus-5.5."""
|
|
721
|
+
if model.startswith("claude-"):
|
|
722
|
+
return "anthropic/" + re.sub(r"-(\d{1,2})-(\d{1,2})$", r"-\1.\2", model)
|
|
723
|
+
return model
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def prompt_hash(model: str, user: str) -> str:
|
|
727
|
+
# The hash covers the model and prompts, not the transport: Claude Code and OpenRouter
|
|
728
|
+
# send the same prompts to the same model, so a bundle cached from one is reused by the
|
|
729
|
+
# other. A different model (or model version) misses the cache.
|
|
730
|
+
return hashlib.sha256(json.dumps([canonical_model(model), SYSTEM, user], ensure_ascii=False).encode()).hexdigest()
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def input_price(model: str) -> float | None:
|
|
734
|
+
"""USD per input token from OpenRouter's model list, or None if it can't be fetched."""
|
|
735
|
+
try:
|
|
736
|
+
resp = httpx.get(MODELS_URL, timeout=15)
|
|
737
|
+
resp.raise_for_status()
|
|
738
|
+
model_info = next(m for m in resp.json()["data"] if m["id"] == model)
|
|
739
|
+
return float(model_info["pricing"]["prompt"])
|
|
740
|
+
except (httpx.HTTPError, ValueError, KeyError, TypeError, StopIteration):
|
|
741
|
+
return None
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def parse_json(text: str) -> dict:
|
|
745
|
+
"""Parse a model reply that should be JSON, tolerating code fences and surrounding prose."""
|
|
746
|
+
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())
|
|
747
|
+
try:
|
|
748
|
+
return json.loads(text)
|
|
749
|
+
except json.JSONDecodeError:
|
|
750
|
+
start, end = text.find("{"), text.rfind("}")
|
|
751
|
+
return json.loads(text[start:end + 1]) if start >= 0 < end else {}
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def findings_of(text: str) -> list[dict]:
|
|
755
|
+
findings = parse_json(text).get("findings", [])
|
|
756
|
+
return findings if isinstance(findings, list) else []
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def ask_openrouter(user: str, model: str, key: str, system: str = SYSTEM) -> tuple[list[dict], dict]:
|
|
760
|
+
"""One chat completion for one bundle prompt; returns (findings, usage)."""
|
|
761
|
+
resp = httpx.post(
|
|
762
|
+
OPENROUTER_URL,
|
|
763
|
+
headers={"Authorization": f"Bearer {key}"},
|
|
764
|
+
json={"model": model, "max_tokens": MAX_TOKENS, "response_format": {"type": "json_object"},
|
|
765
|
+
"usage": {"include": True},
|
|
766
|
+
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]},
|
|
767
|
+
timeout=600,
|
|
768
|
+
)
|
|
769
|
+
resp.raise_for_status()
|
|
770
|
+
body = resp.json()
|
|
771
|
+
usage = body.get("usage") or {}
|
|
772
|
+
return findings_of(body["choices"][0]["message"]["content"] or ""), {
|
|
773
|
+
"in": usage.get("prompt_tokens"), "out": usage.get("completion_tokens"), "cost": usage.get("cost") or 0}
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
class ClaudeCodeError(RuntimeError):
|
|
777
|
+
pass
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def claude_code_once(user: str, model: str, cwd: str, system: str = SYSTEM) -> dict:
|
|
781
|
+
"""Run `claude -p` as a plain model call and return its JSON result envelope."""
|
|
782
|
+
cmd = [
|
|
783
|
+
"claude", "-p", "--model", model, "--output-format", "json", "--max-turns", "1",
|
|
784
|
+
"--system-prompt", system,
|
|
785
|
+
"--tools", "", # no tools: one model turn, nothing else
|
|
786
|
+
"--setting-sources", "", # skip user/project settings, hooks and CLAUDE.md
|
|
787
|
+
"--strict-mcp-config", # no MCP servers
|
|
788
|
+
"--disable-slash-commands", # no skills
|
|
789
|
+
"--no-session-persistence", # keep these out of `claude --resume`
|
|
790
|
+
]
|
|
791
|
+
try:
|
|
792
|
+
proc = subprocess.run(cmd, input=user, capture_output=True, text=True, cwd=cwd,
|
|
793
|
+
timeout=CLAUDE_CODE_TIMEOUT, check=False)
|
|
794
|
+
except subprocess.TimeoutExpired as e:
|
|
795
|
+
raise ClaudeCodeError(f"timed out after {CLAUDE_CODE_TIMEOUT}s") from e
|
|
796
|
+
try:
|
|
797
|
+
env = json.loads(proc.stdout)
|
|
798
|
+
except json.JSONDecodeError:
|
|
799
|
+
env = None
|
|
800
|
+
if proc.returncode or not isinstance(env, dict) or env.get("is_error"):
|
|
801
|
+
detail = (env or {}).get("result") or proc.stderr.strip() or proc.stdout.strip()
|
|
802
|
+
# An API error still reports subtype "success", so show the HTTP status instead.
|
|
803
|
+
status = (env or {}).get("api_error_status")
|
|
804
|
+
why = f"HTTP {status}" if status else (env or {}).get("subtype", "no JSON envelope")
|
|
805
|
+
raise ClaudeCodeError(f"exit {proc.returncode}, {why}: {excerpt(detail, 300)}")
|
|
806
|
+
return env
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def ask_claude_code(user: str, model: str, system: str = SYSTEM,
|
|
810
|
+
parse: Callable[[str], list[dict]] = findings_of) -> tuple[list[dict], dict]:
|
|
811
|
+
"""One headless Claude Code call, retried on failure; returns (parsed items, usage)."""
|
|
812
|
+
# Run from an empty directory so nothing from this project reaches the model.
|
|
813
|
+
with tempfile.TemporaryDirectory(prefix="synth-") as cwd:
|
|
814
|
+
for wait in (*CLAUDE_CODE_RETRIES, None):
|
|
815
|
+
try:
|
|
816
|
+
env = claude_code_once(user, model, cwd, system)
|
|
817
|
+
break
|
|
818
|
+
except ClaudeCodeError as e:
|
|
819
|
+
if wait is None:
|
|
820
|
+
raise
|
|
821
|
+
print(f" claude -p failed ({e}); retrying in {wait}s")
|
|
822
|
+
time.sleep(wait)
|
|
823
|
+
usage = env.get("usage") or {}
|
|
824
|
+
tokens_in = sum(usage.get(k) or 0 for k in
|
|
825
|
+
("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"))
|
|
826
|
+
return parse(env.get("result") or ""), {
|
|
827
|
+
"in": tokens_in, "out": usage.get("output_tokens"), "cost": env.get("total_cost_usd") or 0,
|
|
828
|
+
"seconds": round((env.get("duration_ms") or 0) / 1000)}
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def check_evidence(kind: str, findings: list[dict], allowed: set[tuple]) -> list[dict]:
|
|
832
|
+
"""Keep only refs present in the bundle; drop findings left with none."""
|
|
833
|
+
kept = []
|
|
834
|
+
for f in findings:
|
|
835
|
+
refs = [ref_key(r) for r in f.get("evidence") or [] if isinstance(r, dict)]
|
|
836
|
+
good = list(dict.fromkeys(r for r in refs if r in allowed))
|
|
837
|
+
if len(good) < len(refs):
|
|
838
|
+
print(f" {kind}: dropped {len(refs) - len(good)} unknown ref(s) from {f.get('title')!r}")
|
|
839
|
+
if not good:
|
|
840
|
+
print(f" {kind}: dropped finding with no valid evidence: {f.get('title')!r}")
|
|
841
|
+
continue
|
|
842
|
+
kept.append({**f, "kind": kind, "evidence": [ref_of(r) for r in good]})
|
|
843
|
+
return kept
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
# --- cache -----------------------------------------------------------------------------
|
|
847
|
+
|
|
848
|
+
def load_index() -> dict:
|
|
849
|
+
try:
|
|
850
|
+
return json.loads(INDEX_PATH.read_text())
|
|
851
|
+
except (FileNotFoundError, json.JSONDecodeError):
|
|
852
|
+
return {}
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def cached_run(conn: sqlite3.Connection, index: dict, digest: str, kind: str, name: str = CROSS) -> dict | None:
|
|
856
|
+
"""The index entry whose findings came from this exact prompt, if its run is still the latest for kind.
|
|
857
|
+
|
|
858
|
+
The entry gains "ids", its findings' ids. An entry from before passes existed has no
|
|
859
|
+
"pass" and covers every finding of its kind in its run.
|
|
860
|
+
"""
|
|
861
|
+
entry = index.get(digest)
|
|
862
|
+
if not entry or entry["kind"] != kind or entry.get("pass", CROSS) != name:
|
|
863
|
+
return None
|
|
864
|
+
ids = entry["ids"] if "ids" in entry else [r[0] for r in conn.execute(
|
|
865
|
+
"SELECT finding_id FROM findings WHERE run_id = ? AND kind = ?", (entry["run_id"], kind))]
|
|
866
|
+
count = conn.execute(f"SELECT COUNT(*) FROM findings WHERE run_id = ? AND kind = ? "
|
|
867
|
+
f"AND finding_id IN ({','.join('?' * len(ids))})", (entry["run_id"], kind, *ids)).fetchone()[0]
|
|
868
|
+
latest = conn.execute("SELECT MAX(run_id) FROM findings WHERE kind = ?", (kind,)).fetchone()[0]
|
|
869
|
+
if count != entry["findings"] or (latest and latest > entry["run_id"]):
|
|
870
|
+
return None
|
|
871
|
+
return {**entry, "ids": ids}
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def save_findings(conn: sqlite3.Connection, run_id: str, kind: str, findings: list[dict], now: str) -> list[int]:
|
|
875
|
+
"""Insert findings, each with its "project" (None: it spans projects); returns their ids."""
|
|
876
|
+
ids = [conn.execute(
|
|
877
|
+
"INSERT INTO findings (run_id, kind, title, detail, suggested_fix, evidence, project, created_at) "
|
|
878
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
879
|
+
(run_id, kind, f.get("title"), f.get("detail"), f.get("suggested_fix"),
|
|
880
|
+
json.dumps(f["evidence"]), f.get("project"), now)).lastrowid for f in findings]
|
|
881
|
+
conn.commit()
|
|
882
|
+
return ids
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
# --- main ------------------------------------------------------------------------------
|
|
886
|
+
|
|
887
|
+
# Kinds built only from GitHub data: the table each needs, and what to call it in the skip note.
|
|
888
|
+
NEEDS = {"ci": ("gh_checks", "CI checks"), "review": ("gh_review_comments", "PR review comments")}
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
def missing_data(conn: sqlite3.Connection, kind: str) -> str | None:
|
|
892
|
+
"""Why kind can't be built from this database, or None if it can."""
|
|
893
|
+
table, what = NEEDS.get(kind, (None, None))
|
|
894
|
+
if table and not conn.execute(f"SELECT 1 FROM {table} LIMIT 1").fetchone():
|
|
895
|
+
return f"no {what} in the database (needs GitHub data)"
|
|
896
|
+
return None
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def main(argv: list[str] | None = None) -> None:
|
|
900
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
901
|
+
parser.add_argument("--kind", choices=KINDS, help="Only this finding kind")
|
|
902
|
+
parser.add_argument("--min-confidence", type=float, default=0.7, help="Ignore labels below this confidence")
|
|
903
|
+
parser.add_argument("--max-evidence", type=int, default=40, help="Most evidence items per bundle")
|
|
904
|
+
parser.add_argument("--since", default="", help="Ignore evidence before this ISO date (default: keep all)")
|
|
905
|
+
parser.add_argument("--llm", choices=["claude-code", "openrouter"],
|
|
906
|
+
help="claude-code: `claude -p` on the Claude subscription; openrouter: OpenRouter API. "
|
|
907
|
+
"Default: [synthesize] in config.toml, else claude-code")
|
|
908
|
+
parser.add_argument("--parallel", type=int,
|
|
909
|
+
help="Concurrent model calls (default 2 for claude-code, which shares subscription "
|
|
910
|
+
"limits; one per bundle for openrouter)")
|
|
911
|
+
parser.add_argument("--dry-run", action="store_true", help="Build bundles and print them; no model call")
|
|
912
|
+
parser.add_argument("--force", action="store_true", help="Call the model even when a bundle is cached")
|
|
913
|
+
parser.add_argument("--db", type=Path, help="Database path (default debrief.db)")
|
|
914
|
+
args = parser.parse_args(argv)
|
|
915
|
+
load_env()
|
|
916
|
+
|
|
917
|
+
conn = connect(args.db) if args.db else connect()
|
|
918
|
+
mark_relayed(conn)
|
|
919
|
+
args.llm, model = choose("synthesize", args.llm)
|
|
920
|
+
subscription = args.llm == "claude-code"
|
|
921
|
+
price = None if subscription else input_price(model)
|
|
922
|
+
todo_passes = passes(conn, projects())
|
|
923
|
+
if not todo_passes:
|
|
924
|
+
print("no picked project has sessions: nothing to do. `debrief sync` lists the projects; "
|
|
925
|
+
"`debrief sync --project DIR` picks one.")
|
|
926
|
+
return
|
|
927
|
+
run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
928
|
+
out = BUNDLE_DIR / run_id
|
|
929
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
930
|
+
index = load_index()
|
|
931
|
+
|
|
932
|
+
# Every dict below is keyed by (pass name, kind).
|
|
933
|
+
bundles, prompts, digests, cached, of_pass = {}, {}, {}, {}, {}
|
|
934
|
+
todo_tokens = 0
|
|
935
|
+
for i, p in enumerate(todo_passes):
|
|
936
|
+
kinds = [k for k in p["kinds"] if args.kind in (None, k)]
|
|
937
|
+
if not kinds:
|
|
938
|
+
continue
|
|
939
|
+
name = p["name"]
|
|
940
|
+
print(f"\n== {short_name(name)}: {len(p['sessions'])} sessions, {p['turns']} human turns")
|
|
941
|
+
about = f"Project: {short_name(name)}\n" if p["project"] else ""
|
|
942
|
+
scope(conn, p["sessions"], p["repos"])
|
|
943
|
+
print(f"{'kind':<20} {'items':>11} {'refs':>5} {'~tokens':>8} {'~cost':>8}")
|
|
944
|
+
for kind in kinds:
|
|
945
|
+
if why := missing_data(conn, kind):
|
|
946
|
+
print(f"{kind:<20} skipped: {why}")
|
|
947
|
+
continue
|
|
948
|
+
key = (name, kind)
|
|
949
|
+
of_pass[key] = p
|
|
950
|
+
bundle = bundles[key] = ROLLUPS[kind](conn, args.min_confidence, args.max_evidence, args.since)
|
|
951
|
+
user = prompts[key] = user_prompt(kind, bundle, about)
|
|
952
|
+
digest = digests[key] = prompt_hash(model, user)
|
|
953
|
+
folder = out / f"{i:02d}-{short_name(name).replace(' ', '-')}"
|
|
954
|
+
folder.mkdir(exist_ok=True)
|
|
955
|
+
(folder / f"{kind}.json").write_text(json.dumps(
|
|
956
|
+
{"kind": kind, "pass": name, "model": model, "hash": digest, "bundle": bundle},
|
|
957
|
+
indent=2, ensure_ascii=False))
|
|
958
|
+
tokens = (len(SYSTEM) + len(user)) // 4
|
|
959
|
+
cost = "sub" if subscription else f"${tokens * price:.3f}" if price is not None else "?"
|
|
960
|
+
note = ""
|
|
961
|
+
if not bundle_refs(bundle):
|
|
962
|
+
note = " skipped: no evidence, nothing to send"
|
|
963
|
+
elif not args.force and (hit := cached_run(conn, index, digest, kind, name)):
|
|
964
|
+
cached[key] = hit
|
|
965
|
+
note = f" cached from {hit['run_id']}"
|
|
966
|
+
else:
|
|
967
|
+
todo_tokens += tokens
|
|
968
|
+
items = f"{bundle['totals']['shown']}/{bundle['totals']['items']}"
|
|
969
|
+
print(f"{kind:<20} {items:>11} {len(bundle_refs(bundle)):>5} {tokens:>8} {cost:>8}{note}")
|
|
970
|
+
scope(conn)
|
|
971
|
+
todo = {k: p for k, p in prompts.items() if bundle_refs(bundles[k]) and k not in cached}
|
|
972
|
+
print(f"\n{len(todo)} model calls, {len(cached)} cached")
|
|
973
|
+
if subscription:
|
|
974
|
+
print(f"~{todo_tokens} input tokens to send to {model} via claude -p (subscription)")
|
|
975
|
+
elif price is None:
|
|
976
|
+
print(f"~{todo_tokens} input tokens to send ({model} price unavailable)")
|
|
977
|
+
else:
|
|
978
|
+
print(f"~{todo_tokens} input tokens to send, ~${todo_tokens * price:.2f} at {model} input price "
|
|
979
|
+
f"(output extra, at most {MAX_TOKENS} tokens per call)")
|
|
980
|
+
print(f"bundles in {out}")
|
|
981
|
+
|
|
982
|
+
if args.dry_run:
|
|
983
|
+
if not bundles:
|
|
984
|
+
return
|
|
985
|
+
key = max(bundles, key=lambda k: len(bundle_refs(bundles[k])))
|
|
986
|
+
print(f"\n--- {short_name(key[0])}: {key[1]} ---\n{json.dumps(bundles[key], indent=2, ensure_ascii=False)[:6000]}")
|
|
987
|
+
return
|
|
988
|
+
|
|
989
|
+
if subscription:
|
|
990
|
+
if not shutil.which("claude"):
|
|
991
|
+
sys.exit("claude (Claude Code) is not on PATH")
|
|
992
|
+
ask = partial(ask_claude_code, model=model)
|
|
993
|
+
else:
|
|
994
|
+
key = os.environ.get("OPENROUTER_API_KEY")
|
|
995
|
+
if not key:
|
|
996
|
+
sys.exit("OPENROUTER_API_KEY is not set")
|
|
997
|
+
ask = partial(ask_openrouter, model=model, key=key)
|
|
998
|
+
now = datetime.now(UTC).isoformat()
|
|
999
|
+
total, cost = 0, 0.0
|
|
1000
|
+
failed = set() # names of passes with a failed call
|
|
1001
|
+
|
|
1002
|
+
def entry(key: tuple, ids: list[int]) -> dict:
|
|
1003
|
+
return {"run_id": run_id, "kind": key[1], "pass": key[0], "findings": len(ids), "ids": ids}
|
|
1004
|
+
|
|
1005
|
+
for key, source in cached.items():
|
|
1006
|
+
marks = ",".join("?" * len(source["ids"]))
|
|
1007
|
+
last = conn.execute("SELECT COALESCE(MAX(finding_id), 0) FROM findings").fetchone()[0]
|
|
1008
|
+
conn.execute(
|
|
1009
|
+
"INSERT INTO findings (run_id, kind, title, detail, suggested_fix, evidence, project, created_at) "
|
|
1010
|
+
"SELECT ?, kind, title, detail, suggested_fix, evidence, project, ? FROM findings "
|
|
1011
|
+
f"WHERE finding_id IN ({marks}) ORDER BY finding_id", (run_id, now, *source["ids"]))
|
|
1012
|
+
conn.commit()
|
|
1013
|
+
ids = [r[0] for r in conn.execute("SELECT finding_id FROM findings WHERE finding_id > ?", (last,))]
|
|
1014
|
+
index[digests[key]] = entry(key, ids)
|
|
1015
|
+
total += len(ids)
|
|
1016
|
+
print(f"{short_name(key[0])}: {key[1]}: {len(ids)} findings copied from {source['run_id']}")
|
|
1017
|
+
|
|
1018
|
+
workers = args.parallel or (2 if subscription else len(todo)) or 1
|
|
1019
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
1020
|
+
futures = {k: pool.submit(ask, p) for k, p in todo.items()}
|
|
1021
|
+
for key, fut in futures.items():
|
|
1022
|
+
kind, label = key[1], f"{short_name(key[0])}: {key[1]}"
|
|
1023
|
+
try:
|
|
1024
|
+
raw, usage = fut.result()
|
|
1025
|
+
findings = check_evidence(kind, raw, bundle_refs(bundles[key]))
|
|
1026
|
+
except (httpx.HTTPError, ClaudeCodeError, json.JSONDecodeError, KeyError) as e:
|
|
1027
|
+
print(f"{label}: failed: {e}")
|
|
1028
|
+
failed.add(key[0])
|
|
1029
|
+
continue
|
|
1030
|
+
for f in findings:
|
|
1031
|
+
f["project"] = of_pass[key]["project"]
|
|
1032
|
+
index[digests[key]] = entry(key, save_findings(conn, run_id, kind, findings, now))
|
|
1033
|
+
total += len(findings)
|
|
1034
|
+
cost += usage["cost"]
|
|
1035
|
+
took = f", {usage['seconds']}s" if "seconds" in usage else ""
|
|
1036
|
+
print(f"{label}: {len(findings)} findings, {usage['in']} in / {usage['out']} out tokens, "
|
|
1037
|
+
f"${usage['cost']:.4f}{took}")
|
|
1038
|
+
for f in findings:
|
|
1039
|
+
where = f" {short_name(f['project'])}" if f["project"] else ""
|
|
1040
|
+
print(f"[{kind}{where}] {f.get('title')} ({len(f['evidence'])} refs)\n fix: {f.get('suggested_fix')}")
|
|
1041
|
+
INDEX_PATH.write_text(json.dumps(index, indent=2))
|
|
1042
|
+
if not args.kind: # a whole pass with no failed call: the project is analysed as of this run
|
|
1043
|
+
conn.executemany("INSERT OR REPLACE INTO project_runs VALUES (?, ?, ?)",
|
|
1044
|
+
[(p["project"], run_id, now) for p in todo_passes if p["project"] and p["name"] not in failed])
|
|
1045
|
+
conn.commit()
|
|
1046
|
+
# Claude Code reports what the call would cost at API prices; the subscription pays instead.
|
|
1047
|
+
billed = f"~${cost:.4f} API-equivalent, billed to subscription" if subscription else f"${cost:.4f}"
|
|
1048
|
+
print(f"\n{total} findings saved under run_id {run_id} ({model} via {args.llm}), cost {billed}")
|
|
1049
|
+
|
|
1050
|
+
|
|
1051
|
+
if __name__ == "__main__":
|
|
1052
|
+
main()
|