debrief-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- debrief/__init__.py +0 -0
- debrief/build_artifact.py +83 -0
- debrief/classify.py +812 -0
- debrief/cli.py +291 -0
- debrief/compare_definitions.py +147 -0
- debrief/compare_llm.py +250 -0
- debrief/config.py +204 -0
- debrief/db.py +341 -0
- debrief/export.py +156 -0
- debrief/export_site.py +661 -0
- debrief/github.py +540 -0
- debrief/normalize.py +635 -0
- debrief/paths.py +17 -0
- debrief/prune.py +117 -0
- debrief/recommend.py +271 -0
- debrief/relay.py +83 -0
- debrief/runner.py +163 -0
- debrief/scrape.py +208 -0
- debrief/site/app.js +1613 -0
- debrief/site/index.html +32 -0
- debrief/site/method.js +117 -0
- debrief/site/style.css +520 -0
- debrief/skills.py +107 -0
- debrief/sources.py +341 -0
- debrief/synthesize.py +1052 -0
- debrief/verify_evidence.py +154 -0
- debrief_cli-0.1.0.dist-info/METADATA +205 -0
- debrief_cli-0.1.0.dist-info/RECORD +31 -0
- debrief_cli-0.1.0.dist-info/WHEEL +4 -0
- debrief_cli-0.1.0.dist-info/entry_points.txt +2 -0
- debrief_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
debrief/prune.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Drop sessions last active before a date, and the GitHub rows only they link to.
|
|
2
|
+
|
|
3
|
+
Removes their data/raw/<id>/ folders and their rows in sessions, checkpoints, session_prs,
|
|
4
|
+
messages, tool_calls, turn_labels and session_labels. Then drops GitHub rows no remaining
|
|
5
|
+
session reaches: commit links and seen-markers for commits that are no longer checkpoints
|
|
6
|
+
nor in a kept local session's PR, PRs no remaining commit is in, checks on commits that are neither a kept checkpoint nor a kept PR's
|
|
7
|
+
head, comments on dropped PRs, and event labels of dropped checks and comments. Raw folders
|
|
8
|
+
with no session row go too when their listing.json is older than the date, or when empty.
|
|
9
|
+
findings, recommendations and evidence_checks stay as history. Ends with VACUUM.
|
|
10
|
+
|
|
11
|
+
uv run python -m debrief.prune --before 2026-09-14 --dry-run
|
|
12
|
+
uv run python -m debrief.prune --before 2026-09-14
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import shutil
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from debrief import db
|
|
21
|
+
from debrief.paths import RAW_DIR
|
|
22
|
+
|
|
23
|
+
SESSION_TABLES = ["sessions", "checkpoints", "session_prs", "messages", "tool_calls", "turn_labels", "session_labels"]
|
|
24
|
+
GH_TABLES = ["gh_commit_prs", "gh_commits_seen", "gh_prs", "gh_checks", "gh_review_comments", "event_labels"]
|
|
25
|
+
|
|
26
|
+
# Each runs after the session rows are gone, in this order, since later ones read earlier results.
|
|
27
|
+
KEPT_COMMITS = "SELECT commit_sha FROM checkpoints WHERE commit_sha IS NOT NULL"
|
|
28
|
+
GH_DELETES = [
|
|
29
|
+
(f"DELETE FROM gh_commit_prs WHERE commit_sha NOT IN ({KEPT_COMMITS}) "
|
|
30
|
+
"AND (repo, pr_number) NOT IN (SELECT repo, pr_number FROM session_prs)"),
|
|
31
|
+
f"DELETE FROM gh_commits_seen WHERE commit_sha NOT IN ({KEPT_COMMITS})",
|
|
32
|
+
"DELETE FROM gh_prs WHERE (repo, pr_number) NOT IN (SELECT repo, pr_number FROM gh_commit_prs)",
|
|
33
|
+
# github.py also lists checks on PR heads, which need not be checkpoint commits.
|
|
34
|
+
(f"DELETE FROM gh_checks WHERE commit_sha NOT IN ({KEPT_COMMITS}) "
|
|
35
|
+
"AND commit_sha NOT IN (SELECT head_sha FROM gh_prs WHERE head_sha IS NOT NULL) "
|
|
36
|
+
"AND commit_sha NOT IN (SELECT commit_sha FROM gh_commit_prs)"),
|
|
37
|
+
"DELETE FROM gh_review_comments WHERE (repo, pr_number) NOT IN (SELECT repo, pr_number FROM gh_prs)",
|
|
38
|
+
("DELETE FROM event_labels WHERE event_type = 'review_comment' "
|
|
39
|
+
"AND event_id NOT IN (SELECT comment_id FROM gh_review_comments)"),
|
|
40
|
+
"DELETE FROM event_labels WHERE event_type = 'check_failure' AND event_id NOT IN (SELECT check_id FROM gh_checks)",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def counts(conn) -> dict[str, int]:
|
|
45
|
+
return {t: conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] for t in SESSION_TABLES + GH_TABLES}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def dir_bytes(path: Path) -> int: # 0 when path is missing
|
|
49
|
+
return sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def stale_folders(before: str, dropped: set[str], known: set[str]) -> list[Path]:
|
|
53
|
+
"""Raw folders to remove: dropped sessions, plus session-less folders that are old or empty."""
|
|
54
|
+
out = []
|
|
55
|
+
for folder in sorted(p for p in RAW_DIR.glob("*") if p.is_dir()): # no raw/ when every session is local
|
|
56
|
+
listing = folder / "listing.json"
|
|
57
|
+
orphan = folder.name not in known and (
|
|
58
|
+
not any(folder.iterdir())
|
|
59
|
+
or (listing.exists() and json.loads(listing.read_text()).get("lastActivityAt", "") < before))
|
|
60
|
+
if folder.name in dropped or orphan:
|
|
61
|
+
out.append(folder)
|
|
62
|
+
return out
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def main(argv: list[str] | None = None) -> None:
|
|
66
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
67
|
+
parser.add_argument("--before", required=True, metavar="YYYY-MM-DD", help="Drop sessions last active before this")
|
|
68
|
+
parser.add_argument("--dry-run", action="store_true", help="Report what would go; change nothing")
|
|
69
|
+
args = parser.parse_args(argv)
|
|
70
|
+
assert len(args.before) == 10 and args.before[4] == args.before[7] == "-", "--before must be YYYY-MM-DD"
|
|
71
|
+
|
|
72
|
+
conn = db.connect()
|
|
73
|
+
conn.execute("CREATE TABLE IF NOT EXISTS gh_commits_seen (commit_sha TEXT PRIMARY KEY, looked_up_at TEXT)")
|
|
74
|
+
known = {r[0] for r in conn.execute("SELECT session_id FROM sessions")}
|
|
75
|
+
# ISO timestamps compare as strings; a session with no date is kept.
|
|
76
|
+
dropped = {r[0] for r in conn.execute("SELECT session_id FROM sessions WHERE last_activity_at < ?", (args.before,))}
|
|
77
|
+
folders = stale_folders(args.before, dropped, known)
|
|
78
|
+
before_counts, raw_before, db_before = counts(conn), dir_bytes(RAW_DIR), db.DB_PATH.stat().st_size
|
|
79
|
+
print(f"{len(dropped)} of {len(known)} sessions last active before {args.before}; "
|
|
80
|
+
f"{len(folders)} raw folders to remove ({sum(f.name not in known for f in folders)} with no session row)")
|
|
81
|
+
|
|
82
|
+
conn.execute("BEGIN")
|
|
83
|
+
marks = ",".join("?" * len(dropped))
|
|
84
|
+
for t in SESSION_TABLES:
|
|
85
|
+
conn.execute(f"DELETE FROM {t} WHERE session_id IN ({marks})", list(dropped))
|
|
86
|
+
# A dropped session must stop owning entries, or a resumed copy of it would keep them left out.
|
|
87
|
+
# Its source_logs row stays, so sources.py does not load the unchanged log again.
|
|
88
|
+
conn.execute(f"DELETE FROM source_entries WHERE session_id IN ({marks})", list(dropped))
|
|
89
|
+
for sql in GH_DELETES:
|
|
90
|
+
conn.execute(sql)
|
|
91
|
+
after_counts = counts(conn)
|
|
92
|
+
for t in before_counts:
|
|
93
|
+
print(f" {t:<20} {before_counts[t]:>9} -> {after_counts[t]:>9}")
|
|
94
|
+
# Self-check: no kept session lost rows, and every kept link still resolves.
|
|
95
|
+
assert after_counts["sessions"] == len(known) - len(dropped)
|
|
96
|
+
assert not conn.execute("SELECT 1 FROM sessions WHERE last_activity_at < ?", (args.before,)).fetchone()
|
|
97
|
+
lost = conn.execute(f"SELECT COUNT(DISTINCT commit_sha) FROM ({KEPT_COMMITS}) WHERE commit_sha NOT IN "
|
|
98
|
+
"(SELECT commit_sha FROM gh_commits_seen UNION SELECT commit_sha FROM gh_commit_prs)").fetchone()[0]
|
|
99
|
+
assert not conn.execute("SELECT 1 FROM gh_commit_prs WHERE (repo, pr_number) NOT IN (SELECT repo, pr_number FROM gh_prs)"
|
|
100
|
+
).fetchone(), "a kept commit links to a PR that is gone"
|
|
101
|
+
print(f" {lost} kept checkpoint commits were never looked up (github.py fetches those as before)")
|
|
102
|
+
|
|
103
|
+
if args.dry_run:
|
|
104
|
+
conn.rollback()
|
|
105
|
+
print(f"dry run: nothing changed. data/raw {raw_before / 1e6:.0f} MB, debrief.db {db_before / 1e6:.0f} MB")
|
|
106
|
+
return
|
|
107
|
+
conn.commit()
|
|
108
|
+
for folder in folders:
|
|
109
|
+
shutil.rmtree(folder)
|
|
110
|
+
conn.execute("VACUUM")
|
|
111
|
+
conn.close()
|
|
112
|
+
print(f"data/raw {raw_before / 1e6:.0f} -> {dir_bytes(RAW_DIR) / 1e6:.0f} MB, "
|
|
113
|
+
f"debrief.db {db_before / 1e6:.0f} -> {db.DB_PATH.stat().st_size / 1e6:.0f} MB")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
main()
|
debrief/recommend.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Merge one findings run into recommendations, grouped by the lever that fixes them.
|
|
2
|
+
|
|
3
|
+
Findings come per project and pipeline kind (hotspot, ci, ...), so one problem can show up
|
|
4
|
+
under several kinds. Each project gets one Claude call over its findings, and the findings
|
|
5
|
+
about the person's habits across projects get one more. A call returns recommendations:
|
|
6
|
+
each names the lever (where the fix lives), the change to make, its effort, and the
|
|
7
|
+
findings it covers. Every finding lands in exactly one recommendation, except that a finding
|
|
8
|
+
listing several bugs may back one backlog recommendation per bug; any the model leaves out become their own.
|
|
9
|
+
|
|
10
|
+
uv run python -m debrief.recommend # latest findings run
|
|
11
|
+
uv run python -m debrief.recommend --run 20260925T182434Z --dry-run
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import shutil
|
|
18
|
+
import sys
|
|
19
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
20
|
+
from datetime import UTC, datetime
|
|
21
|
+
|
|
22
|
+
from debrief.config import choose, load_env, projects
|
|
23
|
+
from debrief.db import connect, session_projects
|
|
24
|
+
from debrief.synthesize import ClaudeCodeError, ask_claude_code, parse_json, short_name
|
|
25
|
+
|
|
26
|
+
LEVERS = {
|
|
27
|
+
"agent_config": "Agent instructions: CLAUDE.md, AGENTS.md, skills, slash commands",
|
|
28
|
+
"test_harness": "Tests and the local dev loop: test isolation, fixtures, task runner and scripts",
|
|
29
|
+
"ci_infra": "CI workflows, runners, caching, tool pins, log output",
|
|
30
|
+
"review_tooling": "Automated code review and its config: review bots, their rules and noise",
|
|
31
|
+
"team_practice": "How the people prompt and run sessions: habits, templates, docs for humans",
|
|
32
|
+
"backlog": "A product or code bug an agent flagged in a session and didn't file during it: check it, file it",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
# Where a finding goes when the model leaves it out.
|
|
36
|
+
KIND_LEVER = {"found_not_filed": "backlog", "ci": "ci_infra", "review": "review_tooling",
|
|
37
|
+
"prompt_practice": "team_practice"}
|
|
38
|
+
|
|
39
|
+
SYSTEM = f"""You turn findings about how a developer or a team works with AI coding agents into a short list
|
|
40
|
+
of ideas for them. "team" in the input gives the number of people; it may be one, so don't write as if there
|
|
41
|
+
were several. Take the repo, language and tools from the findings; assume none. Ideas are leads to
|
|
42
|
+
check, not verdicts: the labels behind them are from a classifier and some will be false positives. Suggest;
|
|
43
|
+
don't tell them what they should do.
|
|
44
|
+
|
|
45
|
+
Each finding came from labeled agent sessions and, when the data has them, CI checks or PR review
|
|
46
|
+
comments, and cites evidence.
|
|
47
|
+
Several findings often describe one problem from different angles (a hotspot file, a correction
|
|
48
|
+
pattern and a knowledge gap can all be the same missing doc). Merge those.
|
|
49
|
+
|
|
50
|
+
Projects: "project" on the input says every finding is about that one repo. Without it, the findings
|
|
51
|
+
are about the person's habits across their repos.
|
|
52
|
+
|
|
53
|
+
Levers (where the fix lives):
|
|
54
|
+
{chr(10).join(f"- {k}: {v}" for k, v in LEVERS.items())}
|
|
55
|
+
|
|
56
|
+
Counts:
|
|
57
|
+
- Each finding's "evidence" gives refs, sessions and people: the cited items, the sessions behind them
|
|
58
|
+
and the people who ran those sessions. Use them, and "team", to judge how much weight a finding
|
|
59
|
+
carries. Don't quote them; the dashboard counts reach itself.
|
|
60
|
+
- Counts inside finding text often describe a whole category or an unnamed subset, not one problem.
|
|
61
|
+
|
|
62
|
+
Rules:
|
|
63
|
+
- Assign every finding id to exactly one recommendation. The one exception: if a finding lists several
|
|
64
|
+
unrelated bugs, split it into one backlog recommendation per bug, each citing that finding.
|
|
65
|
+
- Backlog: exactly one recommendation per distinct bug, unless another recommendation's change already
|
|
66
|
+
fixes it. Never group bugs, even small, related or single-session ones, and even if one session flagged
|
|
67
|
+
them together; each backlog title names one bug, never two joined by "and".
|
|
68
|
+
Say the bug was flagged in a session and not filed during it; we can't see the issue tracker, so
|
|
69
|
+
don't say nobody filed it.
|
|
70
|
+
- "change" is the suggested change. For agent_config, give the exact text to add and the file.
|
|
71
|
+
For others, name the file, workflow, setting or command. Markdown is fine; keep it under 120 words.
|
|
72
|
+
- "problem" says what goes wrong, in two or three plain sentences. It may use counts only about this
|
|
73
|
+
specific problem, each with its unit named (messages, sessions, CI runs, review comments). Never quote
|
|
74
|
+
category-wide totals, never a count out of an unnamed subset ("31 of 80 failures"), and never add counts
|
|
75
|
+
of different units together. Never give a share of an unnamed or unsized set ("most of the sampled
|
|
76
|
+
ones", "most of these passed", "most log tails", "many of them"): name the count and its unit ("3 of 5
|
|
77
|
+
CI runs"), or say it without a share ("some passed on re-run"). If unsure, describe it
|
|
78
|
+
without numbers.
|
|
79
|
+
- Never state a rate, ratio, percentage or per-session average ("1.8 corrections per session", "twice as
|
|
80
|
+
often", "40% of runs"), even if a finding gives one: the dashboard computes rates itself, and not always
|
|
81
|
+
the same way. Use only raw counts of this problem's own items, with the unit named ("corrected in 6
|
|
82
|
+
messages across 4 sessions").
|
|
83
|
+
- "impact" is one neutral line saying what the problem is and where it shows up, with no numbers at all
|
|
84
|
+
(e.g. "People paste review bot findings into the session by hand, across several sessions").
|
|
85
|
+
Don't promise an outcome ("removes", "saves").
|
|
86
|
+
- "target" starts with the project's name and a colon when the input names a project
|
|
87
|
+
("myrepo: CLAUDE.md", "myrepo: .github/workflows/ci.yml"); no prefix when it does not.
|
|
88
|
+
- effort: S (under an hour), M (a day or less), L (more).
|
|
89
|
+
- Plain, short sentences. No hype. Do not invent facts that are not in the findings.
|
|
90
|
+
|
|
91
|
+
Reply with JSON only:
|
|
92
|
+
{{"recommendations": [{{"lever": "<lever>", "title": "<imperative, under 70 chars>",
|
|
93
|
+
"problem": "...", "change": "...", "target": "<file or system the change lands in>",
|
|
94
|
+
"effort": "S|M|L", "impact": "...", "finding_ids": [<ids>]}}]}}"""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def latest_run(conn) -> str:
|
|
98
|
+
return conn.execute("SELECT MAX(run_id) FROM findings").fetchone()[0]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def evidence_stats(conn, evidence: list[dict]) -> dict:
|
|
102
|
+
"""Sessions and people behind a finding's evidence, events mapped to sessions via their commits or PR."""
|
|
103
|
+
sessions = {e["session_id"] for e in evidence if "session_id" in e}
|
|
104
|
+
for e in evidence:
|
|
105
|
+
if e.get("event_type") == "check_failure":
|
|
106
|
+
sessions |= {r[0] for r in conn.execute(
|
|
107
|
+
"SELECT DISTINCT cp.session_id FROM gh_checks k JOIN session_commits cp ON cp.commit_sha = k.commit_sha "
|
|
108
|
+
"WHERE k.check_id = ?", (e["event_id"],))}
|
|
109
|
+
elif e.get("event_type") == "review_comment":
|
|
110
|
+
sessions |= {r[0] for r in conn.execute(
|
|
111
|
+
"SELECT DISTINCT l.session_id FROM gh_review_comments c "
|
|
112
|
+
"JOIN session_pr_links l ON l.repo = c.repo AND l.pr_number = c.pr_number "
|
|
113
|
+
"WHERE c.comment_id = ?", (e["event_id"],))}
|
|
114
|
+
people = {r[0] for s in sessions for r in conn.execute(
|
|
115
|
+
"SELECT author FROM sessions WHERE session_id = ? AND author IS NOT NULL", (s,))}
|
|
116
|
+
return {"refs": len(evidence), "sessions": len(sessions), "people": len(people)}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def load_findings(conn, run_id: str) -> list[dict]:
|
|
120
|
+
out = []
|
|
121
|
+
for r in conn.execute("SELECT * FROM findings WHERE run_id = ? ORDER BY finding_id", (run_id,)):
|
|
122
|
+
evidence = json.loads(r["evidence"] or "[]")
|
|
123
|
+
out.append({"id": r["finding_id"], "kind": r["kind"], "title": r["title"], "detail": r["detail"],
|
|
124
|
+
"suggested_fix": r["suggested_fix"], "evidence": evidence_stats(conn, evidence),
|
|
125
|
+
"project": r["project"]})
|
|
126
|
+
return out
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def team_context(conn, picked: set[str]) -> dict:
|
|
130
|
+
"""Sessions, people and human turns in these projects."""
|
|
131
|
+
ids = [s for s, p in session_projects(conn).items() if p in picked]
|
|
132
|
+
conn.execute("CREATE TEMP TABLE IF NOT EXISTS team (session_id TEXT PRIMARY KEY)")
|
|
133
|
+
conn.execute("DELETE FROM team")
|
|
134
|
+
conn.executemany("INSERT INTO team VALUES (?)", [(s,) for s in ids])
|
|
135
|
+
people = conn.execute("SELECT COUNT(DISTINCT author) FROM sessions JOIN team USING (session_id)").fetchone()[0]
|
|
136
|
+
human = conn.execute("SELECT COUNT(*) FROM messages JOIN team USING (session_id) "
|
|
137
|
+
"WHERE role = 'user' AND user_kind = 'human'").fetchone()[0]
|
|
138
|
+
return {"sessions": len(ids), "people": people, "human_turns": human}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def groups(findings: list[dict], picked: list[str]) -> dict[str | None, list[dict]]:
|
|
142
|
+
"""Findings per model call: one per picked project, and one (None) for those about all of them.
|
|
143
|
+
Findings of a project no longer picked are left out."""
|
|
144
|
+
out: dict[str | None, list[dict]] = {}
|
|
145
|
+
for f in findings:
|
|
146
|
+
if f["project"] in (None, *picked):
|
|
147
|
+
out.setdefault(f["project"], []).append(f)
|
|
148
|
+
return out
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def prompt(conn, project: str | None, findings: list[dict]) -> str:
|
|
152
|
+
"""The user message for one call. A project goes by folder name; the full path stays here."""
|
|
153
|
+
body = {"team": team_context(conn, {project} if project else set(projects()))}
|
|
154
|
+
if project:
|
|
155
|
+
body["project"] = short_name(project)
|
|
156
|
+
return json.dumps({**body, "findings": [{k: v for k, v in f.items() if k != "project"} for f in findings]},
|
|
157
|
+
ensure_ascii=False)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
# A backlog idea that reads like a list of bugs rather than one.
|
|
161
|
+
BUNDLED = re.compile(r"\b(bugs|several|multiple|various)\b", re.IGNORECASE)
|
|
162
|
+
# A share of a set the reader can't size.
|
|
163
|
+
SHARE = re.compile(r"\b(most|many) (of|\w+s\b)", re.IGNORECASE)
|
|
164
|
+
# A rate or ratio: a decimal, a percentage, "per session", "3x", "twice as".
|
|
165
|
+
RATE = re.compile(r"\d\.\d|\d\s*%|\bper (session|run|message|turn|pr)\b|\b\d+x\b|\b(twice|thrice) as\b",
|
|
166
|
+
re.IGNORECASE)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def check(recs: list[dict], findings: list[dict]) -> list[dict]:
|
|
170
|
+
"""Keep known finding ids, each in one recommendation (or several backlog ones); add one per orphan."""
|
|
171
|
+
by_id = {f["id"]: f for f in findings}
|
|
172
|
+
owner: dict[int, str] = {} # finding id -> lever of the first recommendation that claimed it
|
|
173
|
+
kept = []
|
|
174
|
+
for r in recs:
|
|
175
|
+
backlog = r.get("lever") == "backlog"
|
|
176
|
+
ids = [i for i in r.get("finding_ids") or []
|
|
177
|
+
if i in by_id and (i not in owner or (backlog and owner[i] == "backlog"))]
|
|
178
|
+
for i in ids:
|
|
179
|
+
owner.setdefault(i, r.get("lever"))
|
|
180
|
+
if not ids:
|
|
181
|
+
print(f"dropped recommendation with no known findings: {r.get('title')!r}")
|
|
182
|
+
continue
|
|
183
|
+
if r.get("lever") not in LEVERS:
|
|
184
|
+
r["lever"] = KIND_LEVER.get(by_id[ids[0]]["kind"], "agent_config")
|
|
185
|
+
kept.append({**r, "finding_ids": ids})
|
|
186
|
+
numbered = [r for r in kept if re.search(r"\b\d", r.get("impact") or "")]
|
|
187
|
+
for r in numbered:
|
|
188
|
+
print(f"warning: impact has a number: {r.get('title')!r}: {r['impact']!r}")
|
|
189
|
+
if numbered:
|
|
190
|
+
print(f"{len(numbered)} of {len(kept)} impact lines have numbers; the prompt asks for none")
|
|
191
|
+
for r in kept:
|
|
192
|
+
if r["lever"] == "backlog" and BUNDLED.search(f"{r.get('title')} {r.get('problem')}"):
|
|
193
|
+
print(f"warning: backlog idea may bundle several bugs: {r.get('title')!r}")
|
|
194
|
+
if m := SHARE.search(f"{r.get('title')} {r.get('impact')} {r.get('problem')}"):
|
|
195
|
+
print(f"warning: share of an unsized set ({m.group(0)!r}): {r.get('title')!r}")
|
|
196
|
+
if m := RATE.search(f"{r.get('title')} {r.get('impact')} {r.get('problem')}"):
|
|
197
|
+
print(f"warning: rate or ratio ({m.group(0)!r}): {r.get('title')!r}")
|
|
198
|
+
for f in findings:
|
|
199
|
+
if f["id"] not in owner:
|
|
200
|
+
print(f"finding {f['id']} left out by the model; kept as its own recommendation")
|
|
201
|
+
kept.append({"lever": KIND_LEVER.get(f["kind"], "agent_config"), "title": f["title"],
|
|
202
|
+
"problem": f["detail"], "change": f["suggested_fix"], "target": "", "effort": "M",
|
|
203
|
+
"impact": "", "finding_ids": [f["id"]]})
|
|
204
|
+
for r in kept:
|
|
205
|
+
r["project"] = by_id[r["finding_ids"][0]]["project"] # one call holds one project's findings
|
|
206
|
+
return kept
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def save(conn, run_id: str, recs: list[dict]) -> None:
|
|
210
|
+
now = datetime.now(UTC).isoformat()
|
|
211
|
+
# The UI sorts by reach; priority just keeps the order the ideas came in.
|
|
212
|
+
for i, r in enumerate(recs, 1):
|
|
213
|
+
r["priority"] = i
|
|
214
|
+
conn.execute("DELETE FROM recommendations WHERE run_id = ?", (run_id,))
|
|
215
|
+
conn.executemany(
|
|
216
|
+
"INSERT INTO recommendations (run_id, priority, lever, title, problem, change, target, effort, impact, "
|
|
217
|
+
"finding_ids, project, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
218
|
+
[(run_id, r["priority"], r["lever"], r.get("title"), r.get("problem"), r.get("change"), r.get("target"),
|
|
219
|
+
r.get("effort"), r.get("impact"), json.dumps(r["finding_ids"]), r.get("project"), now) for r in recs])
|
|
220
|
+
conn.commit()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def main(argv: list[str] | None = None) -> None:
|
|
224
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
225
|
+
parser.add_argument("--run", help="Findings run_id (default: latest)")
|
|
226
|
+
parser.add_argument("--dry-run", action="store_true", help="Print the prompt; no model call")
|
|
227
|
+
args = parser.parse_args(argv)
|
|
228
|
+
load_env()
|
|
229
|
+
|
|
230
|
+
conn = connect()
|
|
231
|
+
run_id = args.run or latest_run(conn)
|
|
232
|
+
if not run_id:
|
|
233
|
+
sys.exit("no findings yet; run synthesize.py")
|
|
234
|
+
prompts = {p: (fs, prompt(conn, p, fs)) for p, fs in groups(load_findings(conn, run_id), projects()).items()}
|
|
235
|
+
print(f"run {run_id}: {len(prompts)} model calls")
|
|
236
|
+
for p, (fs, user) in prompts.items():
|
|
237
|
+
print(f" {short_name(p) if p else 'across projects'}: {len(fs)} findings, "
|
|
238
|
+
f"~{(len(SYSTEM) + len(user)) // 4} input tokens")
|
|
239
|
+
if args.dry_run:
|
|
240
|
+
print(next(iter(prompts.values()), ("", ""))[1][:4000])
|
|
241
|
+
return
|
|
242
|
+
if not shutil.which("claude"):
|
|
243
|
+
sys.exit("claude (Claude Code) is not on PATH")
|
|
244
|
+
|
|
245
|
+
_, model = choose("recommend")
|
|
246
|
+
with ThreadPoolExecutor(max_workers=2) as pool: # 2, as in synthesize: the calls share subscription limits
|
|
247
|
+
futures = {p: pool.submit(ask_claude_code, user, model, system=SYSTEM,
|
|
248
|
+
parse=lambda t: parse_json(t).get("recommendations", []))
|
|
249
|
+
for p, (_, user) in prompts.items()}
|
|
250
|
+
recs, tokens_in, tokens_out, cost, failed = [], 0, 0, 0.0, 0
|
|
251
|
+
for p, fut in futures.items():
|
|
252
|
+
try:
|
|
253
|
+
raw, usage = fut.result()
|
|
254
|
+
except ClaudeCodeError as e:
|
|
255
|
+
# Keep the run whole: a group the model failed on still gets one idea per finding.
|
|
256
|
+
print(f"{short_name(p) if p else 'across projects'}: failed: {e}")
|
|
257
|
+
raw, usage, failed = [], {"in": 0, "out": 0, "cost": 0}, failed + 1
|
|
258
|
+
if failed == len(futures):
|
|
259
|
+
sys.exit("failed: no call succeeded; recommendations left as they were")
|
|
260
|
+
recs += check(raw, prompts[p][0])
|
|
261
|
+
tokens_in, tokens_out, cost = tokens_in + (usage["in"] or 0), tokens_out + (usage["out"] or 0), cost + usage["cost"]
|
|
262
|
+
save(conn, run_id, recs)
|
|
263
|
+
print(f"{len(recs)} recommendations saved for run {run_id} ({tokens_in} in / {tokens_out} out tokens, "
|
|
264
|
+
f"~${cost:.4f} API-equivalent)")
|
|
265
|
+
for r in recs:
|
|
266
|
+
where = f" {short_name(r['project'])}" if r["project"] else ""
|
|
267
|
+
print(f"{r['priority']:>2}. [{r['lever']}/{r.get('effort')}{where}] {r['title']} <- {r['finding_ids']}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
if __name__ == "__main__":
|
|
271
|
+
main()
|
debrief/relay.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Spot human messages that mostly relay a bot's output: review findings, trail comments, CI logs.
|
|
2
|
+
|
|
3
|
+
A person who pastes a review bot's comment or a CI log isn't correcting the agent themselves;
|
|
4
|
+
the agent could have fetched it. is_relayed(text) is True when a paste starts the message, or
|
|
5
|
+
follows a short lead-in, or carries a signature only a bot writes. A person's own words that
|
|
6
|
+
mention a finding ("there is a finding on the trail") don't count. A miss costs little and a
|
|
7
|
+
false hit hides a real correction, so every pattern here must be one people don't type.
|
|
8
|
+
|
|
9
|
+
>>> is_relayed("Entire Trail review comment\\nID: 019ff135\\nSeverity: medium")
|
|
10
|
+
True
|
|
11
|
+
>>> is_relayed("I've got this issue:\\n - [P2] Pick a placement — cmd/x.go:141\\n mirrorReadCluster ...")
|
|
12
|
+
True
|
|
13
|
+
>>> is_relayed("tests are failing and you dismissed a trail finding")
|
|
14
|
+
False
|
|
15
|
+
>>> is_relayed("Copilot reviewed 3 out of 4 changed files in this pull request and generated 2 comments.")
|
|
16
|
+
True
|
|
17
|
+
>>> is_relayed("copilot reviewed this and says the retry loop is wrong, is it?")
|
|
18
|
+
False
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import re
|
|
22
|
+
|
|
23
|
+
# Lines a bot's finding starts with, after any bullet or list number.
|
|
24
|
+
BULLET = r"^[ \t]*(?:\d+\.|[-•*▎])?[ \t]*"
|
|
25
|
+
START = re.compile("|".join([
|
|
26
|
+
BULLET + r"Entire Trail review comment\b",
|
|
27
|
+
BULLET + r"#{1,4}[ \t]*(?:critical|high|medium|low)[ \t]+(?:severity[ \t]+)?finding\b",
|
|
28
|
+
BULLET + r"\[P[0-3]\]", # Codex review: [P2] Title — file.go:12
|
|
29
|
+
BULLET + r"P[0-3][ \t]*(?:—|–|:|[ \t]-[ \t]|[ \t]+\S+\.\w+:\d)", # P2 — Title, P1 file.go:480
|
|
30
|
+
r"^[ \t]*Review comment:[ \t]*$", # Codex review header
|
|
31
|
+
r"^[ \t]*\**Location:?\**:?[ \t]*`?[\w./-]+\.\w+:\d+", # **Location:** `file.go:163`
|
|
32
|
+
r"^[ \t]*In `@[\w./-]+` around lines? \d+", # CodeRabbit "Prompt for AI Agents"
|
|
33
|
+
]), re.IGNORECASE | re.MULTILINE)
|
|
34
|
+
|
|
35
|
+
# Text only a bot writes; relayed wherever it appears.
|
|
36
|
+
SIGNATURE = re.compile("|".join([ # noqa: FLY002 (one pattern per line, each with its source)
|
|
37
|
+
r"^Suggested changeset \d", # GitHub UI, copied with a Copilot comment
|
|
38
|
+
r"^This issue also appears (?:on|in) line",
|
|
39
|
+
r"^Copilot reviewed \d+ out of \d+ changed files", # Copilot review summary
|
|
40
|
+
r"auto-generated (?:comment|reply) by CodeRabbit",
|
|
41
|
+
r"^_?⚠️ Potential issue_?(?:[ \t]*\||[ \t]*$)", # CodeRabbit comment header
|
|
42
|
+
r"^(?:<summary>)?🤖 Prompt for AI Agents",
|
|
43
|
+
r"<!-- (?:BUGBOT_BUG_ID|DESCRIPTION START)\b", # Cursor Bugbot comment markup
|
|
44
|
+
r"^\*\*(?:Low|Medium|High|Critical) Severity\*\*[ \t]*$", # Bugbot severity line
|
|
45
|
+
r"\bSentry observed the following issues:", # Sentry "Suspect Issues" PR comment
|
|
46
|
+
r"\bSeverity: (?:CRITICAL|HIGH|MEDIUM|LOW) \| Confidence: \d", # Sentry bug prediction footer
|
|
47
|
+
]), re.MULTILINE)
|
|
48
|
+
|
|
49
|
+
# Raw GitHub Actions log lines: a 7-digit timestamp prefix or a ##[error] marker. Three of
|
|
50
|
+
# them make a pasted log, with or without a lead-in.
|
|
51
|
+
ACTIONS_LOG = re.compile(r"^(?:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{7}Z |##\[(?:error|group|endgroup|warning)\])",
|
|
52
|
+
re.MULTILINE)
|
|
53
|
+
|
|
54
|
+
# A CI log with a one-line lead-in: "The tests is failing in CI:\nok github.com/..."
|
|
55
|
+
# The lead-in must name CI and three lines must look like test-runner output: both, because
|
|
56
|
+
# either alone is something people write.
|
|
57
|
+
CI_LEAD = re.compile(r"\A[^\n]{0,80}\b(?:CI|pipeline|GitHub Actions)\b[^\n]{0,40}:[ \t]*\n", re.IGNORECASE)
|
|
58
|
+
CI_LOG = re.compile("|".join([ # noqa: FLY002 (one pattern per line, each with its source)
|
|
59
|
+
r"^(?:ok|FAIL|PASS|---|===)[ \t]", # go test, jest
|
|
60
|
+
r"^(?:FAILED|ERROR|PASSED) \S+(?:::|\.py)", # pytest summary lines
|
|
61
|
+
r"^E {3,}\S", # pytest assertion detail
|
|
62
|
+
r"^test \S+ \.\.\. (?:ok|FAILED|ignored)", # cargo test
|
|
63
|
+
r"^error(?:\[E\d+\])?: ", # rustc, many compilers
|
|
64
|
+
r"^npm (?:ERR!|error) ",
|
|
65
|
+
r"^[ \t]*[✓✕✗×●] ", # jest, vitest, mocha
|
|
66
|
+
]), re.MULTILINE)
|
|
67
|
+
|
|
68
|
+
PASTE_TAG = re.compile(r"</?pasted_content[^>]*>")
|
|
69
|
+
LEAD_MAX = 300 # chars of the person's own words before the paste
|
|
70
|
+
LEAD_SHARE = 0.35 # ... and at most this share of the message
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def is_relayed(text: str | None) -> bool:
|
|
74
|
+
"""True when the message is mostly pasted bot, review or CI output."""
|
|
75
|
+
text = PASTE_TAG.sub("", text or "").strip()
|
|
76
|
+
if not text:
|
|
77
|
+
return False
|
|
78
|
+
if SIGNATURE.search(text) or len(ACTIONS_LOG.findall(text)) >= 3:
|
|
79
|
+
return True
|
|
80
|
+
if CI_LEAD.match(text) and len(CI_LOG.findall(text)) >= 3:
|
|
81
|
+
return True
|
|
82
|
+
m = START.search(text)
|
|
83
|
+
return bool(m) and m.start() <= LEAD_MAX and m.start() <= LEAD_SHARE * len(text)
|
debrief/runner.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Run `debrief sync` as a child of `debrief open`: one at a time, watched, and stoppable.
|
|
2
|
+
|
|
3
|
+
The child leads its own process group, so stopping the group also stops the `claude -p`
|
|
4
|
+
calls it started. Its output goes to DEBRIEF_HOME/run.log, one file, overwritten each run.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import signal
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
from collections import deque
|
|
15
|
+
from datetime import UTC, datetime
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from debrief import config
|
|
19
|
+
from debrief.paths import HOME
|
|
20
|
+
|
|
21
|
+
SYNC = [sys.executable, "-u", "-m", f"{__package__}.cli", "sync"] # -u: a pipe would hold prints back
|
|
22
|
+
EXPORT = [sys.executable, "-m", f"{__package__}.cli", "export"]
|
|
23
|
+
# Only the lines `sync` prints: synthesize has "== <project>: ..." headings of its own.
|
|
24
|
+
STAGE = re.compile(r"== (ingest|github|label|synthesize|recommend|export)(?: |$)")
|
|
25
|
+
COUNT = re.compile(r"\s*(\d+)/(\d+) done")
|
|
26
|
+
WOULD = re.compile(r"would label (\d+) items in (\d+) .*? calls(?:.*about (\d+) min)?")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def progress(lines, before: dict | None = None) -> dict:
|
|
30
|
+
"""Where a run is, from its output: {stage, done, total}. Lines it does not know change nothing."""
|
|
31
|
+
at = {"stage": None, "done": None, "total": None} | (before or {})
|
|
32
|
+
for line in lines:
|
|
33
|
+
if m := STAGE.match(line):
|
|
34
|
+
at = {"stage": m[1], "done": None, "total": None}
|
|
35
|
+
elif m := COUNT.match(line):
|
|
36
|
+
at |= {"done": int(m[1]), "total": int(m[2])}
|
|
37
|
+
return at
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def estimate_of(lines) -> dict:
|
|
41
|
+
"""The counts `sync --dry-run` printed; None for what it did not say."""
|
|
42
|
+
m = next((m for line in lines if (m := WOULD.match(line))), None)
|
|
43
|
+
items, calls, minutes = (int(x) if x else None for x in m.groups()) if m else (None, None, None)
|
|
44
|
+
return {"items": items, "calls": calls, "minutes": minutes}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def stop_group(pid: int, wait: float = 5.0) -> None:
|
|
48
|
+
"""Ask the process group to stop; kill what is left after `wait` seconds."""
|
|
49
|
+
for sig in (signal.SIGTERM, signal.SIGKILL):
|
|
50
|
+
end = time.monotonic() + wait
|
|
51
|
+
while time.monotonic() < end:
|
|
52
|
+
try:
|
|
53
|
+
os.killpg(pid, sig)
|
|
54
|
+
except (ProcessLookupError, PermissionError): # gone; macOS says EPERM for a group of zombies
|
|
55
|
+
return
|
|
56
|
+
sig = 0 # from here on only ask whether anyone is left
|
|
57
|
+
time.sleep(0.05)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def now() -> str:
|
|
61
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Runner:
|
|
65
|
+
cmd = SYNC # tests put a fake child here; nothing from HTTP reaches it
|
|
66
|
+
export = EXPORT
|
|
67
|
+
|
|
68
|
+
def __init__(self, home: Path = HOME, hide: tuple[str, ...] = ()):
|
|
69
|
+
self.log, self.pidfile, self.hide = home / "run.log", home / "run.pid", hide
|
|
70
|
+
self.lock = threading.Lock()
|
|
71
|
+
self.proc: subprocess.Popen | None = None
|
|
72
|
+
self.busy = False # a run or an estimate is going
|
|
73
|
+
self.cancelled = False
|
|
74
|
+
self.tail: deque[str] = deque(maxlen=30)
|
|
75
|
+
self.state = {"state": "idle", "started": None, "ended": None, "exit_code": None} | progress([])
|
|
76
|
+
|
|
77
|
+
def status(self) -> dict:
|
|
78
|
+
with self.lock:
|
|
79
|
+
return self.state | {"log": list(self.tail)}
|
|
80
|
+
|
|
81
|
+
def scrub(self, text: str) -> str:
|
|
82
|
+
"""Blank the page token and every OpenRouter key a stage could see."""
|
|
83
|
+
keys = [src.get(config.KEY) for src in
|
|
84
|
+
(os.environ, config.dotenv_values(config.ENV_PATH), config.dotenv_values(config.find_dotenv()))]
|
|
85
|
+
for secret in (*self.hide, *keys):
|
|
86
|
+
if secret:
|
|
87
|
+
text = text.replace(secret, "[redacted]")
|
|
88
|
+
return text
|
|
89
|
+
|
|
90
|
+
def stop_orphan(self) -> None:
|
|
91
|
+
"""Stop a run whose server died: it would go on calling the model with nobody watching."""
|
|
92
|
+
try:
|
|
93
|
+
pid = int(self.pidfile.read_text())
|
|
94
|
+
except (OSError, ValueError):
|
|
95
|
+
return
|
|
96
|
+
ps = subprocess.run(["ps", "-ww", "-o", "command=", "-p", str(pid)], capture_output=True, text=True, check=False).stdout
|
|
97
|
+
if " ".join(self.cmd[1:]) in ps: # pids get reused: stop only what is this command
|
|
98
|
+
stop_group(pid)
|
|
99
|
+
self.pidfile.unlink(missing_ok=True)
|
|
100
|
+
|
|
101
|
+
def start(self) -> bool:
|
|
102
|
+
"""Start a run. False when a run or an estimate is going."""
|
|
103
|
+
with self.lock:
|
|
104
|
+
if self.busy:
|
|
105
|
+
return False
|
|
106
|
+
self.stop_orphan()
|
|
107
|
+
self.log.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
self.proc = subprocess.Popen(self.cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
109
|
+
stderr=subprocess.STDOUT, text=True, errors="replace",
|
|
110
|
+
start_new_session=True)
|
|
111
|
+
self.pidfile.write_text(str(self.proc.pid))
|
|
112
|
+
self.busy, self.cancelled = True, False
|
|
113
|
+
self.tail.clear()
|
|
114
|
+
self.state = {"state": "running", "started": now(), "ended": None, "exit_code": None} | progress([])
|
|
115
|
+
self.reader = threading.Thread(target=self.read, args=(self.proc,), daemon=True)
|
|
116
|
+
self.reader.start()
|
|
117
|
+
return True
|
|
118
|
+
|
|
119
|
+
def read(self, proc: subprocess.Popen) -> None:
|
|
120
|
+
with open(self.log, "w") as log:
|
|
121
|
+
for line in proc.stdout:
|
|
122
|
+
line = self.scrub(line.rstrip("\n"))
|
|
123
|
+
log.write(line + "\n")
|
|
124
|
+
log.flush()
|
|
125
|
+
with self.lock:
|
|
126
|
+
self.tail.append(line)
|
|
127
|
+
self.state |= progress([line], {k: self.state[k] for k in ("stage", "done", "total")})
|
|
128
|
+
code = proc.wait()
|
|
129
|
+
with self.lock:
|
|
130
|
+
end = "cancelled" if self.cancelled else "failed" if code else "done"
|
|
131
|
+
self.state |= {"state": end, "ended": now(), "exit_code": code}
|
|
132
|
+
self.pidfile.unlink(missing_ok=True)
|
|
133
|
+
self.busy = False
|
|
134
|
+
|
|
135
|
+
def cancel(self) -> None:
|
|
136
|
+
"""Stop the run and everything it started, and wait until it is gone. Nothing to do when idle."""
|
|
137
|
+
with self.lock:
|
|
138
|
+
proc = self.proc
|
|
139
|
+
if not proc or proc.poll() is not None:
|
|
140
|
+
return
|
|
141
|
+
self.cancelled = True
|
|
142
|
+
stop_group(proc.pid)
|
|
143
|
+
self.reader.join(10)
|
|
144
|
+
|
|
145
|
+
def estimate(self) -> dict | None:
|
|
146
|
+
"""Run `sync --dry-run`, then export so the page matches what was ingested. None when busy."""
|
|
147
|
+
with self.lock:
|
|
148
|
+
if self.busy:
|
|
149
|
+
return None
|
|
150
|
+
self.busy = True
|
|
151
|
+
try:
|
|
152
|
+
lines, code = [], 0
|
|
153
|
+
for cmd in ([*self.cmd, "--dry-run"], self.export):
|
|
154
|
+
r = subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
155
|
+
text=True, errors="replace", check=False)
|
|
156
|
+
lines += self.scrub(r.stdout).splitlines()
|
|
157
|
+
code = r.returncode
|
|
158
|
+
if code:
|
|
159
|
+
break
|
|
160
|
+
return {"exit_code": code, **estimate_of(lines), "lines": lines}
|
|
161
|
+
finally:
|
|
162
|
+
with self.lock:
|
|
163
|
+
self.busy = False
|