debrief-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- debrief/__init__.py +0 -0
- debrief/build_artifact.py +83 -0
- debrief/classify.py +812 -0
- debrief/cli.py +291 -0
- debrief/compare_definitions.py +147 -0
- debrief/compare_llm.py +250 -0
- debrief/config.py +204 -0
- debrief/db.py +341 -0
- debrief/export.py +156 -0
- debrief/export_site.py +661 -0
- debrief/github.py +540 -0
- debrief/normalize.py +635 -0
- debrief/paths.py +17 -0
- debrief/prune.py +117 -0
- debrief/recommend.py +271 -0
- debrief/relay.py +83 -0
- debrief/runner.py +163 -0
- debrief/scrape.py +208 -0
- debrief/site/app.js +1613 -0
- debrief/site/index.html +32 -0
- debrief/site/method.js +117 -0
- debrief/site/style.css +520 -0
- debrief/skills.py +107 -0
- debrief/sources.py +341 -0
- debrief/synthesize.py +1052 -0
- debrief/verify_evidence.py +154 -0
- debrief_cli-0.1.0.dist-info/METADATA +205 -0
- debrief_cli-0.1.0.dist-info/RECORD +31 -0
- debrief_cli-0.1.0.dist-info/WHEEL +4 -0
- debrief_cli-0.1.0.dist-info/entry_points.txt +2 -0
- debrief_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
debrief/db.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""SQLite schema shared by every stage. One file: debrief.db."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from debrief.paths import DB_PATH
|
|
7
|
+
|
|
8
|
+
SCHEMA = """
|
|
9
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
10
|
+
session_id TEXT PRIMARY KEY,
|
|
11
|
+
agent TEXT, -- Claude Code | Codex | Pi | ...
|
|
12
|
+
model TEXT,
|
|
13
|
+
display_name TEXT,
|
|
14
|
+
author TEXT,
|
|
15
|
+
started_at TEXT,
|
|
16
|
+
last_activity_at TEXT,
|
|
17
|
+
checkpoint_count INTEGER,
|
|
18
|
+
input_tokens INTEGER,
|
|
19
|
+
output_tokens INTEGER,
|
|
20
|
+
agent_lines INTEGER, -- committed lines written by the agent
|
|
21
|
+
human_lines INTEGER, -- committed lines added or changed by a person
|
|
22
|
+
prompt TEXT -- opening prompt
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
CREATE TABLE IF NOT EXISTS checkpoints (
|
|
26
|
+
checkpoint_id TEXT PRIMARY KEY,
|
|
27
|
+
session_id TEXT NOT NULL,
|
|
28
|
+
commit_sha TEXT,
|
|
29
|
+
message TEXT,
|
|
30
|
+
additions INTEGER,
|
|
31
|
+
deletions INTEGER,
|
|
32
|
+
files_changed INTEGER,
|
|
33
|
+
ts TEXT
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
-- One row per message, in order. Tool results that agents log as "user"
|
|
37
|
+
-- messages are folded into tool_calls, not stored here.
|
|
38
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
39
|
+
session_id TEXT NOT NULL,
|
|
40
|
+
idx INTEGER NOT NULL, -- 0-based position within the session
|
|
41
|
+
role TEXT NOT NULL, -- user | assistant
|
|
42
|
+
user_kind TEXT, -- human | injected (skill/command/hook text) ; NULL for assistant
|
|
43
|
+
ts TEXT,
|
|
44
|
+
text TEXT, -- visible text; pasted blocks kept inline
|
|
45
|
+
checkpoint_id TEXT, -- the checkpoint this message falls before (NULL after the last)
|
|
46
|
+
PRIMARY KEY (session_id, idx)
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
CREATE TABLE IF NOT EXISTS tool_calls (
|
|
50
|
+
session_id TEXT NOT NULL,
|
|
51
|
+
msg_idx INTEGER NOT NULL, -- messages.idx of the assistant message that made the call
|
|
52
|
+
call_idx INTEGER NOT NULL,
|
|
53
|
+
name TEXT, -- Bash | Edit | Read | apply_patch | ...
|
|
54
|
+
input TEXT, -- JSON
|
|
55
|
+
output TEXT, -- truncated to ~4 KB
|
|
56
|
+
is_error INTEGER, -- 1 when the call failed (non-zero exit, tool error)
|
|
57
|
+
files TEXT, -- JSON array of repo-relative paths the call read or wrote
|
|
58
|
+
writes INTEGER, -- 1 when the call changed files
|
|
59
|
+
PRIMARY KEY (session_id, msg_idx, call_idx)
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
-- Labels are long-format so the classifier schema can change without migrations.
|
|
63
|
+
CREATE TABLE IF NOT EXISTS turn_labels (
|
|
64
|
+
session_id TEXT NOT NULL,
|
|
65
|
+
msg_idx INTEGER NOT NULL, -- the human user message being labeled
|
|
66
|
+
field TEXT NOT NULL,
|
|
67
|
+
value TEXT,
|
|
68
|
+
confidence REAL, -- probability of the chosen value
|
|
69
|
+
probs TEXT, -- JSON {value: probability} when Jev returns a distribution
|
|
70
|
+
model TEXT,
|
|
71
|
+
input_hash TEXT,
|
|
72
|
+
created_at TEXT,
|
|
73
|
+
PRIMARY KEY (session_id, msg_idx, field)
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
CREATE TABLE IF NOT EXISTS session_labels (
|
|
77
|
+
session_id TEXT NOT NULL,
|
|
78
|
+
field TEXT NOT NULL,
|
|
79
|
+
value TEXT,
|
|
80
|
+
confidence REAL, -- probability of the chosen value
|
|
81
|
+
probs TEXT, -- JSON {value: probability} when Jev returns a distribution
|
|
82
|
+
model TEXT,
|
|
83
|
+
input_hash TEXT,
|
|
84
|
+
created_at TEXT,
|
|
85
|
+
PRIMARY KEY (session_id, field)
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
-- One row per log file debrief.sources has looked at, loaded or not, so a re-run can skip
|
|
89
|
+
-- files that have not changed.
|
|
90
|
+
CREATE TABLE IF NOT EXISTS source_logs (
|
|
91
|
+
path TEXT PRIMARY KEY,
|
|
92
|
+
session_id TEXT,
|
|
93
|
+
agent TEXT,
|
|
94
|
+
size INTEGER,
|
|
95
|
+
mtime INTEGER, -- st_mtime_ns
|
|
96
|
+
status TEXT, -- loaded, or why the file was skipped
|
|
97
|
+
cwd TEXT, -- as the log records it, for the --project filter
|
|
98
|
+
dropped INTEGER -- entries left out because another session owns them
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
-- Which sessions' logs hold each entry. A resumed or forked session's log starts with a
|
|
102
|
+
-- copy of the earlier one; the copies load only into the session that owns them.
|
|
103
|
+
CREATE TABLE IF NOT EXISTS source_entries (
|
|
104
|
+
entry_key TEXT NOT NULL, -- the entry's uuid, or the format's nearest equivalent
|
|
105
|
+
session_id TEXT NOT NULL,
|
|
106
|
+
native INTEGER NOT NULL, -- 1 when the entry names this session as its own
|
|
107
|
+
PRIMARY KEY (entry_key, session_id)
|
|
108
|
+
) WITHOUT ROWID;
|
|
109
|
+
CREATE INDEX IF NOT EXISTS source_entries_session ON source_entries (session_id);
|
|
110
|
+
|
|
111
|
+
-- PRs a local session opened or pushed to, read from its own gh and git output (github.py).
|
|
112
|
+
-- Entire sessions link through checkpoints instead.
|
|
113
|
+
CREATE TABLE IF NOT EXISTS session_prs (
|
|
114
|
+
session_id TEXT NOT NULL,
|
|
115
|
+
repo TEXT NOT NULL, -- owner/name
|
|
116
|
+
pr_number INTEGER NOT NULL,
|
|
117
|
+
source TEXT, -- url (printed by gh pr create) | branch (a branch the session pushed)
|
|
118
|
+
PRIMARY KEY (session_id, repo, pr_number)
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
CREATE TABLE IF NOT EXISTS gh_prs (
|
|
122
|
+
repo TEXT NOT NULL, -- owner/name
|
|
123
|
+
pr_number INTEGER NOT NULL,
|
|
124
|
+
title TEXT,
|
|
125
|
+
state TEXT,
|
|
126
|
+
merged INTEGER,
|
|
127
|
+
author TEXT,
|
|
128
|
+
head_sha TEXT,
|
|
129
|
+
created_at TEXT,
|
|
130
|
+
url TEXT,
|
|
131
|
+
merged_at TEXT,
|
|
132
|
+
closed_at TEXT,
|
|
133
|
+
PRIMARY KEY (repo, pr_number)
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
CREATE TABLE IF NOT EXISTS gh_commit_prs (
|
|
137
|
+
commit_sha TEXT NOT NULL,
|
|
138
|
+
repo TEXT NOT NULL,
|
|
139
|
+
pr_number INTEGER NOT NULL,
|
|
140
|
+
committed_at TEXT, -- set for commits listed from a local session's PR
|
|
141
|
+
PRIMARY KEY (commit_sha, repo, pr_number)
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
-- A PR is settled once merged or closed. Outcome, review and CI analyses should
|
|
145
|
+
-- use only settled PRs; turn-level analyses don't need to.
|
|
146
|
+
CREATE VIEW IF NOT EXISTS settled_prs AS
|
|
147
|
+
SELECT * FROM gh_prs WHERE merged = 1 OR state = 'closed';
|
|
148
|
+
|
|
149
|
+
-- The PRs each session worked on.
|
|
150
|
+
CREATE VIEW IF NOT EXISTS session_pr_links AS
|
|
151
|
+
SELECT session_id, repo, pr_number FROM session_prs
|
|
152
|
+
UNION
|
|
153
|
+
SELECT cp.session_id, g.repo, g.pr_number FROM checkpoints cp JOIN gh_commit_prs g ON g.commit_sha = cp.commit_sha;
|
|
154
|
+
|
|
155
|
+
-- The commits whose checks count for each session: its checkpoints, or the commits of its
|
|
156
|
+
-- PRs made while it ran. Two sessions on one PR each get only their own commits.
|
|
157
|
+
CREATE VIEW IF NOT EXISTS session_commits AS
|
|
158
|
+
SELECT session_id, commit_sha FROM checkpoints WHERE commit_sha IS NOT NULL
|
|
159
|
+
UNION
|
|
160
|
+
SELECT sp.session_id, g.commit_sha FROM session_prs sp
|
|
161
|
+
JOIN sessions s ON s.session_id = sp.session_id
|
|
162
|
+
JOIN gh_commit_prs g ON g.repo = sp.repo AND g.pr_number = sp.pr_number
|
|
163
|
+
WHERE datetime(g.committed_at) BETWEEN datetime(s.started_at, '-10 minutes')
|
|
164
|
+
AND datetime(s.last_activity_at, '+10 minutes');
|
|
165
|
+
|
|
166
|
+
CREATE TABLE IF NOT EXISTS gh_checks (
|
|
167
|
+
check_id INTEGER PRIMARY KEY,
|
|
168
|
+
commit_sha TEXT NOT NULL,
|
|
169
|
+
name TEXT,
|
|
170
|
+
conclusion TEXT, -- success | failure | skipped | ...
|
|
171
|
+
app TEXT,
|
|
172
|
+
run_id INTEGER, -- Actions run, when there is one
|
|
173
|
+
started_at TEXT,
|
|
174
|
+
completed_at TEXT,
|
|
175
|
+
failure_log TEXT, -- tail of the failed step's log, failures only
|
|
176
|
+
repo TEXT
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
CREATE TABLE IF NOT EXISTS gh_review_comments (
|
|
180
|
+
comment_id INTEGER PRIMARY KEY,
|
|
181
|
+
pr_number INTEGER NOT NULL,
|
|
182
|
+
author TEXT,
|
|
183
|
+
is_bot INTEGER,
|
|
184
|
+
path TEXT,
|
|
185
|
+
line INTEGER,
|
|
186
|
+
body TEXT,
|
|
187
|
+
in_reply_to INTEGER,
|
|
188
|
+
created_at TEXT,
|
|
189
|
+
repo TEXT
|
|
190
|
+
);
|
|
191
|
+
|
|
192
|
+
CREATE TABLE IF NOT EXISTS event_labels (
|
|
193
|
+
event_type TEXT NOT NULL, -- review_comment | check_failure
|
|
194
|
+
event_id INTEGER NOT NULL, -- comment_id or check_id
|
|
195
|
+
field TEXT NOT NULL,
|
|
196
|
+
value TEXT,
|
|
197
|
+
confidence REAL, -- probability of the chosen value
|
|
198
|
+
probs TEXT, -- JSON {value: probability} when Jev returns a distribution
|
|
199
|
+
model TEXT,
|
|
200
|
+
input_hash TEXT,
|
|
201
|
+
created_at TEXT,
|
|
202
|
+
PRIMARY KEY (event_type, event_id, field)
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
CREATE TABLE IF NOT EXISTS findings (
|
|
206
|
+
finding_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
207
|
+
run_id TEXT,
|
|
208
|
+
kind TEXT, -- hotspot | knowledge_gap | found_not_filed | correction_pattern | prompt_practice | ci | review
|
|
209
|
+
title TEXT,
|
|
210
|
+
detail TEXT,
|
|
211
|
+
suggested_fix TEXT,
|
|
212
|
+
evidence TEXT, -- JSON array of {session_id, msg_idx} or {event_type, event_id}
|
|
213
|
+
created_at TEXT
|
|
214
|
+
-- project (added below): the folder the finding is about; NULL when it is about all picked projects
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
-- When synthesize last finished each project's pass (it may have found nothing).
|
|
218
|
+
CREATE TABLE IF NOT EXISTS project_runs (
|
|
219
|
+
project TEXT PRIMARY KEY, -- as in findings.project
|
|
220
|
+
run_id TEXT,
|
|
221
|
+
analysed_at TEXT
|
|
222
|
+
);
|
|
223
|
+
|
|
224
|
+
-- Findings from one run merged by the lever that fixes them (recommend.py).
|
|
225
|
+
CREATE TABLE IF NOT EXISTS recommendations (
|
|
226
|
+
rec_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
227
|
+
run_id TEXT, -- the findings run these were built from
|
|
228
|
+
priority INTEGER, -- 1 = do first
|
|
229
|
+
lever TEXT, -- agent_config | test_harness | ci_infra | review_tooling | team_practice | backlog
|
|
230
|
+
title TEXT,
|
|
231
|
+
problem TEXT,
|
|
232
|
+
change TEXT, -- what to do, markdown
|
|
233
|
+
target TEXT, -- file or system the change lands in
|
|
234
|
+
effort TEXT, -- S | M | L
|
|
235
|
+
impact TEXT,
|
|
236
|
+
finding_ids TEXT, -- JSON array of findings.finding_id
|
|
237
|
+
created_at TEXT
|
|
238
|
+
);
|
|
239
|
+
|
|
240
|
+
-- Whether each item an idea cites supports that idea (verify_evidence.py).
|
|
241
|
+
CREATE TABLE IF NOT EXISTS evidence_checks (
|
|
242
|
+
rec_id INTEGER, -- recommendations.rec_id
|
|
243
|
+
ref TEXT, -- m:<session_id>:<msg_idx> or e:<event_type>:<event_id>, as in export_site.py
|
|
244
|
+
supports INTEGER, -- 1 = supports the idea's claim
|
|
245
|
+
reason TEXT,
|
|
246
|
+
model TEXT,
|
|
247
|
+
created_at TEXT,
|
|
248
|
+
PRIMARY KEY (rec_id, ref)
|
|
249
|
+
);
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# Columns added after tables existed. CREATE TABLE IF NOT EXISTS skips existing
|
|
254
|
+
# tables, so connect() adds any that are missing.
|
|
255
|
+
ADDED_COLUMNS = [
|
|
256
|
+
("sessions", "cwd", "TEXT"),
|
|
257
|
+
# GitHub owner/name of the checkout, when the log itself records it (Codex); '' when it does
|
|
258
|
+
# not, NULL in rows from before this column.
|
|
259
|
+
("sessions", "remote", "TEXT"),
|
|
260
|
+
# What the headless rule needs, kept so a re-run can apply it to a log without reading it.
|
|
261
|
+
("source_logs", "launcher", "TEXT"), # as logged: Claude entrypoints, Codex "originator|source"; NULL in rows from before this column
|
|
262
|
+
("source_logs", "human_turns", "INTEGER"), # after copied entries are left out; NULL when the log was not parsed
|
|
263
|
+
("source_logs", "parser", "INTEGER"), # normalize.PARSER_VERSION the log was last read with
|
|
264
|
+
("tool_calls", "exit_code", "INTEGER"), # shell exit code when the format records one
|
|
265
|
+
("gh_review_comments", "kind", "TEXT"), # inline | review | issue (general PR discussion)
|
|
266
|
+
("gh_review_comments", "review_state", "TEXT"), # APPROVED | CHANGES_REQUESTED | COMMENTED | DISMISSED; review rows only
|
|
267
|
+
("gh_review_comments", "is_resolved", "INTEGER"), # thread resolved; inline rows only
|
|
268
|
+
("gh_review_comments", "is_outdated", "INTEGER"), # thread's code has since changed; inline rows only
|
|
269
|
+
# The project (see session_projects) a finding or idea is about; NULL when it is about all of them, and in old runs.
|
|
270
|
+
("findings", "project", "TEXT"),
|
|
271
|
+
("recommendations", "project", "TEXT"),
|
|
272
|
+
]
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def add_repo(conn: sqlite3.Connection) -> None:
|
|
276
|
+
"""Move a database from before the gh_* tables named their repo. It held one repo, read
|
|
277
|
+
here from a PR's URL. SQLite cannot change a primary key, so gh_prs and gh_commit_prs are
|
|
278
|
+
rebuilt; SCHEMA then creates the new tables and the rows are copied in. One transaction."""
|
|
279
|
+
old = {t: [r["name"] for r in conn.execute(f"PRAGMA table_info({t})")] for t in ("gh_prs", "gh_commit_prs")}
|
|
280
|
+
if not old["gh_prs"] or "repo" in old["gh_prs"]:
|
|
281
|
+
return
|
|
282
|
+
url = conn.execute("SELECT url FROM gh_prs WHERE url LIKE 'https://github.com/%/%/pull/%'").fetchone()
|
|
283
|
+
repo = "/".join(url[0].split("/")[3:5]) if url else ""
|
|
284
|
+
create = [stmt for stmt in SCHEMA.split(";") if "CREATE TABLE IF NOT EXISTS gh_prs" in stmt
|
|
285
|
+
or "CREATE TABLE IF NOT EXISTS gh_commit_prs" in stmt]
|
|
286
|
+
conn.execute("BEGIN")
|
|
287
|
+
with conn:
|
|
288
|
+
conn.execute("DROP VIEW IF EXISTS settled_prs") # reads gh_prs; SCHEMA makes it again
|
|
289
|
+
for table in old:
|
|
290
|
+
conn.execute(f"ALTER TABLE {table} RENAME TO {table}_old")
|
|
291
|
+
for stmt in create:
|
|
292
|
+
conn.execute(stmt)
|
|
293
|
+
for table, columns in old.items():
|
|
294
|
+
conn.execute(f"INSERT INTO {table} (repo, {', '.join(columns)}) SELECT ?, * FROM {table}_old", (repo,))
|
|
295
|
+
conn.execute(f"DROP TABLE {table}_old")
|
|
296
|
+
for table in ("gh_checks", "gh_review_comments"):
|
|
297
|
+
conn.execute(f"ALTER TABLE {table} ADD COLUMN repo TEXT")
|
|
298
|
+
conn.execute(f"UPDATE {table} SET repo = ?", (repo,))
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
# What ties a session to a GitHub repo: a PR it worked on, or the remote its own log records.
|
|
302
|
+
REPO_LINKS = """SELECT session_id, repo FROM session_pr_links
|
|
303
|
+
UNION SELECT session_id, remote FROM main.sessions WHERE remote != ''"""
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def repo_projects(conn: sqlite3.Connection) -> dict[str, str]:
|
|
307
|
+
"""Each GitHub repo's project: the folder most of the sessions tied to it (REPO_LINKS) ran in."""
|
|
308
|
+
rows = conn.execute(f"""
|
|
309
|
+
SELECT l.repo, s.cwd FROM main.sessions s JOIN ({REPO_LINKS}) l USING (session_id)
|
|
310
|
+
WHERE s.cwd IS NOT NULL GROUP BY l.repo, s.cwd ORDER BY COUNT(DISTINCT l.session_id), s.cwd DESC""")
|
|
311
|
+
return {repo: cwd for repo, cwd in rows} # the last row per repo has the most sessions
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def session_projects(conn: sqlite3.Connection) -> dict[str, str | None]:
|
|
315
|
+
"""session_id -> project, for every stage. A project is the folder the session ran in.
|
|
316
|
+
|
|
317
|
+
A session tied to a repo (REPO_LINKS) whose project is another folder belongs to that
|
|
318
|
+
project, unless its own folder is some repo's project: a worktree that ingest did not
|
|
319
|
+
resolve still lands in its repo this way.
|
|
320
|
+
"""
|
|
321
|
+
repos = repo_projects(conn)
|
|
322
|
+
out = dict(conn.execute("SELECT session_id, cwd FROM main.sessions"))
|
|
323
|
+
homes = set(repos.values())
|
|
324
|
+
for sid, repo in conn.execute(f"{REPO_LINKS} ORDER BY repo DESC"):
|
|
325
|
+
if out.get(sid) not in homes and repo in repos:
|
|
326
|
+
out[sid] = repos[repo]
|
|
327
|
+
return out
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def connect(path: Path = DB_PATH) -> sqlite3.Connection:
|
|
331
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
332
|
+
conn = sqlite3.connect(path, timeout=30)
|
|
333
|
+
conn.row_factory = sqlite3.Row
|
|
334
|
+
add_repo(conn)
|
|
335
|
+
conn.executescript(SCHEMA)
|
|
336
|
+
for table, column, type_ in ADDED_COLUMNS:
|
|
337
|
+
existing = {row["name"] for row in conn.execute(f"PRAGMA table_info({table})")}
|
|
338
|
+
if column not in existing:
|
|
339
|
+
conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {type_}")
|
|
340
|
+
conn.commit()
|
|
341
|
+
return conn
|
debrief/export.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""Export weak Jev questions as jev-align datasets, and read accepted definitions back.
|
|
2
|
+
|
|
3
|
+
Writes data/jevalign/datasets/<question>.jsonl with rows {"id", "state"}, where state is the exact
|
|
4
|
+
string classify.py sends to Jev, plus data/jevalign/<question>.sh with the matching
|
|
5
|
+
`jeva optimize` command. Reads debrief.db read-only.
|
|
6
|
+
|
|
7
|
+
uv run python -m debrief.export # write all files
|
|
8
|
+
uv run python -m debrief.export --import .jev-align/runs/<run-id> # print the accepted definition
|
|
9
|
+
|
|
10
|
+
The loop, per question:
|
|
11
|
+
1. uv run python -m debrief.export
|
|
12
|
+
2. bash data/jevalign/<question>.sh (needs jeva from the patched jev-align checkout:
|
|
13
|
+
uv tool install --editable ../jev-align). It starts an interactive jeva session.
|
|
14
|
+
3. Label the rows jeva picks; review GEPA's proposal and accept or reject it.
|
|
15
|
+
4. uv run python -m debrief.export --import .jev-align/runs/<run-id>
|
|
16
|
+
jeva writes each run to .jev-align/runs/<timestamp>-<slug>/. Its state.json holds
|
|
17
|
+
current_candidate, the definition last accepted (the seed until you accept one),
|
|
18
|
+
as {instructions, true_criteria, false_criteria} for binary questions or
|
|
19
|
+
{instructions, criteria: {name: description}} for multiclass ones. The command
|
|
20
|
+
prints it as a classify.py noul(...) or choice(...) call.
|
|
21
|
+
5. Paste that over the question in classify.py, then rerun
|
|
22
|
+
`uv run python -m debrief.classify --only turns` (or `--only events` for held_up); the changed
|
|
23
|
+
questions change input_hash, so those items get relabeled.
|
|
24
|
+
|
|
25
|
+
The scripts pass --raw-text with the single --column state, so jeva sends each row's
|
|
26
|
+
state to Jev as the same bare string classify.py sends.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import json
|
|
31
|
+
import shlex
|
|
32
|
+
import sqlite3
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
from debrief.classify import (
|
|
36
|
+
FOUND_NOT_FILED,
|
|
37
|
+
REVIEW_QUESTIONS,
|
|
38
|
+
TURN_QUESTIONS,
|
|
39
|
+
event_items,
|
|
40
|
+
session_items,
|
|
41
|
+
)
|
|
42
|
+
from debrief.db import DB_PATH
|
|
43
|
+
from debrief.paths import HOME
|
|
44
|
+
|
|
45
|
+
OUT = HOME / "jevalign"
|
|
46
|
+
# Datasets live apart from the run scripts: transcripts are full of backticks and $(...),
|
|
47
|
+
# so running a .jsonl by mistake with bash would execute them.
|
|
48
|
+
DATA = OUT / "datasets"
|
|
49
|
+
# GEPA rewrites definitions through headless Claude Code (Opus 5.5) on the Claude
|
|
50
|
+
# subscription; Jev evaluation still runs on OpenRouter.
|
|
51
|
+
REFLECTION_MODEL = "claude-code"
|
|
52
|
+
# GEPA metric calls per round (jeva defaults to 300); lower caps subscription use.
|
|
53
|
+
MAX_METRIC_CALLS = 100
|
|
54
|
+
|
|
55
|
+
# question name -> (definition, which item kinds carry it)
|
|
56
|
+
EXPORTS = {
|
|
57
|
+
"found_not_filed": (FOUND_NOT_FILED, ("turns", "tails")),
|
|
58
|
+
"agent_could_have_found_it": (TURN_QUESTIONS["agent_could_have_found_it"], ("turns",)),
|
|
59
|
+
"design_reversal": (TURN_QUESTIONS["design_reversal"], ("turns",)),
|
|
60
|
+
"human_caught_defect": (TURN_QUESTIONS["human_caught_defect"], ("turns",)),
|
|
61
|
+
"held_up": (REVIEW_QUESTIONS["held_up"], ("review_comments",)),
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def row_id(it: dict) -> str:
|
|
66
|
+
return ":".join(str(k) for k in it["key"])
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def optimize_command(name: str, question: dict, rows: int) -> str:
|
|
70
|
+
lines = [
|
|
71
|
+
["jeva", "optimize", f"{DATA}/{name}.jsonl"],
|
|
72
|
+
["--backend", "openrouter"],
|
|
73
|
+
["--column", "state"],
|
|
74
|
+
["--raw-text"],
|
|
75
|
+
["--reflection-model", REFLECTION_MODEL],
|
|
76
|
+
["--max-metric-calls", str(MAX_METRIC_CALLS)],
|
|
77
|
+
["--question", question["instructions"]],
|
|
78
|
+
]
|
|
79
|
+
if question["type"] == "noul":
|
|
80
|
+
lines += [["--true-criteria", question["criteria"]["true"]],
|
|
81
|
+
["--false-criteria", question["criteria"]["false"]]]
|
|
82
|
+
elif question["type"] == "choice":
|
|
83
|
+
lines += [["--class", f"{label}={description}"] for label, description in question["criteria"].items()]
|
|
84
|
+
else:
|
|
85
|
+
raise ValueError(f"{name}: {question['type']} questions are not exported")
|
|
86
|
+
lines += [["--holdout"], ["--pool-size", str(max(rows, 5))]]
|
|
87
|
+
return " \\\n ".join(shlex.join(line) for line in lines)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def write_script(name: str, question: dict, rows: int) -> Path:
|
|
91
|
+
path = OUT / f"{name}.sh"
|
|
92
|
+
path.write_text(
|
|
93
|
+
"#!/usr/bin/env bash\n"
|
|
94
|
+
"# Generated by export.py. Runs from the insights root so runs land in .jev-align/runs.\n"
|
|
95
|
+
"set -euo pipefail\n"
|
|
96
|
+
'cd "$(dirname "$0")/../.."\n'
|
|
97
|
+
"set -a; . ./.env; set +a\n"
|
|
98
|
+
f"{optimize_command(name, question, rows)}\n"
|
|
99
|
+
)
|
|
100
|
+
path.chmod(0o755)
|
|
101
|
+
return path
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def export() -> None:
|
|
105
|
+
conn = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30)
|
|
106
|
+
conn.row_factory = sqlite3.Row
|
|
107
|
+
turns, tails, _ = session_items(conn, None)
|
|
108
|
+
reviews, _ = event_items(conn)
|
|
109
|
+
kinds = {"turns": turns, "tails": tails, "review_comments": reviews}
|
|
110
|
+
|
|
111
|
+
DATA.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
for name, (question, sources) in EXPORTS.items():
|
|
113
|
+
# Only items where classify.py asks this question; opening turns skip some.
|
|
114
|
+
items = [it for kind in sources for it in kinds[kind] if name in it["questions"]]
|
|
115
|
+
with (DATA / f"{name}.jsonl").open("w") as f:
|
|
116
|
+
for it in items:
|
|
117
|
+
f.write(json.dumps({"id": row_id(it), "state": it["state"]}, ensure_ascii=False) + "\n")
|
|
118
|
+
write_script(name, question, len(items))
|
|
119
|
+
print(f"{name}: {len(items)} rows -> {DATA / name}.jsonl, {OUT / name}.sh")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def import_definition(run_dir: str | Path) -> str:
|
|
123
|
+
"""Return the run's accepted definition as classify.py source."""
|
|
124
|
+
state = json.loads((Path(run_dir) / "state.json").read_text())
|
|
125
|
+
spec = state["current_candidate"]
|
|
126
|
+
if "true_criteria" in spec:
|
|
127
|
+
source = (f"noul(\n {spec['instructions']!r},\n {spec['true_criteria']!r},\n"
|
|
128
|
+
f" {spec['false_criteria']!r},\n)")
|
|
129
|
+
elif isinstance(spec.get("criteria"), dict):
|
|
130
|
+
lines = [f" {spec['instructions']!r},"]
|
|
131
|
+
lines += [f" {label}={description!r}," for label, description in spec["criteria"].items()]
|
|
132
|
+
source = "choice(\n" + "\n".join(lines) + "\n)"
|
|
133
|
+
else:
|
|
134
|
+
raise ValueError("only binary and multiclass definitions map to classify.py questions")
|
|
135
|
+
accepted = sum(1 for h in state.get("history", []) if h.get("decision") == "accepted")
|
|
136
|
+
note = f"# {accepted} accepted proposal(s) in {state['run_id']}"
|
|
137
|
+
if not accepted:
|
|
138
|
+
note += "; this is still the seed definition"
|
|
139
|
+
if state.get("pending_candidate"):
|
|
140
|
+
note += "; a proposal awaits review (resume the run to accept or reject it)"
|
|
141
|
+
return f"{note}\n{source}"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def main(argv: list[str] | None = None) -> None:
|
|
145
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
146
|
+
parser.add_argument("--import", dest="run_dir", metavar="RUN_DIR",
|
|
147
|
+
help="Print the accepted definition of a jeva run as classify.py source")
|
|
148
|
+
args = parser.parse_args(argv)
|
|
149
|
+
if args.run_dir:
|
|
150
|
+
print(import_definition(args.run_dir))
|
|
151
|
+
else:
|
|
152
|
+
export()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
if __name__ == "__main__":
|
|
156
|
+
main()
|