transcripto 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- transcripto-0.1.0.dist-info/METADATA +229 -0
- transcripto-0.1.0.dist-info/RECORD +7 -0
- transcripto-0.1.0.dist-info/WHEEL +5 -0
- transcripto-0.1.0.dist-info/entry_points.txt +2 -0
- transcripto-0.1.0.dist-info/licenses/LICENSE +21 -0
- transcripto-0.1.0.dist-info/top_level.txt +1 -0
- transcripto.py +1378 -0
transcripto.py
ADDED
|
@@ -0,0 +1,1378 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Search everything your coding agents ever did, and find where any file came from.
|
|
3
|
+
|
|
4
|
+
Indexes coding-agent transcripts (Claude Code ~/.claude/projects, Codex ~/.codex)
|
|
5
|
+
into a local SQLite full-text index. Stdlib only. No network. Your data never
|
|
6
|
+
leaves the machine.
|
|
7
|
+
"""
|
|
8
|
+
import sys, os, json, glob, re, sqlite3, argparse
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
|
|
11
|
+
# The tool ships under two console scripts (`transcripto` and `trace`) and is also
|
|
12
|
+
# run as `python3 transcripto.py`. Every hint we print must name the command the
|
|
13
|
+
# reader actually typed, otherwise we tell a stranger to run a binary they do not
|
|
14
|
+
# have on PATH.
|
|
15
|
+
def _prog():
|
|
16
|
+
n = os.path.basename(sys.argv[0] or "")
|
|
17
|
+
if n.endswith(".py"):
|
|
18
|
+
n = n[:-3]
|
|
19
|
+
return n if n in ("transcripto", "trace") else "transcripto"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
PROG = _prog()
|
|
23
|
+
|
|
24
|
+
USAGE = """
|
|
25
|
+
%(p)s index build / refresh the index (incremental)
|
|
26
|
+
%(p)s ask "<topic>" YOUR OWN messages about a topic, newest first + a rollup
|
|
27
|
+
%(p)s search "<query>" full-text search across every session (you + agents)
|
|
28
|
+
%(p)s find <filename> every session that wrote, edited, or read a file
|
|
29
|
+
%(p)s sessions recent sessions, newest first, with their opening ask
|
|
30
|
+
%(p)s stats what you work on most: projects, files, volume
|
|
31
|
+
%(p)s cost what ONE decision of yours costs: spend / turns you typed
|
|
32
|
+
%(p)s coach which of YOUR prompt habits actually survive (a proxy)
|
|
33
|
+
%(p)s coach --harness codex grade your Codex (~/.codex) transcripts instead
|
|
34
|
+
%(p)s coach --verified-human subtract likely-PASTED turns (echoes of agent output)
|
|
35
|
+
""" % {"p": PROG}
|
|
36
|
+
|
|
37
|
+
HOME = os.path.expanduser("~")
|
|
38
|
+
ROOTS = [os.path.join(HOME, ".claude", "projects")]
|
|
39
|
+
DB = os.path.join(HOME, ".trace", "trace.db")
|
|
40
|
+
|
|
41
|
+
FILE_TOOLS = {"Write": "write", "Edit": "edit", "Read": "read",
|
|
42
|
+
"NotebookEdit": "edit", "MultiEdit": "edit"}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def connect():
|
|
46
|
+
os.makedirs(os.path.dirname(DB), exist_ok=True)
|
|
47
|
+
con = sqlite3.connect(DB)
|
|
48
|
+
con.execute("PRAGMA journal_mode=WAL")
|
|
49
|
+
return con
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
SCHEMA_VERSION = 2 # bump when a column/tokenizer change needs a full rebuild
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _needs_rebuild(con):
|
|
56
|
+
"""True if the messages table exists but predates the current schema
|
|
57
|
+
(missing is_human, or an old FTS tokenizer). Triggers a one-time full reindex."""
|
|
58
|
+
t = con.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages'").fetchone()
|
|
59
|
+
if not t:
|
|
60
|
+
return False # fresh db — nothing to migrate
|
|
61
|
+
cols = {r[1] for r in con.execute("PRAGMA table_info(messages)")}
|
|
62
|
+
if "is_human" not in cols:
|
|
63
|
+
return True
|
|
64
|
+
fts = con.execute("SELECT sql FROM sqlite_master WHERE name='messages_fts'").fetchone()
|
|
65
|
+
if fts and "porter" not in (fts[0] or ""):
|
|
66
|
+
return True
|
|
67
|
+
return False
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def init_schema(con):
|
|
71
|
+
if _needs_rebuild(con):
|
|
72
|
+
con.executescript(
|
|
73
|
+
"DROP TABLE IF EXISTS messages_fts; DROP TABLE IF EXISTS messages;"
|
|
74
|
+
"DROP TABLE IF EXISTS files; DROP TABLE IF EXISTS indexed;")
|
|
75
|
+
con.commit()
|
|
76
|
+
con.executescript("""
|
|
77
|
+
CREATE TABLE IF NOT EXISTS messages(
|
|
78
|
+
id INTEGER PRIMARY KEY, session_id TEXT, session_file TEXT, project TEXT,
|
|
79
|
+
ts TEXT, role TEXT, cwd TEXT, git_branch TEXT, text TEXT,
|
|
80
|
+
is_human INTEGER DEFAULT 0, prompt_source TEXT);
|
|
81
|
+
-- porter stemming: `ask "frustration"` also matches frustrated/frustrating.
|
|
82
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
|
83
|
+
text, content='messages', content_rowid='id', tokenize="porter unicode61");
|
|
84
|
+
CREATE TABLE IF NOT EXISTS files(
|
|
85
|
+
id INTEGER PRIMARY KEY, path TEXT, name TEXT, action TEXT,
|
|
86
|
+
session_id TEXT, session_file TEXT, ts TEXT, cwd TEXT);
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_files_name ON files(name);
|
|
88
|
+
CREATE INDEX IF NOT EXISTS idx_msg_human ON messages(is_human, ts);
|
|
89
|
+
CREATE TABLE IF NOT EXISTS indexed(session_file TEXT PRIMARY KEY, mtime REAL);
|
|
90
|
+
|
|
91
|
+
-- Stable READ-ONLY views for consumers (ZUP, Helicon). The contract: consumers
|
|
92
|
+
-- open the db read-only and SELECT from these views; they never write. See READ-CONTRACT.md.
|
|
93
|
+
DROP VIEW IF EXISTS v_sessions;
|
|
94
|
+
CREATE VIEW v_sessions AS
|
|
95
|
+
SELECT m.session_id, m.project,
|
|
96
|
+
MAX(m.ts) AS last_ts, MIN(m.ts) AS first_ts,
|
|
97
|
+
COUNT(*) AS n_messages,
|
|
98
|
+
SUM(CASE WHEN m.role='assistant' THEN 1 ELSE 0 END) AS assistant_turns,
|
|
99
|
+
(SELECT cwd FROM messages c WHERE c.session_id=m.session_id AND c.cwd!=''
|
|
100
|
+
ORDER BY c.ts DESC LIMIT 1) AS cwd
|
|
101
|
+
FROM messages m GROUP BY m.session_id;
|
|
102
|
+
CREATE VIEW IF NOT EXISTS v_file_touches AS
|
|
103
|
+
SELECT name, path, action, session_id, ts, cwd FROM files;
|
|
104
|
+
-- is_human=1 marks a turn Oscar actually typed (promptSource typed/queued, not
|
|
105
|
+
-- injected/tool/peer). See ask_gate() + READ-CONTRACT.md.
|
|
106
|
+
DROP VIEW IF EXISTS v_messages;
|
|
107
|
+
CREATE VIEW v_messages AS
|
|
108
|
+
SELECT id, session_id, project, ts, role, cwd, git_branch, text,
|
|
109
|
+
is_human, prompt_source FROM messages;
|
|
110
|
+
""")
|
|
111
|
+
con.commit()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def extract(d):
|
|
115
|
+
"""Return (role, text, [(action, path)]) from one transcript record."""
|
|
116
|
+
msg = d.get("message") or {}
|
|
117
|
+
role = msg.get("role") or d.get("type")
|
|
118
|
+
parts, files = [], []
|
|
119
|
+
c = msg.get("content")
|
|
120
|
+
if isinstance(c, str):
|
|
121
|
+
parts.append(c)
|
|
122
|
+
elif isinstance(c, list):
|
|
123
|
+
for b in c:
|
|
124
|
+
if not isinstance(b, dict):
|
|
125
|
+
continue
|
|
126
|
+
bt = b.get("type")
|
|
127
|
+
if bt == "text":
|
|
128
|
+
parts.append(b.get("text", ""))
|
|
129
|
+
elif bt == "tool_use":
|
|
130
|
+
name, inp = b.get("name", ""), (b.get("input") or {})
|
|
131
|
+
fp = inp.get("file_path") or inp.get("path") or inp.get("notebook_path")
|
|
132
|
+
if fp:
|
|
133
|
+
files.append((FILE_TOOLS.get(name, name.lower()), fp))
|
|
134
|
+
for k in ("command", "description", "prompt", "pattern", "query", "url", "file_path"):
|
|
135
|
+
v = inp.get(k)
|
|
136
|
+
if isinstance(v, str) and v:
|
|
137
|
+
parts.append("[%s.%s] %s" % (name, k, v))
|
|
138
|
+
elif bt == "tool_result":
|
|
139
|
+
cont = b.get("content")
|
|
140
|
+
if isinstance(cont, str):
|
|
141
|
+
parts.append(cont[:2000])
|
|
142
|
+
elif isinstance(cont, list):
|
|
143
|
+
for x in cont:
|
|
144
|
+
if isinstance(x, dict) and x.get("type") == "text":
|
|
145
|
+
parts.append(x.get("text", "")[:2000])
|
|
146
|
+
return role, "\n".join(p for p in parts if p), files
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def is_human_turn(d):
|
|
150
|
+
"""True iff this transcript record is a message the operator actually TYPED.
|
|
151
|
+
|
|
152
|
+
The measured gate (see reference_transcript_authorship_gate.md): at fleet scale
|
|
153
|
+
~95% of `type: user` records are NOT the operator — tool results, injected skill
|
|
154
|
+
bodies, spawned sub-agent prompts, and cross-session peer messages all arrive as
|
|
155
|
+
`type: user`. The one reliable signal is Claude Code's own `promptSource`.
|
|
156
|
+
keep: promptSource in (typed, queued) — he typed it, live or while busy
|
|
157
|
+
drop: isMeta (skill bodies/images) · toolUseResult (tool output) ·
|
|
158
|
+
isSidechain (spawned agent's prompt) · sdk/system (judges, peers)
|
|
159
|
+
"""
|
|
160
|
+
if d.get("type") != "user":
|
|
161
|
+
return False
|
|
162
|
+
if d.get("promptSource") not in ("typed", "queued"):
|
|
163
|
+
return False
|
|
164
|
+
if d.get("isMeta") or d.get("isSidechain") or d.get("toolUseResult") is not None:
|
|
165
|
+
return False
|
|
166
|
+
return True
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _index_once(con):
|
|
170
|
+
"""Incrementally index every changed/new transcript. Returns (new_sessions, new_msgs)."""
|
|
171
|
+
seen = []
|
|
172
|
+
for root in ROOTS:
|
|
173
|
+
seen += glob.glob(os.path.join(root, "**", "*.jsonl"), recursive=True)
|
|
174
|
+
new = msgs = 0
|
|
175
|
+
for f in sorted(seen):
|
|
176
|
+
mt = os.path.getmtime(f)
|
|
177
|
+
row = con.execute("SELECT mtime FROM indexed WHERE session_file=?", (f,)).fetchone()
|
|
178
|
+
if row and abs(row[0] - mt) < 1e-6:
|
|
179
|
+
continue
|
|
180
|
+
con.execute("DELETE FROM messages WHERE session_file=?", (f,))
|
|
181
|
+
con.execute("DELETE FROM files WHERE session_file=?", (f,))
|
|
182
|
+
proj = os.path.basename(os.path.dirname(f))
|
|
183
|
+
fallback_sid = os.path.basename(f)[:-6]
|
|
184
|
+
for line in open(f, errors="replace"):
|
|
185
|
+
line = line.strip()
|
|
186
|
+
if not line:
|
|
187
|
+
continue
|
|
188
|
+
try:
|
|
189
|
+
d = json.loads(line)
|
|
190
|
+
except Exception:
|
|
191
|
+
continue
|
|
192
|
+
if d.get("type") not in ("user", "assistant"):
|
|
193
|
+
continue
|
|
194
|
+
role, text, fl = extract(d)
|
|
195
|
+
ts = d.get("timestamp") or ""
|
|
196
|
+
cwd = d.get("cwd") or ""
|
|
197
|
+
gb = d.get("gitBranch") or ""
|
|
198
|
+
sid = d.get("sessionId") or fallback_sid
|
|
199
|
+
human = 1 if is_human_turn(d) else 0
|
|
200
|
+
psrc = d.get("promptSource")
|
|
201
|
+
if text:
|
|
202
|
+
cur = con.execute(
|
|
203
|
+
"INSERT INTO messages(session_id,session_file,project,ts,role,cwd,git_branch,text,is_human,prompt_source)"
|
|
204
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?)", (sid, f, proj, ts, role, cwd, gb, text, human, psrc))
|
|
205
|
+
con.execute("INSERT INTO messages_fts(rowid,text) VALUES(?,?)", (cur.lastrowid, text))
|
|
206
|
+
msgs += 1
|
|
207
|
+
for action, path in fl:
|
|
208
|
+
con.execute(
|
|
209
|
+
"INSERT INTO files(path,name,action,session_id,session_file,ts,cwd)"
|
|
210
|
+
" VALUES(?,?,?,?,?,?,?)", (path, os.path.basename(path), action, sid, f, ts, cwd))
|
|
211
|
+
con.execute("INSERT OR REPLACE INTO indexed(session_file,mtime) VALUES(?,?)", (f, mt))
|
|
212
|
+
con.commit(); new += 1
|
|
213
|
+
return new, msgs
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def cmd_index(args):
|
|
217
|
+
con = connect(); init_schema(con)
|
|
218
|
+
new, msgs = _index_once(con)
|
|
219
|
+
tot = con.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
|
|
220
|
+
print("indexed %d changed sessions · +%d messages · %d total searchable" % (new, msgs, tot))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def cmd_watch(args):
|
|
224
|
+
"""Live indexing: pick up new transcripts + trace lines automatically as they land."""
|
|
225
|
+
import time
|
|
226
|
+
con = connect(); init_schema(con)
|
|
227
|
+
_index_once(con)
|
|
228
|
+
tot = con.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
|
|
229
|
+
print(PROG + " watch, live. %d messages indexed; polling %s every %ds. Ctrl-C to stop."
|
|
230
|
+
% (tot, ROOTS[0].replace(HOME, "~"), args.interval), flush=True)
|
|
231
|
+
while True:
|
|
232
|
+
try:
|
|
233
|
+
time.sleep(args.interval)
|
|
234
|
+
except KeyboardInterrupt:
|
|
235
|
+
print("\nstopped.", flush=True); return
|
|
236
|
+
new, msgs = _index_once(con)
|
|
237
|
+
if msgs:
|
|
238
|
+
tot += msgs
|
|
239
|
+
print(" \033[32m+%d\033[0m messages · %d session(s) · %d total" % (msgs, new, tot), flush=True)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _match(query):
|
|
243
|
+
terms = [t for t in query.split() if t]
|
|
244
|
+
return " AND ".join('"%s"' % t.replace('"', '') for t in terms) or '""'
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _day(ts):
|
|
248
|
+
return (ts or "")[:10] or "????-??-??"
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def cmd_search(args):
|
|
252
|
+
con = connect()
|
|
253
|
+
try:
|
|
254
|
+
rows = con.execute(
|
|
255
|
+
"SELECT m.ts,m.project,m.role,m.cwd,snippet(messages_fts,0,'\033[1m','\033[0m','…',14)"
|
|
256
|
+
" FROM messages_fts JOIN messages m ON m.id=messages_fts.rowid"
|
|
257
|
+
" WHERE messages_fts MATCH ? ORDER BY m.ts DESC LIMIT ?",
|
|
258
|
+
(_match(args.query), args.limit)).fetchall()
|
|
259
|
+
except sqlite3.OperationalError as e:
|
|
260
|
+
print("search error:", e); return
|
|
261
|
+
if not rows:
|
|
262
|
+
print("no matches. try `%s index` first, or broader terms." % PROG); return
|
|
263
|
+
for ts, proj, role, cwd, snip in rows:
|
|
264
|
+
repo = os.path.basename(cwd) if cwd else proj
|
|
265
|
+
who = "you" if role == "user" else "agent"
|
|
266
|
+
print("\033[2m%s\033[0m \033[36m%s\033[0m %-5s %s"
|
|
267
|
+
% (_day(ts), repo, who, " ".join(snip.split())))
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _repo(cwd, proj):
|
|
271
|
+
return os.path.basename(cwd) if cwd else proj
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def cmd_ask(args):
|
|
275
|
+
"""YOUR OWN messages about a topic — the question that kills 'did I lose something?'.
|
|
276
|
+
|
|
277
|
+
Filters to turns Oscar actually typed (is_human=1), newest first, each with
|
|
278
|
+
date + repo + session id + snippet, and opens with a deterministic rollup of
|
|
279
|
+
the arc: how many, how long, which repos, and your latest thought on it.
|
|
280
|
+
"""
|
|
281
|
+
con = connect()
|
|
282
|
+
try:
|
|
283
|
+
rows = con.execute(
|
|
284
|
+
"SELECT m.id,m.ts,m.project,m.cwd,m.session_id,m.text,"
|
|
285
|
+
" snippet(messages_fts,0,'\033[1m','\033[0m','…',16)"
|
|
286
|
+
" FROM messages_fts JOIN messages m ON m.id=messages_fts.rowid"
|
|
287
|
+
" WHERE messages_fts MATCH ? AND m.is_human=1"
|
|
288
|
+
" ORDER BY m.ts DESC LIMIT ?",
|
|
289
|
+
(_match(args.query), args.limit)).fetchall()
|
|
290
|
+
except sqlite3.OperationalError as e:
|
|
291
|
+
print("ask error:", e); return
|
|
292
|
+
if not rows:
|
|
293
|
+
# Did the topic exist at all (just not in his own words)? Say so honestly.
|
|
294
|
+
try:
|
|
295
|
+
any_hit = con.execute(
|
|
296
|
+
"SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH ?",
|
|
297
|
+
(_match(args.query),)).fetchone()[0]
|
|
298
|
+
except sqlite3.OperationalError:
|
|
299
|
+
any_hit = 0
|
|
300
|
+
if any_hit:
|
|
301
|
+
print("no messages YOU typed about '%s', but %d agent/tool turns mention it."
|
|
302
|
+
"\ntry `" + PROG + " search \"%s\"` to see those, or `" + PROG + " index` if it's new."
|
|
303
|
+
% (args.query, any_hit, args.query))
|
|
304
|
+
else:
|
|
305
|
+
print("nothing about '%s' yet. try `" + PROG + " index` first, or broader terms."
|
|
306
|
+
% args.query)
|
|
307
|
+
return
|
|
308
|
+
|
|
309
|
+
# ---- rollup: the arc across ALL your matches, not just the shown page ----
|
|
310
|
+
allm = con.execute(
|
|
311
|
+
"SELECT m.ts,m.project,m.cwd,m.session_id"
|
|
312
|
+
" FROM messages_fts JOIN messages m ON m.id=messages_fts.rowid"
|
|
313
|
+
" WHERE messages_fts MATCH ? AND m.is_human=1",
|
|
314
|
+
(_match(args.query),)).fetchall()
|
|
315
|
+
total = len(allm)
|
|
316
|
+
days = sorted(_day(r[0]) for r in allm if r[0])
|
|
317
|
+
sessions = {r[3] for r in allm}
|
|
318
|
+
repos = {}
|
|
319
|
+
for ts, proj, cwd, sid in allm:
|
|
320
|
+
repos[_repo(cwd, proj)] = repos.get(_repo(cwd, proj), 0) + 1
|
|
321
|
+
top = sorted(repos.items(), key=lambda kv: -kv[1])
|
|
322
|
+
span = ("%s → %s" % (days[0], days[-1])) if days else "?"
|
|
323
|
+
shown = len(rows)
|
|
324
|
+
more = " (showing newest %d)" % shown if shown < total else ""
|
|
325
|
+
headcount = ("%d" % total) if shown >= total else ("%d of %d" % (shown, total))
|
|
326
|
+
print("\033[1m%s\033[0m %s message%s you typed · %d session%s · %d repo%s · %s%s"
|
|
327
|
+
% (args.query, headcount, "" if total == 1 else "s",
|
|
328
|
+
len(sessions), "" if len(sessions) == 1 else "s",
|
|
329
|
+
len(top), "" if len(top) == 1 else "s", span, more))
|
|
330
|
+
print("\033[2mwhat you were doing about this\033[0m")
|
|
331
|
+
print(" most active in: " + " · ".join(
|
|
332
|
+
"\033[36m%s\033[0m (%d)" % (r, c) for r, c in top[:4]))
|
|
333
|
+
lid, lts, lproj, lcwd, lsid, ltext, lsnip = rows[0]
|
|
334
|
+
latest = " ".join(ltext.split())[:240]
|
|
335
|
+
print(" latest (\033[2m%s\033[0m %s): \"%s\"" % (_day(lts), _repo(lcwd, lproj), latest))
|
|
336
|
+
|
|
337
|
+
print("\n\033[2myour messages, newest first\033[0m")
|
|
338
|
+
for _id, ts, proj, cwd, sid, text, snip in rows:
|
|
339
|
+
print("%s \033[36m%-20s\033[0m %s \033[2m%s\033[0m"
|
|
340
|
+
% (_day(ts), _repo(cwd, proj)[:20], " ".join(snip.split()), (sid or "")[:8]))
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def cmd_find(args):
|
|
344
|
+
con = connect()
|
|
345
|
+
n = args.name
|
|
346
|
+
rows = con.execute(
|
|
347
|
+
"SELECT ts,action,path,cwd,session_id FROM files"
|
|
348
|
+
" WHERE name=? OR path LIKE ? ORDER BY ts", (n, "%" + n + "%")).fetchall()
|
|
349
|
+
if not rows:
|
|
350
|
+
print("no file matching '%s' in any session. try `%s index`." % (n, PROG)); return
|
|
351
|
+
writes = [r for r in rows if r[1] in ("write", "edit")]
|
|
352
|
+
print("\033[1m%s\033[0m %d touches across sessions (%d were writes/edits)\n"
|
|
353
|
+
% (n, len(rows), len(writes)))
|
|
354
|
+
for ts, action, path, cwd, sid in rows:
|
|
355
|
+
tag = {"write": "\033[32mWROTE\033[0m", "edit": "\033[33mEDIT \033[0m",
|
|
356
|
+
"read": "\033[2mread \033[0m"}.get(action, action.upper())
|
|
357
|
+
print("%s %s %s \033[2m%s\033[0m" % (_day(ts), tag, path, sid[:8]))
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def cmd_sessions(args):
|
|
361
|
+
con = connect()
|
|
362
|
+
rows = con.execute(
|
|
363
|
+
"SELECT session_id,project,MAX(ts) mx,COUNT(*) FROM messages"
|
|
364
|
+
" GROUP BY session_id ORDER BY mx DESC LIMIT ?", (args.limit,)).fetchall()
|
|
365
|
+
for sid, proj, mx, cnt in rows:
|
|
366
|
+
t = con.execute("SELECT text FROM messages WHERE session_id=? AND role='user'"
|
|
367
|
+
" AND text!='' ORDER BY ts LIMIT 1", (sid,)).fetchone()
|
|
368
|
+
title = (t[0][:90].replace("\n", " ") if t else "(no user text)")
|
|
369
|
+
print("\033[2m%s\033[0m \033[36m%-22s\033[0m %4d msg %s"
|
|
370
|
+
% (_day(mx), proj[:22], cnt, title))
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def cmd_stats(args):
|
|
374
|
+
con = connect()
|
|
375
|
+
print("\033[1mprojects by activity\033[0m")
|
|
376
|
+
for proj, c in con.execute("SELECT project,COUNT(*) c FROM messages GROUP BY project"
|
|
377
|
+
" ORDER BY c DESC LIMIT 12"):
|
|
378
|
+
print(" %5d %s" % (c, proj))
|
|
379
|
+
print("\n\033[1mmost-written files\033[0m")
|
|
380
|
+
for name, c in con.execute("SELECT name,COUNT(*) c FROM files WHERE action IN('write','edit')"
|
|
381
|
+
" GROUP BY name ORDER BY c DESC LIMIT 12"):
|
|
382
|
+
print(" %5d %s" % (c, name))
|
|
383
|
+
tot = con.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
|
|
384
|
+
ses = con.execute("SELECT COUNT(DISTINCT session_id) FROM messages").fetchone()[0]
|
|
385
|
+
fil = con.execute("SELECT COUNT(DISTINCT path) FROM files").fetchone()[0]
|
|
386
|
+
print("\n%d messages · %d sessions · %d distinct files tracked" % (tot, ses, fil))
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
390
|
+
# cost — what one HUMAN decision costs
|
|
391
|
+
#
|
|
392
|
+
# ccusage and friends answer "what did I spend". They cannot answer "what did
|
|
393
|
+
# one decision of mine cost", because the denominator does not exist in the
|
|
394
|
+
# transcript: ~95% of `type: user` records at fleet scale are not the operator.
|
|
395
|
+
# trace already has that gate (is_human_turn). Joining it to token spend is the
|
|
396
|
+
# whole feature: spend / decisions-you-actually-made.
|
|
397
|
+
#
|
|
398
|
+
# There is no cost field in a Claude Code transcript — only token counts — so
|
|
399
|
+
# every dollar here is API-EQUIVALENT: what those tokens would cost at
|
|
400
|
+
# Anthropic list price. On a Max/Pro subscription you did not pay it. It is
|
|
401
|
+
# still the only comparable unit, and it is what ccusage reports too.
|
|
402
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
403
|
+
|
|
404
|
+
# USD per 1M tokens (input, output), Anthropic list price, cached 2026-08-25.
|
|
405
|
+
PRICES = {
|
|
406
|
+
"claude-fable-5": (10.0, 50.0),
|
|
407
|
+
"claude-mythos-5": (10.0, 50.0),
|
|
408
|
+
"claude-opus-5": (5.0, 25.0),
|
|
409
|
+
"claude-opus-4-8": (5.0, 25.0),
|
|
410
|
+
"claude-opus-4-7": (5.0, 25.0),
|
|
411
|
+
"claude-opus-4-6": (5.0, 25.0),
|
|
412
|
+
"claude-opus-4-5": (5.0, 25.0),
|
|
413
|
+
"claude-sonnet-5": (3.0, 15.0),
|
|
414
|
+
"claude-sonnet-4-6": (3.0, 15.0),
|
|
415
|
+
"claude-sonnet-4-5": (3.0, 15.0),
|
|
416
|
+
"claude-haiku-4-5": (1.0, 5.0),
|
|
417
|
+
}
|
|
418
|
+
# fast mode is the same model at premium rates — Opus 5 / 4.8 only.
|
|
419
|
+
FAST_PRICES = {"claude-opus-5": (10.0, 50.0), "claude-opus-4-8": (10.0, 50.0)}
|
|
420
|
+
# Sonnet 5 shipped at intro pricing through 2026-08-31.
|
|
421
|
+
INTRO = {"claude-sonnet-5": ((2.0, 10.0), "2026-08-31")}
|
|
422
|
+
CACHE_WRITE_5M, CACHE_WRITE_1H, CACHE_READ = 1.25, 2.0, 0.10
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def normalise_model(model):
|
|
426
|
+
"""`claude-haiku-4-5-20251001` and `claude-haiku-4-5` are the same price.
|
|
427
|
+
Dated snapshots appear in real transcripts; strip the -YYYYMMDD suffix."""
|
|
428
|
+
if not model:
|
|
429
|
+
return "(unknown)"
|
|
430
|
+
head, _, tail = model.rpartition("-")
|
|
431
|
+
if head and len(tail) == 8 and tail.isdigit():
|
|
432
|
+
return head
|
|
433
|
+
return model
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def price_message(model, usage, ts=""):
|
|
437
|
+
"""(usd, tokens) for one API message. usd is None when the model is unpriced —
|
|
438
|
+
an unknown model must show up as an unpriced line, never as a silent $0."""
|
|
439
|
+
u = usage or {}
|
|
440
|
+
inp = u.get("input_tokens") or 0
|
|
441
|
+
out = u.get("output_tokens") or 0
|
|
442
|
+
read = u.get("cache_read_input_tokens") or 0
|
|
443
|
+
cc = u.get("cache_creation") or {}
|
|
444
|
+
w1h = cc.get("ephemeral_1h_input_tokens") or 0
|
|
445
|
+
w5m = cc.get("ephemeral_5m_input_tokens") or 0
|
|
446
|
+
written = u.get("cache_creation_input_tokens") or 0
|
|
447
|
+
if not (w1h or w5m): # older records carry only the aggregate
|
|
448
|
+
w5m = written
|
|
449
|
+
tokens = {"input": inp, "output": out, "cache_read": read,
|
|
450
|
+
"cache_write_5m": w5m, "cache_write_1h": w1h,
|
|
451
|
+
"total": inp + out + read + w5m + w1h}
|
|
452
|
+
model = normalise_model(model)
|
|
453
|
+
rate = None
|
|
454
|
+
if u.get("speed") == "fast" and model in FAST_PRICES:
|
|
455
|
+
rate = FAST_PRICES[model]
|
|
456
|
+
elif model in INTRO and ts and ts[:10] <= INTRO[model][1]:
|
|
457
|
+
rate = INTRO[model][0]
|
|
458
|
+
elif model in PRICES:
|
|
459
|
+
rate = PRICES[model]
|
|
460
|
+
if rate is None:
|
|
461
|
+
return None, tokens
|
|
462
|
+
pin, pout = rate[0] / 1e6, rate[1] / 1e6
|
|
463
|
+
usd = (inp * pin + out * pout + read * pin * CACHE_READ
|
|
464
|
+
+ w5m * pin * CACHE_WRITE_5M + w1h * pin * CACHE_WRITE_1H)
|
|
465
|
+
return usd, tokens
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _msg_key(d, msg):
|
|
469
|
+
"""Claude Code writes one transcript line per content BLOCK of the same API
|
|
470
|
+
message, repeating the identical usage object 2-3x. Summing lines inflates
|
|
471
|
+
spend ~2.7x on a real session. Dedupe on the API message id."""
|
|
472
|
+
return msg.get("id") or d.get("requestId") or d.get("uuid")
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def collect_cost(days=30, roots=None):
|
|
476
|
+
"""Walk the transcripts once. Returns a report dict. Numerator = deduped
|
|
477
|
+
assistant token spend (sub-agent runs included — that is real money).
|
|
478
|
+
Denominator = is_human_turn, the gate `ask` already uses."""
|
|
479
|
+
roots = roots or ROOTS
|
|
480
|
+
cutoff = cut_iso = None
|
|
481
|
+
if days:
|
|
482
|
+
cutoff = datetime.now(timezone.utc).timestamp() - days * 86400
|
|
483
|
+
cut_iso = datetime.fromtimestamp(cutoff, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
|
484
|
+
rep = {"days": days, "usd": 0.0, "decisions": 0, "raw_user_turns": 0, "agent_messages": 0,
|
|
485
|
+
"unpriced_messages": 0, "unpriced_tokens": 0, "sessions": set(),
|
|
486
|
+
"by_model": {}, "by_repo": {}, "tokens": {}, "first_ts": "", "last_ts": ""}
|
|
487
|
+
seen = set()
|
|
488
|
+
for root in roots:
|
|
489
|
+
for f in sorted(glob.glob(os.path.join(root, "**", "*.jsonl"), recursive=True)):
|
|
490
|
+
# append-only files: an mtime before the window means every record is older
|
|
491
|
+
if cutoff and os.path.getmtime(f) < cutoff:
|
|
492
|
+
continue
|
|
493
|
+
for line in open(f, errors="replace"):
|
|
494
|
+
line = line.strip()
|
|
495
|
+
if not line:
|
|
496
|
+
continue
|
|
497
|
+
try:
|
|
498
|
+
d = json.loads(line)
|
|
499
|
+
except Exception:
|
|
500
|
+
continue
|
|
501
|
+
t = d.get("type")
|
|
502
|
+
if t not in ("user", "assistant"):
|
|
503
|
+
continue
|
|
504
|
+
ts = d.get("timestamp") or ""
|
|
505
|
+
if cut_iso and ts and ts[:19] < cut_iso[:19]:
|
|
506
|
+
continue
|
|
507
|
+
repo = _repo(d.get("cwd"), os.path.basename(os.path.dirname(f)))
|
|
508
|
+
if ts:
|
|
509
|
+
rep["first_ts"] = min(rep["first_ts"] or ts, ts)
|
|
510
|
+
rep["last_ts"] = max(rep["last_ts"], ts)
|
|
511
|
+
if t == "user":
|
|
512
|
+
# the naive denominator, kept so the gate's effect is checkable
|
|
513
|
+
# rather than asserted: raw_user_turns / decisions is the factor.
|
|
514
|
+
rep["raw_user_turns"] += 1
|
|
515
|
+
if is_human_turn(d):
|
|
516
|
+
rep["decisions"] += 1
|
|
517
|
+
rep["by_repo"].setdefault(repo, {"usd": 0.0, "decisions": 0})
|
|
518
|
+
rep["by_repo"][repo]["decisions"] += 1
|
|
519
|
+
rep["sessions"].add(d.get("sessionId") or f)
|
|
520
|
+
continue
|
|
521
|
+
msg = d.get("message") or {}
|
|
522
|
+
if not isinstance(msg, dict):
|
|
523
|
+
continue
|
|
524
|
+
usage = msg.get("usage")
|
|
525
|
+
if not usage:
|
|
526
|
+
continue
|
|
527
|
+
key = _msg_key(d, msg)
|
|
528
|
+
if key in seen:
|
|
529
|
+
continue
|
|
530
|
+
seen.add(key)
|
|
531
|
+
model = normalise_model(msg.get("model"))
|
|
532
|
+
usd, tok = price_message(model, usage, ts)
|
|
533
|
+
rep["agent_messages"] += 1
|
|
534
|
+
for k, v in tok.items():
|
|
535
|
+
rep["tokens"][k] = rep["tokens"].get(k, 0) + v
|
|
536
|
+
m = rep["by_model"].setdefault(model, {"usd": 0.0, "messages": 0,
|
|
537
|
+
"tokens": 0, "priced": usd is not None})
|
|
538
|
+
m["messages"] += 1
|
|
539
|
+
m["tokens"] += tok["total"]
|
|
540
|
+
if usd is None:
|
|
541
|
+
if tok["total"]: # a 0-token <synthetic> row costs nothing either way
|
|
542
|
+
rep["unpriced_messages"] += 1
|
|
543
|
+
rep["unpriced_tokens"] += tok["total"]
|
|
544
|
+
continue
|
|
545
|
+
rep["usd"] += usd
|
|
546
|
+
m["usd"] += usd
|
|
547
|
+
rep["by_repo"].setdefault(repo, {"usd": 0.0, "decisions": 0})
|
|
548
|
+
rep["by_repo"][repo]["usd"] += usd
|
|
549
|
+
rep["sessions"] = len(rep["sessions"])
|
|
550
|
+
rep["per_decision"] = (rep["usd"] / rep["decisions"]) if rep["decisions"] else None
|
|
551
|
+
rep["gate_factor"] = (rep["raw_user_turns"] / rep["decisions"]) if rep["decisions"] else None
|
|
552
|
+
rep["turns_per_decision"] = (rep["agent_messages"] / rep["decisions"]) if rep["decisions"] else None
|
|
553
|
+
return rep
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _hm(n):
|
|
557
|
+
for unit, div in (("B", 1e9), ("M", 1e6), ("k", 1e3)):
|
|
558
|
+
if n >= div:
|
|
559
|
+
return "%.1f%s" % (n / div, unit)
|
|
560
|
+
return str(int(n))
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def cmd_cost(args):
|
|
564
|
+
rep = collect_cost(args.days, [os.path.expanduser(args.root)] if args.root else None)
|
|
565
|
+
if args.json:
|
|
566
|
+
print(json.dumps(rep, indent=2, sort_keys=True))
|
|
567
|
+
return
|
|
568
|
+
win = ("last %d days" % args.days) if args.days else "all time"
|
|
569
|
+
span = ""
|
|
570
|
+
if rep["first_ts"]:
|
|
571
|
+
span = " \033[2m%s → %s\033[0m" % (rep["first_ts"][:10], rep["last_ts"][:10])
|
|
572
|
+
print("\033[1mcost per human decision\033[0m %s%s\n" % (win, span))
|
|
573
|
+
if not rep["decisions"]:
|
|
574
|
+
print(" no turns you typed in this window. widen it with --days.")
|
|
575
|
+
return
|
|
576
|
+
print(" API-equivalent spend \033[1m$%s\033[0m" % format(rep["usd"], ",.2f"))
|
|
577
|
+
print(" your decisions \033[1m%d\033[0m \033[2mturns you actually typed"
|
|
578
|
+
" (promptSource typed/queued)\033[0m" % rep["decisions"])
|
|
579
|
+
print(" " + "─" * 52)
|
|
580
|
+
print(" \033[1mcost per human decision $%.2f\033[0m" % rep["per_decision"])
|
|
581
|
+
print("\n %s agent messages · %.0f per decision · %s tokens · %d sessions"
|
|
582
|
+
% (_hm(rep["agent_messages"]), rep["turns_per_decision"],
|
|
583
|
+
_hm(rep["tokens"].get("total", 0)), rep["sessions"]))
|
|
584
|
+
print(" \033[2m%s raw `type: user` records in the same window. dividing by those"
|
|
585
|
+
" instead\n would read $%.2f, %.1fx too cheap.\033[0m"
|
|
586
|
+
% (_hm(rep["raw_user_turns"]), rep["usd"] / rep["raw_user_turns"],
|
|
587
|
+
rep["gate_factor"]))
|
|
588
|
+
if rep["unpriced_messages"]:
|
|
589
|
+
print(" \033[33m%d message(s) unpriced (%s tokens): model not in the price table\033[0m"
|
|
590
|
+
% (rep["unpriced_messages"], _hm(rep["unpriced_tokens"])))
|
|
591
|
+
print("\n\033[1mby model\033[0m")
|
|
592
|
+
for model, m in sorted(rep["by_model"].items(), key=lambda kv: -kv[1]["usd"])[:8]:
|
|
593
|
+
tag = "$%8.2f" % m["usd"] if m["priced"] else "unpriced"
|
|
594
|
+
print(" %10s %-18s %5d msg %6s tok" % (tag, model[:18], m["messages"], _hm(m["tokens"])))
|
|
595
|
+
rows = [(r, v) for r, v in rep["by_repo"].items() if v["decisions"]]
|
|
596
|
+
if rows:
|
|
597
|
+
print("\n\033[1mby repo\033[0m \033[2m(spend attributed by the cwd of each turn)\033[0m")
|
|
598
|
+
for repo, v in sorted(rows, key=lambda kv: -kv[1]["usd"])[:8]:
|
|
599
|
+
per = "$%.2f" % (v["usd"] / v["decisions"])
|
|
600
|
+
print(" %8s %3d decisions %8s / decision \033[36m%s\033[0m"
|
|
601
|
+
% ("$%.2f" % v["usd"], v["decisions"], per, repo[:28]))
|
|
602
|
+
print("\n\033[2mno cost field exists in a transcript. these are list-price equivalents"
|
|
603
|
+
" for the tokens spent.\n on a subscription you did not pay this; it is the"
|
|
604
|
+
" comparable unit, same as ccusage.\033[0m")
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
# ============================================================================
|
|
609
|
+
# coach — rank YOUR OWN prompt habits by whether the work survived
|
|
610
|
+
# ============================================================================
|
|
611
|
+
# Ported from hack-fleet-ata/fleet/coach.py + contract/deterministic.py so it
|
|
612
|
+
# runs here with no imports beyond the stdlib. Same gate (is_human_turn), same
|
|
613
|
+
# survival proxy, same deterministic pattern tags. Offline. Nothing leaves the
|
|
614
|
+
# machine.
|
|
615
|
+
#
|
|
616
|
+
# SURVIVAL IS A PROXY, NOT TRUTH. An episode "survived" iff a Write/Edit landed
|
|
617
|
+
# or a git commit ran after the prompt and nothing reverted it inside the SAME
|
|
618
|
+
# transcript. That is a durable KEYSTROKE, not a durable OUTCOME:
|
|
619
|
+
# - a commit is not proof the code was correct, merged, or kept;
|
|
620
|
+
# - cross-session reverts are invisible to us;
|
|
621
|
+
# - a prompt whose payoff was a decision rather than an edit reads as dead.
|
|
622
|
+
# This caveat travels with every number the command prints. It is a coaching
|
|
623
|
+
# signal, not a verdict.
|
|
624
|
+
|
|
625
|
+
_ABANDON_MARKERS = ("never mind", "forget it", "abandon", "scrap this", "drop it")
|
|
626
|
+
_CORRECTIVE_MARKERS = ("no,", "no ", "no.", "not that", "not this", "not the",
|
|
627
|
+
"i meant", "i mean", "actually", "wait,", "wrong ",
|
|
628
|
+
"other file", "other one")
|
|
629
|
+
_WRITE_TOOLS = {"Write", "Edit", "NotebookEdit", "MultiEdit"}
|
|
630
|
+
_COMMIT_RE = re.compile(r"\bgit\s+commit\b")
|
|
631
|
+
_REVERT_RE = re.compile(r"\bgit\s+(revert|reset\s+--hard)\b")
|
|
632
|
+
_FILE_RE = re.compile(r"\b[\w./-]+\.[A-Za-z]{1,5}\b|\b[\w-]+/[\w./-]+\b")
|
|
633
|
+
_CHECK_RE = re.compile(r"\b(test|tests|verify|verif|prove|proof|done[- ]?when|"
|
|
634
|
+
r"make sure|ensure|confirm|check that|assert|so that|"
|
|
635
|
+
r"screenshot|render)\b")
|
|
636
|
+
_TOKEN_RE = re.compile(r"[A-Za-z0-9_.]+")
|
|
637
|
+
|
|
638
|
+
_INTENT_PHRASES = {"roll back": "REVERT", "rolling back": "REVERT"}
|
|
639
|
+
_INTENT_WORDS = {
|
|
640
|
+
"make": "CHANGE", "let": "CHANGE", "get": "CHANGE", "do": "CHANGE", "have": "CHANGE",
|
|
641
|
+
"fix": "CHANGE", "refactor": "CHANGE", "extract": "CHANGE", "add": "CHANGE",
|
|
642
|
+
"implement": "CHANGE", "build": "CHANGE", "create": "CHANGE", "write": "CHANGE",
|
|
643
|
+
"update": "CHANGE", "bump": "CHANGE", "upgrade": "CHANGE", "change": "CHANGE",
|
|
644
|
+
"edit": "CHANGE", "modify": "CHANGE", "wrap": "CHANGE", "optimize": "CHANGE",
|
|
645
|
+
"optimise": "CHANGE", "improve": "CHANGE", "speed": "CHANGE", "harden": "CHANGE",
|
|
646
|
+
"rename": "CHANGE", "move": "CHANGE", "migrate": "CHANGE", "tidy": "CHANGE",
|
|
647
|
+
"clean": "CHANGE", "cleanup": "CHANGE", "format": "CHANGE", "debug": "CHANGE",
|
|
648
|
+
"diagnose": "CHANGE", "trace": "CHANGE", "reproduce": "CHANGE", "broken": "CHANGE",
|
|
649
|
+
"crash": "CHANGE", "crashes": "CHANGE", "failing": "CHANGE", "handle": "CHANGE",
|
|
650
|
+
"document": "DESCRIBE", "describe": "DESCRIBE", "explain": "DESCRIBE",
|
|
651
|
+
"summarize": "DESCRIBE", "summarise": "DESCRIBE", "comment": "DESCRIBE",
|
|
652
|
+
"review": "DESCRIBE", "audit": "DESCRIBE", "benchmark": "DESCRIBE",
|
|
653
|
+
"measure": "DESCRIBE", "profile": "DESCRIBE", "analyze": "DESCRIBE",
|
|
654
|
+
"analyse": "DESCRIBE", "translate": "DESCRIBE", "localize": "DESCRIBE",
|
|
655
|
+
"localise": "DESCRIBE",
|
|
656
|
+
"revert": "REVERT", "rollback": "REVERT", "undo": "REVERT", "remove": "REVERT",
|
|
657
|
+
"delete": "REVERT", "downgrade": "REVERT", "disable": "REVERT", "deprecate": "REVERT",
|
|
658
|
+
"test": "TEST", "tests": "TEST", "coverage": "TEST", "cover": "TEST",
|
|
659
|
+
"assert": "TEST", "spec": "TEST",
|
|
660
|
+
"deploy": "CHANGE", "ship": "CHANGE", "release": "CHANGE", "publish": "CHANGE",
|
|
661
|
+
}
|
|
662
|
+
_NON_OBJECTS = {"it", "this", "that", "them", "these", "those", "everything",
|
|
663
|
+
"anything", "stuff", "things", "thing", "something", "better",
|
|
664
|
+
"faster", "cleaner", "nicer", "here", "there"}
|
|
665
|
+
_STOP = {"the", "a", "an", "to", "of", "in", "into", "on", "for", "and", "or",
|
|
666
|
+
"but", "with", "from", "by", "at", "as", "is", "are", "be", "been",
|
|
667
|
+
"was", "were", "why", "how", "what", "when", "which", "who", "whose",
|
|
668
|
+
"returns", "return", "yields", "yield", "keep", "green", "show", "me",
|
|
669
|
+
"my", "our", "your", "new", "old", "up", "down", "out", "over",
|
|
670
|
+
"under", "whole", "two", "word", "words", "page", "flow", "layer",
|
|
671
|
+
"module", "so", "if", "then", "before", "after", "please", "can",
|
|
672
|
+
"you", "do", "does", "not", "no", "all", "some", "any", "edge",
|
|
673
|
+
"case", "cases", "set", "result", "nothing", "empty", "load", "box"}
|
|
674
|
+
|
|
675
|
+
MIN_PATTERN_N = 8 # a habit is only rankable once you have this many episodes
|
|
676
|
+
MIN_EPISODES_TO_RANK = 30 # below this, refuse to rank the corpus at all — a
|
|
677
|
+
# rate over a handful of episodes is the exact thing
|
|
678
|
+
# this tool exists to distrust. Raised from 3 after a
|
|
679
|
+
# 9-transcript run printed a ranked table over n=3 buckets.
|
|
680
|
+
_DURABLE = ("commit", "artifact")
|
|
681
|
+
_PROBE = {"commit": "COMMIT-WITNESSED", "artifact": "ARTIFACT-WITNESSED",
|
|
682
|
+
"reverted": "COMMIT-THEN-REVERTED", "none": "NO-DURABLE-RECORD"}
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def _c_tokens(text):
|
|
686
|
+
return [t.lower() for t in _TOKEN_RE.findall(text)]
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _intent(text):
|
|
690
|
+
"""The HEAD intent: the earliest intent token in reading order, or None.
|
|
691
|
+
|
|
692
|
+
Deliberately the FIRST verb, not the strongest. "refactor auth, keep tests
|
|
693
|
+
green" is a refactor with a constraint, not a test task.
|
|
694
|
+
"""
|
|
695
|
+
low, toks, hits = text.lower(), _c_tokens(text), []
|
|
696
|
+
for i, tok in enumerate(toks):
|
|
697
|
+
b = _INTENT_WORDS.get(tok)
|
|
698
|
+
if b:
|
|
699
|
+
hits.append((float(i), b))
|
|
700
|
+
for phrase, bucket in _INTENT_PHRASES.items():
|
|
701
|
+
if phrase in low:
|
|
702
|
+
first = phrase.split()[0]
|
|
703
|
+
if first in toks:
|
|
704
|
+
hits.append((toks.index(first) - 0.5, bucket))
|
|
705
|
+
if not hits:
|
|
706
|
+
return None
|
|
707
|
+
hits.sort(key=lambda x: x[0])
|
|
708
|
+
return hits[0][1]
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def _norm_obj(tok):
|
|
712
|
+
out = []
|
|
713
|
+
for p in re.split(r"[_.]", tok):
|
|
714
|
+
if len(p) < 2:
|
|
715
|
+
continue
|
|
716
|
+
if len(p) > 3 and p.endswith("s"):
|
|
717
|
+
p = p[:-1]
|
|
718
|
+
out.append(p)
|
|
719
|
+
return out
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def _objects(text):
|
|
723
|
+
objs = set()
|
|
724
|
+
for tok in _c_tokens(text):
|
|
725
|
+
if tok in _STOP or tok in _NON_OBJECTS or tok in _INTENT_WORDS:
|
|
726
|
+
continue
|
|
727
|
+
for n in _norm_obj(tok):
|
|
728
|
+
if n not in _STOP and n not in _INTENT_WORDS:
|
|
729
|
+
objs.add(n)
|
|
730
|
+
return objs
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _same_task(a, b):
|
|
734
|
+
"""Offline SAME/DIFFERENT floor. No network, no synonym table, no key.
|
|
735
|
+
|
|
736
|
+
No placeable object in either prompt -> not the same (we refuse to guess).
|
|
737
|
+
Disjoint objects -> different. Overlapping objects but incompatible intent
|
|
738
|
+
families (change vs describe vs revert vs test) -> different.
|
|
739
|
+
"""
|
|
740
|
+
oa, ob = _objects(a), _objects(b)
|
|
741
|
+
if not oa or not ob:
|
|
742
|
+
return False
|
|
743
|
+
hit = bool(oa & ob)
|
|
744
|
+
if not hit:
|
|
745
|
+
for x in oa:
|
|
746
|
+
if len(x) < 5:
|
|
747
|
+
continue
|
|
748
|
+
for y in ob:
|
|
749
|
+
if len(y) >= 5 and x[:4] == y[:4]:
|
|
750
|
+
hit = True
|
|
751
|
+
break
|
|
752
|
+
if hit:
|
|
753
|
+
break
|
|
754
|
+
if not hit:
|
|
755
|
+
return False
|
|
756
|
+
return (_intent(a) or "CHANGE") == (_intent(b) or "CHANGE")
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def _looks_corrective(text):
|
|
760
|
+
low = " " + text.lower().strip()
|
|
761
|
+
return any(low.startswith(" " + m) or (" " + m) in low for m in _CORRECTIVE_MARKERS)
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
def _human_prompt(d):
|
|
765
|
+
"""The text of a genuine human turn, or '' — reuses the measured gate."""
|
|
766
|
+
if not is_human_turn(d):
|
|
767
|
+
return ""
|
|
768
|
+
msg = d.get("message") or {}
|
|
769
|
+
c = msg.get("content")
|
|
770
|
+
if isinstance(c, str):
|
|
771
|
+
return c.strip()
|
|
772
|
+
if isinstance(c, list):
|
|
773
|
+
return "\n".join(b.get("text", "") for b in c
|
|
774
|
+
if isinstance(b, dict) and b.get("type") == "text").strip()
|
|
775
|
+
return ""
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def _tool_uses(rec):
|
|
779
|
+
if rec.get("type") != "assistant":
|
|
780
|
+
return []
|
|
781
|
+
c = (rec.get("message") or {}).get("content")
|
|
782
|
+
return [b for b in c if isinstance(b, dict) and b.get("type") == "tool_use"] \
|
|
783
|
+
if isinstance(c, list) else []
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
# ============================================================================
|
|
787
|
+
# harness connectors — Claude Code is the native shape; Codex is a SECOND source
|
|
788
|
+
# ============================================================================
|
|
789
|
+
# Codex writes rollout transcripts with a different record shape than Claude
|
|
790
|
+
# Code, so we NORMALISE each rollout into the same Claude-shaped rows the coach
|
|
791
|
+
# already understands (is_human_turn / _human_prompt / _tool_uses). One parser,
|
|
792
|
+
# no forked coach — which is also why test_coach.sh stays green by construction.
|
|
793
|
+
#
|
|
794
|
+
# The Codex human gate (measured, see the C1 probe over 77 real rollouts): the
|
|
795
|
+
# reliable, universal signal is a `response_item` `message` with role `user`,
|
|
796
|
+
# MINUS the context blocks Codex threads in wearing the user's role — AGENTS.md,
|
|
797
|
+
# environment_context, permissions, user_instructions, image attachments, the
|
|
798
|
+
# goal context, and file-mention headers. The cleaner `user_message` EVENT is
|
|
799
|
+
# absent from 9 of 77 rollouts (older CLI builds), so the filtered role:user
|
|
800
|
+
# item is the one gate that holds across every build.
|
|
801
|
+
|
|
802
|
+
_CODEX_INJECTED_WRAPPERS = (
|
|
803
|
+
"<environment_context>", "AGENTS.md instructions", "<INSTRUCTIONS>",
|
|
804
|
+
"permissions instructions", "<user_instructions>", "<collaboration_mode>",
|
|
805
|
+
# the auto-continuation Codex injects between turns wearing the user's role
|
|
806
|
+
"The following is the Codex agent history",
|
|
807
|
+
)
|
|
808
|
+
_CODEX_INJECTED_BLOCK_PREFIXES = (
|
|
809
|
+
"<image ", "</image>", "<codex_internal_context",
|
|
810
|
+
"# Files mentioned by the user:",
|
|
811
|
+
)
|
|
812
|
+
# apply_patch envelope: `*** Add File: <path>` / `*** Update File: <path>`.
|
|
813
|
+
_CODEX_ADDFILE_RE = re.compile(r"\*\*\* (?:Add|Update) File: ([^\n\\\"]+)")
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _iter_json(path):
|
|
817
|
+
"""Yield each parseable JSON record from a .jsonl file. Skips blank/bad lines."""
|
|
818
|
+
try:
|
|
819
|
+
f = open(path, "r", errors="replace")
|
|
820
|
+
except OSError:
|
|
821
|
+
return
|
|
822
|
+
with f:
|
|
823
|
+
for line in f:
|
|
824
|
+
line = line.strip()
|
|
825
|
+
if not line:
|
|
826
|
+
continue
|
|
827
|
+
try:
|
|
828
|
+
yield json.loads(line)
|
|
829
|
+
except Exception:
|
|
830
|
+
continue
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _codex_user_text(content):
|
|
834
|
+
"""The genuinely-typed text of a Codex role:user item, or '' if it is one of
|
|
835
|
+
the injected context blocks Codex sends as a user turn."""
|
|
836
|
+
parts = []
|
|
837
|
+
if isinstance(content, str):
|
|
838
|
+
parts.append(content)
|
|
839
|
+
elif isinstance(content, list):
|
|
840
|
+
for b in content:
|
|
841
|
+
if not isinstance(b, dict) or b.get("type") not in ("input_text", "text"):
|
|
842
|
+
continue
|
|
843
|
+
t = b.get("text", "")
|
|
844
|
+
if any(t.lstrip().startswith(p) for p in _CODEX_INJECTED_BLOCK_PREFIXES):
|
|
845
|
+
continue
|
|
846
|
+
parts.append(t)
|
|
847
|
+
txt = "\n".join(parts).strip()
|
|
848
|
+
if not txt or any(w in txt for w in _CODEX_INJECTED_WRAPPERS):
|
|
849
|
+
return ""
|
|
850
|
+
return txt
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def _codex_out_text(content):
|
|
854
|
+
"""Assistant output text of a Codex role:assistant item."""
|
|
855
|
+
if isinstance(content, str):
|
|
856
|
+
return content.strip()
|
|
857
|
+
if isinstance(content, list):
|
|
858
|
+
return "\n".join(b.get("text", "") for b in content
|
|
859
|
+
if isinstance(b, dict)
|
|
860
|
+
and b.get("type") in ("output_text", "text")).strip()
|
|
861
|
+
return ""
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def _codex_tool_block(payload):
|
|
865
|
+
"""Map one Codex tool call to a synthetic Claude tool_use block, so the same
|
|
866
|
+
survival proxy applies: an apply_patch -> Edit (artifact), a `git commit` ->
|
|
867
|
+
a commit, a `git reset --hard`/revert -> a revert, anything else -> read-only
|
|
868
|
+
Bash. We regex the raw serialised call rather than parse the embedded JS."""
|
|
869
|
+
blob = json.dumps(payload)
|
|
870
|
+
m = _CODEX_ADDFILE_RE.search(blob)
|
|
871
|
+
if m:
|
|
872
|
+
return {"type": "tool_use", "name": "Edit",
|
|
873
|
+
"input": {"file_path": m.group(1).strip()}}
|
|
874
|
+
if _COMMIT_RE.search(blob):
|
|
875
|
+
return {"type": "tool_use", "name": "Bash", "input": {"command": "git commit"}}
|
|
876
|
+
if _REVERT_RE.search(blob):
|
|
877
|
+
return {"type": "tool_use", "name": "Bash",
|
|
878
|
+
"input": {"command": "git reset --hard"}}
|
|
879
|
+
return {"type": "tool_use", "name": "Bash", "input": {"command": "exec"}}
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
_CODEX_TOOL_TYPES = ("function_call", "custom_tool_call", "local_shell_call")
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def _codex_rows(path):
|
|
886
|
+
"""Normalise one Codex rollout .jsonl into ordered Claude-shaped rows that
|
|
887
|
+
extract_episodes / _human_prompt / _tool_uses consume unchanged."""
|
|
888
|
+
rows = []
|
|
889
|
+
for d in _iter_json(path):
|
|
890
|
+
if d.get("type") != "response_item":
|
|
891
|
+
continue
|
|
892
|
+
p = d.get("payload") or {}
|
|
893
|
+
pt = p.get("type")
|
|
894
|
+
if pt == "message":
|
|
895
|
+
role = p.get("role")
|
|
896
|
+
if role == "user":
|
|
897
|
+
txt = _codex_user_text(p.get("content"))
|
|
898
|
+
if txt:
|
|
899
|
+
rows.append({"type": "user", "promptSource": "typed",
|
|
900
|
+
"message": {"role": "user", "content": txt}})
|
|
901
|
+
elif role == "assistant":
|
|
902
|
+
txt = _codex_out_text(p.get("content"))
|
|
903
|
+
rows.append({"type": "assistant", "message": {"role": "assistant",
|
|
904
|
+
"content": [{"type": "text", "text": txt}] if txt else []}})
|
|
905
|
+
elif pt in _CODEX_TOOL_TYPES:
|
|
906
|
+
rows.append({"type": "assistant", "message": {"role": "assistant",
|
|
907
|
+
"content": [_codex_tool_block(p)]}})
|
|
908
|
+
return rows
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
def _sniff(path):
|
|
912
|
+
"""'codex' (rollout), 'codex-history' (history.jsonl), or 'claude', from the
|
|
913
|
+
first parseable record. Lets `--root ~/.codex` and mixed dirs just work."""
|
|
914
|
+
for d in _iter_json(path):
|
|
915
|
+
if d.get("type") == "session_meta" or ("payload" in d and "timestamp" in d):
|
|
916
|
+
return "codex"
|
|
917
|
+
if set(d.keys()) == {"session_id", "ts", "text"}:
|
|
918
|
+
return "codex-history"
|
|
919
|
+
return "claude"
|
|
920
|
+
return "claude"
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def _rows_for_file(path):
|
|
924
|
+
"""(rows, harness) for one transcript file, auto-detected. history.jsonl is a
|
|
925
|
+
witness (a typed-input log), not an episode source, so it yields no rows."""
|
|
926
|
+
h = _sniff(path)
|
|
927
|
+
if h == "codex":
|
|
928
|
+
return _codex_rows(path), "codex"
|
|
929
|
+
if h == "codex-history":
|
|
930
|
+
return [], "codex-history"
|
|
931
|
+
return list(_iter_json(path)), "claude"
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def _coach_files(roots, harness):
|
|
935
|
+
"""Discover transcript files. For a Codex tree we name the two real episode
|
|
936
|
+
dirs explicitly — a blind **/*.jsonl over ~/.codex swallows session_index,
|
|
937
|
+
the .tmp scratch, and history.jsonl (double-counted turns)."""
|
|
938
|
+
paths = []
|
|
939
|
+
for r in roots:
|
|
940
|
+
r = os.path.expanduser(r)
|
|
941
|
+
if os.path.isfile(r):
|
|
942
|
+
paths.append(r)
|
|
943
|
+
continue
|
|
944
|
+
base = os.path.basename(r.rstrip("/"))
|
|
945
|
+
if harness == "codex" or base == ".codex":
|
|
946
|
+
got = sorted(glob.glob(os.path.join(r, "archived_sessions", "*.jsonl")))
|
|
947
|
+
got += sorted(glob.glob(os.path.join(r, "sessions", "**", "*.jsonl"),
|
|
948
|
+
recursive=True))
|
|
949
|
+
if not got: # a flat dir of rollouts (e.g. a test fixture)
|
|
950
|
+
got = [p for p in sorted(glob.glob(os.path.join(r, "**", "*.jsonl"),
|
|
951
|
+
recursive=True))
|
|
952
|
+
if os.path.basename(p) not in ("session_index.jsonl",
|
|
953
|
+
"history.jsonl")
|
|
954
|
+
and os.sep + ".tmp" + os.sep not in p]
|
|
955
|
+
paths += got
|
|
956
|
+
else:
|
|
957
|
+
paths += sorted(glob.glob(os.path.join(r, "**", "*.jsonl"), recursive=True))
|
|
958
|
+
return paths
|
|
959
|
+
|
|
960
|
+
|
|
961
|
+
# ============================================================================
|
|
962
|
+
# paste detection — subtract echoed agent output from the human signal
|
|
963
|
+
# ============================================================================
|
|
964
|
+
# A `typed` turn is flagged likely-PASTED when its text is a verbatim substring
|
|
965
|
+
# or a high n-gram overlap of an EARLIER agent/tool message in the SAME session.
|
|
966
|
+
# Cheap tier only: session-scoped, 8-gram overlap, whitespace-normalised. The
|
|
967
|
+
# length floor is load-bearing — a short genuine turn ("run the tests") can never
|
|
968
|
+
# echo a long agent message, and the floor is exactly what keeps it unflagged.
|
|
969
|
+
|
|
970
|
+
_PASTE_MIN_WORDS = 15
|
|
971
|
+
_PASTE_THRESH = 0.6
|
|
972
|
+
_PASTE_N = 8
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def _pnorm(s):
|
|
976
|
+
return " ".join(s.lower().split())
|
|
977
|
+
|
|
978
|
+
|
|
979
|
+
def _pgrams(s, n=_PASTE_N):
|
|
980
|
+
toks = _pnorm(s).split()
|
|
981
|
+
return {" ".join(toks[i:i + n]) for i in range(len(toks) - n + 1)} \
|
|
982
|
+
if len(toks) >= n else set()
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def _is_echo(text, prior, thresh=_PASTE_THRESH, n=_PASTE_N):
|
|
986
|
+
"""True if `text` verbatim-appears in, or shares >= thresh of its n-grams
|
|
987
|
+
with, any earlier agent/tool message. `prior` is a list of precomputed
|
|
988
|
+
(anorm, agrams) pairs — each agent text is normalised and n-grammed ONCE
|
|
989
|
+
when it enters the stream, not re-derived per human turn (was O(n^2))."""
|
|
990
|
+
hnorm, hg = _pnorm(text), _pgrams(text, n)
|
|
991
|
+
for anorm, ag in prior:
|
|
992
|
+
if len(hnorm) >= 40 and hnorm in anorm:
|
|
993
|
+
return True
|
|
994
|
+
if hg and ag and len(hg & ag) / len(hg) >= thresh:
|
|
995
|
+
return True
|
|
996
|
+
return False
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _paste_stream(path, harness):
|
|
1000
|
+
"""Ordered [(kind, text)] with kind in {'human','agent'} for one transcript.
|
|
1001
|
+
'agent' pools everything the operator could have copied FROM: assistant
|
|
1002
|
+
messages, tool commands, and tool results, all EARLIER in the session."""
|
|
1003
|
+
items = []
|
|
1004
|
+
if harness == "codex":
|
|
1005
|
+
for d in _iter_json(path):
|
|
1006
|
+
if d.get("type") != "response_item":
|
|
1007
|
+
continue
|
|
1008
|
+
p = d.get("payload") or {}
|
|
1009
|
+
pt = p.get("type")
|
|
1010
|
+
if pt == "message":
|
|
1011
|
+
role = p.get("role")
|
|
1012
|
+
if role == "user":
|
|
1013
|
+
t = _codex_user_text(p.get("content"))
|
|
1014
|
+
if t:
|
|
1015
|
+
items.append(("human", t))
|
|
1016
|
+
elif role == "assistant":
|
|
1017
|
+
t = _codex_out_text(p.get("content"))
|
|
1018
|
+
if t:
|
|
1019
|
+
items.append(("agent", t))
|
|
1020
|
+
elif pt in _CODEX_TOOL_TYPES:
|
|
1021
|
+
items.append(("agent", json.dumps(p)))
|
|
1022
|
+
elif pt in ("function_call_output", "custom_tool_call_output"):
|
|
1023
|
+
o = p.get("output")
|
|
1024
|
+
t = o if isinstance(o, str) else json.dumps(o)
|
|
1025
|
+
if t:
|
|
1026
|
+
items.append(("agent", t[:4000]))
|
|
1027
|
+
else:
|
|
1028
|
+
for d in _iter_json(path):
|
|
1029
|
+
if is_human_turn(d):
|
|
1030
|
+
t = _human_prompt(d)
|
|
1031
|
+
if t:
|
|
1032
|
+
items.append(("human", t))
|
|
1033
|
+
elif d.get("type") == "assistant":
|
|
1034
|
+
_, t, _ = extract(d)
|
|
1035
|
+
if t:
|
|
1036
|
+
items.append(("agent", t))
|
|
1037
|
+
elif d.get("type") == "user" and d.get("toolUseResult") is not None:
|
|
1038
|
+
_, t, _ = extract(d)
|
|
1039
|
+
if t:
|
|
1040
|
+
items.append(("agent", t))
|
|
1041
|
+
return items
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def detect_pastes(stream, min_words=_PASTE_MIN_WORDS):
|
|
1045
|
+
"""Set of human turn texts that are likely-pasted echoes of earlier output."""
|
|
1046
|
+
flagged, prior = set(), []
|
|
1047
|
+
for kind, text in stream:
|
|
1048
|
+
if kind == "agent":
|
|
1049
|
+
# cache (normalised, n-gram-set) ONCE per agent/tool text so each is
|
|
1050
|
+
# computed a single time, not recomputed for every later human turn.
|
|
1051
|
+
prior.append((_pnorm(text), _pgrams(text)))
|
|
1052
|
+
continue
|
|
1053
|
+
if len(text.split()) < min_words:
|
|
1054
|
+
continue
|
|
1055
|
+
if _is_echo(text, prior):
|
|
1056
|
+
flagged.add(text)
|
|
1057
|
+
return flagged
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
def extract_episodes(rows, source="", pasted=None):
|
|
1061
|
+
"""Split one transcript into episodes: one human intent, opened and closed.
|
|
1062
|
+
A `pasted` set (opener texts flagged by detect_pastes) is treated as non-human
|
|
1063
|
+
so echoed agent output never counts as one of the operator's own prompts."""
|
|
1064
|
+
pasted = pasted or set()
|
|
1065
|
+
eps, n, i = [], len(rows), 0
|
|
1066
|
+
while i < n:
|
|
1067
|
+
opener = _human_prompt(rows[i])
|
|
1068
|
+
if not opener or opener in pasted:
|
|
1069
|
+
i += 1
|
|
1070
|
+
continue
|
|
1071
|
+
start, corrective, assistants = i, 0, 0
|
|
1072
|
+
wrote = committed = reverted = read_only = False
|
|
1073
|
+
witness, j = "", i + 1
|
|
1074
|
+
while j < n:
|
|
1075
|
+
row = rows[j]
|
|
1076
|
+
nxt = _human_prompt(row)
|
|
1077
|
+
if nxt and nxt in pasted:
|
|
1078
|
+
nxt = "" # a pasted turn is agent output, not a new intent
|
|
1079
|
+
if nxt:
|
|
1080
|
+
low = nxt.lower()
|
|
1081
|
+
if any(m in low for m in _ABANDON_MARKERS):
|
|
1082
|
+
break
|
|
1083
|
+
if _looks_corrective(nxt) or _same_task(opener, nxt):
|
|
1084
|
+
corrective += 1
|
|
1085
|
+
j += 1
|
|
1086
|
+
continue
|
|
1087
|
+
break # a genuinely new intent ends the episode
|
|
1088
|
+
if row.get("type") == "assistant":
|
|
1089
|
+
assistants += 1
|
|
1090
|
+
for b in _tool_uses(row):
|
|
1091
|
+
name = b.get("name")
|
|
1092
|
+
if name in _WRITE_TOOLS:
|
|
1093
|
+
wrote = True
|
|
1094
|
+
if not witness:
|
|
1095
|
+
inp = b.get("input") or {}
|
|
1096
|
+
tgt = inp.get("file_path") or inp.get("notebook_path") or "?"
|
|
1097
|
+
witness = "%s %s" % (name, os.path.basename(tgt))
|
|
1098
|
+
elif name == "Bash":
|
|
1099
|
+
cmd = (b.get("input") or {}).get("command", "") or ""
|
|
1100
|
+
if _COMMIT_RE.search(cmd):
|
|
1101
|
+
committed, witness = True, "git commit"
|
|
1102
|
+
elif _REVERT_RE.search(cmd):
|
|
1103
|
+
reverted = True
|
|
1104
|
+
else:
|
|
1105
|
+
read_only = True
|
|
1106
|
+
j += 1
|
|
1107
|
+
|
|
1108
|
+
if committed and reverted:
|
|
1109
|
+
tier = "reverted"
|
|
1110
|
+
elif committed:
|
|
1111
|
+
tier = "commit"
|
|
1112
|
+
elif wrote:
|
|
1113
|
+
tier = "artifact"
|
|
1114
|
+
else:
|
|
1115
|
+
tier = "none"
|
|
1116
|
+
witness = ("read-only Bash only, no file change" if read_only
|
|
1117
|
+
else "no tool ran after the prompt")
|
|
1118
|
+
|
|
1119
|
+
eps.append({"opener": opener, "source": os.path.basename(source),
|
|
1120
|
+
"tier": tier, "probe": _PROBE[tier],
|
|
1121
|
+
"score": 2 if tier == "commit" else (1 if tier == "artifact" else 0),
|
|
1122
|
+
"survived": tier in _DURABLE, "corrective_turns": corrective,
|
|
1123
|
+
"assistant_turns": assistants, "witness": witness})
|
|
1124
|
+
i = j if j > start else start + 1
|
|
1125
|
+
return eps
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def prompt_patterns(text):
|
|
1129
|
+
"""Mechanical, observable tags for one prompt. Every tag is a feature of the
|
|
1130
|
+
text itself, never an inferred 'style'."""
|
|
1131
|
+
tags, wc = [], len(text.split())
|
|
1132
|
+
intent = _intent(text)
|
|
1133
|
+
tags.append("intent:%s" % intent if intent else "intent:none")
|
|
1134
|
+
tags.append("names-a-concrete-object" if _objects(text) else "no-object (pronoun/vague)")
|
|
1135
|
+
tags.append("terse (<8 words)" if wc < 8 else
|
|
1136
|
+
("medium (8-40 words)" if wc <= 40 else "detailed (>40 words)"))
|
|
1137
|
+
if _FILE_RE.search(text):
|
|
1138
|
+
tags.append("cites-a-file-or-path")
|
|
1139
|
+
if _CHECK_RE.search(text.lower()):
|
|
1140
|
+
tags.append("states-a-check-or-done-condition")
|
|
1141
|
+
return tags
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def rank_patterns(episodes):
|
|
1145
|
+
buckets = {}
|
|
1146
|
+
for ep in episodes:
|
|
1147
|
+
for tag in prompt_patterns(ep["opener"]):
|
|
1148
|
+
buckets.setdefault(tag, []).append(ep)
|
|
1149
|
+
out = []
|
|
1150
|
+
for tag, eps in buckets.items():
|
|
1151
|
+
n = len(eps)
|
|
1152
|
+
durable = sum(1 for e in eps if e["survived"])
|
|
1153
|
+
out.append({"pattern": tag, "n": n, "survived": durable,
|
|
1154
|
+
"survival_rate": durable / n if n else 0.0,
|
|
1155
|
+
"rankable": n >= MIN_PATTERN_N})
|
|
1156
|
+
out.sort(key=lambda p: (p["survival_rate"], p["n"]), reverse=True)
|
|
1157
|
+
return out
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
def _pick(episodes, survived, key):
|
|
1161
|
+
cand = [e for e in episodes if e["survived"] is survived and len(e["opener"]) >= 25]
|
|
1162
|
+
if not cand:
|
|
1163
|
+
cand = [e for e in episodes if e["survived"] is survived]
|
|
1164
|
+
return max(cand, key=key) if cand else None
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def _codex_history_overlap(roots, human_texts):
|
|
1168
|
+
"""C1 control: ~/.codex/history.jsonl records the SAME typed turns the rollouts
|
|
1169
|
+
carry as user items, minus all context. Ingest it and check the overlap — it
|
|
1170
|
+
proves the gate reads the rollouts right, without letting a stale, context-free
|
|
1171
|
+
input log double-count into the episode grades. Returns (lines, matched)."""
|
|
1172
|
+
lines = matched = 0
|
|
1173
|
+
full = [t for t in human_texts if t]
|
|
1174
|
+
for r in roots:
|
|
1175
|
+
hp = os.path.join(os.path.expanduser(r), "history.jsonl")
|
|
1176
|
+
if not (os.path.isfile(hp) and _sniff(hp) == "codex-history"):
|
|
1177
|
+
continue
|
|
1178
|
+
for d in _iter_json(hp):
|
|
1179
|
+
txt = (d.get("text") or "").strip()
|
|
1180
|
+
# a one-word input ("n", "yes") is a substring of nearly every turn,
|
|
1181
|
+
# so it would match trivially and make this control unfailable. Only
|
|
1182
|
+
# count lines long enough that a match actually proves the gate read
|
|
1183
|
+
# the rollouts right, and match against the FULL turn, not a head.
|
|
1184
|
+
if len(txt) < 12:
|
|
1185
|
+
continue
|
|
1186
|
+
lines += 1
|
|
1187
|
+
if any(txt in h or h in txt for h in full):
|
|
1188
|
+
matched += 1
|
|
1189
|
+
return lines, matched
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
def _coach_roots(root=None, harness=None):
|
|
1193
|
+
"""The directories coach will read, given an explicit --root and --harness.
|
|
1194
|
+
Single source of truth so the empty-result message names the real path."""
|
|
1195
|
+
if root:
|
|
1196
|
+
return [root]
|
|
1197
|
+
return [os.path.expanduser("~/.codex")] if harness == "codex" else list(ROOTS)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def coach(roots=None, harness=None, verified_human=False):
|
|
1201
|
+
"""Rank the operator's own prompt habits by the survival proxy. Offline.
|
|
1202
|
+
|
|
1203
|
+
harness: None (auto-detect per file), 'claude', or 'codex'. A Codex root is
|
|
1204
|
+
normalised into the same rows, so the survival proxy is identical.
|
|
1205
|
+
verified_human: subtract likely-PASTED turns (echoes of earlier agent/tool
|
|
1206
|
+
output in the same session) from the human signal before grading."""
|
|
1207
|
+
if not roots:
|
|
1208
|
+
roots = _coach_roots(None, harness)
|
|
1209
|
+
paths = _coach_files(roots, harness)
|
|
1210
|
+
episodes, records, humans, pastes = [], 0, 0, 0
|
|
1211
|
+
human_texts, harnesses = [], set()
|
|
1212
|
+
for p in paths:
|
|
1213
|
+
rows, fh = _rows_for_file(p)
|
|
1214
|
+
if fh == "codex-history":
|
|
1215
|
+
continue
|
|
1216
|
+
harnesses.add(fh)
|
|
1217
|
+
records += len(rows)
|
|
1218
|
+
pasted = set()
|
|
1219
|
+
if verified_human:
|
|
1220
|
+
pasted = detect_pastes(_paste_stream(p, fh))
|
|
1221
|
+
pastes += len(pasted)
|
|
1222
|
+
for row in rows:
|
|
1223
|
+
t = _human_prompt(row)
|
|
1224
|
+
if t and t not in pasted:
|
|
1225
|
+
humans += 1
|
|
1226
|
+
human_texts.append(t)
|
|
1227
|
+
episodes += extract_episodes(rows, source=p, pasted=pasted)
|
|
1228
|
+
|
|
1229
|
+
patterns = rank_patterns(episodes)
|
|
1230
|
+
rankable = [p for p in patterns if p["rankable"]]
|
|
1231
|
+
durable = sum(1 for e in episodes if e["survived"])
|
|
1232
|
+
tiers = {t: sum(1 for e in episodes if e["tier"] == t)
|
|
1233
|
+
for t in ("commit", "artifact", "reverted", "none")}
|
|
1234
|
+
resolved = (harness or ("codex" if harnesses == {"codex"}
|
|
1235
|
+
else "claude" if harnesses == {"claude"}
|
|
1236
|
+
else "mixed" if harnesses else "claude"))
|
|
1237
|
+
hist_lines, hist_matched = (_codex_history_overlap(roots, human_texts)
|
|
1238
|
+
if "codex" in harnesses or harness == "codex"
|
|
1239
|
+
else (0, 0))
|
|
1240
|
+
return {
|
|
1241
|
+
"harness": resolved,
|
|
1242
|
+
"verified_human": verified_human, "pastes_flagged": pastes,
|
|
1243
|
+
"history_lines": hist_lines, "history_matched": hist_matched,
|
|
1244
|
+
"files": len(paths), "total_records": records, "human_turns": humans,
|
|
1245
|
+
"human_pct": round(100 * humans / records, 2) if records else 0.0,
|
|
1246
|
+
"episodes": len(episodes), "durable": durable,
|
|
1247
|
+
"durable_rate": round(durable / len(episodes), 3) if episodes else 0.0,
|
|
1248
|
+
"tiers": tiers,
|
|
1249
|
+
"top_patterns": rankable[:5],
|
|
1250
|
+
"bottom_patterns": (rankable[-5:][::-1] if len(rankable) > 5
|
|
1251
|
+
else rankable[::-1][:5]),
|
|
1252
|
+
"best_prompt": _pick(episodes, True,
|
|
1253
|
+
lambda e: (e["score"], -e["corrective_turns"],
|
|
1254
|
+
e["assistant_turns"])),
|
|
1255
|
+
"worst_prompt": _pick(episodes, False,
|
|
1256
|
+
lambda e: (e["corrective_turns"], e["assistant_turns"])),
|
|
1257
|
+
"sparse": len(rankable) < 5,
|
|
1258
|
+
"rankable_corpus": len(episodes) >= MIN_EPISODES_TO_RANK,
|
|
1259
|
+
"proxy": ("survival = a durable Write/Edit or an un-reverted git commit "
|
|
1260
|
+
"in-episode. A PROXY, not proof the work was correct or shipped."),
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
def _one_line(text, width=100):
|
|
1265
|
+
one = " ".join(text.split())
|
|
1266
|
+
return one if len(one) <= width else one[:width - 1] + "…"
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
def cmd_coach(args):
|
|
1270
|
+
r = coach([args.root] if args.root else None, harness=args.harness,
|
|
1271
|
+
verified_human=args.verified_human)
|
|
1272
|
+
if args.json:
|
|
1273
|
+
print(json.dumps(r, indent=2)); return
|
|
1274
|
+
if not r["episodes"]:
|
|
1275
|
+
# A stranger's first run lands here whenever they have no logs for this
|
|
1276
|
+
# harness. Name where we looked, so "it printed nothing" is diagnosable.
|
|
1277
|
+
looked = _coach_roots(args.root, args.harness)
|
|
1278
|
+
print("\n no prompt episodes found for harness '%s'.\n" % r["harness"])
|
|
1279
|
+
print(" looked in: %s" % (", ".join(looked) or "(no default root)"))
|
|
1280
|
+
print("\n this reads transcripts that already exist on your machine; it does not")
|
|
1281
|
+
print(" create them. if that directory is empty, use the agent for a session first.\n")
|
|
1282
|
+
print(" otherwise:")
|
|
1283
|
+
print(" %s coach --harness codex grade Codex (~/.codex) instead" % PROG)
|
|
1284
|
+
print(" %s coach --root <dir> point at a folder of .jsonl transcripts\n" % PROG)
|
|
1285
|
+
return
|
|
1286
|
+
B, D = "\033[1m", "\033[0m"
|
|
1287
|
+
print("\n %sYOUR PROMPT HABITS, GRADED%s (offline, your machine only)\n" % (B, D))
|
|
1288
|
+
print(" harness: %s" % r["harness"])
|
|
1289
|
+
print(" corpus : %s transcript(s), %s records" % (r["files"], format(r["total_records"], ",")))
|
|
1290
|
+
print(" kept : %s prompts you actually typed (%s%% of records)"
|
|
1291
|
+
% (r["human_turns"], r["human_pct"]))
|
|
1292
|
+
if r["verified_human"]:
|
|
1293
|
+
print(" pastes : %s typed turn(s) flagged likely-PASTED and subtracted "
|
|
1294
|
+
"(echoes of earlier agent/tool output)" % r["pastes_flagged"])
|
|
1295
|
+
if r["history_lines"]:
|
|
1296
|
+
print(" codex : history.jsonl ingested: %s/%s of its checkable input "
|
|
1297
|
+
"lines also appear as typed rollout turns (gate control)"
|
|
1298
|
+
% (r["history_matched"], r["history_lines"]))
|
|
1299
|
+
print(" episodes: %s ranked, %s survived (%s%%)"
|
|
1300
|
+
% (r["episodes"], r["durable"], int(round(r["durable_rate"] * 100))))
|
|
1301
|
+
t = r["tiers"]
|
|
1302
|
+
print(" tiers : commit %s | write/edit %s | reverted %s | nothing durable %s"
|
|
1303
|
+
% (t["commit"], t["artifact"], t["reverted"], t["none"]))
|
|
1304
|
+
print("\n \033[2mSURVIVAL IS A PROXY: %s\033[0m" % r["proxy"])
|
|
1305
|
+
if not r["rankable_corpus"]:
|
|
1306
|
+
# The refusal IS the product's argument, not an error. A rate needs a
|
|
1307
|
+
# denominator large enough to mean something; %d episodes is not it, and a
|
|
1308
|
+
# tool that prints "65%%" over three episodes is doing the exact thing this
|
|
1309
|
+
# one exists to catch. So no ranking. The two things below need no sample
|
|
1310
|
+
# size — one episode each — and they are the honest half of the output.
|
|
1311
|
+
print("\n %sNot enough episodes to rank your habits yet.%s" % (B, D))
|
|
1312
|
+
print(" You have %s; habits become rankable at %s. A survival percentage over"
|
|
1313
|
+
% (r["episodes"], MIN_EPISODES_TO_RANK))
|
|
1314
|
+
print(" a handful of episodes is the number this tool was built to distrust,")
|
|
1315
|
+
print(" so it will not print one. Your raw survival and your single best and")
|
|
1316
|
+
print(" worst prompt need no sample size — here they are.")
|
|
1317
|
+
else:
|
|
1318
|
+
if r["sparse"]:
|
|
1319
|
+
print("\n \033[2m(few habits cleared the %s-episode minimum, showing what is "
|
|
1320
|
+
"rankable)\033[0m" % MIN_PATTERN_N)
|
|
1321
|
+
print("\n %sSURVIVES MOST%s do more of these:" % (B, D))
|
|
1322
|
+
for p in r["top_patterns"]:
|
|
1323
|
+
print(" %3d%% (%s/%s) %s" % (round(p["survival_rate"] * 100),
|
|
1324
|
+
p["survived"], p["n"], p["pattern"]))
|
|
1325
|
+
print("\n %sSURVIVES LEAST%s these tend to loop:" % (B, D))
|
|
1326
|
+
for p in r["bottom_patterns"]:
|
|
1327
|
+
print(" %3d%% (%s/%s) %s" % (round(p["survival_rate"] * 100),
|
|
1328
|
+
p["survived"], p["n"], p["pattern"]))
|
|
1329
|
+
b = r["best_prompt"]
|
|
1330
|
+
if b:
|
|
1331
|
+
print("\n \033[32m+\033[0m your best landed prompt, with its witness:")
|
|
1332
|
+
print(" \"%s\"" % _one_line(b["opener"]))
|
|
1333
|
+
print(" %s: %s · corrections: %s" % (b["probe"], b["witness"],
|
|
1334
|
+
b["corrective_turns"]))
|
|
1335
|
+
w = r["worst_prompt"]
|
|
1336
|
+
if w:
|
|
1337
|
+
print("\n \033[31m-\033[0m your worst looped prompt, with its witness:")
|
|
1338
|
+
print(" \"%s\"" % _one_line(w["opener"]))
|
|
1339
|
+
print(" %s: %s · corrections: %s · assistant turns: %s"
|
|
1340
|
+
% (w["probe"], w["witness"], w["corrective_turns"], w["assistant_turns"]))
|
|
1341
|
+
print()
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def main():
|
|
1345
|
+
p = argparse.ArgumentParser(prog=PROG, description=__doc__ + USAGE,
|
|
1346
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
1347
|
+
sub = p.add_subparsers(dest="cmd")
|
|
1348
|
+
sub.add_parser("index").set_defaults(fn=cmd_index)
|
|
1349
|
+
s = sub.add_parser("watch"); s.add_argument("--interval", type=int, default=5); s.set_defaults(fn=cmd_watch)
|
|
1350
|
+
s = sub.add_parser("ask"); s.add_argument("query"); s.add_argument("-n", "--limit", type=int, default=25); s.set_defaults(fn=cmd_ask)
|
|
1351
|
+
s = sub.add_parser("search"); s.add_argument("query"); s.add_argument("-n", "--limit", type=int, default=25); s.set_defaults(fn=cmd_search)
|
|
1352
|
+
s = sub.add_parser("find"); s.add_argument("name"); s.set_defaults(fn=cmd_find)
|
|
1353
|
+
s = sub.add_parser("sessions"); s.add_argument("-n", "--limit", type=int, default=30); s.set_defaults(fn=cmd_sessions)
|
|
1354
|
+
sub.add_parser("stats").set_defaults(fn=cmd_stats)
|
|
1355
|
+
s = sub.add_parser("cost")
|
|
1356
|
+
s.add_argument("--days", type=int, default=30, help="window in days (0 = all time)")
|
|
1357
|
+
s.add_argument("--root", help="scan this transcript dir instead of ~/.claude/projects")
|
|
1358
|
+
s.add_argument("--json", action="store_true", help="machine-readable, for other tools")
|
|
1359
|
+
s.set_defaults(fn=cmd_cost)
|
|
1360
|
+
s = sub.add_parser("coach")
|
|
1361
|
+
s.add_argument("--root", help="grade this transcript dir instead of ~/.claude/projects")
|
|
1362
|
+
s.add_argument("--harness", choices=["claude", "codex"],
|
|
1363
|
+
help="which agent's transcripts to grade. codex reads "
|
|
1364
|
+
"~/.codex (archived_sessions + sessions). default: auto-detect")
|
|
1365
|
+
s.add_argument("--verified-human", dest="verified_human", action="store_true",
|
|
1366
|
+
help="subtract likely-PASTED turns: a typed turn whose text is a "
|
|
1367
|
+
"verbatim/high n-gram echo of an earlier agent or tool message "
|
|
1368
|
+
"in the same session, from the human signal before grading")
|
|
1369
|
+
s.add_argument("--json", action="store_true", help="machine-readable, for other tools")
|
|
1370
|
+
s.set_defaults(fn=cmd_coach)
|
|
1371
|
+
a = p.parse_args()
|
|
1372
|
+
if not getattr(a, "fn", None):
|
|
1373
|
+
p.print_help(); return
|
|
1374
|
+
a.fn(a)
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
if __name__ == "__main__":
|
|
1378
|
+
main()
|