debrief-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
debrief/sources.py ADDED
@@ -0,0 +1,341 @@
1
+ """Load sessions straight from agent logs on disk: no Entire account needed.
2
+
3
+ Reads Claude Code logs (~/.claude/projects), Codex logs (~/.codex/sessions) and any folder
4
+ of .jsonl logs given with --import (Claude Code, Codex or Pi; the format is detected per
5
+ file). A re-run reads only files whose size or modified time changed, and replaces those
6
+ sessions' rows.
7
+
8
+ uv run python -m debrief.sources
9
+ uv run python -m debrief.sources --project ~/Dev/myrepo --since 2026-10-01
10
+ uv run python -m debrief.sources --import ~/Downloads/logs
11
+
12
+ Left out on purpose, and counted in the summary:
13
+ sub-agent logs Claude Code keeps each sub-agent's turns in its own file. No person
14
+ typed them, and the parent session already records the call and its result.
15
+ headless runs `claude -p` and `codex exec`: debrief's own model calls are such runs,
16
+ and must not come back in as sessions. Also sessions a program started
17
+ (through the Claude SDK, or a Codex wrapper app) with fewer than
18
+ two human turns.
19
+ --include-headless loads them all.
20
+ no human turn empty files, metadata-only files, and copies with nothing new in them.
21
+ copied entries a resumed or forked session's log starts with a copy of the earlier
22
+ log. Each entry loads into one session only, so no turn counts twice.
23
+ A session still running loads as far as it has got; the next run replaces it.
24
+ """
25
+
26
+ import argparse
27
+ import hashlib
28
+ import json
29
+ import re
30
+ from collections import Counter
31
+ from datetime import datetime
32
+ from functools import cache
33
+ from pathlib import Path
34
+
35
+ from debrief import db
36
+ from debrief.normalize import PARSER_VERSION, PARSERS, read_jsonl, safe_session_id, write_log
37
+
38
+ CLAUDE_DIR = Path.home() / ".claude" / "projects"
39
+ CODEX_DIR = Path.home() / ".codex" / "sessions"
40
+ # How the agent was started, as its log records it. `claude -p` logs entrypoint "sdk-cli";
41
+ # `codex exec` logs source "exec". Other "sdk-*" entrypoints are programs driving Claude Code:
42
+ # mostly one-prompt jobs, but also wrapper apps a person talks through. Those count as
43
+ # headless only below SDK_MIN_TURNS human turns. Codex is judged the same way by
44
+ # session_meta.originator: its own front ends name themselves "codex...", anything else
45
+ # is a wrapper. (`source` is no help: wrapper apps log "vscode", as the IDE extension does.)
46
+ HEADLESS_CLAUDE = {"sdk-cli"}
47
+ SDK_MIN_TURNS = 2
48
+ HEADLESS_CODEX = {"exec"}
49
+ GITDIR = re.compile(r"gitdir: (.+?)/\.git/worktrees/")
50
+ GITHUB_URL = re.compile(r"github\.com[:/]([\w.-]+/[\w.-]+?)(?:\.git)?/?$")
51
+ # The main repo root an earlier load recorded for a folder: git's own answer, kept for when the folder is gone.
52
+ REMEMBERED = """SELECT s.cwd FROM source_logs l JOIN sessions s USING (session_id)
53
+ WHERE l.status = 'loaded' AND l.cwd = ? AND s.cwd != l.cwd LIMIT 1"""
54
+
55
+ # The owner of an entry is the session the entry itself names (native), else the lowest
56
+ # session ID: a rule that gives the same answer whatever order the logs load in.
57
+ # BEATEN is true for a row `a` of source_entries when another session owns its entry.
58
+ BEATEN = """EXISTS (
59
+ SELECT 1 FROM source_entries b WHERE b.entry_key = a.entry_key AND b.session_id != a.session_id
60
+ AND (b.native > a.native OR (b.native = a.native AND b.session_id < a.session_id)))"""
61
+ FOREIGN = f"SELECT a.entry_key FROM source_entries a WHERE a.session_id = ? AND {BEATEN}"
62
+ # Loaded sessions whose count of such entries is no longer the count they were loaded with.
63
+ # shortcut: compares counts, not sets; exact while logs only grow. Compare sets if agents start rewriting logs.
64
+ STALE = f"""SELECT path, agent FROM source_logs l WHERE status = 'loaded' AND dropped !=
65
+ (SELECT COUNT(*) FROM source_entries a WHERE a.session_id = l.session_id AND {BEATEN})"""
66
+
67
+
68
+ def sniff(entries: list[dict]) -> str | None:
69
+ """Which agent wrote these entries, or None."""
70
+ for e in entries[:50]:
71
+ if e.get("type") == "session_meta":
72
+ return "Codex"
73
+ if e.get("type") == "session" and "cwd" in e:
74
+ return "Pi"
75
+ if "sessionId" in e or "parentUuid" in e:
76
+ return "Claude Code"
77
+ return None
78
+
79
+
80
+ def entry_key(agent: str, e: dict) -> str | None:
81
+ """An ID that a copy of this entry in another log shares; None for entries no parser reads."""
82
+ if agent == "Claude Code":
83
+ return e.get("uuid") if e.get("type") in ("user", "assistant", "attachment", "system") else None
84
+ if agent == "Pi": # Pi IDs are 8 characters, too short to be unique across sessions
85
+ return f"{e['id']}:{e.get('timestamp')}" if e.get("id") and e.get("type") != "session" else None
86
+ if e.get("type") == "response_item": # Codex entries carry no ID
87
+ return hashlib.sha1(json.dumps([e.get("timestamp"), e.get("payload")], sort_keys=True).encode()).hexdigest()
88
+ return None
89
+
90
+
91
+ def most_common(values: list) -> str | None:
92
+ return Counter(values).most_common(1)[0][0] if values else None
93
+
94
+
95
+ def facts(agent: str, entries: list[dict], stem: str) -> dict:
96
+ """What the log says about its session: id, cwd, launcher (as logged; see is_headless),
97
+ model, title, tokens in and out, and remote (the GitHub owner/name of its checkout; only
98
+ Codex logs record one)."""
99
+ f = {"id": stem, "cwd": "", "launcher": "", "model": None, "title": None, "in": None, "out": None, "remote": ""}
100
+ if agent == "Claude Code":
101
+ f["cwd"] = next((e["cwd"] for e in entries if e.get("cwd")), "")
102
+ f["launcher"] = ",".join(sorted({str(e["entrypoint"]) for e in entries if e.get("entrypoint")}))
103
+ for kind, field in (("ai-title", "aiTitle"), ("custom-title", "customTitle")): # a custom title wins
104
+ f["title"] = next((e[field] for e in reversed(entries) if e.get("type") == kind and e.get(field)), f["title"])
105
+ replies = [e["message"] for e in entries if e.get("type") == "assistant" and not e.get("isSidechain")
106
+ and isinstance(e.get("message"), dict) and e["message"].get("model") != "<synthetic>"]
107
+ f["model"] = most_common([m["model"] for m in replies if m.get("model")])
108
+ # One entry per content block, each repeating the reply's usage: keep the last per reply.
109
+ usage = {m.get("id") or i: m["usage"] for i, m in enumerate(replies) if isinstance(m.get("usage"), dict)}
110
+ if usage:
111
+ f["in"] = sum(u.get(k) or 0 for u in usage.values()
112
+ for k in ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"))
113
+ f["out"] = sum(u.get("output_tokens") or 0 for u in usage.values())
114
+ elif agent == "Codex":
115
+ payloads = [(e.get("type"), e["payload"]) for e in entries if isinstance(e.get("payload"), dict)]
116
+ meta = next((p for t, p in payloads if t == "session_meta"), {})
117
+ f["id"], f["cwd"] = meta.get("id") or stem, meta.get("cwd") or ""
118
+ f["launcher"] = f"{meta.get('originator') or ''}|{meta.get('source') or ''}"
119
+ git = meta.get("git") if isinstance(meta.get("git"), dict) else {}
120
+ url = GITHUB_URL.search(str(git.get("repository_url") or ""))
121
+ f["remote"] = url.group(1) if url else ""
122
+ f["model"] = most_common([p["model"] for t, p in payloads if t == "turn_context" and p.get("model")])
123
+ totals = [p["info"]["total_token_usage"] for t, p in payloads if p.get("type") == "token_count"
124
+ and isinstance(p.get("info"), dict) and isinstance(p["info"].get("total_token_usage"), dict)]
125
+ if totals: # a running total; input_tokens already counts cached input
126
+ f["in"], f["out"] = totals[-1].get("input_tokens"), totals[-1].get("output_tokens")
127
+ elif agent == "Pi":
128
+ head = next((e for e in entries if e.get("type") == "session"), {})
129
+ f["id"], f["cwd"] = head.get("id") or stem, head.get("cwd") or ""
130
+ replies = [e["message"] for e in entries if e.get("type") == "message"
131
+ and isinstance(e.get("message"), dict) and e["message"].get("role") == "assistant"]
132
+ f["model"] = most_common([m["model"] for m in replies if m.get("model")])
133
+ usage = [m["usage"] for m in replies if isinstance(m.get("usage"), dict)]
134
+ if usage:
135
+ f["in"] = sum(u.get(k) or 0 for u in usage for k in ("input", "cacheRead", "cacheWrite"))
136
+ f["out"] = sum(u.get("output") or 0 for u in usage)
137
+ return f
138
+
139
+
140
+ @cache
141
+ def repo_roots(cwd: str) -> tuple[str, str]:
142
+ """(checkout root, main repo root) for a directory, read from disk; (cwd, cwd) when it is
143
+ not in a git checkout or no longer exists. The two differ for a git worktree, whose .git
144
+ is a file pointing into the main repo. Sessions of one project share the main repo root."""
145
+ try:
146
+ for d in (Path(cwd), *Path(cwd).parents) if cwd else ():
147
+ git = d / ".git"
148
+ if git.is_dir():
149
+ return str(d), str(d)
150
+ if git.is_file():
151
+ m = GITDIR.match(git.read_text())
152
+ return str(d), str((d / m.group(1)).resolve()) if m else str(d)
153
+ except OSError:
154
+ pass
155
+ return cwd, cwd
156
+
157
+
158
+ def is_headless(agent: str | None, launcher: str, human_turns: int | None, include_headless: bool) -> bool:
159
+ """The headless rule. It reads only what source_logs stores (the launcher as the log gives
160
+ it, and the turn count), so a change to the rule or to --include-headless can be applied
161
+ to a log that has not changed without reading it. human_turns None means not parsed yet."""
162
+ always = wrapped = False
163
+ if agent == "Claude Code": # launcher: the log's entrypoints, comma-separated
164
+ entrypoints = set(filter(None, launcher.split(",")))
165
+ always = bool(entrypoints) and entrypoints <= HEADLESS_CLAUDE
166
+ wrapped = bool(entrypoints) and all(e.startswith("sdk-") for e in entrypoints)
167
+ elif agent == "Codex": # launcher: "originator|source"
168
+ originator, _, source = launcher.partition("|")
169
+ always = source in HEADLESS_CODEX
170
+ wrapped = bool(originator) and not originator.lower().startswith("codex")
171
+ if include_headless:
172
+ return False
173
+ return always or (wrapped and human_turns is not None and human_turns < SDK_MIN_TURNS)
174
+
175
+
176
+ def unload(conn, session_id: str) -> None:
177
+ """Remove a session's rows. A log that is not loaded must not own entries either. Its labels
178
+ go too: they point at message positions, which a later reload would number afresh."""
179
+ with conn:
180
+ for table in ("source_entries", "sessions", "messages", "tool_calls", "turn_labels", "session_labels"):
181
+ conn.execute(f"DELETE FROM {table} WHERE session_id = ?", (session_id,))
182
+
183
+
184
+ def in_project(project: str | None, *dirs: str) -> bool:
185
+ if not project:
186
+ return True
187
+ root = str(Path(project).expanduser().resolve())
188
+ return any(d == root or d.startswith(root + "/") for d in dirs)
189
+
190
+
191
+ def load_log(conn, path: Path, agent: str | None = None, project: str | None = None,
192
+ include_headless: bool = False) -> str:
193
+ """Load one log file and record it in source_logs. Returns its status: "loaded", or why not."""
194
+ stat = path.stat()
195
+ entries = [e for e in read_jsonl(path, tolerant=True) if isinstance(e, dict)]
196
+ agent = agent or sniff(entries)
197
+ f = facts(agent, entries, path.stem) if agent else {"id": None, "cwd": "", "launcher": ""}
198
+ session_id, (checkout, repo) = f["id"], repo_roots(f["cwd"])
199
+ if repo and repo == f["cwd"] and not Path(repo, ".git").exists():
200
+ # Disk no longer says which repo this folder belongs to (a removed worktree). An earlier
201
+ # load of a session run there, or in a folder above it, may have: use its answer.
202
+ known = (conn.execute(REMEMBERED, (str(d),)).fetchone() for d in (Path(repo), *Path(repo).parents))
203
+ repo = next((r[0] for r in known if r), repo)
204
+
205
+ def done(status: str, dropped: int = 0, human_turns: int | None = None) -> str:
206
+ with conn:
207
+ conn.execute(
208
+ "INSERT OR REPLACE INTO source_logs (path, session_id, agent, size, mtime, status, cwd, dropped,"
209
+ " launcher, human_turns, parser) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
210
+ (str(path), session_id, agent, stat.st_size, stat.st_mtime_ns, status, f["cwd"], dropped,
211
+ f["launcher"], human_turns, PARSER_VERSION))
212
+ return status
213
+
214
+ turns = [e for e in entries if e.get("type") in ("user", "assistant")]
215
+ if agent is None:
216
+ return done("unknown format" if entries else "no human turn")
217
+ if agent == "Claude Code" and turns and all(e.get("isSidechain") for e in turns):
218
+ return done("sub-agent log")
219
+ if not safe_session_id(session_id):
220
+ return done("unsafe session ID")
221
+ if is_headless(agent, f["launcher"], None, include_headless):
222
+ return done("headless run") # known from the launcher alone, so not parsed
223
+ if not in_project(project, f["cwd"], repo):
224
+ return done("outside --project")
225
+ # The same session ID in two files (a session moved between project folders): the bigger file wins.
226
+ others = conn.execute("SELECT path, size FROM source_logs WHERE session_id = ? AND path != ? AND status = 'loaded'",
227
+ (session_id, str(path))).fetchall()
228
+ if any((o["size"], str(path)) > (stat.st_size, o["path"]) for o in others if Path(o["path"]).exists()):
229
+ return done("duplicate file")
230
+
231
+ keys = [entry_key(agent, e) for e in entries]
232
+ with conn:
233
+ conn.execute("DELETE FROM source_entries WHERE session_id = ?", (session_id,))
234
+ conn.executemany("INSERT OR REPLACE INTO source_entries VALUES (?,?,?)",
235
+ {(k, session_id, int(e.get("sessionId", session_id) == session_id))
236
+ for k, e in zip(keys, entries) if k})
237
+ foreign = {r[0] for r in conn.execute(FOREIGN, (session_id,))}
238
+ kept = [e for k, e in zip(keys, entries) if k not in foreign]
239
+ log = PARSERS[agent](kept)
240
+ prompts = [m["text"] for m in log.messages if m["user_kind"] == "human"]
241
+ # Judged on every read, so a one-turn wrapped session that gains a second turn loads on the next run.
242
+ if not prompts or is_headless(agent, f["launcher"], len(prompts), include_headless):
243
+ unload(conn, session_id) # it may hold rows from an earlier load
244
+ return done("headless run" if prompts else "no human turn", human_turns=len(prompts))
245
+ prompt = prompts[0]
246
+ f = facts(agent, kept, path.stem) | {k: f[k] for k in ("cwd", "launcher", "remote")} | {"id": session_id}
247
+ stamps = [e["timestamp"] for e in kept if isinstance(e.get("timestamp"), str)]
248
+ lines = prompt.strip().splitlines()
249
+ write_log(conn, (
250
+ session_id, agent, f["model"], f["title"] or (lines[0][:80] if lines else None), None,
251
+ min(stamps, default=None), max(stamps, default=None), None, f["in"], f["out"], None, None, prompt, repo,
252
+ ), log, root=checkout)
253
+ with conn:
254
+ conn.execute("UPDATE sessions SET remote = ? WHERE session_id = ?", (f["remote"], session_id))
255
+ conn.execute("UPDATE source_logs SET status = 'duplicate file' WHERE session_id = ? AND path != ?",
256
+ (session_id, str(path)))
257
+ return done("loaded", len(foreign), len(prompts))
258
+
259
+
260
+ def find_logs(claude_dir: Path | None, codex_dir: Path | None, import_dirs: list[Path]) -> list[tuple[Path, str | None]]:
261
+ """(path, agent) for every log file; agent is None where load_log must detect it."""
262
+ found = []
263
+ for root, agent in ((claude_dir, "Claude Code"), (codex_dir, "Codex"), *((d, None) for d in import_dirs)):
264
+ if root:
265
+ found += [(p, agent) for p in sorted(Path(root).expanduser().glob("**/*.jsonl")) if p.is_file()]
266
+ return found
267
+
268
+
269
+ def sync(conn, claude_dir: Path | None = CLAUDE_DIR, codex_dir: Path | None = CODEX_DIR,
270
+ import_dirs: list[Path] = (), project: str | None = None, since: str | None = None,
271
+ include_headless: bool = False) -> Counter:
272
+ """Load every new or changed log. Returns a count of files per status."""
273
+ cutoff = datetime.fromisoformat(since).timestamp() if since else 0
274
+ known = {r["path"]: r for r in conn.execute("SELECT * FROM source_logs")}
275
+ # Codex sessions loaded before sessions.remote existed: read their logs once more to fill it.
276
+ no_remote = {r[0] for r in conn.execute("SELECT session_id FROM sessions WHERE agent = 'Codex' AND remote IS NULL")}
277
+ counts: Counter = Counter()
278
+ changed = False
279
+ for path, agent in find_logs(claude_dir, codex_dir, list(import_dirs)):
280
+ try:
281
+ stat = path.stat()
282
+ if stat.st_mtime < cutoff:
283
+ counts["older than --since"] += 1
284
+ continue
285
+ old = known.get(str(path))
286
+ # A row from before the launcher column existed says too little, and one read by an
287
+ # older parser may hold rows the parser would no longer produce: read those logs once more.
288
+ if old and old["launcher"] is not None and old["parser"] == PARSER_VERSION and (
289
+ old["size"], old["mtime"]) == (stat.st_size, stat.st_mtime_ns) and not (
290
+ old["status"] == "loaded" and old["session_id"] in no_remote):
291
+ # Unchanged, so not read; but the rule and flags apply afresh to what the row records.
292
+ headless = is_headless(old["agent"], old["launcher"], old["human_turns"], include_headless)
293
+ if old["status"] == "loaded" and headless:
294
+ unload(conn, old["session_id"])
295
+ with conn:
296
+ conn.execute("UPDATE source_logs SET status = 'headless run' WHERE path = ?", (str(path),))
297
+ counts["headless run"] += 1
298
+ changed = True # sessions holding copies of its entries now own them
299
+ continue
300
+ # Read it again only if what kept it out no longer does.
301
+ retry = (old["status"] == "headless run" and not headless) or (
302
+ old["status"] == "outside --project" and in_project(project, old["cwd"], repo_roots(old["cwd"])[1]))
303
+ if not retry:
304
+ counts["unchanged" if old["status"] == "loaded" else old["status"]] += 1
305
+ continue
306
+ status = load_log(conn, path, agent, project, include_headless)
307
+ changed = changed or status == "loaded"
308
+ except (ValueError, KeyError, TypeError, AttributeError, OSError) as e: # one bad log shouldn't stop the rest
309
+ status = "failed"
310
+ print(f"failed {path}: {type(e).__name__}: {e}")
311
+ counts[status] += 1
312
+ # A newly loaded log can take over entries that a session loaded earlier holds a copy of.
313
+ for _ in range(3 if changed else 0):
314
+ stale = [r for r in conn.execute(STALE).fetchall() if Path(r["path"]).exists()]
315
+ if not stale:
316
+ break
317
+ for r in stale:
318
+ load_log(conn, Path(r["path"]), r["agent"], include_headless=include_headless)
319
+ counts["reloaded without copied entries"] += len(stale)
320
+ return counts
321
+
322
+
323
+ def main(argv: list[str] | None = None) -> None:
324
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
325
+ parser.add_argument("--claude-dir", type=Path, default=CLAUDE_DIR, help="Claude Code logs (default: %(default)s)")
326
+ parser.add_argument("--codex-dir", type=Path, default=CODEX_DIR, help="Codex logs (default: %(default)s)")
327
+ parser.add_argument("--import", dest="import_dirs", type=Path, action="append", default=[], metavar="DIR",
328
+ help="Also load .jsonl logs from this folder; repeat for more")
329
+ parser.add_argument("--project", metavar="DIR", help="Only sessions run in this directory or below it")
330
+ parser.add_argument("--since", metavar="YYYY-MM-DD", help="Only logs modified on or after this date")
331
+ parser.add_argument("--include-headless", action="store_true", help="Also load `claude -p`, `codex exec` and one-turn wrapper-app runs")
332
+ args = parser.parse_args(argv)
333
+
334
+ counts = sync(db.connect(), args.claude_dir, args.codex_dir, args.import_dirs, args.project, args.since,
335
+ args.include_headless)
336
+ skipped = {"sub-agent log": "sub-agent logs skipped", "headless run": "headless runs skipped"}
337
+ print(", ".join(f"{n} {skipped.get(status, status)}" for status, n in sorted(counts.items())) or "no logs found")
338
+
339
+
340
+ if __name__ == "__main__":
341
+ main()