debrief-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
debrief/normalize.py ADDED
@@ -0,0 +1,635 @@
1
+ """Load Entire's raw/<session_id>/ folders into the sessions, checkpoints, messages and tool_calls tables.
2
+
3
+ Logs read straight from disk (Claude Code, Codex, Pi) go through debrief.sources, which
4
+ uses the parsers and write_log here.
5
+
6
+ Messages and tool calls come from the agent's native log (raw.jsonl), which keeps tool
7
+ output; checkpoints come from transcript.json and session metadata from listing.json.
8
+ When the raw log is missing (too big to fetch), messages and tool calls come from
9
+ transcript.json instead, mostly without tool output.
10
+ Re-running replaces each session's rows.
11
+
12
+ uv run python -m debrief.normalize
13
+ uv run python -m debrief.normalize --ids efd45cc6-c764-4e22-80eb-cdbc1f9173f9
14
+ """
15
+
16
+ import argparse
17
+ import gzip
18
+ import json
19
+ import posixpath
20
+ import re
21
+ import shlex
22
+ from datetime import datetime
23
+ from pathlib import Path
24
+
25
+ from debrief import db
26
+ from debrief.paths import RAW_DIR
27
+
28
+ OUTPUT_LIMIT = 4000
29
+ # Raise this when a change here alters the rows a log produces: debrief.sources then reads
30
+ # every log again, changed on disk or not. (main() below reloads Entire folders on every run.)
31
+ PARSER_VERSION = 2
32
+
33
+ # Text the harness puts in a user turn. A user message starting with one of these is "injected".
34
+ INJECTED_PREFIXES = (
35
+ "<command-name>", "<command-message>", "<command-args>", "<local-command-stdout>",
36
+ "<local-command-stderr>", "<local-command-caveat>", "Caveat: The messages below",
37
+ "<system-reminder>", "<task-notification>", "<user-prompt-submit-hook>", "<bash-stdout>",
38
+ "<bash-stderr>", "[Request interrupted by user", "Base directory for this skill",
39
+ "This session is being continued from a previous conversation", "Another Claude session sent a message",
40
+ "# AGENTS.md instructions", "<environment_context>", "<turn_aborted>", "<user_instructions>",
41
+ # Claude Code /goal hooks and Codex goal turns: the harness restates or enforces a goal.
42
+ "Stop hook feedback:", "A session-scoped Stop hook is now active", "<codex_internal_context",
43
+ )
44
+
45
+ # Tools whose path argument names a file they read or write (lower-cased names).
46
+ FILE_TOOLS = {"read", "edit", "write", "multiedit", "notebookedit"}
47
+ WRITE_TOOLS = {"edit", "write", "multiedit", "notebookedit", "apply_patch"}
48
+ # Codex scripts embed the patch in a JS string, so a header can end at a literal "\\n".
49
+ PATCH_HEADER = re.compile(r"\*\*\* (?:(?:Add|Update|Delete) File|Move to): (.+?)\s*(?=\\n|\n|\"|$)")
50
+ # Tools that run a shell command; shell_writes() finds the files such a command writes.
51
+ SHELL_TOOLS = {"bash", "shell", "exec_command", "local_shell"}
52
+ # A heredoc: the body is data, not commands, so it goes before the command is read.
53
+ HEREDOC = re.compile(r"(<<-?\s*(['\"]?)(\w+)\2[^\n]*)\n.*?\n[ \t]*\3[ \t]*(?=\n|$)", re.DOTALL)
54
+ SHELL_BREAK = {"|", "||", "&&", ";", ";;", "&", "|&", "(", ")"}
55
+ # Redirect targets that are not files worth counting: devices and the system's temp folders.
56
+ NOT_A_FILE = re.compile(r"/dev/|/tmp/|/private/tmp/|/var/folders/|/private/var/folders/")
57
+ # A shell command inside a Codex exec script, as a plain double-quoted string.
58
+ SCRIPT_CMD = re.compile(r'\bcmd\s*:\s*("(?:[^"\\\n]|\\.)*")')
59
+ SCRIPT_WORKDIR = re.compile(r'\bworkdir\s*:\s*("(?:[^"\\\n]|\\.)*")')
60
+ # Paths inside a git worktree other than the session's cwd, e.g. .../.worktrees/<name>/cmd/x.go.
61
+ WORKTREE = re.compile(r".*/(?:\.worktrees/[^/]+|\.herdr/worktrees/[^/]+/[^/]+)/")
62
+ # Commands for which exit 1 means "nothing matched" or "false" rather than failure.
63
+ # find also exits 1 on real errors, so it only counts when the output shows none.
64
+ SEARCH_CMD = re.compile(r"(?:\w+=\S*\s+)*(?:rg|grep|egrep|fgrep|git\s+grep|test|\[)(?=\s|$)")
65
+ FIND_CMD = re.compile(r"(?:\w+=\S*\s+)*(?:rg|grep|egrep|fgrep|git\s+grep|find|test|\[)(?=\s|$)")
66
+ QUOTED = re.compile(r"'[^']*'|\"(?:[^\"\\\\]|\\\\.)*\"")
67
+ SHELL_SPLIT = re.compile(r"&&|\|\||[;|\n]")
68
+ HOUSEKEEPING_COMMANDS = {
69
+ "login", "logout", "clear", "compact", "model", "config", "status", "cost", "resume",
70
+ "exit", "help", "mcp", "permissions", "memory", "doctor", "fast", "effort", "context", "copy", "reload-skills",
71
+ }
72
+ DUPLICATE_WINDOW = 60 # seconds within which an identical repeated user message counts once
73
+ SESSION_ID = re.compile(r"[0-9a-zA-Z_-]{1,64}")
74
+ ERROR_TEXT = re.compile(r"error|fatal|panic|traceback|no such file|permission denied|\bFAIL\b", re.IGNORECASE)
75
+
76
+
77
+ class Log:
78
+ """Messages and tool calls collected from one raw log, in order."""
79
+
80
+ def __init__(self) -> None:
81
+ self.cwd = ""
82
+ self.messages: list[dict] = []
83
+ self.calls: list[dict] = []
84
+ self.by_id: dict[str, dict] = {}
85
+
86
+ def add_message(self, role: str, ts: str, text: str, user_kind: str | None = None) -> int:
87
+ last = self.messages[-1] if self.messages else None
88
+ if role == "user" and last and last["role"] == "user" and last["text"] == text and within(last["ts"], ts):
89
+ return len(self.messages) - 1 # the same prompt submitted again before any reply
90
+ self.messages.append({"role": role, "user_kind": user_kind, "ts": ts, "text": text})
91
+ return len(self.messages) - 1
92
+
93
+ def assistant_idx(self, ts: str) -> int:
94
+ """The assistant message a tool call belongs to: the last message, or a new empty one."""
95
+ if self.messages and self.messages[-1]["role"] == "assistant":
96
+ return len(self.messages) - 1
97
+ return self.add_message("assistant", ts, "")
98
+
99
+ def add_call(self, msg_idx: int, call_id: str, name: str, args, patch: str = "") -> dict:
100
+ files, written = [], []
101
+ if isinstance(args, dict):
102
+ if name.lower() in FILE_TOOLS:
103
+ path = args.get("file_path") or args.get("path") or args.get("notebook_path")
104
+ if path:
105
+ files.append(path)
106
+ elif name.lower() in SHELL_TOOLS:
107
+ command = args.get("command") or args.get("cmd")
108
+ if isinstance(command, list): # ["bash", "-lc", script], or the argv itself
109
+ command = command[-1] if command[:1] in (["bash"], ["sh"], ["zsh"]) else shlex.join(map(str, command))
110
+ workdir = args.get("workdir") if isinstance(args.get("workdir"), str) else ""
111
+ written = shell_writes(command if isinstance(command, str) else "", workdir or self.cwd)
112
+ elif name == "apply_patch" and not patch: # the patch as a plain argument
113
+ patch = next((v for v in (args.get("input"), args.get("patch")) if isinstance(v, str)), "")
114
+ if name == "exec": # a Codex script: read the shell commands it passes as plain strings
115
+ workdirs = set(SCRIPT_WORKDIR.findall(patch))
116
+ workdir = unquote(workdirs.pop()) if len(workdirs) == 1 else ""
117
+ for literal in SCRIPT_CMD.findall(patch):
118
+ written += shell_writes(unquote(literal), workdir or self.cwd)
119
+ files += written + PATCH_HEADER.findall(patch)
120
+ call = {
121
+ "msg_idx": msg_idx, "name": name, "input": json.dumps(args), "output": None,
122
+ "is_error": 0, "files": files, "writes": int(name.lower() in WRITE_TOOLS or bool(written)),
123
+ "exits": [], # (exit code, real failure) for each shell command the call ran
124
+ }
125
+ self.calls.append(call)
126
+ self.by_id[call_id] = call
127
+ return call
128
+
129
+ def set_output(self, call_id: str | None, output: str, is_error: bool) -> None:
130
+ call = self.by_id.get(call_id)
131
+ if call:
132
+ call["output"] = truncate(output)
133
+ call["is_error"] = int(bool(is_error) or call["is_error"])
134
+
135
+ def add_exit(self, call: dict | None, code: int, command: str, output: str) -> None:
136
+ if call is not None:
137
+ call["exits"].append((code, code != 0 and not quiet_search(code, command, output)))
138
+
139
+
140
+ def unquote(literal: str) -> str:
141
+ """The value of a double-quoted string literal, or "" when it is not plain JSON."""
142
+ try:
143
+ return json.loads(literal)
144
+ except json.JSONDecodeError:
145
+ return ""
146
+
147
+
148
+ def literal_path(token: str) -> bool:
149
+ """A token that names one file outright: plain path characters only, so no variable,
150
+ substitution, glob or space, and not a device or temp file."""
151
+ return bool(re.fullmatch(r"[\w./~@+-]*[\w/]", token)) and not NOT_A_FILE.match(token)
152
+
153
+
154
+ def shell_writes(command: str, cwd: str = "") -> list[str]:
155
+ """Files a shell command writes, for the forms that name the file in the command text:
156
+ `> path` and `>> path` (so `cat > path <<EOF` and `echo x > path` too), `tee path`,
157
+ `sed -i ... path`, and an apply_patch heredoc. A relative path is taken from the last
158
+ `cd` to a literal directory in the command, else from cwd. Anything else a command may
159
+ write (python or perl scripts, cp, mv, git, a path held in a variable) is left out
160
+ rather than guessed. Returns [] for a command that cannot be split into words."""
161
+ found = PATCH_HEADER.findall(command) if "apply_patch" in command else []
162
+ text = HEREDOC.sub(r"\1", command).replace("\\\n", " ")
163
+ lexer = shlex.shlex(text.replace("\n", " ; "), posix=True, punctuation_chars=True)
164
+ lexer.whitespace_split = True
165
+ try:
166
+ tokens = list(lexer)
167
+ except ValueError: # an unclosed quote: not something to guess at
168
+ tokens = []
169
+ here, words = "", [] # the directory a `cd` moved to; the words of the simple command so far
170
+
171
+ def end_command() -> None:
172
+ nonlocal here
173
+ name, args = (words[0], words[1:]) if words else ("", [])
174
+ if name == "cd" and len(args) == 1 and literal_path(args[0]):
175
+ here = posixpath.join(here, args[0])
176
+ elif name == "tee":
177
+ found.extend(posixpath.join(here, a) for a in args if not a.startswith("-") and literal_path(a))
178
+ elif name == "sed" and any(re.match(r"-[a-zA-Z]*i|--in-place", a) for a in args):
179
+ # Operands are the script, then the files; with -e or -f every operand is a file.
180
+ operands, skip = [], False
181
+ for i, a in enumerate(args):
182
+ if skip or not a:
183
+ skip = False
184
+ elif a in ("-e", "-f"):
185
+ skip = True
186
+ elif not a.startswith("-") and not (a.startswith(".") and "/" not in a and re.match(r"-[a-zA-Z]*i$", args[i - 1])):
187
+ operands.append(a) # not an option, nor BSD sed's backup suffix after -i
188
+ has_script_option = any(a in ("-e", "-f") or a.startswith(("--expression", "--file")) for a in args)
189
+ found.extend(posixpath.join(here, a) for a in operands[0 if has_script_option else 1:] if literal_path(a))
190
+ words.clear()
191
+
192
+ i = 0
193
+ while i < len(tokens):
194
+ t = tokens[i]
195
+ if t in (">", ">>") and i + 1 < len(tokens):
196
+ if literal_path(tokens[i + 1]):
197
+ found.append(posixpath.join(here, tokens[i + 1]))
198
+ if words and words[-1].isdigit(): # the 2 of `2> path` is not an argument
199
+ words.pop()
200
+ i += 1
201
+ elif t in SHELL_BREAK:
202
+ end_command()
203
+ elif not set(t) <= set("<>&|;()"): # other redirects (<, <<, >&, &>) are skipped, target and all
204
+ words.append(t)
205
+ elif i + 1 < len(tokens):
206
+ i += 1
207
+ i += 1
208
+ end_command()
209
+ # "~/x" is left as written: the log does not say where home is.
210
+ return list(dict.fromkeys(f if f.startswith("~") or not cwd else posixpath.normpath(posixpath.join(cwd, f)) for f in found))
211
+
212
+
213
+ def quiet_search(code: int, command: str, output: str) -> bool:
214
+ """Exit 1 that means a search matched nothing: the last command was rg/grep/test, or a
215
+ search command ran and the output shows no error."""
216
+ if code != 1:
217
+ return False
218
+ unquoted = QUOTED.sub("''", command) # so a | inside a pattern does not split the command
219
+ segments = [s.strip().lstrip("(! ") for s in SHELL_SPLIT.split(unquoted) if s.strip()]
220
+ if segments and SEARCH_CMD.match(segments[-1]):
221
+ return True
222
+ return any(FIND_CMD.match(s) for s in segments) and not ERROR_TEXT.search(output)
223
+
224
+
225
+ def exit_status(call: dict) -> tuple[int | None, int]:
226
+ """(exit_code, is_error) for a call: the first real failure's code wins, else the highest code."""
227
+ exits = call["exits"]
228
+ failed = [code for code, real in exits if real]
229
+ code = failed[0] if failed else max((c for c, _ in exits), default=None)
230
+ if call["is_error"] is None and not exits: # unknown: the log kept no result
231
+ return None, None
232
+ return code, int(call["is_error"] or bool(failed))
233
+
234
+
235
+ def within(earlier: str | None, later: str | None, seconds: int = DUPLICATE_WINDOW) -> bool:
236
+ a, b = parse_ts(earlier), parse_ts(later)
237
+ return bool(a and b) and abs((b - a).total_seconds()) <= seconds
238
+
239
+
240
+ def truncate(text: str) -> str:
241
+ if len(text) <= OUTPUT_LIMIT:
242
+ return text
243
+ half = OUTPUT_LIMIT // 2
244
+ return f"{text[:half]}\n... [{len(text) - OUTPUT_LIMIT} chars cut] ...\n{text[-half:]}"
245
+
246
+
247
+ def block_text(content, types=("text",)) -> str:
248
+ """Join the text blocks of a message content field (a string or a list of blocks)."""
249
+ if isinstance(content, str):
250
+ return content
251
+ if not isinstance(content, list):
252
+ return ""
253
+ return "\n".join(b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") in types)
254
+
255
+
256
+ def looks_injected(text: str) -> bool:
257
+ return text.lstrip().startswith(INJECTED_PREFIXES)
258
+
259
+
260
+ def safe_session_id(session_id: object) -> bool:
261
+ """A session ID names a folder under RAW_DIR and keys every table, so it may hold only
262
+ letters, digits, "_" and "-": nothing that could climb out of RAW_DIR."""
263
+ if not isinstance(session_id, str) or not SESSION_ID.fullmatch(session_id):
264
+ return False
265
+ return (RAW_DIR / session_id).resolve().parent == RAW_DIR.resolve()
266
+
267
+
268
+ def read_jsonl(path: Path, tolerant: bool = False) -> list[dict]:
269
+ """Read path, or path + '.gz' when scrape.py saved it compressed. tolerant skips lines
270
+ that are not JSON: an agent still writing its log can leave a cut-off last line."""
271
+ gz = path.with_name(path.name + ".gz")
272
+ text = gzip.decompress(gz.read_bytes()).decode() if gz.exists() else path.read_text()
273
+ rows = []
274
+ # Split on "\n" only: splitlines() also breaks on U+2028 and friends, which can sit
275
+ # inside JSON strings (pasted text) and would cut a record in two.
276
+ for line in text.split("\n"):
277
+ if line.strip():
278
+ try:
279
+ rows.append(json.loads(line))
280
+ except json.JSONDecodeError:
281
+ if not tolerant:
282
+ raise
283
+ return rows
284
+
285
+
286
+ # --- Claude Code -------------------------------------------------------------------------------
287
+
288
+ def slash_command(text: str) -> str | None:
289
+ """'/name args' for a slash command a person typed to invoke a skill or custom command.
290
+ Built-in housekeeping commands (/login, /clear, ...) return None: they carry no intent."""
291
+ name = re.search(r"<command-name>/?([^<]+)</command-name>", text)
292
+ if not name or name.group(1).strip() in HOUSEKEEPING_COMMANDS:
293
+ return None
294
+ args = re.search(r"<command-args>(.*?)</command-args>", text, re.DOTALL)
295
+ return f"/{name.group(1).strip()} {args.group(1).strip() if args else ''}".strip()
296
+
297
+
298
+ def claude_user_kind(entry: dict, text: str) -> str:
299
+ if entry.get("isMeta") or entry.get("isCompactSummary") or looks_injected(text):
300
+ return "injected"
301
+ origin = entry.get("origin")
302
+ if origin is not None: # newer logs say who wrote the turn
303
+ return "human" if origin.get("kind") == "human" else "injected"
304
+ return "human"
305
+
306
+
307
+ def parse_claude(entries: list[dict]) -> Log:
308
+ log = Log()
309
+ last_api_id = None # Claude Code logs one entry per content block; merge them by message.id
310
+ for e in entries:
311
+ if e.get("isSidechain"):
312
+ continue
313
+ log.cwd = log.cwd or e.get("cwd", "")
314
+ kind, ts = e.get("type"), e.get("timestamp")
315
+ if kind == "assistant":
316
+ msg = e["message"]
317
+ if msg.get("model") == "<synthetic>": # harness error text such as "API Error: ..."
318
+ continue
319
+ text = block_text(msg["content"])
320
+ if msg.get("id") == last_api_id and log.messages and log.messages[-1]["role"] == "assistant":
321
+ idx = len(log.messages) - 1
322
+ log.messages[idx]["text"] = "\n".join(t for t in (log.messages[idx]["text"], text) if t)
323
+ else:
324
+ idx = log.add_message("assistant", ts, text)
325
+ last_api_id = msg.get("id")
326
+ for block in msg["content"]:
327
+ if block.get("type") == "tool_use":
328
+ log.add_call(idx, block["id"], block["name"], block.get("input"))
329
+ elif kind == "user":
330
+ content = e["message"]["content"]
331
+ for block in content if isinstance(content, list) else []:
332
+ if block.get("type") == "tool_result":
333
+ claude_result(log, block)
334
+ text = block_text(content)
335
+ command = slash_command(text) if not e.get("isMeta") else None
336
+ if command and (e.get("origin") or {}).get("kind", "human") == "human":
337
+ log.add_message("user", ts, command, "human")
338
+ elif text.strip():
339
+ log.add_message("user", ts, text, claude_user_kind(e, text))
340
+ elif kind == "attachment" and e["attachment"].get("type") == "queued_command":
341
+ # A prompt typed while the agent was busy, handed to it mid-turn.
342
+ att = e["attachment"]
343
+ text = block_text(att.get("prompt"))
344
+ if text.strip():
345
+ human = (att.get("origin") or {}).get("kind") == "human" and not looks_injected(text)
346
+ log.add_message("user", ts, text, "human" if human else "injected")
347
+ elif kind == "system" and e.get("subtype") == "local_command":
348
+ log.add_message("user", ts, e.get("content", ""), "injected")
349
+ return log
350
+
351
+
352
+ CLAUDE_EXIT = re.compile(r"\AExit code (\d+)")
353
+
354
+
355
+ def claude_result(log: Log, block: dict) -> None:
356
+ call_id, output = block["tool_use_id"], block_text(block.get("content"))
357
+ call = log.by_id.get(call_id)
358
+ m = CLAUDE_EXIT.match(output)
359
+ if call and call["name"] == "Bash" and (m or not block.get("is_error")):
360
+ log.set_output(call_id, output, False)
361
+ log.add_exit(call, int(m.group(1)) if m else 0, json.loads(call["input"]).get("command", ""), output)
362
+ else:
363
+ log.set_output(call_id, output, block.get("is_error"))
364
+
365
+
366
+ # --- Pi ----------------------------------------------------------------------------------------
367
+
368
+ def parse_pi(entries: list[dict]) -> Log:
369
+ log = Log()
370
+ for e in entries:
371
+ kind, ts = e.get("type"), e.get("timestamp")
372
+ if kind == "session":
373
+ log.cwd = e.get("cwd", "")
374
+ elif kind == "message":
375
+ msg = e["message"]
376
+ role = msg.get("role")
377
+ if role == "user":
378
+ text = block_text(msg["content"])
379
+ if text.strip():
380
+ log.add_message("user", ts, text, "injected" if looks_injected(text) else "human")
381
+ elif role == "assistant":
382
+ idx = log.add_message("assistant", ts, block_text(msg["content"]))
383
+ for block in msg["content"]:
384
+ if block.get("type") == "toolCall":
385
+ args = block.get("arguments") or {}
386
+ patch = args.get("patch") or args.get("input") if isinstance(args, dict) else None
387
+ log.add_call(idx, block["id"], block["name"], args, patch if isinstance(patch, str) else "")
388
+ elif role == "toolResult":
389
+ pi_result(log, msg)
390
+ elif kind == "custom_message": # extension context sent to the model as a user turn
391
+ text = block_text(e.get("content"))
392
+ if text.strip():
393
+ log.add_message("user", ts, text, "injected")
394
+ elif kind == "compaction":
395
+ log.add_message("user", ts, e.get("summary", ""), "injected")
396
+ return log
397
+
398
+
399
+ PI_EXIT = re.compile(r"Command exited with code (\d+)\s*\Z")
400
+
401
+
402
+ def pi_result(log: Log, msg: dict) -> None:
403
+ call_id, output = msg["toolCallId"], block_text(msg.get("content"))
404
+ call = log.by_id.get(call_id)
405
+ m = PI_EXIT.search(output)
406
+ if call and call["name"] == "bash" and (m or not msg.get("isError")):
407
+ log.set_output(call_id, output, False)
408
+ log.add_exit(call, int(m.group(1)) if m else 0, json.loads(call["input"]).get("command", ""), output)
409
+ else:
410
+ log.set_output(call_id, output, msg.get("isError"))
411
+
412
+
413
+ # --- Codex -------------------------------------------------------------------------------------
414
+
415
+ def parse_codex(entries: list[dict]) -> Log:
416
+ log = Log()
417
+ # item_completed UserMessage events hold exactly what the person typed.
418
+ typed = {
419
+ block_text(p["item"].get("content")).strip()
420
+ for p in (e.get("payload") or {} for e in entries)
421
+ if p.get("type") == "item_completed" and p["item"].get("type") == "UserMessage"
422
+ }
423
+ last_exec = None # command and file-change events belong to the exec script that ran them
424
+ for e in entries:
425
+ p, ts = e.get("payload") or {}, e.get("timestamp")
426
+ if e.get("type") == "session_meta":
427
+ log.cwd = p.get("cwd", "")
428
+ elif e.get("type") == "response_item":
429
+ kind = p.get("type")
430
+ if kind == "message" and p.get("role") == "user":
431
+ text = block_text(p["content"], ("input_text",))
432
+ # A <skill> block is Codex expanding a $skill the person typed; that typed
433
+ # message is its own human turn, so the expansion counts as injected.
434
+ human = not text.lstrip().startswith("<skill>") and (
435
+ text.strip() in typed if typed else not looks_injected(text))
436
+ log.add_message("user", ts, text, "human" if human else "injected")
437
+ elif kind == "message" and p.get("role") == "assistant":
438
+ log.add_message("assistant", ts, block_text(p["content"], ("output_text",)))
439
+ elif kind == "agent_message": # a sub-agent reporting back
440
+ log.add_message("user", ts, block_text(p.get("content"), ("input_text",)), "injected")
441
+ elif kind == "custom_tool_call":
442
+ script = p.get("input", "")
443
+ name = "apply_patch" if "tools.apply_patch" in script else p["name"]
444
+ last_exec = log.add_call(log.assistant_idx(ts), p["call_id"], name, {"input": script}, script)
445
+ elif kind == "function_call":
446
+ try:
447
+ args = json.loads(p.get("arguments") or "{}")
448
+ except json.JSONDecodeError:
449
+ args = {"arguments": p.get("arguments")}
450
+ log.add_call(log.assistant_idx(ts), p["call_id"], p["name"], args)
451
+ elif kind in ("custom_tool_call_output", "function_call_output"):
452
+ out = p.get("output")
453
+ text = out if isinstance(out, str) else block_text(out, ("input_text", "output_text", "text"))
454
+ log.set_output(p.get("call_id"), text, text.startswith("Script failed"))
455
+ elif e.get("type") == "event_msg" and p.get("type") == "item_completed" and last_exec:
456
+ item = p["item"]
457
+ if item.get("type") == "CommandExecution" and item.get("exit_code") is not None:
458
+ command = item.get("command") or []
459
+ command = command[-1] if isinstance(command, list) and command else str(command)
460
+ log.add_exit(last_exec, item["exit_code"], command, item.get("aggregated_output") or "")
461
+ elif item.get("type") == "FileChange":
462
+ last_exec["files"] += list(item.get("changes") or {})
463
+ last_exec["writes"] = 1
464
+ if item.get("status") not in ("completed", None):
465
+ last_exec["is_error"] = 1
466
+ return log
467
+
468
+
469
+ PARSERS = {"Claude Code": parse_claude, "Pi": parse_pi, "Codex": parse_codex}
470
+
471
+
472
+ # --- Structured transcript (fallback) ----------------------------------------------------------
473
+
474
+ WORKDIR = re.compile(r"workdir\W+(/[^\"'\\]+)")
475
+
476
+
477
+ def parse_transcript(transcript: dict) -> Log:
478
+ """Build a Log from transcript.json, for sessions whose raw log is too big to fetch.
479
+ It has messages and tool inputs but mostly no tool output, so output, is_error and
480
+ exit codes stay NULL unless a result carries them. A user turn that follows a slash
481
+ command before any reply is the command's expansion, so it counts as injected."""
482
+ log = Log()
483
+ after_command = False
484
+ for m in transcript["sessions"][0]["messages"]:
485
+ ts, text = m.get("timestamp"), m.get("content") if isinstance(m.get("content"), str) else ""
486
+ if m["role"] == "user":
487
+ tags = {t.get("tag"): t.get("content", "") for t in m.get("runtimeTags") or []}
488
+ name = tags.get("command-name", "").lstrip("/").strip()
489
+ if name and name not in HOUSEKEEPING_COMMANDS:
490
+ log.add_message("user", ts, f"/{name} {tags.get('command-args', '')}".strip(), "human")
491
+ after_command = True
492
+ elif tags:
493
+ log.add_message("user", ts, "\n".join(tags.values()), "injected")
494
+ elif text.strip():
495
+ human = not (after_command or looks_injected(text) or text.lstrip().startswith("<skill>"))
496
+ log.add_message("user", ts, text, "human" if human else "injected")
497
+ after_command = False
498
+ continue
499
+ after_command = False
500
+ idx = log.add_message("assistant", ts, text)
501
+ for tc in m.get("toolCalls") or []:
502
+ args = tc.get("input")
503
+ script = args.get("input") or args.get("patch") if isinstance(args, dict) else None
504
+ script = script if isinstance(script, str) else ""
505
+ name = "apply_patch" if "tools.apply_patch" in script else tc["name"]
506
+ call = log.add_call(idx, tc["id"], name, args, script)
507
+ result = tc.get("result") or {}
508
+ if "output" in result:
509
+ log.set_output(tc["id"], result["output"] or "", result.get("type") == "error")
510
+ else:
511
+ call["is_error"] = None
512
+ log.cwd = log.cwd or next(iter(WORKDIR.findall(script)), "")
513
+ return log
514
+
515
+
516
+ # --- Loading -----------------------------------------------------------------------------------
517
+
518
+ def relative(path: str, cwd: str) -> str:
519
+ root = cwd.rstrip("/") + "/"
520
+ if cwd and path.startswith(root):
521
+ return path[len(root):]
522
+ m = WORKTREE.match(path)
523
+ if m:
524
+ return path[m.end():]
525
+ return path.removeprefix("./")
526
+
527
+
528
+ def parse_ts(ts: str | None) -> datetime | None:
529
+ try:
530
+ return datetime.fromisoformat(ts) if ts else None
531
+ except ValueError:
532
+ return None
533
+
534
+
535
+ def session_row(session_id: str, listing: dict, meta: dict, cwd: str) -> tuple:
536
+ get = {**meta, **listing}.get
537
+ human = listing.get("attributionHumanAdded")
538
+ if human is not None:
539
+ human += listing.get("attributionHumanModified") or 0
540
+ return (
541
+ session_id, get("agent"), get("model"), get("displayName"), get("firstCommitAuthorUsername"),
542
+ get("startedAt"), get("lastActivityAt"), get("checkpointCount"), get("inputTokens"),
543
+ get("outputTokens"), get("attributionAgentLines"), human, get("prompt"), cwd,
544
+ )
545
+
546
+
547
+ def write_log(conn, row: tuple, log: Log, checkpoints: list[dict] = (), root: str | None = None) -> None:
548
+ """Replace one session's rows. row is the sessions row, session ID first; root is the
549
+ directory tool-call paths are made relative to (default: the log's cwd)."""
550
+ session_id = row[0]
551
+ root = log.cwd if root is None else root
552
+ cp_times = [(parse_ts(c["timestamp"]), c["checkpointId"]) for c in checkpoints]
553
+
554
+ def checkpoint_for(ts: str | None) -> str | None:
555
+ t = parse_ts(ts)
556
+ return next((cid for ct, cid in cp_times if t and ct >= t), None)
557
+
558
+ with conn:
559
+ for table in ("sessions", "checkpoints", "messages", "tool_calls"):
560
+ conn.execute(f"DELETE FROM {table} WHERE session_id = ?", (session_id,))
561
+ conn.execute(
562
+ "INSERT INTO sessions (session_id, agent, model, display_name, author, started_at, last_activity_at,"
563
+ " checkpoint_count, input_tokens, output_tokens, agent_lines, human_lines, prompt, cwd)"
564
+ " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
565
+ row,
566
+ )
567
+ conn.executemany(
568
+ "INSERT OR REPLACE INTO checkpoints (checkpoint_id, session_id, commit_sha, message, additions,"
569
+ " deletions, files_changed, ts) VALUES (?,?,?,?,?,?,?,?)",
570
+ [(c["checkpointId"], session_id, c.get("commitSha"), c.get("message"), c.get("additions"),
571
+ c.get("deletions"), c.get("filesChanged"), c.get("timestamp")) for c in checkpoints],
572
+ )
573
+ conn.executemany(
574
+ "INSERT INTO messages (session_id, idx, role, user_kind, ts, text, checkpoint_id) VALUES (?,?,?,?,?,?,?)",
575
+ [(session_id, i, m["role"], m["user_kind"], m["ts"], m["text"], checkpoint_for(m["ts"]))
576
+ for i, m in enumerate(log.messages)],
577
+ )
578
+ call_idx: dict[int, int] = {}
579
+ rows = []
580
+ for c in log.calls:
581
+ n = call_idx[c["msg_idx"]] = call_idx.get(c["msg_idx"], -1) + 1
582
+ files = list(dict.fromkeys(relative(f, root) for f in c["files"]))
583
+ exit_code, is_error = exit_status(c)
584
+ rows.append((session_id, c["msg_idx"], n, c["name"], c["input"], c["output"],
585
+ is_error, exit_code, json.dumps(files), c["writes"]))
586
+ conn.executemany(
587
+ "INSERT INTO tool_calls (session_id, msg_idx, call_idx, name, input, output, is_error, exit_code,"
588
+ " files, writes) VALUES (?,?,?,?,?,?,?,?,?,?)",
589
+ rows,
590
+ )
591
+
592
+
593
+ def load_session(conn, folder: Path) -> str:
594
+ session_id = folder.name
595
+ if not safe_session_id(session_id):
596
+ return f"skip {session_id!r}: not a valid session ID"
597
+ listing_path = folder / "listing.json"
598
+ listing = json.loads(listing_path.read_text()) if listing_path.exists() else {}
599
+ transcript = json.loads((folder / "transcript.json").read_text())
600
+ agent = listing.get("agent") or transcript["session"].get("agent")
601
+ parser = PARSERS.get(agent)
602
+ if parser is None:
603
+ return f"skip {session_id}: no parser for agent {agent!r}"
604
+
605
+ raw = folder / "raw.jsonl"
606
+ if raw.exists() or raw.with_name(raw.name + ".gz").exists():
607
+ log = parser(read_jsonl(raw))
608
+ else: # scrape.py saves no raw log when it is over the API's size limit
609
+ log = parse_transcript(transcript)
610
+ checkpoints = sorted(transcript.get("checkpoints", []), key=lambda c: parse_ts(c["timestamp"]))
611
+ write_log(conn, session_row(session_id, listing, transcript.get("session", {}), log.cwd), log, checkpoints)
612
+ return f"loaded {session_id} {agent:<12} {len(log.messages):>4} messages {len(log.calls):>4} tool calls"
613
+
614
+
615
+ def main(argv: list[str] | None = None) -> None:
616
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
617
+ parser.add_argument("--ids", nargs="*", help="Only these session IDs (default: every folder in raw/)")
618
+ args = parser.parse_args(argv)
619
+
620
+ # Only folders scrape.py filled: a home with local sessions alone has no raw/ at all.
621
+ folders = [RAW_DIR / i for i in args.ids] if args.ids else sorted(
622
+ p for p in RAW_DIR.glob("*") if (p / "transcript.json").exists())
623
+ conn = db.connect()
624
+ failed = []
625
+ for folder in folders:
626
+ try:
627
+ print(load_session(conn, folder))
628
+ except (ValueError, KeyError, TypeError, OSError) as e: # one bad log shouldn't stop the rest
629
+ failed.append(folder.name)
630
+ print(f"failed {folder.name}: {type(e).__name__}: {e}")
631
+ print(f"done: {len(folders) - len(failed)} loaded, {len(failed)} failed {failed or ''}")
632
+
633
+
634
+ if __name__ == "__main__":
635
+ main()
debrief/paths.py ADDED
@@ -0,0 +1,17 @@
1
+ """Where debrief keeps its data. Set DEBRIEF_HOME to move it; nothing is written inside the package."""
2
+
3
+ import os
4
+ from pathlib import Path
5
+
6
+ # The one name the program matches on or stores under: data folder, env var prefix, database
7
+ # file, token header. Changing it strands existing users' data: see tests/test_paths.py.
8
+ NAME = "debrief"
9
+ TOKEN_HEADER = f"X-{NAME.title()}-Token"
10
+
11
+ HOME = Path(os.environ.get(f"{NAME.upper()}_HOME") or Path.home() / ".local" / "share" / NAME)
12
+ DB_PATH = Path(os.environ.get(f"{NAME.upper()}_DB") or HOME / f"{NAME}.db")
13
+ RAW_DIR = HOME / "raw" # one folder per session, as fetched or imported
14
+ BUNDLE_DIR = HOME / "bundles" # evidence bundles sent to the model
15
+ SITE_OUT = HOME / "site" # the built site: static files plus data/
16
+ ARTIFACT_OUT = HOME / "artifact"
17
+ SITE_SRC = Path(__file__).parent / "site"