debrief-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
debrief/scrape.py ADDED
@@ -0,0 +1,208 @@
1
+ """Fetch Entire sessions for a repo into data/raw/.
2
+
3
+ Needs a logged-in `entire` CLI (`entire login`); requests go through `entire api`.
4
+
5
+ uv run python -m debrief.scrape --since 2026-08-21 --until 2026-09-18 --limit 100 --dry-run
6
+ uv run python -m debrief.scrape --since 2026-08-21 --until 2026-09-18 --limit 100
7
+ uv run python -m debrief.scrape --ids efd45cc6-c764-4e22-80eb-cdbc1f9173f9 ...
8
+
9
+ Sessions are chosen from the window, not just the most recent: every Codex/Pi session
10
+ up to --max-other, then Claude Code sessions spread evenly across the window's dates.
11
+ A session is fetched again only when its last activity has changed.
12
+ """
13
+
14
+ import argparse
15
+ import gzip
16
+ import json
17
+ import re
18
+ import subprocess
19
+ import sys
20
+ import time
21
+ from collections import Counter
22
+ from datetime import datetime
23
+
24
+ from debrief.normalize import safe_session_id
25
+ from debrief.paths import RAW_DIR
26
+
27
+ SUPPORTED_AGENTS = {"Claude Code", "Codex", "Pi"} # the formats normalize.py parses
28
+ DELAY = 1.0 # seconds between requests
29
+ RETRIES = 3
30
+
31
+ requests_made = 0
32
+
33
+
34
+ class ApiError(Exception):
35
+ pass
36
+
37
+
38
+ def api(path: str) -> str:
39
+ """GET path through `entire api`, one request at a time, retrying 429 and 5xx."""
40
+ global requests_made
41
+ for attempt in range(RETRIES + 1):
42
+ time.sleep(DELAY)
43
+ requests_made += 1
44
+ out = subprocess.run(
45
+ ["entire", "api", "--to", "cell", f"/api/v1{path}"],
46
+ capture_output=True,
47
+ check=False,
48
+ text=True,
49
+ )
50
+ if out.returncode == 0:
51
+ return out.stdout
52
+ message = (out.stderr.strip() or out.stdout[:300]).strip()
53
+ if attempt < RETRIES and re.search(r"\b(429|5\d\d)\b|rate limit", message, re.IGNORECASE):
54
+ wait = 10 * 2**attempt
55
+ print(f" retrying in {wait}s: {message[:120]}")
56
+ time.sleep(wait)
57
+ continue
58
+ raise ApiError(f"entire api {path} failed: {message}")
59
+ raise AssertionError("unreachable")
60
+
61
+
62
+ def api_json(path: str) -> dict:
63
+ return json.loads(api(path))
64
+
65
+
66
+ def to_rfc3339(date: str) -> str:
67
+ return f"{date}T00:00:00Z" if len(date) == 10 else date
68
+
69
+
70
+ def resolve_repo(repo_path: str) -> str:
71
+ return api_json(f"/git/repos/resolve?repoPath={repo_path}")["repoId"]
72
+
73
+
74
+ def list_sessions(repo_id: str, since: str | None, until: str | None, cap: int = 2000) -> list[dict]:
75
+ """Every session in the window, most recent first, 200 per request."""
76
+ window = "".join(f"&{k}={to_rfc3339(v)}" for k, v in (("since", since), ("until", until)) if v)
77
+ sessions, cursor = [], ""
78
+ while len(sessions) < cap:
79
+ page = api_json(f"/repos/{repo_id}/sessions?limit=200&scope=all{window}" + (f"&cursor={cursor}" if cursor else ""))
80
+ sessions += page["sessions"]
81
+ cursor = page.get("next_cursor")
82
+ if not cursor:
83
+ break
84
+ return sessions
85
+
86
+
87
+ def spread(items: list[dict], n: int) -> list[dict]:
88
+ """n items evenly spaced through items sorted by last activity."""
89
+ items = sorted(items, key=lambda s: s["lastActivityAt"])
90
+ if n <= 0:
91
+ return []
92
+ if len(items) <= n:
93
+ return items
94
+ step = len(items) / n
95
+ return [items[int(i * step)] for i in range(n)]
96
+
97
+
98
+ def select(sessions: list[dict], limit: int, max_other: int) -> list[dict]:
99
+ usable = [s for s in sessions if s["agent"] in SUPPORTED_AGENTS and (s.get("stepCount") or 0) >= 2]
100
+ others = spread([s for s in usable if s["agent"] != "Claude Code"], min(max_other, limit))
101
+ claude = spread([s for s in usable if s["agent"] == "Claude Code"], limit - len(others))
102
+ return others + claude
103
+
104
+
105
+ def describe(chosen: list[dict], sessions: list[dict]) -> None:
106
+ def week(s: dict) -> str:
107
+ return datetime.fromisoformat(s["lastActivityAt"]).strftime("%G-W%V")
108
+
109
+ print(f"{len(sessions)} sessions in window; chose {len(chosen)}")
110
+ print("in window by agent:", dict(Counter(s["agent"] for s in sessions)))
111
+ print("chosen by agent: ", dict(Counter(s["agent"] for s in chosen)))
112
+ print("chosen by model: ", dict(Counter(s.get("model") for s in chosen)))
113
+ print("chosen by week: ", dict(sorted(Counter(week(s) for s in chosen).items())))
114
+ steps = sorted(s.get("stepCount") or 0 for s in chosen)
115
+ if steps:
116
+ print(f"steps per session: min {steps[0]}, median {steps[len(steps) // 2]}, max {steps[-1]}")
117
+ new = sum(1 for s in chosen if needs_fetch(s["sessionId"], s))
118
+ print(f"to fetch: {new} sessions, about {2 * new} requests")
119
+
120
+
121
+ def needs_fetch(session_id: str, listing: dict | None) -> bool:
122
+ saved = RAW_DIR / session_id / "listing.json"
123
+ if not (RAW_DIR / session_id / "transcript.json").exists():
124
+ return True
125
+ if listing is None or not saved.exists():
126
+ return False
127
+ return json.loads(saved.read_text()).get("lastActivityAt") != listing.get("lastActivityAt")
128
+
129
+
130
+ def fetch_session(repo_id: str, session_id: str, listing: dict | None) -> None:
131
+ if not safe_session_id(session_id): # main() filters these out; this guards other callers
132
+ raise ApiError(f"unsafe session ID {session_id!r}")
133
+ out = RAW_DIR / session_id
134
+ out.mkdir(parents=True, exist_ok=True)
135
+ base = f"/repos/{repo_id}/sessions/{session_id}"
136
+ # Structured transcript: messages, tool inputs and checkpoints, but no tool output.
137
+ transcript = api_json(f"{base}/transcript")
138
+ # The agent's native log, which keeps tool output. Format varies by agent. `entire api`
139
+ # refuses bodies over 32 MB (and ignores Range), so for those logs keep only the structured
140
+ # transcript; normalize.py falls back to it when raw.jsonl.gz is missing.
141
+ try:
142
+ raw = api(f"{base}/transcript/raw")
143
+ except ApiError as e:
144
+ if "exceeds" not in str(e):
145
+ raise
146
+ raw = None
147
+ print(f" {session_id}: raw log over the size limit; saving the structured transcript only")
148
+ (out / "transcript.json").write_text(json.dumps(transcript, indent=2))
149
+ if raw is None:
150
+ (out / "raw.jsonl.gz").unlink(missing_ok=True) # an older copy would hide the newer transcript
151
+ else:
152
+ (out / "raw.jsonl.gz").write_bytes(gzip.compress(raw.encode()))
153
+ (out / "raw.jsonl").unlink(missing_ok=True)
154
+ if listing: # written last: its presence with a matching lastActivityAt marks the fetch complete
155
+ (out / "listing.json").write_text(json.dumps(listing, indent=2))
156
+ session = transcript["sessions"][0]
157
+ print(f"saved {session_id} {session['agent']:<12} {len(session['messages']):>4} messages "
158
+ f"{len(transcript['checkpoints'])} checkpoints {transcript['session']['displayName']}")
159
+
160
+
161
+ def main(argv: list[str] | None = None) -> None:
162
+ sys.stdout.reconfigure(line_buffering=True) # progress shows up in log files as it happens
163
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
164
+ parser.add_argument("--repo", default="gh/entireio/cli", help="Entire repo path, e.g. gh/owner/name")
165
+ parser.add_argument("--ids", nargs="*", help="Fetch these session IDs instead of choosing from a window")
166
+ parser.add_argument("--since", help="Only sessions last active on or after this date (YYYY-MM-DD)")
167
+ parser.add_argument("--until", help="Only sessions last active before this date (YYYY-MM-DD)")
168
+ parser.add_argument("--limit", type=int, default=100, help="How many sessions to choose from the window")
169
+ parser.add_argument("--max-other", type=int, default=30, help="Cap on Codex/Pi sessions in the choice")
170
+ parser.add_argument("--dry-run", action="store_true", help="Show the choice without fetching sessions")
171
+ args = parser.parse_args(argv)
172
+
173
+ repo_id = resolve_repo(args.repo)
174
+ if args.ids:
175
+ chosen = [{"sessionId": i} for i in args.ids]
176
+ listings: dict[str, dict] = {}
177
+ else:
178
+ sessions = list_sessions(repo_id, args.since, args.until)
179
+ chosen = select(sessions, args.limit, args.max_other)
180
+ listings = {s["sessionId"]: s for s in chosen}
181
+ # The ID comes from the API or the command line and names a folder under RAW_DIR.
182
+ for s in chosen:
183
+ if not safe_session_id(s.get("sessionId")):
184
+ print(f"warning: skipping session with unsafe ID {s.get('sessionId')!r}")
185
+ chosen = [s for s in chosen if safe_session_id(s.get("sessionId"))]
186
+ if not args.ids:
187
+ describe(chosen, sessions)
188
+ if args.dry_run:
189
+ for s in chosen:
190
+ print(f" {s.get('lastActivityAt', '')[:10]} {s.get('agent', ''):<12} {s.get('stepCount', ''):>4} {s.get('displayName', s['sessionId'])}")
191
+ print(f"requests made: {requests_made}")
192
+ return
193
+
194
+ failed = []
195
+ for s in chosen:
196
+ session_id = s["sessionId"]
197
+ if not needs_fetch(session_id, listings.get(session_id)):
198
+ continue
199
+ try:
200
+ fetch_session(repo_id, session_id, listings.get(session_id))
201
+ except ApiError as e:
202
+ failed.append(session_id)
203
+ print(f"failed {session_id}: {e}")
204
+ print(f"requests made: {requests_made}; failed: {len(failed)} {failed or ''}")
205
+
206
+
207
+ if __name__ == "__main__":
208
+ main()