debrief-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
debrief/github.py ADDED
@@ -0,0 +1,540 @@
1
+ """Load GitHub PRs, checks and review comments for the PRs your sessions produced.
2
+
3
+ Optional. Uses the logged-in `gh` CLI, read-only, and writes session_prs, gh_prs,
4
+ gh_commit_prs, gh_checks and gh_review_comments in debrief.db. Without a logged-in `gh`,
5
+ or with nothing to look up, it prints one line and does nothing.
6
+
7
+ With no --repo it links local sessions to PRs from what their own logs show: the PR URL
8
+ `gh pr create` printed, and the branches `git push` sent, looked up as PR heads. A PR closed
9
+ before the session started is ignored. Then it fetches every repo those PRs are in.
10
+ With --repo it looks up the checkpoint commits of sessions imported from Entire in that repo.
11
+
12
+ Re-runs skip what cannot change: branches and commits already looked up, settled PRs,
13
+ checks that have all concluded, and failure logs already stored. Heads of
14
+ unsettled PRs always get their checks re-listed, since a job can be re-run.
15
+
16
+ Check runs are listed with filter=all, so every attempt of a re-run job is a
17
+ separate gh_checks row; the default returns only the latest attempt.
18
+
19
+ uv run python -m debrief.github # local sessions, every repo they pushed to
20
+ uv run python -m debrief.github --dry-run # say what the logs link to; no requests
21
+ uv run python -m debrief.github --repo owner/name # Entire checkpoints in one repo
22
+ uv run python -m debrief.github --refresh # ignore the skips and fetch everything
23
+ uv run python -m debrief.github --repo owner/name --checks-since 2026-08-11
24
+ # only re-list checks on stored commits of sessions active since then,
25
+ # to pick up attempts that the old latest-only listing dropped
26
+ """
27
+
28
+ import argparse
29
+ import functools
30
+ import json
31
+ import re
32
+ import shutil
33
+ import sqlite3
34
+ import subprocess
35
+ import sys
36
+ import threading
37
+ import time
38
+ from concurrent.futures import ThreadPoolExecutor
39
+ from datetime import timedelta
40
+ from urllib.parse import quote
41
+
42
+ from debrief import config, db
43
+ from debrief.normalize import SHELL_TOOLS, parse_ts
44
+
45
+ LOG_LINES = 150
46
+ LOG_BYTES = 12_000
47
+ TIMESTAMP = re.compile(r"^\d{4}-\d\d-\d\dT[\d:.]+Z ")
48
+ ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
49
+ RUN_ID = re.compile(r"/actions/runs/(\d+)")
50
+ # Both go into `gh api` paths, so anything else is refused.
51
+ SHA = re.compile(r"[0-9a-f]{7,40}")
52
+ REPO = re.compile(r"[A-Za-z0-9-]+/(?!\.\.?$)[\w.-]+") # the name may hold dots but not be "." or ".."
53
+ # What a session's shell calls show of its PRs: the URL `gh pr create` prints, the repo and
54
+ # branch in `git push` output ("To github.com:o/n.git", "abc..def local -> branch"), and the
55
+ # branch named in the push command, since `git push -q` prints nothing.
56
+ PR_URL = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/pull/(\d+)")
57
+ GITHUB_REMOTE = r"(?:(?:[\w.-]+@)?github\.com:|(?:https|ssh)://(?:[\w.-]+@)?github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?/?\s*$"
58
+ PUSH_TO = re.compile("^To " + GITHUB_REMOTE, re.MULTILINE)
59
+ PUSH_REF = re.compile(r"^[ +*!=-]*(?:[0-9a-f]{7,40}\.\.\.?[0-9a-f]{7,40}|\[new branch\])\s+\S+\s+->\s+(\S+)", re.MULTILINE)
60
+ PUSH_CMD = re.compile(r"\bgit push\b(?:\s+-\S+)*\s+origin\s+(?:[\w./-]+:)?(\w[\w./-]*)")
61
+ # A PR whose head is one of these is a release or sync PR, not the session's work.
62
+ # shortcut: a fixed list, not each repo's default branch; ask GitHub if a repo's trunk has another name.
63
+ TRUNKS = {"main", "master", "develop", "trunk", "HEAD"}
64
+ RECHECK = timedelta(days=2) # how long after a session a pushed branch may still get its PR
65
+ # Inline comments, reviews and issue comments draw ids from separate sequences
66
+ # that can overlap, so review and issue rows get an offset added to their id.
67
+ REVIEW_ID_OFFSET = 10**15
68
+ ISSUE_ID_OFFSET = 2 * 10**15
69
+ THREADS_QUERY = """
70
+ query($owner: String!, $name: String!, $number: Int!, $endCursor: String) {
71
+ repository(owner: $owner, name: $name) {
72
+ pullRequest(number: $number) {
73
+ reviewThreads(first: 100, after: $endCursor) {
74
+ pageInfo { hasNextPage endCursor }
75
+ nodes { isResolved isOutdated comments(first: 100) { nodes { databaseId } } }
76
+ }
77
+ }
78
+ }
79
+ }
80
+ """
81
+ SEEN_SCHEMA = """
82
+ CREATE TABLE IF NOT EXISTS gh_commits_seen (
83
+ commit_sha TEXT PRIMARY KEY, -- checkpoint commit whose PRs and checks we have fetched
84
+ looked_up_at TEXT
85
+ );
86
+ CREATE TABLE IF NOT EXISTS gh_branches_seen (
87
+ repo TEXT NOT NULL,
88
+ branch TEXT NOT NULL, -- pushed by a local session and looked up as a PR head
89
+ looked_up_at TEXT,
90
+ prs INTEGER, -- how many PRs had it as head
91
+ PRIMARY KEY (repo, branch)
92
+ );
93
+ """
94
+ RETRIES = 5
95
+ requests = 0
96
+ requests_lock = threading.Lock()
97
+
98
+
99
+ def gh(path: str, paginate: bool = False, fields: list[str] | None = None) -> str | None:
100
+ """Return the body of a request, or None on 404/410. `fields` are extra gh flags."""
101
+ # Job logs carry colour codes, which gh refuses to print without this flag.
102
+ cmd = ["gh", "api", path, "--allow-escape-sequences", *(fields or [])] + (["--paginate", "--slurp"] if paginate else [])
103
+ global requests
104
+ for attempt in range(RETRIES):
105
+ with requests_lock:
106
+ requests += 1
107
+ out = subprocess.run(cmd, capture_output=True, text=True, check=False)
108
+ limited = "rate limit" in out.stderr or "HTTP 429" in out.stderr
109
+ if out.returncode == 0 or not limited or attempt == RETRIES - 1:
110
+ break
111
+ wait = 10 * 2**attempt
112
+ print(f"rate limited on {path}; retrying in {wait}s")
113
+ time.sleep(wait)
114
+ if out.returncode != 0:
115
+ if "HTTP 404" in out.stderr or "HTTP 410" in out.stderr:
116
+ return None
117
+ raise RuntimeError(f"gh api {path} failed: {out.stderr.strip()}")
118
+ return out.stdout
119
+
120
+
121
+ def gh_list(path: str, key: str | None = None) -> list[dict]:
122
+ """GET every page of a list endpoint. `key` names the list inside object pages."""
123
+ body = gh(path, paginate=True)
124
+ if body is None:
125
+ return []
126
+ pages = json.loads(body)
127
+ return [item for page in pages for item in (page[key] if key else page)]
128
+
129
+
130
+ def valid_shas(shas) -> list[str]:
131
+ """`shas` in order, without repeats or anything that isn't a hex commit id."""
132
+ return [s for s in dict.fromkeys(shas) if s and SHA.fullmatch(s)]
133
+
134
+
135
+ def load_shas(conn: sqlite3.Connection) -> list[str]:
136
+ return valid_shas(r[0] for r in conn.execute("SELECT commit_sha FROM checkpoints ORDER BY session_id, ts"))
137
+
138
+
139
+ def gh_problem() -> str | None:
140
+ """Why `gh` can't be used, or None when it is installed and logged in."""
141
+ if not shutil.which("gh"):
142
+ return "the gh CLI is not installed"
143
+ if subprocess.run(["gh", "auth", "status"], capture_output=True, check=False).returncode != 0:
144
+ return "gh is not logged in (run `gh auth login`)"
145
+ return None
146
+
147
+
148
+ def is_bot(user: dict | None) -> int:
149
+ user = user or {}
150
+ login = user.get("login") or ""
151
+ return int(user.get("type") == "Bot" or login.endswith("[bot]") or login == "Copilot")
152
+
153
+
154
+ def trim_log(log: str) -> str:
155
+ """Keep the failing step's lines, capped to the last LOG_LINES / LOG_BYTES."""
156
+ lines = [TIMESTAMP.sub("", ANSI.sub("", line)) for line in log.splitlines()]
157
+ err = next((i for i, line in enumerate(lines) if "##[error]" in line), None)
158
+ if err is not None:
159
+ start = max((i for i in range(err) if lines[i].startswith("##[group]Run ")), default=0)
160
+ # Keep any further error lines that follow straight after the first.
161
+ end = err
162
+ while end + 1 < len(lines) and "##[error]" in lines[end + 1]:
163
+ end += 1
164
+ lines = lines[start : end + 1]
165
+ tail = "\n".join(lines[-LOG_LINES:])
166
+ # Parallel test output can push Go's failure report far above the tail.
167
+ fails = go_failures(lines[:-LOG_LINES])
168
+ if fails:
169
+ fails = fails[: LOG_BYTES // 2]
170
+ return f"{fails}\n...\n{tail[-(LOG_BYTES - len(fails) - 5):]}"
171
+ return tail[-LOG_BYTES:]
172
+
173
+
174
+ def go_failures(lines: list[str]) -> str:
175
+ """Each `--- FAIL` line and the indented detail under it."""
176
+ keep = set()
177
+ for i, line in enumerate(lines):
178
+ if line.lstrip().startswith("--- FAIL"):
179
+ keep.add(i)
180
+ j = i + 1
181
+ while j < len(lines) and lines[j].startswith((" ", "\t")):
182
+ keep.add(j)
183
+ j += 1
184
+ return "\n".join(lines[i] for i in sorted(keep))
185
+
186
+
187
+ @functools.cache
188
+ def remote_repo(cwd: str) -> str | None:
189
+ """owner/name of the checkout's GitHub origin, read from disk."""
190
+ out = subprocess.run(["git", "-C", cwd, "remote", "get-url", "origin"], capture_output=True, text=True, check=False)
191
+ m = re.search(GITHUB_REMOTE, out.stdout)
192
+ return m.group(1) if m else None
193
+
194
+
195
+ def find_links(conn: sqlite3.Connection) -> tuple[set, dict]:
196
+ """What the logs say, with no requests: {(session_id, repo, pr_number)} from `gh pr create`
197
+ output, and {(repo, branch): {session_id}} for the branches sessions pushed."""
198
+ calls: dict[str, list] = {}
199
+ marks = ", ".join("?" * (len(SHELL_TOOLS) + 1))
200
+ for r in conn.execute(
201
+ "SELECT t.session_id, s.cwd, t.input, t.output FROM tool_calls t JOIN sessions s USING (session_id) "
202
+ f"WHERE lower(t.name) IN ({marks}) AND (t.input LIKE '%git push%' OR t.input LIKE '%gh pr create%')",
203
+ [*SHELL_TOOLS, "exec"]):
204
+ calls.setdefault(r["session_id"], []).append(r)
205
+ prs, pushes = set(), {}
206
+ for sid, rows in calls.items():
207
+ created = {(repo, int(n)) for r in rows if "gh pr create" in r["input"] for repo, n in PR_URL.findall(r["output"] or "")}
208
+ prs |= {(sid, repo, n) for repo, n in created if REPO.fullmatch(repo)}
209
+ # The session's repo when a push does not print it: the one its other calls name, else the checkout's origin.
210
+ named = {repo for repo, _ in created} | {repo for r in rows for repo in PUSH_TO.findall(r["output"] or "")}
211
+ for r in rows:
212
+ if "git push" not in r["input"]:
213
+ continue
214
+ out = r["output"] or ""
215
+ repo = next(iter(PUSH_TO.findall(out)), None) or (next(iter(named)) if len(named) == 1 else remote_repo(r["cwd"] or ""))
216
+ for branch in {b.removeprefix("refs/heads/") for b in PUSH_REF.findall(out) + PUSH_CMD.findall(r["input"])} - TRUNKS:
217
+ if repo and REPO.fullmatch(repo):
218
+ pushes.setdefault((repo, branch), set()).add(sid)
219
+ return prs, pushes
220
+
221
+
222
+ def link_sessions(conn: sqlite3.Connection, pool: ThreadPoolExecutor, prs: set, pushes: dict, refresh: bool) -> None:
223
+ """Write session_prs: the PRs the logs name, plus the PRs whose head is a branch a session pushed."""
224
+ spans = {r[0]: (r[1] or "", r[2] or "") for r in conn.execute("SELECT session_id, started_at, last_activity_at FROM sessions")}
225
+ seen = {} if refresh else {(r["repo"], r["branch"]): r for r in conn.execute("SELECT * FROM gh_branches_seen")}
226
+
227
+ def settled(key) -> bool:
228
+ """Looked up since the last session that pushed it ended, and a PR was found or RECHECK has passed."""
229
+ row, last = seen.get(key), parse_ts(max(spans[s][1] for s in pushes[key]))
230
+ if not row or not last:
231
+ return False
232
+ waited = parse_ts(row["looked_up_at"]) - last
233
+ return waited > RECHECK or (bool(row["prs"]) and waited > timedelta(0))
234
+
235
+ todo = [key for key in sorted(pushes) if not settled(key)]
236
+ found = list(pool.map(lambda key: json.loads(gh(
237
+ f"repos/{key[0]}/pulls?state=all&per_page=100&head={key[0].split('/')[0]}:{quote(key[1], safe='')}") or "[]"), todo))
238
+ rows = [(sid, repo, n, "url") for sid, repo, n in sorted(prs)]
239
+ for key, heads in zip(todo, found):
240
+ # A PR closed before the session began reused the branch name; it is not this session's.
241
+ rows += [(sid, key[0], p["number"], "branch") for p in heads for sid in sorted(pushes[key])
242
+ if not p.get("closed_at") or p["closed_at"][:19] >= spans[sid][0][:19]]
243
+ now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
244
+ with conn:
245
+ conn.executemany("INSERT OR IGNORE INTO session_prs VALUES (?, ?, ?, ?)", rows)
246
+ conn.executemany("INSERT OR REPLACE INTO gh_branches_seen VALUES (?, ?, ?, ?)",
247
+ [(*key, now, len(heads)) for key, heads in zip(todo, found)])
248
+ linked = conn.execute("SELECT COUNT(DISTINCT session_id), COUNT(DISTINCT repo || '#' || pr_number), "
249
+ "COUNT(DISTINCT repo) FROM session_prs").fetchone()
250
+ print(f"{len(todo)} pushed branches looked up ({len(pushes) - len(todo)} skipped); "
251
+ f"{linked[0]} sessions linked to {linked[1]} PRs in {linked[2]} repos")
252
+
253
+
254
+ def fetch_checks(repo: str, sha: str) -> list[dict]:
255
+ rows = []
256
+ for run in gh_list(f"repos/{repo}/commits/{sha}/check-runs?filter=all", key="check_runs"):
257
+ match = RUN_ID.search(run.get("details_url") or "")
258
+ rows.append({
259
+ "check_id": run["id"],
260
+ "repo": repo,
261
+ "commit_sha": sha,
262
+ "name": run["name"],
263
+ "conclusion": run["conclusion"],
264
+ "app": (run.get("app") or {}).get("slug"),
265
+ "run_id": int(match.group(1)) if match else None,
266
+ "started_at": run["started_at"],
267
+ "completed_at": run["completed_at"],
268
+ "output": run.get("output") or {},
269
+ "failure_log": None,
270
+ })
271
+ return rows
272
+
273
+
274
+ def fetch_failure_log(repo: str, check: dict) -> str | None:
275
+ """Actions jobs: tail of the job log. Other apps: the check's own output text."""
276
+ if check["run_id"]:
277
+ log = gh(f"repos/{repo}/actions/jobs/{check['check_id']}/logs")
278
+ return trim_log(log) if log else None
279
+ out = check["output"]
280
+ text = "\n\n".join(p for p in (out.get("title"), out.get("summary"), out.get("text")) if p)
281
+ return text[-LOG_BYTES:] or None
282
+
283
+
284
+ def fetch_threads(repo: str, pr: int) -> dict[int, tuple[int, int]]:
285
+ """Map inline comment id -> (is_resolved, is_outdated) of its thread."""
286
+ owner, name = repo.split("/")
287
+ fields = ["-f", f"query={THREADS_QUERY}", "-f", f"owner={owner}", "-f", f"name={name}", "-F", f"number={pr}"]
288
+ pages = json.loads(gh("graphql", paginate=True, fields=fields))
289
+ flags = {}
290
+ for page in pages:
291
+ for thread in page["data"]["repository"]["pullRequest"]["reviewThreads"]["nodes"]:
292
+ for comment in thread["comments"]["nodes"]:
293
+ flags[comment["databaseId"]] = (int(thread["isResolved"]), int(thread["isOutdated"]))
294
+ return flags
295
+
296
+
297
+ def comment_row(repo: str, pr: int, kind: str, comment_id: int, item: dict, **extra) -> dict:
298
+ row = {
299
+ "comment_id": comment_id,
300
+ "repo": repo,
301
+ "pr_number": pr,
302
+ "kind": kind,
303
+ "author": (item.get("user") or {}).get("login"),
304
+ "is_bot": is_bot(item.get("user")),
305
+ "path": None,
306
+ "line": None,
307
+ "body": item.get("body") or "",
308
+ "in_reply_to": None,
309
+ "created_at": item.get("created_at") or item.get("submitted_at"),
310
+ "review_state": None,
311
+ "is_resolved": None,
312
+ "is_outdated": None,
313
+ }
314
+ return row | extra
315
+
316
+
317
+ def fetch_comments(repo: str, pr: int) -> list[dict]:
318
+ threads = fetch_threads(repo, pr)
319
+ inline = [
320
+ comment_row(
321
+ repo, pr, "inline", c["id"], c,
322
+ path=c.get("path"),
323
+ line=c.get("line") or c.get("original_line"),
324
+ in_reply_to=c.get("in_reply_to_id"),
325
+ is_resolved=threads.get(c["id"], (None, None))[0],
326
+ is_outdated=threads.get(c["id"], (None, None))[1],
327
+ )
328
+ for c in gh_list(f"repos/{repo}/pulls/{pr}/comments")
329
+ ]
330
+ # Empty-bodied reviews still matter when they approve or block.
331
+ reviews = [
332
+ comment_row(repo, pr, "review", REVIEW_ID_OFFSET + r["id"], r, review_state=r.get("state"))
333
+ for r in gh_list(f"repos/{repo}/pulls/{pr}/reviews")
334
+ if (r.get("body") or "").strip() or r.get("state") in ("APPROVED", "CHANGES_REQUESTED")
335
+ ]
336
+ issue = [
337
+ comment_row(repo, pr, "issue", ISSUE_ID_OFFSET + c["id"], c)
338
+ for c in gh_list(f"repos/{repo}/issues/{pr}/comments")
339
+ ]
340
+ return inline + reviews + issue
341
+
342
+
343
+ def upsert(conn: sqlite3.Connection, table: str, rows: list[dict], columns: list[str]) -> None:
344
+ sql = f"INSERT OR REPLACE INTO {table} ({', '.join(columns)}) VALUES ({', '.join('?' * len(columns))})"
345
+ conn.executemany(sql, [[row[c] for c in columns] for row in rows])
346
+
347
+
348
+ def fetch_pr(repo: str, number: int) -> dict | None:
349
+ """One PR, or None when GitHub has none by that number that this login can see."""
350
+ body = gh(f"repos/{repo}/pulls/{number}")
351
+ if body is None:
352
+ return None
353
+ d = json.loads(body)
354
+ return {
355
+ "repo": repo,
356
+ "pr_number": d["number"],
357
+ "title": d["title"],
358
+ "state": d["state"],
359
+ "merged": int(bool(d.get("merged"))),
360
+ "author": (d.get("user") or {}).get("login"),
361
+ "head_sha": d["head"]["sha"],
362
+ "created_at": d["created_at"],
363
+ "merged_at": d.get("merged_at"),
364
+ "closed_at": d.get("closed_at"),
365
+ "url": d["html_url"],
366
+ }
367
+
368
+
369
+ def stored(conn: sqlite3.Connection, sql: str, params: dict | tuple = ()) -> set:
370
+ return {row[0] for row in conn.execute(sql, params)}
371
+
372
+
373
+ CHECK_COLUMNS = ["check_id", "repo", "commit_sha", "name", "conclusion", "app", "run_id", "started_at", "completed_at",
374
+ "failure_log"]
375
+
376
+
377
+ def fetch_checks_with_logs(pool: ThreadPoolExecutor, repo: str, shas: list[str], logs: dict,
378
+ known: set) -> tuple[list[dict], list[dict]]:
379
+ """Checks on `shas`, reusing stored logs. Fetch logs only for failures whose check_id is not in `known`."""
380
+ checks = [c for rows in pool.map(lambda s: fetch_checks(repo, s), shas) for c in rows]
381
+ for check in checks:
382
+ check["failure_log"] = logs.get(check["check_id"])
383
+ failed = [c for c in checks if c["conclusion"] in ("failure", "timed_out") and c["check_id"] not in known]
384
+ for check, log in zip(failed, pool.map(lambda c: fetch_failure_log(repo, c), failed)):
385
+ check["failure_log"] = log
386
+ return checks, failed
387
+
388
+
389
+ def refetch_checks(conn: sqlite3.Connection, repo: str, since: str, workers: int) -> None:
390
+ """Re-list checks on stored commits tied to sessions active since `since`: their checkpoint
391
+ commits and the heads of their PRs. Commits with no checks stored had none, so they are skipped."""
392
+ shas = valid_shas(sorted(stored(conn, """
393
+ SELECT DISTINCT k.commit_sha FROM gh_checks k WHERE k.commit_sha IN (
394
+ SELECT cp.commit_sha FROM checkpoints cp JOIN sessions s ON s.session_id = cp.session_id
395
+ WHERE s.last_activity_at >= :since
396
+ UNION
397
+ SELECT p.head_sha FROM gh_prs p JOIN gh_commit_prs g ON g.repo = p.repo AND g.pr_number = p.pr_number
398
+ JOIN checkpoints cp ON cp.commit_sha = g.commit_sha JOIN sessions s ON s.session_id = cp.session_id
399
+ WHERE s.last_activity_at >= :since)""", {"since": since})))
400
+ known = stored(conn, "SELECT check_id FROM gh_checks")
401
+ logs = dict(conn.execute("SELECT check_id, failure_log FROM gh_checks WHERE failure_log IS NOT NULL"))
402
+ print(f"re-listing checks on {len(shas)} commits of sessions active since {since}")
403
+ with ThreadPoolExecutor(workers) as pool:
404
+ checks, failed = fetch_checks_with_logs(pool, repo, shas, logs, known)
405
+ with conn:
406
+ upsert(conn, "gh_checks", checks, CHECK_COLUMNS)
407
+ new = [c for c in checks if c["check_id"] not in known]
408
+ print(f"{len(checks)} checks listed, {len(new)} new; {len(failed)} failure logs fetched, "
409
+ f"{sum(c['failure_log'] is not None for c in failed)} found")
410
+ print(f"{requests} gh requests")
411
+
412
+
413
+ def fetch_repo(conn: sqlite3.Connection, pool: ThreadPoolExecutor, repo: str, shas: list[str], refresh: bool) -> None:
414
+ """Fetch one repo: the PRs holding checkpoint commits `shas` and the PRs local sessions link to."""
415
+ local = stored(conn, "SELECT pr_number FROM session_prs WHERE repo = ?", (repo,))
416
+ if refresh:
417
+ seen = settled = concluded = pending = set()
418
+ logs = {}
419
+ else:
420
+ seen = stored(conn, "SELECT commit_sha FROM gh_commit_prs UNION SELECT commit_sha FROM gh_commits_seen")
421
+ settled = stored(conn, "SELECT pr_number FROM settled_prs WHERE repo = ?", (repo,))
422
+ # Commits with checks stored and none of them pending.
423
+ concluded = stored(conn, "SELECT commit_sha FROM gh_checks GROUP BY commit_sha "
424
+ "HAVING COUNT(*) = COUNT(conclusion)")
425
+ # Commits with a check still pending, unless every PR holding them has settled:
426
+ # Entire Gates can stay pending forever on old commits.
427
+ pending = stored(conn, "SELECT commit_sha FROM gh_checks WHERE conclusion IS NULL") - stored(
428
+ conn, "SELECT commit_sha FROM gh_commit_prs g GROUP BY commit_sha HAVING MIN(EXISTS ("
429
+ "SELECT 1 FROM settled_prs sp WHERE sp.repo = g.repo AND sp.pr_number = g.pr_number)) = 1")
430
+ logs = dict(conn.execute("SELECT check_id, failure_log FROM gh_checks WHERE failure_log IS NOT NULL"))
431
+ new_shas = [s for s in shas if s not in seen]
432
+ print(f"{repo}: {len(shas)} checkpoint commits, {len(new_shas)} not looked up before; {len(local)} PRs from local sessions")
433
+
434
+ commit_prs = dict(zip(new_shas, pool.map(lambda s: gh_list(f"repos/{repo}/commits/{s}/pulls"), new_shas)))
435
+ links = [{"commit_sha": s, "pr_number": p["number"], "committed_at": None} for s, found in commit_prs.items() for p in found]
436
+ if new_shas:
437
+ print(f"{sum(not found for found in commit_prs.values())} new commits in no PR")
438
+
439
+ known = stored(conn, "SELECT pr_number FROM gh_commit_prs WHERE repo = ?", (repo,))
440
+ pr_numbers = sorted((known | {link["pr_number"] for link in links} | local) - settled)
441
+ prs = [p for p in pool.map(lambda n: fetch_pr(repo, n), pr_numbers) if p]
442
+ print(f"{len(prs)} PRs refreshed ({len(settled)} settled PRs skipped)")
443
+ if len(prs) < len(pr_numbers):
444
+ print(f"{len(pr_numbers) - len(prs)} PRs not found: gone, or this gh login cannot read {repo}")
445
+ pr_numbers = [p["pr_number"] for p in prs]
446
+ # A local session names its PR, not its commits: list them, with dates, so each session's
447
+ # checks can be told from another's on the same PR (db.session_commits).
448
+ listed = [n for n in pr_numbers if n in local]
449
+ for n, commits in zip(listed, pool.map(lambda n: gh_list(f"repos/{repo}/pulls/{n}/commits"), listed)):
450
+ links += [{"commit_sha": c["sha"], "pr_number": n, "committed_at": c["commit"]["committer"]["date"]}
451
+ for c in commits]
452
+
453
+ # Checks for new commits, commits with pending checks, and heads of unsettled PRs,
454
+ # where a job may have been re-run. A seen commit with no checks stored has none
455
+ # on GitHub, so it is skipped too.
456
+ heads = {p["head_sha"] for p in prs if not (p["merged"] or p["state"] == "closed")}
457
+ candidates = valid_shas(shas + [p["head_sha"] for p in prs] + [link["commit_sha"] for link in links])
458
+ check_shas = [s for s in candidates if s in pending or s in heads or (s not in seen and s not in concluded)]
459
+ checks, failed = fetch_checks_with_logs(pool, repo, check_shas, logs, set(logs))
460
+
461
+ comments = [c for rows in pool.map(lambda n: fetch_comments(repo, n), pr_numbers) for c in rows]
462
+
463
+ now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
464
+ with conn:
465
+ upsert(conn, "gh_prs", prs, ["repo", "pr_number", "title", "state", "merged", "author", "head_sha",
466
+ "created_at", "merged_at", "closed_at", "url"])
467
+ upsert(conn, "gh_commit_prs", [link | {"repo": repo} for link in links],
468
+ ["commit_sha", "repo", "pr_number", "committed_at"])
469
+ upsert(conn, "gh_commits_seen", [{"commit_sha": s, "looked_up_at": now} for s in new_shas],
470
+ ["commit_sha", "looked_up_at"])
471
+ upsert(conn, "gh_checks", checks, CHECK_COLUMNS)
472
+ upsert(conn, "gh_review_comments", comments, ["comment_id", "repo", "pr_number", "author", "is_bot", "path",
473
+ "line", "body", "in_reply_to", "created_at", "kind",
474
+ "review_state", "is_resolved", "is_outdated"])
475
+
476
+ for p in prs:
477
+ print(f" #{p['pr_number']} {p['state']}{' merged' if p['merged'] else ''} {p['title']}")
478
+ print(f"{len(checks)} checks on {len(check_shas)} commits; {len(failed)} failure logs fetched, "
479
+ f"{sum(c['failure_log'] is not None for c in failed)} found")
480
+ bots = sum(c["is_bot"] for c in comments)
481
+ print(f"{len(comments)} review comments on refreshed PRs: {bots} bot, {len(comments) - bots} human")
482
+
483
+
484
+ def main(argv: list[str] | None = None) -> None:
485
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
486
+ parser.add_argument("--repo", help="GitHub owner/name the Entire checkpoint commits belong to "
487
+ "(default: the repos local sessions pushed to)")
488
+ parser.add_argument("--workers", type=int, default=8, help="Parallel gh calls")
489
+ parser.add_argument("--refresh", action="store_true", help="Fetch everything, ignoring what is stored")
490
+ parser.add_argument("--dry-run", action="store_true", help="Say what local sessions link to; make no requests")
491
+ parser.add_argument("--checks-since", metavar="YYYY-MM-DD",
492
+ help="With --repo: only re-list checks on stored commits of sessions active since this date")
493
+ args = parser.parse_args(argv)
494
+ repo = args.repo
495
+ if repo and not REPO.fullmatch(repo):
496
+ sys.exit(f"github: --repo must be owner/name, got {repo!r}")
497
+ if args.checks_since and not repo:
498
+ sys.exit("github: --checks-since needs --repo")
499
+
500
+ conn = db.connect()
501
+ conn.executescript(SEEN_SCHEMA)
502
+ shas = load_shas(conn) if repo else []
503
+ prs, pushes = (set(), {}) if repo else find_links(conn)
504
+ # Only the repos of picked projects: those a session run in a picked folder opened a PR in or
505
+ # pushed to. Other sessions' links to those repos are kept, so a worktree session still links.
506
+ picked = {r[0] for r in conn.execute(
507
+ f"SELECT session_id FROM sessions WHERE cwd IN ({', '.join('?' * len(config.projects()))})", config.projects())}
508
+ repos = {p[1] for p in prs if p[0] in picked} | {k[0] for k, sids in pushes.items() if sids & picked}
509
+ prs, pushes = {p for p in prs if p[1] in repos}, {k: v for k, v in pushes.items() if k[0] in repos}
510
+ if args.dry_run:
511
+ print(f"github: dry run, no requests made. {len({p[1:] for p in prs})} PRs named in the logs of "
512
+ f"{len({p[0] for p in prs})} sessions, {len(pushes)} pushed branches to look up, "
513
+ f"{len({p[1] for p in prs} | {k[0] for k in pushes})} repos")
514
+ conn.close()
515
+ return
516
+ # Nothing to do is not an error: GitHub data is optional and the site hides what is missing.
517
+ if repo:
518
+ skip = None if shas else "no checkpoint commits to look up"
519
+ else:
520
+ skip = None if prs or pushes else "no session of a picked project pushed a branch or opened a pull request"
521
+ skip = skip or gh_problem()
522
+ if skip:
523
+ print(f"github: skipped, {skip}")
524
+ conn.close()
525
+ return
526
+ if args.checks_since:
527
+ refetch_checks(conn, repo, args.checks_since, args.workers)
528
+ conn.close()
529
+ return
530
+ with ThreadPoolExecutor(args.workers) as pool:
531
+ if not repo:
532
+ link_sessions(conn, pool, prs, pushes, args.refresh)
533
+ for name in [repo] if repo else sorted(stored(conn, "SELECT repo FROM session_prs") & repos):
534
+ fetch_repo(conn, pool, name, shas, args.refresh)
535
+ conn.close()
536
+ print(f"{requests} gh requests")
537
+
538
+
539
+ if __name__ == "__main__":
540
+ main()