debrief-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- debrief/__init__.py +0 -0
- debrief/build_artifact.py +83 -0
- debrief/classify.py +812 -0
- debrief/cli.py +291 -0
- debrief/compare_definitions.py +147 -0
- debrief/compare_llm.py +250 -0
- debrief/config.py +204 -0
- debrief/db.py +341 -0
- debrief/export.py +156 -0
- debrief/export_site.py +661 -0
- debrief/github.py +540 -0
- debrief/normalize.py +635 -0
- debrief/paths.py +17 -0
- debrief/prune.py +117 -0
- debrief/recommend.py +271 -0
- debrief/relay.py +83 -0
- debrief/runner.py +163 -0
- debrief/scrape.py +208 -0
- debrief/site/app.js +1613 -0
- debrief/site/index.html +32 -0
- debrief/site/method.js +117 -0
- debrief/site/style.css +520 -0
- debrief/skills.py +107 -0
- debrief/sources.py +341 -0
- debrief/synthesize.py +1052 -0
- debrief/verify_evidence.py +154 -0
- debrief_cli-0.1.0.dist-info/METADATA +205 -0
- debrief_cli-0.1.0.dist-info/RECORD +31 -0
- debrief_cli-0.1.0.dist-info/WHEEL +4 -0
- debrief_cli-0.1.0.dist-info/entry_points.txt +2 -0
- debrief_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
debrief/github.py
ADDED
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
"""Load GitHub PRs, checks and review comments for the PRs your sessions produced.
|
|
2
|
+
|
|
3
|
+
Optional. Uses the logged-in `gh` CLI, read-only, and writes session_prs, gh_prs,
|
|
4
|
+
gh_commit_prs, gh_checks and gh_review_comments in debrief.db. Without a logged-in `gh`,
|
|
5
|
+
or with nothing to look up, it prints one line and does nothing.
|
|
6
|
+
|
|
7
|
+
With no --repo it links local sessions to PRs from what their own logs show: the PR URL
|
|
8
|
+
`gh pr create` printed, and the branches `git push` sent, looked up as PR heads. A PR closed
|
|
9
|
+
before the session started is ignored. Then it fetches every repo those PRs are in.
|
|
10
|
+
With --repo it looks up the checkpoint commits of sessions imported from Entire in that repo.
|
|
11
|
+
|
|
12
|
+
Re-runs skip what cannot change: branches and commits already looked up, settled PRs,
|
|
13
|
+
checks that have all concluded, and failure logs already stored. Heads of
|
|
14
|
+
unsettled PRs always get their checks re-listed, since a job can be re-run.
|
|
15
|
+
|
|
16
|
+
Check runs are listed with filter=all, so every attempt of a re-run job is a
|
|
17
|
+
separate gh_checks row; the default returns only the latest attempt.
|
|
18
|
+
|
|
19
|
+
uv run python -m debrief.github # local sessions, every repo they pushed to
|
|
20
|
+
uv run python -m debrief.github --dry-run # say what the logs link to; no requests
|
|
21
|
+
uv run python -m debrief.github --repo owner/name # Entire checkpoints in one repo
|
|
22
|
+
uv run python -m debrief.github --refresh # ignore the skips and fetch everything
|
|
23
|
+
uv run python -m debrief.github --repo owner/name --checks-since 2026-08-11
|
|
24
|
+
# only re-list checks on stored commits of sessions active since then,
|
|
25
|
+
# to pick up attempts that the old latest-only listing dropped
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
import argparse
|
|
29
|
+
import functools
|
|
30
|
+
import json
|
|
31
|
+
import re
|
|
32
|
+
import shutil
|
|
33
|
+
import sqlite3
|
|
34
|
+
import subprocess
|
|
35
|
+
import sys
|
|
36
|
+
import threading
|
|
37
|
+
import time
|
|
38
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
39
|
+
from datetime import timedelta
|
|
40
|
+
from urllib.parse import quote
|
|
41
|
+
|
|
42
|
+
from debrief import config, db
|
|
43
|
+
from debrief.normalize import SHELL_TOOLS, parse_ts
|
|
44
|
+
|
|
45
|
+
LOG_LINES = 150
|
|
46
|
+
LOG_BYTES = 12_000
|
|
47
|
+
TIMESTAMP = re.compile(r"^\d{4}-\d\d-\d\dT[\d:.]+Z ")
|
|
48
|
+
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
|
|
49
|
+
RUN_ID = re.compile(r"/actions/runs/(\d+)")
|
|
50
|
+
# Both go into `gh api` paths, so anything else is refused.
|
|
51
|
+
SHA = re.compile(r"[0-9a-f]{7,40}")
|
|
52
|
+
REPO = re.compile(r"[A-Za-z0-9-]+/(?!\.\.?$)[\w.-]+") # the name may hold dots but not be "." or ".."
|
|
53
|
+
# What a session's shell calls show of its PRs: the URL `gh pr create` prints, the repo and
|
|
54
|
+
# branch in `git push` output ("To github.com:o/n.git", "abc..def local -> branch"), and the
|
|
55
|
+
# branch named in the push command, since `git push -q` prints nothing.
|
|
56
|
+
PR_URL = re.compile(r"https://github\.com/([\w.-]+/[\w.-]+)/pull/(\d+)")
|
|
57
|
+
GITHUB_REMOTE = r"(?:(?:[\w.-]+@)?github\.com:|(?:https|ssh)://(?:[\w.-]+@)?github\.com/)([\w.-]+/[\w.-]+?)(?:\.git)?/?\s*$"
|
|
58
|
+
PUSH_TO = re.compile("^To " + GITHUB_REMOTE, re.MULTILINE)
|
|
59
|
+
PUSH_REF = re.compile(r"^[ +*!=-]*(?:[0-9a-f]{7,40}\.\.\.?[0-9a-f]{7,40}|\[new branch\])\s+\S+\s+->\s+(\S+)", re.MULTILINE)
|
|
60
|
+
PUSH_CMD = re.compile(r"\bgit push\b(?:\s+-\S+)*\s+origin\s+(?:[\w./-]+:)?(\w[\w./-]*)")
|
|
61
|
+
# A PR whose head is one of these is a release or sync PR, not the session's work.
|
|
62
|
+
# shortcut: a fixed list, not each repo's default branch; ask GitHub if a repo's trunk has another name.
|
|
63
|
+
TRUNKS = {"main", "master", "develop", "trunk", "HEAD"}
|
|
64
|
+
RECHECK = timedelta(days=2) # how long after a session a pushed branch may still get its PR
|
|
65
|
+
# Inline comments, reviews and issue comments draw ids from separate sequences
|
|
66
|
+
# that can overlap, so review and issue rows get an offset added to their id.
|
|
67
|
+
REVIEW_ID_OFFSET = 10**15
|
|
68
|
+
ISSUE_ID_OFFSET = 2 * 10**15
|
|
69
|
+
THREADS_QUERY = """
|
|
70
|
+
query($owner: String!, $name: String!, $number: Int!, $endCursor: String) {
|
|
71
|
+
repository(owner: $owner, name: $name) {
|
|
72
|
+
pullRequest(number: $number) {
|
|
73
|
+
reviewThreads(first: 100, after: $endCursor) {
|
|
74
|
+
pageInfo { hasNextPage endCursor }
|
|
75
|
+
nodes { isResolved isOutdated comments(first: 100) { nodes { databaseId } } }
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
"""
|
|
81
|
+
SEEN_SCHEMA = """
|
|
82
|
+
CREATE TABLE IF NOT EXISTS gh_commits_seen (
|
|
83
|
+
commit_sha TEXT PRIMARY KEY, -- checkpoint commit whose PRs and checks we have fetched
|
|
84
|
+
looked_up_at TEXT
|
|
85
|
+
);
|
|
86
|
+
CREATE TABLE IF NOT EXISTS gh_branches_seen (
|
|
87
|
+
repo TEXT NOT NULL,
|
|
88
|
+
branch TEXT NOT NULL, -- pushed by a local session and looked up as a PR head
|
|
89
|
+
looked_up_at TEXT,
|
|
90
|
+
prs INTEGER, -- how many PRs had it as head
|
|
91
|
+
PRIMARY KEY (repo, branch)
|
|
92
|
+
);
|
|
93
|
+
"""
|
|
94
|
+
RETRIES = 5
|
|
95
|
+
requests = 0
|
|
96
|
+
requests_lock = threading.Lock()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def gh(path: str, paginate: bool = False, fields: list[str] | None = None) -> str | None:
|
|
100
|
+
"""Return the body of a request, or None on 404/410. `fields` are extra gh flags."""
|
|
101
|
+
# Job logs carry colour codes, which gh refuses to print without this flag.
|
|
102
|
+
cmd = ["gh", "api", path, "--allow-escape-sequences", *(fields or [])] + (["--paginate", "--slurp"] if paginate else [])
|
|
103
|
+
global requests
|
|
104
|
+
for attempt in range(RETRIES):
|
|
105
|
+
with requests_lock:
|
|
106
|
+
requests += 1
|
|
107
|
+
out = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
|
108
|
+
limited = "rate limit" in out.stderr or "HTTP 429" in out.stderr
|
|
109
|
+
if out.returncode == 0 or not limited or attempt == RETRIES - 1:
|
|
110
|
+
break
|
|
111
|
+
wait = 10 * 2**attempt
|
|
112
|
+
print(f"rate limited on {path}; retrying in {wait}s")
|
|
113
|
+
time.sleep(wait)
|
|
114
|
+
if out.returncode != 0:
|
|
115
|
+
if "HTTP 404" in out.stderr or "HTTP 410" in out.stderr:
|
|
116
|
+
return None
|
|
117
|
+
raise RuntimeError(f"gh api {path} failed: {out.stderr.strip()}")
|
|
118
|
+
return out.stdout
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def gh_list(path: str, key: str | None = None) -> list[dict]:
|
|
122
|
+
"""GET every page of a list endpoint. `key` names the list inside object pages."""
|
|
123
|
+
body = gh(path, paginate=True)
|
|
124
|
+
if body is None:
|
|
125
|
+
return []
|
|
126
|
+
pages = json.loads(body)
|
|
127
|
+
return [item for page in pages for item in (page[key] if key else page)]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def valid_shas(shas) -> list[str]:
|
|
131
|
+
"""`shas` in order, without repeats or anything that isn't a hex commit id."""
|
|
132
|
+
return [s for s in dict.fromkeys(shas) if s and SHA.fullmatch(s)]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def load_shas(conn: sqlite3.Connection) -> list[str]:
|
|
136
|
+
return valid_shas(r[0] for r in conn.execute("SELECT commit_sha FROM checkpoints ORDER BY session_id, ts"))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def gh_problem() -> str | None:
|
|
140
|
+
"""Why `gh` can't be used, or None when it is installed and logged in."""
|
|
141
|
+
if not shutil.which("gh"):
|
|
142
|
+
return "the gh CLI is not installed"
|
|
143
|
+
if subprocess.run(["gh", "auth", "status"], capture_output=True, check=False).returncode != 0:
|
|
144
|
+
return "gh is not logged in (run `gh auth login`)"
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def is_bot(user: dict | None) -> int:
|
|
149
|
+
user = user or {}
|
|
150
|
+
login = user.get("login") or ""
|
|
151
|
+
return int(user.get("type") == "Bot" or login.endswith("[bot]") or login == "Copilot")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def trim_log(log: str) -> str:
|
|
155
|
+
"""Keep the failing step's lines, capped to the last LOG_LINES / LOG_BYTES."""
|
|
156
|
+
lines = [TIMESTAMP.sub("", ANSI.sub("", line)) for line in log.splitlines()]
|
|
157
|
+
err = next((i for i, line in enumerate(lines) if "##[error]" in line), None)
|
|
158
|
+
if err is not None:
|
|
159
|
+
start = max((i for i in range(err) if lines[i].startswith("##[group]Run ")), default=0)
|
|
160
|
+
# Keep any further error lines that follow straight after the first.
|
|
161
|
+
end = err
|
|
162
|
+
while end + 1 < len(lines) and "##[error]" in lines[end + 1]:
|
|
163
|
+
end += 1
|
|
164
|
+
lines = lines[start : end + 1]
|
|
165
|
+
tail = "\n".join(lines[-LOG_LINES:])
|
|
166
|
+
# Parallel test output can push Go's failure report far above the tail.
|
|
167
|
+
fails = go_failures(lines[:-LOG_LINES])
|
|
168
|
+
if fails:
|
|
169
|
+
fails = fails[: LOG_BYTES // 2]
|
|
170
|
+
return f"{fails}\n...\n{tail[-(LOG_BYTES - len(fails) - 5):]}"
|
|
171
|
+
return tail[-LOG_BYTES:]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def go_failures(lines: list[str]) -> str:
|
|
175
|
+
"""Each `--- FAIL` line and the indented detail under it."""
|
|
176
|
+
keep = set()
|
|
177
|
+
for i, line in enumerate(lines):
|
|
178
|
+
if line.lstrip().startswith("--- FAIL"):
|
|
179
|
+
keep.add(i)
|
|
180
|
+
j = i + 1
|
|
181
|
+
while j < len(lines) and lines[j].startswith((" ", "\t")):
|
|
182
|
+
keep.add(j)
|
|
183
|
+
j += 1
|
|
184
|
+
return "\n".join(lines[i] for i in sorted(keep))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@functools.cache
|
|
188
|
+
def remote_repo(cwd: str) -> str | None:
|
|
189
|
+
"""owner/name of the checkout's GitHub origin, read from disk."""
|
|
190
|
+
out = subprocess.run(["git", "-C", cwd, "remote", "get-url", "origin"], capture_output=True, text=True, check=False)
|
|
191
|
+
m = re.search(GITHUB_REMOTE, out.stdout)
|
|
192
|
+
return m.group(1) if m else None
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def find_links(conn: sqlite3.Connection) -> tuple[set, dict]:
|
|
196
|
+
"""What the logs say, with no requests: {(session_id, repo, pr_number)} from `gh pr create`
|
|
197
|
+
output, and {(repo, branch): {session_id}} for the branches sessions pushed."""
|
|
198
|
+
calls: dict[str, list] = {}
|
|
199
|
+
marks = ", ".join("?" * (len(SHELL_TOOLS) + 1))
|
|
200
|
+
for r in conn.execute(
|
|
201
|
+
"SELECT t.session_id, s.cwd, t.input, t.output FROM tool_calls t JOIN sessions s USING (session_id) "
|
|
202
|
+
f"WHERE lower(t.name) IN ({marks}) AND (t.input LIKE '%git push%' OR t.input LIKE '%gh pr create%')",
|
|
203
|
+
[*SHELL_TOOLS, "exec"]):
|
|
204
|
+
calls.setdefault(r["session_id"], []).append(r)
|
|
205
|
+
prs, pushes = set(), {}
|
|
206
|
+
for sid, rows in calls.items():
|
|
207
|
+
created = {(repo, int(n)) for r in rows if "gh pr create" in r["input"] for repo, n in PR_URL.findall(r["output"] or "")}
|
|
208
|
+
prs |= {(sid, repo, n) for repo, n in created if REPO.fullmatch(repo)}
|
|
209
|
+
# The session's repo when a push does not print it: the one its other calls name, else the checkout's origin.
|
|
210
|
+
named = {repo for repo, _ in created} | {repo for r in rows for repo in PUSH_TO.findall(r["output"] or "")}
|
|
211
|
+
for r in rows:
|
|
212
|
+
if "git push" not in r["input"]:
|
|
213
|
+
continue
|
|
214
|
+
out = r["output"] or ""
|
|
215
|
+
repo = next(iter(PUSH_TO.findall(out)), None) or (next(iter(named)) if len(named) == 1 else remote_repo(r["cwd"] or ""))
|
|
216
|
+
for branch in {b.removeprefix("refs/heads/") for b in PUSH_REF.findall(out) + PUSH_CMD.findall(r["input"])} - TRUNKS:
|
|
217
|
+
if repo and REPO.fullmatch(repo):
|
|
218
|
+
pushes.setdefault((repo, branch), set()).add(sid)
|
|
219
|
+
return prs, pushes
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def link_sessions(conn: sqlite3.Connection, pool: ThreadPoolExecutor, prs: set, pushes: dict, refresh: bool) -> None:
|
|
223
|
+
"""Write session_prs: the PRs the logs name, plus the PRs whose head is a branch a session pushed."""
|
|
224
|
+
spans = {r[0]: (r[1] or "", r[2] or "") for r in conn.execute("SELECT session_id, started_at, last_activity_at FROM sessions")}
|
|
225
|
+
seen = {} if refresh else {(r["repo"], r["branch"]): r for r in conn.execute("SELECT * FROM gh_branches_seen")}
|
|
226
|
+
|
|
227
|
+
def settled(key) -> bool:
|
|
228
|
+
"""Looked up since the last session that pushed it ended, and a PR was found or RECHECK has passed."""
|
|
229
|
+
row, last = seen.get(key), parse_ts(max(spans[s][1] for s in pushes[key]))
|
|
230
|
+
if not row or not last:
|
|
231
|
+
return False
|
|
232
|
+
waited = parse_ts(row["looked_up_at"]) - last
|
|
233
|
+
return waited > RECHECK or (bool(row["prs"]) and waited > timedelta(0))
|
|
234
|
+
|
|
235
|
+
todo = [key for key in sorted(pushes) if not settled(key)]
|
|
236
|
+
found = list(pool.map(lambda key: json.loads(gh(
|
|
237
|
+
f"repos/{key[0]}/pulls?state=all&per_page=100&head={key[0].split('/')[0]}:{quote(key[1], safe='')}") or "[]"), todo))
|
|
238
|
+
rows = [(sid, repo, n, "url") for sid, repo, n in sorted(prs)]
|
|
239
|
+
for key, heads in zip(todo, found):
|
|
240
|
+
# A PR closed before the session began reused the branch name; it is not this session's.
|
|
241
|
+
rows += [(sid, key[0], p["number"], "branch") for p in heads for sid in sorted(pushes[key])
|
|
242
|
+
if not p.get("closed_at") or p["closed_at"][:19] >= spans[sid][0][:19]]
|
|
243
|
+
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
244
|
+
with conn:
|
|
245
|
+
conn.executemany("INSERT OR IGNORE INTO session_prs VALUES (?, ?, ?, ?)", rows)
|
|
246
|
+
conn.executemany("INSERT OR REPLACE INTO gh_branches_seen VALUES (?, ?, ?, ?)",
|
|
247
|
+
[(*key, now, len(heads)) for key, heads in zip(todo, found)])
|
|
248
|
+
linked = conn.execute("SELECT COUNT(DISTINCT session_id), COUNT(DISTINCT repo || '#' || pr_number), "
|
|
249
|
+
"COUNT(DISTINCT repo) FROM session_prs").fetchone()
|
|
250
|
+
print(f"{len(todo)} pushed branches looked up ({len(pushes) - len(todo)} skipped); "
|
|
251
|
+
f"{linked[0]} sessions linked to {linked[1]} PRs in {linked[2]} repos")
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def fetch_checks(repo: str, sha: str) -> list[dict]:
|
|
255
|
+
rows = []
|
|
256
|
+
for run in gh_list(f"repos/{repo}/commits/{sha}/check-runs?filter=all", key="check_runs"):
|
|
257
|
+
match = RUN_ID.search(run.get("details_url") or "")
|
|
258
|
+
rows.append({
|
|
259
|
+
"check_id": run["id"],
|
|
260
|
+
"repo": repo,
|
|
261
|
+
"commit_sha": sha,
|
|
262
|
+
"name": run["name"],
|
|
263
|
+
"conclusion": run["conclusion"],
|
|
264
|
+
"app": (run.get("app") or {}).get("slug"),
|
|
265
|
+
"run_id": int(match.group(1)) if match else None,
|
|
266
|
+
"started_at": run["started_at"],
|
|
267
|
+
"completed_at": run["completed_at"],
|
|
268
|
+
"output": run.get("output") or {},
|
|
269
|
+
"failure_log": None,
|
|
270
|
+
})
|
|
271
|
+
return rows
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def fetch_failure_log(repo: str, check: dict) -> str | None:
|
|
275
|
+
"""Actions jobs: tail of the job log. Other apps: the check's own output text."""
|
|
276
|
+
if check["run_id"]:
|
|
277
|
+
log = gh(f"repos/{repo}/actions/jobs/{check['check_id']}/logs")
|
|
278
|
+
return trim_log(log) if log else None
|
|
279
|
+
out = check["output"]
|
|
280
|
+
text = "\n\n".join(p for p in (out.get("title"), out.get("summary"), out.get("text")) if p)
|
|
281
|
+
return text[-LOG_BYTES:] or None
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def fetch_threads(repo: str, pr: int) -> dict[int, tuple[int, int]]:
|
|
285
|
+
"""Map inline comment id -> (is_resolved, is_outdated) of its thread."""
|
|
286
|
+
owner, name = repo.split("/")
|
|
287
|
+
fields = ["-f", f"query={THREADS_QUERY}", "-f", f"owner={owner}", "-f", f"name={name}", "-F", f"number={pr}"]
|
|
288
|
+
pages = json.loads(gh("graphql", paginate=True, fields=fields))
|
|
289
|
+
flags = {}
|
|
290
|
+
for page in pages:
|
|
291
|
+
for thread in page["data"]["repository"]["pullRequest"]["reviewThreads"]["nodes"]:
|
|
292
|
+
for comment in thread["comments"]["nodes"]:
|
|
293
|
+
flags[comment["databaseId"]] = (int(thread["isResolved"]), int(thread["isOutdated"]))
|
|
294
|
+
return flags
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def comment_row(repo: str, pr: int, kind: str, comment_id: int, item: dict, **extra) -> dict:
|
|
298
|
+
row = {
|
|
299
|
+
"comment_id": comment_id,
|
|
300
|
+
"repo": repo,
|
|
301
|
+
"pr_number": pr,
|
|
302
|
+
"kind": kind,
|
|
303
|
+
"author": (item.get("user") or {}).get("login"),
|
|
304
|
+
"is_bot": is_bot(item.get("user")),
|
|
305
|
+
"path": None,
|
|
306
|
+
"line": None,
|
|
307
|
+
"body": item.get("body") or "",
|
|
308
|
+
"in_reply_to": None,
|
|
309
|
+
"created_at": item.get("created_at") or item.get("submitted_at"),
|
|
310
|
+
"review_state": None,
|
|
311
|
+
"is_resolved": None,
|
|
312
|
+
"is_outdated": None,
|
|
313
|
+
}
|
|
314
|
+
return row | extra
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def fetch_comments(repo: str, pr: int) -> list[dict]:
|
|
318
|
+
threads = fetch_threads(repo, pr)
|
|
319
|
+
inline = [
|
|
320
|
+
comment_row(
|
|
321
|
+
repo, pr, "inline", c["id"], c,
|
|
322
|
+
path=c.get("path"),
|
|
323
|
+
line=c.get("line") or c.get("original_line"),
|
|
324
|
+
in_reply_to=c.get("in_reply_to_id"),
|
|
325
|
+
is_resolved=threads.get(c["id"], (None, None))[0],
|
|
326
|
+
is_outdated=threads.get(c["id"], (None, None))[1],
|
|
327
|
+
)
|
|
328
|
+
for c in gh_list(f"repos/{repo}/pulls/{pr}/comments")
|
|
329
|
+
]
|
|
330
|
+
# Empty-bodied reviews still matter when they approve or block.
|
|
331
|
+
reviews = [
|
|
332
|
+
comment_row(repo, pr, "review", REVIEW_ID_OFFSET + r["id"], r, review_state=r.get("state"))
|
|
333
|
+
for r in gh_list(f"repos/{repo}/pulls/{pr}/reviews")
|
|
334
|
+
if (r.get("body") or "").strip() or r.get("state") in ("APPROVED", "CHANGES_REQUESTED")
|
|
335
|
+
]
|
|
336
|
+
issue = [
|
|
337
|
+
comment_row(repo, pr, "issue", ISSUE_ID_OFFSET + c["id"], c)
|
|
338
|
+
for c in gh_list(f"repos/{repo}/issues/{pr}/comments")
|
|
339
|
+
]
|
|
340
|
+
return inline + reviews + issue
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def upsert(conn: sqlite3.Connection, table: str, rows: list[dict], columns: list[str]) -> None:
|
|
344
|
+
sql = f"INSERT OR REPLACE INTO {table} ({', '.join(columns)}) VALUES ({', '.join('?' * len(columns))})"
|
|
345
|
+
conn.executemany(sql, [[row[c] for c in columns] for row in rows])
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def fetch_pr(repo: str, number: int) -> dict | None:
|
|
349
|
+
"""One PR, or None when GitHub has none by that number that this login can see."""
|
|
350
|
+
body = gh(f"repos/{repo}/pulls/{number}")
|
|
351
|
+
if body is None:
|
|
352
|
+
return None
|
|
353
|
+
d = json.loads(body)
|
|
354
|
+
return {
|
|
355
|
+
"repo": repo,
|
|
356
|
+
"pr_number": d["number"],
|
|
357
|
+
"title": d["title"],
|
|
358
|
+
"state": d["state"],
|
|
359
|
+
"merged": int(bool(d.get("merged"))),
|
|
360
|
+
"author": (d.get("user") or {}).get("login"),
|
|
361
|
+
"head_sha": d["head"]["sha"],
|
|
362
|
+
"created_at": d["created_at"],
|
|
363
|
+
"merged_at": d.get("merged_at"),
|
|
364
|
+
"closed_at": d.get("closed_at"),
|
|
365
|
+
"url": d["html_url"],
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def stored(conn: sqlite3.Connection, sql: str, params: dict | tuple = ()) -> set:
|
|
370
|
+
return {row[0] for row in conn.execute(sql, params)}
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
CHECK_COLUMNS = ["check_id", "repo", "commit_sha", "name", "conclusion", "app", "run_id", "started_at", "completed_at",
|
|
374
|
+
"failure_log"]
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def fetch_checks_with_logs(pool: ThreadPoolExecutor, repo: str, shas: list[str], logs: dict,
|
|
378
|
+
known: set) -> tuple[list[dict], list[dict]]:
|
|
379
|
+
"""Checks on `shas`, reusing stored logs. Fetch logs only for failures whose check_id is not in `known`."""
|
|
380
|
+
checks = [c for rows in pool.map(lambda s: fetch_checks(repo, s), shas) for c in rows]
|
|
381
|
+
for check in checks:
|
|
382
|
+
check["failure_log"] = logs.get(check["check_id"])
|
|
383
|
+
failed = [c for c in checks if c["conclusion"] in ("failure", "timed_out") and c["check_id"] not in known]
|
|
384
|
+
for check, log in zip(failed, pool.map(lambda c: fetch_failure_log(repo, c), failed)):
|
|
385
|
+
check["failure_log"] = log
|
|
386
|
+
return checks, failed
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def refetch_checks(conn: sqlite3.Connection, repo: str, since: str, workers: int) -> None:
|
|
390
|
+
"""Re-list checks on stored commits tied to sessions active since `since`: their checkpoint
|
|
391
|
+
commits and the heads of their PRs. Commits with no checks stored had none, so they are skipped."""
|
|
392
|
+
shas = valid_shas(sorted(stored(conn, """
|
|
393
|
+
SELECT DISTINCT k.commit_sha FROM gh_checks k WHERE k.commit_sha IN (
|
|
394
|
+
SELECT cp.commit_sha FROM checkpoints cp JOIN sessions s ON s.session_id = cp.session_id
|
|
395
|
+
WHERE s.last_activity_at >= :since
|
|
396
|
+
UNION
|
|
397
|
+
SELECT p.head_sha FROM gh_prs p JOIN gh_commit_prs g ON g.repo = p.repo AND g.pr_number = p.pr_number
|
|
398
|
+
JOIN checkpoints cp ON cp.commit_sha = g.commit_sha JOIN sessions s ON s.session_id = cp.session_id
|
|
399
|
+
WHERE s.last_activity_at >= :since)""", {"since": since})))
|
|
400
|
+
known = stored(conn, "SELECT check_id FROM gh_checks")
|
|
401
|
+
logs = dict(conn.execute("SELECT check_id, failure_log FROM gh_checks WHERE failure_log IS NOT NULL"))
|
|
402
|
+
print(f"re-listing checks on {len(shas)} commits of sessions active since {since}")
|
|
403
|
+
with ThreadPoolExecutor(workers) as pool:
|
|
404
|
+
checks, failed = fetch_checks_with_logs(pool, repo, shas, logs, known)
|
|
405
|
+
with conn:
|
|
406
|
+
upsert(conn, "gh_checks", checks, CHECK_COLUMNS)
|
|
407
|
+
new = [c for c in checks if c["check_id"] not in known]
|
|
408
|
+
print(f"{len(checks)} checks listed, {len(new)} new; {len(failed)} failure logs fetched, "
|
|
409
|
+
f"{sum(c['failure_log'] is not None for c in failed)} found")
|
|
410
|
+
print(f"{requests} gh requests")
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def fetch_repo(conn: sqlite3.Connection, pool: ThreadPoolExecutor, repo: str, shas: list[str], refresh: bool) -> None:
|
|
414
|
+
"""Fetch one repo: the PRs holding checkpoint commits `shas` and the PRs local sessions link to."""
|
|
415
|
+
local = stored(conn, "SELECT pr_number FROM session_prs WHERE repo = ?", (repo,))
|
|
416
|
+
if refresh:
|
|
417
|
+
seen = settled = concluded = pending = set()
|
|
418
|
+
logs = {}
|
|
419
|
+
else:
|
|
420
|
+
seen = stored(conn, "SELECT commit_sha FROM gh_commit_prs UNION SELECT commit_sha FROM gh_commits_seen")
|
|
421
|
+
settled = stored(conn, "SELECT pr_number FROM settled_prs WHERE repo = ?", (repo,))
|
|
422
|
+
# Commits with checks stored and none of them pending.
|
|
423
|
+
concluded = stored(conn, "SELECT commit_sha FROM gh_checks GROUP BY commit_sha "
|
|
424
|
+
"HAVING COUNT(*) = COUNT(conclusion)")
|
|
425
|
+
# Commits with a check still pending, unless every PR holding them has settled:
|
|
426
|
+
# Entire Gates can stay pending forever on old commits.
|
|
427
|
+
pending = stored(conn, "SELECT commit_sha FROM gh_checks WHERE conclusion IS NULL") - stored(
|
|
428
|
+
conn, "SELECT commit_sha FROM gh_commit_prs g GROUP BY commit_sha HAVING MIN(EXISTS ("
|
|
429
|
+
"SELECT 1 FROM settled_prs sp WHERE sp.repo = g.repo AND sp.pr_number = g.pr_number)) = 1")
|
|
430
|
+
logs = dict(conn.execute("SELECT check_id, failure_log FROM gh_checks WHERE failure_log IS NOT NULL"))
|
|
431
|
+
new_shas = [s for s in shas if s not in seen]
|
|
432
|
+
print(f"{repo}: {len(shas)} checkpoint commits, {len(new_shas)} not looked up before; {len(local)} PRs from local sessions")
|
|
433
|
+
|
|
434
|
+
commit_prs = dict(zip(new_shas, pool.map(lambda s: gh_list(f"repos/{repo}/commits/{s}/pulls"), new_shas)))
|
|
435
|
+
links = [{"commit_sha": s, "pr_number": p["number"], "committed_at": None} for s, found in commit_prs.items() for p in found]
|
|
436
|
+
if new_shas:
|
|
437
|
+
print(f"{sum(not found for found in commit_prs.values())} new commits in no PR")
|
|
438
|
+
|
|
439
|
+
known = stored(conn, "SELECT pr_number FROM gh_commit_prs WHERE repo = ?", (repo,))
|
|
440
|
+
pr_numbers = sorted((known | {link["pr_number"] for link in links} | local) - settled)
|
|
441
|
+
prs = [p for p in pool.map(lambda n: fetch_pr(repo, n), pr_numbers) if p]
|
|
442
|
+
print(f"{len(prs)} PRs refreshed ({len(settled)} settled PRs skipped)")
|
|
443
|
+
if len(prs) < len(pr_numbers):
|
|
444
|
+
print(f"{len(pr_numbers) - len(prs)} PRs not found: gone, or this gh login cannot read {repo}")
|
|
445
|
+
pr_numbers = [p["pr_number"] for p in prs]
|
|
446
|
+
# A local session names its PR, not its commits: list them, with dates, so each session's
|
|
447
|
+
# checks can be told from another's on the same PR (db.session_commits).
|
|
448
|
+
listed = [n for n in pr_numbers if n in local]
|
|
449
|
+
for n, commits in zip(listed, pool.map(lambda n: gh_list(f"repos/{repo}/pulls/{n}/commits"), listed)):
|
|
450
|
+
links += [{"commit_sha": c["sha"], "pr_number": n, "committed_at": c["commit"]["committer"]["date"]}
|
|
451
|
+
for c in commits]
|
|
452
|
+
|
|
453
|
+
# Checks for new commits, commits with pending checks, and heads of unsettled PRs,
|
|
454
|
+
# where a job may have been re-run. A seen commit with no checks stored has none
|
|
455
|
+
# on GitHub, so it is skipped too.
|
|
456
|
+
heads = {p["head_sha"] for p in prs if not (p["merged"] or p["state"] == "closed")}
|
|
457
|
+
candidates = valid_shas(shas + [p["head_sha"] for p in prs] + [link["commit_sha"] for link in links])
|
|
458
|
+
check_shas = [s for s in candidates if s in pending or s in heads or (s not in seen and s not in concluded)]
|
|
459
|
+
checks, failed = fetch_checks_with_logs(pool, repo, check_shas, logs, set(logs))
|
|
460
|
+
|
|
461
|
+
comments = [c for rows in pool.map(lambda n: fetch_comments(repo, n), pr_numbers) for c in rows]
|
|
462
|
+
|
|
463
|
+
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
464
|
+
with conn:
|
|
465
|
+
upsert(conn, "gh_prs", prs, ["repo", "pr_number", "title", "state", "merged", "author", "head_sha",
|
|
466
|
+
"created_at", "merged_at", "closed_at", "url"])
|
|
467
|
+
upsert(conn, "gh_commit_prs", [link | {"repo": repo} for link in links],
|
|
468
|
+
["commit_sha", "repo", "pr_number", "committed_at"])
|
|
469
|
+
upsert(conn, "gh_commits_seen", [{"commit_sha": s, "looked_up_at": now} for s in new_shas],
|
|
470
|
+
["commit_sha", "looked_up_at"])
|
|
471
|
+
upsert(conn, "gh_checks", checks, CHECK_COLUMNS)
|
|
472
|
+
upsert(conn, "gh_review_comments", comments, ["comment_id", "repo", "pr_number", "author", "is_bot", "path",
|
|
473
|
+
"line", "body", "in_reply_to", "created_at", "kind",
|
|
474
|
+
"review_state", "is_resolved", "is_outdated"])
|
|
475
|
+
|
|
476
|
+
for p in prs:
|
|
477
|
+
print(f" #{p['pr_number']} {p['state']}{' merged' if p['merged'] else ''} {p['title']}")
|
|
478
|
+
print(f"{len(checks)} checks on {len(check_shas)} commits; {len(failed)} failure logs fetched, "
|
|
479
|
+
f"{sum(c['failure_log'] is not None for c in failed)} found")
|
|
480
|
+
bots = sum(c["is_bot"] for c in comments)
|
|
481
|
+
print(f"{len(comments)} review comments on refreshed PRs: {bots} bot, {len(comments) - bots} human")
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def main(argv: list[str] | None = None) -> None:
|
|
485
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
486
|
+
parser.add_argument("--repo", help="GitHub owner/name the Entire checkpoint commits belong to "
|
|
487
|
+
"(default: the repos local sessions pushed to)")
|
|
488
|
+
parser.add_argument("--workers", type=int, default=8, help="Parallel gh calls")
|
|
489
|
+
parser.add_argument("--refresh", action="store_true", help="Fetch everything, ignoring what is stored")
|
|
490
|
+
parser.add_argument("--dry-run", action="store_true", help="Say what local sessions link to; make no requests")
|
|
491
|
+
parser.add_argument("--checks-since", metavar="YYYY-MM-DD",
|
|
492
|
+
help="With --repo: only re-list checks on stored commits of sessions active since this date")
|
|
493
|
+
args = parser.parse_args(argv)
|
|
494
|
+
repo = args.repo
|
|
495
|
+
if repo and not REPO.fullmatch(repo):
|
|
496
|
+
sys.exit(f"github: --repo must be owner/name, got {repo!r}")
|
|
497
|
+
if args.checks_since and not repo:
|
|
498
|
+
sys.exit("github: --checks-since needs --repo")
|
|
499
|
+
|
|
500
|
+
conn = db.connect()
|
|
501
|
+
conn.executescript(SEEN_SCHEMA)
|
|
502
|
+
shas = load_shas(conn) if repo else []
|
|
503
|
+
prs, pushes = (set(), {}) if repo else find_links(conn)
|
|
504
|
+
# Only the repos of picked projects: those a session run in a picked folder opened a PR in or
|
|
505
|
+
# pushed to. Other sessions' links to those repos are kept, so a worktree session still links.
|
|
506
|
+
picked = {r[0] for r in conn.execute(
|
|
507
|
+
f"SELECT session_id FROM sessions WHERE cwd IN ({', '.join('?' * len(config.projects()))})", config.projects())}
|
|
508
|
+
repos = {p[1] for p in prs if p[0] in picked} | {k[0] for k, sids in pushes.items() if sids & picked}
|
|
509
|
+
prs, pushes = {p for p in prs if p[1] in repos}, {k: v for k, v in pushes.items() if k[0] in repos}
|
|
510
|
+
if args.dry_run:
|
|
511
|
+
print(f"github: dry run, no requests made. {len({p[1:] for p in prs})} PRs named in the logs of "
|
|
512
|
+
f"{len({p[0] for p in prs})} sessions, {len(pushes)} pushed branches to look up, "
|
|
513
|
+
f"{len({p[1] for p in prs} | {k[0] for k in pushes})} repos")
|
|
514
|
+
conn.close()
|
|
515
|
+
return
|
|
516
|
+
# Nothing to do is not an error: GitHub data is optional and the site hides what is missing.
|
|
517
|
+
if repo:
|
|
518
|
+
skip = None if shas else "no checkpoint commits to look up"
|
|
519
|
+
else:
|
|
520
|
+
skip = None if prs or pushes else "no session of a picked project pushed a branch or opened a pull request"
|
|
521
|
+
skip = skip or gh_problem()
|
|
522
|
+
if skip:
|
|
523
|
+
print(f"github: skipped, {skip}")
|
|
524
|
+
conn.close()
|
|
525
|
+
return
|
|
526
|
+
if args.checks_since:
|
|
527
|
+
refetch_checks(conn, repo, args.checks_since, args.workers)
|
|
528
|
+
conn.close()
|
|
529
|
+
return
|
|
530
|
+
with ThreadPoolExecutor(args.workers) as pool:
|
|
531
|
+
if not repo:
|
|
532
|
+
link_sessions(conn, pool, prs, pushes, args.refresh)
|
|
533
|
+
for name in [repo] if repo else sorted(stored(conn, "SELECT repo FROM session_prs") & repos):
|
|
534
|
+
fetch_repo(conn, pool, name, shas, args.refresh)
|
|
535
|
+
conn.close()
|
|
536
|
+
print(f"{requests} gh requests")
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
if __name__ == "__main__":
|
|
540
|
+
main()
|