toolroll 0.9.81 → 0.9.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/cli.js +1 -0
- package/dist/commit-extract.d.ts +18 -0
- package/dist/commit-extract.js +39 -1
- package/dist/converse.d.ts +12 -0
- package/dist/converse.js +18 -2
- package/dist/deploy-gate.js +8 -0
- package/dist/eval-gate.d.ts +51 -0
- package/dist/eval-gate.js +199 -0
- package/dist/eval-hooks.d.ts +39 -0
- package/dist/eval-hooks.js +477 -0
- package/dist/eval-oracle-main.d.ts +1 -0
- package/dist/eval-oracle-main.js +31 -0
- package/dist/eval-oracle.d.ts +60 -0
- package/dist/eval-oracle.js +217 -0
- package/dist/eval-world.d.ts +17 -0
- package/dist/eval-world.js +97 -0
- package/dist/evals.d.ts +299 -0
- package/dist/evals.js +964 -0
- package/dist/exec.js +15 -1
- package/dist/flow-draft.js +4 -0
- package/dist/flow-sort.js +4 -0
- package/dist/github-jobs.d.ts +25 -0
- package/dist/github-jobs.js +22 -0
- package/dist/invoke.d.ts +1 -1
- package/dist/invoke.js +14 -0
- package/dist/lead.js +4 -1
- package/dist/memory-pass.js +4 -0
- package/dist/nightly-check.d.ts +4 -1
- package/dist/nightly-check.js +5 -2
- package/dist/nightly-github.d.ts +316 -0
- package/dist/nightly-github.js +1022 -0
- package/dist/operate.d.ts +91 -1
- package/dist/operate.js +158 -1
- package/dist/project-knowledge.d.ts +4 -4
- package/dist/provider.d.ts +3 -0
- package/dist/provider.js +9 -2
- package/dist/release-adapters.d.ts +21 -21
- package/dist/release-adapters.js +52 -19
- package/dist/release-auto.js +26 -0
- package/dist/release-card.js +3 -0
- package/dist/release-cli.js +59 -3
- package/dist/release-view.d.ts +3 -0
- package/dist/release-view.js +2 -0
- package/dist/release.d.ts +16 -0
- package/dist/release.js +16 -0
- package/dist/reported-usage.d.ts +69 -0
- package/dist/reported-usage.js +97 -0
- package/dist/server/chat.js +4 -0
- package/dist/server/render-tasks.js +1 -0
- package/dist/server/tasks.js +8 -1
- package/dist/store.d.ts +109 -1
- package/dist/store.js +394 -4
- package/dist/subagents.js +4 -0
- package/dist/subscription-chat.d.ts +2 -0
- package/dist/subscription-chat.js +52 -13
- package/dist/task-sizing.js +4 -0
- package/dist/teardown-git.d.ts +9 -2
- package/dist/teardown-git.js +56 -33
- package/evals/cases/catalog/case.json +25 -0
- package/evals/cases/catalog/fixture/README.md +3 -0
- package/evals/cases/catalog/fixture/package.json +8 -0
- package/evals/cases/catalog/fixture/test/catalog.test.mjs +4 -0
- package/evals/cases/catalog/oracle.mjs +23 -0
- package/evals/cases/catalog/probe.mjs +10 -0
- package/evals/cases/catalog/source.md +9 -0
- package/evals/cases/csv-quoting/case.json +23 -0
- package/evals/cases/csv-quoting/fixture/README.md +3 -0
- package/evals/cases/csv-quoting/fixture/csv.js +1 -0
- package/evals/cases/csv-quoting/fixture/package.json +8 -0
- package/evals/cases/csv-quoting/fixture/test/csv.test.mjs +5 -0
- package/evals/cases/csv-quoting/oracle.mjs +14 -0
- package/evals/cases/csv-quoting/probe.mjs +9 -0
- package/evals/cases/csv-quoting/source.md +9 -0
- package/evals/cases/dedupe/case.json +23 -0
- package/evals/cases/dedupe/fixture/README.md +3 -0
- package/evals/cases/dedupe/fixture/dedupe.js +1 -0
- package/evals/cases/dedupe/fixture/package.json +8 -0
- package/evals/cases/dedupe/fixture/test/dedupe.test.mjs +5 -0
- package/evals/cases/dedupe/oracle.mjs +16 -0
- package/evals/cases/dedupe/probe.mjs +13 -0
- package/evals/cases/dedupe/source.md +9 -0
- package/evals/cases/endpoint/case.json +25 -0
- package/evals/cases/endpoint/fixture/README.md +3 -0
- package/evals/cases/endpoint/fixture/package.json +8 -0
- package/evals/cases/endpoint/fixture/test/endpoint.test.mjs +4 -0
- package/evals/cases/endpoint/oracle.mjs +27 -0
- package/evals/cases/endpoint/probe.mjs +10 -0
- package/evals/cases/endpoint/source.md +11 -0
- package/evals/cases/math-divide/case.json +23 -0
- package/evals/cases/math-divide/fixture/README.md +3 -0
- package/evals/cases/math-divide/fixture/package.json +8 -0
- package/evals/cases/math-divide/fixture/src/math.js +1 -0
- package/evals/cases/math-divide/fixture/test/math.test.js +5 -0
- package/evals/cases/math-divide/oracle.mjs +13 -0
- package/evals/cases/math-divide/probe.mjs +9 -0
- package/evals/cases/math-divide/source.md +9 -0
- package/evals/cases/math-multiply/case.json +23 -0
- package/evals/cases/math-multiply/fixture/README.md +3 -0
- package/evals/cases/math-multiply/fixture/package.json +8 -0
- package/evals/cases/math-multiply/fixture/src/math.js +2 -0
- package/evals/cases/math-multiply/fixture/test/math.test.js +5 -0
- package/evals/cases/math-multiply/oracle.mjs +12 -0
- package/evals/cases/math-multiply/probe.mjs +9 -0
- package/evals/cases/math-multiply/source.md +9 -0
- package/evals/cases/math-subtract/case.json +23 -0
- package/evals/cases/math-subtract/fixture/README.md +3 -0
- package/evals/cases/math-subtract/fixture/package.json +8 -0
- package/evals/cases/math-subtract/fixture/src/math.js +1 -0
- package/evals/cases/math-subtract/fixture/test/math.test.js +5 -0
- package/evals/cases/math-subtract/oracle.mjs +12 -0
- package/evals/cases/math-subtract/probe.mjs +9 -0
- package/evals/cases/math-subtract/source.md +9 -0
- package/evals/cases/merge-config/case.json +23 -0
- package/evals/cases/merge-config/fixture/README.md +3 -0
- package/evals/cases/merge-config/fixture/config.js +1 -0
- package/evals/cases/merge-config/fixture/package.json +8 -0
- package/evals/cases/merge-config/fixture/test/config.test.mjs +5 -0
- package/evals/cases/merge-config/oracle.mjs +28 -0
- package/evals/cases/merge-config/probe.mjs +15 -0
- package/evals/cases/merge-config/source.md +9 -0
- package/evals/cases/paginate/case.json +23 -0
- package/evals/cases/paginate/fixture/README.md +3 -0
- package/evals/cases/paginate/fixture/package.json +8 -0
- package/evals/cases/paginate/fixture/paginate.js +1 -0
- package/evals/cases/paginate/fixture/test/paginate.test.mjs +5 -0
- package/evals/cases/paginate/oracle.mjs +18 -0
- package/evals/cases/paginate/probe.mjs +13 -0
- package/evals/cases/paginate/source.md +9 -0
- package/evals/cases/round2-asks/case.json +24 -0
- package/evals/cases/round2-asks/fixture/README.md +3 -0
- package/evals/cases/round2-asks/fixture/package.json +8 -0
- package/evals/cases/round2-asks/fixture/src/math.js +1 -0
- package/evals/cases/round2-asks/fixture/test/math.test.js +5 -0
- package/evals/cases/round2-asks/source.md +11 -0
- package/evals/manifest.json +15 -0
- package/package.json +2 -1
|
@@ -0,0 +1,1022 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub's nightly as release debt (docs/plans/evals-nightly.md, part A). The plane reads every Nightly run and every
|
|
3
|
+
* attempt on main itself, through its own gh, and a lane that failed its tests becomes debt.
|
|
4
|
+
*
|
|
5
|
+
* - Opting in (`toolroll release nightly --github`) is a person's step, proved now: it records the GitHub repository
|
|
6
|
+
* from the project's recorded remote and seeds one stable lane per required check (release.ts REQUIRED_PR_CHECKS).
|
|
7
|
+
* A rename needs a person's alias; a lane retires only when no required check names it, by a person.
|
|
8
|
+
* - Reading keeps a paging cursor (at most NIGHTLY_PAGES_PER_READ pages a read, resumed on the next) and a watch list
|
|
9
|
+
* of every counted run created in the last 30 days (GitHub's re-run horizon), each re-read for new attempts; a run
|
|
10
|
+
* that hasn't finished stays watched whatever its age. A run counts only from `.github/workflows/nightly.yml`
|
|
11
|
+
* exactly, in the project's own repository, on main, by schedule or dispatch, at a commit on main's first-parent
|
|
12
|
+
* history as the plane read it (nightly-check.ts trustedMainOf). Every attempt's completed jobs are recorded once.
|
|
13
|
+
* - A lane's failure is a test failure unless GitHub's own step records show the runner failed (github-jobs.ts):
|
|
14
|
+
* before the lane's own steps, or, for an aggregate lane failing only at "Every shard passed", a positively
|
|
15
|
+
* established chain of readable shard failures in the same attempt, each before its tests. The coverage verifier
|
|
16
|
+
* ("Every shard ran the whole suite") failing is always a test failure. A cause that can't be read is a failure.
|
|
17
|
+
* - A failure clears only when a different counted nightly run, started after the failure completed, passes the same
|
|
18
|
+
* lane on a commit that contains the failed one. Nothing read later — a cancellation, a missing job, an unreadable
|
|
19
|
+
* page, a newer attempt — ever removes a recorded failure.
|
|
20
|
+
*
|
|
21
|
+
* Trust: a lane's conclusion is GitHub's record of the job, not independent evidence (the tests are the repository's
|
|
22
|
+
* code). It only ever adds holds, and never trusts more than the required-checks gate already does.
|
|
23
|
+
*/
|
|
24
|
+
import { authenticateApprover, personProof } from "./scope.js";
|
|
25
|
+
import { verifiedFirstParentAncestry } from "./commit-extract.js";
|
|
26
|
+
import { projectGitDir } from "./check-report.js";
|
|
27
|
+
import { failedBeforeTests } from "./github-jobs.js";
|
|
28
|
+
import { GITHUB_NIGHTLY_ON, TRUSTED_MAIN_CLEAR_MS, failureSummary, trustedMainOf } from "./nightly-check.js";
|
|
29
|
+
import { REQUIRED_PR_CHECKS, REQUIRED_PR_CHECKS_WORKFLOW } from "./release.js";
|
|
30
|
+
import { releaseRemote } from "./release-git.js";
|
|
31
|
+
import { TRUSTED_MAIN_ACTION, releaseAuthorityOfTask, trustedRemoteIdentity } from "./store.js";
|
|
32
|
+
export { GITHUB_NIGHTLY_ON };
|
|
33
|
+
/** The one workflow file whose runs count: exactly this path, never one with an `@` suffix or another folder's. */
|
|
34
|
+
export const NIGHTLY_WORKFLOW_PATH = ".github/workflows/nightly.yml";
|
|
35
|
+
/** GitHub's re-run horizon: a counted run is read again for new attempts for this long. */
|
|
36
|
+
export const NIGHTLY_WATCH_MS = 30 * 24 * 60 * 60_000;
|
|
37
|
+
/** Listing never goes further back than this: no run older than it can still be unfinished. */
|
|
38
|
+
export const NIGHTLY_LIST_HORIZON_MS = 40 * 24 * 60 * 60_000;
|
|
39
|
+
/** Pages of runs one read lists at most; a longer backlog resumes on the next read. */
|
|
40
|
+
export const NIGHTLY_PAGES_PER_READ = 10;
|
|
41
|
+
const PAGE = 100;
|
|
42
|
+
/** GitHub answers at most this many runs to one filtered listing. */
|
|
43
|
+
const LISTING_CAP = 1000;
|
|
44
|
+
/** A boundary's one bounded read (deploy, merge, pre-tag), and the background read's overall deadline. */
|
|
45
|
+
export const NIGHTLY_BOUNDARY_READ_MS = 20_000;
|
|
46
|
+
export const NIGHTLY_BACKGROUND_READ_MS = 60_000;
|
|
47
|
+
/** How often the watch loop reads each opted-in project's nightly. */
|
|
48
|
+
export const NIGHTLY_BACKGROUND_EVERY_MS = 30 * 60_000;
|
|
49
|
+
/** One call never waits longer than this, inside the read's deadline. */
|
|
50
|
+
const CALL_MS = 15_000;
|
|
51
|
+
/** How far back main's first-parent history is walked. */
|
|
52
|
+
const MAIN_WALK = 20_000;
|
|
53
|
+
const SHA = /^[0-9a-f]{40}$/;
|
|
54
|
+
/** The job-name prefix GitHub gives a lane of each called workflow inside nightly.yml (its job names "nightly" and
|
|
55
|
+
* "nightly floor"). */
|
|
56
|
+
const LANE_PREFIX = {
|
|
57
|
+
".github/workflows/ci.yml": "nightly / ",
|
|
58
|
+
".github/workflows/release-candidate.yml": "nightly floor / ",
|
|
59
|
+
};
|
|
60
|
+
/** An aggregate lane: the join of a sharded unit suite. */
|
|
61
|
+
const AGGREGATE = /^node (\d+) · (ubuntu|macos)-latest$/;
|
|
62
|
+
export const SHARDS_PASSED_STEP = "Every shard passed";
|
|
63
|
+
export const SHARDS_COVERED_STEP = "Every shard ran the whole suite";
|
|
64
|
+
const object = (value) => (value !== null && typeof value === "object" && !Array.isArray(value) ? value : null);
|
|
65
|
+
const isoOf = (value) => {
|
|
66
|
+
if (typeof value !== "string" || value === "")
|
|
67
|
+
return null;
|
|
68
|
+
const at = Date.parse(value);
|
|
69
|
+
return Number.isFinite(at) ? new Date(at).toISOString() : null;
|
|
70
|
+
};
|
|
71
|
+
const positive = (value) => Number.isSafeInteger(value) && value > 0;
|
|
72
|
+
// ---- opting in, lanes and aliases ----------------------------------------------------------------------------------
|
|
73
|
+
/** Whether a project opted in, as the plane recorded a person's step: the newest such ledger row, read strictly. */
|
|
74
|
+
export function githubNightlyOf(store, repo) {
|
|
75
|
+
const row = store.handle.prepare("SELECT detail FROM action_ledger WHERE repo = ? AND action = ? ORDER BY id DESC LIMIT 1").get(repo, GITHUB_NIGHTLY_ON);
|
|
76
|
+
if (row === undefined)
|
|
77
|
+
return null;
|
|
78
|
+
try {
|
|
79
|
+
const read = JSON.parse(String(row["detail"]));
|
|
80
|
+
if (read["v"] !== 1 || typeof read["slug"] !== "string" || typeof read["url"] !== "string" || !Number.isSafeInteger(read["record"])
|
|
81
|
+
|| typeof read["by"] !== "string" || typeof read["at"] !== "string" || typeof read["proof"] !== "string" || !personProof(read["proof"]))
|
|
82
|
+
return null;
|
|
83
|
+
return { slug: read["slug"], url: read["url"], record: Number(read["record"]), by: read["by"], at: read["at"] };
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** Every project that ever opted in (a broken or superseded record still counts here: its debt still holds). */
|
|
90
|
+
export function githubNightlyRepos(store) {
|
|
91
|
+
return store.handle.prepare("SELECT DISTINCT repo FROM action_ledger WHERE action = ? AND repo IS NOT NULL").all(GITHUB_NIGHTLY_ON).map(row => String(row["repo"]));
|
|
92
|
+
}
|
|
93
|
+
/** A lane's stable id: its workflow file and the required check's name when it was seeded. */
|
|
94
|
+
export const laneIdOf = (workflow, name) => `${workflow.replace(/^\.github\/workflows\//, "")}:${name}`;
|
|
95
|
+
/** One lane per required check whose name no lane (by its name or an alias) answers to yet. A retired lane whose name
|
|
96
|
+
* is required again opens again. */
|
|
97
|
+
export function seedLanes(store, repo, now) {
|
|
98
|
+
for (const name of REQUIRED_PR_CHECKS) {
|
|
99
|
+
const workflow = REQUIRED_PR_CHECKS_WORKFLOW[name];
|
|
100
|
+
if (resolveLane(store, repo, workflow, name) !== null)
|
|
101
|
+
continue;
|
|
102
|
+
store.handle.prepare(`INSERT INTO nightly_lane (repo, lane_id, workflow, display_name, seeded_at) VALUES (?, ?, ?, ?, ?)
|
|
103
|
+
ON CONFLICT (repo, lane_id) DO UPDATE SET retired_at = NULL, retired_by = NULL, retired_proof = NULL WHERE display_name = excluded.display_name AND workflow = excluded.workflow`)
|
|
104
|
+
.run(repo, laneIdOf(workflow, name), workflow, name, now.toISOString());
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** The live lane a job of `workflow` named `name` belongs to: a person's alias first (the newest for that name), then
|
|
108
|
+
* the lane seeded under it. Never a guess: anything else is no lane. */
|
|
109
|
+
export function resolveLane(store, repo, workflow, name) {
|
|
110
|
+
const aliased = store.handle.prepare(`SELECT a.lane_id FROM nightly_lane_alias a JOIN nightly_lane l ON l.repo = a.repo AND l.lane_id = a.lane_id
|
|
111
|
+
WHERE a.repo = ? AND a.name = ? AND l.workflow = ? AND l.retired_at IS NULL ORDER BY a.id DESC LIMIT 1`).get(repo, name, workflow);
|
|
112
|
+
if (aliased !== undefined)
|
|
113
|
+
return String(aliased["lane_id"]);
|
|
114
|
+
const own = store.handle.prepare("SELECT lane_id FROM nightly_lane WHERE repo = ? AND workflow = ? AND display_name = ? AND retired_at IS NULL").get(repo, workflow, name);
|
|
115
|
+
return own === undefined ? null : String(own["lane_id"]);
|
|
116
|
+
}
|
|
117
|
+
/** Every name a lane answers to: its seeded name and each alias. */
|
|
118
|
+
function laneNames(store, repo, laneId) {
|
|
119
|
+
const lane = store.handle.prepare("SELECT display_name FROM nightly_lane WHERE repo = ? AND lane_id = ?").get(repo, laneId);
|
|
120
|
+
const aliases = store.handle.prepare("SELECT name FROM nightly_lane_alias WHERE repo = ? AND lane_id = ? ORDER BY id").all(repo, laneId).map(row => String(row["name"]));
|
|
121
|
+
return [...new Set([...(lane === undefined ? [] : [String(lane["display_name"])]), ...aliases])];
|
|
122
|
+
}
|
|
123
|
+
/** The name a lane is shown by: its newest alias, else the name it was seeded under. */
|
|
124
|
+
export function laneName(store, repo, laneId) {
|
|
125
|
+
const alias = store.handle.prepare("SELECT name FROM nightly_lane_alias WHERE repo = ? AND lane_id = ? ORDER BY id DESC LIMIT 1").get(repo, laneId);
|
|
126
|
+
if (alias !== undefined)
|
|
127
|
+
return String(alias["name"]);
|
|
128
|
+
const lane = store.handle.prepare("SELECT display_name FROM nightly_lane WHERE repo = ? AND lane_id = ?").get(repo, laneId);
|
|
129
|
+
return lane === undefined ? laneId : String(lane["display_name"]);
|
|
130
|
+
}
|
|
131
|
+
/** A person proved now — called INSIDE the mutation's own transaction (Codex rounds 1 and 2, finding 7): the password
|
|
132
|
+
* is checked, and the approver's standing and access read, under the same lock as the write, so nothing (a
|
|
133
|
+
* revocation, a credential rotation, an access change) can land between the proof and the change. */
|
|
134
|
+
function provePerson(store, repo, as, token, what) {
|
|
135
|
+
const proved = authenticateApprover(store, as, token, repo);
|
|
136
|
+
if (!proved.ok || !personProof(proved.method))
|
|
137
|
+
return { ok: false, message: `Only an approver of this project, proved as a person, ${what}.` };
|
|
138
|
+
return { ok: true, by: as, proof: proved.method };
|
|
139
|
+
}
|
|
140
|
+
/** `toolroll release nightly --github`: a person turns GitHub's nightly into release debt for this project. The
|
|
141
|
+
* repository is the project's recorded remote's, never the checkout's. */
|
|
142
|
+
export function optInGithubNightly(store, input, now) {
|
|
143
|
+
return store.transact(() => {
|
|
144
|
+
const proved = provePerson(store, input.repo, input.as, input.token, "turns on GitHub's nightly");
|
|
145
|
+
if (!proved.ok)
|
|
146
|
+
return proved;
|
|
147
|
+
// The recorded remote as it stands inside this transaction.
|
|
148
|
+
const identity = trustedRemoteIdentity(store, input.repo);
|
|
149
|
+
const remote = identity === null ? null : releaseRemote(identity.url);
|
|
150
|
+
if (identity === null || remote === null)
|
|
151
|
+
return { ok: false, message: "The project's recorded remote isn't a GitHub repository, so its nightly can't be read." };
|
|
152
|
+
store.recordAction({ at: now.toISOString(), actor: proved.by, repo: input.repo, taskId: null, runId: null, action: GITHUB_NIGHTLY_ON, outcome: "on", source: "request",
|
|
153
|
+
detail: JSON.stringify({ v: 1, slug: remote.slug, url: identity.url, record: identity.record, workflow: NIGHTLY_WORKFLOW_PATH, by: proved.by, proof: proved.proof, at: now.toISOString() }) });
|
|
154
|
+
seedLanes(store, input.repo, now);
|
|
155
|
+
store.handle.prepare("INSERT OR IGNORE INTO github_nightly_cursor (repo, page_cursor) VALUES (?, ?)").run(input.repo, JSON.stringify(EMPTY_CURSOR));
|
|
156
|
+
const lanes = Number(store.handle.prepare("SELECT COUNT(*) AS n FROM nightly_lane WHERE repo = ? AND retired_at IS NULL").get(input.repo)?.["n"] ?? 0);
|
|
157
|
+
return { ok: true, slug: remote.slug, lanes };
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
/** `--lane-alias <old-id> <new name>`: a person says a renamed job is the same lane. Open debt goes with the lane. */
|
|
161
|
+
export function aliasNightlyLane(store, input, now) {
|
|
162
|
+
const name = input.name.trim();
|
|
163
|
+
if (name === "" || name.length > 200 || /[\u0000-\u001f\u007f]/.test(name))
|
|
164
|
+
return { ok: false, message: "That isn't a job name." };
|
|
165
|
+
return store.transact(() => {
|
|
166
|
+
const proved = provePerson(store, input.repo, input.as, input.token, "renames a nightly lane");
|
|
167
|
+
if (!proved.ok)
|
|
168
|
+
return proved;
|
|
169
|
+
const lane = store.handle.prepare("SELECT workflow, retired_at FROM nightly_lane WHERE repo = ? AND lane_id = ?").get(input.repo, input.laneId);
|
|
170
|
+
if (lane === undefined || lane["retired_at"] !== null)
|
|
171
|
+
return { ok: false, message: `No live lane ${input.laneId} in this project.` };
|
|
172
|
+
// Only a name no other lane answers to: never one another live lane resolves (whatever its debt), nor a required
|
|
173
|
+
// check's name that another lane owns, so a rename can't re-route a live lane's results onto this one.
|
|
174
|
+
const workflow = String(lane["workflow"]);
|
|
175
|
+
const other = resolveLane(store, input.repo, workflow, name);
|
|
176
|
+
if (other !== null && other !== input.laneId)
|
|
177
|
+
return { ok: false, message: `"${name}" is already lane ${other}; it can't also be ${input.laneId}.` };
|
|
178
|
+
const required = REQUIRED_PR_CHECKS.includes(name) ? laneIdOf(REQUIRED_PR_CHECKS_WORKFLOW[name], name) : null;
|
|
179
|
+
if (required !== null && required !== input.laneId)
|
|
180
|
+
return { ok: false, message: `"${name}" is a required check of its own lane (${required}); it can't be renamed onto ${input.laneId}.` };
|
|
181
|
+
store.handle.prepare("INSERT INTO nightly_lane_alias (repo, lane_id, name, by, proof, at) VALUES (?, ?, ?, ?, ?, ?)").run(input.repo, input.laneId, name, proved.by, proved.proof, now.toISOString());
|
|
182
|
+
store.recordAction({ at: now.toISOString(), actor: proved.by, repo: input.repo, taskId: null, runId: null, action: "nightly lane renamed", outcome: "aliased", source: "request", detail: `${input.laneId} → ${name}` });
|
|
183
|
+
return { ok: true };
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
/** `--lane-retire <lane-id>`: a person retires a lane no required check names any more; its open debt closes,
|
|
187
|
+
* recorded as theirs. */
|
|
188
|
+
export function retireNightlyLane(store, input, now) {
|
|
189
|
+
return store.transact(() => {
|
|
190
|
+
const proved = provePerson(store, input.repo, input.as, input.token, "retires a nightly lane");
|
|
191
|
+
if (!proved.ok)
|
|
192
|
+
return proved;
|
|
193
|
+
const lane = store.handle.prepare("SELECT retired_at FROM nightly_lane WHERE repo = ? AND lane_id = ?").get(input.repo, input.laneId);
|
|
194
|
+
if (lane === undefined || lane["retired_at"] !== null)
|
|
195
|
+
return { ok: false, message: `No live lane ${input.laneId} in this project.` };
|
|
196
|
+
const required = laneNames(store, input.repo, input.laneId).filter(name => REQUIRED_PR_CHECKS.includes(name));
|
|
197
|
+
if (required.length > 0)
|
|
198
|
+
return { ok: false, message: `"${required[0]}" is still a required check, so its lane can't be retired.` };
|
|
199
|
+
const at = now.toISOString();
|
|
200
|
+
store.handle.prepare("UPDATE nightly_lane SET retired_at = ?, retired_by = ?, retired_proof = ? WHERE repo = ? AND lane_id = ?").run(at, proved.by, proved.proof, input.repo, input.laneId);
|
|
201
|
+
const closed = Number(store.handle.prepare("UPDATE nightly_lane_failure SET cleared_at = ?, closed_why = ? WHERE repo = ? AND lane_id = ? AND cleared_at IS NULL")
|
|
202
|
+
.run(at, `retired by ${proved.by}`, input.repo, input.laneId).changes);
|
|
203
|
+
store.recordAction({ at, actor: proved.by, repo: input.repo, taskId: null, runId: null, action: "nightly lane retired", outcome: "retired", source: "request", detail: `${input.laneId} · ${closed} open failure${closed === 1 ? "" : "s"} closed` });
|
|
204
|
+
return { ok: true, closed };
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
function runOf(value) {
|
|
208
|
+
const one = object(value);
|
|
209
|
+
if (one === null || !positive(one["id"]) || typeof one["path"] !== "string" || typeof one["head_sha"] !== "string" || typeof one["event"] !== "string"
|
|
210
|
+
|| typeof one["status"] !== "string" || !positive(one["run_attempt"]))
|
|
211
|
+
return null;
|
|
212
|
+
const createdAt = isoOf(one["created_at"]);
|
|
213
|
+
if (createdAt === null)
|
|
214
|
+
return null;
|
|
215
|
+
const full = (value) => { const repo = object(value); return repo !== null && typeof repo["full_name"] === "string" ? repo["full_name"] : null; };
|
|
216
|
+
return { id: Number(one["id"]), path: one["path"], repository: full(one["repository"]), headRepository: full(one["head_repository"]),
|
|
217
|
+
headBranch: typeof one["head_branch"] === "string" ? one["head_branch"] : null, headSha: one["head_sha"], event: one["event"], status: one["status"],
|
|
218
|
+
runAttempt: Number(one["run_attempt"]), createdAt };
|
|
219
|
+
}
|
|
220
|
+
function jobOf(value) {
|
|
221
|
+
const one = object(value);
|
|
222
|
+
if (one === null || !positive(one["id"]) || typeof one["name"] !== "string" || typeof one["status"] !== "string" || typeof one["head_sha"] !== "string" || !positive(one["run_id"]))
|
|
223
|
+
return null;
|
|
224
|
+
const rawSteps = one["steps"];
|
|
225
|
+
let steps = null;
|
|
226
|
+
if (Array.isArray(rawSteps)) {
|
|
227
|
+
const read = rawSteps.map(object);
|
|
228
|
+
steps = read.every(step => step !== null && Number.isSafeInteger(step["number"]) && typeof step["name"] === "string" && typeof step["status"] === "string")
|
|
229
|
+
? read.map(step => ({ number: Number(step["number"]), name: String(step["name"]), status: String(step["status"]), conclusion: typeof step["conclusion"] === "string" ? step["conclusion"] : null }))
|
|
230
|
+
: null;
|
|
231
|
+
}
|
|
232
|
+
return { id: Number(one["id"]), name: one["name"], status: one["status"], conclusion: typeof one["conclusion"] === "string" && one["conclusion"] !== "" ? one["conclusion"] : null,
|
|
233
|
+
startedAt: isoOf(one["started_at"]), completedAt: isoOf(one["completed_at"]), headSha: one["head_sha"], runId: Number(one["run_id"]),
|
|
234
|
+
runAttempt: positive(one["run_attempt"]) ? Number(one["run_attempt"]) : null, steps };
|
|
235
|
+
}
|
|
236
|
+
/** Whether a run is one of the project's own nightly runs on main (everything but main's history, checked apart). */
|
|
237
|
+
export function nightlyRunBinds(run, slug) {
|
|
238
|
+
return run.path === NIGHTLY_WORKFLOW_PATH && run.repository === slug && run.headRepository === slug && run.headBranch === "main"
|
|
239
|
+
&& (run.event === "schedule" || run.event === "workflow_dispatch") && SHA.test(run.headSha);
|
|
240
|
+
}
|
|
241
|
+
/** A lane job's workflow and its name inside that workflow, from its name in the nightly ("nightly / node 24 · …"). */
|
|
242
|
+
function laneJobName(name) {
|
|
243
|
+
for (const [workflow, prefix] of Object.entries(LANE_PREFIX))
|
|
244
|
+
if (name.startsWith(prefix))
|
|
245
|
+
return { workflow, name: name.slice(prefix.length) };
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Why a completed lane job ended as it did, from GitHub's own job and step records in the same attempt
|
|
250
|
+
* (evals-nightly.md, "Cause of a failure"). Fails closed: anything not positively the runner's is a test failure.
|
|
251
|
+
*/
|
|
252
|
+
export function causeOf(job, attempt) {
|
|
253
|
+
if (job.conclusion === "success")
|
|
254
|
+
return "passed";
|
|
255
|
+
if (job.conclusion === "cancelled" || job.conclusion === "skipped")
|
|
256
|
+
return "not-run";
|
|
257
|
+
if (job.steps === null)
|
|
258
|
+
return "test-failure";
|
|
259
|
+
const record = (one) => ({ id: one.id, status: one.status, conclusion: one.conclusion, steps: one.steps ?? [] });
|
|
260
|
+
if (failedBeforeTests(record(job)))
|
|
261
|
+
return "runner-failure";
|
|
262
|
+
const lane = laneJobName(job.name);
|
|
263
|
+
const aggregate = lane === null ? null : AGGREGATE.exec(lane.name);
|
|
264
|
+
if (lane === null || aggregate === null)
|
|
265
|
+
return "test-failure";
|
|
266
|
+
// The coverage verifier failing is always the tests' (it can fail with every shard green: a file that ran nowhere).
|
|
267
|
+
const failedSteps = job.steps.filter(step => step.conclusion !== null && step.conclusion !== "success" && step.conclusion !== "skipped");
|
|
268
|
+
if (failedSteps.length !== 1 || failedSteps[0].name !== SHARDS_PASSED_STEP || failedSteps[0].conclusion !== "failure")
|
|
269
|
+
return "test-failure";
|
|
270
|
+
// Only "Every shard passed" failed: the runner's only on a positively established, non-empty chain — every failed
|
|
271
|
+
// shard of the same attempt readable and ended before its tests.
|
|
272
|
+
const prefix = LANE_PREFIX[lane.workflow];
|
|
273
|
+
const shard = new RegExp(`^node ${aggregate[1]} · ${aggregate[2] === "macos" ? "macos " : ""}shard \\d+/\\d+$`);
|
|
274
|
+
const shards = attempt.filter(one => one.name.startsWith(prefix) && shard.test(one.name.slice(prefix.length)));
|
|
275
|
+
const failed = shards.filter(one => one.conclusion !== "success");
|
|
276
|
+
if (failed.length === 0)
|
|
277
|
+
return "test-failure";
|
|
278
|
+
const chain = failed.every(one => one.status === "completed" && one.steps !== null && (one.conclusion === "failure" || one.conclusion === "timed_out" || one.conclusion === "cancelled")
|
|
279
|
+
&& failedBeforeTests(record(one)));
|
|
280
|
+
return chain ? "runner-failure" : "test-failure";
|
|
281
|
+
}
|
|
282
|
+
const EMPTY_CURSOR = { v: 1, done: 0, scan: null };
|
|
283
|
+
function cursorOf(store, repo) {
|
|
284
|
+
const row = store.handle.prepare("SELECT page_cursor FROM github_nightly_cursor WHERE repo = ?").get(repo);
|
|
285
|
+
if (row === undefined)
|
|
286
|
+
return EMPTY_CURSOR;
|
|
287
|
+
try {
|
|
288
|
+
const read = JSON.parse(String(row["page_cursor"]));
|
|
289
|
+
const scan = object(read["scan"]);
|
|
290
|
+
if (read["v"] !== 1 || !Number.isSafeInteger(read["done"]) || Number(read["done"]) < 0)
|
|
291
|
+
return EMPTY_CURSOR;
|
|
292
|
+
return { v: 1, done: Number(read["done"]), scan: scan !== null && positive(scan["top"]) && isoOf(scan["below"]) !== null ? { top: Number(scan["top"]), below: isoOf(scan["below"]) } : null };
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return EMPTY_CURSOR;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
function saveCursor(store, repo, cursor) {
|
|
299
|
+
store.handle.prepare("INSERT INTO github_nightly_cursor (repo, page_cursor) VALUES (?, ?) ON CONFLICT (repo) DO UPDATE SET page_cursor = excluded.page_cursor").run(repo, JSON.stringify(cursor));
|
|
300
|
+
}
|
|
301
|
+
/** Why the project's nightly can't be read now (not opted in, or its remote changed since), or its repository. */
|
|
302
|
+
function readableSlug(store, repo) {
|
|
303
|
+
const on = githubNightlyOf(store, repo);
|
|
304
|
+
if (on === null)
|
|
305
|
+
return { ok: false, why: "GitHub's nightly isn't turned on for this project" };
|
|
306
|
+
const identity = trustedRemoteIdentity(store, repo);
|
|
307
|
+
if (identity === null || identity.url !== on.url || releaseRemote(identity.url)?.slug !== on.slug)
|
|
308
|
+
return { ok: false, why: "the project's recorded remote changed since GitHub's nightly was turned on; turn it on again" };
|
|
309
|
+
return { ok: true, slug: on.slug };
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* One read of the project's GitHub nightly within `budgetMs`: list runs from the cursor, re-read the watch list (new
|
|
313
|
+
* and unfinished runs first, then the oldest-checked), record every completed lane job, open and clear failures. A
|
|
314
|
+
* failed call stops where it is; what was recorded stays, and the cursor never moves past a page it couldn't read.
|
|
315
|
+
*/
|
|
316
|
+
export async function readGithubNightly(store, repo, deps, budgetMs) {
|
|
317
|
+
const started = deps.now();
|
|
318
|
+
const deadline = started.getTime() + budgetMs;
|
|
319
|
+
const readable = readableSlug(store, repo);
|
|
320
|
+
if (!readable.ok) {
|
|
321
|
+
noteRead(store, repo, readable.why, started);
|
|
322
|
+
return readable;
|
|
323
|
+
}
|
|
324
|
+
const slug = readable.slug;
|
|
325
|
+
seedLanes(store, repo, started);
|
|
326
|
+
const get = async (path) => {
|
|
327
|
+
const left = deadline - deps.now().getTime();
|
|
328
|
+
if (left <= 0)
|
|
329
|
+
return { ok: false, why: "the read ran out of time" };
|
|
330
|
+
try {
|
|
331
|
+
return await deps.gh(path, Math.min(left, CALL_MS));
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
return { ok: false, why: String(error?.message ?? error).slice(0, 200) };
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
// Local history work (main's first-parent walks) is bounded by the same budget on the wall clock.
|
|
338
|
+
const wall = Date.now() + budgetMs;
|
|
339
|
+
let problem = null;
|
|
340
|
+
const listing = await listRuns(store, repo, slug, get, deps.now);
|
|
341
|
+
if (!listing.ok)
|
|
342
|
+
problem = listing.why;
|
|
343
|
+
const watched = await revisitWatchList(store, repo, slug, get, deps, () => deps.now().getTime() >= deadline || Date.now() >= wall, wall);
|
|
344
|
+
if (problem === null && !watched.ok)
|
|
345
|
+
problem = watched.why;
|
|
346
|
+
const settled = settleGithubNightly(store, repo, deps, deps.now(), wall);
|
|
347
|
+
const complete = listing.ok && listing.complete && watched.ok && watched.complete && settled.complete;
|
|
348
|
+
// Partial progress is never a whole read (Codex round 1, finding 5): what is still unresolved is said, and the last
|
|
349
|
+
// whole read's time stays as it was.
|
|
350
|
+
const unresolved = problem ?? (complete ? null : watched.ok && watched.unresolved !== null ? watched.unresolved : settled.unresolved ?? "some runs are still to be read");
|
|
351
|
+
noteRead(store, repo, unresolved, deps.now());
|
|
352
|
+
return problem === null ? { ok: true, listed: listing.listed, checked: watched.checked, complete } : { ok: false, why: problem };
|
|
353
|
+
}
|
|
354
|
+
function noteRead(store, repo, why, now) {
|
|
355
|
+
store.handle.prepare(`INSERT INTO github_nightly_cursor (repo, page_cursor, last_ok_at, last_why) VALUES (?, ?, ?, ?)
|
|
356
|
+
ON CONFLICT (repo) DO UPDATE SET last_ok_at = COALESCE(excluded.last_ok_at, last_ok_at), last_why = excluded.last_why`)
|
|
357
|
+
.run(repo, JSON.stringify(EMPTY_CURSOR), why === null ? now.toISOString() : null, why === null ? null : why.slice(0, 300));
|
|
358
|
+
}
|
|
359
|
+
/** List nightly.yml's runs on main from the cursor, at most NIGHTLY_PAGES_PER_READ pages, putting every run that binds
|
|
360
|
+
* on the watch list page by page. A page that can't be read whole stops here, the cursor before it. */
|
|
361
|
+
async function listRuns(store, repo, slug, get, now) {
|
|
362
|
+
const cursor = cursorOf(store, repo);
|
|
363
|
+
const filter = cursor.scan?.below ?? null;
|
|
364
|
+
let top = cursor.scan?.top ?? cursor.done, below = filter, total = null, listed = 0;
|
|
365
|
+
for (let page = 1; page <= NIGHTLY_PAGES_PER_READ; page++) {
|
|
366
|
+
const query = `branch=main&per_page=${PAGE}&page=${page}${filter === null ? "" : `&created=${encodeURIComponent(`<=${filter}`)}`}`;
|
|
367
|
+
const got = await get(`repos/${slug}/actions/workflows/nightly.yml/runs?${query}`);
|
|
368
|
+
if (!got.ok)
|
|
369
|
+
return { ok: false, why: `GitHub's nightly runs couldn't be listed: ${got.why}`, listed };
|
|
370
|
+
const body = object(got.body);
|
|
371
|
+
const rows = body === null ? null : body["workflow_runs"];
|
|
372
|
+
const count = body === null ? null : body["total_count"];
|
|
373
|
+
if (!Array.isArray(rows) || rows.length > PAGE || !Number.isSafeInteger(count) || Number(count) < 0)
|
|
374
|
+
return { ok: false, why: "a page of GitHub's nightly runs couldn't be read", listed };
|
|
375
|
+
if (total !== null && total !== count)
|
|
376
|
+
return { ok: false, why: "GitHub's list of nightly runs changed while it was read", listed };
|
|
377
|
+
total = Number(count);
|
|
378
|
+
const expected = Math.max(0, Math.min(PAGE, Math.min(total, LISTING_CAP) - (page - 1) * PAGE));
|
|
379
|
+
const runs = rows.map(runOf);
|
|
380
|
+
if (rows.length !== expected || runs.some(one => one === null) || new Set(runs.map(one => one.id)).size !== runs.length) {
|
|
381
|
+
return { ok: false, why: "a page of GitHub's nightly runs came back cut short or unreadable", listed };
|
|
382
|
+
}
|
|
383
|
+
let end = rows.length < PAGE;
|
|
384
|
+
const at = now();
|
|
385
|
+
store.transact(() => {
|
|
386
|
+
for (const run of runs) {
|
|
387
|
+
if (run.id <= cursor.done) {
|
|
388
|
+
end = true;
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
const age = at.getTime() - Date.parse(run.createdAt);
|
|
392
|
+
if (age > NIGHTLY_LIST_HORIZON_MS && run.status === "completed") {
|
|
393
|
+
end = true;
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
if (!nightlyRunBinds(run, slug) || (age > NIGHTLY_WATCH_MS && run.status === "completed"))
|
|
397
|
+
continue;
|
|
398
|
+
listed += Number(store.handle.prepare("INSERT OR IGNORE INTO github_nightly_watch (repo, gh_run, created_at) VALUES (?, ?, ?)").run(repo, run.id, run.createdAt).changes);
|
|
399
|
+
}
|
|
400
|
+
for (const run of runs) {
|
|
401
|
+
top = Math.max(top, run.id);
|
|
402
|
+
if (below === null || run.createdAt < below)
|
|
403
|
+
below = run.createdAt;
|
|
404
|
+
}
|
|
405
|
+
saveCursor(store, repo, end || rows.length === 0 ? { v: 1, done: Math.max(cursor.done, top), scan: null } : { v: 1, done: cursor.done, scan: { top, below: below } });
|
|
406
|
+
});
|
|
407
|
+
if (end || rows.length === 0)
|
|
408
|
+
return { ok: true, listed, complete: true };
|
|
409
|
+
}
|
|
410
|
+
return { ok: true, listed, complete: false };
|
|
411
|
+
}
|
|
412
|
+
/** Read one run by id, strictly. */
|
|
413
|
+
async function readRun(get, slug, id) {
|
|
414
|
+
const got = await get(`repos/${slug}/actions/runs/${id}`);
|
|
415
|
+
if (!got.ok)
|
|
416
|
+
return null;
|
|
417
|
+
const run = runOf(got.body);
|
|
418
|
+
return run !== null && run.id === id ? run : null;
|
|
419
|
+
}
|
|
420
|
+
/** Every job of one exact attempt of one run, every page, each GitHub's record of that run, attempt and commit. Null
|
|
421
|
+
* when any page can't be read whole. */
|
|
422
|
+
export async function readAttemptJobs(get, slug, run, attempt, sha) {
|
|
423
|
+
const all = [];
|
|
424
|
+
let total = null;
|
|
425
|
+
for (let page = 1; page <= 10; page++) {
|
|
426
|
+
const got = await get(`repos/${slug}/actions/runs/${run}/attempts/${attempt}/jobs?per_page=${PAGE}&page=${page}`);
|
|
427
|
+
if (!got.ok)
|
|
428
|
+
return null;
|
|
429
|
+
const body = object(got.body);
|
|
430
|
+
const rows = body === null ? null : body["jobs"];
|
|
431
|
+
const count = body === null ? null : body["total_count"];
|
|
432
|
+
if (!Array.isArray(rows) || rows.length > PAGE || !Number.isSafeInteger(count) || (total !== null && total !== count))
|
|
433
|
+
return null;
|
|
434
|
+
total = Number(count);
|
|
435
|
+
for (const row of rows) {
|
|
436
|
+
const job = jobOf(row);
|
|
437
|
+
if (job === null || job.runId !== run || job.headSha !== sha || (job.runAttempt !== null && job.runAttempt !== attempt))
|
|
438
|
+
return null;
|
|
439
|
+
all.push(job);
|
|
440
|
+
}
|
|
441
|
+
if (rows.length < PAGE || all.length >= total)
|
|
442
|
+
break;
|
|
443
|
+
}
|
|
444
|
+
if (total === null || all.length !== total || new Set(all.map(one => one.id)).size !== all.length)
|
|
445
|
+
return null;
|
|
446
|
+
return all;
|
|
447
|
+
}
|
|
448
|
+
/** Re-read every watched run within the budget (never-checked first, then unfinished, then the oldest-checked): new
|
|
449
|
+
* attempts' jobs are recorded; a run leaves once it is 30 days old, finished and fully read. */
|
|
450
|
+
async function revisitWatchList(store, repo, slug, get, deps, outOfTime, wall) {
|
|
451
|
+
// Never-checked first, then strictly the oldest-checked (Codex round 1, finding 4: unfinished runs never starve a
|
|
452
|
+
// finished one whose rerun is waiting to be seen).
|
|
453
|
+
const rows = store.handle.prepare(`SELECT gh_run, created_at, attempts_read, finished FROM github_nightly_watch WHERE repo = ?
|
|
454
|
+
ORDER BY last_checked_at IS NOT NULL, last_checked_at, gh_run`).all(repo);
|
|
455
|
+
let problem = null, checked = 0, complete = true, unresolved = null;
|
|
456
|
+
for (const row of rows) {
|
|
457
|
+
if (outOfTime()) {
|
|
458
|
+
complete = false;
|
|
459
|
+
break;
|
|
460
|
+
}
|
|
461
|
+
const id = Number(row["gh_run"]), attemptsRead = Number(row["attempts_read"]);
|
|
462
|
+
const mark = (fields = {}) => store.handle.prepare("UPDATE github_nightly_watch SET attempts_read = MAX(attempts_read, ?), finished = ?, last_checked_at = ? WHERE repo = ? AND gh_run = ?")
|
|
463
|
+
.run(fields.attemptsRead ?? attemptsRead, (fields.finished ?? Number(row["finished"]) === 1) ? 1 : 0, deps.now().toISOString(), repo, id);
|
|
464
|
+
const drop = () => store.handle.prepare("DELETE FROM github_nightly_watch WHERE repo = ? AND gh_run = ?").run(repo, id);
|
|
465
|
+
const first = await readRun(get, slug, id);
|
|
466
|
+
if (first === null) {
|
|
467
|
+
problem ??= `nightly run ${id} couldn't be read`;
|
|
468
|
+
complete = false;
|
|
469
|
+
mark();
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
checked++;
|
|
473
|
+
// A run that doesn't bind never counts; one not on main's history (as the plane read main) never counts either.
|
|
474
|
+
if (!nightlyRunBinds(first, slug)) {
|
|
475
|
+
drop();
|
|
476
|
+
continue;
|
|
477
|
+
}
|
|
478
|
+
const old = deps.now().getTime() - Date.parse(first.createdAt) > NIGHTLY_WATCH_MS;
|
|
479
|
+
// "No" only from a read of main made after the run was created, and only for a finished run; anything less certain
|
|
480
|
+
// keeps the run watched (never dropped on a stale read of main: its failure would never be recorded).
|
|
481
|
+
const onMain = deps.onMain(first.headSha, first.createdAt, wall);
|
|
482
|
+
if (onMain === "no" && first.status === "completed") {
|
|
483
|
+
drop();
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
if (onMain !== "yes") {
|
|
487
|
+
complete = false;
|
|
488
|
+
unresolved ??= `nightly run ${id}'s commit ${first.headSha.slice(0, 7)} couldn't be placed on main's history yet`;
|
|
489
|
+
mark();
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
if (first.runAttempt < attemptsRead) {
|
|
493
|
+
problem ??= `nightly run ${id}'s attempts went backwards`;
|
|
494
|
+
mark();
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
497
|
+
let read = attemptsRead, whole = true;
|
|
498
|
+
for (let attempt = attemptsRead + 1; attempt <= first.runAttempt; attempt++) {
|
|
499
|
+
const jobs = await readAttemptJobs(get, slug, id, attempt, first.headSha);
|
|
500
|
+
if (jobs === null) {
|
|
501
|
+
problem ??= `nightly run ${id} attempt ${attempt}'s jobs couldn't be read`;
|
|
502
|
+
whole = false;
|
|
503
|
+
complete = false;
|
|
504
|
+
break;
|
|
505
|
+
}
|
|
506
|
+
recordAttempt(store, repo, { run: id, attempt, sha: first.headSha, createdAt: first.createdAt, jobs }, deps.now());
|
|
507
|
+
// An earlier attempt than the newest is over: GitHub re-runs only a finished run.
|
|
508
|
+
if (attempt < first.runAttempt)
|
|
509
|
+
read = attempt;
|
|
510
|
+
}
|
|
511
|
+
let finished = false;
|
|
512
|
+
if (whole) {
|
|
513
|
+
// Read again: an attempt or status that changed mid-read is read whole next time; what was recorded stays.
|
|
514
|
+
const again = await readRun(get, slug, id);
|
|
515
|
+
if (again !== null && again.runAttempt === first.runAttempt && again.status === first.status && first.status === "completed") {
|
|
516
|
+
read = first.runAttempt;
|
|
517
|
+
finished = true;
|
|
518
|
+
}
|
|
519
|
+
else if (again === null) {
|
|
520
|
+
problem ??= `nightly run ${id} couldn't be read again`;
|
|
521
|
+
complete = false;
|
|
522
|
+
}
|
|
523
|
+
else
|
|
524
|
+
complete = false;
|
|
525
|
+
}
|
|
526
|
+
if (finished && old && read === first.runAttempt) {
|
|
527
|
+
drop();
|
|
528
|
+
continue;
|
|
529
|
+
}
|
|
530
|
+
mark({ attemptsRead: read, finished });
|
|
531
|
+
}
|
|
532
|
+
return problem === null ? { ok: true, checked, complete, unresolved } : { ok: false, why: problem, checked };
|
|
533
|
+
}
|
|
534
|
+
/** Record one attempt's completed lane jobs once each, and open a failure for each test failure. */
|
|
535
|
+
function recordAttempt(store, repo, attempt, now) {
|
|
536
|
+
const at = now.toISOString();
|
|
537
|
+
store.transact(() => {
|
|
538
|
+
for (const job of attempt.jobs) {
|
|
539
|
+
if (job.status !== "completed")
|
|
540
|
+
continue;
|
|
541
|
+
const lane = laneJobName(job.name);
|
|
542
|
+
const laneId = lane === null ? null : resolveLane(store, repo, lane.workflow, lane.name);
|
|
543
|
+
if (laneId === null)
|
|
544
|
+
continue;
|
|
545
|
+
const cause = causeOf(job, attempt.jobs);
|
|
546
|
+
store.handle.prepare(`INSERT OR IGNORE INTO nightly_lane_job (repo, lane_id, gh_run, run_created_at, attempt, job_id, sha, started_at, completed_at, conclusion, cause, read_at)
|
|
547
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(repo, laneId, attempt.run, attempt.createdAt, attempt.attempt, job.id, attempt.sha, job.startedAt, job.completedAt, job.conclusion, cause, at);
|
|
548
|
+
if (cause !== "test-failure")
|
|
549
|
+
continue;
|
|
550
|
+
// No completion time on record: the failure counts as completed now, so only a pass started later clears it.
|
|
551
|
+
const opened = store.handle.prepare(`INSERT OR IGNORE INTO nightly_lane_failure (repo, lane_id, gh_run, run_created_at, attempt, job_id, sha, completed_at, opened_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
552
|
+
.run(repo, laneId, attempt.run, attempt.createdAt, attempt.attempt, job.id, attempt.sha, job.completedAt ?? at, at);
|
|
553
|
+
// One attention notice per failure, when it is first recorded.
|
|
554
|
+
if (Number(opened.changes) === 1) {
|
|
555
|
+
const lane = laneName(store, repo, laneId);
|
|
556
|
+
store.enqueueNotification({ dedupeKey: `github-nightly-failed:${Number(opened.lastInsertRowid)}`, kind: "github-nightly-failed", subject: "GitHub's nightly failed on main",
|
|
557
|
+
body: `Nightly checks failed: ${lane}, on main at ${attempt.sha.slice(0, 7)}. Releases wait until it passes again.`, pushClass: "attention",
|
|
558
|
+
// The project it belongs to, so its people's chats may carry it (Codex round 1, finding 8).
|
|
559
|
+
source: { project: repo }, link: "/board" }, now);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
export function openFailures(store, repo) {
|
|
565
|
+
return store.handle.prepare("SELECT * FROM nightly_lane_failure WHERE repo = ? AND cleared_at IS NULL ORDER BY id").all(repo).map(row => ({
|
|
566
|
+
id: Number(row["id"]), repo: String(row["repo"]), laneId: String(row["lane_id"]), ghRun: Number(row["gh_run"]), attempt: Number(row["attempt"]), jobId: Number(row["job_id"]),
|
|
567
|
+
sha: String(row["sha"]), completedAt: String(row["completed_at"]), openedAt: String(row["opened_at"]),
|
|
568
|
+
}));
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Clear each open failure that a later counted nightly run passed: a run created after the failed run (a rerun of an
|
|
572
|
+
* older run never clears newer debt; Codex round 1, finding 2), the same lane, a job started after the failed one
|
|
573
|
+
* completed, on a commit that contains the failed one. A re-run inside the same run never clears. Local history work
|
|
574
|
+
* stops at `wall` (wall-clock ms); what is left is settled on a later read.
|
|
575
|
+
*/
|
|
576
|
+
export function settleGithubNightly(store, repo, history, now, wall = Date.now() + NIGHTLY_BACKGROUND_READ_MS) {
|
|
577
|
+
let cleared = 0, unresolved = null;
|
|
578
|
+
const late = () => ({ cleared, complete: false, unresolved: unresolved ?? "settling the open failures ran out of time; the rest is settled on the next read" });
|
|
579
|
+
for (const failure of openFailures(store, repo)) {
|
|
580
|
+
if (Date.now() >= wall)
|
|
581
|
+
return late();
|
|
582
|
+
const passes = store.handle.prepare(`SELECT j.gh_run, j.attempt, j.job_id, j.sha FROM nightly_lane_job j JOIN nightly_lane_failure f ON f.id = ?
|
|
583
|
+
WHERE j.repo = ? AND j.lane_id = ? AND j.cause = 'passed' AND j.gh_run <> f.gh_run AND j.run_created_at > f.run_created_at
|
|
584
|
+
AND j.started_at IS NOT NULL AND j.started_at > f.completed_at ORDER BY j.started_at, j.id`).all(failure.id, repo, failure.laneId);
|
|
585
|
+
// Three-state, candidate by candidate: an unreadable or timed-out ancestry is never "doesn't contain" (Codex round 2,
|
|
586
|
+
// finding 5) — the failure stays open and the read says it is unresolved.
|
|
587
|
+
let pass;
|
|
588
|
+
for (const one of passes) {
|
|
589
|
+
if (Date.now() >= wall)
|
|
590
|
+
return late();
|
|
591
|
+
const contains = String(one["sha"]) === failure.sha ? "yes" : history.mainContains(String(one["sha"]), failure.sha, wall);
|
|
592
|
+
if (contains === "yes") {
|
|
593
|
+
pass = one;
|
|
594
|
+
break;
|
|
595
|
+
}
|
|
596
|
+
if (contains === "unknown")
|
|
597
|
+
unresolved ??= `whether nightly run ${Number(one["gh_run"])} contains ${failure.sha.slice(0, 7)} couldn't be read yet`;
|
|
598
|
+
}
|
|
599
|
+
if (Date.now() >= wall)
|
|
600
|
+
return late();
|
|
601
|
+
if (pass === undefined)
|
|
602
|
+
continue;
|
|
603
|
+
const by = `${Number(pass["gh_run"])}/${Number(pass["attempt"])}/${Number(pass["job_id"])}`;
|
|
604
|
+
store.transact(() => {
|
|
605
|
+
if (Number(store.handle.prepare("UPDATE nightly_lane_failure SET cleared_at = ?, cleared_by_job = ?, closed_why = 'passed' WHERE id = ? AND cleared_at IS NULL").run(now.toISOString(), by, failure.id).changes) !== 1)
|
|
606
|
+
return;
|
|
607
|
+
store.recordAction({ at: now.toISOString(), actor: "toolroll", repo, taskId: null, runId: null, action: "nightly lane failure cleared", outcome: "cleared", source: "work",
|
|
608
|
+
detail: `${laneName(store, repo, failure.laneId)} at ${failure.sha.slice(0, 7)} · by nightly run ${by}` });
|
|
609
|
+
cleared++;
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
return unresolved === null ? { cleared, complete: true, unresolved: null } : { cleared, complete: false, unresolved };
|
|
613
|
+
}
|
|
614
|
+
/** Main's history as the plane read it from the project's recorded remote (nightly-check.ts trustedMainOf), from the
|
|
615
|
+
* verified objects in the project's own Git store; never a ref a build could move. */
|
|
616
|
+
export function mainHistoryOf(store, repo, now) {
|
|
617
|
+
const main = trustedMainOf(store, repo, now, TRUSTED_MAIN_CLEAR_MS);
|
|
618
|
+
// When that read of main was made: the same ledger row trustedMainOf stands on.
|
|
619
|
+
const readAt = main === null ? null : String(store.handle.prepare("SELECT at FROM action_ledger WHERE repo = ? AND action = ? ORDER BY id DESC LIMIT 1").get(repo, TRUSTED_MAIN_ACTION)?.["at"] ?? "");
|
|
620
|
+
const gitDir = projectGitDir(repo);
|
|
621
|
+
return {
|
|
622
|
+
onMain(sha, since, deadline) {
|
|
623
|
+
if (main === null || gitDir === null || !SHA.test(sha))
|
|
624
|
+
return "unknown";
|
|
625
|
+
if (sha === main)
|
|
626
|
+
return "yes";
|
|
627
|
+
// "No" only when proven: every commit back to the root read and verified without meeting it (Codex round 1,
|
|
628
|
+
// finding 1) — and only by a read of main made after the run was created. A missing or unreadable object, the
|
|
629
|
+
// walk's limit or the deadline is "unknown": the run stays watched.
|
|
630
|
+
const walked = verifiedFirstParentAncestry(gitDir, main, sha, { deadline, max: MAIN_WALK });
|
|
631
|
+
if (walked !== "no")
|
|
632
|
+
return walked;
|
|
633
|
+
return readAt !== null && Number.isFinite(Date.parse(readAt)) && Date.parse(readAt) > Date.parse(since) ? "no" : "unknown";
|
|
634
|
+
},
|
|
635
|
+
mainContains(sha, ancestor, deadline) {
|
|
636
|
+
if (gitDir === null || !SHA.test(sha) || !SHA.test(ancestor))
|
|
637
|
+
return "unknown";
|
|
638
|
+
return verifiedFirstParentAncestry(gitDir, sha, ancestor, { deadline, max: MAIN_WALK });
|
|
639
|
+
},
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
/**
|
|
643
|
+
* The watch loop's background read (operate.ts refreshGithubNightlies): each opted-in project at most every
|
|
644
|
+
* NIGHTLY_BACKGROUND_EVERY_MS, claimed in the store so two processes don't both read, within one overall deadline.
|
|
645
|
+
*/
|
|
646
|
+
export async function readGithubNightlies(store, deps, budgetMs = NIGHTLY_BACKGROUND_READ_MS) {
|
|
647
|
+
const started = deps.now();
|
|
648
|
+
const deadline = started.getTime() + budgetMs;
|
|
649
|
+
let read = 0;
|
|
650
|
+
for (const repo of githubNightlyRepos(store)) {
|
|
651
|
+
const left = deadline - deps.now().getTime();
|
|
652
|
+
if (left <= 0)
|
|
653
|
+
break;
|
|
654
|
+
if (githubNightlyOf(store, repo) === null)
|
|
655
|
+
continue;
|
|
656
|
+
const at = deps.now();
|
|
657
|
+
store.handle.prepare("INSERT OR IGNORE INTO github_nightly_cursor (repo, page_cursor) VALUES (?, ?)").run(repo, JSON.stringify(EMPTY_CURSOR));
|
|
658
|
+
const claimed = store.handle.prepare("UPDATE github_nightly_cursor SET last_tried_at = ? WHERE repo = ? AND (last_tried_at IS NULL OR last_tried_at <= ?)")
|
|
659
|
+
.run(at.toISOString(), repo, new Date(at.getTime() - NIGHTLY_BACKGROUND_EVERY_MS).toISOString());
|
|
660
|
+
if (Number(claimed.changes) !== 1)
|
|
661
|
+
continue;
|
|
662
|
+
const history = deps.history === undefined ? mainHistoryOf(store, repo, at) : deps.history(repo, at);
|
|
663
|
+
await readGithubNightly(store, repo, { ...history, gh: deps.gh, now: deps.now }, left);
|
|
664
|
+
read++;
|
|
665
|
+
}
|
|
666
|
+
return read;
|
|
667
|
+
}
|
|
668
|
+
/** How long a boundary's re-read of a candidate's exemption stands for the stored checks inside the reservation, the
|
|
669
|
+
* deployment gate and the installed runtime's deploy check after it. */
|
|
670
|
+
export const EXEMPTION_FRESH_MS = 10 * 60_000;
|
|
671
|
+
const STAMP = new Intl.DateTimeFormat("en-US", { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", hourCycle: "h23", timeZone: "UTC" });
|
|
672
|
+
/** "Oct 30, 06:17 UTC". */
|
|
673
|
+
export const nightlyStamp = (iso) => `${STAMP.format(new Date(iso))} UTC`;
|
|
674
|
+
/** The words of a nightly hold: the first open failure, how many more, and the way out at this point. */
|
|
675
|
+
export function nightlyDebtWords(store, held, phase, release) {
|
|
676
|
+
const first = held[0];
|
|
677
|
+
const more = held.length - 1;
|
|
678
|
+
const lanes = `GitHub's nightly failed "${laneName(store, first.repo, first.laneId)}" on main at ${first.sha.slice(0, 7)} (${nightlyStamp(first.completedAt)})${more > 0 ? `, and ${more} more lane${more === 1 ? "" : "s"}` : ""}.`;
|
|
679
|
+
const who = release ? "This release" : "This candidate";
|
|
680
|
+
const what = phase === "deploy" ? "can't deploy" : phase === "merge" ? "can't merge" : phase === "tag" ? "can't be tagged or published" : "can't be approved automatically";
|
|
681
|
+
// After the merge the release's own CI can't help (the release proof refuses jobs finished after the merge).
|
|
682
|
+
const way = release && phase !== "tag" ? "re-run this release's CI, or wait for tonight's nightly" : "wait for tonight's nightly, or dispatch Nightly on main";
|
|
683
|
+
return `${lanes} ${who} ${what} until ${more > 0 ? "those lanes pass" : "that lane passes"} again: ${way}.`;
|
|
684
|
+
}
|
|
685
|
+
/** Whether this exact candidate of this task holds a fresh exemption for one failure. */
|
|
686
|
+
function exempted(store, taskId, candidate, failure, now) {
|
|
687
|
+
const row = store.handle.prepare("SELECT read_at FROM nightly_exemption WHERE task = ? AND candidate = ? AND failure_id = ?").get(taskId, candidate, failure);
|
|
688
|
+
return row !== undefined && Date.parse(String(row["read_at"])) >= now.getTime() - EXEMPTION_FRESH_MS && Date.parse(String(row["read_at"])) <= now.getTime() + 60_000;
|
|
689
|
+
}
|
|
690
|
+
/**
|
|
691
|
+
* The stored check every boundary makes, inside the caller's own transaction where it has one (the deployment gate, the
|
|
692
|
+
* deploy and merge reservations, the pre-tag step, an automatic approval): the project's open lane failures, less those
|
|
693
|
+
* this exact task and candidate hold a fresh exemption for. Null when nothing holds. A project that never opted in has
|
|
694
|
+
* no lane failures.
|
|
695
|
+
*/
|
|
696
|
+
export function nightlyDebtHold(store, input) {
|
|
697
|
+
const held = openFailures(store, input.repo).filter(failure => input.taskId === null || input.candidate === null || !exempted(store, input.taskId, input.candidate, failure.id, input.now));
|
|
698
|
+
return held.length === 0 ? null : { why: nightlyDebtWords(store, held, input.phase, input.release), failures: held };
|
|
699
|
+
}
|
|
700
|
+
/** No nightly run seen for this long reads "hasn't run since". */
|
|
701
|
+
export const NIGHTLY_STALE_MS = 36 * 60 * 60_000;
|
|
702
|
+
export function githubNightlyStatus(store, repo, now) {
|
|
703
|
+
const on = githubNightlyOf(store, repo);
|
|
704
|
+
const cursor = store.handle.prepare("SELECT last_ok_at, last_why FROM github_nightly_cursor WHERE repo = ?").get(repo);
|
|
705
|
+
const ran = store.handle.prepare(`SELECT MAX(at) AS at FROM (SELECT MAX(created_at) AS at FROM github_nightly_watch WHERE repo = ?
|
|
706
|
+
UNION ALL SELECT MAX(started_at) FROM nightly_lane_job WHERE repo = ?)`).get(repo, repo);
|
|
707
|
+
const lastRunAt = ran?.["at"] == null ? null : String(ran["at"]);
|
|
708
|
+
const since = lastRunAt ?? on?.at ?? null;
|
|
709
|
+
return { on: on !== null, lastOkAt: cursor?.["last_ok_at"] == null ? null : String(cursor["last_ok_at"]), lastWhy: cursor?.["last_why"] == null ? null : String(cursor["last_why"]),
|
|
710
|
+
lastRunAt, stale: on !== null && since !== null && now.getTime() - Date.parse(since) > NIGHTLY_STALE_MS };
|
|
711
|
+
}
|
|
712
|
+
const agoWords = (iso, now) => {
|
|
713
|
+
if (iso === null)
|
|
714
|
+
return "never read";
|
|
715
|
+
const minutes = Math.max(0, Math.round((now.getTime() - Date.parse(iso)) / 60_000));
|
|
716
|
+
return minutes < 60 ? `last read ${minutes} min ago` : minutes < 48 * 60 ? `last read ${Math.round(minutes / 60)} h ago` : `last read ${Math.round(minutes / 1440)} days ago`;
|
|
717
|
+
};
|
|
718
|
+
/** A warning that never holds anything (Decision 1): GitHub couldn't be read, or the nightly hasn't run lately. */
|
|
719
|
+
export function nightlyWarning(status, now) {
|
|
720
|
+
if (!status.on)
|
|
721
|
+
return null;
|
|
722
|
+
if (status.lastWhy !== null)
|
|
723
|
+
return `GitHub nightly: couldn't be read (${status.lastWhy}), ${agoWords(status.lastOkAt, now)}. Not holding the release.`;
|
|
724
|
+
if (status.stale)
|
|
725
|
+
return status.lastRunAt === null ? "GitHub nightly: hasn't run since it was turned on. Not holding the release." : `GitHub nightly: hasn't run since ${nightlyStamp(status.lastRunAt)}. Not holding the release.`;
|
|
726
|
+
return null;
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* A release candidate's own pull-request CI, as an exemption needs it (modelled on scripts/release-proof.mjs): for each
|
|
730
|
+
* workflow, the newest run on `head` from that pull request's `pull_request` event, that run's latest attempt's jobs
|
|
731
|
+
* with their times, then the run read again. The newest run is chosen first: one that doesn't bind to this pull
|
|
732
|
+
* request never lets an older run stand. `null` for a workflow with no such run; "unreadable" when anything can't be
|
|
733
|
+
* read whole or changed during the read.
|
|
734
|
+
*/
|
|
735
|
+
export async function readLaneJobs(get, slug, pr, head, workflows) {
|
|
736
|
+
const listed = [];
|
|
737
|
+
const raw = new Map();
|
|
738
|
+
let total = null;
|
|
739
|
+
for (let page = 1; page <= 10; page++) {
|
|
740
|
+
const got = await get(`repos/${slug}/actions/runs?head_sha=${head}&event=pull_request&per_page=${PAGE}&page=${page}`);
|
|
741
|
+
if (!got.ok)
|
|
742
|
+
return "unreadable";
|
|
743
|
+
const body = object(got.body);
|
|
744
|
+
const rows = body === null ? null : body["workflow_runs"];
|
|
745
|
+
const count = body === null ? null : body["total_count"];
|
|
746
|
+
if (!Array.isArray(rows) || rows.length > PAGE || !Number.isSafeInteger(count) || (total !== null && total !== count))
|
|
747
|
+
return "unreadable";
|
|
748
|
+
total = Number(count);
|
|
749
|
+
for (const row of rows) {
|
|
750
|
+
const run = runOf(row);
|
|
751
|
+
if (run === null || raw.has(run.id))
|
|
752
|
+
return "unreadable";
|
|
753
|
+
listed.push(run);
|
|
754
|
+
raw.set(run.id, object(row));
|
|
755
|
+
}
|
|
756
|
+
if (rows.length < PAGE || listed.length >= total)
|
|
757
|
+
break;
|
|
758
|
+
}
|
|
759
|
+
if (total === null || listed.length !== total)
|
|
760
|
+
return "unreadable";
|
|
761
|
+
const found = new Map();
|
|
762
|
+
for (const workflow of new Set(workflows)) {
|
|
763
|
+
const newest = listed.filter(run => run.path === workflow && run.headSha === head && run.event === "pull_request").sort((a, b) => b.id - a.id)[0];
|
|
764
|
+
if (newest === undefined) {
|
|
765
|
+
found.set(workflow, null);
|
|
766
|
+
continue;
|
|
767
|
+
}
|
|
768
|
+
const bound = raw.get(newest.id)["pull_requests"];
|
|
769
|
+
const binds = newest.repository === slug && newest.headRepository === slug && Array.isArray(bound) && bound.length === 1 && (() => {
|
|
770
|
+
const one = object(bound[0]), base = object(one?.["base"]), baseRepo = object(base?.["repo"]), headRef = object(one?.["head"]);
|
|
771
|
+
return one?.["number"] === pr && base?.["ref"] === "main" && baseRepo?.["url"] === `https://api.github.com/repos/${slug}` && headRef?.["sha"] === head;
|
|
772
|
+
})();
|
|
773
|
+
if (!binds) {
|
|
774
|
+
found.set(workflow, null);
|
|
775
|
+
continue;
|
|
776
|
+
}
|
|
777
|
+
if (newest.status !== "completed") {
|
|
778
|
+
found.set(workflow, { run: newest.id, attempt: newest.runAttempt, status: newest.status, jobs: [] });
|
|
779
|
+
continue;
|
|
780
|
+
}
|
|
781
|
+
const jobs = await readAttemptJobs(get, slug, newest.id, newest.runAttempt, head);
|
|
782
|
+
if (jobs === null)
|
|
783
|
+
return "unreadable";
|
|
784
|
+
const again = await readRun(get, slug, newest.id);
|
|
785
|
+
if (again === null || again.runAttempt !== newest.runAttempt || again.status !== newest.status)
|
|
786
|
+
return "unreadable";
|
|
787
|
+
found.set(workflow, { run: newest.id, attempt: newest.runAttempt, status: newest.status, jobs });
|
|
788
|
+
}
|
|
789
|
+
// The runs again: a newer run or attempt that appeared mid-read voids the read.
|
|
790
|
+
const relisted = await get(`repos/${slug}/actions/runs?head_sha=${head}&event=pull_request&per_page=${PAGE}&page=1`);
|
|
791
|
+
const relistedBody = relisted.ok ? object(relisted.body) : null;
|
|
792
|
+
const relistedRows = relistedBody === null ? null : relistedBody["workflow_runs"];
|
|
793
|
+
if (!Array.isArray(relistedRows) || relistedBody["total_count"] !== total)
|
|
794
|
+
return "unreadable";
|
|
795
|
+
if (total <= PAGE) {
|
|
796
|
+
const newestOf = (runs, workflow) => runs.filter((run) => run !== null && run.path === workflow && run.headSha === head && run.event === "pull_request").sort((a, b) => b.id - a.id)[0];
|
|
797
|
+
for (const workflow of found.keys()) {
|
|
798
|
+
const before = newestOf(listed, workflow), after = newestOf(relistedRows.map(runOf), workflow);
|
|
799
|
+
if (before?.id !== after?.id || before?.runAttempt !== after?.runAttempt || before?.status !== after?.status)
|
|
800
|
+
return "unreadable";
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
return found;
|
|
804
|
+
}
|
|
805
|
+
/** The files a lane's job runs that the candidate could otherwise rewrite to make a job of that name pass: its workflow
|
|
806
|
+
* file and the CI scripts the workflows call. An exemption stands only when each is byte-identical (the same Git
|
|
807
|
+
* blob) on the candidate and at the failed commit. The tests themselves are the candidate's, as for every required
|
|
808
|
+
* check (the trust the required-checks gate already places in them). */
|
|
809
|
+
export const LANE_PINNED_FILES = ["scripts/ci-shards.mjs", "scripts/ci-scope.mjs", "scripts/ci-git.sh", "scripts/ci-no-skips.mjs"];
|
|
810
|
+
/** Whether every pinned file of a lane's workflow is the same blob on `candidate` as at `failed`, by GitHub's own
|
|
811
|
+
* contents API; null when any can't be read. */
|
|
812
|
+
async function lanePinned(get, slug, workflow, candidate, failed) {
|
|
813
|
+
for (const path of [workflow, ...LANE_PINNED_FILES]) {
|
|
814
|
+
const blobs = [];
|
|
815
|
+
for (const ref of [candidate, failed]) {
|
|
816
|
+
const got = await get(`repos/${slug}/contents/${path}?ref=${ref}`);
|
|
817
|
+
const body = got.ok ? object(got.body) : null;
|
|
818
|
+
if (body === null || body["type"] !== "file" || typeof body["sha"] !== "string" || !/^[0-9a-f]{40,64}$/.test(body["sha"]))
|
|
819
|
+
return null;
|
|
820
|
+
blobs.push(body["sha"]);
|
|
821
|
+
}
|
|
822
|
+
if (blobs[0] !== blobs[1])
|
|
823
|
+
return { same: false, path };
|
|
824
|
+
}
|
|
825
|
+
return { same: true };
|
|
826
|
+
}
|
|
827
|
+
/** Whether `candidate` contains `ancestor`, by GitHub's own comparison; null when it can't be read. */
|
|
828
|
+
async function candidateContains(get, slug, candidate, ancestor) {
|
|
829
|
+
if (candidate === ancestor)
|
|
830
|
+
return true;
|
|
831
|
+
const got = await get(`repos/${slug}/compare/${ancestor}...${candidate}`);
|
|
832
|
+
if (!got.ok)
|
|
833
|
+
return null;
|
|
834
|
+
const body = object(got.body);
|
|
835
|
+
if (body === null || typeof body["status"] !== "string" || !Number.isSafeInteger(body["behind_by"]))
|
|
836
|
+
return null;
|
|
837
|
+
return body["behind_by"] === 0 && (body["status"] === "ahead" || body["status"] === "identical");
|
|
838
|
+
}
|
|
839
|
+
/** A read time no stored check accepts: an exemption whose last re-read failed. */
|
|
840
|
+
const STALE_READ = "1970-01-01T00:00:00.000Z";
|
|
841
|
+
/**
|
|
842
|
+
* A boundary (the coordinator's deploy, merge and pre-tag steps): one bounded read of GitHub's nightly, then this
|
|
843
|
+
* candidate's exemptions read again from its own pull-request CI — minted only before the merge (`deploy`, `merge`);
|
|
844
|
+
* at the tag an exemption obtained before the merge is re-read, never made — then the stored debt. A read that fails is
|
|
845
|
+
* a warning and the stored debt still holds; an exemption that can't be re-read is no exemption here.
|
|
846
|
+
*/
|
|
847
|
+
export async function nightlyBoundary(store, input, deps, budgetMs = NIGHTLY_BOUNDARY_READ_MS) {
|
|
848
|
+
const started = deps.now().getTime();
|
|
849
|
+
const deadline = started + budgetMs;
|
|
850
|
+
if (githubNightlyOf(store, input.repo) === null && openFailures(store, input.repo).length === 0)
|
|
851
|
+
return { ok: true, warning: null };
|
|
852
|
+
const get = async (path) => {
|
|
853
|
+
const left = deadline - deps.now().getTime();
|
|
854
|
+
if (left <= 0)
|
|
855
|
+
return { ok: false, why: "the read ran out of time" };
|
|
856
|
+
try {
|
|
857
|
+
return await deps.gh(path, Math.min(left, CALL_MS));
|
|
858
|
+
}
|
|
859
|
+
catch (error) {
|
|
860
|
+
return { ok: false, why: String(error?.message ?? error).slice(0, 200) };
|
|
861
|
+
}
|
|
862
|
+
};
|
|
863
|
+
// The nightly itself, within about two thirds of the budget; the rest is the candidate's own CI.
|
|
864
|
+
if (githubNightlyOf(store, input.repo) !== null)
|
|
865
|
+
await readGithubNightly(store, input.repo, deps, Math.max(0, Math.floor(budgetMs * 2 / 3) - (deps.now().getTime() - started)));
|
|
866
|
+
const warning = nightlyWarning(githubNightlyStatus(store, input.repo, deps.now()), deps.now());
|
|
867
|
+
const open = openFailures(store, input.repo);
|
|
868
|
+
const unverified = new Set();
|
|
869
|
+
// A re-read that fails takes the stored permission away too (Codex round 1, finding 3): the row stays, so a later
|
|
870
|
+
// boundary that reads it whole again (the tag included) can restore it, but no stored check accepts it meanwhile.
|
|
871
|
+
const invalidate = (failure) => store.handle.prepare("UPDATE nightly_exemption SET read_at = ? WHERE task = ? AND candidate = ? AND failure_id = ?")
|
|
872
|
+
.run(STALE_READ, input.taskId, input.candidate, failure);
|
|
873
|
+
/** Failures this candidate can't be exempted from: it changes how the lane runs (the path that differs). */
|
|
874
|
+
const changed = new Map();
|
|
875
|
+
const slug = githubNightlyOf(store, input.repo)?.slug ?? null;
|
|
876
|
+
if (open.length > 0 && slug !== null && SHA.test(input.candidate) && positive(input.pr)) {
|
|
877
|
+
const lanes = new Map(open.map(failure => [failure.id, store.handle.prepare("SELECT workflow FROM nightly_lane WHERE repo = ? AND lane_id = ?").get(input.repo, failure.laneId)?.["workflow"]]));
|
|
878
|
+
const read = await readLaneJobs(get, slug, input.pr, input.candidate, [...new Set([...lanes.values()].filter((one) => typeof one === "string"))]);
|
|
879
|
+
for (const failure of open) {
|
|
880
|
+
const workflow = lanes.get(failure.id);
|
|
881
|
+
const row = store.handle.prepare("SELECT gh_run, attempt, job_id FROM nightly_exemption WHERE task = ? AND candidate = ? AND failure_id = ?").get(input.taskId, input.candidate, failure.id);
|
|
882
|
+
if (read === "unreadable" || typeof workflow !== "string") {
|
|
883
|
+
unverified.add(failure.id);
|
|
884
|
+
invalidate(failure.id);
|
|
885
|
+
continue;
|
|
886
|
+
}
|
|
887
|
+
const found = read.get(workflow) ?? null;
|
|
888
|
+
const names = laneNames(store, input.repo, failure.laneId);
|
|
889
|
+
const named = found === null ? [] : found.jobs.filter(job => names.includes(job.name));
|
|
890
|
+
const job = named.length === 1 ? named[0] : null;
|
|
891
|
+
const valid = found !== null && found.status === "completed" && job !== null && job.status === "completed" && job.conclusion === "success" && job.headSha === input.candidate
|
|
892
|
+
&& job.startedAt !== null && job.completedAt !== null && job.startedAt > failure.completedAt;
|
|
893
|
+
const same = row !== undefined && found !== null && job !== null && Number(row["gh_run"]) === found.run && Number(row["attempt"]) === found.attempt && Number(row["job_id"]) === job.id;
|
|
894
|
+
const at = deps.now().toISOString();
|
|
895
|
+
// The newest run or attempt differs, is pending, or holds a failure for that lane: the exemption is void.
|
|
896
|
+
if (row !== undefined && !(same && valid))
|
|
897
|
+
store.handle.prepare("DELETE FROM nightly_exemption WHERE task = ? AND candidate = ? AND failure_id = ?").run(input.taskId, input.candidate, failure.id);
|
|
898
|
+
if (!valid)
|
|
899
|
+
continue;
|
|
900
|
+
if (same) {
|
|
901
|
+
store.handle.prepare("UPDATE nightly_exemption SET read_at = ? WHERE task = ? AND candidate = ? AND failure_id = ?").run(at, input.taskId, input.candidate, failure.id);
|
|
902
|
+
continue;
|
|
903
|
+
}
|
|
904
|
+
if (input.phase === "tag")
|
|
905
|
+
continue;
|
|
906
|
+
const contains = await candidateContains(get, slug, input.candidate, failure.sha);
|
|
907
|
+
if (contains !== true) {
|
|
908
|
+
if (contains === null) {
|
|
909
|
+
unverified.add(failure.id);
|
|
910
|
+
invalidate(failure.id);
|
|
911
|
+
}
|
|
912
|
+
continue;
|
|
913
|
+
}
|
|
914
|
+
// A job named like the lane proves nothing if the candidate rewrote what that job runs.
|
|
915
|
+
const pinned = await lanePinned(get, slug, workflow, input.candidate, failure.sha);
|
|
916
|
+
if (pinned === null) {
|
|
917
|
+
unverified.add(failure.id);
|
|
918
|
+
invalidate(failure.id);
|
|
919
|
+
continue;
|
|
920
|
+
}
|
|
921
|
+
if (!pinned.same) {
|
|
922
|
+
changed.set(failure.id, pinned.path);
|
|
923
|
+
continue;
|
|
924
|
+
}
|
|
925
|
+
store.handle.prepare(`INSERT INTO nightly_exemption (task, candidate, failure_id, pr, gh_run, attempt, job_id, started_at, completed_at, read_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
926
|
+
ON CONFLICT (task, candidate, failure_id) DO UPDATE SET pr = excluded.pr, gh_run = excluded.gh_run, attempt = excluded.attempt, job_id = excluded.job_id,
|
|
927
|
+
started_at = excluded.started_at, completed_at = excluded.completed_at, read_at = excluded.read_at`)
|
|
928
|
+
.run(input.taskId, input.candidate, failure.id, input.pr, found.run, found.attempt, job.id, job.startedAt, job.completedAt, at);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
const now = deps.now();
|
|
932
|
+
const hold = store.transact(() => {
|
|
933
|
+
const stored = nightlyDebtHold(store, { repo: input.repo, taskId: input.taskId, candidate: input.candidate, phase: input.phase, release: true, now });
|
|
934
|
+
// An exemption that couldn't be re-read now never counts at this boundary, whatever is stored.
|
|
935
|
+
const held = openFailures(store, input.repo).filter(failure => unverified.has(failure.id) || stored?.failures.some(one => one.id === failure.id) === true);
|
|
936
|
+
if (held.length === 0)
|
|
937
|
+
return null;
|
|
938
|
+
const rewrote = held.map(failure => changed.get(failure.id)).find((path) => path !== undefined);
|
|
939
|
+
return rewrote === undefined ? nightlyDebtWords(store, held, input.phase, true)
|
|
940
|
+
: `${nightlyDebtWords(store, held, input.phase, true)} This release changes ${rewrote}, which that lane runs, so its own CI can't stand in for the nightly: wait for tonight's nightly, or dispatch Nightly on main.`;
|
|
941
|
+
});
|
|
942
|
+
return hold === null ? { ok: true, warning } : { ok: false, why: hold, warning };
|
|
943
|
+
}
|
|
944
|
+
/**
|
|
945
|
+
* Needs you, for the projects `shown` admits: one item per open GitHub nightly lane failure (linking to its job on
|
|
946
|
+
* GitHub, under the opted-in repository the plane recorded), and one per nightly check of main with open failures
|
|
947
|
+
* (linking to its task). Oldest first.
|
|
948
|
+
*/
|
|
949
|
+
export function nightlyNeedsYou(store, shown) {
|
|
950
|
+
const needs = [];
|
|
951
|
+
const failed = store.handle.prepare("SELECT DISTINCT repo FROM nightly_lane_failure WHERE cleared_at IS NULL").all().map(row => String(row["repo"])).filter(shown);
|
|
952
|
+
for (const repo of failed) {
|
|
953
|
+
const slug = githubNightlyOf(store, repo)?.slug ?? null;
|
|
954
|
+
for (const failure of openFailures(store, repo)) {
|
|
955
|
+
needs.push({ key: `github-nightly-${failure.id}`, repo, title: `Nightly checks failed: ${laneName(store, repo, failure.laneId)}`,
|
|
956
|
+
reason: `On main at ${failure.sha.slice(0, 7)}. Releases wait until it passes again.`,
|
|
957
|
+
href: slug === null ? "/board" : `https://github.com/${slug}/actions/runs/${failure.ghRun}/job/${failure.jobId}`, since: failure.openedAt });
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
// The plane's own nightly check of main (nightly-check.ts): one item per failed check, its items summed up.
|
|
961
|
+
const rows = store.handle.prepare("SELECT repo, run, sha, MIN(opened_at) AS since, COUNT(*) AS n FROM main_check_failure WHERE cleared_at IS NULL GROUP BY repo, run, sha ORDER BY since").all();
|
|
962
|
+
for (const row of rows) {
|
|
963
|
+
const repo = String(row["repo"]);
|
|
964
|
+
if (!shown(repo))
|
|
965
|
+
continue;
|
|
966
|
+
const run = store.getRun(Number(row["run"]));
|
|
967
|
+
const task = run === null ? null : store.refById(run.taskRef)?.externalId ?? null;
|
|
968
|
+
const items = store.handle.prepare("SELECT item FROM main_check_failure WHERE repo = ? AND run = ? AND cleared_at IS NULL ORDER BY item").all(repo, Number(row["run"])).map(one => String(one["item"]));
|
|
969
|
+
needs.push({ key: `main-check-${Number(row["run"])}`, repo, title: `Nightly check of main failed: ${failureSummary(items)}`,
|
|
970
|
+
reason: `On main at ${String(row["sha"]).slice(0, 7)}. Composed releases wait until a full check passes.`,
|
|
971
|
+
href: task === null ? "/board" : `/t/${encodeURIComponent(task)}`, since: String(row["since"]) });
|
|
972
|
+
}
|
|
973
|
+
return needs.sort((a, b) => a.since.localeCompare(b.since));
|
|
974
|
+
}
|
|
975
|
+
/** The release card's GitHub nightly lines: each open lane failure (held, or exempted for this candidate), then a
|
|
976
|
+
* warning when GitHub couldn't be read or the nightly hasn't run lately. Empty when there is nothing to say. */
|
|
977
|
+
export function nightlyReleaseLines(store, input) {
|
|
978
|
+
const lines = openFailures(store, input.repo).map(failure => {
|
|
979
|
+
const lane = `GitHub nightly: failed ${laneName(store, input.repo, failure.laneId)} (${nightlyStamp(failure.completedAt)}).`;
|
|
980
|
+
return input.candidate !== null && exempted(store, input.taskId, input.candidate, failure.id, input.now)
|
|
981
|
+
? `${lane} This release re-ran it after the failure and passed.` : `${lane} This release waits until it passes again.`;
|
|
982
|
+
});
|
|
983
|
+
const warning = nightlyWarning(githubNightlyStatus(store, input.repo, input.now), input.now);
|
|
984
|
+
return warning === null ? lines : [...lines, warning];
|
|
985
|
+
}
|
|
986
|
+
/**
|
|
987
|
+
* A deploy run's own boundary (scripts/deploy-browser.mjs, through the installed runtime; Codex round 1, finding 3):
|
|
988
|
+
* the same bounded read as the coordinator's — GitHub's nightly, then this candidate's exemptions re-read — before the
|
|
989
|
+
* stored check, so a deploy started by hand neither misses a new failure nor rides an exemption its CI no longer
|
|
990
|
+
* supports. A task no release covers has no exemption to re-read; its nightly is still read. Anything that fails here
|
|
991
|
+
* takes away the candidate's stored exemptions; the stored check that follows decides.
|
|
992
|
+
*/
|
|
993
|
+
export async function refreshNightlyForDeploy(store, input) {
|
|
994
|
+
const run = store.getRun(input.runId);
|
|
995
|
+
const ref = run === null ? null : store.refById(run.taskRef);
|
|
996
|
+
if (run === null || ref === null || ref.repo === null)
|
|
997
|
+
return null;
|
|
998
|
+
const repo = ref.repo;
|
|
999
|
+
if (githubNightlyOf(store, repo) === null && openFailures(store, repo).length === 0)
|
|
1000
|
+
return null;
|
|
1001
|
+
const candidate = store.getScope(ref.externalId)?.candidate ?? run.headRevision;
|
|
1002
|
+
const at = input.now();
|
|
1003
|
+
try {
|
|
1004
|
+
const gh = input.gh ?? await (async () => {
|
|
1005
|
+
const [{ exec, githubReads }, { releaseTools }] = await Promise.all([import("./release-adapters.js"), import("./release-git.js")]);
|
|
1006
|
+
return githubReads(exec, releaseTools().gh);
|
|
1007
|
+
})();
|
|
1008
|
+
const deps = { ...mainHistoryOf(store, repo, at), gh, now: input.now };
|
|
1009
|
+
const pr = releaseAuthorityOfTask(store, ref.externalId)?.authority.pr ?? null;
|
|
1010
|
+
if (candidate === null || pr === null) {
|
|
1011
|
+
if (githubNightlyOf(store, repo) !== null)
|
|
1012
|
+
await readGithubNightly(store, repo, deps, input.budgetMs ?? NIGHTLY_BOUNDARY_READ_MS);
|
|
1013
|
+
return null;
|
|
1014
|
+
}
|
|
1015
|
+
return await nightlyBoundary(store, { repo, taskId: ref.externalId, candidate, pr, phase: "deploy" }, deps, input.budgetMs ?? NIGHTLY_BOUNDARY_READ_MS);
|
|
1016
|
+
}
|
|
1017
|
+
catch {
|
|
1018
|
+
if (candidate !== null)
|
|
1019
|
+
store.handle.prepare("UPDATE nightly_exemption SET read_at = ? WHERE task = ? AND candidate = ?").run(STALE_READ, ref.externalId, candidate);
|
|
1020
|
+
return null;
|
|
1021
|
+
}
|
|
1022
|
+
}
|