toolroll 0.9.76 → 0.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +273 -0
- package/README.md +11 -4
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/agent-fence.js +46 -10
- package/dist/assignment.js +3 -0
- package/dist/backup-ui.d.ts +9 -1
- package/dist/backup-ui.js +28 -3
- package/dist/backup.js +4 -3
- package/dist/browser/THIRD_PARTY_NOTICES.txt +5 -5
- package/dist/browser/workspace.css +2 -2
- package/dist/browser/workspace.js +31 -31
- package/dist/browser-workspace.d.ts +2 -0
- package/dist/builder.d.ts +18 -3
- package/dist/builder.js +152 -47
- package/dist/chat-decide.js +4 -0
- package/dist/check-failure.d.ts +92 -0
- package/dist/check-failure.js +295 -0
- package/dist/check-progress.d.ts +28 -1
- package/dist/check-progress.js +105 -6
- package/dist/check-report.d.ts +67 -6
- package/dist/check-report.js +254 -9
- package/dist/checkout-cleanup.js +2 -2
- package/dist/claim.d.ts +9 -0
- package/dist/claim.js +9 -1
- package/dist/cli.js +8 -5
- package/dist/coding-handoff.d.ts +1 -0
- package/dist/coding-handoff.js +20 -11
- package/dist/commit-extract.d.ts +6 -0
- package/dist/commit-extract.js +63 -8
- package/dist/console-chrome.d.ts +41 -0
- package/dist/console-chrome.js +166 -0
- package/dist/contracts/console-api.d.ts +1 -1
- package/dist/contracts/console-api.js +1 -1
- package/dist/contracts/release-check.d.ts +12 -0
- package/dist/contracts/release-check.js +17 -0
- package/dist/contracts/release-terms.d.ts +41 -0
- package/dist/contracts/release-terms.js +53 -0
- package/dist/contracts/verification-receipt.d.ts +20 -2
- package/dist/contracts/verification-receipt.js +15 -3
- package/dist/contracts/verification-view.d.ts +8 -0
- package/dist/controller-service.js +4 -1
- package/dist/controller-supervisor.d.ts +8 -0
- package/dist/controller-supervisor.js +33 -1
- package/dist/demo.js +3 -2
- package/dist/deploy-gate.js +5 -0
- package/dist/deploy-rescue.d.ts +41 -0
- package/dist/deploy-rescue.js +94 -0
- package/dist/desktop-update.d.ts +1 -1
- package/dist/dispose.d.ts +3 -0
- package/dist/dispose.js +9 -5
- package/dist/doctor.d.ts +7 -0
- package/dist/doctor.js +10 -2
- package/dist/envelope.js +2 -1
- package/dist/evidence.d.ts +9 -0
- package/dist/evidence.js +19 -4
- package/dist/exec.d.ts +9 -0
- package/dist/exec.js +22 -2
- package/dist/git-floor.d.ts +16 -0
- package/dist/git-floor.js +36 -0
- package/dist/grant.js +5 -1
- package/dist/guides.js +22 -11
- package/dist/latency-timeline.d.ts +46 -0
- package/dist/latency-timeline.js +120 -0
- package/dist/live-backup.d.ts +39 -0
- package/dist/live-backup.js +271 -0
- package/dist/operate.d.ts +13 -0
- package/dist/operate.js +969 -328
- package/dist/planner.js +2 -2
- package/dist/prepared-evidence.d.ts +2 -1
- package/dist/prepared-evidence.js +8 -3
- package/dist/project-knowledge.js +6 -2
- package/dist/project-learning.js +2 -2
- package/dist/proof.js +6 -0
- package/dist/read-executor-worker.js +8 -4
- package/dist/read-executor.d.ts +18 -1
- package/dist/read-executor.js +8 -0
- package/dist/release-adapters.d.ts +80 -2
- package/dist/release-adapters.js +441 -16
- package/dist/release-authority.js +6 -2
- package/dist/release-auto.d.ts +38 -7
- package/dist/release-auto.js +131 -15
- package/dist/release-card.d.ts +15 -3
- package/dist/release-card.js +25 -9
- package/dist/release-carry.d.ts +21 -1
- package/dist/release-carry.js +22 -0
- package/dist/release-ci-hold.d.ts +41 -0
- package/dist/release-ci-hold.js +53 -0
- package/dist/release-cli.d.ts +3 -0
- package/dist/release-cli.js +95 -17
- package/dist/release-coverage.js +5 -2
- package/dist/release-gate.d.ts +5 -0
- package/dist/release-gate.js +1 -1
- package/dist/release-recheck.d.ts +1 -1
- package/dist/release-recheck.js +8 -2
- package/dist/release.d.ts +144 -14
- package/dist/release.js +421 -81
- package/dist/review-context.js +8 -6
- package/dist/runner-process.d.ts +67 -0
- package/dist/runner-process.js +219 -0
- package/dist/runner.d.ts +48 -6
- package/dist/runner.js +73 -9
- package/dist/sandbox-runtime.d.ts +8 -0
- package/dist/sandbox-runtime.js +14 -2
- package/dist/scope.d.ts +19 -1
- package/dist/scope.js +26 -3
- package/dist/scout.js +2 -2
- package/dist/secret-mint.d.ts +1 -0
- package/dist/secret-mint.js +14 -0
- package/dist/serve.js +112 -51
- package/dist/server/chat.js +5 -5
- package/dist/server/http.d.ts +2 -0
- package/dist/server/pages.js +7 -3
- package/dist/server/remote-hooks.d.ts +1 -1
- package/dist/server/remote-hooks.js +5 -3
- package/dist/server/render-results.d.ts +2 -0
- package/dist/server/render-results.js +1 -0
- package/dist/server/render-tasks.d.ts +2 -0
- package/dist/server/render-tasks.js +2 -1
- package/dist/server/request-context.d.ts +8 -0
- package/dist/server/runtime.d.ts +2 -0
- package/dist/server/settings.js +22 -2
- package/dist/server/task-views.js +7 -2
- package/dist/server/tasks.js +4 -1
- package/dist/setup-guide.d.ts +2 -2
- package/dist/storage-sweep.js +1 -1
- package/dist/storage.d.ts +5 -0
- package/dist/storage.js +28 -1
- package/dist/store.d.ts +135 -2
- package/dist/store.js +275 -13
- package/dist/surface.js +3 -1
- package/dist/task-base.d.ts +4 -0
- package/dist/task-base.js +22 -2
- package/dist/teardown-git.d.ts +52 -3
- package/dist/teardown-git.js +145 -33
- package/dist/test-infra.js +1 -1
- package/dist/toolroll-update.js +2 -2
- package/dist/up-processes.d.ts +137 -0
- package/dist/up-processes.js +165 -0
- package/dist/up-role-child.d.ts +1 -0
- package/dist/up-role-child.js +81 -0
- package/dist/verification-evidence.d.ts +1 -0
- package/dist/verification-evidence.js +13 -1
- package/dist/workspace-revision.js +3 -1
- package/package.json +2 -1
package/dist/release.js
CHANGED
|
@@ -36,6 +36,7 @@ import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, re
|
|
|
36
36
|
import { randomUUID } from "node:crypto";
|
|
37
37
|
import { dirname, join } from "node:path";
|
|
38
38
|
import { personProof } from "./person-proof.js";
|
|
39
|
+
import { RELEASE_CI_RUNNING_WORDS as CI_RUNNING_WORDS } from "./release-ci-hold.js";
|
|
39
40
|
/** The pull-request checks a release waits for, by exact job name (.github/workflows/ci.yml). Every one must have
|
|
40
41
|
* concluded `success` on the gated commit; src/ci-workflow.test.ts holds the workflow to this list. */
|
|
41
42
|
export const REQUIRED_PR_CHECKS = [
|
|
@@ -44,16 +45,42 @@ export const REQUIRED_PR_CHECKS = [
|
|
|
44
45
|
"linux native containment · delegated cgroup v2",
|
|
45
46
|
"linux fence · sandbox-runtime",
|
|
46
47
|
"Windows Job Object containment",
|
|
48
|
+
// The git lanes (ci.yml job "git"): the declared floor (git-floor.ts) and Debian trixie's 2.47.
|
|
49
|
+
"git 2.41 · debian trixie",
|
|
50
|
+
"git 2.47 · debian trixie",
|
|
51
|
+
// The unit suite on macOS (ci.yml jobs "macos-shard" and "macos"; pipeline-speed.md, item 3a): tested on the candidate
|
|
52
|
+
// before it merges, no longer only on the tag after.
|
|
53
|
+
"node 22 · macos-latest",
|
|
54
|
+
"node 24 · macos-latest",
|
|
47
55
|
];
|
|
48
|
-
/** The
|
|
49
|
-
|
|
56
|
+
/** The workflow file every REQUIRED_PR_CHECKS job belongs to: a job of that name from any other workflow (a lookalike
|
|
57
|
+
* added beside it) never stands for it (Codex review round 1, inherited CI limitation). */
|
|
58
|
+
export const REQUIRED_PR_CHECKS_WORKFLOW = ".github/workflows/ci.yml";
|
|
59
|
+
/**
|
|
60
|
+
* The pull-request checks main's GitHub protection is not yet required to name (eval-harness review 2, finding 1): the
|
|
61
|
+
* git lanes join main's protection once they have run there, so a protection without them is reported, never refused.
|
|
62
|
+
* A later release empties this list; from then on main's protection must name every REQUIRED_PR_CHECKS check.
|
|
63
|
+
*/
|
|
64
|
+
export const PROTECTION_REPORT_ONLY_CHECKS = ["git 2.41 · debian trixie", "git 2.47 · debian trixie", "node 22 · macos-latest", "node 24 · macos-latest"];
|
|
65
|
+
/** The checks main's protection must require at minimum: every required check that isn't report-only. */
|
|
66
|
+
export const PROTECTION_MINIMUM_CHECKS = REQUIRED_PR_CHECKS.filter(name => !PROTECTION_REPORT_ONLY_CHECKS.includes(name));
|
|
67
|
+
/** The checks a release's publication waits for on its tag (.github/workflows/publish.yml, pipeline-speed.md item 3):
|
|
68
|
+
* the publish job and the macOS legs it needs, each named. Nothing else reported on the merge commit (main's own CI push
|
|
69
|
+
* run) holds publication. */
|
|
70
|
+
export const RELEASE_TAG_CHECKS = ["publish", "tag macOS · node 22", "tag macOS · node 24"];
|
|
71
|
+
export const RELEASE_TAG_WORKFLOW = ".github/workflows/publish.yml";
|
|
50
72
|
export const RELEASE_STEPS = ["gate", "approval", "check", "complete", "quiet", "deploy", "link", "ci", "merge", "tree", "tag", "publish", "homebrew", "done"];
|
|
51
73
|
/** How long each waiting step may take, in minutes, before the release stops there (rerunning continues from it). */
|
|
52
74
|
export const RELEASE_LIMITS = { gate: 60, approval: 24 * 60, check: 120, quiet: 60, deploy: 30, ci: 60, publish: 45, homebrew: 45 };
|
|
53
75
|
const POLL_MS = 15_000;
|
|
54
|
-
/**
|
|
55
|
-
*
|
|
56
|
-
|
|
76
|
+
/**
|
|
77
|
+
* Failure classes replace a flat flake allowance (docs/plans/eval-harness.md; check-failure.ts says the classes): only a
|
|
78
|
+
* failure the plane itself confirmed as infrastructure is ever run again under the same approval — at most once per
|
|
79
|
+
* incident and twice per candidate — for a release check and for a GitHub CI job alike. A failing test, a typecheck, an
|
|
80
|
+
* attack test, a migration: none tolerated. An unexplained failure blocks, and a person decides.
|
|
81
|
+
*/
|
|
82
|
+
export const INFRA_RERUNS_PER_INCIDENT = 1;
|
|
83
|
+
export const INFRA_RERUNS_PER_CANDIDATE = 2;
|
|
57
84
|
/** A stop the release reports: the step it stopped at and why, in words. */
|
|
58
85
|
class Stop extends Error {
|
|
59
86
|
reason;
|
|
@@ -65,6 +92,9 @@ class Stop extends Error {
|
|
|
65
92
|
this.build = build;
|
|
66
93
|
}
|
|
67
94
|
}
|
|
95
|
+
/** The tap change's checks failed: Homebrew's own state, never a stop of a published release. */
|
|
96
|
+
class HomebrewFailed extends Error {
|
|
97
|
+
}
|
|
68
98
|
/** The branch moved before the deploy reservation: carry the approval to the new head, or file it for a person. */
|
|
69
99
|
class Restart extends Error {
|
|
70
100
|
head;
|
|
@@ -160,6 +190,18 @@ export function checksVerdict(runs, required) {
|
|
|
160
190
|
return { state: "failed", failed };
|
|
161
191
|
return waiting.length > 0 ? { state: "pending", waiting } : { state: "passed" };
|
|
162
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* Publication's verdict on the tag commit (pipeline-speed.md, item 3): only RELEASE_TAG_CHECKS, from GitHub Actions
|
|
195
|
+
* runs of publish.yml bound to the tag's push, each by its newest attempt. Every other check on the merge commit — main's
|
|
196
|
+
* own CI run — is not waited for: the candidate's checks already passed on its exact head before the merge, and the
|
|
197
|
+
* merged tree was proved equal to it.
|
|
198
|
+
*/
|
|
199
|
+
export function tagChecksVerdict(runs) {
|
|
200
|
+
const names = RELEASE_TAG_CHECKS;
|
|
201
|
+
// Only GitHub Actions runs of publish.yml bound to the tag's own push (release-adapters.ts readCheckRuns): a check of
|
|
202
|
+
// the same name from another workflow, an unbound run or another app is never counted (security scan).
|
|
203
|
+
return checksVerdict(runs.filter(run => names.includes(run.name) && run.app === "github-actions" && workflowIs(run.workflowPath, RELEASE_TAG_WORKFLOW)), RELEASE_TAG_CHECKS);
|
|
204
|
+
}
|
|
163
205
|
/** Run the release from wherever its journal stopped. */
|
|
164
206
|
export async function runRelease(options, adapters) {
|
|
165
207
|
const file = journalFile(options.stateDir, options.branch);
|
|
@@ -234,6 +276,8 @@ export async function runRelease(options, adapters) {
|
|
|
234
276
|
say(`Released ${journal.tag} (${journal.candidate.sha.slice(0, 7)}) — nothing left to do.`);
|
|
235
277
|
return { ok: true, journal, file };
|
|
236
278
|
}
|
|
279
|
+
// Published, with Homebrew still to finish: this run finishes it, waiting for the tap's checks within its limit.
|
|
280
|
+
const finishingHomebrew = journal !== null && journal.step === "homebrew" && journal.published !== null && options.fresh !== true;
|
|
237
281
|
if (!await adapters.github.authenticated())
|
|
238
282
|
throw new Stop("github-auth", "GitHub CLI is not signed in: run `gh auth login`, then rerun.");
|
|
239
283
|
if (journal === null) {
|
|
@@ -261,6 +305,9 @@ export async function runRelease(options, adapters) {
|
|
|
261
305
|
else
|
|
262
306
|
say(`Resuming the release of ${journal.tag} (${journal.candidate.sha.slice(0, 7)}) at ${journal.step}.`);
|
|
263
307
|
const j = journal;
|
|
308
|
+
// A journal an older coordinator filed ahead of CI: its deploy reads CI fresh too.
|
|
309
|
+
if (j.ciAhead === true && j.ciBeforeDeploy !== true)
|
|
310
|
+
j.ciBeforeDeploy = true;
|
|
264
311
|
if (j.checkout !== options.checkout)
|
|
265
312
|
throw new Stop("checkout", `This release runs in ${j.checkout}; rerun with --repo ${j.checkout}.`);
|
|
266
313
|
save();
|
|
@@ -274,6 +321,89 @@ export async function runRelease(options, adapters) {
|
|
|
274
321
|
throw new Stop("drift", `Pull request #${j.pr} no longer has ${j.candidate.sha.slice(0, 7)} at its head.`);
|
|
275
322
|
return pr;
|
|
276
323
|
};
|
|
324
|
+
// ---- Homebrew, finished apart from publication (pipeline-speed.md, P1) ------------------------------------------
|
|
325
|
+
let homebrewLeft = null;
|
|
326
|
+
/**
|
|
327
|
+
* One look at the tap: propose the formula change when it's missing, judge its checks (once, or within the Homebrew
|
|
328
|
+
* limit when `wait`), and merge exactly the proposed head once every check passed. A tap that already names a newer
|
|
329
|
+
* version is never overwritten, checked again right before the merge. Checksums come from npm's own tarball
|
|
330
|
+
* (release-adapters.ts propose), unchanged.
|
|
331
|
+
*/
|
|
332
|
+
const homebrewPass = async (wait) => {
|
|
333
|
+
const brew = adapters.homebrew;
|
|
334
|
+
let state = await brew.state(j.packageVersion);
|
|
335
|
+
if (state.kind === "newer")
|
|
336
|
+
return { state: "superseded", why: `the tap already names ${state.version}` };
|
|
337
|
+
if (state.kind === "current")
|
|
338
|
+
return { state: "current", why: null };
|
|
339
|
+
// A change this release already proposed is the only one it ever merges (security scan): another pull request for
|
|
340
|
+
// the same version, a moved head or a closed change is failed, never adopted or merged.
|
|
341
|
+
const journaled = j.homebrew?.pr != null && j.homebrew.head != null ? { pr: j.homebrew.pr, head: j.homebrew.head } : null;
|
|
342
|
+
if (journaled !== null && state.kind === "missing")
|
|
343
|
+
return { state: "failed", why: `${brew.repo} #${journaled.pr} was closed before it merged` };
|
|
344
|
+
if (journaled !== null && state.kind === "proposed" && (state.pr !== journaled.pr || state.head !== journaled.head)) {
|
|
345
|
+
return { state: "failed", why: `${brew.repo}'s change for ${j.packageVersion} is now #${state.pr} at ${state.head.slice(0, 7)}, not the #${journaled.pr} at ${journaled.head.slice(0, 7)} this release proposed` };
|
|
346
|
+
}
|
|
347
|
+
if (state.kind === "missing") {
|
|
348
|
+
intend("homebrew");
|
|
349
|
+
const proposed = await brew.propose(j.packageVersion);
|
|
350
|
+
state = { kind: "proposed", ...proposed };
|
|
351
|
+
}
|
|
352
|
+
const proposed = state;
|
|
353
|
+
j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false, state: "pending", why: null };
|
|
354
|
+
save();
|
|
355
|
+
// Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
|
|
356
|
+
// on a change, and only when it's still empty a poll later.
|
|
357
|
+
const checked = await brew.checked(proposed.head);
|
|
358
|
+
let emptyLooks = 0;
|
|
359
|
+
const probe = async () => {
|
|
360
|
+
// Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
|
|
361
|
+
// Two apps may name a check alike; each app's attempts stand for themselves.
|
|
362
|
+
const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
|
|
363
|
+
const statuses = await brew.statuses(proposed.head);
|
|
364
|
+
const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
|
|
365
|
+
const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
|
|
366
|
+
if (failed.length > 0)
|
|
367
|
+
throw new HomebrewFailed(`the tap change's checks failed: ${failed.join(", ")}`);
|
|
368
|
+
if (verdicts.some(one => one.state === "pending"))
|
|
369
|
+
return null;
|
|
370
|
+
// GitHub registers checks asynchronously; an empty set says nothing about success.
|
|
371
|
+
// A workflow is owed: only its own run counts, not another app's green check.
|
|
372
|
+
if (checked && !runs.some(run => run.app === "github-actions"))
|
|
373
|
+
return null;
|
|
374
|
+
if (runs.length === 0 && statuses.length === 0)
|
|
375
|
+
return ++emptyLooks >= 2 ? true : null;
|
|
376
|
+
return true;
|
|
377
|
+
};
|
|
378
|
+
try {
|
|
379
|
+
const passed = wait ? await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, probe) : await probe();
|
|
380
|
+
if (passed === null)
|
|
381
|
+
return { state: "pending", why: `${brew.repo} #${proposed.pr}'s checks are still running` };
|
|
382
|
+
}
|
|
383
|
+
catch (error) {
|
|
384
|
+
if (error instanceof HomebrewFailed)
|
|
385
|
+
return { state: "failed", why: `${brew.repo} #${proposed.pr}: ${error.message}` };
|
|
386
|
+
// Waiting ran out: still pending, never a stop of a published release.
|
|
387
|
+
if (error instanceof Stop && error.reason === "timeout")
|
|
388
|
+
return { state: "pending", why: error.message };
|
|
389
|
+
throw error;
|
|
390
|
+
}
|
|
391
|
+
// Ordering, once more right before the merge: a newer formula landed meanwhile is never overwritten.
|
|
392
|
+
const now = await brew.state(j.packageVersion);
|
|
393
|
+
if (now.kind === "newer")
|
|
394
|
+
return { state: "superseded", why: `the tap already names ${now.version}` };
|
|
395
|
+
if (now.kind === "missing")
|
|
396
|
+
return { state: "failed", why: `${brew.repo} #${proposed.pr} was closed before it merged` };
|
|
397
|
+
if (now.kind === "proposed" && now.head !== proposed.head)
|
|
398
|
+
return { state: "failed", why: `${brew.repo} #${proposed.pr} moved to ${now.head.slice(0, 7)} after its checks were read` };
|
|
399
|
+
if (now.kind === "current")
|
|
400
|
+
return { state: "current", why: null };
|
|
401
|
+
const verified = await brew.verified(proposed.pr, proposed.head, j.packageVersion);
|
|
402
|
+
if (!verified.ok)
|
|
403
|
+
return { state: "failed", why: verified.why };
|
|
404
|
+
await brew.merge(proposed.pr, proposed.head);
|
|
405
|
+
return { state: "current", why: null };
|
|
406
|
+
};
|
|
277
407
|
// ---- one approval per release (release-once.md) -------------------------------------------------------------
|
|
278
408
|
const rel = adapters.release;
|
|
279
409
|
let epoch = null;
|
|
@@ -387,12 +517,14 @@ export async function runRelease(options, adapters) {
|
|
|
387
517
|
? `${what}. Nothing was merged. The deploy had already started, so this commit needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`
|
|
388
518
|
: `${what}. Nothing was deployed or merged. This commit needs a person's approval: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
389
519
|
};
|
|
390
|
-
/** An automatic approval's CI allowance:
|
|
391
|
-
|
|
520
|
+
/** An automatic approval's CI allowance: only attempts GitHub shows failing before the tests ran, at most
|
|
521
|
+
* INFRA_RERUNS_PER_CANDIDATE on the commit; a person's approval judges each job by its latest attempt. */
|
|
522
|
+
const autoCap = () => (j.auto?.state === "approved" ? { auto: true } : {});
|
|
392
523
|
/**
|
|
393
524
|
* An automatically approved release stands only while its CI does (auto-approve-green.md): the complete, current
|
|
394
525
|
* checks on the exact commit are read again right before each step that can't be taken back — the deploy, the CLI
|
|
395
|
-
* relink, the merge and the tag. Anything but green (failed, a job re-run and pending,
|
|
526
|
+
* relink, the merge and the tag. Anything but green (failed, a job re-run and pending, an earlier failure GitHub
|
|
527
|
+
* doesn't show as the runner's own) stops the
|
|
396
528
|
* release, revokes its approval and tells the owner; the commit then needs a person.
|
|
397
529
|
*/
|
|
398
530
|
const autoCiHolds = async (before) => {
|
|
@@ -400,9 +532,11 @@ export async function runRelease(options, adapters) {
|
|
|
400
532
|
return;
|
|
401
533
|
const read = await adapters.github.checks(j.github, j.candidate.sha);
|
|
402
534
|
const whole = adapters.github.complete === undefined ? false : await adapters.github.complete(j.github, j.candidate.sha);
|
|
403
|
-
const verdict = whole ? prChecksVerdict(read, j.candidate.sha, REQUIRED_PR_CHECKS, {
|
|
404
|
-
if (verdict.state === "passed")
|
|
535
|
+
const verdict = whole ? prChecksVerdict(read, j.candidate.sha, REQUIRED_PR_CHECKS, { auto: true, workflow: REQUIRED_PR_CHECKS_WORKFLOW }) : { state: "failed", failed: ["GitHub's check history of this commit can't be read whole"] };
|
|
536
|
+
if (verdict.state === "passed") {
|
|
537
|
+
await liftCiHold(false, `The release cannot continue before ${before}`);
|
|
405
538
|
return;
|
|
539
|
+
}
|
|
406
540
|
if (before === "the tag") {
|
|
407
541
|
// Already merged: nothing to revoke. It isn't tagged or published, and a person decides — durably, in the plane.
|
|
408
542
|
const why = `Pull request checks on ${j.candidate.sha.slice(0, 7)} stopped passing after the merge`;
|
|
@@ -472,6 +606,65 @@ export async function runRelease(options, adapters) {
|
|
|
472
606
|
await rel?.auto?.giveUp(j.task.id, why);
|
|
473
607
|
throw new Stop("approval", `Not approved automatically: ${why}. A person approves task ${j.task.id} (\`toolroll task approve ${j.task.id} --digest ${j.task.scopeDigest}\`, or the console), then rerun \`${resume}\`.`);
|
|
474
608
|
};
|
|
609
|
+
/**
|
|
610
|
+
* CI passed on the candidate as the coordinator read it: the plane reads it itself and lifts the approval hold it
|
|
611
|
+
* placed when the task was filed ahead of CI (round 1, finding 4). A refusal stops here; nobody was asked.
|
|
612
|
+
*/
|
|
613
|
+
const liftCiHold = async (ask, nothing) => {
|
|
614
|
+
const lift = rel?.auto?.ciPassed;
|
|
615
|
+
if (lift === undefined)
|
|
616
|
+
return;
|
|
617
|
+
const lifted = await lift(j.task.id, j.candidate.sha, ask);
|
|
618
|
+
if (!lifted.ok)
|
|
619
|
+
throw new Stop("ci-failed", `The plane didn't confirm CI passed on ${j.candidate.sha.slice(0, 7)}: ${lifted.why}. ${nothing}. Rerun \`${resume}\` once every pull request check passes.`);
|
|
620
|
+
};
|
|
621
|
+
/**
|
|
622
|
+
* One look at the pull request's required checks on the exact commit while CI may still run: a failed check stops
|
|
623
|
+
* (`nothing` says what hasn't happened), passed ends the plane's "CI still running" note, pending (or GitHub not
|
|
624
|
+
* answering) answers false. Judged as the ci step judges a person's release (prChecksVerdict, ci.yml, exact head).
|
|
625
|
+
*/
|
|
626
|
+
const ciWhileRunning = async (nothing, ask) => {
|
|
627
|
+
let runs;
|
|
628
|
+
try {
|
|
629
|
+
runs = await adapters.github.checks(j.github, j.candidate.sha);
|
|
630
|
+
}
|
|
631
|
+
catch (error) {
|
|
632
|
+
if (transientNetworkError(error))
|
|
633
|
+
return false;
|
|
634
|
+
throw error;
|
|
635
|
+
}
|
|
636
|
+
const verdict = prChecksVerdict(runs, j.candidate.sha, REQUIRED_PR_CHECKS, { workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
637
|
+
if (verdict.state === "failed") {
|
|
638
|
+
throw new Stop("ci-failed", `GitHub's checks failed on ${j.candidate.sha.slice(0, 7)}: ${verdict.failed.join(", ")}. ${nothing}. ${j.approval === null ? "" : `The approval covers ${j.candidate.sha.slice(0, 7)} only. `}Rerun the failed job on GitHub and run \`${resume}\` again, or fix the branch and release the new commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
639
|
+
}
|
|
640
|
+
if (verdict.state !== "passed")
|
|
641
|
+
return false;
|
|
642
|
+
// The plane's note can survive a crash before the journal saved ciAhead. Clear it on the plane's own reading,
|
|
643
|
+
// even on an older/resumed journal whose flags say nothing about CI.
|
|
644
|
+
await liftCiHold(ask, nothing);
|
|
645
|
+
if (j.ciAhead === true) {
|
|
646
|
+
j.ciAhead = false;
|
|
647
|
+
save();
|
|
648
|
+
}
|
|
649
|
+
return true;
|
|
650
|
+
};
|
|
651
|
+
/** Every person's release reads exact-head CI at deployment, even if it was green at the gate or an older
|
|
652
|
+
* journal has no ciBeforeDeploy flag. An approval and the release check can outlive that earlier green reading. */
|
|
653
|
+
const personCiBeforeDeploy = async () => {
|
|
654
|
+
if (j.auto?.state === "approved")
|
|
655
|
+
return;
|
|
656
|
+
let said = false;
|
|
657
|
+
await waitFor("ci", `the pull request checks on ${j.candidate.sha.slice(0, 7)} before the deploy`, async () => {
|
|
658
|
+
await watch("deploy");
|
|
659
|
+
if (await ciWhileRunning("Nothing was deployed, merged, tagged or published", false))
|
|
660
|
+
return true;
|
|
661
|
+
if (!said) {
|
|
662
|
+
say(` ${CI_RUNNING_WORDS}`);
|
|
663
|
+
said = true;
|
|
664
|
+
}
|
|
665
|
+
return null;
|
|
666
|
+
});
|
|
667
|
+
};
|
|
475
668
|
/**
|
|
476
669
|
* The task is approved for `action`: a release task (a carried commit, or any task once the release has an
|
|
477
670
|
* authority) only when the live-release predicate applies to it AND accepts it — fail closed, whatever its own seal
|
|
@@ -501,7 +694,7 @@ export async function runRelease(options, adapters) {
|
|
|
501
694
|
if (gate.result?.level !== "full")
|
|
502
695
|
throw new Stop("check-level", `A Full check is required for this release (run ${gate.result?.run ?? "unknown"}: ${gate.result?.level ?? "unknown"}). Quick and Off checks cannot release a build.`);
|
|
503
696
|
if (gate.result.check !== "passed")
|
|
504
|
-
throw new Stop("check-failed",
|
|
697
|
+
throw new Stop("check-failed", checkFailedWords({ sha: j.candidate.sha, run: gate.result.run, class: null, why: `its check reads ${gate.result.check}`, branch: j.branch, checkout: j.checkout, taskId: j.task.id }));
|
|
505
698
|
};
|
|
506
699
|
/** The commit the passing build recorded: what the deployed runtime is stamped with. */
|
|
507
700
|
const builtHead = () => j.check?.head ?? j.candidate.sha;
|
|
@@ -515,7 +708,7 @@ export async function runRelease(options, adapters) {
|
|
|
515
708
|
throw new Stop("ci-failed", `A pull request check failed after the deploy, so ${j.candidate.sha.slice(0, 7)} needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
516
709
|
// A resumed release that already has an authority takes its lease and settles every open operation first.
|
|
517
710
|
await hold();
|
|
518
|
-
while (j.step !== "done") {
|
|
711
|
+
while (j.step !== "done" && homebrewLeft === null) {
|
|
519
712
|
step = j.step;
|
|
520
713
|
say(`• ${j.step}`);
|
|
521
714
|
try {
|
|
@@ -538,6 +731,11 @@ export async function runRelease(options, adapters) {
|
|
|
538
731
|
}
|
|
539
732
|
}
|
|
540
733
|
}
|
|
734
|
+
// The tree CI tested (the pull request's head, not GitHub's merge commit) is the tree that merges only when the
|
|
735
|
+
// branch already contains main's head: refused here, before anything is filed, and again right before merging.
|
|
736
|
+
await containsMain(adapters, j, "Nothing was filed or sent for approval");
|
|
737
|
+
if (j.intents.gate === undefined)
|
|
738
|
+
await mainProtection(adapters, j, false, "Nothing was filed or sent for approval");
|
|
541
739
|
const existing = await adapters.toolroll.gate(j.task.id);
|
|
542
740
|
// A fresh task this journal never filed has appeared under its id: it isn't ours, and isn't a fresh approval.
|
|
543
741
|
if (existing.exists && j.freshTask === true && j.intents.gate === undefined)
|
|
@@ -549,7 +747,20 @@ export async function runRelease(options, adapters) {
|
|
|
549
747
|
j.task.scopeDigest = existing.scopeDigest;
|
|
550
748
|
}
|
|
551
749
|
else {
|
|
552
|
-
|
|
750
|
+
// The gate only stops on a check that already failed: the task is filed and a person asked while CI runs, and
|
|
751
|
+
// the release check runs beside it (pipeline-speed.md, D2 and "Decision: a person's release asks during
|
|
752
|
+
// CI"). The automatic approval waits for CI (the check step); nothing deploys before it passes (the deploy).
|
|
753
|
+
const settled = await earlyChecks(adapters, j, (what, probe) => waitFor("gate", what, probe), { pendingIsEnough: true });
|
|
754
|
+
// Recorded before the task exists, so its approval card says CI still runs from the first moment.
|
|
755
|
+
if (!settled)
|
|
756
|
+
await rel?.auto?.ciRunning?.(j.task.id, j.candidate.sha);
|
|
757
|
+
j.ciAhead = !settled;
|
|
758
|
+
if (!settled)
|
|
759
|
+
j.ciBeforeDeploy = true;
|
|
760
|
+
save();
|
|
761
|
+
if (!settled) {
|
|
762
|
+
say(` Pull request checks on ${j.candidate.sha.slice(0, 7)} are still running: the release check runs beside them, and nothing deploys before they pass`);
|
|
763
|
+
}
|
|
553
764
|
intend("gate");
|
|
554
765
|
await adapters.repo.sync(j.branch);
|
|
555
766
|
const filed = await adapters.toolroll.fileGate({
|
|
@@ -587,9 +798,19 @@ export async function runRelease(options, adapters) {
|
|
|
587
798
|
say(` Not approved automatically: ${authorized.why}. A person approves it.`);
|
|
588
799
|
}
|
|
589
800
|
}
|
|
801
|
+
// Filed while CI still runs: the person is asked now (Alex, 2026-10-29), and told nothing deploys until CI passes.
|
|
802
|
+
// A check that already failed asks nobody (as at the gate).
|
|
803
|
+
if (j.ciAhead === true)
|
|
804
|
+
await ciWhileRunning("Nothing was deployed or merged", true);
|
|
590
805
|
say(` The release scope needs your approval: \`toolroll task approve ${j.task.id} --digest ${j.task.scopeDigest}\`, or open the task in the console.`);
|
|
806
|
+
if (j.ciAhead === true)
|
|
807
|
+
say(` ${CI_RUNNING_WORDS}`);
|
|
591
808
|
const approved = await waitFor("approval", `a person to approve task ${j.task.id}`, async () => {
|
|
592
809
|
await watch("dispatch");
|
|
810
|
+
// While asking: a check that failed stops here (nobody approves a red commit), and CI passing ends the card's
|
|
811
|
+
// "still running" note (and lifts a hold an older coordinator placed).
|
|
812
|
+
if (j.ciAhead === true)
|
|
813
|
+
await ciWhileRunning("Nothing was deployed or merged", true);
|
|
593
814
|
const gate = await adapters.toolroll.gate(j.task.id);
|
|
594
815
|
if (gate.scopeDigest !== j.task.scopeDigest)
|
|
595
816
|
throw new Stop("scope-changed", `The scope of ${j.task.id} changed after it was filed; the release only runs on the scope it filed.`);
|
|
@@ -623,6 +844,18 @@ export async function runRelease(options, adapters) {
|
|
|
623
844
|
// A rerun of the same approved commit is on its way: the failure before it is not this release's answer.
|
|
624
845
|
if (gate.pending === true && gate.result?.check !== "passed")
|
|
625
846
|
return null;
|
|
847
|
+
// A failed Full check: the plane says why (its own records only) and reruns it under the same approval only
|
|
848
|
+
// for confirmed infrastructure; every attempt stays in the evidence. Anything else stops, saying its class.
|
|
849
|
+
if (gate.failed === null && gate.result !== null && gate.result.level === "full" && gate.result.check === "failed" && rel?.recheck !== undefined) {
|
|
850
|
+
const answer = await rel.recheck(j.task.id);
|
|
851
|
+
if (answer.action === "pending")
|
|
852
|
+
return null;
|
|
853
|
+
if (answer.action === "rerun") {
|
|
854
|
+
say(` run ${gate.result.run} failed: ${answer.why}. Toolroll confirmed it as its own infrastructure, so the check runs again on ${j.candidate.sha.slice(0, 7)} under the same approval`);
|
|
855
|
+
return null;
|
|
856
|
+
}
|
|
857
|
+
throw new Stop("check-failed", checkFailedWords({ sha: j.candidate.sha, run: gate.result.run, class: answer.class, why: answer.why, branch: j.branch, checkout: j.checkout, taskId: j.task.id }));
|
|
858
|
+
}
|
|
626
859
|
if (gate.failed !== null)
|
|
627
860
|
throw new Stop("check-failed", `The release build of ${j.task.id} failed: ${gate.failed}. Fix it and release the new commit with --new.`, true);
|
|
628
861
|
if (gate.result === null || gate.result.check === "running" || gate.result.check === "pending")
|
|
@@ -636,6 +869,17 @@ export async function runRelease(options, adapters) {
|
|
|
636
869
|
fullCheck(gate);
|
|
637
870
|
if (gate.result.worktree === null)
|
|
638
871
|
throw new Stop("worktree", `Run ${gate.result.run} names no checkout to deploy from.`);
|
|
872
|
+
// A pass stands only when no earlier attempt on this commit failed for a reason other than confirmed
|
|
873
|
+
// infrastructure the release reran; what passed only on a retry is said (and an automatic approval refuses it).
|
|
874
|
+
if (rel?.standing !== undefined) {
|
|
875
|
+
const standing = await rel.standing(j.task.id, gate.result.run);
|
|
876
|
+
if (!standing.ok)
|
|
877
|
+
throw new Stop("check-failed", `Run ${gate.result.run} passed on ${j.candidate.sha.slice(0, 7)}, but ${standing.why}: approve this commit again with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`, or fix the branch and release the new commit the same way.`);
|
|
878
|
+
if (standing.retried === null)
|
|
879
|
+
say(` run ${gate.result.run} passed, but its report can't be read, so whether anything passed only on a retry is unknown`);
|
|
880
|
+
else if (standing.retried.length > 0)
|
|
881
|
+
say(` run ${gate.result.run} passed, but only on a retry: ${standing.retried.join(", ")}`);
|
|
882
|
+
}
|
|
639
883
|
return { run: gate.result.run, receipt: gate.result.receipt, worktree: gate.result.worktree, head };
|
|
640
884
|
};
|
|
641
885
|
const result = await waitFor("check", `the release check of ${j.candidate.sha.slice(0, 7)}`, async () => {
|
|
@@ -655,6 +899,26 @@ export async function runRelease(options, adapters) {
|
|
|
655
899
|
j.check = result;
|
|
656
900
|
say(` run ${result.run} passed on ${j.candidate.sha.slice(0, 7)}`);
|
|
657
901
|
if (j.auto?.state === "checking" && !j.authority) {
|
|
902
|
+
// The check ran beside CI (the gate didn't wait): it waits for every pull request check now. A failure ends the
|
|
903
|
+
// automatic road, and the person's road refuses it too (the approval step waits for CI first).
|
|
904
|
+
if (j.ciAhead === true) {
|
|
905
|
+
try {
|
|
906
|
+
await earlyChecks(adapters, j, (what, probe) => waitFor("gate", what, probe), { nothing: "Nothing was approved" });
|
|
907
|
+
await liftCiHold(false, "Nothing was approved");
|
|
908
|
+
}
|
|
909
|
+
catch (error) {
|
|
910
|
+
if (error instanceof Stop && error.reason === "ci-failed") {
|
|
911
|
+
j.auto = { state: "refused", why: error.message };
|
|
912
|
+
j.check = null;
|
|
913
|
+
j.step = "approval";
|
|
914
|
+
save();
|
|
915
|
+
await rel?.auto?.giveUp(j.task.id, error.message.replace(/\.$/, ""));
|
|
916
|
+
}
|
|
917
|
+
throw error;
|
|
918
|
+
}
|
|
919
|
+
j.ciAhead = false;
|
|
920
|
+
save();
|
|
921
|
+
}
|
|
658
922
|
// The release approval, sealed only now: the plane reads CI on the exact commit itself and proves the check.
|
|
659
923
|
const sealed = rel?.auto === undefined ? { ok: false, why: "this plane can't approve releases automatically" }
|
|
660
924
|
: await rel.auto.approve({ taskId: j.task.id, digest: j.task.scopeDigest, candidate: j.candidate.sha, run: result.run, receipt: result.receipt });
|
|
@@ -714,6 +978,7 @@ export async function runRelease(options, adapters) {
|
|
|
714
978
|
if (runtime !== null && rel !== undefined && j.authority && epoch !== null) {
|
|
715
979
|
const authority = await rel.authority(j.task.id);
|
|
716
980
|
if (authority !== null && authority.state === "open") {
|
|
981
|
+
await personCiBeforeDeploy();
|
|
717
982
|
await watch("deploy");
|
|
718
983
|
// Every deploy reservation, whatever the journal saved: CI must still be green on the exact commit.
|
|
719
984
|
await autoCiHolds("the deploy");
|
|
@@ -727,6 +992,9 @@ export async function runRelease(options, adapters) {
|
|
|
727
992
|
}
|
|
728
993
|
if (runtime === null) {
|
|
729
994
|
// The last moment a moved head restarts; then the deploy reservation freezes the candidate in the store.
|
|
995
|
+
// A person's release: every required check passed on the exact commit, read now (the approval may have come
|
|
996
|
+
// while CI ran). A check that failed stops here, before anything is deployed.
|
|
997
|
+
await personCiBeforeDeploy();
|
|
730
998
|
await watch("deploy");
|
|
731
999
|
await pinned();
|
|
732
1000
|
// Every deploy issued, whatever the journal saved (a retry after a rollback included): CI still green.
|
|
@@ -793,7 +1061,7 @@ export async function runRelease(options, adapters) {
|
|
|
793
1061
|
case "ci": {
|
|
794
1062
|
await waitFor("ci", `the pull request checks on ${j.candidate.sha.slice(0, 7)}`, async () => {
|
|
795
1063
|
await pinned();
|
|
796
|
-
const verdict = prChecksVerdict(await adapters.github.checks(j.github, j.candidate.sha), j.candidate.sha, REQUIRED_PR_CHECKS, autoCap());
|
|
1064
|
+
const verdict = prChecksVerdict(await adapters.github.checks(j.github, j.candidate.sha), j.candidate.sha, REQUIRED_PR_CHECKS, { ...autoCap(), workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
797
1065
|
if (verdict.state === "failed")
|
|
798
1066
|
await ciFailedAfterDeploy(`Pull request checks failed on ${j.candidate.sha.slice(0, 7)}: ${verdict.failed.join(", ")}`);
|
|
799
1067
|
return verdict.state === "passed" ? true : null;
|
|
@@ -816,15 +1084,19 @@ export async function runRelease(options, adapters) {
|
|
|
816
1084
|
if (rel !== undefined && j.authority)
|
|
817
1085
|
await identity("merge", false);
|
|
818
1086
|
await pinned();
|
|
819
|
-
// A squash merge only reproduces the gated tree when main hasn't moved past the branch's base.
|
|
820
|
-
await adapters
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
throw new Stop("base-moved", `main moved to ${main.slice(0, 7)}, which ${j.branch} doesn't contain, so the merge wouldn't be the gated tree. Nothing was merged: update the branch and release the new commit with --new.`);
|
|
1087
|
+
// A squash merge only reproduces the gated (and CI-tested) tree when main hasn't moved past the branch's base.
|
|
1088
|
+
const main = await containsMain(adapters, j, "Nothing was merged");
|
|
1089
|
+
// Every release merges only where GitHub itself refuses a branch behind main.
|
|
1090
|
+
await mainProtection(adapters, j, j.auto?.state === "approved", "Nothing was merged");
|
|
824
1091
|
// The checks again, at the moment of merging: a rerun that failed since must not merge.
|
|
825
|
-
const verdict = prChecksVerdict(await adapters.github.checks(j.github, j.candidate.sha), j.candidate.sha, REQUIRED_PR_CHECKS, autoCap());
|
|
1092
|
+
const verdict = prChecksVerdict(await adapters.github.checks(j.github, j.candidate.sha), j.candidate.sha, REQUIRED_PR_CHECKS, { ...autoCap(), workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
826
1093
|
if (verdict.state !== "passed")
|
|
827
1094
|
await ciFailedAfterDeploy(`Pull request checks are no longer all passing on ${j.candidate.sha.slice(0, 7)}`);
|
|
1095
|
+
// Main once more, from GitHub, the last thing before the merge is reserved: moved since the comparison, it
|
|
1096
|
+
// stops here (finding 4). GitHub's own up-to-date rule refuses the rest of the window.
|
|
1097
|
+
const now = await adapters.github.mainHead(j.github);
|
|
1098
|
+
if (now !== main)
|
|
1099
|
+
throw new Stop("base-moved", `The branch is behind main; update it and push. Main moved to ${now.slice(0, 7)} while ${j.branch} was being merged, so what merges wouldn't be the tree CI tested. Nothing was merged: release the updated commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
828
1100
|
let operation;
|
|
829
1101
|
if (rel !== undefined && j.authority && epoch !== null) {
|
|
830
1102
|
const reserved = await rel.reserve({ authority: j.authority.id, kind: "merge", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
@@ -890,7 +1162,7 @@ export async function runRelease(options, adapters) {
|
|
|
890
1162
|
await waitFor("publish", `npm, the GitHub release and the tag's checks for ${j.tag}`, async () => {
|
|
891
1163
|
if (await adapters.github.tag(j.github, j.tag) !== j.tagged.sha)
|
|
892
1164
|
throw new Stop("conflict", `${j.tag} no longer points at the verified merge commit. Homebrew was not updated.`);
|
|
893
|
-
const verdict =
|
|
1165
|
+
const verdict = tagChecksVerdict(await adapters.github.checks(j.github, j.tagged.sha, { tag: j.tag }));
|
|
894
1166
|
if (verdict.state === "failed")
|
|
895
1167
|
throw new Stop("publish-failed", `The release tag's checks failed: ${verdict.failed.join(", ")}. Homebrew was not updated.`);
|
|
896
1168
|
if (verdict.state === "pending")
|
|
@@ -903,43 +1175,16 @@ export async function runRelease(options, adapters) {
|
|
|
903
1175
|
}
|
|
904
1176
|
case "homebrew": {
|
|
905
1177
|
await publishHold();
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false };
|
|
916
|
-
save();
|
|
917
|
-
// Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
|
|
918
|
-
// on a change, and only when it's still empty a poll later.
|
|
919
|
-
const checked = await brew.checked(proposed.head);
|
|
920
|
-
let emptyLooks = 0;
|
|
921
|
-
await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, async () => {
|
|
922
|
-
// Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
|
|
923
|
-
// Two apps may name a check alike; each app's attempts stand for themselves.
|
|
924
|
-
const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
|
|
925
|
-
const statuses = await brew.statuses(proposed.head);
|
|
926
|
-
const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
|
|
927
|
-
const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
|
|
928
|
-
if (failed.length > 0)
|
|
929
|
-
throw new Stop("homebrew-failed", `The Homebrew change's checks failed: ${failed.join(", ")}.`);
|
|
930
|
-
if (verdicts.some(one => one.state === "pending"))
|
|
931
|
-
return null;
|
|
932
|
-
// GitHub registers checks asynchronously; an empty set says nothing about success.
|
|
933
|
-
// A workflow is owed: only its own run counts, not another app's green check.
|
|
934
|
-
if (checked && !runs.some(run => run.app === "github-actions"))
|
|
935
|
-
return null;
|
|
936
|
-
if (runs.length === 0 && statuses.length === 0)
|
|
937
|
-
return ++emptyLooks >= 2 ? true : null;
|
|
938
|
-
return true;
|
|
939
|
-
});
|
|
940
|
-
await brew.merge(proposed.pr, proposed.head);
|
|
1178
|
+
// Homebrew is finished apart from publication (pipeline-speed.md, P1): npm and the GitHub release are verified,
|
|
1179
|
+
// so the release is published. This run proposes the tap's change and reads its checks once; only a run that
|
|
1180
|
+
// resumes to finish Homebrew waits for them. Pending or failed, the state is kept and the run ends published.
|
|
1181
|
+
const progress = await homebrewPass(finishingHomebrew);
|
|
1182
|
+
j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: progress.state === "current" && j.homebrew?.head != null ? true : j.homebrew?.merged ?? false, state: progress.state, why: progress.why };
|
|
1183
|
+
save();
|
|
1184
|
+
if (progress.state === "pending" || progress.state === "failed") {
|
|
1185
|
+
homebrewLeft = { state: progress.state, why: progress.why };
|
|
1186
|
+
break;
|
|
941
1187
|
}
|
|
942
|
-
j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: true };
|
|
943
1188
|
advance("done");
|
|
944
1189
|
break;
|
|
945
1190
|
}
|
|
@@ -951,8 +1196,13 @@ export async function runRelease(options, adapters) {
|
|
|
951
1196
|
await restart(error.head);
|
|
952
1197
|
}
|
|
953
1198
|
}
|
|
954
|
-
|
|
955
|
-
if (
|
|
1199
|
+
const left = homebrewLeft;
|
|
1200
|
+
if (left === null)
|
|
1201
|
+
say(`Released ${j.tag}: deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, published and ${j.homebrew?.state === "superseded" ? `left Homebrew as it is (${j.homebrew.why ?? "the tap names a newer version"})` : "on Homebrew"}.`);
|
|
1202
|
+
else
|
|
1203
|
+
say(`Published ${j.tag} (deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, on npm and GitHub); Homebrew ${left.state}${left.why === null ? "" : `: ${left.why}`}. Finish it with \`${resume}\`.`);
|
|
1204
|
+
// The automatic road's note, once: when this run published (a run that only finishes Homebrew already said it).
|
|
1205
|
+
if (j.auto?.state === "approved" && !finishingHomebrew) {
|
|
956
1206
|
try {
|
|
957
1207
|
await rel?.auto?.released(j.task.id, j.tag);
|
|
958
1208
|
}
|
|
@@ -1047,12 +1297,54 @@ async function deployed(adapters, candidate) {
|
|
|
1047
1297
|
* running are named, and the gate waits for them; ones that never finish within the gate's limit stop here too. GitHub
|
|
1048
1298
|
* not answering is waited out the same way, never taken for a pass. The ci step still checks again before merging.
|
|
1049
1299
|
*/
|
|
1050
|
-
|
|
1300
|
+
/**
|
|
1301
|
+
* The branch must contain main's head as GitHub's API answers now: CI tests the pull request's head commit,
|
|
1302
|
+
* so only a branch that already holds main merges into exactly the tested tree. Refused in plain words otherwise.
|
|
1303
|
+
*/
|
|
1304
|
+
async function containsMain(adapters, j, nothing) {
|
|
1305
|
+
// GitHub's own answers (eval-harness review 1, finding 4): main's head and the comparison are read from its API by
|
|
1306
|
+
// this coordinator, never from refs/remotes/origin/main in a checkout a build can write. The checkout is still synced
|
|
1307
|
+
// for what the gate reads from it later.
|
|
1308
|
+
await adapters.repo.sync(j.branch);
|
|
1309
|
+
const main = await adapters.github.mainHead(j.github);
|
|
1310
|
+
if (!/^[0-9a-f]{40}$/.test(main))
|
|
1311
|
+
throw new Stop("base-moved", `GitHub named no commit for main. ${nothing}.`);
|
|
1312
|
+
if (await adapters.github.contains(j.github, j.candidate.sha, main))
|
|
1313
|
+
return main;
|
|
1314
|
+
throw new Stop("base-moved", `The branch is behind main; update it and push. ${j.branch} at ${j.candidate.sha.slice(0, 7)} doesn't contain main's head ${main.slice(0, 7)}, so what merges wouldn't be the tree CI tested. ${nothing}: release the updated commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
1315
|
+
}
|
|
1316
|
+
/**
|
|
1317
|
+
* GitHub must refuse a branch behind main for every release (eval-harness review 1, finding 4). A person's approval
|
|
1318
|
+
* cannot make the final main read and merge atomic, so absent or unreadable protection stops that release too.
|
|
1319
|
+
*/
|
|
1320
|
+
async function mainProtection(adapters, j, auto, nothing) {
|
|
1321
|
+
let protection;
|
|
1322
|
+
try {
|
|
1323
|
+
protection = await adapters.github.upToDateRequired(j.github);
|
|
1324
|
+
}
|
|
1325
|
+
catch {
|
|
1326
|
+
protection = null;
|
|
1327
|
+
}
|
|
1328
|
+
// The git lanes main's protection doesn't name yet are said, never refused (PROTECTION_REPORT_ONLY_CHECKS).
|
|
1329
|
+
if (protection !== null && protection.notYetRequired.length > 0)
|
|
1330
|
+
adapters.say(` Note: main's protection doesn't require ${protection.notYetRequired.join(", ")} yet; add ${protection.notYetRequired.length === 1 ? "it" : "them"} to its required checks.`);
|
|
1331
|
+
if (protection?.effective === true)
|
|
1332
|
+
return;
|
|
1333
|
+
const lacking = protection === null ? "GitHub's branch protection for main couldn't be read" : `main's protection doesn't hold: ${protection.missing.join("; ")}`;
|
|
1334
|
+
throw new Stop("base-unprotected", `${lacking}, so this release can't merge${auto ? " automatically" : ""}. ${nothing}. Restore effective main protection and try again.`);
|
|
1335
|
+
}
|
|
1336
|
+
/**
|
|
1337
|
+
* Wait for every required pull request check on the candidate to pass; stop on one that failed. `pendingIsEnough` (the
|
|
1338
|
+
* automatic road's gate, pipeline-speed.md D2): checks still running are no reason to wait — returns false then, true
|
|
1339
|
+
* once they all passed. `nothing` says what hasn't happened yet when a check failed.
|
|
1340
|
+
*/
|
|
1341
|
+
async function earlyChecks(adapters, j, wait, options = {}) {
|
|
1342
|
+
const nothing = options.nothing ?? "Nothing was filed or sent for approval";
|
|
1051
1343
|
const sha = j.candidate.sha.slice(0, 7);
|
|
1052
1344
|
const required = REQUIRED_PR_CHECKS;
|
|
1053
1345
|
let named = "", unread = false;
|
|
1054
1346
|
try {
|
|
1055
|
-
await wait(`the pull request checks on ${sha}`, async () => {
|
|
1347
|
+
return await wait(`the pull request checks on ${sha}`, async () => {
|
|
1056
1348
|
let runs;
|
|
1057
1349
|
try {
|
|
1058
1350
|
runs = await adapters.github.checks(j.github, j.candidate.sha);
|
|
@@ -1063,12 +1355,14 @@ async function earlyChecks(adapters, j, wait) {
|
|
|
1063
1355
|
unread = true;
|
|
1064
1356
|
return null;
|
|
1065
1357
|
}
|
|
1066
|
-
const verdict = prChecksVerdict(runs.filter(run => required.includes(run.name)), j.candidate.sha, required);
|
|
1358
|
+
const verdict = prChecksVerdict(runs.filter(run => required.includes(run.name)), j.candidate.sha, required, { workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
1067
1359
|
if (verdict.state === "failed") {
|
|
1068
|
-
throw new Stop("ci-failed", `Pull request checks already failed on ${sha}: ${verdict.failed.join(", ")}.
|
|
1360
|
+
throw new Stop("ci-failed", `Pull request checks already failed on ${sha}: ${verdict.failed.join(", ")}. ${nothing}. Fix the branch, then release the new commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\` (or rerun the failed job on GitHub and run this command again).`);
|
|
1069
1361
|
}
|
|
1070
1362
|
if (verdict.state === "passed")
|
|
1071
1363
|
return true;
|
|
1364
|
+
if (options.pendingIsEnough === true)
|
|
1365
|
+
return false;
|
|
1072
1366
|
const now = verdict.waiting.join(", ");
|
|
1073
1367
|
if (now !== named) {
|
|
1074
1368
|
adapters.say(` Pull request checks on ${sha} not finished yet: ${now}.`);
|
|
@@ -1080,7 +1374,7 @@ async function earlyChecks(adapters, j, wait) {
|
|
|
1080
1374
|
catch (error) {
|
|
1081
1375
|
if (!(error instanceof Stop) || error.reason !== "timeout")
|
|
1082
1376
|
throw error;
|
|
1083
|
-
throw new Stop("ci-missing", `Pull request checks on ${sha} didn't all finish in time${named === "" ? (unread ? " (GitHub didn't answer)" : "") : `: ${named}`}.
|
|
1377
|
+
throw new Stop("ci-missing", `Pull request checks on ${sha} didn't all finish in time${named === "" ? (unread ? " (GitHub didn't answer)" : "") : `: ${named}`}. ${nothing}. Start or rerun them on GitHub, then run this command again.`);
|
|
1084
1378
|
}
|
|
1085
1379
|
}
|
|
1086
1380
|
/** The network failures a release's waits ride out: git or gh timing out (or cut off) on the way to GitHub. */
|
|
@@ -1100,8 +1394,10 @@ function networkWords(error) {
|
|
|
1100
1394
|
* each reported attempt must be filed under `head` with a check-run id; a required check counts only from GitHub
|
|
1101
1395
|
* Actions (an app that names a check alike can't stand in for it); attempts are grouped by the job they belong to — the
|
|
1102
1396
|
* app, its check suite (a workflow run, whose re-runs stay in it) and the name — and every group is judged by its
|
|
1103
|
-
* latest attempt; every required check needs a group. With `
|
|
1104
|
-
*
|
|
1397
|
+
* latest attempt; every required check needs a group. With `auto` (an automatic approval), every other failed attempt on
|
|
1398
|
+
* the commit — an earlier attempt, or a job a newer run renamed or dropped — must be one GitHub's own job record shows ending before any workflow step ran (`beforeTests`: confirmed
|
|
1399
|
+
* infrastructure), and at most INFRA_RERUNS_PER_CANDIDATE of them on the commit; any other earlier failure — a test,
|
|
1400
|
+
* a typecheck, or one nothing explains — fails the verdict, so a person decides. The failures it tolerated are returned.
|
|
1105
1401
|
*/
|
|
1106
1402
|
export function prChecksVerdict(runs, head, required, options = {}) {
|
|
1107
1403
|
const failed = [], waiting = [], failures = [];
|
|
@@ -1119,6 +1415,12 @@ export function prChecksVerdict(runs, head, required, options = {}) {
|
|
|
1119
1415
|
failed.push(`${run.name} (reported by ${run.app ?? "an unknown app"}, not GitHub Actions)`);
|
|
1120
1416
|
continue;
|
|
1121
1417
|
}
|
|
1418
|
+
// A required name counts only from the workflow that defines it: a job of the same name in another workflow (or
|
|
1419
|
+
// one whose workflow GitHub doesn't name) fails the verdict rather than stand in for it.
|
|
1420
|
+
if (options.workflow !== undefined && required.includes(run.name) && !workflowIs(run.workflowPath, options.workflow)) {
|
|
1421
|
+
failed.push(`${run.name} (from ${run.workflowPath ?? "a workflow GitHub doesn't name"}, not ${options.workflow})`);
|
|
1422
|
+
continue;
|
|
1423
|
+
}
|
|
1122
1424
|
if (run.app === "github-actions" && (typeof run.suite !== "number" || !Number.isSafeInteger(run.suite))) {
|
|
1123
1425
|
failed.push(`${run.name} (no workflow run)`);
|
|
1124
1426
|
continue;
|
|
@@ -1143,16 +1445,20 @@ export function prChecksVerdict(runs, head, required, options = {}) {
|
|
|
1143
1445
|
groups.set(key, [...(groups.get(key) ?? []), run]);
|
|
1144
1446
|
}
|
|
1145
1447
|
const judged = new Set();
|
|
1448
|
+
// The attempt each current job stands on (its latest in the current run); every other attempt on the commit is
|
|
1449
|
+
// history, whatever its job is called now.
|
|
1450
|
+
const standing = new Set();
|
|
1146
1451
|
for (const attempts of groups.values()) {
|
|
1147
1452
|
const ordered = [...attempts].sort((a, b) => a.id - b.id);
|
|
1148
1453
|
const now = current.get(workflowOf(ordered[0])) ?? null;
|
|
1149
1454
|
const live = ordered.filter(one => (one.suite ?? null) === now);
|
|
1150
|
-
// A job only an older run had (the newer run no longer has it
|
|
1151
|
-
// required check.
|
|
1455
|
+
// A job only an older run had (the newer run no longer has it, or calls it something else) doesn't stand for a
|
|
1456
|
+
// required check; its failures are still history below.
|
|
1152
1457
|
if (live.length === 0)
|
|
1153
1458
|
continue;
|
|
1154
1459
|
const latest = live.at(-1), name = latest.name;
|
|
1155
1460
|
judged.add(name);
|
|
1461
|
+
standing.add(latest);
|
|
1156
1462
|
if (latest.status !== "completed") {
|
|
1157
1463
|
waiting.push(`${name} (${latest.status})`);
|
|
1158
1464
|
continue;
|
|
@@ -1161,20 +1467,31 @@ export function prChecksVerdict(runs, head, required, options = {}) {
|
|
|
1161
1467
|
failed.push(`${name} (${latest.conclusion ?? "no conclusion"})`);
|
|
1162
1468
|
continue;
|
|
1163
1469
|
}
|
|
1164
|
-
const earlier = ordered.filter(one => one !== latest);
|
|
1165
|
-
// An automatic approval also waits out an earlier attempt of the current run still running (it counts toward the cap).
|
|
1166
|
-
if (options.cap !== undefined && earlier.some(one => (one.suite ?? null) === now && one.status !== "completed")) {
|
|
1167
|
-
waiting.push(`${name} (an earlier attempt still running)`);
|
|
1168
|
-
continue;
|
|
1169
|
-
}
|
|
1170
|
-
// The cap counts every failed attempt of the job, in the current run and in the runs it superseded.
|
|
1171
|
-
const flaked = earlier.filter(one => one.status === "completed" && one.conclusion !== "success");
|
|
1172
|
-
if (options.cap !== undefined && flaked.length > options.cap) {
|
|
1173
|
-
failed.push(`${name} (failed ${flaked.length} times before it passed)`);
|
|
1174
|
-
continue;
|
|
1175
|
-
}
|
|
1176
|
-
failures.push(...flaked.map(one => ({ id: one.id, name, conclusion: one.conclusion })));
|
|
1177
1470
|
}
|
|
1471
|
+
// Every failed attempt on the commit that no current job stands on — an earlier attempt of a job, a job a newer run
|
|
1472
|
+
// renamed, dropped, or never ran again — is classified before anything is superseded (eval-harness review 1, finding
|
|
1473
|
+
// 1): a renamed or vanished job never erases a failure. An automatic approval tolerates only confirmed
|
|
1474
|
+
// infrastructure (GitHub's job record shows the runner failing before any workflow step ran), and waits out any
|
|
1475
|
+
// earlier attempt still running.
|
|
1476
|
+
const history = valid.filter(one => !standing.has(one));
|
|
1477
|
+
if (options.auto === true) {
|
|
1478
|
+
for (const one of history)
|
|
1479
|
+
if (one.status !== "completed")
|
|
1480
|
+
waiting.push(`${one.name} (an earlier attempt still running)`);
|
|
1481
|
+
}
|
|
1482
|
+
const flaked = history.filter(one => one.status === "completed" && one.conclusion !== "success");
|
|
1483
|
+
if (options.auto === true) {
|
|
1484
|
+
const unexplained = new Map();
|
|
1485
|
+
for (const one of flaked)
|
|
1486
|
+
if (one.beforeTests !== true)
|
|
1487
|
+
unexplained.set(one.name, (unexplained.get(one.name) ?? 0) + 1);
|
|
1488
|
+
for (const [name, count] of unexplained)
|
|
1489
|
+
failed.push(`${name} (failed ${count === 1 ? "once" : `${count} times`} on this commit, and GitHub doesn't show its runner failing before the tests ran)`);
|
|
1490
|
+
}
|
|
1491
|
+
const infrastructure = flaked.filter(one => one.beforeTests === true).length;
|
|
1492
|
+
failures.push(...flaked.map(one => ({ id: one.id, name: one.name, conclusion: one.conclusion })));
|
|
1493
|
+
if (options.auto === true && infrastructure > INFRA_RERUNS_PER_CANDIDATE)
|
|
1494
|
+
failed.push(`GitHub's runners failed ${infrastructure} times on this commit before the tests ran (at most ${INFRA_RERUNS_PER_CANDIDATE} are rerun)`);
|
|
1178
1495
|
for (const name of required)
|
|
1179
1496
|
if (!judged.has(name))
|
|
1180
1497
|
waiting.push(`${name} (not started)`);
|
|
@@ -1182,3 +1499,26 @@ export function prChecksVerdict(runs, head, required, options = {}) {
|
|
|
1182
1499
|
return { state: "failed", failed };
|
|
1183
1500
|
return waiting.length > 0 ? { state: "pending", waiting } : { state: "passed", failures };
|
|
1184
1501
|
}
|
|
1502
|
+
/**
|
|
1503
|
+
* What a failed release check means and what happens next, by its class (check-failure.ts), in words. Only confirmed
|
|
1504
|
+
* infrastructure reruns on its own; nothing else is rerun to pass.
|
|
1505
|
+
*/
|
|
1506
|
+
export function checkFailedWords(input) {
|
|
1507
|
+
const short = input.sha.slice(0, 7), again = `\`toolroll release ${input.branch} --repo ${input.checkout} --new\``;
|
|
1508
|
+
switch (input.class) {
|
|
1509
|
+
case "deterministic":
|
|
1510
|
+
return `The Full release check of ${short} failed (run ${input.run}): ${input.why}. That is a failure in the code or its tests, so it isn't run again: fix the branch, then release the new commit with ${again}.`;
|
|
1511
|
+
case "agent-behaviour":
|
|
1512
|
+
return `The Full release check of ${short} failed (run ${input.run}): ${input.why}. Real-model journeys are counted, never rerun until they pass, so a person decides: fix the branch and release the new commit with ${again}, or approve this commit again with ${again}.`;
|
|
1513
|
+
case "infrastructure":
|
|
1514
|
+
return `The Full release check of ${short} failed (run ${input.run}): ${input.why}. It was already run again as often as a commit is (${INFRA_RERUNS_PER_CANDIDATE} times), so a person decides: approve this commit again with ${again}.`;
|
|
1515
|
+
case "unknown":
|
|
1516
|
+
return `The Full release check of ${short} failed (run ${input.run}) with no failure Toolroll could explain: ${input.why}. It isn't rerun to pass. To look into it, \`toolroll task regate ${input.taskId}\` runs it again on the same commit, but a pass then doesn't erase this failure: a person decides, and approves this commit again with ${again}, or fixes the branch and releases the new commit the same way.`;
|
|
1517
|
+
default:
|
|
1518
|
+
return `The Full release check of ${short} did not pass (run ${input.run}): ${input.why}. Fix the branch and release the new commit with ${again}.`;
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
/** Whether GitHub's workflow-run `path` names exactly `file` (it may carry the ref it ran from, after an `@`). */
|
|
1522
|
+
export function workflowIs(path, file) {
|
|
1523
|
+
return typeof path === "string" && (path === file || path.startsWith(`${file}@`));
|
|
1524
|
+
}
|