toolroll 0.9.75 → 0.9.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +241 -0
- package/README.md +11 -4
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/agent-fence.js +36 -8
- package/dist/approval-authority.js +6 -2
- package/dist/approval-rules-ui.d.ts +11 -0
- package/dist/approval-rules-ui.js +13 -1
- package/dist/assignment.js +3 -0
- package/dist/batch-checks.js +3 -1
- package/dist/browser/THIRD_PARTY_NOTICES.txt +5 -5
- package/dist/browser/workspace.css +2 -2
- package/dist/browser/workspace.js +31 -31
- package/dist/builder.d.ts +14 -0
- package/dist/builder.js +141 -42
- package/dist/chat-decide.js +4 -0
- package/dist/check-failure.d.ts +92 -0
- package/dist/check-failure.js +267 -0
- package/dist/check-progress.d.ts +28 -1
- package/dist/check-progress.js +105 -6
- package/dist/check-report.d.ts +52 -6
- package/dist/check-report.js +193 -8
- package/dist/checkout-cleanup.js +2 -2
- package/dist/cli.js +8 -5
- package/dist/coding-handoff.d.ts +1 -0
- package/dist/coding-handoff.js +20 -11
- package/dist/coding-workspace.js +7 -5
- package/dist/commit-extract.d.ts +6 -0
- package/dist/commit-extract.js +63 -8
- package/dist/console-chrome.d.ts +41 -0
- package/dist/console-chrome.js +166 -0
- package/dist/contracts/console-api.d.ts +1 -0
- package/dist/contracts/console-api.js +1 -0
- package/dist/contracts/release-check.d.ts +12 -0
- package/dist/contracts/release-check.js +17 -0
- package/dist/contracts/verification-receipt.d.ts +20 -2
- package/dist/contracts/verification-receipt.js +15 -3
- package/dist/contracts/verification-view.d.ts +8 -0
- package/dist/controller-supervisor.d.ts +8 -0
- package/dist/controller-supervisor.js +33 -1
- package/dist/demo.js +3 -2
- package/dist/deploy-gate.d.ts +3 -0
- package/dist/deploy-gate.js +8 -2
- package/dist/desktop-update.d.ts +1 -1
- package/dist/discover.js +6 -1
- package/dist/dispose.d.ts +3 -0
- package/dist/dispose.js +6 -3
- package/dist/doctor.d.ts +7 -0
- package/dist/doctor.js +10 -2
- package/dist/envelope.js +2 -1
- package/dist/evidence.js +4 -1
- package/dist/exec.d.ts +7 -0
- package/dist/exec.js +7 -1
- package/dist/first-run.js +3 -1
- package/dist/git-floor.d.ts +16 -0
- package/dist/git-floor.js +36 -0
- package/dist/git-incident.d.ts +49 -0
- package/dist/git-incident.js +69 -0
- package/dist/git-views.d.ts +20 -0
- package/dist/git-views.js +120 -0
- package/dist/grant.js +7 -1
- package/dist/guides.js +4 -3
- package/dist/operate.d.ts +13 -0
- package/dist/operate.js +779 -319
- package/dist/planner.js +5 -1
- package/dist/prepared-evidence.d.ts +2 -1
- package/dist/prepared-evidence.js +8 -3
- package/dist/project-knowledge.js +6 -2
- package/dist/project-learning.js +23 -3
- package/dist/proof.js +6 -0
- package/dist/pull-request-flow.js +4 -0
- package/dist/read-executor-worker.js +8 -4
- package/dist/read-executor.d.ts +18 -1
- package/dist/read-executor.js +8 -0
- package/dist/release-adapters.d.ts +71 -2
- package/dist/release-adapters.js +471 -12
- package/dist/release-authority.d.ts +46 -0
- package/dist/release-authority.js +68 -0
- package/dist/release-auto.d.ts +184 -0
- package/dist/release-auto.js +612 -0
- package/dist/release-card.d.ts +15 -3
- package/dist/release-card.js +25 -9
- package/dist/release-ci-hold.d.ts +41 -0
- package/dist/release-ci-hold.js +53 -0
- package/dist/release-cli.js +77 -14
- package/dist/release-coverage.js +5 -2
- package/dist/release-recheck.d.ts +1 -1
- package/dist/release-recheck.js +6 -1
- package/dist/release.d.ts +207 -6
- package/dist/release.js +495 -20
- package/dist/repository-context.js +4 -6
- package/dist/result-follow-ups.js +3 -1
- package/dist/review-context.js +8 -6
- package/dist/runner.js +2 -1
- package/dist/sandbox-runtime.d.ts +8 -0
- package/dist/sandbox-runtime.js +14 -2
- package/dist/scope.d.ts +31 -2
- package/dist/scope.js +51 -4
- package/dist/scout.js +5 -1
- package/dist/secret-mint.d.ts +1 -0
- package/dist/secret-mint.js +14 -0
- package/dist/serve.js +101 -51
- package/dist/server/chat.js +5 -5
- package/dist/server/render-tasks.d.ts +12 -0
- package/dist/server/render-tasks.js +25 -14
- package/dist/server/request-context.d.ts +6 -0
- package/dist/server/route-table.js +3 -0
- package/dist/server/runtime.d.ts +2 -0
- package/dist/server/settings.js +29 -2
- package/dist/server/task-views.js +15 -2
- package/dist/setup-guide.d.ts +2 -2
- package/dist/storage.d.ts +5 -0
- package/dist/storage.js +28 -1
- package/dist/store.d.ts +11 -1
- package/dist/store.js +44 -5
- package/dist/task-base.d.ts +4 -0
- package/dist/task-base.js +22 -2
- package/dist/teardown-git.d.ts +90 -13
- package/dist/teardown-git.js +1027 -129
- package/dist/test-infra.d.ts +26 -0
- package/dist/test-infra.js +321 -0
- package/dist/up-processes.d.ts +137 -0
- package/dist/up-processes.js +165 -0
- package/dist/up-role-child.d.ts +1 -0
- package/dist/up-role-child.js +77 -0
- package/dist/verification-evidence.d.ts +1 -0
- package/dist/verification-evidence.js +13 -1
- package/dist/worktree.d.ts +4 -0
- package/dist/worktree.js +13 -7
- package/package.json +2 -1
package/dist/release.js
CHANGED
|
@@ -36,6 +36,7 @@ import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, re
|
|
|
36
36
|
import { randomUUID } from "node:crypto";
|
|
37
37
|
import { dirname, join } from "node:path";
|
|
38
38
|
import { personProof } from "./person-proof.js";
|
|
39
|
+
import { RELEASE_CI_RUNNING_WORDS as CI_RUNNING_WORDS } from "./release-ci-hold.js";
|
|
39
40
|
/** The pull-request checks a release waits for, by exact job name (.github/workflows/ci.yml). Every one must have
|
|
40
41
|
* concluded `success` on the gated commit; src/ci-workflow.test.ts holds the workflow to this list. */
|
|
41
42
|
export const REQUIRED_PR_CHECKS = [
|
|
@@ -44,19 +45,44 @@ export const REQUIRED_PR_CHECKS = [
|
|
|
44
45
|
"linux native containment · delegated cgroup v2",
|
|
45
46
|
"linux fence · sandbox-runtime",
|
|
46
47
|
"Windows Job Object containment",
|
|
48
|
+
// The git lanes (ci.yml job "git"): the declared floor (git-floor.ts) and Debian trixie's 2.47.
|
|
49
|
+
"git 2.41 · debian trixie",
|
|
50
|
+
"git 2.47 · debian trixie",
|
|
47
51
|
];
|
|
52
|
+
/** The workflow file every REQUIRED_PR_CHECKS job belongs to: a job of that name from any other workflow (a lookalike
|
|
53
|
+
* added beside it) never stands for it (Codex review round 1, inherited CI limitation). */
|
|
54
|
+
export const REQUIRED_PR_CHECKS_WORKFLOW = ".github/workflows/ci.yml";
|
|
55
|
+
/**
|
|
56
|
+
* The pull-request checks main's GitHub protection is not yet required to name (eval-harness review 2, finding 1): the
|
|
57
|
+
* git lanes join main's protection once they have run there, so a protection without them is reported, never refused.
|
|
58
|
+
* A later release empties this list; from then on main's protection must name every REQUIRED_PR_CHECKS check.
|
|
59
|
+
*/
|
|
60
|
+
export const PROTECTION_REPORT_ONLY_CHECKS = ["git 2.41 · debian trixie", "git 2.47 · debian trixie"];
|
|
61
|
+
/** The checks main's protection must require at minimum: every required check that isn't report-only. */
|
|
62
|
+
export const PROTECTION_MINIMUM_CHECKS = REQUIRED_PR_CHECKS.filter(name => !PROTECTION_REPORT_ONLY_CHECKS.includes(name));
|
|
48
63
|
/** The checks a release tag runs (.github/workflows/publish.yml), awaited before Homebrew moves. */
|
|
49
64
|
export const RELEASE_TAG_CHECKS = ["node 22 · macos-latest", "node 24 · macos-latest", "publish"];
|
|
50
65
|
export const RELEASE_STEPS = ["gate", "approval", "check", "complete", "quiet", "deploy", "link", "ci", "merge", "tree", "tag", "publish", "homebrew", "done"];
|
|
51
66
|
/** How long each waiting step may take, in minutes, before the release stops there (rerunning continues from it). */
|
|
52
67
|
export const RELEASE_LIMITS = { gate: 60, approval: 24 * 60, check: 120, quiet: 60, deploy: 30, ci: 60, publish: 45, homebrew: 45 };
|
|
53
68
|
const POLL_MS = 15_000;
|
|
69
|
+
/**
|
|
70
|
+
* Failure classes replace a flat flake allowance (docs/plans/eval-harness.md; check-failure.ts says the classes): only a
|
|
71
|
+
* failure the plane itself confirmed as infrastructure is ever run again under the same approval — at most once per
|
|
72
|
+
* incident and twice per candidate — for a release check and for a GitHub CI job alike. A failing test, a typecheck, an
|
|
73
|
+
* attack test, a migration: none tolerated. An unexplained failure blocks, and a person decides.
|
|
74
|
+
*/
|
|
75
|
+
export const INFRA_RERUNS_PER_INCIDENT = 1;
|
|
76
|
+
export const INFRA_RERUNS_PER_CANDIDATE = 2;
|
|
54
77
|
/** A stop the release reports: the step it stopped at and why, in words. */
|
|
55
78
|
class Stop extends Error {
|
|
56
79
|
reason;
|
|
57
|
-
|
|
80
|
+
build;
|
|
81
|
+
/** `build`: the release build itself failed before its check (not a failed check). */
|
|
82
|
+
constructor(reason, message, build = false) {
|
|
58
83
|
super(message);
|
|
59
84
|
this.reason = reason;
|
|
85
|
+
this.build = build;
|
|
60
86
|
}
|
|
61
87
|
}
|
|
62
88
|
/** The branch moved before the deploy reservation: carry the approval to the new head, or file it for a person. */
|
|
@@ -255,6 +281,9 @@ export async function runRelease(options, adapters) {
|
|
|
255
281
|
else
|
|
256
282
|
say(`Resuming the release of ${journal.tag} (${journal.candidate.sha.slice(0, 7)}) at ${journal.step}.`);
|
|
257
283
|
const j = journal;
|
|
284
|
+
// A journal an older coordinator filed ahead of CI: its deploy reads CI fresh too.
|
|
285
|
+
if (j.ciAhead === true && j.ciBeforeDeploy !== true)
|
|
286
|
+
j.ciBeforeDeploy = true;
|
|
258
287
|
if (j.checkout !== options.checkout)
|
|
259
288
|
throw new Stop("checkout", `This release runs in ${j.checkout}; rerun with --repo ${j.checkout}.`);
|
|
260
289
|
save();
|
|
@@ -367,14 +396,54 @@ export async function runRelease(options, adapters) {
|
|
|
367
396
|
* release needs a person's fresh approval, even for the same commit. It is persisted (the authority stops; the
|
|
368
397
|
* journal says so) before the release stops; `--new` files the commit for a person again.
|
|
369
398
|
*/
|
|
370
|
-
const ciFailedAfterDeploy = async (what) => {
|
|
399
|
+
const ciFailedAfterDeploy = async (what, deploy = "after") => {
|
|
371
400
|
j.needsApproval = true;
|
|
372
401
|
save();
|
|
402
|
+
// An automatic approval never outlives a CI failure: the owner is told, and a person approves again.
|
|
403
|
+
if (j.auto?.state === "approved")
|
|
404
|
+
await rel?.auto?.giveUp(j.task.id, deploy === "after" ? `${what}, after the deploy` : what);
|
|
373
405
|
// Fail closed: the authority must have stopped; otherwise its approval could still carry the commit.
|
|
374
|
-
if (rel !== undefined && j.authority && (epoch === null || !await rel.needsApproval(j.authority.id, "a pull request check failed after the deploy", epoch))) {
|
|
406
|
+
if (rel !== undefined && j.authority && (epoch === null || !await rel.needsApproval(j.authority.id, deploy === "after" ? "a pull request check failed after the deploy" : "a pull request check failed before the deploy", epoch))) {
|
|
375
407
|
throw new Stop("lease", `${what}. Nothing was merged, but this release's approval couldn't be stopped (another coordinator holds it, or a merge is reserved). Check it with \`toolroll release ${j.branch} --repo ${j.checkout}\` before going on.`);
|
|
376
408
|
}
|
|
377
|
-
throw new Stop("ci-failed",
|
|
409
|
+
throw new Stop("ci-failed", deploy === "after"
|
|
410
|
+
? `${what}. Nothing was merged. The deploy had already started, so this commit needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`
|
|
411
|
+
: `${what}. Nothing was deployed or merged. This commit needs a person's approval: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
412
|
+
};
|
|
413
|
+
/** An automatic approval's CI allowance: only attempts GitHub shows failing before the tests ran, at most
|
|
414
|
+
* INFRA_RERUNS_PER_CANDIDATE on the commit; a person's approval judges each job by its latest attempt. */
|
|
415
|
+
const autoCap = () => (j.auto?.state === "approved" ? { auto: true } : {});
|
|
416
|
+
/**
|
|
417
|
+
* An automatically approved release stands only while its CI does (auto-approve-green.md): the complete, current
|
|
418
|
+
* checks on the exact commit are read again right before each step that can't be taken back — the deploy, the CLI
|
|
419
|
+
* relink, the merge and the tag. Anything but green (failed, a job re-run and pending, an earlier failure GitHub
|
|
420
|
+
* doesn't show as the runner's own) stops the
|
|
421
|
+
* release, revokes its approval and tells the owner; the commit then needs a person.
|
|
422
|
+
*/
|
|
423
|
+
const autoCiHolds = async (before) => {
|
|
424
|
+
if (j.auto?.state !== "approved")
|
|
425
|
+
return;
|
|
426
|
+
const read = await adapters.github.checks(j.github, j.candidate.sha);
|
|
427
|
+
const whole = adapters.github.complete === undefined ? false : await adapters.github.complete(j.github, j.candidate.sha);
|
|
428
|
+
const verdict = whole ? prChecksVerdict(read, j.candidate.sha, REQUIRED_PR_CHECKS, { auto: true, workflow: REQUIRED_PR_CHECKS_WORKFLOW }) : { state: "failed", failed: ["GitHub's check history of this commit can't be read whole"] };
|
|
429
|
+
if (verdict.state === "passed") {
|
|
430
|
+
await liftCiHold(false, `The release cannot continue before ${before}`);
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
if (before === "the tag") {
|
|
434
|
+
// Already merged: nothing to revoke. It isn't tagged or published, and a person decides — durably, in the plane.
|
|
435
|
+
const why = `Pull request checks on ${j.candidate.sha.slice(0, 7)} stopped passing after the merge`;
|
|
436
|
+
await rel?.auto?.holdPublish(j.task.id, why);
|
|
437
|
+
await rel?.auto?.giveUp(j.task.id, why);
|
|
438
|
+
throw new Stop("ci-failed", `Pull request checks on ${j.candidate.sha.slice(0, 7)} are no longer all passing: ${(verdict.state === "failed" ? verdict.failed : verdict.waiting).join(", ")}. It was merged, but nothing was tagged or published; a person decides how to go on.`);
|
|
439
|
+
}
|
|
440
|
+
await ciFailedAfterDeploy(`Pull request checks on ${j.candidate.sha.slice(0, 7)} are no longer all passing before ${before}: ${(verdict.state === "failed" ? verdict.failed : verdict.waiting).join(", ")}`, before === "the deploy" && j.deploy === null ? "before" : "after");
|
|
441
|
+
};
|
|
442
|
+
/** A publication held for a person (the plane's record) stops the tag, the publication and Homebrew. */
|
|
443
|
+
const publishHold = async () => {
|
|
444
|
+
const held = rel?.auto === undefined ? null : await rel.auto.publishHeld(j.task.id);
|
|
445
|
+
if (held !== null)
|
|
446
|
+
throw new Stop("approval", `${held}. It was merged; nothing more is tagged, published or sent to Homebrew until a person lets it go on: \`toolroll release publish-ok ${j.task.id} --as <you> --token <password>\`, then rerun.`);
|
|
378
447
|
};
|
|
379
448
|
/** identity(), on a plane that keeps release authority; a legacy plane keeps the old rules. */
|
|
380
449
|
const watch = async (action) => { if (rel !== undefined)
|
|
@@ -409,6 +478,7 @@ export async function runRelease(options, adapters) {
|
|
|
409
478
|
j.check = null;
|
|
410
479
|
j.completion = null;
|
|
411
480
|
j.authority = null;
|
|
481
|
+
j.auto = null;
|
|
412
482
|
epoch = null;
|
|
413
483
|
delete j.intents.gate;
|
|
414
484
|
delete j.intents.check;
|
|
@@ -417,6 +487,77 @@ export async function runRelease(options, adapters) {
|
|
|
417
487
|
j.stopped = null;
|
|
418
488
|
save();
|
|
419
489
|
};
|
|
490
|
+
/**
|
|
491
|
+
* The automatic road ended without an approval: the plane takes its `auto-check` back (the task asks a person, its
|
|
492
|
+
* passing check kept) and tells the owner; the release waits at approval for a person, as without the setting.
|
|
493
|
+
*/
|
|
494
|
+
const autoRefused = async (why) => {
|
|
495
|
+
j.auto = { state: "refused", why };
|
|
496
|
+
j.check = null;
|
|
497
|
+
j.step = "approval";
|
|
498
|
+
save();
|
|
499
|
+
await rel?.auto?.giveUp(j.task.id, why);
|
|
500
|
+
throw new Stop("approval", `Not approved automatically: ${why}. A person approves task ${j.task.id} (\`toolroll task approve ${j.task.id} --digest ${j.task.scopeDigest}\`, or the console), then rerun \`${resume}\`.`);
|
|
501
|
+
};
|
|
502
|
+
/**
|
|
503
|
+
* CI passed on the candidate as the coordinator read it: the plane reads it itself and lifts the approval hold it
|
|
504
|
+
* placed when the task was filed ahead of CI (round 1, finding 4). A refusal stops here; nobody was asked.
|
|
505
|
+
*/
|
|
506
|
+
const liftCiHold = async (ask, nothing) => {
|
|
507
|
+
const lift = rel?.auto?.ciPassed;
|
|
508
|
+
if (lift === undefined)
|
|
509
|
+
return;
|
|
510
|
+
const lifted = await lift(j.task.id, j.candidate.sha, ask);
|
|
511
|
+
if (!lifted.ok)
|
|
512
|
+
throw new Stop("ci-failed", `The plane didn't confirm CI passed on ${j.candidate.sha.slice(0, 7)}: ${lifted.why}. ${nothing}. Rerun \`${resume}\` once every pull request check passes.`);
|
|
513
|
+
};
|
|
514
|
+
/**
|
|
515
|
+
* One look at the pull request's required checks on the exact commit while CI may still run: a failed check stops
|
|
516
|
+
* (`nothing` says what hasn't happened), passed ends the plane's "CI still running" note, pending (or GitHub not
|
|
517
|
+
* answering) answers false. Judged as the ci step judges a person's release (prChecksVerdict, ci.yml, exact head).
|
|
518
|
+
*/
|
|
519
|
+
const ciWhileRunning = async (nothing, ask) => {
|
|
520
|
+
let runs;
|
|
521
|
+
try {
|
|
522
|
+
runs = await adapters.github.checks(j.github, j.candidate.sha);
|
|
523
|
+
}
|
|
524
|
+
catch (error) {
|
|
525
|
+
if (transientNetworkError(error))
|
|
526
|
+
return false;
|
|
527
|
+
throw error;
|
|
528
|
+
}
|
|
529
|
+
const verdict = prChecksVerdict(runs, j.candidate.sha, REQUIRED_PR_CHECKS, { workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
530
|
+
if (verdict.state === "failed") {
|
|
531
|
+
throw new Stop("ci-failed", `GitHub's checks failed on ${j.candidate.sha.slice(0, 7)}: ${verdict.failed.join(", ")}. ${nothing}. ${j.approval === null ? "" : `The approval covers ${j.candidate.sha.slice(0, 7)} only. `}Rerun the failed job on GitHub and run \`${resume}\` again, or fix the branch and release the new commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
532
|
+
}
|
|
533
|
+
if (verdict.state !== "passed")
|
|
534
|
+
return false;
|
|
535
|
+
// The plane's note can survive a crash before the journal saved ciAhead. Clear it on the plane's own reading,
|
|
536
|
+
// even on an older/resumed journal whose flags say nothing about CI.
|
|
537
|
+
await liftCiHold(ask, nothing);
|
|
538
|
+
if (j.ciAhead === true) {
|
|
539
|
+
j.ciAhead = false;
|
|
540
|
+
save();
|
|
541
|
+
}
|
|
542
|
+
return true;
|
|
543
|
+
};
|
|
544
|
+
/** Every person's release reads exact-head CI at deployment, even if it was green at the gate or an older
|
|
545
|
+
* journal has no ciBeforeDeploy flag. An approval and the release check can outlive that earlier green reading. */
|
|
546
|
+
const personCiBeforeDeploy = async () => {
|
|
547
|
+
if (j.auto?.state === "approved")
|
|
548
|
+
return;
|
|
549
|
+
let said = false;
|
|
550
|
+
await waitFor("ci", `the pull request checks on ${j.candidate.sha.slice(0, 7)} before the deploy`, async () => {
|
|
551
|
+
await watch("deploy");
|
|
552
|
+
if (await ciWhileRunning("Nothing was deployed, merged, tagged or published", false))
|
|
553
|
+
return true;
|
|
554
|
+
if (!said) {
|
|
555
|
+
say(` ${CI_RUNNING_WORDS}`);
|
|
556
|
+
said = true;
|
|
557
|
+
}
|
|
558
|
+
return null;
|
|
559
|
+
});
|
|
560
|
+
};
|
|
420
561
|
/**
|
|
421
562
|
* The task is approved for `action`: a release task (a carried commit, or any task once the release has an
|
|
422
563
|
* authority) only when the live-release predicate applies to it AND accepts it — fail closed, whatever its own seal
|
|
@@ -425,6 +566,13 @@ export async function runRelease(options, adapters) {
|
|
|
425
566
|
const approved = async (gate, action = "dispatch") => {
|
|
426
567
|
if (gate.candidate !== j.candidate.sha || gate.scopeDigest !== j.task.scopeDigest)
|
|
427
568
|
throw new Stop("scope-changed", `Task ${j.task.id}'s candidate or scope changed after approval.`);
|
|
569
|
+
// Checking automatically: the plane's `auto-check` seal must still stand (the setting on, its evidence, this digest).
|
|
570
|
+
if (j.auto?.state === "checking" && !j.authority) {
|
|
571
|
+
const standing = rel?.auto === undefined ? { ok: false, why: "this plane can't approve releases automatically" } : await rel.auto.checking(j.task.id);
|
|
572
|
+
if (standing.ok)
|
|
573
|
+
return;
|
|
574
|
+
return await autoRefused(standing.why);
|
|
575
|
+
}
|
|
428
576
|
if (rel !== undefined && (j.authority || gate.approval?.basis === "carried")) {
|
|
429
577
|
const live = await rel.live(j.task.id, action);
|
|
430
578
|
if (live.applies && live.ok)
|
|
@@ -439,7 +587,7 @@ export async function runRelease(options, adapters) {
|
|
|
439
587
|
if (gate.result?.level !== "full")
|
|
440
588
|
throw new Stop("check-level", `A Full check is required for this release (run ${gate.result?.run ?? "unknown"}: ${gate.result?.level ?? "unknown"}). Quick and Off checks cannot release a build.`);
|
|
441
589
|
if (gate.result.check !== "passed")
|
|
442
|
-
throw new Stop("check-failed",
|
|
590
|
+
throw new Stop("check-failed", checkFailedWords({ sha: j.candidate.sha, run: gate.result.run, class: null, why: `its check reads ${gate.result.check}`, branch: j.branch, checkout: j.checkout, taskId: j.task.id }));
|
|
443
591
|
};
|
|
444
592
|
/** The commit the passing build recorded: what the deployed runtime is stamped with. */
|
|
445
593
|
const builtHead = () => j.check?.head ?? j.candidate.sha;
|
|
@@ -476,6 +624,11 @@ export async function runRelease(options, adapters) {
|
|
|
476
624
|
}
|
|
477
625
|
}
|
|
478
626
|
}
|
|
627
|
+
// The tree CI tested (the pull request's head, not GitHub's merge commit) is the tree that merges only when the
|
|
628
|
+
// branch already contains main's head: refused here, before anything is filed, and again right before merging.
|
|
629
|
+
await containsMain(adapters, j, "Nothing was filed or sent for approval");
|
|
630
|
+
if (j.intents.gate === undefined)
|
|
631
|
+
await mainProtection(adapters, j, false, "Nothing was filed or sent for approval");
|
|
479
632
|
const existing = await adapters.toolroll.gate(j.task.id);
|
|
480
633
|
// A fresh task this journal never filed has appeared under its id: it isn't ours, and isn't a fresh approval.
|
|
481
634
|
if (existing.exists && j.freshTask === true && j.intents.gate === undefined)
|
|
@@ -487,7 +640,20 @@ export async function runRelease(options, adapters) {
|
|
|
487
640
|
j.task.scopeDigest = existing.scopeDigest;
|
|
488
641
|
}
|
|
489
642
|
else {
|
|
490
|
-
|
|
643
|
+
// The gate only stops on a check that already failed: the task is filed and a person asked while CI runs, and
|
|
644
|
+
// the release check runs beside it (pipeline-speed.md, D2 and "Decision: a person's release asks during
|
|
645
|
+
// CI"). The automatic approval waits for CI (the check step); nothing deploys before it passes (the deploy).
|
|
646
|
+
const settled = await earlyChecks(adapters, j, (what, probe) => waitFor("gate", what, probe), { pendingIsEnough: true });
|
|
647
|
+
// Recorded before the task exists, so its approval card says CI still runs from the first moment.
|
|
648
|
+
if (!settled)
|
|
649
|
+
await rel?.auto?.ciRunning?.(j.task.id, j.candidate.sha);
|
|
650
|
+
j.ciAhead = !settled;
|
|
651
|
+
if (!settled)
|
|
652
|
+
j.ciBeforeDeploy = true;
|
|
653
|
+
save();
|
|
654
|
+
if (!settled) {
|
|
655
|
+
say(` Pull request checks on ${j.candidate.sha.slice(0, 7)} are still running: the release check runs beside them, and nothing deploys before they pass`);
|
|
656
|
+
}
|
|
491
657
|
intend("gate");
|
|
492
658
|
await adapters.repo.sync(j.branch);
|
|
493
659
|
const filed = await adapters.toolroll.fileGate({
|
|
@@ -502,9 +668,42 @@ export async function runRelease(options, adapters) {
|
|
|
502
668
|
break;
|
|
503
669
|
}
|
|
504
670
|
case "approval": {
|
|
671
|
+
// Approve automatically when checks pass (auto-approve-green.md): with the project's setting on, the plane seals
|
|
672
|
+
// `auto-check` so the release check runs first; the approval itself is sealed after it passed. A fresh task
|
|
673
|
+
// after a CI failure after the deploy, or an automatic refusal, always asks a person.
|
|
674
|
+
const auto = rel?.auto;
|
|
675
|
+
if (auto !== undefined && j.freshTask !== true && j.auto?.state !== "refused" && j.task.scopeDigest !== null && await auto.setting(j.task.id)) {
|
|
676
|
+
await watch("dispatch");
|
|
677
|
+
const gate = await adapters.toolroll.gate(j.task.id);
|
|
678
|
+
if (gate.scopeDigest !== j.task.scopeDigest)
|
|
679
|
+
throw new Stop("scope-changed", `The scope of ${j.task.id} changed after it was filed; the release only runs on the scope it filed.`);
|
|
680
|
+
if (!ownerApproval(gate, j.task.scopeDigest).ok) {
|
|
681
|
+
const authorized = await auto.authorizeCheck({ taskId: j.task.id, digest: j.task.scopeDigest, candidate: j.candidate.sha });
|
|
682
|
+
if (authorized.ok) {
|
|
683
|
+
j.auto = { state: "checking" };
|
|
684
|
+
say(` Approving automatically once CI and the release check pass on ${j.candidate.sha.slice(0, 7)} (this project's release approval setting)`);
|
|
685
|
+
advance("check");
|
|
686
|
+
break;
|
|
687
|
+
}
|
|
688
|
+
j.auto = { state: "refused", why: authorized.why };
|
|
689
|
+
save();
|
|
690
|
+
await auto.giveUp(j.task.id, authorized.why);
|
|
691
|
+
say(` Not approved automatically: ${authorized.why}. A person approves it.`);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
// Filed while CI still runs: the person is asked now (Alex, 2026-10-29), and told nothing deploys until CI passes.
|
|
695
|
+
// A check that already failed asks nobody (as at the gate).
|
|
696
|
+
if (j.ciAhead === true)
|
|
697
|
+
await ciWhileRunning("Nothing was deployed or merged", true);
|
|
505
698
|
say(` The release scope needs your approval: \`toolroll task approve ${j.task.id} --digest ${j.task.scopeDigest}\`, or open the task in the console.`);
|
|
699
|
+
if (j.ciAhead === true)
|
|
700
|
+
say(` ${CI_RUNNING_WORDS}`);
|
|
506
701
|
const approved = await waitFor("approval", `a person to approve task ${j.task.id}`, async () => {
|
|
507
702
|
await watch("dispatch");
|
|
703
|
+
// While asking: a check that failed stops here (nobody approves a red commit), and CI passing ends the card's
|
|
704
|
+
// "still running" note (and lifts a hold an older coordinator placed).
|
|
705
|
+
if (j.ciAhead === true)
|
|
706
|
+
await ciWhileRunning("Nothing was deployed or merged", true);
|
|
508
707
|
const gate = await adapters.toolroll.gate(j.task.id);
|
|
509
708
|
if (gate.scopeDigest !== j.task.scopeDigest)
|
|
510
709
|
throw new Stop("scope-changed", `The scope of ${j.task.id} changed after it was filed; the release only runs on the scope it filed.`);
|
|
@@ -530,7 +729,7 @@ export async function runRelease(options, adapters) {
|
|
|
530
729
|
break;
|
|
531
730
|
}
|
|
532
731
|
case "check": {
|
|
533
|
-
const
|
|
732
|
+
const checkProbe = async () => {
|
|
534
733
|
// Before acting on an old candidate's failure: a moved head restarts the release first.
|
|
535
734
|
await watch("dispatch");
|
|
536
735
|
const gate = await adapters.toolroll.gate(j.task.id);
|
|
@@ -538,8 +737,20 @@ export async function runRelease(options, adapters) {
|
|
|
538
737
|
// A rerun of the same approved commit is on its way: the failure before it is not this release's answer.
|
|
539
738
|
if (gate.pending === true && gate.result?.check !== "passed")
|
|
540
739
|
return null;
|
|
740
|
+
// A failed Full check: the plane says why (its own records only) and reruns it under the same approval only
|
|
741
|
+
// for confirmed infrastructure; every attempt stays in the evidence. Anything else stops, saying its class.
|
|
742
|
+
if (gate.failed === null && gate.result !== null && gate.result.level === "full" && gate.result.check === "failed" && rel?.recheck !== undefined) {
|
|
743
|
+
const answer = await rel.recheck(j.task.id);
|
|
744
|
+
if (answer.action === "pending")
|
|
745
|
+
return null;
|
|
746
|
+
if (answer.action === "rerun") {
|
|
747
|
+
say(` run ${gate.result.run} failed: ${answer.why}. Toolroll confirmed it as its own infrastructure, so the check runs again on ${j.candidate.sha.slice(0, 7)} under the same approval`);
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
throw new Stop("check-failed", checkFailedWords({ sha: j.candidate.sha, run: gate.result.run, class: answer.class, why: answer.why, branch: j.branch, checkout: j.checkout, taskId: j.task.id }));
|
|
751
|
+
}
|
|
541
752
|
if (gate.failed !== null)
|
|
542
|
-
throw new Stop("check-failed", `The release build of ${j.task.id} failed: ${gate.failed}. Fix it and release the new commit with --new
|
|
753
|
+
throw new Stop("check-failed", `The release build of ${j.task.id} failed: ${gate.failed}. Fix it and release the new commit with --new.`, true);
|
|
543
754
|
if (gate.result === null || gate.result.check === "running" || gate.result.check === "pending")
|
|
544
755
|
return null;
|
|
545
756
|
// The build records a prepared candidate as a commit of its own (its message names the task): it must hold
|
|
@@ -551,10 +762,74 @@ export async function runRelease(options, adapters) {
|
|
|
551
762
|
fullCheck(gate);
|
|
552
763
|
if (gate.result.worktree === null)
|
|
553
764
|
throw new Stop("worktree", `Run ${gate.result.run} names no checkout to deploy from.`);
|
|
765
|
+
// A pass stands only when no earlier attempt on this commit failed for a reason other than confirmed
|
|
766
|
+
// infrastructure the release reran; what passed only on a retry is said (and an automatic approval refuses it).
|
|
767
|
+
if (rel?.standing !== undefined) {
|
|
768
|
+
const standing = await rel.standing(j.task.id, gate.result.run);
|
|
769
|
+
if (!standing.ok)
|
|
770
|
+
throw new Stop("check-failed", `Run ${gate.result.run} passed on ${j.candidate.sha.slice(0, 7)}, but ${standing.why}: approve this commit again with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`, or fix the branch and release the new commit the same way.`);
|
|
771
|
+
if (standing.retried === null)
|
|
772
|
+
say(` run ${gate.result.run} passed, but its report can't be read, so whether anything passed only on a retry is unknown`);
|
|
773
|
+
else if (standing.retried.length > 0)
|
|
774
|
+
say(` run ${gate.result.run} passed, but only on a retry: ${standing.retried.join(", ")}`);
|
|
775
|
+
}
|
|
554
776
|
return { run: gate.result.run, receipt: gate.result.receipt, worktree: gate.result.worktree, head };
|
|
777
|
+
};
|
|
778
|
+
const result = await waitFor("check", `the release check of ${j.candidate.sha.slice(0, 7)}`, async () => {
|
|
779
|
+
try {
|
|
780
|
+
return await checkProbe();
|
|
781
|
+
}
|
|
782
|
+
catch (error) {
|
|
783
|
+
// A failed check (or a changed candidate) ends the automatic road: the owner is told, and a person decides.
|
|
784
|
+
// A failed check is told, and stays checkable again on the same commit (a flake); anything else ends the road.
|
|
785
|
+
if (error instanceof Stop && j.auto?.state === "checking" && error.reason === "check-failed" && !error.build)
|
|
786
|
+
await rel?.auto?.checkFailed(j.task.id, error.message.replace(/\.$/, ""));
|
|
787
|
+
else if (error instanceof Stop && j.auto?.state === "checking" && error.reason !== "approval")
|
|
788
|
+
await rel?.auto?.giveUp(j.task.id, error.message.replace(/\.$/, ""));
|
|
789
|
+
throw error;
|
|
790
|
+
}
|
|
555
791
|
});
|
|
556
792
|
j.check = result;
|
|
557
793
|
say(` run ${result.run} passed on ${j.candidate.sha.slice(0, 7)}`);
|
|
794
|
+
if (j.auto?.state === "checking" && !j.authority) {
|
|
795
|
+
// The check ran beside CI (the gate didn't wait): it waits for every pull request check now. A failure ends the
|
|
796
|
+
// automatic road, and the person's road refuses it too (the approval step waits for CI first).
|
|
797
|
+
if (j.ciAhead === true) {
|
|
798
|
+
try {
|
|
799
|
+
await earlyChecks(adapters, j, (what, probe) => waitFor("gate", what, probe), { nothing: "Nothing was approved" });
|
|
800
|
+
await liftCiHold(false, "Nothing was approved");
|
|
801
|
+
}
|
|
802
|
+
catch (error) {
|
|
803
|
+
if (error instanceof Stop && error.reason === "ci-failed") {
|
|
804
|
+
j.auto = { state: "refused", why: error.message };
|
|
805
|
+
j.check = null;
|
|
806
|
+
j.step = "approval";
|
|
807
|
+
save();
|
|
808
|
+
await rel?.auto?.giveUp(j.task.id, error.message.replace(/\.$/, ""));
|
|
809
|
+
}
|
|
810
|
+
throw error;
|
|
811
|
+
}
|
|
812
|
+
j.ciAhead = false;
|
|
813
|
+
save();
|
|
814
|
+
}
|
|
815
|
+
// The release approval, sealed only now: the plane reads CI on the exact commit itself and proves the check.
|
|
816
|
+
const sealed = rel?.auto === undefined ? { ok: false, why: "this plane can't approve releases automatically" }
|
|
817
|
+
: await rel.auto.approve({ taskId: j.task.id, digest: j.task.scopeDigest, candidate: j.candidate.sha, run: result.run, receipt: result.receipt });
|
|
818
|
+
if (!sealed.ok)
|
|
819
|
+
await autoRefused(sealed.why);
|
|
820
|
+
else {
|
|
821
|
+
j.auto = { state: "approved", ci: sealed.ci.map(one => one.id) };
|
|
822
|
+
j.approval = { by: `${sealed.by} (automatic)`, at: sealed.at };
|
|
823
|
+
const view = await rel.authority(j.task.id);
|
|
824
|
+
if (view === null)
|
|
825
|
+
throw new Stop("release", `The automatic approval of ${j.task.id} opened no release record; a person approves it.`);
|
|
826
|
+
j.authority = { id: view.id, generation: view.generation };
|
|
827
|
+
j.landed = view.landed;
|
|
828
|
+
save();
|
|
829
|
+
await hold();
|
|
830
|
+
say(` Approved automatically: CI and the release check passed on ${j.candidate.sha.slice(0, 7)}`);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
558
833
|
advance("complete");
|
|
559
834
|
break;
|
|
560
835
|
}
|
|
@@ -596,7 +871,10 @@ export async function runRelease(options, adapters) {
|
|
|
596
871
|
if (runtime !== null && rel !== undefined && j.authority && epoch !== null) {
|
|
597
872
|
const authority = await rel.authority(j.task.id);
|
|
598
873
|
if (authority !== null && authority.state === "open") {
|
|
874
|
+
await personCiBeforeDeploy();
|
|
599
875
|
await watch("deploy");
|
|
876
|
+
// Every deploy reservation, whatever the journal saved: CI must still be green on the exact commit.
|
|
877
|
+
await autoCiHolds("the deploy");
|
|
600
878
|
const reserved = await rel.reserve({ authority: j.authority.id, kind: "deploy", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
601
879
|
if (!reserved.ok)
|
|
602
880
|
throw new Stop("release", `The deploy couldn't be reserved: ${reserved.why}`);
|
|
@@ -607,8 +885,13 @@ export async function runRelease(options, adapters) {
|
|
|
607
885
|
}
|
|
608
886
|
if (runtime === null) {
|
|
609
887
|
// The last moment a moved head restarts; then the deploy reservation freezes the candidate in the store.
|
|
888
|
+
// A person's release: every required check passed on the exact commit, read now (the approval may have come
|
|
889
|
+
// while CI ran). A check that failed stops here, before anything is deployed.
|
|
890
|
+
await personCiBeforeDeploy();
|
|
610
891
|
await watch("deploy");
|
|
611
892
|
await pinned();
|
|
893
|
+
// Every deploy issued, whatever the journal saved (a retry after a rollback included): CI still green.
|
|
894
|
+
await autoCiHolds("the deploy");
|
|
612
895
|
let operation, secret = "";
|
|
613
896
|
if (rel !== undefined && j.authority && epoch !== null) {
|
|
614
897
|
const reserved = await rel.reserve({ authority: j.authority.id, kind: "deploy", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
@@ -648,6 +931,7 @@ export async function runRelease(options, adapters) {
|
|
|
648
931
|
}
|
|
649
932
|
case "link": {
|
|
650
933
|
const runtime = j.deploy.runtime;
|
|
934
|
+
await autoCiHolds("the CLI names move");
|
|
651
935
|
if (await served(adapters, builtHead()) !== runtime)
|
|
652
936
|
throw new Stop("drift", `The console and worker no longer run ${runtime}; the CLI names were not moved.`);
|
|
653
937
|
if (j.links === undefined || j.links === null) {
|
|
@@ -670,7 +954,7 @@ export async function runRelease(options, adapters) {
|
|
|
670
954
|
case "ci": {
|
|
671
955
|
await waitFor("ci", `the pull request checks on ${j.candidate.sha.slice(0, 7)}`, async () => {
|
|
672
956
|
await pinned();
|
|
673
|
-
const verdict =
|
|
957
|
+
const verdict = prChecksVerdict(await adapters.github.checks(j.github, j.candidate.sha), j.candidate.sha, REQUIRED_PR_CHECKS, { ...autoCap(), workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
674
958
|
if (verdict.state === "failed")
|
|
675
959
|
await ciFailedAfterDeploy(`Pull request checks failed on ${j.candidate.sha.slice(0, 7)}: ${verdict.failed.join(", ")}`);
|
|
676
960
|
return verdict.state === "passed" ? true : null;
|
|
@@ -693,15 +977,19 @@ export async function runRelease(options, adapters) {
|
|
|
693
977
|
if (rel !== undefined && j.authority)
|
|
694
978
|
await identity("merge", false);
|
|
695
979
|
await pinned();
|
|
696
|
-
// A squash merge only reproduces the gated tree when main hasn't moved past the branch's base.
|
|
697
|
-
await adapters
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
throw new Stop("base-moved", `main moved to ${main.slice(0, 7)}, which ${j.branch} doesn't contain, so the merge wouldn't be the gated tree. Nothing was merged: update the branch and release the new commit with --new.`);
|
|
980
|
+
// A squash merge only reproduces the gated (and CI-tested) tree when main hasn't moved past the branch's base.
|
|
981
|
+
const main = await containsMain(adapters, j, "Nothing was merged");
|
|
982
|
+
// Every release merges only where GitHub itself refuses a branch behind main.
|
|
983
|
+
await mainProtection(adapters, j, j.auto?.state === "approved", "Nothing was merged");
|
|
701
984
|
// The checks again, at the moment of merging: a rerun that failed since must not merge.
|
|
702
|
-
const verdict =
|
|
985
|
+
const verdict = prChecksVerdict(await adapters.github.checks(j.github, j.candidate.sha), j.candidate.sha, REQUIRED_PR_CHECKS, { ...autoCap(), workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
703
986
|
if (verdict.state !== "passed")
|
|
704
987
|
await ciFailedAfterDeploy(`Pull request checks are no longer all passing on ${j.candidate.sha.slice(0, 7)}`);
|
|
988
|
+
// Main once more, from GitHub, the last thing before the merge is reserved: moved since the comparison, it
|
|
989
|
+
// stops here (finding 4). GitHub's own up-to-date rule refuses the rest of the window.
|
|
990
|
+
const now = await adapters.github.mainHead(j.github);
|
|
991
|
+
if (now !== main)
|
|
992
|
+
throw new Stop("base-moved", `The branch is behind main; update it and push. Main moved to ${now.slice(0, 7)} while ${j.branch} was being merged, so what merges wouldn't be the tree CI tested. Nothing was merged: release the updated commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
705
993
|
let operation;
|
|
706
994
|
if (rel !== undefined && j.authority && epoch !== null) {
|
|
707
995
|
const reserved = await rel.reserve({ authority: j.authority.id, kind: "merge", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
@@ -747,10 +1035,12 @@ export async function runRelease(options, adapters) {
|
|
|
747
1035
|
}
|
|
748
1036
|
case "tag": {
|
|
749
1037
|
const merge = j.merge;
|
|
1038
|
+
await publishHold();
|
|
750
1039
|
const at = await adapters.github.tag(j.github, j.tag);
|
|
751
1040
|
if (at !== null && at !== merge.sha)
|
|
752
1041
|
throw new Stop("conflict", `${j.tag} already points at ${at.slice(0, 7)}, not the merge ${merge.sha.slice(0, 7)}.`);
|
|
753
1042
|
if (at === null) {
|
|
1043
|
+
await autoCiHolds("the tag");
|
|
754
1044
|
intend("tag");
|
|
755
1045
|
await adapters.github.createTag(j.github, j.tag, merge.sha);
|
|
756
1046
|
}
|
|
@@ -759,6 +1049,7 @@ export async function runRelease(options, adapters) {
|
|
|
759
1049
|
break;
|
|
760
1050
|
}
|
|
761
1051
|
case "publish": {
|
|
1052
|
+
await publishHold();
|
|
762
1053
|
if (j.intents.publish === undefined)
|
|
763
1054
|
intend("publish");
|
|
764
1055
|
await waitFor("publish", `npm, the GitHub release and the tag's checks for ${j.tag}`, async () => {
|
|
@@ -776,6 +1067,7 @@ export async function runRelease(options, adapters) {
|
|
|
776
1067
|
break;
|
|
777
1068
|
}
|
|
778
1069
|
case "homebrew": {
|
|
1070
|
+
await publishHold();
|
|
779
1071
|
const brew = adapters.homebrew;
|
|
780
1072
|
let state = await brew.state(j.packageVersion);
|
|
781
1073
|
if (state.kind === "missing") {
|
|
@@ -825,6 +1117,12 @@ export async function runRelease(options, adapters) {
|
|
|
825
1117
|
}
|
|
826
1118
|
}
|
|
827
1119
|
say(`Released ${j.tag}: deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, published and on Homebrew.`);
|
|
1120
|
+
if (j.auto?.state === "approved") {
|
|
1121
|
+
try {
|
|
1122
|
+
await rel?.auto?.released(j.task.id, j.tag);
|
|
1123
|
+
}
|
|
1124
|
+
catch { /* the release is done; the note is best effort */ }
|
|
1125
|
+
}
|
|
828
1126
|
for (const line of landedWords(j.landed ?? []))
|
|
829
1127
|
say(line);
|
|
830
1128
|
return { ok: true, journal: j, file };
|
|
@@ -914,12 +1212,54 @@ async function deployed(adapters, candidate) {
|
|
|
914
1212
|
* running are named, and the gate waits for them; ones that never finish within the gate's limit stop here too. GitHub
|
|
915
1213
|
* not answering is waited out the same way, never taken for a pass. The ci step still checks again before merging.
|
|
916
1214
|
*/
|
|
917
|
-
|
|
1215
|
+
/**
|
|
1216
|
+
* The branch must contain main's head as GitHub's API answers now: CI tests the pull request's head commit,
|
|
1217
|
+
* so only a branch that already holds main merges into exactly the tested tree. Refused in plain words otherwise.
|
|
1218
|
+
*/
|
|
1219
|
+
async function containsMain(adapters, j, nothing) {
|
|
1220
|
+
// GitHub's own answers (eval-harness review 1, finding 4): main's head and the comparison are read from its API by
|
|
1221
|
+
// this coordinator, never from refs/remotes/origin/main in a checkout a build can write. The checkout is still synced
|
|
1222
|
+
// for what the gate reads from it later.
|
|
1223
|
+
await adapters.repo.sync(j.branch);
|
|
1224
|
+
const main = await adapters.github.mainHead(j.github);
|
|
1225
|
+
if (!/^[0-9a-f]{40}$/.test(main))
|
|
1226
|
+
throw new Stop("base-moved", `GitHub named no commit for main. ${nothing}.`);
|
|
1227
|
+
if (await adapters.github.contains(j.github, j.candidate.sha, main))
|
|
1228
|
+
return main;
|
|
1229
|
+
throw new Stop("base-moved", `The branch is behind main; update it and push. ${j.branch} at ${j.candidate.sha.slice(0, 7)} doesn't contain main's head ${main.slice(0, 7)}, so what merges wouldn't be the tree CI tested. ${nothing}: release the updated commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
1230
|
+
}
|
|
1231
|
+
/**
|
|
1232
|
+
* GitHub must refuse a branch behind main for every release (eval-harness review 1, finding 4). A person's approval
|
|
1233
|
+
* cannot make the final main read and merge atomic, so absent or unreadable protection stops that release too.
|
|
1234
|
+
*/
|
|
1235
|
+
async function mainProtection(adapters, j, auto, nothing) {
|
|
1236
|
+
let protection;
|
|
1237
|
+
try {
|
|
1238
|
+
protection = await adapters.github.upToDateRequired(j.github);
|
|
1239
|
+
}
|
|
1240
|
+
catch {
|
|
1241
|
+
protection = null;
|
|
1242
|
+
}
|
|
1243
|
+
// The git lanes main's protection doesn't name yet are said, never refused (PROTECTION_REPORT_ONLY_CHECKS).
|
|
1244
|
+
if (protection !== null && protection.notYetRequired.length > 0)
|
|
1245
|
+
adapters.say(` Note: main's protection doesn't require ${protection.notYetRequired.join(", ")} yet; add ${protection.notYetRequired.length === 1 ? "it" : "them"} to its required checks.`);
|
|
1246
|
+
if (protection?.effective === true)
|
|
1247
|
+
return;
|
|
1248
|
+
const lacking = protection === null ? "GitHub's branch protection for main couldn't be read" : `main's protection doesn't hold: ${protection.missing.join("; ")}`;
|
|
1249
|
+
throw new Stop("base-unprotected", `${lacking}, so this release can't merge${auto ? " automatically" : ""}. ${nothing}. Restore effective main protection and try again.`);
|
|
1250
|
+
}
|
|
1251
|
+
/**
|
|
1252
|
+
* Wait for every required pull request check on the candidate to pass; stop on one that failed. `pendingIsEnough` (the
|
|
1253
|
+
* automatic road's gate, pipeline-speed.md D2): checks still running are no reason to wait — returns false then, true
|
|
1254
|
+
* once they all passed. `nothing` says what hasn't happened yet when a check failed.
|
|
1255
|
+
*/
|
|
1256
|
+
async function earlyChecks(adapters, j, wait, options = {}) {
|
|
1257
|
+
const nothing = options.nothing ?? "Nothing was filed or sent for approval";
|
|
918
1258
|
const sha = j.candidate.sha.slice(0, 7);
|
|
919
1259
|
const required = REQUIRED_PR_CHECKS;
|
|
920
1260
|
let named = "", unread = false;
|
|
921
1261
|
try {
|
|
922
|
-
await wait(`the pull request checks on ${sha}`, async () => {
|
|
1262
|
+
return await wait(`the pull request checks on ${sha}`, async () => {
|
|
923
1263
|
let runs;
|
|
924
1264
|
try {
|
|
925
1265
|
runs = await adapters.github.checks(j.github, j.candidate.sha);
|
|
@@ -930,12 +1270,14 @@ async function earlyChecks(adapters, j, wait) {
|
|
|
930
1270
|
unread = true;
|
|
931
1271
|
return null;
|
|
932
1272
|
}
|
|
933
|
-
const verdict =
|
|
1273
|
+
const verdict = prChecksVerdict(runs.filter(run => required.includes(run.name)), j.candidate.sha, required, { workflow: REQUIRED_PR_CHECKS_WORKFLOW });
|
|
934
1274
|
if (verdict.state === "failed") {
|
|
935
|
-
throw new Stop("ci-failed", `Pull request checks already failed on ${sha}: ${verdict.failed.join(", ")}.
|
|
1275
|
+
throw new Stop("ci-failed", `Pull request checks already failed on ${sha}: ${verdict.failed.join(", ")}. ${nothing}. Fix the branch, then release the new commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\` (or rerun the failed job on GitHub and run this command again).`);
|
|
936
1276
|
}
|
|
937
1277
|
if (verdict.state === "passed")
|
|
938
1278
|
return true;
|
|
1279
|
+
if (options.pendingIsEnough === true)
|
|
1280
|
+
return false;
|
|
939
1281
|
const now = verdict.waiting.join(", ");
|
|
940
1282
|
if (now !== named) {
|
|
941
1283
|
adapters.say(` Pull request checks on ${sha} not finished yet: ${now}.`);
|
|
@@ -947,7 +1289,7 @@ async function earlyChecks(adapters, j, wait) {
|
|
|
947
1289
|
catch (error) {
|
|
948
1290
|
if (!(error instanceof Stop) || error.reason !== "timeout")
|
|
949
1291
|
throw error;
|
|
950
|
-
throw new Stop("ci-missing", `Pull request checks on ${sha} didn't all finish in time${named === "" ? (unread ? " (GitHub didn't answer)" : "") : `: ${named}`}.
|
|
1292
|
+
throw new Stop("ci-missing", `Pull request checks on ${sha} didn't all finish in time${named === "" ? (unread ? " (GitHub didn't answer)" : "") : `: ${named}`}. ${nothing}. Start or rerun them on GitHub, then run this command again.`);
|
|
951
1293
|
}
|
|
952
1294
|
}
|
|
953
1295
|
/** The network failures a release's waits ride out: git or gh timing out (or cut off) on the way to GitHub. */
|
|
@@ -962,3 +1304,136 @@ export function transientNetworkError(error) {
|
|
|
962
1304
|
function networkWords(error) {
|
|
963
1305
|
return TRANSIENT_NETWORK.exec(String(error?.message ?? error))?.[0] ?? "a network failure";
|
|
964
1306
|
}
|
|
1307
|
+
/**
|
|
1308
|
+
* The pull request's checks on exactly `head`, every attempt (the adapter asks GitHub for all of them, filter=all):
|
|
1309
|
+
* each reported attempt must be filed under `head` with a check-run id; a required check counts only from GitHub
|
|
1310
|
+
* Actions (an app that names a check alike can't stand in for it); attempts are grouped by the job they belong to — the
|
|
1311
|
+
* app, its check suite (a workflow run, whose re-runs stay in it) and the name — and every group is judged by its
|
|
1312
|
+
* latest attempt; every required check needs a group. With `auto` (an automatic approval), every other failed attempt on
|
|
1313
|
+
* the commit — an earlier attempt, or a job a newer run renamed or dropped — must be one GitHub's own job record shows ending before any workflow step ran (`beforeTests`: confirmed
|
|
1314
|
+
* infrastructure), and at most INFRA_RERUNS_PER_CANDIDATE of them on the commit; any other earlier failure — a test,
|
|
1315
|
+
* a typecheck, or one nothing explains — fails the verdict, so a person decides. The failures it tolerated are returned.
|
|
1316
|
+
*/
|
|
1317
|
+
export function prChecksVerdict(runs, head, required, options = {}) {
|
|
1318
|
+
const failed = [], waiting = [], failures = [];
|
|
1319
|
+
const valid = [];
|
|
1320
|
+
for (const run of runs) {
|
|
1321
|
+
if (run.head !== head) {
|
|
1322
|
+
failed.push(`${run.name} (reported for ${run.head == null ? "no commit" : run.head.slice(0, 7)}, not ${head.slice(0, 7)})`);
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
if (typeof run.id !== "number" || !Number.isSafeInteger(run.id) || run.id <= 0) {
|
|
1326
|
+
failed.push(`${run.name} (no check run id)`);
|
|
1327
|
+
continue;
|
|
1328
|
+
}
|
|
1329
|
+
if (required.includes(run.name) && run.app !== "github-actions") {
|
|
1330
|
+
failed.push(`${run.name} (reported by ${run.app ?? "an unknown app"}, not GitHub Actions)`);
|
|
1331
|
+
continue;
|
|
1332
|
+
}
|
|
1333
|
+
// A required name counts only from the workflow that defines it: a job of the same name in another workflow (or
|
|
1334
|
+
// one whose workflow GitHub doesn't name) fails the verdict rather than stand in for it.
|
|
1335
|
+
if (options.workflow !== undefined && required.includes(run.name) && !workflowIs(run.workflowPath, options.workflow)) {
|
|
1336
|
+
failed.push(`${run.name} (from ${run.workflowPath ?? "a workflow GitHub doesn't name"}, not ${options.workflow})`);
|
|
1337
|
+
continue;
|
|
1338
|
+
}
|
|
1339
|
+
if (run.app === "github-actions" && (typeof run.suite !== "number" || !Number.isSafeInteger(run.suite))) {
|
|
1340
|
+
failed.push(`${run.name} (no workflow run)`);
|
|
1341
|
+
continue;
|
|
1342
|
+
}
|
|
1343
|
+
valid.push(run);
|
|
1344
|
+
}
|
|
1345
|
+
// Supersession by trusted workflow identity (GitHub's workflow-run API, read by the adapter): a newer run of the same
|
|
1346
|
+
// workflow on this commit replaces the older ones — its jobs are the current ones. A suite whose workflow isn't known
|
|
1347
|
+
// is its own workflow (nothing supersedes it), and another app's suites stand on their own.
|
|
1348
|
+
const workflowOf = (run) => run.app === "github-actions" && typeof run.workflow === "number" ? `w:${run.workflow}` : `s:${run.app ?? ""}:${run.suite ?? ""}`;
|
|
1349
|
+
const current = new Map();
|
|
1350
|
+
for (const run of valid) {
|
|
1351
|
+
const key = workflowOf(run), suite = run.suite ?? null;
|
|
1352
|
+
const was = current.get(key);
|
|
1353
|
+
if (was === undefined || (suite !== null && (was === null || suite > was)))
|
|
1354
|
+
current.set(key, suite);
|
|
1355
|
+
}
|
|
1356
|
+
// Jobs: a workflow and a name, every attempt in every suite of it; judged by the latest attempt in the current suite.
|
|
1357
|
+
const groups = new Map();
|
|
1358
|
+
for (const run of valid) {
|
|
1359
|
+
const key = JSON.stringify([workflowOf(run), run.name]);
|
|
1360
|
+
groups.set(key, [...(groups.get(key) ?? []), run]);
|
|
1361
|
+
}
|
|
1362
|
+
const judged = new Set();
|
|
1363
|
+
// The attempt each current job stands on (its latest in the current run); every other attempt on the commit is
|
|
1364
|
+
// history, whatever its job is called now.
|
|
1365
|
+
const standing = new Set();
|
|
1366
|
+
for (const attempts of groups.values()) {
|
|
1367
|
+
const ordered = [...attempts].sort((a, b) => a.id - b.id);
|
|
1368
|
+
const now = current.get(workflowOf(ordered[0])) ?? null;
|
|
1369
|
+
const live = ordered.filter(one => (one.suite ?? null) === now);
|
|
1370
|
+
// A job only an older run had (the newer run no longer has it, or calls it something else) doesn't stand for a
|
|
1371
|
+
// required check; its failures are still history below.
|
|
1372
|
+
if (live.length === 0)
|
|
1373
|
+
continue;
|
|
1374
|
+
const latest = live.at(-1), name = latest.name;
|
|
1375
|
+
judged.add(name);
|
|
1376
|
+
standing.add(latest);
|
|
1377
|
+
if (latest.status !== "completed") {
|
|
1378
|
+
waiting.push(`${name} (${latest.status})`);
|
|
1379
|
+
continue;
|
|
1380
|
+
}
|
|
1381
|
+
if (latest.conclusion !== "success") {
|
|
1382
|
+
failed.push(`${name} (${latest.conclusion ?? "no conclusion"})`);
|
|
1383
|
+
continue;
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
// Every failed attempt on the commit that no current job stands on — an earlier attempt of a job, a job a newer run
|
|
1387
|
+
// renamed, dropped, or never ran again — is classified before anything is superseded (eval-harness review 1, finding
|
|
1388
|
+
// 1): a renamed or vanished job never erases a failure. An automatic approval tolerates only confirmed
|
|
1389
|
+
// infrastructure (GitHub's job record shows the runner failing before any workflow step ran), and waits out any
|
|
1390
|
+
// earlier attempt still running.
|
|
1391
|
+
const history = valid.filter(one => !standing.has(one));
|
|
1392
|
+
if (options.auto === true) {
|
|
1393
|
+
for (const one of history)
|
|
1394
|
+
if (one.status !== "completed")
|
|
1395
|
+
waiting.push(`${one.name} (an earlier attempt still running)`);
|
|
1396
|
+
}
|
|
1397
|
+
const flaked = history.filter(one => one.status === "completed" && one.conclusion !== "success");
|
|
1398
|
+
if (options.auto === true) {
|
|
1399
|
+
const unexplained = new Map();
|
|
1400
|
+
for (const one of flaked)
|
|
1401
|
+
if (one.beforeTests !== true)
|
|
1402
|
+
unexplained.set(one.name, (unexplained.get(one.name) ?? 0) + 1);
|
|
1403
|
+
for (const [name, count] of unexplained)
|
|
1404
|
+
failed.push(`${name} (failed ${count === 1 ? "once" : `${count} times`} on this commit, and GitHub doesn't show its runner failing before the tests ran)`);
|
|
1405
|
+
}
|
|
1406
|
+
const infrastructure = flaked.filter(one => one.beforeTests === true).length;
|
|
1407
|
+
failures.push(...flaked.map(one => ({ id: one.id, name: one.name, conclusion: one.conclusion })));
|
|
1408
|
+
if (options.auto === true && infrastructure > INFRA_RERUNS_PER_CANDIDATE)
|
|
1409
|
+
failed.push(`GitHub's runners failed ${infrastructure} times on this commit before the tests ran (at most ${INFRA_RERUNS_PER_CANDIDATE} are rerun)`);
|
|
1410
|
+
for (const name of required)
|
|
1411
|
+
if (!judged.has(name))
|
|
1412
|
+
waiting.push(`${name} (not started)`);
|
|
1413
|
+
if (failed.length > 0)
|
|
1414
|
+
return { state: "failed", failed };
|
|
1415
|
+
return waiting.length > 0 ? { state: "pending", waiting } : { state: "passed", failures };
|
|
1416
|
+
}
|
|
1417
|
+
/**
|
|
1418
|
+
* What a failed release check means and what happens next, by its class (check-failure.ts), in words. Only confirmed
|
|
1419
|
+
* infrastructure reruns on its own; nothing else is rerun to pass.
|
|
1420
|
+
*/
|
|
1421
|
+
export function checkFailedWords(input) {
|
|
1422
|
+
const short = input.sha.slice(0, 7), again = `\`toolroll release ${input.branch} --repo ${input.checkout} --new\``;
|
|
1423
|
+
switch (input.class) {
|
|
1424
|
+
case "deterministic":
|
|
1425
|
+
return `The Full release check of ${short} failed (run ${input.run}): ${input.why}. That is a failure in the code or its tests, so it isn't run again: fix the branch, then release the new commit with ${again}.`;
|
|
1426
|
+
case "agent-behaviour":
|
|
1427
|
+
return `The Full release check of ${short} failed (run ${input.run}): ${input.why}. Real-model journeys are counted, never rerun until they pass, so a person decides: fix the branch and release the new commit with ${again}, or approve this commit again with ${again}.`;
|
|
1428
|
+
case "infrastructure":
|
|
1429
|
+
return `The Full release check of ${short} failed (run ${input.run}): ${input.why}. It was already run again as often as a commit is (${INFRA_RERUNS_PER_CANDIDATE} times), so a person decides: approve this commit again with ${again}.`;
|
|
1430
|
+
case "unknown":
|
|
1431
|
+
return `The Full release check of ${short} failed (run ${input.run}) with no failure Toolroll could explain: ${input.why}. It isn't rerun to pass. To look into it, \`toolroll task regate ${input.taskId}\` runs it again on the same commit, but a pass then doesn't erase this failure: a person decides, and approves this commit again with ${again}, or fixes the branch and releases the new commit the same way.`;
|
|
1432
|
+
default:
|
|
1433
|
+
return `The Full release check of ${short} did not pass (run ${input.run}): ${input.why}. Fix the branch and release the new commit with ${again}.`;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
/** Whether GitHub's workflow-run `path` names exactly `file` (it may carry the ref it ran from, after an `@`). */
|
|
1437
|
+
export function workflowIs(path, file) {
|
|
1438
|
+
return typeof path === "string" && (path === file || path.startsWith(`${file}@`));
|
|
1439
|
+
}
|