toolroll 0.9.77 → 0.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/dist/agent-fence.js +18 -2
- package/dist/backup-ui.d.ts +9 -1
- package/dist/backup-ui.js +28 -3
- package/dist/backup.js +4 -3
- package/dist/browser/workspace.css +1 -1
- package/dist/browser/workspace.js +18 -18
- package/dist/browser-workspace.d.ts +2 -0
- package/dist/builder.d.ts +4 -3
- package/dist/builder.js +14 -8
- package/dist/check-failure.js +31 -3
- package/dist/check-report.d.ts +18 -3
- package/dist/check-report.js +67 -7
- package/dist/claim.d.ts +9 -0
- package/dist/claim.js +9 -1
- package/dist/contracts/console-api.d.ts +1 -1
- package/dist/contracts/console-api.js +1 -1
- package/dist/contracts/release-terms.d.ts +41 -0
- package/dist/contracts/release-terms.js +53 -0
- package/dist/controller-service.js +4 -1
- package/dist/deploy-rescue.d.ts +41 -0
- package/dist/deploy-rescue.js +94 -0
- package/dist/desktop-update.d.ts +1 -1
- package/dist/dispose.js +4 -3
- package/dist/evidence.d.ts +9 -0
- package/dist/evidence.js +15 -3
- package/dist/exec.d.ts +2 -0
- package/dist/exec.js +15 -1
- package/dist/guides.js +18 -8
- package/dist/latency-timeline.d.ts +46 -0
- package/dist/latency-timeline.js +120 -0
- package/dist/live-backup.d.ts +39 -0
- package/dist/live-backup.js +271 -0
- package/dist/operate.js +203 -16
- package/dist/planner.js +2 -2
- package/dist/release-adapters.d.ts +32 -1
- package/dist/release-adapters.js +99 -7
- package/dist/release-carry.d.ts +21 -1
- package/dist/release-carry.js +22 -0
- package/dist/release-cli.d.ts +3 -0
- package/dist/release-cli.js +74 -5
- package/dist/release-gate.d.ts +5 -0
- package/dist/release-gate.js +1 -1
- package/dist/release-recheck.js +3 -2
- package/dist/release.d.ts +37 -4
- package/dist/release.js +128 -43
- package/dist/runner-process.d.ts +67 -0
- package/dist/runner-process.js +219 -0
- package/dist/runner.d.ts +48 -6
- package/dist/runner.js +71 -8
- package/dist/scope.d.ts +19 -1
- package/dist/scope.js +19 -2
- package/dist/scout.js +2 -2
- package/dist/serve.js +11 -0
- package/dist/server/http.d.ts +2 -0
- package/dist/server/pages.js +7 -3
- package/dist/server/remote-hooks.d.ts +1 -1
- package/dist/server/remote-hooks.js +5 -3
- package/dist/server/render-results.d.ts +2 -0
- package/dist/server/render-results.js +1 -0
- package/dist/server/request-context.d.ts +2 -0
- package/dist/server/settings.js +22 -2
- package/dist/server/tasks.js +4 -1
- package/dist/storage-sweep.js +1 -1
- package/dist/store.d.ts +125 -2
- package/dist/store.js +250 -12
- package/dist/surface.js +3 -1
- package/dist/teardown-git.js +5 -5
- package/dist/toolroll-update.js +2 -2
- package/dist/up-role-child.js +6 -2
- package/dist/workspace-revision.js +3 -1
- package/package.json +1 -1
package/dist/release.js
CHANGED
|
@@ -48,6 +48,10 @@ export const REQUIRED_PR_CHECKS = [
|
|
|
48
48
|
// The git lanes (ci.yml job "git"): the declared floor (git-floor.ts) and Debian trixie's 2.47.
|
|
49
49
|
"git 2.41 · debian trixie",
|
|
50
50
|
"git 2.47 · debian trixie",
|
|
51
|
+
// The unit suite on macOS (ci.yml jobs "macos-shard" and "macos"; pipeline-speed.md, item 3a): tested on the candidate
|
|
52
|
+
// before it merges, no longer only on the tag after.
|
|
53
|
+
"node 22 · macos-latest",
|
|
54
|
+
"node 24 · macos-latest",
|
|
51
55
|
];
|
|
52
56
|
/** The workflow file every REQUIRED_PR_CHECKS job belongs to: a job of that name from any other workflow (a lookalike
|
|
53
57
|
* added beside it) never stands for it (Codex review round 1, inherited CI limitation). */
|
|
@@ -57,11 +61,14 @@ export const REQUIRED_PR_CHECKS_WORKFLOW = ".github/workflows/ci.yml";
|
|
|
57
61
|
* git lanes join main's protection once they have run there, so a protection without them is reported, never refused.
|
|
58
62
|
* A later release empties this list; from then on main's protection must name every REQUIRED_PR_CHECKS check.
|
|
59
63
|
*/
|
|
60
|
-
export const PROTECTION_REPORT_ONLY_CHECKS = ["git 2.41 · debian trixie", "git 2.47 · debian trixie"];
|
|
64
|
+
export const PROTECTION_REPORT_ONLY_CHECKS = ["git 2.41 · debian trixie", "git 2.47 · debian trixie", "node 22 · macos-latest", "node 24 · macos-latest"];
|
|
61
65
|
/** The checks main's protection must require at minimum: every required check that isn't report-only. */
|
|
62
66
|
export const PROTECTION_MINIMUM_CHECKS = REQUIRED_PR_CHECKS.filter(name => !PROTECTION_REPORT_ONLY_CHECKS.includes(name));
|
|
63
|
-
/** The checks a release tag
|
|
64
|
-
|
|
67
|
+
/** The checks a release's publication waits for on its tag (.github/workflows/publish.yml, pipeline-speed.md item 3):
|
|
68
|
+
* the publish job and the macOS legs it needs, each named. Nothing else reported on the merge commit (main's own CI push
|
|
69
|
+
* run) holds publication. */
|
|
70
|
+
export const RELEASE_TAG_CHECKS = ["publish", "tag macOS · node 22", "tag macOS · node 24"];
|
|
71
|
+
export const RELEASE_TAG_WORKFLOW = ".github/workflows/publish.yml";
|
|
65
72
|
export const RELEASE_STEPS = ["gate", "approval", "check", "complete", "quiet", "deploy", "link", "ci", "merge", "tree", "tag", "publish", "homebrew", "done"];
|
|
66
73
|
/** How long each waiting step may take, in minutes, before the release stops there (rerunning continues from it). */
|
|
67
74
|
export const RELEASE_LIMITS = { gate: 60, approval: 24 * 60, check: 120, quiet: 60, deploy: 30, ci: 60, publish: 45, homebrew: 45 };
|
|
@@ -85,6 +92,9 @@ class Stop extends Error {
|
|
|
85
92
|
this.build = build;
|
|
86
93
|
}
|
|
87
94
|
}
|
|
95
|
+
/** The tap change's checks failed: Homebrew's own state, never a stop of a published release. */
|
|
96
|
+
class HomebrewFailed extends Error {
|
|
97
|
+
}
|
|
88
98
|
/** The branch moved before the deploy reservation: carry the approval to the new head, or file it for a person. */
|
|
89
99
|
class Restart extends Error {
|
|
90
100
|
head;
|
|
@@ -180,6 +190,18 @@ export function checksVerdict(runs, required) {
|
|
|
180
190
|
return { state: "failed", failed };
|
|
181
191
|
return waiting.length > 0 ? { state: "pending", waiting } : { state: "passed" };
|
|
182
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* Publication's verdict on the tag commit (pipeline-speed.md, item 3): only RELEASE_TAG_CHECKS, from GitHub Actions
|
|
195
|
+
* runs of publish.yml bound to the tag's push, each by its newest attempt. Every other check on the merge commit — main's
|
|
196
|
+
* own CI run — is not waited for: the candidate's checks already passed on its exact head before the merge, and the
|
|
197
|
+
* merged tree was proved equal to it.
|
|
198
|
+
*/
|
|
199
|
+
export function tagChecksVerdict(runs) {
|
|
200
|
+
const names = RELEASE_TAG_CHECKS;
|
|
201
|
+
// Only GitHub Actions runs of publish.yml bound to the tag's own push (release-adapters.ts readCheckRuns): a check of
|
|
202
|
+
// the same name from another workflow, an unbound run or another app is never counted (security scan).
|
|
203
|
+
return checksVerdict(runs.filter(run => names.includes(run.name) && run.app === "github-actions" && workflowIs(run.workflowPath, RELEASE_TAG_WORKFLOW)), RELEASE_TAG_CHECKS);
|
|
204
|
+
}
|
|
183
205
|
/** Run the release from wherever its journal stopped. */
|
|
184
206
|
export async function runRelease(options, adapters) {
|
|
185
207
|
const file = journalFile(options.stateDir, options.branch);
|
|
@@ -254,6 +276,8 @@ export async function runRelease(options, adapters) {
|
|
|
254
276
|
say(`Released ${journal.tag} (${journal.candidate.sha.slice(0, 7)}) — nothing left to do.`);
|
|
255
277
|
return { ok: true, journal, file };
|
|
256
278
|
}
|
|
279
|
+
// Published, with Homebrew still to finish: this run finishes it, waiting for the tap's checks within its limit.
|
|
280
|
+
const finishingHomebrew = journal !== null && journal.step === "homebrew" && journal.published !== null && options.fresh !== true;
|
|
257
281
|
if (!await adapters.github.authenticated())
|
|
258
282
|
throw new Stop("github-auth", "GitHub CLI is not signed in: run `gh auth login`, then rerun.");
|
|
259
283
|
if (journal === null) {
|
|
@@ -297,6 +321,89 @@ export async function runRelease(options, adapters) {
|
|
|
297
321
|
throw new Stop("drift", `Pull request #${j.pr} no longer has ${j.candidate.sha.slice(0, 7)} at its head.`);
|
|
298
322
|
return pr;
|
|
299
323
|
};
|
|
324
|
+
// ---- Homebrew, finished apart from publication (pipeline-speed.md, P1) ------------------------------------------
|
|
325
|
+
let homebrewLeft = null;
|
|
326
|
+
/**
|
|
327
|
+
* One look at the tap: propose the formula change when it's missing, judge its checks (once, or within the Homebrew
|
|
328
|
+
* limit when `wait`), and merge exactly the proposed head once every check passed. A tap that already names a newer
|
|
329
|
+
* version is never overwritten, checked again right before the merge. Checksums come from npm's own tarball
|
|
330
|
+
* (release-adapters.ts propose), unchanged.
|
|
331
|
+
*/
|
|
332
|
+
const homebrewPass = async (wait) => {
|
|
333
|
+
const brew = adapters.homebrew;
|
|
334
|
+
let state = await brew.state(j.packageVersion);
|
|
335
|
+
if (state.kind === "newer")
|
|
336
|
+
return { state: "superseded", why: `the tap already names ${state.version}` };
|
|
337
|
+
if (state.kind === "current")
|
|
338
|
+
return { state: "current", why: null };
|
|
339
|
+
// A change this release already proposed is the only one it ever merges (security scan): another pull request for
|
|
340
|
+
// the same version, a moved head or a closed change is failed, never adopted or merged.
|
|
341
|
+
const journaled = j.homebrew?.pr != null && j.homebrew.head != null ? { pr: j.homebrew.pr, head: j.homebrew.head } : null;
|
|
342
|
+
if (journaled !== null && state.kind === "missing")
|
|
343
|
+
return { state: "failed", why: `${brew.repo} #${journaled.pr} was closed before it merged` };
|
|
344
|
+
if (journaled !== null && state.kind === "proposed" && (state.pr !== journaled.pr || state.head !== journaled.head)) {
|
|
345
|
+
return { state: "failed", why: `${brew.repo}'s change for ${j.packageVersion} is now #${state.pr} at ${state.head.slice(0, 7)}, not the #${journaled.pr} at ${journaled.head.slice(0, 7)} this release proposed` };
|
|
346
|
+
}
|
|
347
|
+
if (state.kind === "missing") {
|
|
348
|
+
intend("homebrew");
|
|
349
|
+
const proposed = await brew.propose(j.packageVersion);
|
|
350
|
+
state = { kind: "proposed", ...proposed };
|
|
351
|
+
}
|
|
352
|
+
const proposed = state;
|
|
353
|
+
j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false, state: "pending", why: null };
|
|
354
|
+
save();
|
|
355
|
+
// Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
|
|
356
|
+
// on a change, and only when it's still empty a poll later.
|
|
357
|
+
const checked = await brew.checked(proposed.head);
|
|
358
|
+
let emptyLooks = 0;
|
|
359
|
+
const probe = async () => {
|
|
360
|
+
// Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
|
|
361
|
+
// Two apps may name a check alike; each app's attempts stand for themselves.
|
|
362
|
+
const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
|
|
363
|
+
const statuses = await brew.statuses(proposed.head);
|
|
364
|
+
const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
|
|
365
|
+
const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
|
|
366
|
+
if (failed.length > 0)
|
|
367
|
+
throw new HomebrewFailed(`the tap change's checks failed: ${failed.join(", ")}`);
|
|
368
|
+
if (verdicts.some(one => one.state === "pending"))
|
|
369
|
+
return null;
|
|
370
|
+
// GitHub registers checks asynchronously; an empty set says nothing about success.
|
|
371
|
+
// A workflow is owed: only its own run counts, not another app's green check.
|
|
372
|
+
if (checked && !runs.some(run => run.app === "github-actions"))
|
|
373
|
+
return null;
|
|
374
|
+
if (runs.length === 0 && statuses.length === 0)
|
|
375
|
+
return ++emptyLooks >= 2 ? true : null;
|
|
376
|
+
return true;
|
|
377
|
+
};
|
|
378
|
+
try {
|
|
379
|
+
const passed = wait ? await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, probe) : await probe();
|
|
380
|
+
if (passed === null)
|
|
381
|
+
return { state: "pending", why: `${brew.repo} #${proposed.pr}'s checks are still running` };
|
|
382
|
+
}
|
|
383
|
+
catch (error) {
|
|
384
|
+
if (error instanceof HomebrewFailed)
|
|
385
|
+
return { state: "failed", why: `${brew.repo} #${proposed.pr}: ${error.message}` };
|
|
386
|
+
// Waiting ran out: still pending, never a stop of a published release.
|
|
387
|
+
if (error instanceof Stop && error.reason === "timeout")
|
|
388
|
+
return { state: "pending", why: error.message };
|
|
389
|
+
throw error;
|
|
390
|
+
}
|
|
391
|
+
// Ordering, once more right before the merge: a newer formula landed meanwhile is never overwritten.
|
|
392
|
+
const now = await brew.state(j.packageVersion);
|
|
393
|
+
if (now.kind === "newer")
|
|
394
|
+
return { state: "superseded", why: `the tap already names ${now.version}` };
|
|
395
|
+
if (now.kind === "missing")
|
|
396
|
+
return { state: "failed", why: `${brew.repo} #${proposed.pr} was closed before it merged` };
|
|
397
|
+
if (now.kind === "proposed" && now.head !== proposed.head)
|
|
398
|
+
return { state: "failed", why: `${brew.repo} #${proposed.pr} moved to ${now.head.slice(0, 7)} after its checks were read` };
|
|
399
|
+
if (now.kind === "current")
|
|
400
|
+
return { state: "current", why: null };
|
|
401
|
+
const verified = await brew.verified(proposed.pr, proposed.head, j.packageVersion);
|
|
402
|
+
if (!verified.ok)
|
|
403
|
+
return { state: "failed", why: verified.why };
|
|
404
|
+
await brew.merge(proposed.pr, proposed.head);
|
|
405
|
+
return { state: "current", why: null };
|
|
406
|
+
};
|
|
300
407
|
// ---- one approval per release (release-once.md) -------------------------------------------------------------
|
|
301
408
|
const rel = adapters.release;
|
|
302
409
|
let epoch = null;
|
|
@@ -601,7 +708,7 @@ export async function runRelease(options, adapters) {
|
|
|
601
708
|
throw new Stop("ci-failed", `A pull request check failed after the deploy, so ${j.candidate.sha.slice(0, 7)} needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
602
709
|
// A resumed release that already has an authority takes its lease and settles every open operation first.
|
|
603
710
|
await hold();
|
|
604
|
-
while (j.step !== "done") {
|
|
711
|
+
while (j.step !== "done" && homebrewLeft === null) {
|
|
605
712
|
step = j.step;
|
|
606
713
|
say(`• ${j.step}`);
|
|
607
714
|
try {
|
|
@@ -1055,7 +1162,7 @@ export async function runRelease(options, adapters) {
|
|
|
1055
1162
|
await waitFor("publish", `npm, the GitHub release and the tag's checks for ${j.tag}`, async () => {
|
|
1056
1163
|
if (await adapters.github.tag(j.github, j.tag) !== j.tagged.sha)
|
|
1057
1164
|
throw new Stop("conflict", `${j.tag} no longer points at the verified merge commit. Homebrew was not updated.`);
|
|
1058
|
-
const verdict =
|
|
1165
|
+
const verdict = tagChecksVerdict(await adapters.github.checks(j.github, j.tagged.sha, { tag: j.tag }));
|
|
1059
1166
|
if (verdict.state === "failed")
|
|
1060
1167
|
throw new Stop("publish-failed", `The release tag's checks failed: ${verdict.failed.join(", ")}. Homebrew was not updated.`);
|
|
1061
1168
|
if (verdict.state === "pending")
|
|
@@ -1068,43 +1175,16 @@ export async function runRelease(options, adapters) {
|
|
|
1068
1175
|
}
|
|
1069
1176
|
case "homebrew": {
|
|
1070
1177
|
await publishHold();
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false };
|
|
1081
|
-
save();
|
|
1082
|
-
// Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
|
|
1083
|
-
// on a change, and only when it's still empty a poll later.
|
|
1084
|
-
const checked = await brew.checked(proposed.head);
|
|
1085
|
-
let emptyLooks = 0;
|
|
1086
|
-
await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, async () => {
|
|
1087
|
-
// Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
|
|
1088
|
-
// Two apps may name a check alike; each app's attempts stand for themselves.
|
|
1089
|
-
const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
|
|
1090
|
-
const statuses = await brew.statuses(proposed.head);
|
|
1091
|
-
const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
|
|
1092
|
-
const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
|
|
1093
|
-
if (failed.length > 0)
|
|
1094
|
-
throw new Stop("homebrew-failed", `The Homebrew change's checks failed: ${failed.join(", ")}.`);
|
|
1095
|
-
if (verdicts.some(one => one.state === "pending"))
|
|
1096
|
-
return null;
|
|
1097
|
-
// GitHub registers checks asynchronously; an empty set says nothing about success.
|
|
1098
|
-
// A workflow is owed: only its own run counts, not another app's green check.
|
|
1099
|
-
if (checked && !runs.some(run => run.app === "github-actions"))
|
|
1100
|
-
return null;
|
|
1101
|
-
if (runs.length === 0 && statuses.length === 0)
|
|
1102
|
-
return ++emptyLooks >= 2 ? true : null;
|
|
1103
|
-
return true;
|
|
1104
|
-
});
|
|
1105
|
-
await brew.merge(proposed.pr, proposed.head);
|
|
1178
|
+
// Homebrew is finished apart from publication (pipeline-speed.md, P1): npm and the GitHub release are verified,
|
|
1179
|
+
// so the release is published. This run proposes the tap's change and reads its checks once; only a run that
|
|
1180
|
+
// resumes to finish Homebrew waits for them. Pending or failed, the state is kept and the run ends published.
|
|
1181
|
+
const progress = await homebrewPass(finishingHomebrew);
|
|
1182
|
+
j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: progress.state === "current" && j.homebrew?.head != null ? true : j.homebrew?.merged ?? false, state: progress.state, why: progress.why };
|
|
1183
|
+
save();
|
|
1184
|
+
if (progress.state === "pending" || progress.state === "failed") {
|
|
1185
|
+
homebrewLeft = { state: progress.state, why: progress.why };
|
|
1186
|
+
break;
|
|
1106
1187
|
}
|
|
1107
|
-
j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: true };
|
|
1108
1188
|
advance("done");
|
|
1109
1189
|
break;
|
|
1110
1190
|
}
|
|
@@ -1116,8 +1196,13 @@ export async function runRelease(options, adapters) {
|
|
|
1116
1196
|
await restart(error.head);
|
|
1117
1197
|
}
|
|
1118
1198
|
}
|
|
1119
|
-
|
|
1120
|
-
if (
|
|
1199
|
+
const left = homebrewLeft;
|
|
1200
|
+
if (left === null)
|
|
1201
|
+
say(`Released ${j.tag}: deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, published and ${j.homebrew?.state === "superseded" ? `left Homebrew as it is (${j.homebrew.why ?? "the tap names a newer version"})` : "on Homebrew"}.`);
|
|
1202
|
+
else
|
|
1203
|
+
say(`Published ${j.tag} (deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, on npm and GitHub); Homebrew ${left.state}${left.why === null ? "" : `: ${left.why}`}. Finish it with \`${resume}\`.`);
|
|
1204
|
+
// The automatic road's note, once: when this run published (a run that only finishes Homebrew already said it).
|
|
1205
|
+
if (j.auto?.state === "approved" && !finishingHomebrew) {
|
|
1121
1206
|
try {
|
|
1122
1207
|
await rel?.auto?.released(j.task.id, j.tag);
|
|
1123
1208
|
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { Runner } from "./runner.js";
|
|
2
|
+
export type RunnerProcessRole = "up" | "console" | "worker";
|
|
3
|
+
export type RunnerProcess = {
|
|
4
|
+
pid: number;
|
|
5
|
+
role: RunnerProcessRole;
|
|
6
|
+
started: string;
|
|
7
|
+
};
|
|
8
|
+
export type RunnerProcessRecord = {
|
|
9
|
+
v: 1;
|
|
10
|
+
runner: string;
|
|
11
|
+
host: string;
|
|
12
|
+
registeredAt: string;
|
|
13
|
+
/** A digest of the plane's credential hash for this registration: it names the incarnation, it proves nothing. */
|
|
14
|
+
credential: string;
|
|
15
|
+
processes: RunnerProcess[];
|
|
16
|
+
/** Authenticated with the plane's stored credential hash; legacy/forged records prove nothing. */
|
|
17
|
+
signature?: string;
|
|
18
|
+
};
|
|
19
|
+
export declare const RUNNER_PROCESS_DIR = "runner-processes";
|
|
20
|
+
/** The namespace is created before builds launch and denied by name, including in a shared state folder. */
|
|
21
|
+
export declare function ensureRunnerProcessFolder(folder: string): void;
|
|
22
|
+
export declare function sealRunnerProcessRecord(record: RunnerProcessRecord, credentialHash: string): RunnerProcessRecord;
|
|
23
|
+
/** Where a runner's record lives: beside the database, under a name derived from the runner's (never the name itself). */
|
|
24
|
+
export declare function runnerProcessFile(databaseFile: string, runner: string): string;
|
|
25
|
+
/** The incarnation's name in a record: a digest of the stored credential hash, so the record never holds that hash. */
|
|
26
|
+
export declare function credentialDigest(credentialHash: string): string;
|
|
27
|
+
/** `ps -o lstart=` (LC_ALL=C, TZ=UTC) as an ISO second, or null when it isn't exactly that shape. */
|
|
28
|
+
export declare function parseProcessStart(raw: string): string | null;
|
|
29
|
+
/** When a process started (whole seconds), or null when there is no such process or it can't be read. */
|
|
30
|
+
export declare function processStart(pid: number): string | null;
|
|
31
|
+
/** The probes a proof uses; real by default, supplied by tests. */
|
|
32
|
+
export type ProcessProbe = {
|
|
33
|
+
exists: (pid: number) => "absent" | "present" | "unknown";
|
|
34
|
+
start: (pid: number) => string | null;
|
|
35
|
+
};
|
|
36
|
+
export declare const realProcessProbe: ProcessProbe;
|
|
37
|
+
/** Proved gone: no process has this id, or the one that has it started at another time (the id was reused). */
|
|
38
|
+
export declare function processGone(entry: RunnerProcess, probe?: ProcessProbe): boolean;
|
|
39
|
+
/** A record read strictly: a regular private file of bounded size, every field its exact type, or null. */
|
|
40
|
+
export declare function readRunnerProcessRecord(file: string): RunnerProcessRecord | null;
|
|
41
|
+
/** Replace the record atomically: a private temp file in a private folder, renamed into place. */
|
|
42
|
+
export declare function writeRunnerProcessRecord(file: string, record: RunnerProcessRecord): void;
|
|
43
|
+
/**
|
|
44
|
+
* Whether the plane's current holder of a runner name is proved gone: the record is for exactly that registration,
|
|
45
|
+
* on this host, and every process it names has exited (by identity). `detail` says which processes, for the ledger.
|
|
46
|
+
*/
|
|
47
|
+
export declare function holderProvedGone(record: RunnerProcessRecord | null, holder: {
|
|
48
|
+
runner: Runner;
|
|
49
|
+
credentialHash: string;
|
|
50
|
+
}, host: string, probe?: ProcessProbe): {
|
|
51
|
+
gone: true;
|
|
52
|
+
detail: string;
|
|
53
|
+
} | {
|
|
54
|
+
gone: false;
|
|
55
|
+
why: string;
|
|
56
|
+
};
|
|
57
|
+
/** Re-read after observing exit: an up may append a child between the first read and its own exit. */
|
|
58
|
+
export declare function recordedHolderProvedGone(file: string, holder: {
|
|
59
|
+
runner: Runner;
|
|
60
|
+
credentialHash: string;
|
|
61
|
+
}, host: string, probe?: ProcessProbe): {
|
|
62
|
+
gone: true;
|
|
63
|
+
detail: string;
|
|
64
|
+
} | {
|
|
65
|
+
gone: false;
|
|
66
|
+
why: string;
|
|
67
|
+
};
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which processes hold a runner's credential, kept beside the plane's database so a later `up` can prove the previous
|
|
3
|
+
* holder is gone (docs: the 2026-10-29 deploy incident). `toolroll up` registers its runner, then records itself and
|
|
4
|
+
* every role child it hands the token to, each by process id AND start time, before the child receives the token.
|
|
5
|
+
*
|
|
6
|
+
* A fresh `up` whose runner name is still held (its heartbeat looks fresh, or its watch lease has not lapsed) asks
|
|
7
|
+
* this record whether that holder is gone: the record must be for exactly the registration the plane holds (name,
|
|
8
|
+
* host, registration time and credential), and every process in it must have exited, each proved by identity (no
|
|
9
|
+
* such process, or a process with that id started at a different time). Then the plane retires the stale holder in
|
|
10
|
+
* the same transaction that proved it (runner.ts registerRunnerIfIdle). Anything missing, unreadable or ambiguous
|
|
11
|
+
* proves nothing, and the door keeps refusing until the heartbeat ages out, exactly as before.
|
|
12
|
+
*
|
|
13
|
+
* The record is the plane's own state (the folder that holds the database). It grants nothing by itself: it can
|
|
14
|
+
* only let a name whose every recorded holder has exited be taken over early.
|
|
15
|
+
*/
|
|
16
|
+
import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto";
|
|
17
|
+
import { execFileSync } from "node:child_process";
|
|
18
|
+
import { closeSync, fstatSync, lstatSync, mkdirSync, openSync, readSync, realpathSync, renameSync, rmSync, writeFileSync, fsyncSync, constants } from "node:fs";
|
|
19
|
+
import { dirname, join, resolve } from "node:path";
|
|
20
|
+
export const RUNNER_PROCESS_DIR = "runner-processes";
|
|
21
|
+
/** The namespace is created before builds launch and denied by name, including in a shared state folder. */
|
|
22
|
+
export function ensureRunnerProcessFolder(folder) {
|
|
23
|
+
try {
|
|
24
|
+
mkdirSync(folder, { mode: 0o700 });
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
if (error.code !== "EEXIST")
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
const stat = lstatSync(folder);
|
|
31
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0
|
|
32
|
+
|| (process.getuid && stat.uid !== process.getuid()) || realpathSync(folder) !== resolve(folder))
|
|
33
|
+
throw Error("Untrusted runner process folder");
|
|
34
|
+
}
|
|
35
|
+
export function sealRunnerProcessRecord(record, credentialHash) {
|
|
36
|
+
const { signature: _signature, ...plain } = record;
|
|
37
|
+
return { ...plain, signature: createHmac("sha256", credentialHash).update(JSON.stringify(plain)).digest("hex") };
|
|
38
|
+
}
|
|
39
|
+
const MAX_RECORD = 64 * 1024;
|
|
40
|
+
const MAX_PROCESSES = 64;
|
|
41
|
+
const ROLES = new Set(["up", "console", "worker"]);
|
|
42
|
+
/** Where a runner's record lives: beside the database, under a name derived from the runner's (never the name itself). */
|
|
43
|
+
export function runnerProcessFile(databaseFile, runner) {
|
|
44
|
+
let database = databaseFile;
|
|
45
|
+
try {
|
|
46
|
+
database = realpathSync(databaseFile);
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
if (error.code !== "ENOENT")
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
return join(realpathSync(dirname(database)), RUNNER_PROCESS_DIR, `${createHash("sha256").update(runner, "utf8").digest("hex").slice(0, 32)}.json`);
|
|
53
|
+
}
|
|
54
|
+
/** The incarnation's name in a record: a digest of the stored credential hash, so the record never holds that hash. */
|
|
55
|
+
export function credentialDigest(credentialHash) {
|
|
56
|
+
return createHash("sha256").update(`runner-process:${credentialHash}`, "utf8").digest("hex").slice(0, 32);
|
|
57
|
+
}
|
|
58
|
+
const LSTART = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{1,2}) (\d{2}:\d{2}:\d{2}) (\d{4})$/;
|
|
59
|
+
/** `ps -o lstart=` (LC_ALL=C, TZ=UTC) as an ISO second, or null when it isn't exactly that shape. */
|
|
60
|
+
export function parseProcessStart(raw) {
|
|
61
|
+
const match = LSTART.exec(raw.trim().replace(/ +/g, " "));
|
|
62
|
+
if (match === null)
|
|
63
|
+
return null;
|
|
64
|
+
const utc = `${match[1]}, ${match[3].padStart(2, "0")} ${match[2]} ${match[5]} ${match[4]} GMT`;
|
|
65
|
+
const at = Date.parse(utc);
|
|
66
|
+
if (!Number.isFinite(at) || new Date(at).toUTCString() !== utc)
|
|
67
|
+
return null;
|
|
68
|
+
return new Date(at).toISOString();
|
|
69
|
+
}
|
|
70
|
+
/** When a process started (whole seconds), or null when there is no such process or it can't be read. */
|
|
71
|
+
export function processStart(pid) {
|
|
72
|
+
if (!Number.isSafeInteger(pid) || pid <= 1 || process.platform === "win32")
|
|
73
|
+
return null;
|
|
74
|
+
try {
|
|
75
|
+
const raw = execFileSync("/bin/ps", ["-p", String(pid), "-o", "lstart="], {
|
|
76
|
+
encoding: "utf8", env: { PATH: "/usr/bin:/bin", LC_ALL: "C", TZ: "UTC" }, stdio: ["ignore", "pipe", "ignore"],
|
|
77
|
+
maxBuffer: 1024, timeout: 2_000, killSignal: "SIGKILL",
|
|
78
|
+
});
|
|
79
|
+
return parseProcessStart(raw);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
export const realProcessProbe = {
|
|
86
|
+
exists: pid => {
|
|
87
|
+
try {
|
|
88
|
+
process.kill(pid, 0);
|
|
89
|
+
return "present";
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
const code = error.code;
|
|
93
|
+
return code === "ESRCH" ? "absent" : code === "EPERM" ? "present" : "unknown";
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
start: processStart,
|
|
97
|
+
};
|
|
98
|
+
/** Proved gone: no process has this id, or the one that has it started at another time (the id was reused). */
|
|
99
|
+
export function processGone(entry, probe = realProcessProbe) {
|
|
100
|
+
const exists = probe.exists(entry.pid);
|
|
101
|
+
if (exists === "absent")
|
|
102
|
+
return true;
|
|
103
|
+
if (exists === "unknown")
|
|
104
|
+
return false;
|
|
105
|
+
const started = probe.start(entry.pid);
|
|
106
|
+
if (started === null)
|
|
107
|
+
return probe.exists(entry.pid) === "absent";
|
|
108
|
+
return started !== entry.started;
|
|
109
|
+
}
|
|
110
|
+
/** A record read strictly: a regular private file of bounded size, every field its exact type, or null. */
|
|
111
|
+
export function readRunnerProcessRecord(file) {
|
|
112
|
+
let fd = null;
|
|
113
|
+
try {
|
|
114
|
+
const folder = lstatSync(dirname(file));
|
|
115
|
+
if (!folder.isDirectory() || folder.isSymbolicLink() || (folder.mode & 0o077) !== 0
|
|
116
|
+
|| (process.getuid && folder.uid !== process.getuid()) || realpathSync(dirname(file)) !== resolve(dirname(file)))
|
|
117
|
+
return null;
|
|
118
|
+
const link = lstatSync(file);
|
|
119
|
+
if (!link.isFile() || link.isSymbolicLink() || link.size > MAX_RECORD)
|
|
120
|
+
return null;
|
|
121
|
+
fd = openSync(file, constants.O_RDONLY | constants.O_NONBLOCK | (constants.O_NOFOLLOW ?? 0));
|
|
122
|
+
const stat = fstatSync(fd);
|
|
123
|
+
if (!stat.isFile() || stat.size > MAX_RECORD || stat.ino !== link.ino || stat.dev !== link.dev || (stat.mode & 0o077) !== 0)
|
|
124
|
+
return null;
|
|
125
|
+
if (typeof process.getuid === "function" && stat.uid !== process.getuid())
|
|
126
|
+
return null;
|
|
127
|
+
const buffer = Buffer.alloc(stat.size);
|
|
128
|
+
let read = 0;
|
|
129
|
+
while (read < stat.size) {
|
|
130
|
+
const n = readSync(fd, buffer, read, stat.size - read, read);
|
|
131
|
+
if (n === 0)
|
|
132
|
+
break;
|
|
133
|
+
read += n;
|
|
134
|
+
}
|
|
135
|
+
return parseRecord(JSON.parse(buffer.subarray(0, read).toString("utf8")));
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
if (fd !== null)
|
|
142
|
+
closeSync(fd);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
|
146
|
+
function parseRecord(value) {
|
|
147
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
148
|
+
return null;
|
|
149
|
+
const v = value;
|
|
150
|
+
const text = (one, max = 512) => typeof one === "string" && one !== "" && one.length <= max && !one.includes("\0");
|
|
151
|
+
if (v["v"] !== 1 || !text(v["runner"]) || !text(v["host"]) || !text(v["registeredAt"]) || !ISO.test(v["registeredAt"]) || !text(v["credential"], 64))
|
|
152
|
+
return null;
|
|
153
|
+
const list = v["processes"];
|
|
154
|
+
if (!Array.isArray(list) || list.length === 0 || list.length > MAX_PROCESSES)
|
|
155
|
+
return null;
|
|
156
|
+
const processes = [];
|
|
157
|
+
for (const one of list) {
|
|
158
|
+
if (typeof one !== "object" || one === null)
|
|
159
|
+
return null;
|
|
160
|
+
const p = one;
|
|
161
|
+
if (!Number.isSafeInteger(p["pid"]) || p["pid"] <= 1 || !ROLES.has(p["role"]) || !text(p["started"], 32) || !ISO.test(p["started"]))
|
|
162
|
+
return null;
|
|
163
|
+
processes.push({ pid: p["pid"], role: p["role"], started: p["started"] });
|
|
164
|
+
}
|
|
165
|
+
return { v: 1, runner: v["runner"], host: v["host"], registeredAt: v["registeredAt"], credential: v["credential"], processes,
|
|
166
|
+
...(typeof v["signature"] === "string" ? { signature: v["signature"] } : {}) };
|
|
167
|
+
}
|
|
168
|
+
/** Replace the record atomically: a private temp file in a private folder, renamed into place. */
|
|
169
|
+
export function writeRunnerProcessRecord(file, record) {
|
|
170
|
+
ensureRunnerProcessFolder(dirname(file));
|
|
171
|
+
const temp = `${file}.${randomUUID()}.tmp`;
|
|
172
|
+
try {
|
|
173
|
+
const fd = openSync(temp, "wx", 0o600);
|
|
174
|
+
try {
|
|
175
|
+
writeFileSync(fd, JSON.stringify(record));
|
|
176
|
+
fsyncSync(fd);
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
closeSync(fd);
|
|
180
|
+
}
|
|
181
|
+
renameSync(temp, file);
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
rmSync(temp, { force: true });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Whether the plane's current holder of a runner name is proved gone: the record is for exactly that registration,
|
|
189
|
+
* on this host, and every process it names has exited (by identity). `detail` says which processes, for the ledger.
|
|
190
|
+
*/
|
|
191
|
+
export function holderProvedGone(record, holder, host, probe = realProcessProbe) {
|
|
192
|
+
if (record === null)
|
|
193
|
+
return { gone: false, why: "no readable record of its processes" };
|
|
194
|
+
const expected = sealRunnerProcessRecord(record, holder.credentialHash).signature;
|
|
195
|
+
if (typeof record.signature !== "string" || !/^[a-f0-9]{64}$/.test(record.signature)
|
|
196
|
+
|| !timingSafeEqual(Buffer.from(record.signature, "hex"), Buffer.from(expected, "hex")))
|
|
197
|
+
return { gone: false, why: "its process record is not authenticated" };
|
|
198
|
+
if (record.runner !== holder.runner.name || record.host !== holder.runner.host || record.host !== host)
|
|
199
|
+
return { gone: false, why: "its record is for another worker or machine" };
|
|
200
|
+
if (record.registeredAt !== holder.runner.registeredAt || record.credential !== credentialDigest(holder.credentialHash))
|
|
201
|
+
return { gone: false, why: "its record is from an earlier start" };
|
|
202
|
+
if (!record.processes.some(one => one.role === "up"))
|
|
203
|
+
return { gone: false, why: "its record names no `up` process" };
|
|
204
|
+
const left = record.processes.filter(one => !processGone(one, probe));
|
|
205
|
+
if (left.length > 0)
|
|
206
|
+
return { gone: false, why: `process ${left.map(one => one.pid).join(", ")} is still running` };
|
|
207
|
+
return { gone: true, detail: `${record.runner}: process ${record.processes.map(one => `${one.pid} (${one.role})`).join(", ")} proved gone` };
|
|
208
|
+
}
|
|
209
|
+
/** Re-read after observing exit: an up may append a child between the first read and its own exit. */
|
|
210
|
+
export function recordedHolderProvedGone(file, holder, host, probe = realProcessProbe) {
|
|
211
|
+
const record = readRunnerProcessRecord(file);
|
|
212
|
+
const proof = holderProvedGone(record, holder, host, probe);
|
|
213
|
+
if (!proof.gone)
|
|
214
|
+
return proof;
|
|
215
|
+
const after = readRunnerProcessRecord(file);
|
|
216
|
+
if (after === null || JSON.stringify(after) !== JSON.stringify(record))
|
|
217
|
+
return { gone: false, why: "its process record changed during the exit proof" };
|
|
218
|
+
return proof;
|
|
219
|
+
}
|
package/dist/runner.d.ts
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
import { type AcpProviderId } from "./acp-agents.js";
|
|
23
23
|
import type { AcpAttestation } from "./store.js";
|
|
24
|
-
import type
|
|
24
|
+
import { type Store, type Mutation } from "./store.js";
|
|
25
25
|
import { type ProviderId } from "./provider.js";
|
|
26
26
|
import type { ReadinessObservation } from "./phase-routing.js";
|
|
27
27
|
/** A runner's repo binding, in the same canonical form task filing uses —
|
|
@@ -59,11 +59,18 @@ export declare const RUNNER_TOKEN_MS: number;
|
|
|
59
59
|
/**
|
|
60
60
|
* How long a runner may go quiet before it is presumed gone.
|
|
61
61
|
*
|
|
62
|
-
*
|
|
63
|
-
* machine dead
|
|
64
|
-
*
|
|
62
|
+
* Four heartbeats (RUNNER_HEARTBEAT_MS): one slow check-in never declares a busy
|
|
63
|
+
* machine dead, and a machine that really is gone holds its name for a minute,
|
|
64
|
+
* not three (docs/plans/actors-lessons.md). A clean stop doesn't wait even that
|
|
65
|
+
* long: a draining or stopped holder never blocks the next start
|
|
66
|
+
* (registerRunnerIfIdle).
|
|
65
67
|
*/
|
|
66
|
-
export declare const DEFAULT_LIVENESS_MS
|
|
68
|
+
export declare const DEFAULT_LIVENESS_MS = 60000;
|
|
69
|
+
/** How often a live runner says so: `up`'s runner heartbeat and every watch lease renewal. */
|
|
70
|
+
export declare const RUNNER_HEARTBEAT_MS = 15000;
|
|
71
|
+
/** The longest a watch lease may be taken or renewed for (durable-actors' MAX_HOST_LEASE_DURATION_MS): a crashed
|
|
72
|
+
* holder's projects are free within a minute. The store clamps every ttl to it. */
|
|
73
|
+
export { MAX_WATCH_LEASE_MS } from "./store.js";
|
|
67
74
|
export type RegisterOptions = {
|
|
68
75
|
name: string;
|
|
69
76
|
host: string;
|
|
@@ -140,13 +147,35 @@ export declare function normalizeRunnerName(raw: string): string;
|
|
|
140
147
|
* live claim owns), reclaims, and rotates. Two racers serialize: one wins,
|
|
141
148
|
* the other gets the typed refusal.
|
|
142
149
|
*/
|
|
143
|
-
export declare function registerRunnerIfIdle(store: Store, options: RegisterOptions
|
|
150
|
+
export declare function registerRunnerIfIdle(store: Store, options: RegisterOptions & {
|
|
151
|
+
/**
|
|
152
|
+
* Whether the name's current holder is proved gone (runner-process.ts): every process it handed its credential to
|
|
153
|
+
* has exited, verified by identity. Asked only while the holder still blocks the name, inside this transaction;
|
|
154
|
+
* a proof retires that holder and lapses its watch leases here, so a stale heartbeat never outlives its processes.
|
|
155
|
+
*/
|
|
156
|
+
previousGone?: (holder: {
|
|
157
|
+
runner: Runner;
|
|
158
|
+
credentialHash: string;
|
|
159
|
+
}) => {
|
|
160
|
+
gone: true;
|
|
161
|
+
detail: string;
|
|
162
|
+
} | {
|
|
163
|
+
gone: false;
|
|
164
|
+
why: string;
|
|
165
|
+
};
|
|
166
|
+
}): (Registration & {
|
|
144
167
|
ok: true;
|
|
145
168
|
recoveredRuns: number;
|
|
169
|
+
signedOff: string | null;
|
|
146
170
|
}) | {
|
|
147
171
|
ok: false;
|
|
148
172
|
reason: "runner-alive" | "watch-live";
|
|
149
173
|
detail: string;
|
|
174
|
+
} | {
|
|
175
|
+
ok: false;
|
|
176
|
+
reason: "runner-draining";
|
|
177
|
+
detail: string;
|
|
178
|
+
until: string;
|
|
150
179
|
};
|
|
151
180
|
/**
|
|
152
181
|
* The authenticated lease door (arc 2 findings 15/26): credential verified
|
|
@@ -280,3 +309,16 @@ export declare function reportProviderReadinessAuthed(store: Store, args: {
|
|
|
280
309
|
token: string;
|
|
281
310
|
observations: readonly Omit<ReadinessObservation, "runner" | "observedAt">[];
|
|
282
311
|
}, now: Date): AuthResult;
|
|
312
|
+
/** A worker's lifecycle row (service_lifecycle). */
|
|
313
|
+
export declare const workerComponent: (name: string) => string;
|
|
314
|
+
/**
|
|
315
|
+
* The stop begins (durable-actors' draining report), fenced by this holder's credential: from now this runner takes
|
|
316
|
+
* no new work (claim.ts acquire refuses). `until` is a schedule, never proof that the holder exited.
|
|
317
|
+
*/
|
|
318
|
+
export declare function drainRunnerIfCurrent(store: Store, name: string, token: string, now: Date, until: Date): boolean;
|
|
319
|
+
/** A runner's drain as its own lifecycle row reports it (only the current registration's row counts): null when it
|
|
320
|
+
* isn't draining; ended once its bound passed or it reported stopped. */
|
|
321
|
+
export declare function drainOf(store: Store, runner: Runner, now: Date): {
|
|
322
|
+
until: string;
|
|
323
|
+
ended: boolean;
|
|
324
|
+
} | null;
|