toolroll 0.9.77 → 0.9.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/dist/agent-fence.js +18 -2
  3. package/dist/backup-ui.d.ts +9 -1
  4. package/dist/backup-ui.js +28 -3
  5. package/dist/backup.js +4 -3
  6. package/dist/browser/workspace.css +1 -1
  7. package/dist/browser/workspace.js +18 -18
  8. package/dist/browser-workspace.d.ts +2 -0
  9. package/dist/builder.d.ts +4 -3
  10. package/dist/builder.js +14 -8
  11. package/dist/check-failure.js +31 -3
  12. package/dist/check-report.d.ts +18 -3
  13. package/dist/check-report.js +67 -7
  14. package/dist/claim.d.ts +9 -0
  15. package/dist/claim.js +9 -1
  16. package/dist/contracts/console-api.d.ts +1 -1
  17. package/dist/contracts/console-api.js +1 -1
  18. package/dist/contracts/release-terms.d.ts +41 -0
  19. package/dist/contracts/release-terms.js +53 -0
  20. package/dist/controller-service.js +4 -1
  21. package/dist/deploy-rescue.d.ts +41 -0
  22. package/dist/deploy-rescue.js +94 -0
  23. package/dist/desktop-update.d.ts +1 -1
  24. package/dist/dispose.js +4 -3
  25. package/dist/evidence.d.ts +9 -0
  26. package/dist/evidence.js +15 -3
  27. package/dist/exec.d.ts +2 -0
  28. package/dist/exec.js +15 -1
  29. package/dist/guides.js +18 -8
  30. package/dist/latency-timeline.d.ts +46 -0
  31. package/dist/latency-timeline.js +120 -0
  32. package/dist/live-backup.d.ts +39 -0
  33. package/dist/live-backup.js +271 -0
  34. package/dist/operate.js +203 -16
  35. package/dist/planner.js +2 -2
  36. package/dist/release-adapters.d.ts +32 -1
  37. package/dist/release-adapters.js +99 -7
  38. package/dist/release-carry.d.ts +21 -1
  39. package/dist/release-carry.js +22 -0
  40. package/dist/release-cli.d.ts +3 -0
  41. package/dist/release-cli.js +74 -5
  42. package/dist/release-gate.d.ts +5 -0
  43. package/dist/release-gate.js +1 -1
  44. package/dist/release-recheck.js +3 -2
  45. package/dist/release.d.ts +37 -4
  46. package/dist/release.js +128 -43
  47. package/dist/runner-process.d.ts +67 -0
  48. package/dist/runner-process.js +219 -0
  49. package/dist/runner.d.ts +48 -6
  50. package/dist/runner.js +71 -8
  51. package/dist/scope.d.ts +19 -1
  52. package/dist/scope.js +19 -2
  53. package/dist/scout.js +2 -2
  54. package/dist/serve.js +11 -0
  55. package/dist/server/http.d.ts +2 -0
  56. package/dist/server/pages.js +7 -3
  57. package/dist/server/remote-hooks.d.ts +1 -1
  58. package/dist/server/remote-hooks.js +5 -3
  59. package/dist/server/render-results.d.ts +2 -0
  60. package/dist/server/render-results.js +1 -0
  61. package/dist/server/request-context.d.ts +2 -0
  62. package/dist/server/settings.js +22 -2
  63. package/dist/server/tasks.js +4 -1
  64. package/dist/storage-sweep.js +1 -1
  65. package/dist/store.d.ts +125 -2
  66. package/dist/store.js +250 -12
  67. package/dist/surface.js +3 -1
  68. package/dist/teardown-git.js +5 -5
  69. package/dist/toolroll-update.js +2 -2
  70. package/dist/up-role-child.js +6 -2
  71. package/dist/workspace-revision.js +3 -1
  72. package/package.json +1 -1
package/dist/release.js CHANGED
@@ -48,6 +48,10 @@ export const REQUIRED_PR_CHECKS = [
48
48
  // The git lanes (ci.yml job "git"): the declared floor (git-floor.ts) and Debian trixie's 2.47.
49
49
  "git 2.41 · debian trixie",
50
50
  "git 2.47 · debian trixie",
51
+ // The unit suite on macOS (ci.yml jobs "macos-shard" and "macos"; pipeline-speed.md, item 3a): tested on the candidate
52
+ // before it merges, no longer only on the tag after.
53
+ "node 22 · macos-latest",
54
+ "node 24 · macos-latest",
51
55
  ];
52
56
  /** The workflow file every REQUIRED_PR_CHECKS job belongs to: a job of that name from any other workflow (a lookalike
53
57
  * added beside it) never stands for it (Codex review round 1, inherited CI limitation). */
@@ -57,11 +61,14 @@ export const REQUIRED_PR_CHECKS_WORKFLOW = ".github/workflows/ci.yml";
57
61
  * git lanes join main's protection once they have run there, so a protection without them is reported, never refused.
58
62
  * A later release empties this list; from then on main's protection must name every REQUIRED_PR_CHECKS check.
59
63
  */
60
- export const PROTECTION_REPORT_ONLY_CHECKS = ["git 2.41 · debian trixie", "git 2.47 · debian trixie"];
64
+ export const PROTECTION_REPORT_ONLY_CHECKS = ["git 2.41 · debian trixie", "git 2.47 · debian trixie", "node 22 · macos-latest", "node 24 · macos-latest"];
61
65
  /** The checks main's protection must require at minimum: every required check that isn't report-only. */
62
66
  export const PROTECTION_MINIMUM_CHECKS = REQUIRED_PR_CHECKS.filter(name => !PROTECTION_REPORT_ONLY_CHECKS.includes(name));
63
- /** The checks a release tag runs (.github/workflows/publish.yml), awaited before Homebrew moves. */
64
- export const RELEASE_TAG_CHECKS = ["node 22 · macos-latest", "node 24 · macos-latest", "publish"];
67
+ /** The checks a release's publication waits for on its tag (.github/workflows/publish.yml, pipeline-speed.md item 3):
68
+ * the publish job and the macOS legs it needs, each named. Nothing else reported on the merge commit (main's own CI push
69
+ * run) holds publication. */
70
+ export const RELEASE_TAG_CHECKS = ["publish", "tag macOS · node 22", "tag macOS · node 24"];
71
+ export const RELEASE_TAG_WORKFLOW = ".github/workflows/publish.yml";
65
72
  export const RELEASE_STEPS = ["gate", "approval", "check", "complete", "quiet", "deploy", "link", "ci", "merge", "tree", "tag", "publish", "homebrew", "done"];
66
73
  /** How long each waiting step may take, in minutes, before the release stops there (rerunning continues from it). */
67
74
  export const RELEASE_LIMITS = { gate: 60, approval: 24 * 60, check: 120, quiet: 60, deploy: 30, ci: 60, publish: 45, homebrew: 45 };
@@ -85,6 +92,9 @@ class Stop extends Error {
85
92
  this.build = build;
86
93
  }
87
94
  }
95
+ /** The tap change's checks failed: Homebrew's own state, never a stop of a published release. */
96
+ class HomebrewFailed extends Error {
97
+ }
88
98
  /** The branch moved before the deploy reservation: carry the approval to the new head, or file it for a person. */
89
99
  class Restart extends Error {
90
100
  head;
@@ -180,6 +190,18 @@ export function checksVerdict(runs, required) {
180
190
  return { state: "failed", failed };
181
191
  return waiting.length > 0 ? { state: "pending", waiting } : { state: "passed" };
182
192
  }
193
+ /**
194
+ * Publication's verdict on the tag commit (pipeline-speed.md, item 3): only RELEASE_TAG_CHECKS, from GitHub Actions
195
+ * runs of publish.yml bound to the tag's push, each by its newest attempt. Every other check on the merge commit — main's
196
+ * own CI run — is not waited for: the candidate's checks already passed on its exact head before the merge, and the
197
+ * merged tree was proved equal to it.
198
+ */
199
+ export function tagChecksVerdict(runs) {
200
+ const names = RELEASE_TAG_CHECKS;
201
+ // Only GitHub Actions runs of publish.yml bound to the tag's own push (release-adapters.ts readCheckRuns): a check of
202
+ // the same name from another workflow, an unbound run or another app is never counted (security scan).
203
+ return checksVerdict(runs.filter(run => names.includes(run.name) && run.app === "github-actions" && workflowIs(run.workflowPath, RELEASE_TAG_WORKFLOW)), RELEASE_TAG_CHECKS);
204
+ }
183
205
  /** Run the release from wherever its journal stopped. */
184
206
  export async function runRelease(options, adapters) {
185
207
  const file = journalFile(options.stateDir, options.branch);
@@ -254,6 +276,8 @@ export async function runRelease(options, adapters) {
254
276
  say(`Released ${journal.tag} (${journal.candidate.sha.slice(0, 7)}) — nothing left to do.`);
255
277
  return { ok: true, journal, file };
256
278
  }
279
+ // Published, with Homebrew still to finish: this run finishes it, waiting for the tap's checks within its limit.
280
+ const finishingHomebrew = journal !== null && journal.step === "homebrew" && journal.published !== null && options.fresh !== true;
257
281
  if (!await adapters.github.authenticated())
258
282
  throw new Stop("github-auth", "GitHub CLI is not signed in: run `gh auth login`, then rerun.");
259
283
  if (journal === null) {
@@ -297,6 +321,89 @@ export async function runRelease(options, adapters) {
297
321
  throw new Stop("drift", `Pull request #${j.pr} no longer has ${j.candidate.sha.slice(0, 7)} at its head.`);
298
322
  return pr;
299
323
  };
324
+ // ---- Homebrew, finished apart from publication (pipeline-speed.md, P1) ------------------------------------------
325
+ let homebrewLeft = null;
326
+ /**
327
+ * One look at the tap: propose the formula change when it's missing, judge its checks (once, or within the Homebrew
328
+ * limit when `wait`), and merge exactly the proposed head once every check passed. A tap that already names a newer
329
+ * version is never overwritten, checked again right before the merge. Checksums come from npm's own tarball
330
+ * (release-adapters.ts propose), unchanged.
331
+ */
332
+ const homebrewPass = async (wait) => {
333
+ const brew = adapters.homebrew;
334
+ let state = await brew.state(j.packageVersion);
335
+ if (state.kind === "newer")
336
+ return { state: "superseded", why: `the tap already names ${state.version}` };
337
+ if (state.kind === "current")
338
+ return { state: "current", why: null };
339
+ // A change this release already proposed is the only one it ever merges (security scan): another pull request for
340
+ // the same version, a moved head or a closed change is failed, never adopted or merged.
341
+ const journaled = j.homebrew?.pr != null && j.homebrew.head != null ? { pr: j.homebrew.pr, head: j.homebrew.head } : null;
342
+ if (journaled !== null && state.kind === "missing")
343
+ return { state: "failed", why: `${brew.repo} #${journaled.pr} was closed before it merged` };
344
+ if (journaled !== null && state.kind === "proposed" && (state.pr !== journaled.pr || state.head !== journaled.head)) {
345
+ return { state: "failed", why: `${brew.repo}'s change for ${j.packageVersion} is now #${state.pr} at ${state.head.slice(0, 7)}, not the #${journaled.pr} at ${journaled.head.slice(0, 7)} this release proposed` };
346
+ }
347
+ if (state.kind === "missing") {
348
+ intend("homebrew");
349
+ const proposed = await brew.propose(j.packageVersion);
350
+ state = { kind: "proposed", ...proposed };
351
+ }
352
+ const proposed = state;
353
+ j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false, state: "pending", why: null };
354
+ save();
355
+ // Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
356
+ // on a change, and only when it's still empty a poll later.
357
+ const checked = await brew.checked(proposed.head);
358
+ let emptyLooks = 0;
359
+ const probe = async () => {
360
+ // Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
361
+ // Two apps may name a check alike; each app's attempts stand for themselves.
362
+ const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
363
+ const statuses = await brew.statuses(proposed.head);
364
+ const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
365
+ const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
366
+ if (failed.length > 0)
367
+ throw new HomebrewFailed(`the tap change's checks failed: ${failed.join(", ")}`);
368
+ if (verdicts.some(one => one.state === "pending"))
369
+ return null;
370
+ // GitHub registers checks asynchronously; an empty set says nothing about success.
371
+ // A workflow is owed: only its own run counts, not another app's green check.
372
+ if (checked && !runs.some(run => run.app === "github-actions"))
373
+ return null;
374
+ if (runs.length === 0 && statuses.length === 0)
375
+ return ++emptyLooks >= 2 ? true : null;
376
+ return true;
377
+ };
378
+ try {
379
+ const passed = wait ? await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, probe) : await probe();
380
+ if (passed === null)
381
+ return { state: "pending", why: `${brew.repo} #${proposed.pr}'s checks are still running` };
382
+ }
383
+ catch (error) {
384
+ if (error instanceof HomebrewFailed)
385
+ return { state: "failed", why: `${brew.repo} #${proposed.pr}: ${error.message}` };
386
+ // Waiting ran out: still pending, never a stop of a published release.
387
+ if (error instanceof Stop && error.reason === "timeout")
388
+ return { state: "pending", why: error.message };
389
+ throw error;
390
+ }
391
+ // Ordering, once more right before the merge: a newer formula landed meanwhile is never overwritten.
392
+ const now = await brew.state(j.packageVersion);
393
+ if (now.kind === "newer")
394
+ return { state: "superseded", why: `the tap already names ${now.version}` };
395
+ if (now.kind === "missing")
396
+ return { state: "failed", why: `${brew.repo} #${proposed.pr} was closed before it merged` };
397
+ if (now.kind === "proposed" && now.head !== proposed.head)
398
+ return { state: "failed", why: `${brew.repo} #${proposed.pr} moved to ${now.head.slice(0, 7)} after its checks were read` };
399
+ if (now.kind === "current")
400
+ return { state: "current", why: null };
401
+ const verified = await brew.verified(proposed.pr, proposed.head, j.packageVersion);
402
+ if (!verified.ok)
403
+ return { state: "failed", why: verified.why };
404
+ await brew.merge(proposed.pr, proposed.head);
405
+ return { state: "current", why: null };
406
+ };
300
407
  // ---- one approval per release (release-once.md) -------------------------------------------------------------
301
408
  const rel = adapters.release;
302
409
  let epoch = null;
@@ -601,7 +708,7 @@ export async function runRelease(options, adapters) {
601
708
  throw new Stop("ci-failed", `A pull request check failed after the deploy, so ${j.candidate.sha.slice(0, 7)} needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
602
709
  // A resumed release that already has an authority takes its lease and settles every open operation first.
603
710
  await hold();
604
- while (j.step !== "done") {
711
+ while (j.step !== "done" && homebrewLeft === null) {
605
712
  step = j.step;
606
713
  say(`• ${j.step}`);
607
714
  try {
@@ -1055,7 +1162,7 @@ export async function runRelease(options, adapters) {
1055
1162
  await waitFor("publish", `npm, the GitHub release and the tag's checks for ${j.tag}`, async () => {
1056
1163
  if (await adapters.github.tag(j.github, j.tag) !== j.tagged.sha)
1057
1164
  throw new Stop("conflict", `${j.tag} no longer points at the verified merge commit. Homebrew was not updated.`);
1058
- const verdict = checksVerdict(await adapters.github.checks(j.github, j.tagged.sha), RELEASE_TAG_CHECKS);
1165
+ const verdict = tagChecksVerdict(await adapters.github.checks(j.github, j.tagged.sha, { tag: j.tag }));
1059
1166
  if (verdict.state === "failed")
1060
1167
  throw new Stop("publish-failed", `The release tag's checks failed: ${verdict.failed.join(", ")}. Homebrew was not updated.`);
1061
1168
  if (verdict.state === "pending")
@@ -1068,43 +1175,16 @@ export async function runRelease(options, adapters) {
1068
1175
  }
1069
1176
  case "homebrew": {
1070
1177
  await publishHold();
1071
- const brew = adapters.homebrew;
1072
- let state = await brew.state(j.packageVersion);
1073
- if (state.kind === "missing") {
1074
- intend("homebrew");
1075
- const proposed = await brew.propose(j.packageVersion);
1076
- state = { kind: "proposed", ...proposed };
1077
- }
1078
- if (state.kind === "proposed") {
1079
- const proposed = state;
1080
- j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false };
1081
- save();
1082
- // Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
1083
- // on a change, and only when it's still empty a poll later.
1084
- const checked = await brew.checked(proposed.head);
1085
- let emptyLooks = 0;
1086
- await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, async () => {
1087
- // Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
1088
- // Two apps may name a check alike; each app's attempts stand for themselves.
1089
- const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
1090
- const statuses = await brew.statuses(proposed.head);
1091
- const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
1092
- const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
1093
- if (failed.length > 0)
1094
- throw new Stop("homebrew-failed", `The Homebrew change's checks failed: ${failed.join(", ")}.`);
1095
- if (verdicts.some(one => one.state === "pending"))
1096
- return null;
1097
- // GitHub registers checks asynchronously; an empty set says nothing about success.
1098
- // A workflow is owed: only its own run counts, not another app's green check.
1099
- if (checked && !runs.some(run => run.app === "github-actions"))
1100
- return null;
1101
- if (runs.length === 0 && statuses.length === 0)
1102
- return ++emptyLooks >= 2 ? true : null;
1103
- return true;
1104
- });
1105
- await brew.merge(proposed.pr, proposed.head);
1178
+ // Homebrew is finished apart from publication (pipeline-speed.md, P1): npm and the GitHub release are verified,
1179
+ // so the release is published. This run proposes the tap's change and reads its checks once; only a run that
1180
+ // resumes to finish Homebrew waits for them. Pending or failed, the state is kept and the run ends published.
1181
+ const progress = await homebrewPass(finishingHomebrew);
1182
+ j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: progress.state === "current" && j.homebrew?.head != null ? true : j.homebrew?.merged ?? false, state: progress.state, why: progress.why };
1183
+ save();
1184
+ if (progress.state === "pending" || progress.state === "failed") {
1185
+ homebrewLeft = { state: progress.state, why: progress.why };
1186
+ break;
1106
1187
  }
1107
- j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: true };
1108
1188
  advance("done");
1109
1189
  break;
1110
1190
  }
@@ -1116,8 +1196,13 @@ export async function runRelease(options, adapters) {
1116
1196
  await restart(error.head);
1117
1197
  }
1118
1198
  }
1119
- say(`Released ${j.tag}: deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, published and on Homebrew.`);
1120
- if (j.auto?.state === "approved") {
1199
+ const left = homebrewLeft;
1200
+ if (left === null)
1201
+ say(`Released ${j.tag}: deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, published and ${j.homebrew?.state === "superseded" ? `left Homebrew as it is (${j.homebrew.why ?? "the tap names a newer version"})` : "on Homebrew"}.`);
1202
+ else
1203
+ say(`Published ${j.tag} (deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, on npm and GitHub); Homebrew ${left.state}${left.why === null ? "" : `: ${left.why}`}. Finish it with \`${resume}\`.`);
1204
+ // The automatic road's note, once: when this run published (a run that only finishes Homebrew already said it).
1205
+ if (j.auto?.state === "approved" && !finishingHomebrew) {
1121
1206
  try {
1122
1207
  await rel?.auto?.released(j.task.id, j.tag);
1123
1208
  }
@@ -0,0 +1,67 @@
1
+ import type { Runner } from "./runner.js";
2
+ export type RunnerProcessRole = "up" | "console" | "worker";
3
+ export type RunnerProcess = {
4
+ pid: number;
5
+ role: RunnerProcessRole;
6
+ started: string;
7
+ };
8
+ export type RunnerProcessRecord = {
9
+ v: 1;
10
+ runner: string;
11
+ host: string;
12
+ registeredAt: string;
13
+ /** A digest of the plane's credential hash for this registration: it names the incarnation, it proves nothing. */
14
+ credential: string;
15
+ processes: RunnerProcess[];
16
+ /** Authenticated with the plane's stored credential hash; legacy/forged records prove nothing. */
17
+ signature?: string;
18
+ };
19
+ export declare const RUNNER_PROCESS_DIR = "runner-processes";
20
+ /** The namespace is created before builds launch and denied by name, including in a shared state folder. */
21
+ export declare function ensureRunnerProcessFolder(folder: string): void;
22
+ export declare function sealRunnerProcessRecord(record: RunnerProcessRecord, credentialHash: string): RunnerProcessRecord;
23
+ /** Where a runner's record lives: beside the database, under a name derived from the runner's (never the name itself). */
24
+ export declare function runnerProcessFile(databaseFile: string, runner: string): string;
25
+ /** The incarnation's name in a record: a digest of the stored credential hash, so the record never holds that hash. */
26
+ export declare function credentialDigest(credentialHash: string): string;
27
+ /** `ps -o lstart=` (LC_ALL=C, TZ=UTC) as an ISO second, or null when it isn't exactly that shape. */
28
+ export declare function parseProcessStart(raw: string): string | null;
29
+ /** When a process started (whole seconds), or null when there is no such process or it can't be read. */
30
+ export declare function processStart(pid: number): string | null;
31
+ /** The probes a proof uses; real by default, supplied by tests. */
32
+ export type ProcessProbe = {
33
+ exists: (pid: number) => "absent" | "present" | "unknown";
34
+ start: (pid: number) => string | null;
35
+ };
36
+ export declare const realProcessProbe: ProcessProbe;
37
+ /** Proved gone: no process has this id, or the one that has it started at another time (the id was reused). */
38
+ export declare function processGone(entry: RunnerProcess, probe?: ProcessProbe): boolean;
39
+ /** A record read strictly: a regular private file of bounded size, every field its exact type, or null. */
40
+ export declare function readRunnerProcessRecord(file: string): RunnerProcessRecord | null;
41
+ /** Replace the record atomically: a private temp file in a private folder, renamed into place. */
42
+ export declare function writeRunnerProcessRecord(file: string, record: RunnerProcessRecord): void;
43
+ /**
44
+ * Whether the plane's current holder of a runner name is proved gone: the record is for exactly that registration,
45
+ * on this host, and every process it names has exited (by identity). `detail` says which processes, for the ledger.
46
+ */
47
+ export declare function holderProvedGone(record: RunnerProcessRecord | null, holder: {
48
+ runner: Runner;
49
+ credentialHash: string;
50
+ }, host: string, probe?: ProcessProbe): {
51
+ gone: true;
52
+ detail: string;
53
+ } | {
54
+ gone: false;
55
+ why: string;
56
+ };
57
+ /** Re-read after observing exit: an up may append a child between the first read and its own exit. */
58
+ export declare function recordedHolderProvedGone(file: string, holder: {
59
+ runner: Runner;
60
+ credentialHash: string;
61
+ }, host: string, probe?: ProcessProbe): {
62
+ gone: true;
63
+ detail: string;
64
+ } | {
65
+ gone: false;
66
+ why: string;
67
+ };
@@ -0,0 +1,219 @@
1
+ /**
2
+ * Which processes hold a runner's credential, kept beside the plane's database so a later `up` can prove the previous
3
+ * holder is gone (docs: the 2026-10-29 deploy incident). `toolroll up` registers its runner, then records itself and
4
+ * every role child it hands the token to, each by process id AND start time, before the child receives the token.
5
+ *
6
+ * A fresh `up` whose runner name is still held (its heartbeat looks fresh, or its watch lease has not lapsed) asks
7
+ * this record whether that holder is gone: the record must be for exactly the registration the plane holds (name,
8
+ * host, registration time and credential), and every process in it must have exited, each proved by identity (no
9
+ * such process, or a process with that id started at a different time). Then the plane retires the stale holder in
10
+ * the same transaction that proved it (runner.ts registerRunnerIfIdle). Anything missing, unreadable or ambiguous
11
+ * proves nothing, and the door keeps refusing until the heartbeat ages out, exactly as before.
12
+ *
13
+ * The record is the plane's own state (the folder that holds the database). It grants nothing by itself: it can
14
+ * only let a name whose every recorded holder has exited be taken over early.
15
+ */
16
+ import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto";
17
+ import { execFileSync } from "node:child_process";
18
+ import { closeSync, fstatSync, lstatSync, mkdirSync, openSync, readSync, realpathSync, renameSync, rmSync, writeFileSync, fsyncSync, constants } from "node:fs";
19
+ import { dirname, join, resolve } from "node:path";
20
+ export const RUNNER_PROCESS_DIR = "runner-processes";
21
+ /** The namespace is created before builds launch and denied by name, including in a shared state folder. */
22
+ export function ensureRunnerProcessFolder(folder) {
23
+ try {
24
+ mkdirSync(folder, { mode: 0o700 });
25
+ }
26
+ catch (error) {
27
+ if (error.code !== "EEXIST")
28
+ throw error;
29
+ }
30
+ const stat = lstatSync(folder);
31
+ if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0
32
+ || (process.getuid && stat.uid !== process.getuid()) || realpathSync(folder) !== resolve(folder))
33
+ throw Error("Untrusted runner process folder");
34
+ }
35
+ export function sealRunnerProcessRecord(record, credentialHash) {
36
+ const { signature: _signature, ...plain } = record;
37
+ return { ...plain, signature: createHmac("sha256", credentialHash).update(JSON.stringify(plain)).digest("hex") };
38
+ }
39
+ const MAX_RECORD = 64 * 1024;
40
+ const MAX_PROCESSES = 64;
41
+ const ROLES = new Set(["up", "console", "worker"]);
42
+ /** Where a runner's record lives: beside the database, under a name derived from the runner's (never the name itself). */
43
+ export function runnerProcessFile(databaseFile, runner) {
44
+ let database = databaseFile;
45
+ try {
46
+ database = realpathSync(databaseFile);
47
+ }
48
+ catch (error) {
49
+ if (error.code !== "ENOENT")
50
+ throw error;
51
+ }
52
+ return join(realpathSync(dirname(database)), RUNNER_PROCESS_DIR, `${createHash("sha256").update(runner, "utf8").digest("hex").slice(0, 32)}.json`);
53
+ }
54
+ /** The incarnation's name in a record: a digest of the stored credential hash, so the record never holds that hash. */
55
+ export function credentialDigest(credentialHash) {
56
+ return createHash("sha256").update(`runner-process:${credentialHash}`, "utf8").digest("hex").slice(0, 32);
57
+ }
58
+ const LSTART = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{1,2}) (\d{2}:\d{2}:\d{2}) (\d{4})$/;
59
+ /** `ps -o lstart=` (LC_ALL=C, TZ=UTC) as an ISO second, or null when it isn't exactly that shape. */
60
+ export function parseProcessStart(raw) {
61
+ const match = LSTART.exec(raw.trim().replace(/ +/g, " "));
62
+ if (match === null)
63
+ return null;
64
+ const utc = `${match[1]}, ${match[3].padStart(2, "0")} ${match[2]} ${match[5]} ${match[4]} GMT`;
65
+ const at = Date.parse(utc);
66
+ if (!Number.isFinite(at) || new Date(at).toUTCString() !== utc)
67
+ return null;
68
+ return new Date(at).toISOString();
69
+ }
70
+ /** When a process started (whole seconds), or null when there is no such process or it can't be read. */
71
+ export function processStart(pid) {
72
+ if (!Number.isSafeInteger(pid) || pid <= 1 || process.platform === "win32")
73
+ return null;
74
+ try {
75
+ const raw = execFileSync("/bin/ps", ["-p", String(pid), "-o", "lstart="], {
76
+ encoding: "utf8", env: { PATH: "/usr/bin:/bin", LC_ALL: "C", TZ: "UTC" }, stdio: ["ignore", "pipe", "ignore"],
77
+ maxBuffer: 1024, timeout: 2_000, killSignal: "SIGKILL",
78
+ });
79
+ return parseProcessStart(raw);
80
+ }
81
+ catch {
82
+ return null;
83
+ }
84
+ }
85
+ export const realProcessProbe = {
86
+ exists: pid => {
87
+ try {
88
+ process.kill(pid, 0);
89
+ return "present";
90
+ }
91
+ catch (error) {
92
+ const code = error.code;
93
+ return code === "ESRCH" ? "absent" : code === "EPERM" ? "present" : "unknown";
94
+ }
95
+ },
96
+ start: processStart,
97
+ };
98
+ /** Proved gone: no process has this id, or the one that has it started at another time (the id was reused). */
99
+ export function processGone(entry, probe = realProcessProbe) {
100
+ const exists = probe.exists(entry.pid);
101
+ if (exists === "absent")
102
+ return true;
103
+ if (exists === "unknown")
104
+ return false;
105
+ const started = probe.start(entry.pid);
106
+ if (started === null)
107
+ return probe.exists(entry.pid) === "absent";
108
+ return started !== entry.started;
109
+ }
110
+ /** A record read strictly: a regular private file of bounded size, every field its exact type, or null. */
111
+ export function readRunnerProcessRecord(file) {
112
+ let fd = null;
113
+ try {
114
+ const folder = lstatSync(dirname(file));
115
+ if (!folder.isDirectory() || folder.isSymbolicLink() || (folder.mode & 0o077) !== 0
116
+ || (process.getuid && folder.uid !== process.getuid()) || realpathSync(dirname(file)) !== resolve(dirname(file)))
117
+ return null;
118
+ const link = lstatSync(file);
119
+ if (!link.isFile() || link.isSymbolicLink() || link.size > MAX_RECORD)
120
+ return null;
121
+ fd = openSync(file, constants.O_RDONLY | constants.O_NONBLOCK | (constants.O_NOFOLLOW ?? 0));
122
+ const stat = fstatSync(fd);
123
+ if (!stat.isFile() || stat.size > MAX_RECORD || stat.ino !== link.ino || stat.dev !== link.dev || (stat.mode & 0o077) !== 0)
124
+ return null;
125
+ if (typeof process.getuid === "function" && stat.uid !== process.getuid())
126
+ return null;
127
+ const buffer = Buffer.alloc(stat.size);
128
+ let read = 0;
129
+ while (read < stat.size) {
130
+ const n = readSync(fd, buffer, read, stat.size - read, read);
131
+ if (n === 0)
132
+ break;
133
+ read += n;
134
+ }
135
+ return parseRecord(JSON.parse(buffer.subarray(0, read).toString("utf8")));
136
+ }
137
+ catch {
138
+ return null;
139
+ }
140
+ finally {
141
+ if (fd !== null)
142
+ closeSync(fd);
143
+ }
144
+ }
145
+ const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
146
+ function parseRecord(value) {
147
+ if (typeof value !== "object" || value === null || Array.isArray(value))
148
+ return null;
149
+ const v = value;
150
+ const text = (one, max = 512) => typeof one === "string" && one !== "" && one.length <= max && !one.includes("\0");
151
+ if (v["v"] !== 1 || !text(v["runner"]) || !text(v["host"]) || !text(v["registeredAt"]) || !ISO.test(v["registeredAt"]) || !text(v["credential"], 64))
152
+ return null;
153
+ const list = v["processes"];
154
+ if (!Array.isArray(list) || list.length === 0 || list.length > MAX_PROCESSES)
155
+ return null;
156
+ const processes = [];
157
+ for (const one of list) {
158
+ if (typeof one !== "object" || one === null)
159
+ return null;
160
+ const p = one;
161
+ if (!Number.isSafeInteger(p["pid"]) || p["pid"] <= 1 || !ROLES.has(p["role"]) || !text(p["started"], 32) || !ISO.test(p["started"]))
162
+ return null;
163
+ processes.push({ pid: p["pid"], role: p["role"], started: p["started"] });
164
+ }
165
+ return { v: 1, runner: v["runner"], host: v["host"], registeredAt: v["registeredAt"], credential: v["credential"], processes,
166
+ ...(typeof v["signature"] === "string" ? { signature: v["signature"] } : {}) };
167
+ }
168
+ /** Replace the record atomically: a private temp file in a private folder, renamed into place. */
169
+ export function writeRunnerProcessRecord(file, record) {
170
+ ensureRunnerProcessFolder(dirname(file));
171
+ const temp = `${file}.${randomUUID()}.tmp`;
172
+ try {
173
+ const fd = openSync(temp, "wx", 0o600);
174
+ try {
175
+ writeFileSync(fd, JSON.stringify(record));
176
+ fsyncSync(fd);
177
+ }
178
+ finally {
179
+ closeSync(fd);
180
+ }
181
+ renameSync(temp, file);
182
+ }
183
+ finally {
184
+ rmSync(temp, { force: true });
185
+ }
186
+ }
187
+ /**
188
+ * Whether the plane's current holder of a runner name is proved gone: the record is for exactly that registration,
189
+ * on this host, and every process it names has exited (by identity). `detail` says which processes, for the ledger.
190
+ */
191
+ export function holderProvedGone(record, holder, host, probe = realProcessProbe) {
192
+ if (record === null)
193
+ return { gone: false, why: "no readable record of its processes" };
194
+ const expected = sealRunnerProcessRecord(record, holder.credentialHash).signature;
195
+ if (typeof record.signature !== "string" || !/^[a-f0-9]{64}$/.test(record.signature)
196
+ || !timingSafeEqual(Buffer.from(record.signature, "hex"), Buffer.from(expected, "hex")))
197
+ return { gone: false, why: "its process record is not authenticated" };
198
+ if (record.runner !== holder.runner.name || record.host !== holder.runner.host || record.host !== host)
199
+ return { gone: false, why: "its record is for another worker or machine" };
200
+ if (record.registeredAt !== holder.runner.registeredAt || record.credential !== credentialDigest(holder.credentialHash))
201
+ return { gone: false, why: "its record is from an earlier start" };
202
+ if (!record.processes.some(one => one.role === "up"))
203
+ return { gone: false, why: "its record names no `up` process" };
204
+ const left = record.processes.filter(one => !processGone(one, probe));
205
+ if (left.length > 0)
206
+ return { gone: false, why: `process ${left.map(one => one.pid).join(", ")} is still running` };
207
+ return { gone: true, detail: `${record.runner}: process ${record.processes.map(one => `${one.pid} (${one.role})`).join(", ")} proved gone` };
208
+ }
209
+ /** Re-read after observing exit: an up may append a child between the first read and its own exit. */
210
+ export function recordedHolderProvedGone(file, holder, host, probe = realProcessProbe) {
211
+ const record = readRunnerProcessRecord(file);
212
+ const proof = holderProvedGone(record, holder, host, probe);
213
+ if (!proof.gone)
214
+ return proof;
215
+ const after = readRunnerProcessRecord(file);
216
+ if (after === null || JSON.stringify(after) !== JSON.stringify(record))
217
+ return { gone: false, why: "its process record changed during the exit proof" };
218
+ return proof;
219
+ }
package/dist/runner.d.ts CHANGED
@@ -21,7 +21,7 @@
21
21
  */
22
22
  import { type AcpProviderId } from "./acp-agents.js";
23
23
  import type { AcpAttestation } from "./store.js";
24
- import type { Store, Mutation } from "./store.js";
24
+ import { type Store, type Mutation } from "./store.js";
25
25
  import { type ProviderId } from "./provider.js";
26
26
  import type { ReadinessObservation } from "./phase-routing.js";
27
27
  /** A runner's repo binding, in the same canonical form task filing uses —
@@ -59,11 +59,18 @@ export declare const RUNNER_TOKEN_MS: number;
59
59
  /**
60
60
  * How long a runner may go quiet before it is presumed gone.
61
61
  *
62
- * Comfortably more than any sensible heartbeat interval: declaring a busy
63
- * machine dead because one check-in was slow costs more than waiting another
64
- * minute for a machine that really is.
62
+ * Four heartbeats (RUNNER_HEARTBEAT_MS): one slow check-in never declares a busy
63
+ * machine dead, and a machine that really is gone holds its name for a minute,
64
+ * not three (docs/plans/actors-lessons.md). A clean stop doesn't wait even that
65
+ * long: a draining or stopped holder never blocks the next start
66
+ * (registerRunnerIfIdle).
65
67
  */
66
- export declare const DEFAULT_LIVENESS_MS: number;
68
+ export declare const DEFAULT_LIVENESS_MS = 60000;
69
+ /** How often a live runner says so: `up`'s runner heartbeat and every watch lease renewal. */
70
+ export declare const RUNNER_HEARTBEAT_MS = 15000;
71
+ /** The longest a watch lease may be taken or renewed for (durable-actors' MAX_HOST_LEASE_DURATION_MS): a crashed
72
+ * holder's projects are free within a minute. The store clamps every ttl to it. */
73
+ export { MAX_WATCH_LEASE_MS } from "./store.js";
67
74
  export type RegisterOptions = {
68
75
  name: string;
69
76
  host: string;
@@ -140,13 +147,35 @@ export declare function normalizeRunnerName(raw: string): string;
140
147
  * live claim owns), reclaims, and rotates. Two racers serialize: one wins,
141
148
  * the other gets the typed refusal.
142
149
  */
143
- export declare function registerRunnerIfIdle(store: Store, options: RegisterOptions): (Registration & {
150
+ export declare function registerRunnerIfIdle(store: Store, options: RegisterOptions & {
151
+ /**
152
+ * Whether the name's current holder is proved gone (runner-process.ts): every process it handed its credential to
153
+ * has exited, verified by identity. Asked only while the holder still blocks the name, inside this transaction;
154
+ * a proof retires that holder and lapses its watch leases here, so a stale heartbeat never outlives its processes.
155
+ */
156
+ previousGone?: (holder: {
157
+ runner: Runner;
158
+ credentialHash: string;
159
+ }) => {
160
+ gone: true;
161
+ detail: string;
162
+ } | {
163
+ gone: false;
164
+ why: string;
165
+ };
166
+ }): (Registration & {
144
167
  ok: true;
145
168
  recoveredRuns: number;
169
+ signedOff: string | null;
146
170
  }) | {
147
171
  ok: false;
148
172
  reason: "runner-alive" | "watch-live";
149
173
  detail: string;
174
+ } | {
175
+ ok: false;
176
+ reason: "runner-draining";
177
+ detail: string;
178
+ until: string;
150
179
  };
151
180
  /**
152
181
  * The authenticated lease door (arc 2 findings 15/26): credential verified
@@ -280,3 +309,16 @@ export declare function reportProviderReadinessAuthed(store: Store, args: {
280
309
  token: string;
281
310
  observations: readonly Omit<ReadinessObservation, "runner" | "observedAt">[];
282
311
  }, now: Date): AuthResult;
312
+ /** A worker's lifecycle row (service_lifecycle). */
313
+ export declare const workerComponent: (name: string) => string;
314
+ /**
315
+ * The stop begins (durable-actors' draining report), fenced by this holder's credential: from now this runner takes
316
+ * no new work (claim.ts acquire refuses). `until` is a schedule, never proof that the holder exited.
317
+ */
318
+ export declare function drainRunnerIfCurrent(store: Store, name: string, token: string, now: Date, until: Date): boolean;
319
+ /** A runner's drain as its own lifecycle row reports it (only the current registration's row counts): null when it
320
+ * isn't draining; ended once its bound passed or it reported stopped. */
321
+ export declare function drainOf(store: Store, runner: Runner, now: Date): {
322
+ until: string;
323
+ ended: boolean;
324
+ } | null;