toolroll 0.9.76 → 0.9.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/CHANGELOG.md +273 -0
  2. package/README.md +11 -4
  3. package/THIRD_PARTY_NOTICES.md +5 -5
  4. package/dist/agent-fence.js +46 -10
  5. package/dist/assignment.js +3 -0
  6. package/dist/backup-ui.d.ts +9 -1
  7. package/dist/backup-ui.js +28 -3
  8. package/dist/backup.js +4 -3
  9. package/dist/browser/THIRD_PARTY_NOTICES.txt +5 -5
  10. package/dist/browser/workspace.css +2 -2
  11. package/dist/browser/workspace.js +31 -31
  12. package/dist/browser-workspace.d.ts +2 -0
  13. package/dist/builder.d.ts +18 -3
  14. package/dist/builder.js +152 -47
  15. package/dist/chat-decide.js +4 -0
  16. package/dist/check-failure.d.ts +92 -0
  17. package/dist/check-failure.js +295 -0
  18. package/dist/check-progress.d.ts +28 -1
  19. package/dist/check-progress.js +105 -6
  20. package/dist/check-report.d.ts +67 -6
  21. package/dist/check-report.js +254 -9
  22. package/dist/checkout-cleanup.js +2 -2
  23. package/dist/claim.d.ts +9 -0
  24. package/dist/claim.js +9 -1
  25. package/dist/cli.js +8 -5
  26. package/dist/coding-handoff.d.ts +1 -0
  27. package/dist/coding-handoff.js +20 -11
  28. package/dist/commit-extract.d.ts +6 -0
  29. package/dist/commit-extract.js +63 -8
  30. package/dist/console-chrome.d.ts +41 -0
  31. package/dist/console-chrome.js +166 -0
  32. package/dist/contracts/console-api.d.ts +1 -1
  33. package/dist/contracts/console-api.js +1 -1
  34. package/dist/contracts/release-check.d.ts +12 -0
  35. package/dist/contracts/release-check.js +17 -0
  36. package/dist/contracts/release-terms.d.ts +41 -0
  37. package/dist/contracts/release-terms.js +53 -0
  38. package/dist/contracts/verification-receipt.d.ts +20 -2
  39. package/dist/contracts/verification-receipt.js +15 -3
  40. package/dist/contracts/verification-view.d.ts +8 -0
  41. package/dist/controller-service.js +4 -1
  42. package/dist/controller-supervisor.d.ts +8 -0
  43. package/dist/controller-supervisor.js +33 -1
  44. package/dist/demo.js +3 -2
  45. package/dist/deploy-gate.js +5 -0
  46. package/dist/deploy-rescue.d.ts +41 -0
  47. package/dist/deploy-rescue.js +94 -0
  48. package/dist/desktop-update.d.ts +1 -1
  49. package/dist/dispose.d.ts +3 -0
  50. package/dist/dispose.js +9 -5
  51. package/dist/doctor.d.ts +7 -0
  52. package/dist/doctor.js +10 -2
  53. package/dist/envelope.js +2 -1
  54. package/dist/evidence.d.ts +9 -0
  55. package/dist/evidence.js +19 -4
  56. package/dist/exec.d.ts +9 -0
  57. package/dist/exec.js +22 -2
  58. package/dist/git-floor.d.ts +16 -0
  59. package/dist/git-floor.js +36 -0
  60. package/dist/grant.js +5 -1
  61. package/dist/guides.js +22 -11
  62. package/dist/latency-timeline.d.ts +46 -0
  63. package/dist/latency-timeline.js +120 -0
  64. package/dist/live-backup.d.ts +39 -0
  65. package/dist/live-backup.js +271 -0
  66. package/dist/operate.d.ts +13 -0
  67. package/dist/operate.js +969 -328
  68. package/dist/planner.js +2 -2
  69. package/dist/prepared-evidence.d.ts +2 -1
  70. package/dist/prepared-evidence.js +8 -3
  71. package/dist/project-knowledge.js +6 -2
  72. package/dist/project-learning.js +2 -2
  73. package/dist/proof.js +6 -0
  74. package/dist/read-executor-worker.js +8 -4
  75. package/dist/read-executor.d.ts +18 -1
  76. package/dist/read-executor.js +8 -0
  77. package/dist/release-adapters.d.ts +80 -2
  78. package/dist/release-adapters.js +441 -16
  79. package/dist/release-authority.js +6 -2
  80. package/dist/release-auto.d.ts +38 -7
  81. package/dist/release-auto.js +131 -15
  82. package/dist/release-card.d.ts +15 -3
  83. package/dist/release-card.js +25 -9
  84. package/dist/release-carry.d.ts +21 -1
  85. package/dist/release-carry.js +22 -0
  86. package/dist/release-ci-hold.d.ts +41 -0
  87. package/dist/release-ci-hold.js +53 -0
  88. package/dist/release-cli.d.ts +3 -0
  89. package/dist/release-cli.js +95 -17
  90. package/dist/release-coverage.js +5 -2
  91. package/dist/release-gate.d.ts +5 -0
  92. package/dist/release-gate.js +1 -1
  93. package/dist/release-recheck.d.ts +1 -1
  94. package/dist/release-recheck.js +8 -2
  95. package/dist/release.d.ts +144 -14
  96. package/dist/release.js +421 -81
  97. package/dist/review-context.js +8 -6
  98. package/dist/runner-process.d.ts +67 -0
  99. package/dist/runner-process.js +219 -0
  100. package/dist/runner.d.ts +48 -6
  101. package/dist/runner.js +73 -9
  102. package/dist/sandbox-runtime.d.ts +8 -0
  103. package/dist/sandbox-runtime.js +14 -2
  104. package/dist/scope.d.ts +19 -1
  105. package/dist/scope.js +26 -3
  106. package/dist/scout.js +2 -2
  107. package/dist/secret-mint.d.ts +1 -0
  108. package/dist/secret-mint.js +14 -0
  109. package/dist/serve.js +112 -51
  110. package/dist/server/chat.js +5 -5
  111. package/dist/server/http.d.ts +2 -0
  112. package/dist/server/pages.js +7 -3
  113. package/dist/server/remote-hooks.d.ts +1 -1
  114. package/dist/server/remote-hooks.js +5 -3
  115. package/dist/server/render-results.d.ts +2 -0
  116. package/dist/server/render-results.js +1 -0
  117. package/dist/server/render-tasks.d.ts +2 -0
  118. package/dist/server/render-tasks.js +2 -1
  119. package/dist/server/request-context.d.ts +8 -0
  120. package/dist/server/runtime.d.ts +2 -0
  121. package/dist/server/settings.js +22 -2
  122. package/dist/server/task-views.js +7 -2
  123. package/dist/server/tasks.js +4 -1
  124. package/dist/setup-guide.d.ts +2 -2
  125. package/dist/storage-sweep.js +1 -1
  126. package/dist/storage.d.ts +5 -0
  127. package/dist/storage.js +28 -1
  128. package/dist/store.d.ts +135 -2
  129. package/dist/store.js +275 -13
  130. package/dist/surface.js +3 -1
  131. package/dist/task-base.d.ts +4 -0
  132. package/dist/task-base.js +22 -2
  133. package/dist/teardown-git.d.ts +52 -3
  134. package/dist/teardown-git.js +145 -33
  135. package/dist/test-infra.js +1 -1
  136. package/dist/toolroll-update.js +2 -2
  137. package/dist/up-processes.d.ts +137 -0
  138. package/dist/up-processes.js +165 -0
  139. package/dist/up-role-child.d.ts +1 -0
  140. package/dist/up-role-child.js +81 -0
  141. package/dist/verification-evidence.d.ts +1 -0
  142. package/dist/verification-evidence.js +13 -1
  143. package/dist/workspace-revision.js +3 -1
  144. package/package.json +2 -1
@@ -804,6 +804,8 @@ export type BrowserResultView = {
804
804
  efficiency?: import("./run-efficiency.js").RunEfficiency | null;
805
805
  /** What its network was and what its proxy saw (egress.ts): display only. */
806
806
  network?: import("./egress.js").RunNetworkRecord | null;
807
+ /** Where the time went: its release check's parts and the task's deploys (latency-timeline.ts), display only. */
808
+ timing?: import("./latency-timeline.js").TimingView | null;
807
809
  /** Words for the result's Accept: Accept and finish when offered, else the Needs you acceptance. */
808
810
  decision: BrowserResultDecision | null;
809
811
  checks: {
package/dist/builder.d.ts CHANGED
@@ -34,6 +34,20 @@ export declare function approvedCommandShell(command: string, platform?: NodeJS.
34
34
  * must not turn an ordinary product failure into environment recovery.
35
35
  */
36
36
  export declare function verificationExecutableMissing(result: ExecResult, platform?: NodeJS.Platform): boolean;
37
+ /**
38
+ * The approved command's own program — its first word — when the plane, looking it up itself before the check starts,
39
+ * finds no such executable; null when it exists or can't be told (eval-harness review 2, finding 5): a first word with
40
+ * any quoting, expansion, assignment or redirection, a shell builtin, or Windows (cmd.exe's lookup isn't modelled).
41
+ * A path is looked up from the checkout; a bare name on `path`'s absolute folders.
42
+ */
43
+ export declare function approvedProgramMissing(command: string, cwd: string, path: string | undefined, platform?: NodeJS.Platform): string | null;
44
+ /**
45
+ * Whether a check's first attempt failed at its launch, so the approved recovery setup may run and the check run again
46
+ * (eval-harness review 1, finding 6; review 2, finding 5). Only a result the plane made without spawning qualifies.
47
+ * Any shell output, including a perfect "not found" line after a missing-program lookup, remains a check failure.
48
+ * A later passing check still counts as a retry (check-failure.ts), which an automatic release approval refuses.
49
+ */
50
+ export declare function launchFailedBeforeCheck(result: ExecResult, nonce: string | null, missing: string | null, platform?: NodeJS.Platform): boolean;
37
51
  export type BuildRequest = {
38
52
  taskId: string;
39
53
  taskRef: number;
@@ -160,10 +174,11 @@ export declare const REPAIR_MAX_TURNS = 4;
160
174
  export declare const REPAIR_TIMEOUT_MS: number;
161
175
  /**
162
176
  * How often a running build says "still here" — extending its lease and its
163
- * runner's liveness in one beat. A minute against a three-minute liveness
164
- * window means two beats can be lost to load before anything looks dead.
177
+ * runner's liveness in one beat. Every 15 s against a one-minute liveness
178
+ * window (runner.ts) means three beats can be lost to load before anything
179
+ * looks dead.
165
180
  */
166
- export declare const DEFAULT_PULSE_MS = 60000;
181
+ export declare const DEFAULT_PULSE_MS = 15000;
167
182
  /** Branches an unattended agent may never commit to, whatever it was asked. */
168
183
  export declare const PROTECTED: Set<string>;
169
184
  /**
package/dist/builder.js CHANGED
@@ -8,14 +8,15 @@ import { queueBatchCheck } from "./batch-checks.js";
8
8
  import { batchesFullCheck } from "./batch-policy.js";
9
9
  import { skillsContext } from "./project-skills.js";
10
10
  import { failedVerificationEvidence, sealVerificationReceipt, verificationEvidence, reuseObservationVerification } from "./verification-evidence.js";
11
- import { CHECK_INVENTORY_CAPTURE, CHECK_REPORT_CAPTURE, prepareCheckAttempt, sealCheckOutcome } from "./check-report.js";
11
+ import { CHECK_INVENTORY_CAPTURE, CHECK_REPORT_CAPTURE, CheckBaselineUnverifiable, prepareCheckAttempt, reportFromOutput, sealCheckOutcome } from "./check-report.js";
12
+ import { retriedInside } from "./check-failure.js";
12
13
  import { planTargetedRecheck, settleCoverage } from "./release-recheck.js";
13
14
  import { effectiveCheckFacts } from "./verification-evidence.js";
14
15
  import { learningContext } from "./project-learning.js";
15
16
  import { knowledgeContext } from "./project-knowledge.js";
16
17
  import { flowGoalCuts } from "./flow-engine.js";
17
18
  import { readCodingHandoff, verifyCodingHandoffBase } from "./coding-handoff.js";
18
- import { PREPARED_EVIDENCE_FILE, PREPARED_EVIDENCE_GIT, preparedScreenshotMatches, readPreparedEvidence, writePreparedEvidence } from "./prepared-evidence.js";
19
+ import { PREPARED_EVIDENCE_FILE, preparedEvidenceGit, preparedScreenshotMatches, readPreparedEvidence, writePreparedEvidence } from "./prepared-evidence.js";
19
20
  /**
20
21
  * The first thing here that runs an agent.
21
22
  *
@@ -46,21 +47,22 @@ import { PREPARED_EVIDENCE_FILE, PREPARED_EVIDENCE_GIT, preparedScreenshotMatche
46
47
  * watchdog and a high runaway-turn breaker still stop a pathological loop;
47
48
  * repair remains narrowly time-bounded because its job is narrowly scoped.
48
49
  */
49
- import { existsSync, realpathSync, unlinkSync, writeFileSync } from "node:fs";
50
+ import { accessSync, constants as fsConstants, existsSync, realpathSync, statSync, unlinkSync, writeFileSync } from "node:fs";
50
51
  import { homedir } from "node:os";
51
- import { join, resolve } from "node:path";
52
+ import { delimiter, isAbsolute, join, resolve } from "node:path";
52
53
  import { run, runClaudeStreamJsonl, runOwnerTag, streamsInput } from "./exec.js";
53
54
  import { stopRequestedFor, stopWords, underStopWatch } from "./task-control.js";
54
55
  import { witnessedRunner } from "./process-custody.js";
55
56
  import { runWithIsolatedDatabase } from "./child-database.js";
56
57
  import { recordWorktreeProcess, saveWorkPatch } from "./worktree.js";
57
58
  import { launchSandboxOf, protectedChanges, protectedLinks } from "./agent-fence.js";
58
- import { approvalOf, digestOf, idleClock, isAcpProfile, profileDigestOf, routeParityProblem, watchdogsOf } from "./scope.js";
59
+ import { approvalOf, digestOf, idleClock, isAcpProfile, profileDigestOf, releaseCheckScope, routeParityProblem, watchdogsOf } from "./scope.js";
59
60
  import { legOf, routeDigestOf, routeFromJson } from "./phase-routing.js";
60
61
  import { execFileSync } from "node:child_process";
61
62
  import { currentClaim, heartbeat, SYNC_MAX_AGE_MS, finalize } from "./claim.js";
62
63
  import { missingCapability } from "./dispatch.js";
63
- import { heartbeat as runnerHeartbeat } from "./runner.js";
64
+ import { heartbeat as runnerHeartbeat, RUNNER_HEARTBEAT_MS } from "./runner.js";
65
+ import { keepCheckTimeline } from "./latency-timeline.js";
64
66
  import { MARKER as LEASE_MARKER } from "./worktree.js";
65
67
  import { parseDecision, parseHandoff, repairPrompt, HEADLESS_RULE, HANDOFF_CONCLUSION_CAP, HANDOFF_ITEM_CAP, HANDOFF_LIST_CAP, HANDOFF_PAYLOAD_CAP } from "./decision.js";
66
68
  import { createHash, randomBytes, randomUUID } from "node:crypto";
@@ -74,7 +76,7 @@ import { closeSteerMailbox, openSteerMailbox, STEER_HANDLED } from "./steer-mail
74
76
  import { LiveInput } from "./live-input.js";
75
77
  import { LIVE_TICK_MS, tickLiveSteer } from "./steer-live.js";
76
78
  import { withRunActivity } from "./task-activity.js";
77
- import { CheckProgressTracker } from "./check-progress.js";
79
+ import { CheckProgressTracker, coalescedProgressWriter } from "./check-progress.js";
78
80
  import { captureParkEvidence, captureTerminalDiff, captureBaseTree, evidenceRoot, handoffName, isImagePath, WORKTREE_EVIDENCE_DIR, storeHandoffArtifact, looksLikeProtocolFile, mailboxName, progressFileName, steerFolderName, statusFileName, proofFileName, rubricFileName, proposalFileName, quarantineMailboxes, readMailbox, readVerifiedArtifact, scanForSecrets, redactSecretLines, storeEvidence, validateScreenshotBytes, imageDimensions, SCREENSHOT_BYTE_CAP, boundStreamHeadTail } from "./evidence.js";
79
81
  import { attemptKeyOf, closeRunQuestions, noteOpenTool, recordAgentTurn } from "./agent-board.js";
80
82
  import { openStatusFile, pulseAgentBoard, removeStatusFile, statusBrief } from "./agent-pulse.js";
@@ -125,6 +127,57 @@ export function verificationExecutableMissing(result, platform = process.platfor
125
127
  return false;
126
128
  return /^'[^'\r\n]+' is not recognized as an internal or external command,\s+operable program or batch file\.\s*$/i.test(result.stderr.trim());
127
129
  }
130
+ /** Shell words that are never a program on PATH: a command starting with one has no program the plane can look up. */
131
+ const SHELL_WORDS = new Set(["!", ".", ":", "[", "alias", "bg", "break", "case", "cd", "command", "continue", "eval", "exec", "exit", "export", "false", "fg", "for",
132
+ "function", "getopts", "hash", "if", "jobs", "kill", "local", "pwd", "read", "readonly", "return", "set", "shift", "source", "test", "times", "trap", "true", "type",
133
+ "ulimit", "umask", "unalias", "unset", "until", "wait", "while", "{", "time", "echo", "printf"]);
134
+ /**
135
+ * The approved command's own program — its first word — when the plane, looking it up itself before the check starts,
136
+ * finds no such executable; null when it exists or can't be told (eval-harness review 2, finding 5): a first word with
137
+ * any quoting, expansion, assignment or redirection, a shell builtin, or Windows (cmd.exe's lookup isn't modelled).
138
+ * A path is looked up from the checkout; a bare name on `path`'s absolute folders.
139
+ */
140
+ export function approvedProgramMissing(command, cwd, path, platform = process.platform) {
141
+ if (platform === "win32")
142
+ return null;
143
+ // Only a single plain invocation: shell expansion or another command could execute before a missing first word.
144
+ if (!/^[ \t]*[A-Za-z0-9._+/@-]+(?:[ \t]+[A-Za-z0-9._+/@,:=%-]+)*[ \t]*$/.test(command))
145
+ return null;
146
+ const first = /^\s*([A-Za-z0-9._+/@-]+)(?:\s|$)/.exec(command)?.[1];
147
+ if (first === undefined || SHELL_WORDS.has(first))
148
+ return null;
149
+ const runnable = (file) => { try {
150
+ accessSync(file, fsConstants.X_OK);
151
+ return statSync(file).isFile();
152
+ }
153
+ catch {
154
+ return false;
155
+ } };
156
+ if (first.includes("/"))
157
+ return runnable(resolve(cwd, first)) ? null : first;
158
+ if (path === undefined || path.split(delimiter).some(dir => !isAbsolute(dir)))
159
+ return null;
160
+ for (const dir of (path ?? "").split(delimiter))
161
+ if (isAbsolute(dir) && runnable(join(dir, first)))
162
+ return null;
163
+ return first;
164
+ }
165
+ // Only the plane can issue this evidence, and only when it never spawns the command. A lookup followed by a shell
166
+ // launch is racy: the build can install a program between them and forge the shell's entire diagnostic.
167
+ const unstartedChecks = new WeakMap();
168
+ /**
169
+ * Whether a check's first attempt failed at its launch, so the approved recovery setup may run and the check run again
170
+ * (eval-harness review 1, finding 6; review 2, finding 5). Only a result the plane made without spawning qualifies.
171
+ * Any shell output, including a perfect "not found" line after a missing-program lookup, remains a check failure.
172
+ * A later passing check still counts as a retry (check-failure.ts), which an automatic release approval refuses.
173
+ */
174
+ export function launchFailedBeforeCheck(result, nonce, missing, platform = process.platform) {
175
+ if (missing === null || unstartedChecks.get(result) !== missing || platform === "win32" || result.code !== 127 || !verificationExecutableMissing(result, platform))
176
+ return false;
177
+ if (result.stdout.trim() !== "" || (nonce !== null && reportFromOutput(result.stdout, nonce).ok))
178
+ return false;
179
+ return true;
180
+ }
128
181
  /** Long enough for real work; short enough that a stuck build ends the same night. */
129
182
  export const DEFAULT_BUILD_TIMEOUT_MS = 30 * 60_000;
130
183
  export const DEFAULT_MAX_TURNS = 40;
@@ -139,10 +192,11 @@ export const REPAIR_MAX_TURNS = 4;
139
192
  export const REPAIR_TIMEOUT_MS = 5 * 60_000;
140
193
  /**
141
194
  * How often a running build says "still here" — extending its lease and its
142
- * runner's liveness in one beat. A minute against a three-minute liveness
143
- * window means two beats can be lost to load before anything looks dead.
195
+ * runner's liveness in one beat. Every 15 s against a one-minute liveness
196
+ * window (runner.ts) means three beats can be lost to load before anything
197
+ * looks dead.
144
198
  */
145
- export const DEFAULT_PULSE_MS = 60_000;
199
+ export const DEFAULT_PULSE_MS = RUNNER_HEARTBEAT_MS;
146
200
  /** Branches an unattended agent may never commit to, whatever it was asked. */
147
201
  export const PROTECTED = new Set(["main", "master", "trunk", "develop", "release"]);
148
202
  const GIT = "git";
@@ -326,12 +380,13 @@ export function proveApprovedProfile(scope, given) {
326
380
  acceptance: scope.acceptance, candidate: scope.candidate ?? null,
327
381
  qualityMode: scope.qualityMode ?? "default",
328
382
  releaseTerms: scope.releaseTerms ?? null,
383
+ releaseCheck: scope.releaseCheck ?? null,
329
384
  }, snapshot,
330
385
  // v47: the SEALED route is part of the signed bytes whenever it
331
386
  // says more than the legacy resolution — re-derived here from
332
387
  // the seal's own snapshot, never from mutable configuration.
333
388
  routeFromJson(scope.approvedRouteJson ?? null))
334
- : digestOf({ goal: scope.goal, outOfScope: scope.outOfScope, touches: scope.touches, budgetMicrousd: scope.budgetMicrousd, acceptance: scope.acceptance, candidate: scope.candidate ?? null, releaseTerms: scope.releaseTerms ?? null });
389
+ : digestOf({ goal: scope.goal, outOfScope: scope.outOfScope, touches: scope.touches, budgetMicrousd: scope.budgetMicrousd, acceptance: scope.acceptance, candidate: scope.candidate ?? null, releaseTerms: scope.releaseTerms ?? null, releaseCheck: scope.releaseCheck ?? null });
335
390
  if (rederived !== scope.approvedDigest) {
336
391
  return { ok: false, message: "the approval record does not verify against the stored terms — re-approve (stale-approval)" };
337
392
  }
@@ -647,7 +702,7 @@ export async function build(store, request) {
647
702
  if (refusal !== null)
648
703
  return { ok: false, reason: "no-op", message: refusal };
649
704
  // Check the exact candidate before setup or the full gate spends anything.
650
- const listed = await git(GIT, [...PREPARED_EVIDENCE_GIT, "ls-tree", "-z", preparedCandidate, "--", PREPARED_EVIDENCE_FILE], { cwd: worktree, maxBuffer: 2048 });
705
+ const listed = await git(GIT, [...preparedEvidenceGit(), "ls-tree", "-z", preparedCandidate, "--", PREPARED_EVIDENCE_FILE], { cwd: worktree, maxBuffer: 2048 });
651
706
  if (listed.code !== 0)
652
707
  return { ok: false, reason: "no-op", message: "The committed screenshot inventory could not be inspected. Restore the saved candidate before review." };
653
708
  const required = scope?.acceptance.some(criterion => criterion.evidence.includes("screenshot")) ?? false;
@@ -1936,11 +1991,16 @@ export async function settleProviderOutcome(captured, result) {
1936
1991
  store.setRunPhase(request.runId, "verifying-proof");
1937
1992
  await settleProof(captured, diffEvidence.statId, head);
1938
1993
  }
1939
- catch {
1994
+ catch (error) {
1940
1995
  // Adjudication itself must never fail the attempt — if even the
1941
1996
  // catch-all inside settleProof somehow throws, the build still
1942
1997
  // stands; the run simply has no verdict, which every surface
1943
- // treats the same as "no proof was written".
1998
+ // treats the same as "no proof was written". It is said, though:
1999
+ // release check 3264 ended with no log and no word why.
2000
+ try {
2001
+ store.addRunNote(request.runId, "Toolroll", `The check's result could not be saved: ${error instanceof Error ? error.message : String(error)}`.slice(0, 500), clock());
2002
+ }
2003
+ catch { /* the missing verdict still says enough */ }
1944
2004
  }
1945
2005
  }
1946
2006
  }
@@ -2226,9 +2286,12 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2226
2286
  const safe = hits.length > 0 ? redactSecretLines(value, hits) : value;
2227
2287
  return boundStreamHeadTail(safe, cap);
2228
2288
  };
2229
- const attemptOutcome = (label, result) => `${label}: (exit ${result.code}${result.notFound ? " · could not start" : ""}${result.timedOut ? " · timed out" : ""})`;
2289
+ // How the command ended, each fact apart: a signal or the plane's own hard stop is never read as a plain exit
2290
+ // (release check 3261 was recorded as "exit 1" when a signal had ended it).
2291
+ const endedBy = (result, sep) => `${result.signal ? `${sep}ended by ${result.signal}` : ""}${result.stoppedByPlane === true ? `${sep}stopped by Toolroll's own shutdown` : ""}`;
2292
+ const attemptOutcome = (label, result) => `${label}: (exit ${result.code}${result.notFound ? " · could not start" : ""}${result.timedOut ? " · timed out" : ""}${endedBy(result, " · ")})`;
2230
2293
  const attemptLog = (label, command, result) => {
2231
- const prefix = `=== ${label} ===\n$ ${command}\n(exit ${result.code}${result.notFound ? ", could not start" : ""}${result.timedOut ? ", timed out" : ""})\n\n--- stdout ---\n`;
2294
+ const prefix = `=== ${label} ===\n$ ${command}\n(exit ${result.code}${result.notFound ? ", could not start" : ""}${result.timedOut ? ", timed out" : ""}${endedBy(result, ", ")})\n\n--- stdout ---\n`;
2232
2295
  const between = "\n\n--- stderr ---\n";
2233
2296
  checkLogSourceBodyBytes += Buffer.byteLength(prefix) + Buffer.byteLength(result.stdout) + Buffer.byteLength(between) + Buffer.byteLength(result.stderr);
2234
2297
  return `${prefix}${boundedAttemptStream(result.stdout)}${between}${boundedAttemptStream(result.stderr)}`;
@@ -2303,6 +2366,7 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2303
2366
  else {
2304
2367
  const verifyRunner = request.verify ?? run;
2305
2368
  const verifyShell = approvedCommandShell(configured.command);
2369
+ let baselineUnverifiable = null;
2306
2370
  // A Full check of a prepared candidate (a release gate, the nightly check): its attempt is sealed before the command
2307
2371
  // runs — intended mode, nonce, inventory, context — and the handshake goes on its stdin (check-report.ts).
2308
2372
  // A release task's check is targeted when the release's last full check allows it (release-recheck.ts): its plan and
@@ -2310,36 +2374,55 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2310
2374
  if (repo !== null && chosenCheck?.level === "full" && (store.getScope(request.taskId)?.candidate ?? null) !== null) {
2311
2375
  const access = checkLaunchOf(store, runId, captured.effective.profile).access;
2312
2376
  const decision = planTargetedRecheck(store, root, { runId, taskId: request.taskId, repo, head: sealedHead, command: configured, access, now: now() });
2313
- checkAttempt = prepareCheckAttempt(store, root, { runId, repo, head: sealedHead, command: configured, access, authority: decision.authority,
2314
- ...(decision.mode === "targeted" ? { plan: decision.plan, snapshot: decision.snapshot } : {}), watch: decision.watch }, now());
2377
+ // A release's check (signed terms or an open authority): CI never stands in for its unit suite.
2378
+ // A release's check: signed terms, a release check purpose (pipeline-speed.md, P0) or an open authority.
2379
+ const release = decision.authority !== null || releaseCheckScope(store.getScope(request.taskId));
2380
+ try {
2381
+ checkAttempt = prepareCheckAttempt(store, root, { runId, repo, head: sealedHead, command: configured, access, authority: decision.authority,
2382
+ ...(decision.mode === "targeted" ? { plan: decision.plan, snapshot: decision.snapshot } : {}), watch: decision.watch, release }, now());
2383
+ }
2384
+ catch (error) {
2385
+ // A recorded last release that can't be verified (eval-harness review 2, finding 4): nothing runs.
2386
+ if (!(error instanceof CheckBaselineUnverifiable))
2387
+ throw error;
2388
+ checkAttempt = null;
2389
+ baselineUnverifiable = error.message;
2390
+ recordCheckNote(error.message);
2391
+ }
2315
2392
  if (checkAttempt !== null && decision.mode === "targeted")
2316
2393
  store.addRunNote(runId, "Toolroll", `Re-running only what failed at run #${decision.plan.base}: ${decision.plan.units.length} unit files, ${decision.plan.groups.reduce((n, one) => n + one.groups.length, 0)} journey groups, ${decision.plan.versions.length} upgrade versions${decision.plan.parts.length > 0 ? `, and in full: ${decision.plan.parts.join(", ")}` : ""}.`, now());
2317
2394
  }
2318
2395
  else
2319
2396
  checkAttempt = null;
2320
- const checkProgress = new CheckProgressTracker(snapshot => {
2321
- store.saveCheckProgress(runId, snapshot, now());
2322
- });
2323
- const runVerification = async (label) => {
2397
+ // Live progress is advisory (release check 3264 lost its log to one refused snapshot): display snapshots are
2398
+ // coalesced (at most one write a second), the final one is kept until it is written, and a failing save never
2399
+ // throws out of the writer, so the check's log, report and receipt are always sealed.
2400
+ const progressWrites = coalescedProgressWriter(snapshot => { store.saveCheckProgress(runId, snapshot, now()); });
2401
+ const checkProgress = new CheckProgressTracker(snapshot => progressWrites.push(snapshot));
2402
+ const runVerification = async (label, missing = null) => {
2324
2403
  const streamed = { stdout: false, stderr: false };
2325
2404
  // The check runs under the stop watch (v52), owned by this run.
2326
- const result = await underStopWatch(store, runId, () => runWithIsolatedDatabase(witnessedRunner(store, runId, now, commandRunner(store, runId, captured.effective.profile, verifyRunner)), verifyShell.file, verifyShell.args, {
2327
- cwd: worktree,
2328
- timeoutMs: configured.timeoutMs,
2329
- processGroup: true,
2330
- owner: runOwnerTag(store, runId),
2331
- beforeSpawn: () => !stopRequestedFor(store, runId, request.shouldStop),
2332
- onSpawn: pid => {
2333
- request.onProviderSpawn?.(pid);
2334
- if (stopRequestedFor(store, runId, request.shouldStop))
2335
- throw new Error("the attempt was stopped before spawn custody completed");
2336
- },
2337
- envAllowlist: SETUP_ENV_ALLOWLIST,
2338
- omitEnv: SETUP_ENV_DENYLIST,
2339
- ...(checkAttempt === null ? {} : { stdin: checkAttempt.stdin }),
2340
- onStdout: chunk => { streamed.stdout = true; checkProgress.feed(chunk, "stdout"); },
2341
- onStderr: chunk => { streamed.stderr = true; checkProgress.feed(chunk, "stderr"); },
2342
- }));
2405
+ const result = missing !== null
2406
+ ? { code: 127, stdout: "", stderr: `Toolroll did not start the check: ${missing} wasn't found.`, timedOut: false, notFound: false }
2407
+ : await underStopWatch(store, runId, () => runWithIsolatedDatabase(witnessedRunner(store, runId, now, commandRunner(store, runId, captured.effective.profile, verifyRunner)), verifyShell.file, verifyShell.args, {
2408
+ cwd: worktree,
2409
+ timeoutMs: configured.timeoutMs,
2410
+ processGroup: true,
2411
+ owner: runOwnerTag(store, runId),
2412
+ beforeSpawn: () => !stopRequestedFor(store, runId, request.shouldStop),
2413
+ onSpawn: pid => {
2414
+ request.onProviderSpawn?.(pid);
2415
+ if (stopRequestedFor(store, runId, request.shouldStop))
2416
+ throw new Error("the attempt was stopped before spawn custody completed");
2417
+ },
2418
+ envAllowlist: SETUP_ENV_ALLOWLIST,
2419
+ omitEnv: SETUP_ENV_DENYLIST,
2420
+ ...(checkAttempt === null ? {} : { stdin: checkAttempt.stdin }),
2421
+ onStdout: chunk => { streamed.stdout = true; checkProgress.feed(chunk, "stdout"); },
2422
+ onStderr: chunk => { streamed.stderr = true; checkProgress.feed(chunk, "stderr"); },
2423
+ }));
2424
+ if (missing !== null)
2425
+ unstartedChecks.set(result, missing);
2343
2426
  if (!streamed.stdout)
2344
2427
  checkProgress.feed(result.stdout, "stdout");
2345
2428
  if (!streamed.stderr)
@@ -2388,15 +2471,21 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2388
2471
  return "own-install-failed";
2389
2472
  return await sealedTreeState(buildCheckoutGit(request.git ?? run, worktree)) === "clean" ? null : "setup-changed-files";
2390
2473
  };
2391
- const ownInstall = await ownInstallIfChanged();
2392
- const first = ownInstall === null ? await runVerification("Project check · attempt 1") : null;
2474
+ const ownInstall = baselineUnverifiable === null ? await ownInstallIfChanged() : null;
2475
+ // A missing program short-circuits the first launch; never execute build code and then infer that nothing ran.
2476
+ const missingProgram = configured.recoverySetupDigest === null ? null : approvedProgramMissing(configured.command, worktree, process.env["PATH"]);
2477
+ const first = baselineUnverifiable === null && ownInstall === null ? await runVerification("Project check · attempt 1", missingProgram) : null;
2393
2478
  if (first === null) {
2394
- verifyCommand = { configured: true, ran: false, attemptFailed: true, failure: ownInstall ?? "own-install-failed" };
2479
+ verifyCommand = { configured: true, ran: false, attemptFailed: true, failure: baselineUnverifiable !== null ? "baseline-unverifiable" : ownInstall ?? "own-install-failed" };
2395
2480
  }
2396
2481
  else if (first.notFound || first.timedOut) {
2397
2482
  verifyCommand = { configured: true, ran: false, attemptFailed: true, failure: first.timedOut ? "timed-out" : "spawn-failed" };
2398
2483
  }
2399
- else if (!verificationExecutableMissing(first)) {
2484
+ else if (first.stoppedByPlane === true) {
2485
+ // This worker's own hard stop ended it (a shutdown): it never finished, and its exit says nothing about the code.
2486
+ verifyCommand = { configured: true, ran: false, attemptFailed: true, failure: "stopped-by-toolroll" };
2487
+ }
2488
+ else if (!launchFailedBeforeCheck(first, checkAttempt?.nonce ?? null, missingProgram)) {
2400
2489
  verifyCommand = { configured: true, ran: true, exitCode: first.code };
2401
2490
  }
2402
2491
  else {
@@ -2511,7 +2600,8 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2511
2600
  verifyCommand = { configured: true, ran: false, attemptFailed: true, failure: "dependency-still-missing" };
2512
2601
  }
2513
2602
  else {
2514
- verifyCommand = { configured: true, ran: true, exitCode: retried.code, setupReplayed: true };
2603
+ // The first attempt, classified on its own: it never started (its program was missing).
2604
+ verifyCommand = { configured: true, ran: true, exitCode: retried.code, setupReplayed: true, firstAttempt: { class: "launch", program: missingProgram, exitCode: first.code } };
2515
2605
  }
2516
2606
  }
2517
2607
  }
@@ -2519,11 +2609,21 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2519
2609
  }
2520
2610
  }
2521
2611
  }
2522
- checkProgress.finish();
2612
+ // The display's final snapshot (Codex review round 1, finding 6): the writer keeps it until a write succeeds and
2613
+ // never throws; a display failure of any kind never keeps the check's outcome below from being sealed.
2614
+ try {
2615
+ checkProgress.finish();
2616
+ }
2617
+ catch { /* display only */ }
2618
+ if (!progressWrites.flush())
2619
+ progressWrites.flush();
2523
2620
  // The check's own report (the last line carrying the nonce), sealed beside what was sealed before it ran.
2524
2621
  const last = verification.last;
2622
+ let sealedReport = null;
2525
2623
  if (checkAttempt !== null && last !== null && !last.notFound) {
2526
- sealCheckOutcome(store, root, checkAttempt, { stdout: last.stdout, exitCode: last.timedOut ? null : last.code }, now());
2624
+ sealedReport = sealCheckOutcome(store, root, checkAttempt, { stdout: last.stdout, exitCode: last.timedOut ? null : last.code }, now());
2625
+ // Where the check's time went (latency-timeline.ts): each part's start and end, display only.
2626
+ keepCheckTimeline(store, { runId, taskId: request.taskId, report: sealedReport, stdout: last.stdout, now: now() });
2527
2627
  }
2528
2628
  // What the check covered (receipt version 3): full, composed from the release's last full check, partial, or
2529
2629
  // identity-removed (which stops the release carrying).
@@ -2532,10 +2632,15 @@ async function settleProof(captured, statArtifactId, sealedHead) {
2532
2632
  const sha = (capture) => { const found = store.artifactsFor(runId).filter(a => a.kind === "structured-output" && a.capture === capture); return found.length === 1 ? found[0].sha256 : null; };
2533
2633
  // The one place coverage meets the outcome: a partial or identity-removed check is a failed attempt for every
2534
2634
  // reader from here on (the verdict, the run's check, the receipt, reuse, the deploy gates).
2535
- const effective = effectiveCheckFacts(verifyCommand, { coverage: settled.coverage, check: { attempt: checkAttempt.mode, report: sha(CHECK_REPORT_CAPTURE), inventory: sha(CHECK_INVENTORY_CAPTURE) }, composed: settled.composed });
2635
+ const effective = effectiveCheckFacts(verifyCommand, { coverage: settled.coverage, check: { attempt: checkAttempt.mode, report: sha(CHECK_REPORT_CAPTURE), inventory: sha(CHECK_INVENTORY_CAPTURE), retried: retriedInside(sealedReport) }, composed: settled.composed });
2536
2636
  verifyCommand = effective.result;
2537
2637
  releaseFacts = effective.facts;
2538
2638
  }
2639
+ else if (verifyCommand.configured && verifyCommand.ran && chosenCheck?.level === "full" && releaseCheckScope(store.getScope(request.taskId))) {
2640
+ // A release's Full check that couldn't be sealed before it ran (no nonce, no inventory) has no result it can count.
2641
+ verifyCommand = { configured: true, ran: false, attemptFailed: true, failure: "no-result" };
2642
+ recordCheckNote("The release check ran without a sealed attempt (its commit couldn't be read before it ran), so it has no result that can count.");
2643
+ }
2539
2644
  }
2540
2645
  if (configured !== null && checkLog.length > 0) {
2541
2646
  const summary = `=== Attempt summary ===\n${checkOutcomes.length === 0 ? "No command started." : checkOutcomes.map(one => `- ${one}`).join("\n")}`;
@@ -30,6 +30,7 @@ import { requestResultChanges } from "./result-actions.js";
30
30
  import { revisionSourceOf } from "./result-review.js";
31
31
  import { APPROVAL_PROOF, approvalOf, approvalProofDetail, deployGateScope, quarantinedAuthor } from "./scope.js";
32
32
  import { releaseCardOf } from "./release-card.js";
33
+ import { releaseCiHoldOf, releaseCiHoldWords } from "./release-ci-hold.js";
33
34
  import { phoneText } from "./telegram-status.js";
34
35
  /** The words, said once. */
35
36
  export const DECIDE_LABELS = {
@@ -662,6 +663,9 @@ export function applyDecideTap(store, seat, input) {
662
663
  }
663
664
  case "approve": {
664
665
  if (row.digest.startsWith(RELEASE_DIGEST)) {
666
+ const ciHold = releaseCiHoldOf(store, row.taskId);
667
+ if (ciHold !== null)
668
+ return notDone(releaseCiHoldWords(ciHold));
665
669
  const release = releaseInChat(store, row.taskId, who.name, now, seat.channel, seat, input.root);
666
670
  if (!release.ok)
667
671
  return notDone(release.why);
@@ -0,0 +1,92 @@
1
+ import { type CheckReport } from "./contracts/release-check.js";
2
+ import { INFRA_RERUNS_PER_CANDIDATE, INFRA_RERUNS_PER_INCIDENT, type FailureClassName } from "./release.js";
3
+ import type { Store } from "./store.js";
4
+ export type FailureClass = FailureClassName;
5
+ export type ClassifiedFailure = {
6
+ class: FailureClass;
7
+ why: string;
8
+ };
9
+ /** The plane's limits for confirmed infrastructure failures (one rerun per incident, two per candidate). */
10
+ export { INFRA_RERUNS_PER_CANDIDATE, INFRA_RERUNS_PER_INCIDENT };
11
+ /** The ledger action that records each rerun the release asked for, and why. */
12
+ export declare const RELEASE_RECHECK_ACTION = "release check rerun";
13
+ /** What a sealed verification receipt says about the check (receipt.result, and its coverage). */
14
+ export type CheckFacts = {
15
+ result: {
16
+ configured: boolean;
17
+ ran?: boolean;
18
+ exitCode?: number;
19
+ attemptFailed?: boolean;
20
+ failure?: string;
21
+ };
22
+ coverage?: string | undefined;
23
+ };
24
+ /**
25
+ * One failed check, classified. `facts` come from the plane's sealed receipt; `report` (the check's own report, sealed
26
+ * when valid) only ever picks among the classes that block. Null when the check passed.
27
+ */
28
+ export declare function classifyCheckFailure(facts: CheckFacts, report: CheckReport | null): ClassifiedFailure | null;
29
+ /** What passed only on a retry inside a passing check (a real-model journey group, a unit file): counted and reported,
30
+ * never silently part of a pass. Null when there is no valid report to read it from: unknown, never "no retries". */
31
+ export declare function retriedInside(report: CheckReport | null): string[] | null;
32
+ /** A run's check, classified from its sealed receipt and sealed report; null when it passed, or can't be read (then
33
+ * `unreadable` says why — an unreadable receipt is itself unknown). */
34
+ export declare function classifyRunCheck(store: Store, root: string, runId: number): ClassifiedFailure | null;
35
+ /**
36
+ * What passed only on a retry inside a run's check (eval-harness review 1, finding 3): the summary sealed into its
37
+ * protected receipt, which a damaged or lost report can't take away; for an older receipt without one, read from the
38
+ * sealed report. Null — unknown — when neither can be read, or when a readable report disagrees with the receipt.
39
+ */
40
+ export declare function retriedOfRun(store: Store, root: string, runId: number): string[] | null;
41
+ /** How a check rerun after the approved recovery setup reads among the retries. */
42
+ export declare const SETUP_REPLAY_RETRY = "the whole check, rerun after the approved setup";
43
+ /**
44
+ * The rerun's words with its first attempt's own class (eval-harness review 2, finding 5): a launch failure names the
45
+ * missing program; a receipt that doesn't classify it (an older one) says so, never that it was a launch failure.
46
+ */
47
+ export declare function setupReplayWords(first: unknown): string;
48
+ /**
49
+ * Whether a passing check of a release task stands, given every earlier attempt on the same candidate and signed scope:
50
+ * each earlier failure must be confirmed infrastructure the release itself reran (at most two), and nothing in the
51
+ * passing run may have passed only on a retry. A deterministic, unknown or agent failure before it isn't erased by a
52
+ * later pass: a person decides. `retried` is reported even when it stands (a person's approval reads it; an automatic
53
+ * one refuses on it).
54
+ */
55
+ export declare function releaseCheckStanding(store: Store, root: string, input: {
56
+ taskId: string;
57
+ run: number;
58
+ }): {
59
+ ok: true;
60
+ earlier: {
61
+ run: number;
62
+ why: string;
63
+ }[];
64
+ retried: string[] | null;
65
+ } | {
66
+ ok: false;
67
+ why: string;
68
+ retried: string[] | null;
69
+ };
70
+ export declare const classWords: (kind: FailureClass) => string;
71
+ export type RecheckAnswer = {
72
+ action: "rerun";
73
+ class: "infrastructure";
74
+ why: string;
75
+ source: number;
76
+ run: number;
77
+ } | {
78
+ action: "pending";
79
+ source: number;
80
+ } | {
81
+ action: "stop";
82
+ class: FailureClass | null;
83
+ why: string;
84
+ source: number | null;
85
+ };
86
+ /**
87
+ * After a failed release check: the plane classifies the task's newest attempt from its own records and, only for
88
+ * confirmed infrastructure within its limits (one rerun per incident, two per candidate), runs the check again on the
89
+ * same candidate under the same approval (dispose.ts regateTask keeps candidate, scope and approval). Every attempt
90
+ * stays in the evidence; each rerun is in the ledger with its class. Anything else stops, saying which class and why.
91
+ */
92
+ export declare function recheckReleaseTask(store: Store, root: string, taskId: string, now: Date): RecheckAnswer;