toolroll 0.9.71 → 0.9.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -0
- package/dist/agent-fence.d.ts +10 -2
- package/dist/agent-fence.js +47 -9
- package/dist/approval-authority.d.ts +20 -0
- package/dist/approval-authority.js +67 -0
- package/dist/assignment.js +26 -4
- package/dist/browser/workspace.css +1 -1
- package/dist/browser/workspace.js +28 -28
- package/dist/builder.js +48 -6
- package/dist/chat-actions.js +13 -8
- package/dist/chat-decide.d.ts +47 -1
- package/dist/chat-decide.js +186 -5
- package/dist/check-access.js +2 -2
- package/dist/check-report.d.ts +109 -0
- package/dist/check-report.js +205 -0
- package/dist/claim.js +1 -1
- package/dist/coding-provider.js +4 -2
- package/dist/commit-extract.d.ts +111 -0
- package/dist/commit-extract.js +514 -0
- package/dist/contracts/console-api.d.ts +4 -4
- package/dist/contracts/console-api.js +5 -4
- package/dist/contracts/flow.d.ts +16 -0
- package/dist/contracts/flow.js +3 -0
- package/dist/contracts/release-check.d.ts +335 -0
- package/dist/contracts/release-check.js +159 -0
- package/dist/contracts/release-terms.d.ts +66 -0
- package/dist/contracts/release-terms.js +74 -0
- package/dist/contracts/verification-receipt.d.ts +42 -8
- package/dist/contracts/verification-receipt.js +24 -7
- package/dist/contracts/verification-view.d.ts +1 -0
- package/dist/deploy-gate.d.ts +17 -0
- package/dist/deploy-gate.js +68 -0
- package/dist/dispatch.js +3 -10
- package/dist/dispose.js +4 -1
- package/dist/evidence-pack.d.ts +1 -1
- package/dist/evidence-pack.js +20 -2
- package/dist/evidence.d.ts +1 -0
- package/dist/evidence.js +2 -1
- package/dist/exec.d.ts +1 -1
- package/dist/exec.js +7 -3
- package/dist/flow-engine.js +5 -1
- package/dist/flow-gallery-ui.js +4 -2
- package/dist/flow-gallery.d.ts +3 -0
- package/dist/flow-gallery.js +6 -2
- package/dist/flow-pull-request.d.ts +5 -2
- package/dist/flow-pull-request.js +14 -5
- package/dist/flow-schedule.js +25 -6
- package/dist/flow-triggers.d.ts +8 -0
- package/dist/flow-triggers.js +15 -0
- package/dist/flows-cli.js +13 -3
- package/dist/flows-ui.js +7 -1
- package/dist/guides.js +10 -1
- package/dist/invoke.d.ts +17 -6
- package/dist/invoke.js +48 -18
- package/dist/kits-ui.js +4 -2
- package/dist/kits.d.ts +6 -4
- package/dist/kits.js +9 -11
- package/dist/lead-status.js +5 -2
- package/dist/live-terminal.d.ts +10 -6
- package/dist/live-terminal.js +55 -14
- package/dist/mcp-connect.d.ts +10 -0
- package/dist/mcp-connect.js +2 -0
- package/dist/nightly-check.d.ts +104 -0
- package/dist/nightly-check.js +362 -0
- package/dist/operate.js +39 -2
- package/dist/person-proof.d.ts +2 -0
- package/dist/person-proof.js +14 -0
- package/dist/plan-auto.js +4 -4
- package/dist/planner-source.d.ts +1 -1
- package/dist/project-delete-ui.js +1 -1
- package/dist/project-tools.js +3 -1
- package/dist/proof.js +2 -0
- package/dist/protected-storage.d.ts +67 -0
- package/dist/protected-storage.js +398 -0
- package/dist/publish.js +156 -144
- package/dist/pull-request-flow.d.ts +2 -0
- package/dist/pull-request-flow.js +24 -2
- package/dist/release-adapters.d.ts +16 -1
- package/dist/release-adapters.js +292 -29
- package/dist/release-authority.d.ts +118 -0
- package/dist/release-authority.js +135 -0
- package/dist/release-card.d.ts +76 -0
- package/dist/release-card.js +252 -0
- package/dist/release-carry.d.ts +236 -0
- package/dist/release-carry.js +664 -0
- package/dist/release-cli.d.ts +3 -0
- package/dist/release-cli.js +158 -6
- package/dist/release-coverage.js +3 -0
- package/dist/release-git.d.ts +43 -0
- package/dist/release-git.js +106 -0
- package/dist/release-recheck.d.ts +146 -0
- package/dist/release-recheck.js +657 -0
- package/dist/release-view.d.ts +34 -0
- package/dist/release-view.js +27 -0
- package/dist/release.d.ts +139 -3
- package/dist/release.js +641 -250
- package/dist/remote-main.d.ts +2 -0
- package/dist/remote-main.js +1 -1
- package/dist/result-completion.d.ts +6 -0
- package/dist/result-completion.js +15 -1
- package/dist/result-follow-ups.js +29 -1
- package/dist/sandbox-runtime.d.ts +16 -5
- package/dist/sandbox-runtime.js +19 -6
- package/dist/scope.d.ts +122 -5
- package/dist/scope.js +236 -7
- package/dist/server/flows.js +40 -10
- package/dist/server/render-settings.js +1 -1
- package/dist/server/render-tasks.d.ts +8 -0
- package/dist/server/render-tasks.js +50 -9
- package/dist/server/route-table.js +1 -0
- package/dist/server/settings.js +26 -5
- package/dist/server/task-views.js +4 -0
- package/dist/server/tasks.js +12 -0
- package/dist/setup-guide.d.ts +2 -2
- package/dist/store.d.ts +230 -5
- package/dist/store.js +733 -37
- package/dist/subagent-memory.d.ts +1 -0
- package/dist/subagent-memory.js +1 -1
- package/dist/subagent-tools.d.ts +22 -6
- package/dist/subagent-tools.js +44 -8
- package/dist/subagent-work.d.ts +2 -0
- package/dist/subagent-work.js +5 -2
- package/dist/subagents-ui.js +6 -4
- package/dist/task-outcome-cli.js +7 -3
- package/dist/telegram.d.ts +6 -1
- package/dist/telegram.js +58 -6
- package/dist/toolroll-update.js +2 -2
- package/dist/verification-evidence.d.ts +25 -1
- package/dist/verification-evidence.js +57 -2
- package/dist/work-index.js +3 -6
- package/package.json +1 -1
package/dist/release.js
CHANGED
|
@@ -5,7 +5,11 @@
|
|
|
5
5
|
*
|
|
6
6
|
* gate → approval → check → complete → quiet → deploy → link → ci → merge → tree → tag → publish → homebrew → done
|
|
7
7
|
*
|
|
8
|
-
* gate
|
|
8
|
+
* gate, before filing a task for that commit, looks at the pull request's required checks on it: one that failed stops
|
|
9
|
+
* the release before anything is filed or sent for approval; it waits, naming them, for checks not started or still
|
|
10
|
+
* running, and stops if they don't finish within its limit (a task already filed is not looked at again; ci checks
|
|
11
|
+
* once more before merging). It then files a prepared-candidate task pinned to the branch's exact commit, with the full
|
|
12
|
+
* release check. approval
|
|
9
13
|
* waits for an active approver's yes to that exact scope (never a mode's, an AI's or the lead's). check requires a Full
|
|
10
14
|
* result of that commit: a failed check stops the release and stays visible, nothing reruns it. complete marks that
|
|
11
15
|
* exact result Complete through the task's own authorization. quiet waits until nothing builds. deploy runs the
|
|
@@ -31,6 +35,7 @@
|
|
|
31
35
|
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
32
36
|
import { randomUUID } from "node:crypto";
|
|
33
37
|
import { dirname, join } from "node:path";
|
|
38
|
+
import { personProof } from "./person-proof.js";
|
|
34
39
|
/** The pull-request checks a release waits for, by exact job name (.github/workflows/ci.yml). Every one must have
|
|
35
40
|
* concluded `success` on the gated commit; src/ci-workflow.test.ts holds the workflow to this list. */
|
|
36
41
|
export const REQUIRED_PR_CHECKS = [
|
|
@@ -44,7 +49,7 @@ export const REQUIRED_PR_CHECKS = [
|
|
|
44
49
|
export const RELEASE_TAG_CHECKS = ["node 22 · macos-latest", "node 24 · macos-latest", "publish"];
|
|
45
50
|
export const RELEASE_STEPS = ["gate", "approval", "check", "complete", "quiet", "deploy", "link", "ci", "merge", "tree", "tag", "publish", "homebrew", "done"];
|
|
46
51
|
/** How long each waiting step may take, in minutes, before the release stops there (rerunning continues from it). */
|
|
47
|
-
export const RELEASE_LIMITS = { approval: 24 * 60, check: 120, quiet: 60, deploy: 30, ci: 60, publish: 45, homebrew: 45 };
|
|
52
|
+
export const RELEASE_LIMITS = { gate: 60, approval: 24 * 60, check: 120, quiet: 60, deploy: 30, ci: 60, publish: 45, homebrew: 45 };
|
|
48
53
|
const POLL_MS = 15_000;
|
|
49
54
|
/** A stop the release reports: the step it stopped at and why, in words. */
|
|
50
55
|
class Stop extends Error {
|
|
@@ -54,6 +59,21 @@ class Stop extends Error {
|
|
|
54
59
|
this.reason = reason;
|
|
55
60
|
}
|
|
56
61
|
}
|
|
62
|
+
/** The branch moved before the deploy reservation: carry the approval to the new head, or file it for a person. */
|
|
63
|
+
class Restart extends Error {
|
|
64
|
+
head;
|
|
65
|
+
constructor(head) {
|
|
66
|
+
super(`restart at ${head}`);
|
|
67
|
+
this.head = head;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/** Every commit that landed after the approval, in words: the summary and the merge message list them all. */
|
|
71
|
+
export function landedWords(landed) {
|
|
72
|
+
if (landed.length === 0)
|
|
73
|
+
return [];
|
|
74
|
+
return [`Landed after the approval (${landed.length} commit${landed.length === 1 ? "" : "s"}):`,
|
|
75
|
+
...landed.map(one => `- ${one.sha.slice(0, 7)}: ${one.files.map(file => `${file.path} (+${file.added} −${file.deleted})`).join(", ")}`)];
|
|
76
|
+
}
|
|
57
77
|
const branchFile = (branch) => branch.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
58
78
|
export const journalFile = (stateDir, branch) => join(stateDir, "releases", `${branchFile(branch)}.json`);
|
|
59
79
|
export const resumeCommand = (branch, checkout) => `toolroll release ${branch} --repo ${checkout}`;
|
|
@@ -98,12 +118,22 @@ export function ownerApproval(gate, digest) {
|
|
|
98
118
|
return { ok: false, why: "not approved yet" };
|
|
99
119
|
if (digest === null || approval.digest !== digest)
|
|
100
120
|
return { ok: false, why: "the approval is for different scope text" };
|
|
121
|
+
if (approval.basis === "lead")
|
|
122
|
+
return { ok: false, why: "approved by the lead, not by a person" };
|
|
101
123
|
if (approval.basis !== "password" && approval.basis !== "chat")
|
|
102
124
|
return { ok: false, why: `approved by ${approval.basis ?? "an unknown road"}, not by a person` };
|
|
103
125
|
if (/ \(AI\)$/.test(approval.by) || /^mode /.test(approval.by) || /^lead\b/.test(approval.by) || approval.by === "routine")
|
|
104
126
|
return { ok: false, why: `approved as ${approval.by}, not by a person` };
|
|
105
127
|
if (approval.role !== "approver" || !approval.active)
|
|
106
128
|
return { ok: false, why: `approved by ${approval.by}, who is not an active approver` };
|
|
129
|
+
// A person's proof, recorded for these exact bytes (approval policy §1.5): a lead token or a remote API token sealed
|
|
130
|
+
// through the person's road before the fix said `password` too. A plane that can't say is not a yes.
|
|
131
|
+
if (approval.person === undefined)
|
|
132
|
+
return { ok: false, why: "the plane can't show a person gave this approval; update it first" };
|
|
133
|
+
if (!approval.person.ok)
|
|
134
|
+
return { ok: false, why: approval.person.why };
|
|
135
|
+
if (!personProof(approval.person.method))
|
|
136
|
+
return { ok: false, why: "approved with a machine credential, not by a person" };
|
|
107
137
|
return { ok: true, by: approval.by, at: approval.at };
|
|
108
138
|
}
|
|
109
139
|
/** Every required check present, and every reported check's newest attempt completed successfully. */
|
|
@@ -133,6 +163,8 @@ export async function runRelease(options, adapters) {
|
|
|
133
163
|
const say = adapters.say;
|
|
134
164
|
let journal = null;
|
|
135
165
|
let step = "preflight";
|
|
166
|
+
let freshApproval = false;
|
|
167
|
+
let supersededTask = null;
|
|
136
168
|
const save = () => { journal.updatedAt = adapters.now().toISOString(); writeJournal(file, journal); };
|
|
137
169
|
const advance = (next) => { journal.step = next; journal.stopped = null; save(); };
|
|
138
170
|
const intend = (at) => { journal.intents[at] = adapters.now().toISOString(); save(); };
|
|
@@ -149,16 +181,29 @@ export async function runRelease(options, adapters) {
|
|
|
149
181
|
delete j.intents.deploy;
|
|
150
182
|
save();
|
|
151
183
|
};
|
|
152
|
-
/** Ask `probe` until it answers, within this step's limit.
|
|
184
|
+
/** Ask `probe` until it answers, within this step's limit. A short network failure on the way to GitHub (git or gh
|
|
185
|
+
* timing out) is no answer yet, never the release's end: the next poll asks again, until the step's own limit. */
|
|
153
186
|
const waitFor = async (at, what, probe) => {
|
|
154
187
|
const deadline = adapters.now().getTime() + limits[at] * 60_000;
|
|
155
|
-
let said = false;
|
|
188
|
+
let said = false, unreachable = null;
|
|
156
189
|
for (;;) {
|
|
157
|
-
|
|
190
|
+
let answer;
|
|
191
|
+
try {
|
|
192
|
+
answer = await probe();
|
|
193
|
+
unreachable = null;
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (!transientNetworkError(error))
|
|
197
|
+
throw error;
|
|
198
|
+
if (unreachable === null)
|
|
199
|
+
say(` GitHub didn't answer in time (${networkWords(error)}); asking again`);
|
|
200
|
+
unreachable = networkWords(error);
|
|
201
|
+
answer = null;
|
|
202
|
+
}
|
|
158
203
|
if (answer !== null)
|
|
159
204
|
return answer;
|
|
160
205
|
if (adapters.now().getTime() >= deadline)
|
|
161
|
-
throw new Stop("timeout", `Still waiting for ${what} after ${limits[at]} minutes.`);
|
|
206
|
+
throw new Stop("timeout", `Still waiting for ${what} after ${limits[at]} minutes${unreachable === null ? "" : ` (GitHub last didn't answer: ${unreachable})`}.`);
|
|
162
207
|
if (!said) {
|
|
163
208
|
say(` waiting for ${what} (up to ${limits[at]} min)`);
|
|
164
209
|
said = true;
|
|
@@ -175,6 +220,8 @@ export async function runRelease(options, adapters) {
|
|
|
175
220
|
const aside = file.replace(/\.json$/, `.superseded-${adapters.now().toISOString().replace(/[:.]/g, "-")}.json`);
|
|
176
221
|
renameSync(file, aside);
|
|
177
222
|
say(`Kept the earlier release journal at ${aside}.`);
|
|
223
|
+
freshApproval = journal.needsApproval === true;
|
|
224
|
+
supersededTask = journal.task.id;
|
|
178
225
|
journal = null;
|
|
179
226
|
}
|
|
180
227
|
if (journal?.step === "done") {
|
|
@@ -183,8 +230,28 @@ export async function runRelease(options, adapters) {
|
|
|
183
230
|
}
|
|
184
231
|
if (!await adapters.github.authenticated())
|
|
185
232
|
throw new Stop("github-auth", "GitHub CLI is not signed in: run `gh auth login`, then rerun.");
|
|
186
|
-
if (journal === null)
|
|
233
|
+
if (journal === null) {
|
|
187
234
|
journal = await begin(options, adapters, adapters.now());
|
|
235
|
+
// The earlier attempt needs a person's fresh approval (a CI failure after its deploy): this one files a task of its
|
|
236
|
+
// own for the same commit, so the old approval can't carry it.
|
|
237
|
+
// Only an id that has never existed, numbered after the superseded one: never a task whose approval still stands.
|
|
238
|
+
if (freshApproval) {
|
|
239
|
+
const base = journal.task.id;
|
|
240
|
+
const suffix = supersededTask === null ? null : supersededTask === base ? 1 : supersededTask.startsWith(`${base}-`) ? Number(supersededTask.slice(base.length + 1)) : null;
|
|
241
|
+
let picked = null;
|
|
242
|
+
for (let n = suffix !== null && Number.isInteger(suffix) && suffix > 0 ? suffix + 1 : 2; n < 1000; n++) {
|
|
243
|
+
const id = `${base}-${n}`;
|
|
244
|
+
if (!(await adapters.toolroll.gate(id)).exists) {
|
|
245
|
+
picked = id;
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (picked === null)
|
|
250
|
+
throw new Stop("approval", `No unused task id is left for a fresh approval of ${base}.`);
|
|
251
|
+
journal.task = { id: picked, scopeDigest: null };
|
|
252
|
+
journal.freshTask = true;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
188
255
|
else
|
|
189
256
|
say(`Resuming the release of ${journal.tag} (${journal.candidate.sha.slice(0, 7)}) at ${journal.step}.`);
|
|
190
257
|
const j = journal;
|
|
@@ -201,9 +268,169 @@ export async function runRelease(options, adapters) {
|
|
|
201
268
|
throw new Stop("drift", `Pull request #${j.pr} no longer has ${j.candidate.sha.slice(0, 7)} at its head.`);
|
|
202
269
|
return pr;
|
|
203
270
|
};
|
|
204
|
-
|
|
271
|
+
// ---- one approval per release (release-once.md) -------------------------------------------------------------
|
|
272
|
+
const rel = adapters.release;
|
|
273
|
+
let epoch = null;
|
|
274
|
+
// This invocation's own lease owner (review round 1): two runs of the same journal never share an epoch.
|
|
275
|
+
const owner = `${j.id}:${randomUUID()}`;
|
|
276
|
+
/** Take the release's lease (a takeover raises its epoch), then settle every operation still open before any new work. */
|
|
277
|
+
const hold = async () => {
|
|
278
|
+
if (rel === undefined || !j.authority)
|
|
279
|
+
return;
|
|
280
|
+
const lease = await rel.lease(j.authority.id, owner);
|
|
281
|
+
if (!lease.ok)
|
|
282
|
+
throw new Stop("lease", lease.why);
|
|
283
|
+
epoch = lease.epoch;
|
|
284
|
+
await reconcile();
|
|
285
|
+
};
|
|
286
|
+
const reconcile = async () => {
|
|
287
|
+
if (rel === undefined || !j.authority)
|
|
288
|
+
return;
|
|
289
|
+
for (const op of (await rel.operations(j.authority.id)).filter(one => one.state === "reserved" || one.state === "running" || one.state === "unknown")) {
|
|
290
|
+
// A started call whose issuer may still be alive may still land (review round 2): nothing is settled for it on
|
|
291
|
+
// elapsed time. It stays as it is, and a person looks.
|
|
292
|
+
const inFlight = () => new Stop("operation-in-flight", `An earlier ${op.kind} (operation ${op.id}) was started by a process that may still be running. Nothing was settled: check that process, then rerun.`);
|
|
293
|
+
if (op.kind === "deploy") {
|
|
294
|
+
// Reserved and never started: nothing to undo (a start after this takeover is refused: its epoch moved).
|
|
295
|
+
if (op.state === "reserved") {
|
|
296
|
+
await rel.finish(op.id, "failed", { detail: "the call never started" }, epoch);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
if (await served(adapters, builtHead()) !== null) {
|
|
300
|
+
await rel.finish(op.id, "succeeded", { detail: "reconciled: the services run the candidate" }, epoch);
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
if (!op.issuerGone)
|
|
304
|
+
throw inFlight();
|
|
305
|
+
if (op.direct) {
|
|
306
|
+
// A person's own deploy: its process is gone; its lock, update pause and journal must say nothing is half done.
|
|
307
|
+
if (!await adapters.toolroll.deploymentIdle(op.stage))
|
|
308
|
+
throw new Stop("deploy-recovery", `A person's deploy of this release (operation ${op.id}) ended without finishing${op.stage === null ? "" : `; its journal is in ${op.stage}`}. Recover it, then rerun.`);
|
|
309
|
+
await rel.finish(op.id, "failed", { detail: "reconciled: the deploy's process ended, and nothing is deployed or half done" }, epoch);
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
await recoverDeploy();
|
|
313
|
+
await rel.finish(op.id, "failed", { detail: "reconciled: recovered the previous build" }, epoch);
|
|
314
|
+
}
|
|
315
|
+
else {
|
|
316
|
+
// The pull request decides an unknown merge: merged from the gated head is a success, whoever issued it.
|
|
317
|
+
const pr = await adapters.github.pullRequest(j.github, j.branch);
|
|
318
|
+
if (pr?.state === "merged" && pr.number === j.pr && pr.head === op.candidate && pr.mergeCommit !== null)
|
|
319
|
+
await rel.finish(op.id, "succeeded", { externalRef: pr.mergeCommit, detail: "reconciled: the pull request is merged from the gated head" }, epoch);
|
|
320
|
+
else if (op.state === "reserved")
|
|
321
|
+
await rel.finish(op.id, "failed", { detail: "reconciled: the merge never started" }, epoch);
|
|
322
|
+
else if (!op.issuerGone)
|
|
323
|
+
throw inFlight();
|
|
324
|
+
else if (pr?.state === "open" && pr.number === j.pr && pr.head === op.candidate)
|
|
325
|
+
await rel.finish(op.id, "failed", { detail: "reconciled: the merge's process ended and the pull request is still open at the gated head" }, epoch);
|
|
326
|
+
else
|
|
327
|
+
throw new Stop("merge-unknown", `The merge of pull request #${j.pr} can't be reconciled: it is ${pr?.state ?? "gone"}. Check it on GitHub before rerunning.`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
/**
|
|
332
|
+
* Who and what the release is, read again: the repository, the pull request (open, its number, base main) and its
|
|
333
|
+
* head, and the live-release predicate for the coming action. Before the deploy reservation a moved head restarts
|
|
334
|
+
* the release for the new commit; after it, it stops, and the new commit needs a person.
|
|
335
|
+
*/
|
|
336
|
+
const identity = async (action, beforeReservation) => {
|
|
337
|
+
if (rel === undefined) {
|
|
338
|
+
await pinned();
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
if (await adapters.repo.github() !== j.github)
|
|
342
|
+
throw new Stop("drift", `The release's repository is no longer ${j.github}.`);
|
|
343
|
+
const head = await adapters.repo.remoteHead(j.branch);
|
|
344
|
+
const pr = await adapters.github.pullRequest(j.github, j.branch);
|
|
345
|
+
if (pr === null || pr.number !== j.pr || pr.state !== "open")
|
|
346
|
+
throw new Stop("drift", `Pull request #${j.pr} is no longer the open pull request of ${j.branch}; the release stops.`);
|
|
347
|
+
if (pr.base !== "main")
|
|
348
|
+
throw new Stop("drift", `Pull request #${j.pr} now merges into ${pr.base}, not main; the release stops.`);
|
|
349
|
+
if (head !== j.candidate.sha || pr.head !== j.candidate.sha) {
|
|
350
|
+
if (beforeReservation && head !== null && pr.head === head)
|
|
351
|
+
throw new Restart(head);
|
|
352
|
+
throw new Stop("drift", `${j.branch} moved from ${j.candidate.sha.slice(0, 7)} to ${head?.slice(0, 7) ?? "nothing"} after the deploy began. The new commit needs a person's approval: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
353
|
+
}
|
|
354
|
+
if (j.authority) {
|
|
355
|
+
const live = await rel.live(j.task.id, action);
|
|
356
|
+
// Fail closed: once the release has an authority, its task must be one the predicate applies to AND accepts.
|
|
357
|
+
if (!live.applies || !live.ok)
|
|
358
|
+
throw new Stop("release", `Task ${j.task.id} can't ${action === "dispatch" ? "go on" : action}: ${live.applies ? live.why ?? "its release approval doesn't stand" : "it belongs to no release approval this plane can find"}.`);
|
|
359
|
+
if (epoch !== null && !await rel.renew(j.authority.id, owner, epoch))
|
|
360
|
+
throw new Stop("lease", "Another release coordinator took over this release.");
|
|
361
|
+
}
|
|
362
|
+
};
|
|
363
|
+
/** The head moved before the deploy reservation: carry the approval to it, or file it for a person. Nothing kept from
|
|
364
|
+
* the old step is used afterwards: the journal is rewritten and the state machine runs again. */
|
|
365
|
+
/**
|
|
366
|
+
* A pull request check failed after the deploy reservation (Alex, 2026-10-25): the candidate froze at deploy, so the
|
|
367
|
+
* release needs a person's fresh approval, even for the same commit. It is persisted (the authority stops; the
|
|
368
|
+
* journal says so) before the release stops; `--new` files the commit for a person again.
|
|
369
|
+
*/
|
|
370
|
+
const ciFailedAfterDeploy = async (what) => {
|
|
371
|
+
j.needsApproval = true;
|
|
372
|
+
save();
|
|
373
|
+
// Fail closed: the authority must have stopped; otherwise its approval could still carry the commit.
|
|
374
|
+
if (rel !== undefined && j.authority && (epoch === null || !await rel.needsApproval(j.authority.id, "a pull request check failed after the deploy", epoch))) {
|
|
375
|
+
throw new Stop("lease", `${what}. Nothing was merged, but this release's approval couldn't be stopped (another coordinator holds it, or a merge is reserved). Check it with \`toolroll release ${j.branch} --repo ${j.checkout}\` before going on.`);
|
|
376
|
+
}
|
|
377
|
+
throw new Stop("ci-failed", `${what}. Nothing was merged. The deploy had already started, so this commit needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
378
|
+
};
|
|
379
|
+
/** identity(), on a plane that keeps release authority; a legacy plane keeps the old rules. */
|
|
380
|
+
const watch = async (action) => { if (rel !== undefined)
|
|
381
|
+
await identity(action, true); };
|
|
382
|
+
const restart = async (head) => {
|
|
383
|
+
const tree = await adapters.repo.tree(head);
|
|
384
|
+
if (rel !== undefined && j.authority && epoch !== null) {
|
|
385
|
+
say(` ${j.branch} moved to ${head.slice(0, 7)}: checking whether the approval carries`);
|
|
386
|
+
const carried = await rel.carry(j.authority.id, head, randomUUID(), epoch);
|
|
387
|
+
if (carried.ok) {
|
|
388
|
+
const view = await rel.authority(carried.taskId);
|
|
389
|
+
j.candidate = { sha: head, tree };
|
|
390
|
+
j.task = { id: carried.taskId, scopeDigest: (await adapters.toolroll.gate(carried.taskId)).scopeDigest };
|
|
391
|
+
j.check = null;
|
|
392
|
+
j.completion = null;
|
|
393
|
+
j.landed = view?.landed ?? j.landed ?? [];
|
|
394
|
+
delete j.intents.check;
|
|
395
|
+
delete j.intents.complete;
|
|
396
|
+
j.step = "check";
|
|
397
|
+
j.stopped = null;
|
|
398
|
+
save();
|
|
399
|
+
say(` carried the approval to ${head.slice(0, 7)} (task ${carried.taskId})`);
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
say(` ${head.slice(0, 7)} needs a person's approval: ${carried.message}`);
|
|
403
|
+
}
|
|
404
|
+
else
|
|
405
|
+
say(` ${j.branch} moved to ${head.slice(0, 7)}: filing it for approval`);
|
|
406
|
+
j.candidate = { sha: head, tree };
|
|
407
|
+
j.task = { id: gateTaskId(j.packageVersion, head), scopeDigest: null };
|
|
408
|
+
j.approval = null;
|
|
409
|
+
j.check = null;
|
|
410
|
+
j.completion = null;
|
|
411
|
+
j.authority = null;
|
|
412
|
+
epoch = null;
|
|
413
|
+
delete j.intents.gate;
|
|
414
|
+
delete j.intents.check;
|
|
415
|
+
delete j.intents.complete;
|
|
416
|
+
j.step = "gate";
|
|
417
|
+
j.stopped = null;
|
|
418
|
+
save();
|
|
419
|
+
};
|
|
420
|
+
/**
|
|
421
|
+
* The task is approved for `action`: a release task (a carried commit, or any task once the release has an
|
|
422
|
+
* authority) only when the live-release predicate applies to it AND accepts it — fail closed, whatever its own seal
|
|
423
|
+
* says. Anything else needs the owner's own approval of these exact words (which a carried seal never is).
|
|
424
|
+
*/
|
|
425
|
+
const approved = async (gate, action = "dispatch") => {
|
|
205
426
|
if (gate.candidate !== j.candidate.sha || gate.scopeDigest !== j.task.scopeDigest)
|
|
206
427
|
throw new Stop("scope-changed", `Task ${j.task.id}'s candidate or scope changed after approval.`);
|
|
428
|
+
if (rel !== undefined && (j.authority || gate.approval?.basis === "carried")) {
|
|
429
|
+
const live = await rel.live(j.task.id, action);
|
|
430
|
+
if (live.applies && live.ok)
|
|
431
|
+
return;
|
|
432
|
+
throw new Stop("release", `Task ${j.task.id} can't ${action === "dispatch" ? "go on" : action}: ${live.applies ? live.why ?? "its release approval doesn't stand" : "it belongs to no release approval this plane can find"}.`);
|
|
433
|
+
}
|
|
207
434
|
const owner = ownerApproval(gate, j.task.scopeDigest);
|
|
208
435
|
if (!owner.ok)
|
|
209
436
|
throw new Stop("approval", `Task ${j.task.id}: ${owner.why}. An active approver must approve the release scope.`);
|
|
@@ -216,277 +443,387 @@ export async function runRelease(options, adapters) {
|
|
|
216
443
|
};
|
|
217
444
|
/** The commit the passing build recorded: what the deployed runtime is stamped with. */
|
|
218
445
|
const builtHead = () => j.check?.head ?? j.candidate.sha;
|
|
219
|
-
const checked = (gate) => {
|
|
220
|
-
approved(gate);
|
|
446
|
+
const checked = async (gate, action = "dispatch") => {
|
|
447
|
+
await approved(gate, action);
|
|
221
448
|
if (gate.result?.head !== builtHead() || gate.result?.receipt !== j.check?.receipt)
|
|
222
449
|
throw new Stop("drift", `Task ${j.task.id}'s result changed since its check passed.`);
|
|
223
450
|
fullCheck(gate);
|
|
224
451
|
};
|
|
452
|
+
if (j.needsApproval === true)
|
|
453
|
+
throw new Stop("ci-failed", `A pull request check failed after the deploy, so ${j.candidate.sha.slice(0, 7)} needs a person's approval again: run \`toolroll release ${j.branch} --repo ${j.checkout} --new\`.`);
|
|
454
|
+
// A resumed release that already has an authority takes its lease and settles every open operation first.
|
|
455
|
+
await hold();
|
|
225
456
|
while (j.step !== "done") {
|
|
226
457
|
step = j.step;
|
|
227
458
|
say(`• ${j.step}`);
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
459
|
+
try {
|
|
460
|
+
switch (j.step) {
|
|
461
|
+
case "gate": {
|
|
462
|
+
await pinned();
|
|
463
|
+
// The same commit's earlier gate belongs to a release that no longer stands (a CI failure after its deploy, a
|
|
464
|
+
// revoke): a fresh task asks a person again, and their yes opens a new generation.
|
|
465
|
+
if (rel !== undefined && j.freshTask !== true) {
|
|
466
|
+
const base = gateTaskId(j.packageVersion, j.candidate.sha);
|
|
467
|
+
for (let n = 1; n < 50; n++) {
|
|
468
|
+
const id = n === 1 ? base : `${base}-${n}`;
|
|
469
|
+
const live = await rel.live(id, "dispatch");
|
|
470
|
+
if (!live.applies || live.ok) {
|
|
471
|
+
if (j.task.id !== id) {
|
|
472
|
+
j.task = { id, scopeDigest: null };
|
|
473
|
+
save();
|
|
474
|
+
}
|
|
475
|
+
break;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
const existing = await adapters.toolroll.gate(j.task.id);
|
|
480
|
+
// A fresh task this journal never filed has appeared under its id: it isn't ours, and isn't a fresh approval.
|
|
481
|
+
if (existing.exists && j.freshTask === true && j.intents.gate === undefined)
|
|
482
|
+
throw new Stop("approval", `Task ${j.task.id} appeared before this release filed it; rerun with --new.`);
|
|
483
|
+
if (existing.exists) {
|
|
484
|
+
// Filed already (a resume after a lost answer): the task stands, and the ci step judges its checks.
|
|
485
|
+
if (existing.candidate !== j.candidate.sha)
|
|
486
|
+
throw new Stop("conflict", `Task ${j.task.id} exists for a different commit (${existing.candidate?.slice(0, 7) ?? "none"}).`);
|
|
487
|
+
j.task.scopeDigest = existing.scopeDigest;
|
|
488
|
+
}
|
|
489
|
+
else {
|
|
490
|
+
await earlyChecks(adapters, j, (what, probe) => waitFor("gate", what, probe));
|
|
491
|
+
intend("gate");
|
|
492
|
+
await adapters.repo.sync(j.branch);
|
|
493
|
+
const filed = await adapters.toolroll.fileGate({
|
|
494
|
+
taskId: j.task.id, title: `Release ${j.packageVersion}`, checkout: j.checkout, candidate: j.candidate.sha,
|
|
495
|
+
goal: `Verify ${j.packageName} ${j.packageVersion} (${j.branch} at ${j.candidate.sha}) passes the approved release check before it is deployed, merged, tagged and published.`,
|
|
496
|
+
acceptance: "The approved release check passes on the candidate commit|check",
|
|
497
|
+
...(rel === undefined ? {} : { release: { version: j.packageVersion, pr: j.pr, branch: j.branch, noCarry: options.noCarry === true } }),
|
|
498
|
+
});
|
|
499
|
+
j.task.scopeDigest = filed.scopeDigest;
|
|
500
|
+
}
|
|
501
|
+
advance("approval");
|
|
502
|
+
break;
|
|
236
503
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
await
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
504
|
+
case "approval": {
|
|
505
|
+
say(` The release scope needs your approval: \`toolroll task approve ${j.task.id} --digest ${j.task.scopeDigest}\`, or open the task in the console.`);
|
|
506
|
+
const approved = await waitFor("approval", `a person to approve task ${j.task.id}`, async () => {
|
|
507
|
+
await watch("dispatch");
|
|
508
|
+
const gate = await adapters.toolroll.gate(j.task.id);
|
|
509
|
+
if (gate.scopeDigest !== j.task.scopeDigest)
|
|
510
|
+
throw new Stop("scope-changed", `The scope of ${j.task.id} changed after it was filed; the release only runs on the scope it filed.`);
|
|
511
|
+
const owner = ownerApproval(gate, j.task.scopeDigest);
|
|
512
|
+
if (!owner.ok && gate.approval !== null && owner.why !== "the approval is for different scope text")
|
|
513
|
+
throw new Stop("approval", `Task ${j.task.id} was ${owner.why}; the release needs a person to approve it.`);
|
|
514
|
+
return owner.ok ? owner : null;
|
|
244
515
|
});
|
|
245
|
-
j.
|
|
516
|
+
j.approval = { by: approved.by, at: approved.at };
|
|
517
|
+
say(` approved by ${approved.by}`);
|
|
518
|
+
// The person's yes to release terms opened the release's authority: hold its lease from here on.
|
|
519
|
+
if (rel !== undefined) {
|
|
520
|
+
const view = await rel.authority(j.task.id);
|
|
521
|
+
if (view !== null) {
|
|
522
|
+
j.authority = { id: view.id, generation: view.generation };
|
|
523
|
+
j.landed = view.landed;
|
|
524
|
+
save();
|
|
525
|
+
await hold();
|
|
526
|
+
say(` later test-only commits on #${j.pr} carry this approval (release ${view.id})`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
advance("check");
|
|
530
|
+
break;
|
|
246
531
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
532
|
+
case "check": {
|
|
533
|
+
const result = await waitFor("check", `the release check of ${j.candidate.sha.slice(0, 7)}`, async () => {
|
|
534
|
+
// Before acting on an old candidate's failure: a moved head restarts the release first.
|
|
535
|
+
await watch("dispatch");
|
|
536
|
+
const gate = await adapters.toolroll.gate(j.task.id);
|
|
537
|
+
await approved(gate);
|
|
538
|
+
if (gate.failed !== null)
|
|
539
|
+
throw new Stop("check-failed", `The release build of ${j.task.id} failed: ${gate.failed}. Fix it and release the new commit with --new.`);
|
|
540
|
+
if (gate.result === null || gate.result.check === "running" || gate.result.check === "pending")
|
|
541
|
+
return null;
|
|
542
|
+
// The build records a prepared candidate as a commit of its own (its message names the task): it must hold
|
|
543
|
+
// exactly the candidate's tree, or it isn't the release.
|
|
544
|
+
const head = gate.result.head;
|
|
545
|
+
if (head === null || (head !== j.candidate.sha && await adapters.repo.tree(head) !== await adapters.repo.tree(j.candidate.sha))) {
|
|
546
|
+
throw new Stop("drift", `The release result is for ${head?.slice(0, 7) ?? "no commit"}, which doesn't hold exactly ${j.candidate.sha.slice(0, 7)}'s files.`);
|
|
547
|
+
}
|
|
548
|
+
fullCheck(gate);
|
|
549
|
+
if (gate.result.worktree === null)
|
|
550
|
+
throw new Stop("worktree", `Run ${gate.result.run} names no checkout to deploy from.`);
|
|
551
|
+
return { run: gate.result.run, receipt: gate.result.receipt, worktree: gate.result.worktree, head };
|
|
552
|
+
});
|
|
553
|
+
j.check = result;
|
|
554
|
+
say(` run ${result.run} passed on ${j.candidate.sha.slice(0, 7)}`);
|
|
555
|
+
advance("complete");
|
|
556
|
+
break;
|
|
557
|
+
}
|
|
558
|
+
case "complete": {
|
|
559
|
+
await watch("complete");
|
|
560
|
+
const check = j.check;
|
|
268
561
|
const gate = await adapters.toolroll.gate(j.task.id);
|
|
269
|
-
|
|
270
|
-
if (gate.
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
// exactly the candidate's tree, or it isn't the release.
|
|
276
|
-
const head = gate.result.head;
|
|
277
|
-
if (head === null || (head !== j.candidate.sha && await adapters.repo.tree(head) !== await adapters.repo.tree(j.candidate.sha))) {
|
|
278
|
-
throw new Stop("drift", `The release result is for ${head?.slice(0, 7) ?? "no commit"}, which doesn't hold exactly ${j.candidate.sha.slice(0, 7)}'s files.`);
|
|
562
|
+
await checked(gate, "complete");
|
|
563
|
+
if (gate.completion?.digest !== check.receipt) {
|
|
564
|
+
intend("complete");
|
|
565
|
+
const done = await adapters.toolroll.complete(j.task.id, check.receipt);
|
|
566
|
+
if (!done.ok)
|
|
567
|
+
throw new Stop("complete", `Couldn't mark task ${j.task.id} complete: ${done.message} Mark that exact result complete in the console, then rerun.`);
|
|
279
568
|
}
|
|
280
|
-
|
|
281
|
-
if (
|
|
282
|
-
throw new Stop("
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
say(` run ${result.run} passed on ${j.candidate.sha.slice(0, 7)}`);
|
|
287
|
-
advance("complete");
|
|
288
|
-
break;
|
|
289
|
-
}
|
|
290
|
-
case "complete": {
|
|
291
|
-
const check = j.check;
|
|
292
|
-
const gate = await adapters.toolroll.gate(j.task.id);
|
|
293
|
-
checked(gate);
|
|
294
|
-
if (gate.completion?.digest !== check.receipt) {
|
|
295
|
-
intend("complete");
|
|
296
|
-
const done = await adapters.toolroll.complete(j.task.id, check.receipt);
|
|
297
|
-
if (!done.ok)
|
|
298
|
-
throw new Stop("complete", `Couldn't mark task ${j.task.id} complete: ${done.message} Mark that exact result complete in the console, then rerun.`);
|
|
569
|
+
const after = await adapters.toolroll.gate(j.task.id);
|
|
570
|
+
if (after.completion?.digest !== check.receipt)
|
|
571
|
+
throw new Stop("complete", `Task ${j.task.id} is not marked complete for result ${check.receipt.slice(0, 12)}.`);
|
|
572
|
+
j.completion = { actor: after.completion.actor, at: after.completion.at };
|
|
573
|
+
advance("quiet");
|
|
574
|
+
break;
|
|
299
575
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
await waitFor("quiet", "running work to finish", async () => {
|
|
309
|
-
const busy = await adapters.toolroll.busy();
|
|
310
|
-
return busy.running === 0 ? true : null;
|
|
311
|
-
});
|
|
312
|
-
advance("deploy");
|
|
313
|
-
break;
|
|
314
|
-
}
|
|
315
|
-
case "deploy": {
|
|
316
|
-
const check = j.check;
|
|
317
|
-
await recoverDeploy();
|
|
318
|
-
checked(await adapters.toolroll.gate(j.task.id));
|
|
319
|
-
// A deployment that finished but whose answer was lost: the services already run a runtime of this commit.
|
|
320
|
-
let runtime = await served(adapters, builtHead());
|
|
321
|
-
if (runtime === null) {
|
|
322
|
-
await pinned();
|
|
323
|
-
j.deployAttempt = { stage: join(options.stateDir, "staged-upgrades", `browser-${j.candidate.sha.slice(0, 7)}-${randomUUID()}`) };
|
|
324
|
-
intend("deploy");
|
|
325
|
-
const result = await adapters.toolroll.deploy({ run: check.run, worktree: check.worktree, stage: j.deployAttempt.stage });
|
|
326
|
-
if (!result.ok)
|
|
327
|
-
throw deployStop("deploy", `The deployment of run ${check.run} stopped: ${result.message}`, result);
|
|
328
|
-
if (!/[\\/]dist$/.test(result.runtime))
|
|
329
|
-
throw new Stop("deploy", `The deployment named ${result.runtime} as its runtime, which is not a dist folder.`);
|
|
330
|
-
runtime = result.runtime;
|
|
331
|
-
// Saved at once: everything after this (relinking, a rerun) uses the runtime the deployer named.
|
|
332
|
-
j.deploy = { runtime, at: adapters.now().toISOString() };
|
|
333
|
-
save();
|
|
576
|
+
case "quiet": {
|
|
577
|
+
await waitFor("quiet", "running work to finish", async () => {
|
|
578
|
+
await watch("complete");
|
|
579
|
+
const busy = await adapters.toolroll.busy();
|
|
580
|
+
return busy.running === 0 ? true : null;
|
|
581
|
+
});
|
|
582
|
+
advance("deploy");
|
|
583
|
+
break;
|
|
334
584
|
}
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
585
|
+
case "deploy": {
|
|
586
|
+
const check = j.check;
|
|
587
|
+
await recoverDeploy();
|
|
588
|
+
await checked(await adapters.toolroll.gate(j.task.id), "deploy");
|
|
589
|
+
// A deployment that finished but whose answer was lost: the services already run a runtime of this commit.
|
|
590
|
+
let runtime = await served(adapters, builtHead());
|
|
591
|
+
// Already running this candidate (a new generation after a CI failure, or a lost answer): the release still
|
|
592
|
+
// reserves its deploy, so its merge has a deploy to stand on.
|
|
593
|
+
if (runtime !== null && rel !== undefined && j.authority && epoch !== null) {
|
|
594
|
+
const authority = await rel.authority(j.task.id);
|
|
595
|
+
if (authority !== null && authority.state === "open") {
|
|
596
|
+
await watch("deploy");
|
|
597
|
+
const reserved = await rel.reserve({ authority: j.authority.id, kind: "deploy", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
598
|
+
if (!reserved.ok)
|
|
599
|
+
throw new Stop("release", `The deploy couldn't be reserved: ${reserved.why}`);
|
|
600
|
+
if (!await rel.start(reserved.operation, epoch))
|
|
601
|
+
throw new Stop("lease", "Another release coordinator took over this release.");
|
|
602
|
+
await rel.finish(reserved.operation, "succeeded", { detail: "the services already run this candidate" }, epoch);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
if (runtime === null) {
|
|
606
|
+
// The last moment a moved head restarts; then the deploy reservation freezes the candidate in the store.
|
|
607
|
+
await watch("deploy");
|
|
608
|
+
await pinned();
|
|
609
|
+
let operation, secret = "";
|
|
610
|
+
if (rel !== undefined && j.authority && epoch !== null) {
|
|
611
|
+
const reserved = await rel.reserve({ authority: j.authority.id, kind: "deploy", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
612
|
+
if (!reserved.ok)
|
|
613
|
+
throw new Stop("release", `The deploy couldn't be reserved: ${reserved.why}`);
|
|
614
|
+
operation = reserved.operation;
|
|
615
|
+
secret = reserved.secret ?? "";
|
|
616
|
+
j.operations = { ...j.operations, deploy: operation };
|
|
617
|
+
save();
|
|
618
|
+
}
|
|
619
|
+
j.deployAttempt = { stage: join(options.stateDir, "staged-upgrades", `browser-${j.candidate.sha.slice(0, 7)}-${randomUUID()}`) };
|
|
620
|
+
intend("deploy");
|
|
621
|
+
// The issuing fence, immediately before the deploy is spawned (review round 2): lease, epoch and the live
|
|
622
|
+
// release again, in one transaction that marks the operation as issued by this process.
|
|
623
|
+
if (operation !== undefined && !await rel.start(operation, epoch))
|
|
624
|
+
throw new Stop("lease", "Another release coordinator took over this release; the deploy didn't start.");
|
|
625
|
+
const result = await adapters.toolroll.deploy({ run: check.run, worktree: check.worktree, stage: j.deployAttempt.stage, ...(operation === undefined ? {} : { operation: { id: operation, secret } }) });
|
|
626
|
+
if (operation !== undefined)
|
|
627
|
+
await rel.finish(operation, result.ok ? "succeeded" : "failed", { detail: result.ok ? result.runtime : result.message.slice(0, 500) }, epoch);
|
|
628
|
+
if (!result.ok)
|
|
629
|
+
throw deployStop("deploy", `The deployment of run ${check.run} stopped: ${result.message}`, result);
|
|
630
|
+
if (!/[\\/]dist$/.test(result.runtime))
|
|
631
|
+
throw new Stop("deploy", `The deployment named ${result.runtime} as its runtime, which is not a dist folder.`);
|
|
632
|
+
runtime = result.runtime;
|
|
633
|
+
// Saved at once: everything after this (relinking, a rerun) uses the runtime the deployer named.
|
|
634
|
+
j.deploy = { runtime, at: adapters.now().toISOString() };
|
|
635
|
+
save();
|
|
636
|
+
}
|
|
637
|
+
const want = runtime;
|
|
638
|
+
await waitFor("deploy", `the console and worker to run the gated commit from ${want}`, async () => (await served(adapters, builtHead())) === want ? true : null);
|
|
639
|
+
j.deploy = { runtime: want, at: j.deploy?.runtime === want ? j.deploy.at : adapters.now().toISOString() };
|
|
640
|
+
j.deployAttempt = null;
|
|
641
|
+
delete j.intents.deploy;
|
|
642
|
+
say(` the service runs ${j.packageVersion} from ${want}`);
|
|
643
|
+
advance("link");
|
|
644
|
+
break;
|
|
351
645
|
}
|
|
352
|
-
|
|
353
|
-
|
|
646
|
+
case "link": {
|
|
647
|
+
const runtime = j.deploy.runtime;
|
|
648
|
+
if (await served(adapters, builtHead()) !== runtime)
|
|
649
|
+
throw new Stop("drift", `The console and worker no longer run ${runtime}; the CLI names were not moved.`);
|
|
650
|
+
if (j.links === undefined || j.links === null) {
|
|
651
|
+
j.links = await adapters.toolroll.cliLinks();
|
|
652
|
+
intend("link");
|
|
653
|
+
}
|
|
654
|
+
try {
|
|
655
|
+
await adapters.toolroll.switchLinks(j.links, join(runtime, "bin.js"), j.packageVersion);
|
|
656
|
+
}
|
|
657
|
+
catch (error) {
|
|
658
|
+
throw new Stop("link", `The CLI names were not moved to ${runtime} (they point where they did): ${error.message}. The service already runs the new build; rerun to try again.`);
|
|
659
|
+
}
|
|
660
|
+
const everywhere = await deployed(adapters, builtHead());
|
|
661
|
+
if (everywhere !== runtime)
|
|
662
|
+
throw new Stop("mixed-runtime", `After relinking, the console, worker and both CLI names don't all run ${runtime} at ${j.candidate.sha.slice(0, 7)}.`);
|
|
663
|
+
say(` toolroll and standing-orders run ${join(runtime, "bin.js")}`);
|
|
664
|
+
advance("ci");
|
|
665
|
+
break;
|
|
354
666
|
}
|
|
355
|
-
|
|
356
|
-
|
|
667
|
+
case "ci": {
|
|
668
|
+
await waitFor("ci", `the pull request checks on ${j.candidate.sha.slice(0, 7)}`, async () => {
|
|
669
|
+
await pinned();
|
|
670
|
+
const verdict = checksVerdict(await adapters.github.checks(j.github, j.candidate.sha), REQUIRED_PR_CHECKS);
|
|
671
|
+
if (verdict.state === "failed")
|
|
672
|
+
await ciFailedAfterDeploy(`Pull request checks failed on ${j.candidate.sha.slice(0, 7)}: ${verdict.failed.join(", ")}`);
|
|
673
|
+
return verdict.state === "passed" ? true : null;
|
|
674
|
+
});
|
|
675
|
+
j.ci = { checks: [...REQUIRED_PR_CHECKS], at: adapters.now().toISOString() };
|
|
676
|
+
advance("merge");
|
|
677
|
+
break;
|
|
357
678
|
}
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
679
|
+
case "merge": {
|
|
680
|
+
const pr = await adapters.github.pullRequest(j.github, j.branch);
|
|
681
|
+
if (pr?.state === "merged" && pr.number === j.pr && pr.mergeCommit !== null) {
|
|
682
|
+
if (pr.head !== j.candidate.sha || pr.base !== "main")
|
|
683
|
+
throw new Stop("drift", `Merged pull request #${j.pr} does not match the gated commit and base.`);
|
|
684
|
+
if (j.intents.merge === undefined)
|
|
685
|
+
intend("merge");
|
|
686
|
+
j.merge = { sha: pr.mergeCommit, tree: null };
|
|
687
|
+
}
|
|
688
|
+
else {
|
|
689
|
+
// After the deploy reservation a moved head, a changed pull request or a stopped release stops it here.
|
|
690
|
+
if (rel !== undefined && j.authority)
|
|
691
|
+
await identity("merge", false);
|
|
692
|
+
await pinned();
|
|
693
|
+
// A squash merge only reproduces the gated tree when main hasn't moved past the branch's base.
|
|
694
|
+
await adapters.repo.sync(j.branch);
|
|
695
|
+
const main = await adapters.repo.mainHead();
|
|
696
|
+
if (!await adapters.repo.contains(j.candidate.sha, main))
|
|
697
|
+
throw new Stop("base-moved", `main moved to ${main.slice(0, 7)}, which ${j.branch} doesn't contain, so the merge wouldn't be the gated tree. Nothing was merged: update the branch and release the new commit with --new.`);
|
|
698
|
+
// The checks again, at the moment of merging: a rerun that failed since must not merge.
|
|
699
|
+
const verdict = checksVerdict(await adapters.github.checks(j.github, j.candidate.sha), REQUIRED_PR_CHECKS);
|
|
700
|
+
if (verdict.state !== "passed")
|
|
701
|
+
await ciFailedAfterDeploy(`Pull request checks are no longer all passing on ${j.candidate.sha.slice(0, 7)}`);
|
|
702
|
+
let operation;
|
|
703
|
+
if (rel !== undefined && j.authority && epoch !== null) {
|
|
704
|
+
const reserved = await rel.reserve({ authority: j.authority.id, kind: "merge", taskId: j.task.id, candidate: j.candidate.sha, epoch });
|
|
705
|
+
if (!reserved.ok)
|
|
706
|
+
throw new Stop("release", `The merge couldn't be reserved: ${reserved.why}`);
|
|
707
|
+
operation = reserved.operation;
|
|
708
|
+
j.operations = { ...j.operations, merge: operation };
|
|
709
|
+
save();
|
|
710
|
+
}
|
|
383
711
|
intend("merge");
|
|
384
|
-
|
|
712
|
+
const message = landedWords(j.landed ?? []).join("\n");
|
|
713
|
+
// The issuing fence, immediately before gh is spawned (review round 2).
|
|
714
|
+
if (operation !== undefined && !await rel.start(operation, epoch))
|
|
715
|
+
throw new Stop("lease", "Another release coordinator took over this release; the merge didn't start.");
|
|
716
|
+
let merged;
|
|
717
|
+
try {
|
|
718
|
+
merged = await adapters.github.merge(j.github, j.pr, j.candidate.sha, ...(message === "" ? [] : [message]));
|
|
719
|
+
}
|
|
720
|
+
catch (error) {
|
|
721
|
+
if (operation !== undefined)
|
|
722
|
+
await rel.finish(operation, "unknown", { detail: String(error?.message ?? error).slice(0, 500) }, epoch);
|
|
723
|
+
throw error;
|
|
724
|
+
}
|
|
725
|
+
if (operation !== undefined)
|
|
726
|
+
await rel.finish(operation, "succeeded", { externalRef: merged }, epoch);
|
|
727
|
+
j.merge = { sha: merged, tree: null };
|
|
728
|
+
}
|
|
729
|
+
save();
|
|
730
|
+
// This also runs after discovering a merge whose response was lost. Never delete a reused branch.
|
|
731
|
+
await adapters.repo.deleteBranch(j.branch, j.candidate.sha);
|
|
732
|
+
advance("tree");
|
|
733
|
+
break;
|
|
385
734
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
// A squash merge only reproduces the gated tree when main hasn't moved past the branch's base.
|
|
735
|
+
case "tree": {
|
|
736
|
+
const merge = j.merge;
|
|
389
737
|
await adapters.repo.sync(j.branch);
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
throw new Stop("ci-failed", `Pull request checks are no longer all passing on ${j.candidate.sha.slice(0, 7)}; nothing was merged.`);
|
|
397
|
-
intend("merge");
|
|
398
|
-
j.merge = { sha: await adapters.github.merge(j.github, j.pr, j.candidate.sha), tree: null };
|
|
399
|
-
}
|
|
400
|
-
save();
|
|
401
|
-
// This also runs after discovering a merge whose response was lost. Never delete a reused branch.
|
|
402
|
-
await adapters.repo.deleteBranch(j.branch, j.candidate.sha);
|
|
403
|
-
advance("tree");
|
|
404
|
-
break;
|
|
405
|
-
}
|
|
406
|
-
case "tree": {
|
|
407
|
-
const merge = j.merge;
|
|
408
|
-
await adapters.repo.sync(j.branch);
|
|
409
|
-
const tree = await adapters.repo.tree(merge.sha);
|
|
410
|
-
merge.tree = tree;
|
|
411
|
-
if (tree !== j.candidate.tree)
|
|
412
|
-
throw new Stop("tree-mismatch", `Main's merge commit ${merge.sha.slice(0, 7)} has tree ${tree.slice(0, 12)}, not the gated ${j.candidate.tree.slice(0, 12)}: main changed what was checked. Nothing was tagged; the deployed build stays as it is.`);
|
|
413
|
-
advance("tag");
|
|
414
|
-
break;
|
|
415
|
-
}
|
|
416
|
-
case "tag": {
|
|
417
|
-
const merge = j.merge;
|
|
418
|
-
const at = await adapters.github.tag(j.github, j.tag);
|
|
419
|
-
if (at !== null && at !== merge.sha)
|
|
420
|
-
throw new Stop("conflict", `${j.tag} already points at ${at.slice(0, 7)}, not the merge ${merge.sha.slice(0, 7)}.`);
|
|
421
|
-
if (at === null) {
|
|
422
|
-
intend("tag");
|
|
423
|
-
await adapters.github.createTag(j.github, j.tag, merge.sha);
|
|
738
|
+
const tree = await adapters.repo.tree(merge.sha);
|
|
739
|
+
merge.tree = tree;
|
|
740
|
+
if (tree !== j.candidate.tree)
|
|
741
|
+
throw new Stop("tree-mismatch", `Main's merge commit ${merge.sha.slice(0, 7)} has tree ${tree.slice(0, 12)}, not the gated ${j.candidate.tree.slice(0, 12)}: main changed what was checked. Nothing was tagged; the deployed build stays as it is.`);
|
|
742
|
+
advance("tag");
|
|
743
|
+
break;
|
|
424
744
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
throw new Stop("publish-failed", `The release tag's checks failed: ${verdict.failed.join(", ")}. Homebrew was not updated.`);
|
|
438
|
-
if (verdict.state === "pending")
|
|
439
|
-
return null;
|
|
440
|
-
return await adapters.registry.published(j.packageName, j.packageVersion) && await adapters.github.release(j.github, j.tag) ? true : null;
|
|
441
|
-
});
|
|
442
|
-
j.published = { at: adapters.now().toISOString() };
|
|
443
|
-
advance("homebrew");
|
|
444
|
-
break;
|
|
445
|
-
}
|
|
446
|
-
case "homebrew": {
|
|
447
|
-
const brew = adapters.homebrew;
|
|
448
|
-
let state = await brew.state(j.packageVersion);
|
|
449
|
-
if (state.kind === "missing") {
|
|
450
|
-
intend("homebrew");
|
|
451
|
-
const proposed = await brew.propose(j.packageVersion);
|
|
452
|
-
state = { kind: "proposed", ...proposed };
|
|
745
|
+
case "tag": {
|
|
746
|
+
const merge = j.merge;
|
|
747
|
+
const at = await adapters.github.tag(j.github, j.tag);
|
|
748
|
+
if (at !== null && at !== merge.sha)
|
|
749
|
+
throw new Stop("conflict", `${j.tag} already points at ${at.slice(0, 7)}, not the merge ${merge.sha.slice(0, 7)}.`);
|
|
750
|
+
if (at === null) {
|
|
751
|
+
intend("tag");
|
|
752
|
+
await adapters.github.createTag(j.github, j.tag, merge.sha);
|
|
753
|
+
}
|
|
754
|
+
j.tagged = { sha: merge.sha };
|
|
755
|
+
advance("publish");
|
|
756
|
+
break;
|
|
453
757
|
}
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
// Two apps may name a check alike; each app's attempts stand for themselves.
|
|
465
|
-
const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
|
|
466
|
-
const statuses = await brew.statuses(proposed.head);
|
|
467
|
-
const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
|
|
468
|
-
const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
|
|
469
|
-
if (failed.length > 0)
|
|
470
|
-
throw new Stop("homebrew-failed", `The Homebrew change's checks failed: ${failed.join(", ")}.`);
|
|
471
|
-
if (verdicts.some(one => one.state === "pending"))
|
|
472
|
-
return null;
|
|
473
|
-
// GitHub registers checks asynchronously; an empty set says nothing about success.
|
|
474
|
-
// A workflow is owed: only its own run counts, not another app's green check.
|
|
475
|
-
if (checked && !runs.some(run => run.app === "github-actions"))
|
|
758
|
+
case "publish": {
|
|
759
|
+
if (j.intents.publish === undefined)
|
|
760
|
+
intend("publish");
|
|
761
|
+
await waitFor("publish", `npm, the GitHub release and the tag's checks for ${j.tag}`, async () => {
|
|
762
|
+
if (await adapters.github.tag(j.github, j.tag) !== j.tagged.sha)
|
|
763
|
+
throw new Stop("conflict", `${j.tag} no longer points at the verified merge commit. Homebrew was not updated.`);
|
|
764
|
+
const verdict = checksVerdict(await adapters.github.checks(j.github, j.tagged.sha), RELEASE_TAG_CHECKS);
|
|
765
|
+
if (verdict.state === "failed")
|
|
766
|
+
throw new Stop("publish-failed", `The release tag's checks failed: ${verdict.failed.join(", ")}. Homebrew was not updated.`);
|
|
767
|
+
if (verdict.state === "pending")
|
|
476
768
|
return null;
|
|
477
|
-
|
|
478
|
-
return ++emptyLooks >= 2 ? true : null;
|
|
479
|
-
return true;
|
|
769
|
+
return await adapters.registry.published(j.packageName, j.packageVersion) && await adapters.github.release(j.github, j.tag) ? true : null;
|
|
480
770
|
});
|
|
481
|
-
|
|
771
|
+
j.published = { at: adapters.now().toISOString() };
|
|
772
|
+
advance("homebrew");
|
|
773
|
+
break;
|
|
774
|
+
}
|
|
775
|
+
case "homebrew": {
|
|
776
|
+
const brew = adapters.homebrew;
|
|
777
|
+
let state = await brew.state(j.packageVersion);
|
|
778
|
+
if (state.kind === "missing") {
|
|
779
|
+
intend("homebrew");
|
|
780
|
+
const proposed = await brew.propose(j.packageVersion);
|
|
781
|
+
state = { kind: "proposed", ...proposed };
|
|
782
|
+
}
|
|
783
|
+
if (state.kind === "proposed") {
|
|
784
|
+
const proposed = state;
|
|
785
|
+
j.homebrew = { pr: proposed.pr, head: proposed.head, merged: false };
|
|
786
|
+
save();
|
|
787
|
+
// Whatever checks do register are always judged; an empty set passes only for a tap whose workflows never run
|
|
788
|
+
// on a change, and only when it's still empty a poll later.
|
|
789
|
+
const checked = await brew.checked(proposed.head);
|
|
790
|
+
let emptyLooks = 0;
|
|
791
|
+
await waitFor("homebrew", `the Homebrew change's checks (${brew.repo} #${proposed.pr})`, async () => {
|
|
792
|
+
// Check runs and commit statuses are judged apart, so neither stands in for the other or hides its failure.
|
|
793
|
+
// Two apps may name a check alike; each app's attempts stand for themselves.
|
|
794
|
+
const runs = (await adapters.github.checks(brew.repo, proposed.head)).map(run => ({ ...run, name: `${run.app ?? "unknown app"}: ${run.name}` }));
|
|
795
|
+
const statuses = await brew.statuses(proposed.head);
|
|
796
|
+
const verdicts = [runs, statuses].filter(one => one.length > 0).map(one => checksVerdict(one, [...new Set(one.map(run => run.name))]));
|
|
797
|
+
const failed = verdicts.flatMap(one => one.state === "failed" ? one.failed : []);
|
|
798
|
+
if (failed.length > 0)
|
|
799
|
+
throw new Stop("homebrew-failed", `The Homebrew change's checks failed: ${failed.join(", ")}.`);
|
|
800
|
+
if (verdicts.some(one => one.state === "pending"))
|
|
801
|
+
return null;
|
|
802
|
+
// GitHub registers checks asynchronously; an empty set says nothing about success.
|
|
803
|
+
// A workflow is owed: only its own run counts, not another app's green check.
|
|
804
|
+
if (checked && !runs.some(run => run.app === "github-actions"))
|
|
805
|
+
return null;
|
|
806
|
+
if (runs.length === 0 && statuses.length === 0)
|
|
807
|
+
return ++emptyLooks >= 2 ? true : null;
|
|
808
|
+
return true;
|
|
809
|
+
});
|
|
810
|
+
await brew.merge(proposed.pr, proposed.head);
|
|
811
|
+
}
|
|
812
|
+
j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: true };
|
|
813
|
+
advance("done");
|
|
814
|
+
break;
|
|
482
815
|
}
|
|
483
|
-
j.homebrew = { pr: j.homebrew?.pr ?? null, head: j.homebrew?.head ?? null, merged: true };
|
|
484
|
-
advance("done");
|
|
485
|
-
break;
|
|
486
816
|
}
|
|
487
817
|
}
|
|
818
|
+
catch (error) {
|
|
819
|
+
if (!(error instanceof Restart))
|
|
820
|
+
throw error;
|
|
821
|
+
await restart(error.head);
|
|
822
|
+
}
|
|
488
823
|
}
|
|
489
824
|
say(`Released ${j.tag}: deployed, merged as ${j.merge.sha.slice(0, 7)}, tagged, published and on Homebrew.`);
|
|
825
|
+
for (const line of landedWords(j.landed ?? []))
|
|
826
|
+
say(line);
|
|
490
827
|
return { ok: true, journal: j, file };
|
|
491
828
|
}
|
|
492
829
|
catch (error) {
|
|
@@ -568,3 +905,57 @@ async function deployed(adapters, candidate) {
|
|
|
568
905
|
const [runtime] = runtimes;
|
|
569
906
|
return runtime !== undefined && runtime !== "" && await adapters.toolroll.runtimeCommit(runtime) === candidate ? runtime : null;
|
|
570
907
|
}
|
|
908
|
+
/**
|
|
909
|
+
* The gate's look at the pull request's required checks on the exact commit, before anything is filed or anyone is
|
|
910
|
+
* asked to approve (#237): a required check that failed or was cancelled stops here; checks not started or still
|
|
911
|
+
* running are named, and the gate waits for them; ones that never finish within the gate's limit stop here too. GitHub
|
|
912
|
+
* not answering is waited out the same way, never taken for a pass. The ci step still checks again before merging.
|
|
913
|
+
*/
|
|
914
|
+
async function earlyChecks(adapters, j, wait) {
|
|
915
|
+
const sha = j.candidate.sha.slice(0, 7);
|
|
916
|
+
const required = REQUIRED_PR_CHECKS;
|
|
917
|
+
let named = "", unread = false;
|
|
918
|
+
try {
|
|
919
|
+
await wait(`the pull request checks on ${sha}`, async () => {
|
|
920
|
+
let runs;
|
|
921
|
+
try {
|
|
922
|
+
runs = await adapters.github.checks(j.github, j.candidate.sha);
|
|
923
|
+
}
|
|
924
|
+
catch (error) {
|
|
925
|
+
if (!unread)
|
|
926
|
+
adapters.say(` Couldn't read the pull request checks on ${sha} (${error?.message ?? error}); trying again.`);
|
|
927
|
+
unread = true;
|
|
928
|
+
return null;
|
|
929
|
+
}
|
|
930
|
+
const verdict = checksVerdict(runs.filter(run => required.includes(run.name)), required);
|
|
931
|
+
if (verdict.state === "failed") {
|
|
932
|
+
throw new Stop("ci-failed", `Pull request checks already failed on ${sha}: ${verdict.failed.join(", ")}. Nothing was filed or sent for approval. Fix the branch, then release the new commit with \`toolroll release ${j.branch} --repo ${j.checkout} --new\` (or rerun the failed job on GitHub and run this command again).`);
|
|
933
|
+
}
|
|
934
|
+
if (verdict.state === "passed")
|
|
935
|
+
return true;
|
|
936
|
+
const now = verdict.waiting.join(", ");
|
|
937
|
+
if (now !== named) {
|
|
938
|
+
adapters.say(` Pull request checks on ${sha} not finished yet: ${now}.`);
|
|
939
|
+
named = now;
|
|
940
|
+
}
|
|
941
|
+
return null;
|
|
942
|
+
});
|
|
943
|
+
}
|
|
944
|
+
catch (error) {
|
|
945
|
+
if (!(error instanceof Stop) || error.reason !== "timeout")
|
|
946
|
+
throw error;
|
|
947
|
+
throw new Stop("ci-missing", `Pull request checks on ${sha} didn't all finish in time${named === "" ? (unread ? " (GitHub didn't answer)" : "") : `: ${named}`}. Nothing was filed or sent for approval. Start or rerun them on GitHub, then run this command again.`);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
/** The network failures a release's waits ride out: git or gh timing out (or cut off) on the way to GitHub. */
|
|
951
|
+
const TRANSIENT_NETWORK = /Recv failure: (?:Operation timed out|Connection reset by peer)|TLS handshake timeout|Operation timed out|Connection timed out|i\/o timeout|Connection reset by peer|Failed to connect to [^\s]*github\.com port \d+/i;
|
|
952
|
+
/** A probe's failure that is only the network to GitHub being slow, never a decision (a Stop) or another error. */
|
|
953
|
+
export function transientNetworkError(error) {
|
|
954
|
+
if (error instanceof Stop || error instanceof Restart)
|
|
955
|
+
return false;
|
|
956
|
+
return TRANSIENT_NETWORK.test(String(error?.message ?? error));
|
|
957
|
+
}
|
|
958
|
+
/** The part of a network failure worth saying: its matching phrase. */
|
|
959
|
+
function networkWords(error) {
|
|
960
|
+
return TRANSIENT_NETWORK.exec(String(error?.message ?? error))?.[0] ?? "a network failure";
|
|
961
|
+
}
|