toolroll 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +5 -5
- package/dist/agent-onboard.d.ts +45 -18
- package/dist/agent-onboard.js +146 -57
- package/dist/browser/workspace.css +2 -2
- package/dist/browser/workspace.js +14 -13
- package/dist/checkout-cleanup.d.ts +84 -0
- package/dist/checkout-cleanup.js +245 -0
- package/dist/daemon.d.ts +6 -2
- package/dist/daemon.js +6 -5
- package/dist/first-run.d.ts +8 -2
- package/dist/first-run.js +12 -8
- package/dist/install-method.d.ts +1 -1
- package/dist/install-method.js +4 -0
- package/dist/mcp.js +26 -0
- package/dist/operate.d.ts +2 -4
- package/dist/operate.js +167 -30
- package/dist/prompt.d.ts +11 -0
- package/dist/prompt.js +41 -0
- package/dist/serve.js +103 -28
- package/dist/storage-ui.d.ts +10 -0
- package/dist/storage-ui.js +67 -0
- package/dist/storage.d.ts +15 -1
- package/dist/storage.js +62 -7
- package/dist/store.d.ts +6 -6
- package/dist/store.js +23 -20
- package/dist/toolroll-update-cli.js +7 -3
- package/dist/toolroll-update-ui.d.ts +2 -0
- package/dist/toolroll-update-ui.js +7 -6
- package/dist/toolroll-update.d.ts +69 -15
- package/dist/toolroll-update.js +572 -110
- package/dist/worktree.d.ts +24 -10
- package/dist/worktree.js +84 -44
- package/package.json +1 -1
package/dist/toolroll-update.js
CHANGED
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
* runtimes folder (`staged-upgrades/`, as deploy-browser stages), never over
|
|
14
14
|
* the running one. npm's own signature and attestation check covers what it
|
|
15
15
|
* installed, the installed bytes must be the ones downloaded and hashed, and
|
|
16
|
-
* their provenance must name ap9000/toolroll
|
|
16
|
+
* the certificate that signed their provenance must name ap9000/toolroll, its
|
|
17
|
+
* publish workflow and GitHub Actions, checked by npm's own Sigstore verifier. The
|
|
17
18
|
* service is stopped, and its processes seen gone, before the final backup,
|
|
18
19
|
* so nothing written before the switch is lost. A failed health check puts
|
|
19
20
|
* back the previous runtime, database and coding catalog on its own, keeping
|
|
@@ -21,9 +22,9 @@
|
|
|
21
22
|
* previous runtime and its backups. Every update, rollback, refusal and
|
|
22
23
|
* failure is a ledger entry.
|
|
23
24
|
*/
|
|
24
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
25
|
+
import { createHash, randomUUID, verify as signatureValid, X509Certificate } from "node:crypto";
|
|
25
26
|
import { spawnSync } from "node:child_process";
|
|
26
|
-
import { chmodSync, closeSync, copyFileSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
27
|
+
import { accessSync, chmodSync, closeSync, constants, copyFileSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, statfsSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
27
28
|
import { homedir, userInfo } from "node:os";
|
|
28
29
|
import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
|
|
29
30
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
@@ -41,6 +42,9 @@ function sqlite() {
|
|
|
41
42
|
}
|
|
42
43
|
export const PROVENANCE_REPOSITORY = "https://github.com/ap9000/toolroll";
|
|
43
44
|
export const PROVENANCE_WORKFLOW = ".github/workflows/publish.yml";
|
|
45
|
+
export const PROVENANCE_ISSUER = "https://token.actions.githubusercontent.com";
|
|
46
|
+
/** How long a watch's `toolroll up` may take to exit once launchd stops it (controller-supervisor's shutdown). */
|
|
47
|
+
const UP_EXIT_MS = 45_000;
|
|
44
48
|
/** The steps a person sees, in order. */
|
|
45
49
|
export const UPDATE_STEPS = ["verifying", "draining", "backing-up", "rehearsing", "switching", "restarting", "health"];
|
|
46
50
|
export const STEP_WORDS = {
|
|
@@ -54,6 +58,8 @@ export const UPDATE_JOB_LABEL = "com.toolroll.update";
|
|
|
54
58
|
const TERMINAL = ["complete", "restored", "refused", "cancelled"];
|
|
55
59
|
export const runtimeUpdateTerminal = (phase) => TERMINAL.includes(phase);
|
|
56
60
|
const journalFile = (stateDir) => join(stateDir, "toolroll-update.json");
|
|
61
|
+
/** The last completed update, kept apart from the journal: what `--rollback` returns from. */
|
|
62
|
+
const lastUpdateFile = (stateDir) => join(stateDir, "toolroll-update.last.json");
|
|
57
63
|
const cancelFile = (j) => join(j.stageDir, "cancel-request.json");
|
|
58
64
|
const sha = (bytes, algorithm = "sha256") => createHash(algorithm).update(bytes).digest("hex");
|
|
59
65
|
const fileHash = (file) => sha(readFileSync(file));
|
|
@@ -61,8 +67,7 @@ const quote = (s) => '"' + s.replaceAll('"', '""') + '"';
|
|
|
61
67
|
const codingFile = (databaseFile) => `${databaseFile}.coding.sqlite`;
|
|
62
68
|
class Refusal extends Error {
|
|
63
69
|
}
|
|
64
|
-
export function readRuntimeUpdate(stateDir) {
|
|
65
|
-
const file = journalFile(stateDir);
|
|
70
|
+
export function readRuntimeUpdate(stateDir, file = journalFile(stateDir)) {
|
|
66
71
|
if (!existsSync(file))
|
|
67
72
|
return null;
|
|
68
73
|
const stat = lstatSync(file);
|
|
@@ -79,6 +84,23 @@ export function readRuntimeUpdate(stateDir) {
|
|
|
79
84
|
throw Error("The saved update record is invalid. Preserve it and its backups; nothing was changed.");
|
|
80
85
|
return j;
|
|
81
86
|
}
|
|
87
|
+
/** The update `--rollback` returns from: the last one that completed, whatever was attempted since. */
|
|
88
|
+
export function lastCompletedUpdate(stateDir) {
|
|
89
|
+
const kept = existsSync(lastUpdateFile(stateDir)) ? readRuntimeUpdate(stateDir, lastUpdateFile(stateDir)) : readRuntimeUpdate(stateDir);
|
|
90
|
+
return kept?.kind === "update" && kept.phase === "complete" ? kept : null;
|
|
91
|
+
}
|
|
92
|
+
/** Before a new attempt replaces the saved journal: a completed update is kept as what `--rollback` returns from.
|
|
93
|
+
* 0.8.0 and older never wrote `.last.json`, so the update that brought this version would otherwise be lost to the
|
|
94
|
+
* first refused or failed attempt. */
|
|
95
|
+
function keepCompletedUpdate(stateDir) {
|
|
96
|
+
const saved = readRuntimeUpdate(stateDir);
|
|
97
|
+
if (saved?.kind !== "update" || saved.phase !== "complete")
|
|
98
|
+
return;
|
|
99
|
+
const kept = existsSync(lastUpdateFile(stateDir)) ? readRuntimeUpdate(stateDir, lastUpdateFile(stateDir)) : null;
|
|
100
|
+
if (kept && (kept.finishedAt ?? "") >= (saved.finishedAt ?? ""))
|
|
101
|
+
return;
|
|
102
|
+
durableJson(lastUpdateFile(stateDir), saved);
|
|
103
|
+
}
|
|
82
104
|
function save(j, phase, detail, now) {
|
|
83
105
|
if (j.phase !== phase)
|
|
84
106
|
j.steps.push({ phase, at: now.toISOString() });
|
|
@@ -93,6 +115,9 @@ function save(j, phase, detail, now) {
|
|
|
93
115
|
/** Every update, rollback, refusal and failure. Plain SQL any build can run,
|
|
94
116
|
* written after any database restore so the restore cannot erase it. */
|
|
95
117
|
function ledger(databaseFile, now, actor, action, outcome, detail) {
|
|
118
|
+
// Never an empty database where the live one should be.
|
|
119
|
+
if (!existsSync(databaseFile))
|
|
120
|
+
throw Error("The database is not in place.");
|
|
96
121
|
const db = new (sqlite().DatabaseSync)(databaseFile);
|
|
97
122
|
try {
|
|
98
123
|
db.exec("PRAGMA busy_timeout=5000");
|
|
@@ -115,33 +140,121 @@ export function runningWorkWords(db) {
|
|
|
115
140
|
const parts = [...(runs.length ? [`running ${runs.join(", ")}${more > 0 ? ` and ${more} more` : ""}`] : []), ...other];
|
|
116
141
|
return parts.join("; ");
|
|
117
142
|
}
|
|
118
|
-
/**
|
|
119
|
-
|
|
120
|
-
|
|
143
|
+
/** Sigstore's certificate extensions (Fulcio's OID registry) and subjectAltName. */
|
|
144
|
+
const OID = { issuerV1: "1.3.6.1.4.1.57264.1.1", repositoryV1: "1.3.6.1.4.1.57264.1.5", issuer: "1.3.6.1.4.1.57264.1.8", buildSigner: "1.3.6.1.4.1.57264.1.9", sourceRepository: "1.3.6.1.4.1.57264.1.12", subjectAltName: "2.5.29.17" };
|
|
145
|
+
/** One DER element at `at`: its tag and where its contents start and end. */
|
|
146
|
+
function derAt(der, at) {
|
|
147
|
+
let length = der[at + 1] ?? 0, start = at + 2;
|
|
148
|
+
if (length & 0x80) {
|
|
149
|
+
const bytes = length & 0x7f;
|
|
150
|
+
if (bytes < 1 || bytes > 4)
|
|
151
|
+
throw Error("unsupported DER length");
|
|
152
|
+
length = 0;
|
|
153
|
+
for (let i = 0; i < bytes; i++)
|
|
154
|
+
length = length * 256 + (der[start + i] ?? 0);
|
|
155
|
+
start += bytes;
|
|
156
|
+
}
|
|
157
|
+
if (at >= der.length || start + length > der.length)
|
|
158
|
+
throw Error("truncated DER");
|
|
159
|
+
return { tag: der[at], start, end: start + length };
|
|
160
|
+
}
|
|
161
|
+
function derChildren(der, parent) {
|
|
162
|
+
const children = [];
|
|
163
|
+
for (let at = parent.start; at < parent.end;) {
|
|
164
|
+
const child = derAt(der, at);
|
|
165
|
+
children.push(child);
|
|
166
|
+
at = child.end;
|
|
167
|
+
}
|
|
168
|
+
return children;
|
|
169
|
+
}
|
|
170
|
+
function oidText(bytes) {
|
|
171
|
+
const parts = [Math.floor(bytes[0] / 40), bytes[0] % 40];
|
|
172
|
+
let n = 0;
|
|
173
|
+
for (const byte of bytes.subarray(1)) {
|
|
174
|
+
n = n * 128 + (byte & 0x7f);
|
|
175
|
+
if (!(byte & 0x80)) {
|
|
176
|
+
parts.push(n);
|
|
177
|
+
n = 0;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return parts.join(".");
|
|
181
|
+
}
|
|
182
|
+
/** A certificate's extensions by OID: each one's value octets. Node's X509Certificate does not expose them. */
|
|
183
|
+
function certificateExtensions(der) {
|
|
184
|
+
const tbs = derChildren(der, derAt(der, 0))[0];
|
|
185
|
+
const found = new Map();
|
|
186
|
+
const wrapper = derChildren(der, tbs).find(element => element.tag === 0xa3);
|
|
187
|
+
if (!wrapper)
|
|
188
|
+
return found;
|
|
189
|
+
for (const extension of derChildren(der, derChildren(der, wrapper)[0])) {
|
|
190
|
+
const parts = derChildren(der, extension), oid = parts[0], value = parts[parts.length - 1];
|
|
191
|
+
if (oid.tag !== 0x06 || value.tag !== 0x04)
|
|
192
|
+
throw Error("malformed extension");
|
|
193
|
+
found.set(oidText(der.subarray(oid.start, oid.end)), der.subarray(value.start, value.end));
|
|
194
|
+
}
|
|
195
|
+
return found;
|
|
196
|
+
}
|
|
197
|
+
/** The signing certificate's own words: who asked Sigstore for it (issuer), which repository and which workflow. */
|
|
198
|
+
function certificateIdentity(der) {
|
|
199
|
+
const extensions = certificateExtensions(der);
|
|
200
|
+
const utf8 = (oid) => { const value = extensions.get(oid); if (!value)
|
|
201
|
+
return null; const one = derAt(value, 0); if (one.tag !== 0x0c)
|
|
202
|
+
throw Error("not a UTF8String"); return value.subarray(one.start, one.end).toString("utf8"); };
|
|
203
|
+
const raw = (oid) => extensions.get(oid)?.toString("utf8") ?? null;
|
|
204
|
+
const san = extensions.get(OID.subjectAltName);
|
|
205
|
+
const uris = san ? derChildren(san, derAt(san, 0)).filter(name => name.tag === 0x86).map(name => san.subarray(name.start, name.end).toString("ascii")) : [];
|
|
206
|
+
return { issuer: utf8(OID.issuer) ?? raw(OID.issuerV1), repository: utf8(OID.sourceRepository), repositoryV1: raw(OID.repositoryV1), signer: utf8(OID.buildSigner), uris };
|
|
207
|
+
}
|
|
208
|
+
/** The npm provenance for exactly these bytes of Toolroll itself must be signed by a certificate that names the
|
|
209
|
+
* Toolroll repository, its publish workflow and GitHub Actions as the issuer. The statement's own claims name
|
|
210
|
+
* nobody: only the certificate, whose key must have signed the statement. Sigstore's chain and transparency log are
|
|
211
|
+
* then checked by npm's own verifier (UpdateSystem.sigstore) for the identity returned here. */
|
|
121
212
|
export function checkProvenance(attestations, version, sha512Hex) {
|
|
122
213
|
const list = attestations?.attestations;
|
|
123
214
|
if (!Array.isArray(list) || list.length === 0)
|
|
124
215
|
throw new Refusal(`Toolroll ${version} has no npm provenance, so it cannot be verified. Nothing was changed.`);
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
216
|
+
const found = list.find(one => String(one?.predicateType ?? "").startsWith("https://slsa.dev/provenance/"));
|
|
217
|
+
if (!found)
|
|
218
|
+
throw new Refusal(`Toolroll ${version} has no build provenance statement. Nothing was changed.`);
|
|
219
|
+
const bundle = found.bundle ?? {}, envelope = bundle.dsseEnvelope ?? {};
|
|
220
|
+
let statement, certificate, who;
|
|
221
|
+
const payload = Buffer.from(String(envelope.payload ?? ""), "base64"), payloadType = String(envelope.payloadType ?? "");
|
|
222
|
+
try {
|
|
223
|
+
const material = bundle.verificationMaterial;
|
|
224
|
+
const der = Buffer.from(String(material?.certificate?.rawBytes ?? material?.x509CertificateChain?.certificates?.[0]?.rawBytes ?? ""), "base64");
|
|
225
|
+
certificate = new X509Certificate(der);
|
|
226
|
+
who = certificateIdentity(der);
|
|
227
|
+
statement = JSON.parse(payload.toString("utf8"));
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
throw new Refusal(`Toolroll ${version}'s provenance could not be read. Nothing was changed.`);
|
|
231
|
+
}
|
|
232
|
+
// DSSE: the signature covers the pre-authentication encoding of the payload and its type.
|
|
233
|
+
const signed = Buffer.concat([Buffer.from(`DSSEv1 ${Buffer.byteLength(payloadType)} ${payloadType} ${payload.length} `), payload]);
|
|
234
|
+
const signatures = envelope.signatures ?? [];
|
|
235
|
+
let valid = false;
|
|
236
|
+
try {
|
|
237
|
+
valid = signatures.length === 1 && typeof signatures[0].sig === "string" && signatureValid("sha256", signed, certificate.publicKey, Buffer.from(signatures[0].sig, "base64"));
|
|
143
238
|
}
|
|
144
|
-
|
|
239
|
+
catch {
|
|
240
|
+
valid = false;
|
|
241
|
+
}
|
|
242
|
+
if (!valid)
|
|
243
|
+
throw new Refusal(`Toolroll ${version}'s provenance is not signed by the certificate it carries. Nothing was changed.`);
|
|
244
|
+
const subject = statement.subject?.find(s => s.name === `pkg:npm/${NAME}@${version}`);
|
|
245
|
+
if (!subject || subject.digest?.sha512 !== sha512Hex)
|
|
246
|
+
throw new Refusal(`Toolroll ${version}'s provenance is for different bytes than the package downloaded. Nothing was changed.`);
|
|
247
|
+
const workflow = `${PROVENANCE_REPOSITORY}/${PROVENANCE_WORKFLOW}@`;
|
|
248
|
+
const identity = who.uris.length === 1 ? who.uris[0] : null;
|
|
249
|
+
const repository = who.repository ?? (who.repositoryV1 === null ? null : `https://github.com/${who.repositoryV1}`);
|
|
250
|
+
const named = who.issuer === PROVENANCE_ISSUER && repository === PROVENANCE_REPOSITORY
|
|
251
|
+
&& (who.repositoryV1 === null || `https://github.com/${who.repositoryV1}` === PROVENANCE_REPOSITORY)
|
|
252
|
+
&& identity !== null && identity.startsWith(workflow) && (who.signer === null || who.signer.startsWith(workflow));
|
|
253
|
+
if (!named || identity === null || who.issuer === null) {
|
|
254
|
+
const by = identity ?? repository ?? "an unnamed workflow";
|
|
255
|
+
throw new Refusal(`Toolroll ${version} was built by ${by}${who.issuer === PROVENANCE_ISSUER ? "" : ` (signed in by ${who.issuer ?? "no issuer"})`}, not ${PROVENANCE_REPOSITORY.replace("https://github.com/", "")} (${PROVENANCE_WORKFLOW}) on GitHub Actions. Nothing was changed.`);
|
|
256
|
+
}
|
|
257
|
+
return { repository: PROVENANCE_REPOSITORY, workflow: PROVENANCE_WORKFLOW, issuer: who.issuer, identity, bundle };
|
|
145
258
|
}
|
|
146
259
|
/** Headlines of this version's changelog section: its bold lead phrases. */
|
|
147
260
|
export function releaseNotes(changelog, version) {
|
|
@@ -223,17 +336,53 @@ async function rehearse(j, system, source) {
|
|
|
223
336
|
rmSync(copy + suffix, { force: true });
|
|
224
337
|
}
|
|
225
338
|
}
|
|
226
|
-
/** Put a verified backup in place of a live SQLite file. Only once the service's processes are gone.
|
|
339
|
+
/** Put a verified backup in place of a live SQLite file. Only once the service's processes are gone. The live file
|
|
340
|
+
* is replaced only by a complete copy: a copy that fails leaves it as it was. */
|
|
227
341
|
function restoreFile(live, backup, hash, what) {
|
|
228
342
|
if (fileHash(backup) !== hash)
|
|
229
343
|
throw Error(`The retained ${what} backup changed. It was not put back.`);
|
|
230
344
|
const temp = `${live}.${randomUUID()}.restore`;
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
345
|
+
try {
|
|
346
|
+
copyFileSync(backup, temp);
|
|
347
|
+
chmodSync(temp, 0o600);
|
|
348
|
+
for (const suffix of ["-wal", "-shm"])
|
|
349
|
+
rmSync(live + suffix, { force: true });
|
|
350
|
+
durableRename(temp, live);
|
|
351
|
+
}
|
|
352
|
+
finally {
|
|
353
|
+
rmSync(temp, { force: true });
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/** Before anything is moved or replaced: every backup is still the one recorded, and there is room for the copy
|
|
357
|
+
* kept aside and for the backups put back. */
|
|
358
|
+
function assertRestorable(j, from, system) {
|
|
359
|
+
if (fileHash(from.path) !== from.hash)
|
|
360
|
+
throw Error("The retained database backup changed. Nothing was put back; the live database is as it was.");
|
|
361
|
+
if (from.codingPath && fileHash(from.codingPath) !== from.codingHash)
|
|
362
|
+
throw Error("The retained coding catalog backup changed. Nothing was put back; the live database is as it was.");
|
|
363
|
+
const size = (file) => { try {
|
|
364
|
+
return statSync(file).size;
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
return 0;
|
|
368
|
+
} };
|
|
369
|
+
const live = size(j.databaseFile) + size(`${j.databaseFile}-wal`);
|
|
370
|
+
const needs = new Map();
|
|
371
|
+
const need = (dir, bytes) => { const dev = statSync(dir).dev; const at = needs.get(dev) ?? { dir, bytes: 0 }; at.bytes += bytes; needs.set(dev, at); };
|
|
372
|
+
need(j.stageDir, live);
|
|
373
|
+
need(dirname(j.databaseFile), size(from.path) + (from.codingPath ? size(from.codingPath) : 0));
|
|
374
|
+
const free = system.freeBytes ?? freeBytes;
|
|
375
|
+
for (const { dir, bytes } of needs.values()) {
|
|
376
|
+
const available = free(dir);
|
|
377
|
+
if (available < bytes)
|
|
378
|
+
throw Error(`${dir} has ${megabytes(available)} free and the restore needs ${megabytes(bytes)}. Nothing was put back; the live database is as it was. Free some space, then run toolroll update --resume.`);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
function freeBytes(dir) {
|
|
382
|
+
const fs = statfsSync(dir);
|
|
383
|
+
return Number(fs.bavail) * Number(fs.bsize);
|
|
236
384
|
}
|
|
385
|
+
const megabytes = (bytes) => `${Math.ceil(bytes / 1_048_576)} MB`;
|
|
237
386
|
/** The database and, when one was backed up, the coding catalog, then this run's coding gate lifted. */
|
|
238
387
|
function restoreDatabase(j, from) {
|
|
239
388
|
restoreFile(j.databaseFile, from.path, from.hash, "database");
|
|
@@ -241,27 +390,119 @@ function restoreDatabase(j, from) {
|
|
|
241
390
|
restoreFile(codingFile(j.databaseFile), from.codingPath, from.codingHash, "coding catalog");
|
|
242
391
|
ungateCoding(j);
|
|
243
392
|
}
|
|
244
|
-
/**
|
|
393
|
+
/** SQLite's own answer for a file it cannot read as a database: corrupt, or not a database at all. Anything else
|
|
394
|
+
* (busy, a full disk, a permission) is not a reason to move the live database. */
|
|
395
|
+
function unreadableDatabase(error) {
|
|
396
|
+
const code = error.errcode;
|
|
397
|
+
return typeof code === "number" && [11, 26].includes(code & 0xff);
|
|
398
|
+
}
|
|
399
|
+
/** The kept-aside copies a journal names, oldest first; a journal written by 0.8.1 names only its newest. */
|
|
400
|
+
function keptOf(j) {
|
|
401
|
+
return j.kept ?? (j.keptAside ? [{ path: j.keptAside, unreadable: j.keptAsideUnreadable === true }] : []);
|
|
402
|
+
}
|
|
403
|
+
function recordKept(j, path, unreadable) {
|
|
404
|
+
j.kept = [...keptOf(j), { path, unreadable }];
|
|
405
|
+
j.keptAside = path;
|
|
406
|
+
if (unreadable)
|
|
407
|
+
j.keptAsideUnreadable = true;
|
|
408
|
+
else
|
|
409
|
+
delete j.keptAsideUnreadable;
|
|
410
|
+
}
|
|
411
|
+
function forgetKept(j, path) {
|
|
412
|
+
j.kept = keptOf(j).filter(one => one.path !== path);
|
|
413
|
+
const newest = j.kept[j.kept.length - 1];
|
|
414
|
+
if (newest) {
|
|
415
|
+
j.keptAside = newest.path;
|
|
416
|
+
if (newest.unreadable)
|
|
417
|
+
j.keptAsideUnreadable = true;
|
|
418
|
+
else
|
|
419
|
+
delete j.keptAsideUnreadable;
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
delete j.keptAside;
|
|
423
|
+
delete j.keptAsideUnreadable;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
/** A private copy of the live database before a restore replaces it: what the new version wrote is kept, and named.
|
|
427
|
+
* Only a database SQLite says is corrupt or not a database is moved aside whole, with its WAL; any other failure
|
|
428
|
+
* stops the restore with the live database where it was. Returns what was moved, so a failed restore moves it back. */
|
|
245
429
|
async function keepAside(j) {
|
|
246
|
-
|
|
247
|
-
|
|
430
|
+
// Nothing at the live path: an earlier attempt moved it aside and stopped before putting the backup back. That
|
|
431
|
+
// copy stays named in `kept`.
|
|
432
|
+
if (!existsSync(j.databaseFile))
|
|
433
|
+
return [];
|
|
434
|
+
const id = randomUUID().slice(0, 8), kept = join(j.stageDir, `orders.kept.${id}.db`);
|
|
248
435
|
try {
|
|
249
|
-
|
|
436
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile, { readOnly: true });
|
|
437
|
+
try {
|
|
438
|
+
await sqlite().backup(db, kept);
|
|
439
|
+
}
|
|
440
|
+
finally {
|
|
441
|
+
db.close();
|
|
442
|
+
}
|
|
443
|
+
chmodSync(kept, 0o600);
|
|
444
|
+
fsyncPath(kept);
|
|
445
|
+
recordKept(j, kept, false);
|
|
446
|
+
return [];
|
|
250
447
|
}
|
|
251
|
-
|
|
252
|
-
|
|
448
|
+
catch (error) {
|
|
449
|
+
rmSync(kept, { force: true });
|
|
450
|
+
if (!unreadableDatabase(error))
|
|
451
|
+
throw Error(`The live database could not be copied aside before the restore (${error.message || "no reason given"}). It was left as it was, and nothing was put back.`);
|
|
253
452
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
453
|
+
const unreadable = join(j.stageDir, `orders.unreadable.${id}.db`);
|
|
454
|
+
const moved = [];
|
|
455
|
+
try {
|
|
456
|
+
for (const suffix of ["-wal", "-shm", ""]) {
|
|
457
|
+
if (!existsSync(j.databaseFile + suffix))
|
|
458
|
+
continue;
|
|
459
|
+
durableRename(j.databaseFile + suffix, unreadable + suffix);
|
|
460
|
+
moved.push({ from: j.databaseFile + suffix, to: unreadable + suffix });
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
catch (error) {
|
|
464
|
+
moveBack(moved);
|
|
465
|
+
throw Error(`The unreadable live database could not be moved aside (${error.message}). It was left as it was, and nothing was put back.`);
|
|
466
|
+
}
|
|
467
|
+
recordKept(j, unreadable, true);
|
|
468
|
+
return moved;
|
|
469
|
+
}
|
|
470
|
+
/** Undo keepAside's moves: the live database goes back where it was, never leaving the path empty. */
|
|
471
|
+
function moveBack(moved) {
|
|
472
|
+
for (const one of [...moved].reverse())
|
|
473
|
+
if (!existsSync(one.from) && existsSync(one.to))
|
|
474
|
+
durableRename(one.to, one.from);
|
|
257
475
|
}
|
|
258
476
|
// ---- the service: stop, and see it gone -------------------------------------
|
|
259
|
-
/**
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
477
|
+
/** The service and watch daemons this run stops, switches and restarts: once switched, the ones it switched. */
|
|
478
|
+
function unitsOf(j) {
|
|
479
|
+
return { main: j.switched?.unit?.path ?? j.service?.unit ?? null, watches: j.switched?.watches?.map(w => w.path) ?? j.watches?.map(w => w.unit) ?? [] };
|
|
480
|
+
}
|
|
481
|
+
/** The watch daemons launchd had loaded before the update: the only ones stopped and started again. One a person
|
|
482
|
+
* unloaded stays unloaded (its definition is still switched, so it starts the new version when they load it). */
|
|
483
|
+
const loadedWatches = (j, watches) => watches.filter(unit => j.watches?.find(w => w.unit === unit)?.loaded !== false);
|
|
484
|
+
/** Stop the service and every loaded watch daemon, and wait until every process they ran has exited: only then may
|
|
485
|
+
* a file they write be replaced. */
|
|
486
|
+
async function stopServices(j, system, main, watches) {
|
|
487
|
+
if (!main && watches.length === 0)
|
|
488
|
+
return;
|
|
489
|
+
const pidsOf = async (unit, earlier) => [...new Set([...earlier, ...await system.servicePids(unit)])];
|
|
490
|
+
if (main)
|
|
491
|
+
j.service = { unit: main, pids: await pidsOf(main, j.service?.unit === main ? j.service.pids : []) };
|
|
492
|
+
const watching = [];
|
|
493
|
+
for (const unit of watches) {
|
|
494
|
+
const earlier = j.watches?.find(w => w.unit === unit);
|
|
495
|
+
// Recorded once, before the first stop: a resumed run must not read its own stop as the person's choice.
|
|
496
|
+
const loaded = earlier ? earlier.loaded !== false : await system.serviceLoaded(unit);
|
|
497
|
+
watching.push({ unit, pids: loaded ? await pidsOf(unit, earlier?.pids ?? []) : [], loaded });
|
|
498
|
+
}
|
|
499
|
+
if (watching.length > 0)
|
|
500
|
+
j.watches = watching;
|
|
263
501
|
save(j, j.phase, j.detail, system.now());
|
|
264
|
-
|
|
502
|
+
for (const unit of [main, ...loadedWatches(j, watches)])
|
|
503
|
+
if (unit)
|
|
504
|
+
await system.stopService(unit);
|
|
505
|
+
const pids = [...(main ? j.service.pids : []), ...watching.flatMap(w => w.pids)];
|
|
265
506
|
const deadline = system.now().getTime() + (system.exitTimeoutMs ?? 60_000);
|
|
266
507
|
for (;;) {
|
|
267
508
|
const alive = pids.filter(pid => system.processAlive(pid));
|
|
@@ -280,8 +521,8 @@ function pointLink(path, target) {
|
|
|
280
521
|
}
|
|
281
522
|
/** What runs the current version: its service definition and every command on PATH. */
|
|
282
523
|
function switchable(j, system) {
|
|
283
|
-
const unit = system.serviceUnit(j.from), links = system.commands(j.from);
|
|
284
|
-
if (!unit && links.length === 0)
|
|
524
|
+
const unit = system.serviceUnit(j.from), watches = system.watchUnits(j.from), links = system.commands(j.from);
|
|
525
|
+
if (!unit && watches.length === 0 && links.length === 0)
|
|
285
526
|
throw new Refusal(`No background service or toolroll command runs ${j.from.version} from ${j.from.dist}, so there is nothing to switch. Nothing was changed.`);
|
|
286
527
|
const real = (path) => { try {
|
|
287
528
|
return realpathSync(path);
|
|
@@ -296,44 +537,76 @@ function switchable(j, system) {
|
|
|
296
537
|
const target = real(link);
|
|
297
538
|
if (target === null || !target.startsWith(root + sep))
|
|
298
539
|
throw new Refusal(`${link} runs a different Toolroll (${target ?? "a missing file"}), not ${j.from.version}, so it cannot be switched. Remove it, then update again. Nothing was changed.`);
|
|
540
|
+
// A link is replaced by a rename beside it: its folder must take one, or the switch would stop halfway.
|
|
541
|
+
try {
|
|
542
|
+
accessSync(dirname(link), constants.W_OK);
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
throw new Refusal(`${dirname(link)} cannot be written, so ${link} would keep running ${j.from.version}. Make that folder writable, then update again. Nothing was changed.`);
|
|
546
|
+
}
|
|
299
547
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
548
|
+
for (const one of [unit, ...watches])
|
|
549
|
+
if (one && !readFileSync(one, "utf8").includes(j.from.dist))
|
|
550
|
+
throw new Refusal("The service definition does not name the current runtime. Nothing was changed.");
|
|
551
|
+
return { unit, watches, links };
|
|
303
552
|
}
|
|
304
553
|
/** Every change is recorded before it is made; `revert` undoes exactly those.
|
|
305
554
|
* A rollback puts the earlier database back only once the service is gone. */
|
|
306
555
|
async function switchRuntime(j, system) {
|
|
307
556
|
// A resumed switch keeps what it first recorded, so a revert still reaches the original.
|
|
308
557
|
if (!j.switched) {
|
|
309
|
-
const { unit, links } = switchable(j, system);
|
|
310
|
-
j.switched = { links: links.map(path => ({ path, previous: readlinkSync(path) })), unit: null };
|
|
558
|
+
const { unit, watches, links } = switchable(j, system);
|
|
559
|
+
j.switched = { links: links.map(path => ({ path, previous: readlinkSync(path) })), unit: null, watches: [] };
|
|
311
560
|
if (unit) {
|
|
312
561
|
const saved = join(j.stageDir, "service.saved.plist");
|
|
313
562
|
writeFileDurably(saved, readFileSync(unit));
|
|
314
563
|
j.switched.unit = { path: unit, saved };
|
|
315
564
|
}
|
|
316
|
-
|
|
565
|
+
for (const [i, path] of watches.entries()) {
|
|
566
|
+
const saved = join(j.stageDir, `watch.${i}.saved.plist`);
|
|
567
|
+
writeFileDurably(saved, readFileSync(path));
|
|
568
|
+
j.switched.watches.push({ path, saved });
|
|
569
|
+
}
|
|
570
|
+
save(j, "switching", `Pointing ${unit || watches.length > 0 ? "the service and " : ""}the toolroll and standing-orders commands at ${j.to.version}.`, system.now());
|
|
317
571
|
}
|
|
318
|
-
const
|
|
572
|
+
const { main, watches } = unitsOf(j);
|
|
319
573
|
if (j.restoreFrom && !j.switched.databaseRestored) {
|
|
320
|
-
|
|
321
|
-
|
|
574
|
+
await stopServices(j, system, main, watches);
|
|
575
|
+
refuseWhileWatching(j, system, true);
|
|
576
|
+
assertRestorable(j, j.restoreFrom, system);
|
|
322
577
|
restoreDatabase(j, j.restoreFrom);
|
|
323
578
|
gate(j);
|
|
324
579
|
j.switched.databaseRestored = true;
|
|
325
580
|
save(j, "switching", j.detail, system.now());
|
|
326
581
|
}
|
|
327
|
-
|
|
328
|
-
|
|
582
|
+
for (const one of [j.switched.unit, ...j.switched.watches ?? []])
|
|
583
|
+
if (one)
|
|
584
|
+
writeFileDurably(one.path, readFileSync(one.saved, "utf8").replaceAll(j.from.dist, j.to.dist), 0o644);
|
|
329
585
|
for (const link of j.switched.links)
|
|
330
586
|
pointLink(link.path, join(j.to.dist, "bin.js"));
|
|
331
587
|
}
|
|
588
|
+
/** Put back every definition and command it can; one it cannot write is skipped and named, so the restore goes on. */
|
|
332
589
|
function revertSwitch(j) {
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
590
|
+
const stuck = [];
|
|
591
|
+
for (const one of [j.switched?.unit, ...j.switched?.watches ?? []]) {
|
|
592
|
+
if (!one || !existsSync(one.saved))
|
|
593
|
+
continue;
|
|
594
|
+
try {
|
|
595
|
+
writeFileDurably(one.path, readFileSync(one.saved), 0o644);
|
|
596
|
+
}
|
|
597
|
+
catch (error) {
|
|
598
|
+
stuck.push(`${one.path} (${error.message})`);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
for (const link of j.switched?.links ?? []) {
|
|
602
|
+
try {
|
|
603
|
+
pointLink(link.path, link.previous);
|
|
604
|
+
}
|
|
605
|
+
catch (error) {
|
|
606
|
+
stuck.push(`${link.path} (${error.message})`);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return stuck;
|
|
337
610
|
}
|
|
338
611
|
function gate(j) {
|
|
339
612
|
const db = new (sqlite().DatabaseSync)(j.databaseFile);
|
|
@@ -371,17 +644,29 @@ function ungateCoding(j) {
|
|
|
371
644
|
db.close();
|
|
372
645
|
}
|
|
373
646
|
}
|
|
374
|
-
/** A foreground `toolroll up` holds a live watch lease; the runtime under it must not be replaced.
|
|
375
|
-
|
|
376
|
-
|
|
647
|
+
/** A foreground `toolroll up` holds a live watch lease; the runtime or database under it must not be replaced.
|
|
648
|
+
* (A watch daemon releases its lease when it is stopped.) `restoring`: checked right before a backup is put back,
|
|
649
|
+
* where a database that cannot be read has no watch on it to protect. */
|
|
650
|
+
function refuseWhileWatching(j, system, restoring = false) {
|
|
651
|
+
let row;
|
|
377
652
|
try {
|
|
378
|
-
const
|
|
379
|
-
|
|
380
|
-
|
|
653
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile, { readOnly: true });
|
|
654
|
+
try {
|
|
655
|
+
row = db.prepare("SELECT runner, repo FROM watch_lease WHERE expires_at > ? LIMIT 1").get(system.now().toISOString());
|
|
656
|
+
}
|
|
657
|
+
finally {
|
|
658
|
+
db.close();
|
|
659
|
+
}
|
|
381
660
|
}
|
|
382
|
-
|
|
383
|
-
|
|
661
|
+
catch (error) {
|
|
662
|
+
if (!restoring)
|
|
663
|
+
throw error;
|
|
664
|
+
row = undefined;
|
|
384
665
|
}
|
|
666
|
+
if (!row)
|
|
667
|
+
return;
|
|
668
|
+
const what = `toolroll up is running for ${String(row["repo"])} (${String(row["runner"])}).`;
|
|
669
|
+
throw new Refusal(restoring ? `${what} The database was not put back under it; stop it first.` : `${what} Stop it first; ${j.kind === "update" ? "an update replaces the version it runs" : "a rollback replaces the database"}. Nothing was changed.`);
|
|
385
670
|
}
|
|
386
671
|
function nextAt(hhmm, now) {
|
|
387
672
|
const [h, m] = hhmm.split(":").map(Number);
|
|
@@ -410,6 +695,7 @@ export function prepareRuntimeUpdate(o, now) {
|
|
|
410
695
|
return { refused: `Toolroll ${o.version} is not newer than ${o.current.version}. To go back to an older release, run toolroll update --version ${o.version} --allow-downgrade.` };
|
|
411
696
|
try {
|
|
412
697
|
assertNoneActive(o.stateDir);
|
|
698
|
+
keepCompletedUpdate(o.stateDir);
|
|
413
699
|
}
|
|
414
700
|
catch (error) {
|
|
415
701
|
return { refused: error.message };
|
|
@@ -435,12 +721,13 @@ export async function startRuntimeRollback(o, system) {
|
|
|
435
721
|
let last;
|
|
436
722
|
try {
|
|
437
723
|
assertNoneActive(o.stateDir);
|
|
438
|
-
|
|
724
|
+
keepCompletedUpdate(o.stateDir);
|
|
725
|
+
last = lastCompletedUpdate(o.stateDir);
|
|
439
726
|
}
|
|
440
727
|
catch (error) {
|
|
441
728
|
return { ok: false, phase: "refused", message: error.message, journal: null };
|
|
442
729
|
}
|
|
443
|
-
if (!last ||
|
|
730
|
+
if (!last || !last.backupPath || !last.backupHash)
|
|
444
731
|
return { ok: false, phase: "refused", message: "There is no completed update to roll back.", journal: null };
|
|
445
732
|
if (!existsSync(join(last.from.dist, "bin.js")) || !existsSync(last.backupPath) || (last.codingBackupPath && !existsSync(last.codingBackupPath)))
|
|
446
733
|
return { ok: false, phase: "refused", message: `The previous runtime (${last.from.version}) or its backup is no longer on this machine. Nothing was changed.`, journal: null };
|
|
@@ -465,14 +752,49 @@ export function abandonRuntimeUpdate(stateDir, id, why, now) {
|
|
|
465
752
|
if (j?.id === id && j.phase === "scheduled")
|
|
466
753
|
save(j, "refused", why, now);
|
|
467
754
|
}
|
|
468
|
-
|
|
755
|
+
const CANCELLABLE = ["scheduled", "verifying", "draining"];
|
|
756
|
+
/** `locked`: a test's seam, called once the updater's lock is held and before the journal is read again. */
|
|
757
|
+
export function requestRuntimeUpdateCancel(stateDir, now = new Date(), seams = {}) {
|
|
469
758
|
const j = readRuntimeUpdate(stateDir);
|
|
470
759
|
if (!j || runtimeUpdateTerminal(j.phase))
|
|
471
760
|
return "No update is in progress.";
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
761
|
+
const tooLate = (at) => `The update to ${at.to.version} is past the point it can be cancelled (${at.phase}); it will finish or restore on its own.`;
|
|
762
|
+
if (!CANCELLABLE.includes(j.phase))
|
|
763
|
+
return tooLate(j);
|
|
764
|
+
// No updater is running to see the request: cancel here, so the admission pause does not outlive it.
|
|
765
|
+
const lock = sqliteLock(join(j.stageDir, "worker.sqlite"));
|
|
766
|
+
if (!lock) {
|
|
767
|
+
durableJson(cancelFile(j), { id: j.id, action: "cancel" });
|
|
768
|
+
return `Cancelling the update to ${j.to.version}. Nothing is switched; new work resumes.`;
|
|
769
|
+
}
|
|
770
|
+
try {
|
|
771
|
+
seams.locked?.();
|
|
772
|
+
// An updater may have moved on, finished or been replaced between the first read and the lock.
|
|
773
|
+
const held = readRuntimeUpdate(stateDir);
|
|
774
|
+
if (!held || runtimeUpdateTerminal(held.phase))
|
|
775
|
+
return "No update is in progress.";
|
|
776
|
+
if (held.id !== j.id)
|
|
777
|
+
return "A different update was saved while cancelling. Nothing was cancelled.";
|
|
778
|
+
if (!CANCELLABLE.includes(held.phase))
|
|
779
|
+
return tooLate(held);
|
|
780
|
+
durableJson(cancelFile(held), { id: held.id, action: "cancel" });
|
|
781
|
+
try {
|
|
782
|
+
ungate(held);
|
|
783
|
+
}
|
|
784
|
+
catch { /* no pause yet */ }
|
|
785
|
+
if (held.kind === "update")
|
|
786
|
+
rmSync(join(held.stageDir, "runtime"), { recursive: true, force: true });
|
|
787
|
+
const noun = held.kind === "update" ? "update" : "rollback";
|
|
788
|
+
save(held, "cancelled", `The ${noun} to ${held.to.version} was cancelled. Nothing changed; new work resumed.`, now);
|
|
789
|
+
try {
|
|
790
|
+
ledger(held.databaseFile, now, held.actor, `toolroll ${noun} cancelled`, "cancelled", `${held.from.version} → ${held.to.version}`);
|
|
791
|
+
}
|
|
792
|
+
catch { /* the journal still says what happened */ }
|
|
793
|
+
return `Cancelled the ${noun} to ${held.to.version}. Nothing was switched; new work resumes.`;
|
|
794
|
+
}
|
|
795
|
+
finally {
|
|
796
|
+
lock.close();
|
|
797
|
+
}
|
|
476
798
|
}
|
|
477
799
|
const cancelRequested = (j) => existsSync(cancelFile(j));
|
|
478
800
|
async function driveRuntimeUpdate(j, system) {
|
|
@@ -512,8 +834,8 @@ async function driveRuntimeUpdate(j, system) {
|
|
|
512
834
|
else if (fileHash(j.restoreFrom.path) !== j.restoreFrom.hash || (j.restoreFrom.codingPath && fileHash(j.restoreFrom.codingPath) !== j.restoreFrom.codingHash))
|
|
513
835
|
throw new Refusal("The update's backup changed since it was made. Nothing was changed.");
|
|
514
836
|
}
|
|
515
|
-
const unit = switchable(j, system)
|
|
516
|
-
if (!unit)
|
|
837
|
+
const { unit, watches } = switchable(j, system);
|
|
838
|
+
if (!unit && watches.length === 0)
|
|
517
839
|
refuseWhileWatching(j, system);
|
|
518
840
|
if (resumedAt <= at("draining")) {
|
|
519
841
|
step("draining", j.when === "now" ? "Checking that no work is running, then pausing new work." : "Pausing new work and letting running work finish. Nothing is being cancelled.");
|
|
@@ -524,9 +846,8 @@ async function driveRuntimeUpdate(j, system) {
|
|
|
524
846
|
}
|
|
525
847
|
}
|
|
526
848
|
// The service stops before the backup, so nothing it writes afterwards can be lost by a restore.
|
|
527
|
-
step("backing-up", unit ? "Stopping the background service, then backing up the database." : "Backing up the database.");
|
|
528
|
-
|
|
529
|
-
await stopService(j, system, unit);
|
|
849
|
+
step("backing-up", unit || watches.length > 0 ? "Stopping the background service, then backing up the database." : "Backing up the database.");
|
|
850
|
+
await stopServices(j, system, unit, watches);
|
|
530
851
|
refuseWhileWatching(j, system);
|
|
531
852
|
const db = new (sqlite().DatabaseSync)(j.databaseFile, { readOnly: true });
|
|
532
853
|
try {
|
|
@@ -547,10 +868,11 @@ async function driveRuntimeUpdate(j, system) {
|
|
|
547
868
|
await switchRuntime(j, system);
|
|
548
869
|
}
|
|
549
870
|
if (resumedAt <= at("restarting")) {
|
|
550
|
-
const
|
|
551
|
-
step("restarting",
|
|
552
|
-
|
|
553
|
-
|
|
871
|
+
const { main, watches } = unitsOf(j);
|
|
872
|
+
step("restarting", main || watches.length > 0 ? "Restarting the background service." : "No background service runs here; the commands now start the new version.");
|
|
873
|
+
for (const unit of [main, ...loadedWatches(j, watches)])
|
|
874
|
+
if (unit)
|
|
875
|
+
await system.restartService(unit);
|
|
554
876
|
}
|
|
555
877
|
step("health", `Checking that ${j.to.version} is running and healthy.`);
|
|
556
878
|
const deadline = system.now().getTime() + (system.healthTimeoutMs ?? 90_000);
|
|
@@ -566,6 +888,15 @@ async function driveRuntimeUpdate(j, system) {
|
|
|
566
888
|
const done = j.kind === "update"
|
|
567
889
|
? finish(true, "complete", `Toolroll ${j.to.version} is running. Your previous version (${j.from.version}) and its backup are kept; toolroll update --rollback returns to them.`, "toolroll updated", "complete", verb)
|
|
568
890
|
: finish(true, "complete", `Back on Toolroll ${j.to.version} with the database from before the update. The records made since are kept in ${basename(j.backupPath ?? "")}.`, "toolroll rolled back", "complete", verb);
|
|
891
|
+
// What --rollback returns from, kept apart from the journal a later refused or failed attempt replaces.
|
|
892
|
+
// The update is done: failing to write this must never send it back through the restore below.
|
|
893
|
+
try {
|
|
894
|
+
if (j.kind === "update")
|
|
895
|
+
durableJson(lastUpdateFile(j.stateDir), j);
|
|
896
|
+
else
|
|
897
|
+
rmSync(lastUpdateFile(j.stateDir), { force: true });
|
|
898
|
+
}
|
|
899
|
+
catch { /* the journal still holds it until the next attempt */ }
|
|
569
900
|
try {
|
|
570
901
|
pruneRuntimes(j.stateDir, [j.to.dist, j.from.dist]);
|
|
571
902
|
}
|
|
@@ -585,14 +916,16 @@ async function driveRuntimeUpdate(j, system) {
|
|
|
585
916
|
// Nothing uses a refused download; the database backup is kept.
|
|
586
917
|
if (j.kind === "update")
|
|
587
918
|
rmSync(join(j.stageDir, "runtime"), { recursive: true, force: true });
|
|
588
|
-
// A service stopped for the backup
|
|
919
|
+
// A service and watch daemons stopped for the backup start again, unchanged.
|
|
589
920
|
let restarted = "";
|
|
590
|
-
|
|
921
|
+
for (const unit of [j.service?.unit, ...(j.watches ?? []).filter(w => w.loaded !== false).map(w => w.unit)]) {
|
|
922
|
+
if (!unit)
|
|
923
|
+
continue;
|
|
591
924
|
try {
|
|
592
|
-
await system.restartService(
|
|
925
|
+
await system.restartService(unit);
|
|
593
926
|
}
|
|
594
927
|
catch (again) {
|
|
595
|
-
restarted
|
|
928
|
+
restarted += ` ${unit === j.service?.unit ? "The background service" : `The watch ${basename(unit, ".plist")}`} did not start again: ${again.message}`;
|
|
596
929
|
}
|
|
597
930
|
}
|
|
598
931
|
const refused = error instanceof Refusal;
|
|
@@ -621,24 +954,46 @@ async function restore(j, system, finish, verb, why) {
|
|
|
621
954
|
try {
|
|
622
955
|
save(j, "rolling-back", `${why} Restoring ${j.from.version} and the database backup.`, system.now());
|
|
623
956
|
system.checkpoint?.("rolling-back");
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
957
|
+
// The commands go back first, so a stop that fails never leaves them on the failed version.
|
|
958
|
+
const stuck = revertSwitch(j);
|
|
959
|
+
const { main, watches } = unitsOf(j);
|
|
960
|
+
await stopServices(j, system, main, watches);
|
|
628
961
|
if (!j.backupPath || !j.backupHash)
|
|
629
962
|
throw Error("No verified backup was recorded.");
|
|
630
963
|
if (!j.restoredDatabase) {
|
|
964
|
+
refuseWhileWatching(j, system, true);
|
|
965
|
+
const from = { path: j.backupPath, hash: j.backupHash, ...(j.codingBackupPath && j.codingBackupHash ? { codingPath: j.codingBackupPath, codingHash: j.codingBackupHash } : {}) };
|
|
966
|
+
// Nothing moves until the backups are the ones recorded and there is room to put them back.
|
|
967
|
+
assertRestorable(j, from, system);
|
|
631
968
|
// A fresh copy before EVERY attempt that is about to replace the database, not only the first: a retry
|
|
632
969
|
// after a failure earlier in the restore must not overwrite writes made since without keeping them.
|
|
633
|
-
|
|
970
|
+
const moved = await keepAside(j);
|
|
634
971
|
save(j, "rolling-back", j.detail, system.now());
|
|
635
|
-
|
|
972
|
+
try {
|
|
973
|
+
system.checkpoint?.("kept-aside");
|
|
974
|
+
restoreDatabase(j, from);
|
|
975
|
+
}
|
|
976
|
+
catch (error) {
|
|
977
|
+
// The backup did not go in: the live database comes back to its own path, and is no longer named as kept.
|
|
978
|
+
if (moved.length > 0 && !existsSync(j.databaseFile)) {
|
|
979
|
+
moveBack(moved);
|
|
980
|
+
forgetKept(j, j.keptAside);
|
|
981
|
+
save(j, "rolling-back", j.detail, system.now());
|
|
982
|
+
}
|
|
983
|
+
throw error;
|
|
984
|
+
}
|
|
636
985
|
j.restoredDatabase = true;
|
|
637
986
|
save(j, "rolling-back", j.detail, system.now());
|
|
638
987
|
}
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
988
|
+
for (const unit of [main, ...loadedWatches(j, watches)])
|
|
989
|
+
if (unit)
|
|
990
|
+
await system.restartService(unit);
|
|
991
|
+
const copies = keptOf(j), readable = copies.filter(one => !one.unreadable).map(one => one.path), unreadable = copies.filter(one => one.unreadable).map(one => one.path);
|
|
992
|
+
const kept = (unreadable.length > 0 ? ` The live database could not be read, so it was left as it was at ${unreadable.join(" and ")}.` : "") + (readable.length > 0 ? ` Anything written since the backup is kept in ${readable.join(" and ")}.` : "");
|
|
993
|
+
const restored = `${why} Toolroll ${j.from.version} and its database were restored.${kept}`;
|
|
994
|
+
if (stuck.length > 0)
|
|
995
|
+
return finish(false, "needs-attention", `${restored} ${stuck.join("; ")} could not be pointed back at ${j.from.version}. Make ${stuck.length === 1 ? "its folder" : "those folders"} writable, then run toolroll update --resume.`, `toolroll ${noun} failed`, "needs attention", `${verb}: ${why} / ${stuck.join("; ")}`.slice(0, 300));
|
|
996
|
+
return finish(false, "restored", restored, `toolroll ${noun} failed`, "restored", `${verb}: ${why}`.slice(0, 300));
|
|
642
997
|
}
|
|
643
998
|
catch (error) {
|
|
644
999
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -651,17 +1006,20 @@ async function verify(j, system) {
|
|
|
651
1006
|
throw new Refusal(`The registry answered with ${release.version}, not ${j.to.version}. Nothing was changed.`);
|
|
652
1007
|
if (!release.attestations)
|
|
653
1008
|
throw new Refusal(`Toolroll ${j.to.version} has no npm provenance, so it cannot be verified. Nothing was changed.`);
|
|
1009
|
+
if (!onRegistry(release.attestations))
|
|
1010
|
+
throw new Refusal(`Toolroll ${j.to.version}'s provenance is not served by the npm registry (${release.attestations.slice(0, 120)}). Nothing was changed.`);
|
|
654
1011
|
const bytes = await system.download(release.tarball);
|
|
655
1012
|
const sha512 = createHash("sha512").update(bytes).digest();
|
|
656
1013
|
if (release.integrity !== `sha512-${sha512.toString("base64")}`)
|
|
657
1014
|
throw new Refusal(`The downloaded package does not match the registry's checksum. Nothing was changed.`);
|
|
658
1015
|
const provenance = checkProvenance(await system.attestations(release.attestations), j.to.version, sha512.toString("hex"));
|
|
1016
|
+
await system.sigstore(provenance.bundle, { issuer: provenance.issuer, identity: provenance.identity });
|
|
659
1017
|
const runtime = join(j.stageDir, "runtime");
|
|
660
1018
|
mkdirSync(runtime, { recursive: true, mode: 0o700 });
|
|
661
1019
|
j.to.dist = await system.install(runtime, release);
|
|
662
1020
|
if (!j.to.dist.startsWith(runtime + sep) || !existsSync(join(j.to.dist, "bin.js")))
|
|
663
1021
|
throw new Refusal("The staged runtime has no Toolroll command. Nothing was changed.");
|
|
664
|
-
j.package = { sha512: sha512.toString("hex"),
|
|
1022
|
+
j.package = { sha512: sha512.toString("hex"), repository: provenance.repository, workflow: provenance.workflow };
|
|
665
1023
|
try {
|
|
666
1024
|
j.notes = releaseNotes(readFileSync(join(dirname(j.to.dist), "CHANGELOG.md"), "utf8"), j.to.version);
|
|
667
1025
|
}
|
|
@@ -669,6 +1027,16 @@ async function verify(j, system) {
|
|
|
669
1027
|
j.notes = [];
|
|
670
1028
|
}
|
|
671
1029
|
}
|
|
1030
|
+
/** Served over HTTPS by the registry itself, as the package is. */
|
|
1031
|
+
function onRegistry(url) {
|
|
1032
|
+
try {
|
|
1033
|
+
const at = new URL(url), registry = new URL(REGISTRY);
|
|
1034
|
+
return at.protocol === "https:" && at.host === registry.host && !at.username && !at.password;
|
|
1035
|
+
}
|
|
1036
|
+
catch {
|
|
1037
|
+
return false;
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
672
1040
|
async function drain(j, system) {
|
|
673
1041
|
const db = new (sqlite().DatabaseSync)(j.databaseFile);
|
|
674
1042
|
try {
|
|
@@ -717,7 +1085,14 @@ export function pruneRuntimes(stateDir, keep) {
|
|
|
717
1085
|
for (const dir of releases)
|
|
718
1086
|
if (kept.size < KEEP_RUNTIMES)
|
|
719
1087
|
kept.add(dir);
|
|
720
|
-
|
|
1088
|
+
// A database a failed update kept aside is the only copy of what was written then: never pruned.
|
|
1089
|
+
const holdsKeptAside = (dir) => { try {
|
|
1090
|
+
return readdirSync(dir).some(name => /^orders\.(?:kept|unreadable)\./.test(name));
|
|
1091
|
+
}
|
|
1092
|
+
catch {
|
|
1093
|
+
return true;
|
|
1094
|
+
} };
|
|
1095
|
+
const removed = releases.filter(dir => !kept.has(dir) && !holdsKeptAside(dir));
|
|
721
1096
|
for (const dir of removed)
|
|
722
1097
|
rmSync(dir, { recursive: true, force: true });
|
|
723
1098
|
return removed;
|
|
@@ -737,7 +1112,14 @@ export function runtimeUpdateStatus(stateDir) {
|
|
|
737
1112
|
lock?.close();
|
|
738
1113
|
}
|
|
739
1114
|
const whatsNew = j && j.kind === "update" && j.phase === "complete" && !j.seen ? { version: j.to.version, notes: j.notes ?? [] } : null;
|
|
740
|
-
|
|
1115
|
+
let last = null;
|
|
1116
|
+
try {
|
|
1117
|
+
last = lastCompletedUpdate(stateDir);
|
|
1118
|
+
}
|
|
1119
|
+
catch {
|
|
1120
|
+
last = null;
|
|
1121
|
+
}
|
|
1122
|
+
return { journal: j, running, whatsNew, lastUpdate: last ? { from: last.from.version, to: last.to.version } : null };
|
|
741
1123
|
}
|
|
742
1124
|
export function markWhatsNewSeen(stateDir) {
|
|
743
1125
|
const j = readRuntimeUpdate(stateDir);
|
|
@@ -747,7 +1129,7 @@ export function markWhatsNewSeen(stateDir) {
|
|
|
747
1129
|
durableJson(journalFile(stateDir), j);
|
|
748
1130
|
durableJson(join(j.stageDir, "update.json"), j);
|
|
749
1131
|
}
|
|
750
|
-
const exec = (command, args, options = {}) => spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 300_000, maxBuffer: 16 * 1024 * 1024, ...(options.cwd ? { cwd: options.cwd } : {}) });
|
|
1132
|
+
const exec = (command, args, options = {}) => spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 300_000, maxBuffer: 16 * 1024 * 1024, ...(options.cwd ? { cwd: options.cwd } : {}), ...(options.input !== undefined ? { input: options.input } : {}) });
|
|
751
1133
|
const LABELS = ["com.toolroll.browser", "com.standing-orders.browser"];
|
|
752
1134
|
export function currentRuntime(version) {
|
|
753
1135
|
return { version, dist: dirname(fileURLToPath(import.meta.url)) };
|
|
@@ -767,6 +1149,11 @@ function processAlive(pid) {
|
|
|
767
1149
|
return error.code === "EPERM";
|
|
768
1150
|
}
|
|
769
1151
|
}
|
|
1152
|
+
/** The Sigstore verifier npm itself ships (npm install and npm audit signatures use it): no dependency of Toolroll's.
|
|
1153
|
+
* It checks the certificate chain, the transparency log entry and that the certificate names this identity. */
|
|
1154
|
+
const SIGSTORE_SCRIPT = `const input = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
|
|
1155
|
+
require(input.verifier).verify(input.bundle, { certificateIssuer: input.issuer, certificateIdentityURI: input.identity })
|
|
1156
|
+
.then(() => process.exit(0), error => { process.stderr.write(String(error && error.message || error)); process.exit(1); });`;
|
|
770
1157
|
export function machineSystem(home = homedir(), env = process.env, seams = {}) {
|
|
771
1158
|
const sh = seams.exec ?? exec;
|
|
772
1159
|
const supervise = seams.run ?? (async (file, args, options) => (await import("./exec.js")).run(file, args, options));
|
|
@@ -806,7 +1193,7 @@ export function machineSystem(home = homedir(), env = process.env, seams = {}) {
|
|
|
806
1193
|
// tarball is not checked), and the lockfile records exactly which bytes were installed.
|
|
807
1194
|
writeFileSync(join(runtime, "package.json"), JSON.stringify({ name: `${NAME}-runtime`, private: true, dependencies: { [NAME]: release.version } }, null, 2));
|
|
808
1195
|
const registry = `--registry=${REGISTRY}/`;
|
|
809
|
-
const installed = sh("npm", ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund", registry], { cwd: runtime });
|
|
1196
|
+
const installed = sh("npm", ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund", "--no-color", registry], { cwd: runtime });
|
|
810
1197
|
if (installed.status !== 0)
|
|
811
1198
|
throw new Refusal(`npm could not install Toolroll ${release.version}: ${(installed.stderr || installed.stdout).trim().slice(-400)} Nothing was changed.`);
|
|
812
1199
|
let locked;
|
|
@@ -818,7 +1205,7 @@ export function machineSystem(home = homedir(), env = process.env, seams = {}) {
|
|
|
818
1205
|
}
|
|
819
1206
|
if (locked?.version !== release.version || locked.integrity !== release.integrity || !String(locked.resolved ?? "").startsWith(`${REGISTRY}/`))
|
|
820
1207
|
throw new Refusal(`npm installed different bytes than the verified Toolroll ${release.version}. Nothing was changed.`);
|
|
821
|
-
const signatures = sh("npm", ["audit", "signatures", registry], { cwd: runtime });
|
|
1208
|
+
const signatures = sh("npm", ["audit", "signatures", "--no-color", registry], { cwd: runtime });
|
|
822
1209
|
const output = `${signatures.stdout}\n${signatures.stderr}`;
|
|
823
1210
|
if (signatures.status !== 0)
|
|
824
1211
|
throw new Refusal(`npm could not verify the package signatures: ${output.trim().slice(-400)} Nothing was changed.`);
|
|
@@ -846,16 +1233,49 @@ export function machineSystem(home = homedir(), env = process.env, seams = {}) {
|
|
|
846
1233
|
}
|
|
847
1234
|
return [...found];
|
|
848
1235
|
},
|
|
1236
|
+
sigstore: async (bundle, identity) => {
|
|
1237
|
+
const verifier = findSigstoreVerifier({
|
|
1238
|
+
path: env["PATH"] ?? "", execPath: seams.execPath ?? process.execPath,
|
|
1239
|
+
npmRoot: () => { const root = sh("npm", ["root", "--global", "--no-color"], { timeout: 30_000 }); return root.status === 0 ? root.stdout.trim() : null; },
|
|
1240
|
+
});
|
|
1241
|
+
if (!verifier)
|
|
1242
|
+
throw new Refusal("Toolroll could not find the Sigstore verifier that npm ships (npm 10 or newer), so its provenance cannot be checked. Nothing was changed.");
|
|
1243
|
+
const checked = sh(process.execPath, ["--no-warnings", "-e", SIGSTORE_SCRIPT], { input: JSON.stringify({ verifier, bundle, ...identity }), timeout: 120_000 });
|
|
1244
|
+
if (checked.status !== 0)
|
|
1245
|
+
throw new Refusal(`Sigstore did not verify Toolroll's provenance for ${identity.identity}: ${(checked.stderr || checked.stdout).trim().slice(-400)} Nothing was changed.`);
|
|
1246
|
+
},
|
|
849
1247
|
serviceUnit: from => unitFor(from),
|
|
1248
|
+
watchUnits: from => {
|
|
1249
|
+
const dir = join(home, "Library", "LaunchAgents");
|
|
1250
|
+
let names = [];
|
|
1251
|
+
try {
|
|
1252
|
+
names = readdirSync(dir);
|
|
1253
|
+
}
|
|
1254
|
+
catch {
|
|
1255
|
+
return [];
|
|
1256
|
+
}
|
|
1257
|
+
return names.filter(name => /^com\.(?:toolroll|standing-orders)\.watch\..+\.plist$/.test(name)).map(name => join(dir, name))
|
|
1258
|
+
.filter(unit => { try {
|
|
1259
|
+
return readFileSync(unit, "utf8").includes(from.dist);
|
|
1260
|
+
}
|
|
1261
|
+
catch {
|
|
1262
|
+
return false;
|
|
1263
|
+
} });
|
|
1264
|
+
},
|
|
1265
|
+
serviceLoaded: async (unit) => (await supervise("launchctl", ["print", `${domain}/${labelOf(unit)}`])).code === 0,
|
|
850
1266
|
servicePids: async (unit) => {
|
|
851
1267
|
const printed = await supervise("launchctl", ["print", `${domain}/${labelOf(unit)}`]);
|
|
852
1268
|
const pid = Number(printed.stdout.match(/\n\s*pid = (\d+)/)?.[1]);
|
|
853
1269
|
return printed.code === 0 && pid > 1 ? [pid] : [];
|
|
854
1270
|
},
|
|
855
1271
|
stopService: async (unit) => {
|
|
856
|
-
|
|
857
|
-
|
|
1272
|
+
// As long as a watch's toolroll up takes to exit.
|
|
1273
|
+
const stopped = await stopLaunchdService(definition(unit), supervise, { waitMs: UP_EXIT_MS, ...(seams.sleep ? { sleep: seams.sleep } : {}) });
|
|
1274
|
+
if (!stopped.ok) {
|
|
1275
|
+
// The stop disabled the label: enabled again before anything kickstarts it, or it could never start.
|
|
1276
|
+
await supervise("launchctl", ["enable", `${domain}/${labelOf(unit)}`]);
|
|
858
1277
|
throw Error(`${stopped.message}. Nothing was replaced.`);
|
|
1278
|
+
}
|
|
859
1279
|
},
|
|
860
1280
|
restartService: async (unit) => {
|
|
861
1281
|
const started = await installLaunchdService(definition(unit), supervise);
|
|
@@ -893,12 +1313,51 @@ export function machineSystem(home = homedir(), env = process.env, seams = {}) {
|
|
|
893
1313
|
},
|
|
894
1314
|
};
|
|
895
1315
|
}
|
|
1316
|
+
/** npm's own Sigstore verifier. Found from npm itself (the `npm` on PATH, then the one beside node), never only from
|
|
1317
|
+
* `npm root --global`: with a custom global prefix (npm's advice for permission errors) that folder holds the global
|
|
1318
|
+
* packages, not npm. */
|
|
1319
|
+
export function findSigstoreVerifier(o) {
|
|
1320
|
+
const npmPackage = (start) => {
|
|
1321
|
+
for (let dir = dirname(start); dir !== dirname(dir); dir = dirname(dir)) {
|
|
1322
|
+
try {
|
|
1323
|
+
if (JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).name === "npm")
|
|
1324
|
+
return dir;
|
|
1325
|
+
}
|
|
1326
|
+
catch { /* not the package folder yet */ }
|
|
1327
|
+
}
|
|
1328
|
+
return null;
|
|
1329
|
+
};
|
|
1330
|
+
const candidates = [
|
|
1331
|
+
...o.path.split(":").filter(dir => isAbsolute(dir)).map(dir => () => { try {
|
|
1332
|
+
return npmPackage(realpathSync(join(dir, "npm")));
|
|
1333
|
+
}
|
|
1334
|
+
catch {
|
|
1335
|
+
return null;
|
|
1336
|
+
} }),
|
|
1337
|
+
() => join(dirname(o.execPath), "..", "lib", "node_modules", "npm"),
|
|
1338
|
+
() => { const root = o.npmRoot(); return root && isAbsolute(root) ? join(root, "npm") : null; },
|
|
1339
|
+
];
|
|
1340
|
+
for (const candidate of candidates) {
|
|
1341
|
+
const npm = candidate();
|
|
1342
|
+
const verifier = npm ? join(npm, "node_modules", "sigstore") : null;
|
|
1343
|
+
if (verifier && existsSync(join(verifier, "package.json")))
|
|
1344
|
+
return resolve(verifier);
|
|
1345
|
+
}
|
|
1346
|
+
return null;
|
|
1347
|
+
}
|
|
896
1348
|
const jobUnit = (home) => join(home, "Library", "LaunchAgents", `${UPDATE_JOB_LABEL}.plist`);
|
|
897
1349
|
/** The console starts the updater for one prepared journal as its own one-off
|
|
898
1350
|
* launchd job, so the service it restarts is not its parent. The job resumes
|
|
899
1351
|
* that id only; it does not run at login (no RunAtLoad: launchd starts it
|
|
900
1352
|
* with a kickstart) and removes its definition when it finishes. Elsewhere it
|
|
901
1353
|
* is a detached process. */
|
|
1354
|
+
/** Where npm puts global commands (`npm prefix --global`/bin), or null when npm does not say. Asked without blocking:
|
|
1355
|
+
* the console's request handler starts the job. */
|
|
1356
|
+
async function npmPrefixBin(run) {
|
|
1357
|
+
const answered = await run("npm", ["prefix", "--global", "--no-color"], { timeoutMs: 30_000 }).catch(() => null);
|
|
1358
|
+
const prefix = answered?.code === 0 ? answered.stdout.trim() : "";
|
|
1359
|
+
return isAbsolute(prefix) ? join(prefix, "bin") : null;
|
|
1360
|
+
}
|
|
902
1361
|
export async function launchRuntimeUpdate(args, seams = {}) {
|
|
903
1362
|
const dist = args.dist ?? dirname(fileURLToPath(import.meta.url));
|
|
904
1363
|
const command = [process.execPath, join(dist, "bin.js"), "update", "--resume", "--id", args.id, "--db", args.databaseFile];
|
|
@@ -907,7 +1366,10 @@ export async function launchRuntimeUpdate(args, seams = {}) {
|
|
|
907
1366
|
if ((seams.platform ?? process.platform) === "darwin") {
|
|
908
1367
|
const home = seams.home ?? homedir();
|
|
909
1368
|
const run = seams.run ?? (await import("./exec.js")).run;
|
|
910
|
-
|
|
1369
|
+
// Everywhere a toolroll command is usually linked: the job switches every one it finds on this PATH.
|
|
1370
|
+
const npmBin = await (seams.npmBin ?? (() => npmPrefixBin(run)))();
|
|
1371
|
+
const pathEnv = [...new Set([dirname(process.execPath), ...(npmBin ? [npmBin] : []), join(home, ".local", "bin"), join(home, "bin"), join(home, "Library", "pnpm"), "/usr/local/bin", "/opt/homebrew/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin"])].join(":");
|
|
1372
|
+
const unit = launchdPlist({ label: UPDATE_JOB_LABEL, command, workingDirectory: stateDir, logPath: log, pathEnv, keepAlive: false, runAtLoad: false });
|
|
911
1373
|
const started = await installLaunchdService({ platform: "darwin", label: UPDATE_JOB_LABEL, bin: process.execPath, entry: join(dist, "bin.js"), logPath: log, unitPath: jobUnit(home), unitContent: unit }, run);
|
|
912
1374
|
if (!started.ok)
|
|
913
1375
|
throw Error(started.message);
|