toolroll 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +413 -0
- package/README.md +15 -0
- package/dist/agent-onboard.d.ts +106 -0
- package/dist/agent-onboard.js +282 -0
- package/dist/browser/workspace.css +2 -2
- package/dist/browser/workspace.js +23 -23
- package/dist/browser-shell.js +5 -3
- package/dist/browser-workspace.d.ts +11 -0
- package/dist/cli.d.ts +3 -1
- package/dist/cli.js +8 -1
- package/dist/control-setup.d.ts +4 -4
- package/dist/daemon.d.ts +4 -0
- package/dist/daemon.js +28 -4
- package/dist/desktop-host.js +2 -2
- package/dist/desktop-update.d.ts +8 -2
- package/dist/desktop-update.js +39 -22
- package/dist/evidence.d.ts +24 -0
- package/dist/evidence.js +103 -5
- package/dist/first-run.d.ts +54 -0
- package/dist/first-run.js +101 -0
- package/dist/install-method.d.ts +1 -1
- package/dist/install-method.js +4 -0
- package/dist/operate.d.ts +9 -0
- package/dist/operate.js +68 -4
- package/dist/releases.d.ts +9 -0
- package/dist/releases.js +14 -1
- package/dist/serve.d.ts +16 -0
- package/dist/serve.js +151 -82
- package/dist/skills.d.ts +2 -0
- package/dist/skills.js +3 -1
- package/dist/store.d.ts +3 -3
- package/dist/store.js +9 -4
- package/dist/surface.js +2 -0
- package/dist/toolroll-update-cli.d.ts +15 -0
- package/dist/toolroll-update-cli.js +155 -0
- package/dist/toolroll-update-ui.d.ts +38 -0
- package/dist/toolroll-update-ui.js +89 -0
- package/dist/toolroll-update.d.ts +235 -0
- package/dist/toolroll-update.js +929 -0
- package/package.json +2 -1
|
@@ -0,0 +1,929 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `toolroll update`: a verified, drained, undoable update of an npm-installed
|
|
3
|
+
* Toolroll. It reuses the desktop update engine's pieces (desktop-update.ts):
|
|
4
|
+
* the SQLite admission gate that lets running work finish, the verified
|
|
5
|
+
* private backup (the coding catalog's inside the same reservation), the
|
|
6
|
+
* durable journal written before every irreversible step, the OS lock that
|
|
7
|
+
* shows whether an updater is alive, and resume after a crash. What differs
|
|
8
|
+
* is what gets swapped: a staged npm runtime instead of an app bundle.
|
|
9
|
+
*
|
|
10
|
+
* verify → drain → stop and back up → rehearse → switch → restart → health
|
|
11
|
+
*
|
|
12
|
+
* The new release is installed from the npm registry into a Toolroll-managed
|
|
13
|
+
* runtimes folder (`staged-upgrades/`, as deploy-browser stages), never over
|
|
14
|
+
* the running one. npm's own signature and attestation check covers what it
|
|
15
|
+
* installed, the installed bytes must be the ones downloaded and hashed, and
|
|
16
|
+
* their provenance must name ap9000/toolroll and its publish workflow. The
|
|
17
|
+
* service is stopped, and its processes seen gone, before the final backup,
|
|
18
|
+
* so nothing written before the switch is lost. A failed health check puts
|
|
19
|
+
* back the previous runtime, database and coding catalog on its own, keeping
|
|
20
|
+
* what the new version wrote in a named copy. `--rollback` returns to the
|
|
21
|
+
* previous runtime and its backups. Every update, rollback, refusal and
|
|
22
|
+
* failure is a ledger entry.
|
|
23
|
+
*/
|
|
24
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
25
|
+
import { spawnSync } from "node:child_process";
|
|
26
|
+
import { chmodSync, closeSync, copyFileSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
27
|
+
import { homedir, userInfo } from "node:os";
|
|
28
|
+
import { basename, dirname, isAbsolute, join, resolve, sep } from "node:path";
|
|
29
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
30
|
+
import { createRequire } from "node:module";
|
|
31
|
+
import { durableJson, lingeringWork, sqliteLock, verifiedDatabaseBackup } from "./desktop-update.js";
|
|
32
|
+
import { activeUpdateWork, freezeUpdateGate, installUpdateGate, removeUpdateGate, updateAdmissionPaused, updateGateOwned } from "./desktop-update-gate.js";
|
|
33
|
+
import { assertCodingUpdateStopped, backupCodingCatalog, codingCatalogExists, removeCodingUpdateGate } from "./coding-update.js";
|
|
34
|
+
import { installLaunchdService, launchdPlist, stopLaunchdService, writeFileDurably } from "./daemon.js";
|
|
35
|
+
import { NAME } from "./names.js";
|
|
36
|
+
import { isNewer, REGISTRY } from "./releases.js";
|
|
37
|
+
/** Loaded on first use (as backup.ts and store.ts do), so modules that only import this one (the console,
|
|
38
|
+
* and tests that load it in a browser-like environment) never need `node:sqlite` itself. */
|
|
39
|
+
function sqlite() {
|
|
40
|
+
return createRequire(import.meta.url)("node:sqlite");
|
|
41
|
+
}
|
|
42
|
+
export const PROVENANCE_REPOSITORY = "https://github.com/ap9000/toolroll";
|
|
43
|
+
export const PROVENANCE_WORKFLOW = ".github/workflows/publish.yml";
|
|
44
|
+
/** The steps a person sees, in order. */
|
|
45
|
+
export const UPDATE_STEPS = ["verifying", "draining", "backing-up", "rehearsing", "switching", "restarting", "health"];
|
|
46
|
+
export const STEP_WORDS = {
|
|
47
|
+
verifying: "Verify the package", draining: "Let running work finish", "backing-up": "Stop and back up",
|
|
48
|
+
rehearsing: "Rehearse the migration", switching: "Switch to the new version", restarting: "Restart", health: "Health check",
|
|
49
|
+
};
|
|
50
|
+
/** How many release-* runtimes stay on disk: the running one and the one before it. */
|
|
51
|
+
export const KEEP_RUNTIMES = 2;
|
|
52
|
+
/** The one-off launchd job the console starts the updater as. */
|
|
53
|
+
export const UPDATE_JOB_LABEL = "com.toolroll.update";
|
|
54
|
+
const TERMINAL = ["complete", "restored", "refused", "cancelled"];
|
|
55
|
+
export const runtimeUpdateTerminal = (phase) => TERMINAL.includes(phase);
|
|
56
|
+
const journalFile = (stateDir) => join(stateDir, "toolroll-update.json");
|
|
57
|
+
const cancelFile = (j) => join(j.stageDir, "cancel-request.json");
|
|
58
|
+
const sha = (bytes, algorithm = "sha256") => createHash(algorithm).update(bytes).digest("hex");
|
|
59
|
+
const fileHash = (file) => sha(readFileSync(file));
|
|
60
|
+
const quote = (s) => '"' + s.replaceAll('"', '""') + '"';
|
|
61
|
+
const codingFile = (databaseFile) => `${databaseFile}.coding.sqlite`;
|
|
62
|
+
class Refusal extends Error {
|
|
63
|
+
}
|
|
64
|
+
export function readRuntimeUpdate(stateDir) {
|
|
65
|
+
const file = journalFile(stateDir);
|
|
66
|
+
if (!existsSync(file))
|
|
67
|
+
return null;
|
|
68
|
+
const stat = lstatSync(file);
|
|
69
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
70
|
+
throw Error("The saved update record is not a regular file. Nothing was changed.");
|
|
71
|
+
const j = JSON.parse(readFileSync(file, "utf8"));
|
|
72
|
+
const real = (path) => { try {
|
|
73
|
+
return realpathSync(path);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return resolve(path);
|
|
77
|
+
} };
|
|
78
|
+
if (j.version !== 1 || !/^[a-f0-9-]{36}$/.test(j.id) || typeof j.stateDir !== "string" || real(j.stateDir) !== real(stateDir) || !isAbsolute(j.stageDir ?? "") || real(dirname(j.stageDir)) !== real(join(stateDir, "staged-upgrades")))
|
|
79
|
+
throw Error("The saved update record is invalid. Preserve it and its backups; nothing was changed.");
|
|
80
|
+
return j;
|
|
81
|
+
}
|
|
82
|
+
function save(j, phase, detail, now) {
|
|
83
|
+
if (j.phase !== phase)
|
|
84
|
+
j.steps.push({ phase, at: now.toISOString() });
|
|
85
|
+
j.phase = phase;
|
|
86
|
+
j.detail = detail;
|
|
87
|
+
j.updatedAt = now.toISOString();
|
|
88
|
+
if (runtimeUpdateTerminal(phase))
|
|
89
|
+
j.finishedAt = now.toISOString();
|
|
90
|
+
durableJson(join(j.stageDir, "update.json"), j);
|
|
91
|
+
durableJson(journalFile(j.stateDir), j);
|
|
92
|
+
}
|
|
93
|
+
/** Every update, rollback, refusal and failure. Plain SQL any build can run,
|
|
94
|
+
* written after any database restore so the restore cannot erase it. */
|
|
95
|
+
function ledger(databaseFile, now, actor, action, outcome, detail) {
|
|
96
|
+
const db = new (sqlite().DatabaseSync)(databaseFile);
|
|
97
|
+
try {
|
|
98
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
99
|
+
db.prepare("INSERT INTO action_ledger(at,actor,repo,task_id,run_id,action,outcome,source,detail) VALUES (?,?,NULL,NULL,NULL,?,?,'policy',?)")
|
|
100
|
+
.run(now.toISOString(), actor, action, outcome, detail.replace(/[\u0000-\u001f\u007f]+/g, " ").slice(0, 300));
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
db.close();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/** Running work by name, for a refusal a person can act on. */
|
|
107
|
+
export function runningWorkWords(db) {
|
|
108
|
+
const active = activeUpdateWork(db);
|
|
109
|
+
if (Object.values(active).every(n => n === 0))
|
|
110
|
+
return null;
|
|
111
|
+
const runs = db.prepare("SELECT tr.external_id id, COALESCE(t.title, tr.external_id) title FROM run r JOIN task_ref tr ON tr.id = r.task_ref LEFT JOIN task t ON t.id = tr.external_id WHERE r.outcome IS NULL ORDER BY r.id LIMIT 3").all()
|
|
112
|
+
.map(row => `${String(row["id"])} (${String(row["title"]).slice(0, 60)})`);
|
|
113
|
+
const more = Number(active["runs"] ?? 0) - runs.length;
|
|
114
|
+
const other = Object.entries(active).filter(([key, n]) => key !== "runs" && n > 0).map(([key, n]) => `${n} ${key === "claims" ? "claimed task" : key === "conversations" ? "chat request" : key === "sessions" ? "held session" : key === "stopping" ? "run still stopping" : key === "codingDeliveries" ? "unconfirmed coding message" : "coding session"}${n === 1 ? "" : "s"}`);
|
|
115
|
+
const parts = [...(runs.length ? [`running ${runs.join(", ")}${more > 0 ? ` and ${more} more` : ""}`] : []), ...other];
|
|
116
|
+
return parts.join("; ");
|
|
117
|
+
}
|
|
118
|
+
/** The npm provenance statement for exactly these bytes must name the
|
|
119
|
+
* Toolroll repository and its publish workflow. Anything else is refused.
|
|
120
|
+
* npm's own check (UpdateSystem.install) verifies the statement's signature. */
|
|
121
|
+
export function checkProvenance(attestations, version, sha512Hex) {
|
|
122
|
+
const list = attestations?.attestations;
|
|
123
|
+
if (!Array.isArray(list) || list.length === 0)
|
|
124
|
+
throw new Refusal(`Toolroll ${version} has no npm provenance, so it cannot be verified. Nothing was changed.`);
|
|
125
|
+
for (const one of list) {
|
|
126
|
+
if (!String(one.predicateType ?? "").startsWith("https://slsa.dev/provenance/"))
|
|
127
|
+
continue;
|
|
128
|
+
let statement;
|
|
129
|
+
try {
|
|
130
|
+
statement = JSON.parse(Buffer.from(String(one.bundle?.dsseEnvelope?.payload ?? ""), "base64").toString("utf8"));
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
throw new Refusal(`Toolroll ${version}'s provenance could not be read. Nothing was changed.`);
|
|
134
|
+
}
|
|
135
|
+
const subject = statement.subject?.find(s => s.name === `pkg:npm/${NAME}@${version}`);
|
|
136
|
+
if (!subject || subject.digest?.sha512 !== sha512Hex)
|
|
137
|
+
throw new Refusal(`Toolroll ${version}'s provenance is for different bytes than the package downloaded. Nothing was changed.`);
|
|
138
|
+
const workflow = statement.predicate?.buildDefinition?.externalParameters?.workflow;
|
|
139
|
+
const repository = String(workflow?.repository ?? "").replace(/\.git$/, "");
|
|
140
|
+
if (repository !== PROVENANCE_REPOSITORY || workflow?.path !== PROVENANCE_WORKFLOW)
|
|
141
|
+
throw new Refusal(`Toolroll ${version} was built by ${repository || "an unnamed repository"} (${workflow?.path ?? "no workflow"}), not ${PROVENANCE_REPOSITORY.replace("https://github.com/", "")} (${PROVENANCE_WORKFLOW}). Nothing was changed.`);
|
|
142
|
+
return { repository, workflow: workflow.path };
|
|
143
|
+
}
|
|
144
|
+
throw new Refusal(`Toolroll ${version} has no build provenance statement. Nothing was changed.`);
|
|
145
|
+
}
|
|
146
|
+
/** Headlines of this version's changelog section: its bold lead phrases. */
|
|
147
|
+
export function releaseNotes(changelog, version) {
|
|
148
|
+
const lines = changelog.split("\n");
|
|
149
|
+
const start = lines.findIndex(line => new RegExp(`^## ${version.replaceAll(".", "\\.")}( |$)`).test(line));
|
|
150
|
+
if (start < 0)
|
|
151
|
+
return [];
|
|
152
|
+
const end = lines.findIndex((line, i) => i > start && line.startsWith("## "));
|
|
153
|
+
const section = lines.slice(start + 1, end < 0 ? undefined : end).join("\n");
|
|
154
|
+
return [...section.matchAll(/^- \*\*(.+?)\*\*/gm)].map(m => m[1].replace(/\.$/, "")).slice(0, 6);
|
|
155
|
+
}
|
|
156
|
+
// ---- durable files ----------------------------------------------------------
|
|
157
|
+
function fsyncPath(path) {
|
|
158
|
+
const fd = openSync(path, "r");
|
|
159
|
+
try {
|
|
160
|
+
fsyncSync(fd);
|
|
161
|
+
}
|
|
162
|
+
finally {
|
|
163
|
+
closeSync(fd);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/** Every rename here: what is renamed is flushed first (a file itself, a link
|
|
167
|
+
* by its directory), and the directory after, so a crash never leaves a torn
|
|
168
|
+
* database, catalog or command. */
|
|
169
|
+
export function durableRename(temp, target) {
|
|
170
|
+
fsyncPath(lstatSync(temp).isSymbolicLink() ? dirname(temp) : temp);
|
|
171
|
+
renameSync(temp, target);
|
|
172
|
+
if (process.platform !== "win32")
|
|
173
|
+
fsyncPath(dirname(target));
|
|
174
|
+
}
|
|
175
|
+
function tableDigest(db, name, columns) {
|
|
176
|
+
const rows = db.prepare("SELECT " + columns.map(quote).join(",") + " FROM " + quote(name)).all().map(r => JSON.stringify(r)).sort();
|
|
177
|
+
return { count: rows.length, hash: sha(rows.join("\n")) };
|
|
178
|
+
}
|
|
179
|
+
/** Every historical row, table by table (deploy-browser's rehearsal check). */
|
|
180
|
+
export function historySnapshot(db) {
|
|
181
|
+
return db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name <> 'schema_version' ORDER BY name").all()
|
|
182
|
+
.map(row => String(row["name"])).map(name => {
|
|
183
|
+
const columns = db.prepare("PRAGMA table_info(" + quote(name) + ")").all().map(r => String(r["name"]));
|
|
184
|
+
return { name, columns, ...tableDigest(db, name, columns) };
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
export function changedHistory(db, before) {
|
|
188
|
+
const tables = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(r => String(r["name"])));
|
|
189
|
+
return before.filter(t => {
|
|
190
|
+
if (!tables.has(t.name))
|
|
191
|
+
return true;
|
|
192
|
+
const present = new Set(db.prepare("PRAGMA table_info(" + quote(t.name) + ")").all().map(r => String(r["name"])));
|
|
193
|
+
if (t.columns.some(c => !present.has(c)))
|
|
194
|
+
return true;
|
|
195
|
+
const after = tableDigest(db, t.name, t.columns);
|
|
196
|
+
return after.count !== t.count || after.hash !== t.hash;
|
|
197
|
+
}).map(t => t.name);
|
|
198
|
+
}
|
|
199
|
+
async function rehearse(j, system, source) {
|
|
200
|
+
const copy = join(j.stageDir, `rehearsal.${randomUUID()}.db`);
|
|
201
|
+
copyFileSync(source, copy);
|
|
202
|
+
chmodSync(copy, 0o600);
|
|
203
|
+
try {
|
|
204
|
+
let db = new (sqlite().DatabaseSync)(copy, { readOnly: true });
|
|
205
|
+
const before = historySnapshot(db);
|
|
206
|
+
db.close();
|
|
207
|
+
await system.rehearse(j.to.dist, copy);
|
|
208
|
+
db = new (sqlite().DatabaseSync)(copy, { readOnly: true });
|
|
209
|
+
try {
|
|
210
|
+
const changed = changedHistory(db, before);
|
|
211
|
+
if (changed.length > 0)
|
|
212
|
+
throw new Refusal(`Toolroll ${j.to.version} would change saved history in ${changed.slice(0, 4).join(", ")}${changed.length > 4 ? ` and ${changed.length - 4} more` : ""}. Nothing was changed.`);
|
|
213
|
+
if (db.prepare("PRAGMA integrity_check").get()?.["integrity_check"] !== "ok")
|
|
214
|
+
throw new Refusal(`The rehearsed database failed its integrity check under ${j.to.version}. Nothing was changed.`);
|
|
215
|
+
}
|
|
216
|
+
finally {
|
|
217
|
+
db.close();
|
|
218
|
+
}
|
|
219
|
+
return { tables: before.length, rows: before.reduce((n, t) => n + t.count, 0) };
|
|
220
|
+
}
|
|
221
|
+
finally {
|
|
222
|
+
for (const suffix of ["", "-wal", "-shm"])
|
|
223
|
+
rmSync(copy + suffix, { force: true });
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/** Put a verified backup in place of a live SQLite file. Only once the service's processes are gone. */
|
|
227
|
+
function restoreFile(live, backup, hash, what) {
|
|
228
|
+
if (fileHash(backup) !== hash)
|
|
229
|
+
throw Error(`The retained ${what} backup changed. It was not put back.`);
|
|
230
|
+
const temp = `${live}.${randomUUID()}.restore`;
|
|
231
|
+
copyFileSync(backup, temp);
|
|
232
|
+
chmodSync(temp, 0o600);
|
|
233
|
+
for (const suffix of ["-wal", "-shm"])
|
|
234
|
+
rmSync(live + suffix, { force: true });
|
|
235
|
+
durableRename(temp, live);
|
|
236
|
+
}
|
|
237
|
+
/** The database and, when one was backed up, the coding catalog, then this run's coding gate lifted. */
|
|
238
|
+
function restoreDatabase(j, from) {
|
|
239
|
+
restoreFile(j.databaseFile, from.path, from.hash, "database");
|
|
240
|
+
if (from.codingPath && from.codingHash)
|
|
241
|
+
restoreFile(codingFile(j.databaseFile), from.codingPath, from.codingHash, "coding catalog");
|
|
242
|
+
ungateCoding(j);
|
|
243
|
+
}
|
|
244
|
+
/** A private copy of the live database before a restore replaces it: what the new version wrote is kept, and named. */
|
|
245
|
+
async function keepAside(j) {
|
|
246
|
+
const kept = join(j.stageDir, `orders.kept.${randomUUID().slice(0, 8)}.db`);
|
|
247
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile, { readOnly: true });
|
|
248
|
+
try {
|
|
249
|
+
await sqlite().backup(db, kept);
|
|
250
|
+
}
|
|
251
|
+
finally {
|
|
252
|
+
db.close();
|
|
253
|
+
}
|
|
254
|
+
chmodSync(kept, 0o600);
|
|
255
|
+
fsyncPath(kept);
|
|
256
|
+
return kept;
|
|
257
|
+
}
|
|
258
|
+
// ---- the service: stop, and see it gone -------------------------------------
|
|
259
|
+
/** Stop the service and wait until every process it ran has exited: only then may a file it writes be replaced. */
|
|
260
|
+
async function stopService(j, system, unit) {
|
|
261
|
+
const pids = [...new Set([...(j.service?.unit === unit ? j.service.pids : []), ...await system.servicePids(unit)])];
|
|
262
|
+
j.service = { unit, pids };
|
|
263
|
+
save(j, j.phase, j.detail, system.now());
|
|
264
|
+
await system.stopService(unit);
|
|
265
|
+
const deadline = system.now().getTime() + (system.exitTimeoutMs ?? 60_000);
|
|
266
|
+
for (;;) {
|
|
267
|
+
const alive = pids.filter(pid => system.processAlive(pid));
|
|
268
|
+
if (alive.length === 0)
|
|
269
|
+
return;
|
|
270
|
+
if (system.now().getTime() >= deadline)
|
|
271
|
+
throw Error(`The background service is still running (process ${alive.join(", ")}) after it was stopped. Nothing was replaced.`);
|
|
272
|
+
await system.sleep(250);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// ---- runtime switch: service definition and commands ----------------------
|
|
276
|
+
function pointLink(path, target) {
|
|
277
|
+
const temp = join(dirname(path), `.${basename(path)}.${randomUUID().slice(0, 8)}`);
|
|
278
|
+
symlinkSync(target, temp);
|
|
279
|
+
durableRename(temp, path);
|
|
280
|
+
}
|
|
281
|
+
/** What runs the current version: its service definition and every command on PATH. */
|
|
282
|
+
function switchable(j, system) {
|
|
283
|
+
const unit = system.serviceUnit(j.from), links = system.commands(j.from);
|
|
284
|
+
if (!unit && links.length === 0)
|
|
285
|
+
throw new Refusal(`No background service or toolroll command runs ${j.from.version} from ${j.from.dist}, so there is nothing to switch. Nothing was changed.`);
|
|
286
|
+
const real = (path) => { try {
|
|
287
|
+
return realpathSync(path);
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return null;
|
|
291
|
+
} };
|
|
292
|
+
const root = real(dirname(j.from.dist)) ?? dirname(j.from.dist);
|
|
293
|
+
for (const link of links) {
|
|
294
|
+
if (!lstatSync(link).isSymbolicLink())
|
|
295
|
+
throw new Refusal(`${link} is not a link Toolroll can switch (a shim or a copy), so it would keep running ${j.from.version}. Remove it or reinstall with npm, then update again. Nothing was changed.`);
|
|
296
|
+
const target = real(link);
|
|
297
|
+
if (target === null || !target.startsWith(root + sep))
|
|
298
|
+
throw new Refusal(`${link} runs a different Toolroll (${target ?? "a missing file"}), not ${j.from.version}, so it cannot be switched. Remove it, then update again. Nothing was changed.`);
|
|
299
|
+
}
|
|
300
|
+
if (unit && !readFileSync(unit, "utf8").includes(j.from.dist))
|
|
301
|
+
throw new Refusal("The service definition does not name the current runtime. Nothing was changed.");
|
|
302
|
+
return { unit, links };
|
|
303
|
+
}
|
|
304
|
+
/** Every change is recorded before it is made; `revert` undoes exactly those.
|
|
305
|
+
* A rollback puts the earlier database back only once the service is gone. */
|
|
306
|
+
async function switchRuntime(j, system) {
|
|
307
|
+
// A resumed switch keeps what it first recorded, so a revert still reaches the original.
|
|
308
|
+
if (!j.switched) {
|
|
309
|
+
const { unit, links } = switchable(j, system);
|
|
310
|
+
j.switched = { links: links.map(path => ({ path, previous: readlinkSync(path) })), unit: null };
|
|
311
|
+
if (unit) {
|
|
312
|
+
const saved = join(j.stageDir, "service.saved.plist");
|
|
313
|
+
writeFileDurably(saved, readFileSync(unit));
|
|
314
|
+
j.switched.unit = { path: unit, saved };
|
|
315
|
+
}
|
|
316
|
+
save(j, "switching", `Pointing ${unit ? "the service and " : ""}the toolroll and standing-orders commands at ${j.to.version}.`, system.now());
|
|
317
|
+
}
|
|
318
|
+
const unit = j.switched.unit?.path ?? null;
|
|
319
|
+
if (j.restoreFrom && !j.switched.databaseRestored) {
|
|
320
|
+
if (unit)
|
|
321
|
+
await stopService(j, system, unit);
|
|
322
|
+
restoreDatabase(j, j.restoreFrom);
|
|
323
|
+
gate(j);
|
|
324
|
+
j.switched.databaseRestored = true;
|
|
325
|
+
save(j, "switching", j.detail, system.now());
|
|
326
|
+
}
|
|
327
|
+
if (unit)
|
|
328
|
+
writeFileDurably(unit, readFileSync(j.switched.unit.saved, "utf8").replaceAll(j.from.dist, j.to.dist), 0o644);
|
|
329
|
+
for (const link of j.switched.links)
|
|
330
|
+
pointLink(link.path, join(j.to.dist, "bin.js"));
|
|
331
|
+
}
|
|
332
|
+
function revertSwitch(j) {
|
|
333
|
+
if (j.switched?.unit && existsSync(j.switched.unit.saved))
|
|
334
|
+
writeFileDurably(j.switched.unit.path, readFileSync(j.switched.unit.saved), 0o644);
|
|
335
|
+
for (const link of j.switched?.links ?? [])
|
|
336
|
+
pointLink(link.path, link.previous);
|
|
337
|
+
}
|
|
338
|
+
function gate(j) {
|
|
339
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile);
|
|
340
|
+
try {
|
|
341
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
342
|
+
installUpdateGate(db, j.id);
|
|
343
|
+
freezeUpdateGate(db, j.id);
|
|
344
|
+
}
|
|
345
|
+
finally {
|
|
346
|
+
db.close();
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
/** Lift this run's admission pause, in the database and the coding catalog. A restored database has no pause of
|
|
350
|
+
* its own (its backup dropped it), so the catalog's is lifted on its own. */
|
|
351
|
+
function ungate(j) {
|
|
352
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile);
|
|
353
|
+
try {
|
|
354
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
355
|
+
if (updateGateOwned(db, j.id))
|
|
356
|
+
removeUpdateGate(db, j.id);
|
|
357
|
+
else
|
|
358
|
+
removeCodingUpdateGate(db, j.id);
|
|
359
|
+
}
|
|
360
|
+
finally {
|
|
361
|
+
db.close();
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
function ungateCoding(j) {
|
|
365
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile);
|
|
366
|
+
try {
|
|
367
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
368
|
+
removeCodingUpdateGate(db, j.id);
|
|
369
|
+
}
|
|
370
|
+
finally {
|
|
371
|
+
db.close();
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
/** A foreground `toolroll up` holds a live watch lease; the runtime under it must not be replaced. */
|
|
375
|
+
function refuseWhileWatching(j, system) {
|
|
376
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile, { readOnly: true });
|
|
377
|
+
try {
|
|
378
|
+
const row = db.prepare("SELECT runner, repo FROM watch_lease WHERE expires_at > ? LIMIT 1").get(system.now().toISOString());
|
|
379
|
+
if (row)
|
|
380
|
+
throw new Refusal(`toolroll up is running for ${String(row["repo"])} (${String(row["runner"])}). Stop it first; ${j.kind === "update" ? "an update replaces the version it runs" : "a rollback replaces the database"}. Nothing was changed.`);
|
|
381
|
+
}
|
|
382
|
+
finally {
|
|
383
|
+
db.close();
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
function nextAt(hhmm, now) {
|
|
387
|
+
const [h, m] = hhmm.split(":").map(Number);
|
|
388
|
+
const at = new Date(now);
|
|
389
|
+
at.setHours(h, m, 0, 0);
|
|
390
|
+
if (at <= now)
|
|
391
|
+
at.setDate(at.getDate() + 1);
|
|
392
|
+
return at;
|
|
393
|
+
}
|
|
394
|
+
function newJournal(kind, o, now) {
|
|
395
|
+
const id = randomUUID(), stageDir = join(resolve(o.stateDir), "staged-upgrades", `${kind === "update" ? "release" : "rollback"}-${o.to.version}-${id.slice(0, 8)}`);
|
|
396
|
+
mkdirSync(stageDir, { recursive: true, mode: 0o700 });
|
|
397
|
+
chmodSync(stageDir, 0o700);
|
|
398
|
+
return { version: 1, id, kind, stateDir: resolve(o.stateDir), databaseFile: o.databaseFile, stageDir, from: o.from, to: o.to, when: o.when, at: o.at, actor: o.actor, phase: "scheduled", detail: "", steps: [], startedAt: now.toISOString(), updatedAt: now.toISOString() };
|
|
399
|
+
}
|
|
400
|
+
function assertNoneActive(stateDir) {
|
|
401
|
+
const existing = readRuntimeUpdate(stateDir);
|
|
402
|
+
if (existing && !runtimeUpdateTerminal(existing.phase))
|
|
403
|
+
throw new Refusal(`An update to ${existing.to.version} is already ${existing.phase === "scheduled" ? "scheduled" : "under way"} (${existing.detail}) Cancel it with toolroll update --cancel, or resume it with toolroll update --resume.`);
|
|
404
|
+
}
|
|
405
|
+
/** Record an update without running it: the console's job resumes exactly this journal id. */
|
|
406
|
+
export function prepareRuntimeUpdate(o, now) {
|
|
407
|
+
if (!/^\d+\.\d+\.\d+$/.test(o.version))
|
|
408
|
+
return { refused: `${o.version} is not a release version (x.y.z).` };
|
|
409
|
+
if (!o.allowDowngrade && !isNewer(o.version, o.current.version))
|
|
410
|
+
return { refused: `Toolroll ${o.version} is not newer than ${o.current.version}. To go back to an older release, run toolroll update --version ${o.version} --allow-downgrade.` };
|
|
411
|
+
try {
|
|
412
|
+
assertNoneActive(o.stateDir);
|
|
413
|
+
}
|
|
414
|
+
catch (error) {
|
|
415
|
+
return { refused: error.message };
|
|
416
|
+
}
|
|
417
|
+
const at = o.when === "at" ? nextAt(o.at ?? "03:00", now).toISOString() : null;
|
|
418
|
+
const j = newJournal("update", { stateDir: o.stateDir, databaseFile: o.databaseFile, from: o.current, to: { version: o.version, dist: "" }, when: o.when, at, actor: o.actor }, now);
|
|
419
|
+
save(j, "scheduled", at ? `Scheduled for ${at}.` : "Starting.", now);
|
|
420
|
+
if (at)
|
|
421
|
+
ledger(j.databaseFile, now, j.actor, "toolroll update scheduled", "scheduled", `${j.from.version} → ${j.to.version} at ${at}`);
|
|
422
|
+
return j;
|
|
423
|
+
}
|
|
424
|
+
/** Start an update: stage, verify, then run the journaled steps. */
|
|
425
|
+
export async function startRuntimeUpdate(o, system) {
|
|
426
|
+
if (o.version === o.current.version)
|
|
427
|
+
return { ok: true, phase: "complete", message: `Toolroll ${o.version} is current.`, journal: null };
|
|
428
|
+
const j = prepareRuntimeUpdate(o, system.now());
|
|
429
|
+
if ("refused" in j)
|
|
430
|
+
return { ok: false, phase: "refused", message: j.refused, journal: null };
|
|
431
|
+
return driveRuntimeUpdate(j, system);
|
|
432
|
+
}
|
|
433
|
+
/** Return to the runtime and database backup an update replaced. */
|
|
434
|
+
export async function startRuntimeRollback(o, system) {
|
|
435
|
+
let last;
|
|
436
|
+
try {
|
|
437
|
+
assertNoneActive(o.stateDir);
|
|
438
|
+
last = readRuntimeUpdate(o.stateDir);
|
|
439
|
+
}
|
|
440
|
+
catch (error) {
|
|
441
|
+
return { ok: false, phase: "refused", message: error.message, journal: null };
|
|
442
|
+
}
|
|
443
|
+
if (!last || last.kind !== "update" || last.phase !== "complete" || !last.backupPath || !last.backupHash)
|
|
444
|
+
return { ok: false, phase: "refused", message: "There is no completed update to roll back.", journal: null };
|
|
445
|
+
if (!existsSync(join(last.from.dist, "bin.js")) || !existsSync(last.backupPath) || (last.codingBackupPath && !existsSync(last.codingBackupPath)))
|
|
446
|
+
return { ok: false, phase: "refused", message: `The previous runtime (${last.from.version}) or its backup is no longer on this machine. Nothing was changed.`, journal: null };
|
|
447
|
+
const now = system.now();
|
|
448
|
+
const j = newJournal("rollback", { stateDir: o.stateDir, databaseFile: o.databaseFile, from: last.to, to: last.from, when: o.when, at: null, actor: o.actor }, now);
|
|
449
|
+
j.restoreFrom = { path: last.backupPath, hash: last.backupHash, updateId: last.id, ...(last.codingBackupPath && last.codingBackupHash ? { codingPath: last.codingBackupPath, codingHash: last.codingBackupHash } : {}) };
|
|
450
|
+
return driveRuntimeUpdate(j, system);
|
|
451
|
+
}
|
|
452
|
+
/** Continue a saved update after a crash, where it left off. With `id` (the console's job), only that update:
|
|
453
|
+
* a job never starts a fresh one, and a finished, replaced or rolled-back one is left alone. */
|
|
454
|
+
export async function resumeRuntimeUpdate(stateDir, system, id) {
|
|
455
|
+
const j = readRuntimeUpdate(stateDir);
|
|
456
|
+
if (id !== undefined && j?.id !== id)
|
|
457
|
+
return { ok: true, phase: j?.phase ?? "complete", message: "The update this job was started for is no longer the saved one. Nothing was changed.", journal: j };
|
|
458
|
+
if (!j || runtimeUpdateTerminal(j.phase))
|
|
459
|
+
return { ok: true, phase: j?.phase ?? "complete", message: "No update is in progress.", journal: j };
|
|
460
|
+
return driveRuntimeUpdate(j, system);
|
|
461
|
+
}
|
|
462
|
+
/** A prepared update whose job could not start. */
|
|
463
|
+
export function abandonRuntimeUpdate(stateDir, id, why, now) {
|
|
464
|
+
const j = readRuntimeUpdate(stateDir);
|
|
465
|
+
if (j?.id === id && j.phase === "scheduled")
|
|
466
|
+
save(j, "refused", why, now);
|
|
467
|
+
}
|
|
468
|
+
export function requestRuntimeUpdateCancel(stateDir) {
|
|
469
|
+
const j = readRuntimeUpdate(stateDir);
|
|
470
|
+
if (!j || runtimeUpdateTerminal(j.phase))
|
|
471
|
+
return "No update is in progress.";
|
|
472
|
+
if (!["scheduled", "verifying", "draining"].includes(j.phase))
|
|
473
|
+
return `The update to ${j.to.version} is past the point it can be cancelled (${j.phase}); it will finish or restore on its own.`;
|
|
474
|
+
durableJson(cancelFile(j), { id: j.id, action: "cancel" });
|
|
475
|
+
return `Cancelling the update to ${j.to.version}. Nothing is switched; new work resumes.`;
|
|
476
|
+
}
|
|
477
|
+
const cancelRequested = (j) => existsSync(cancelFile(j));
|
|
478
|
+
async function driveRuntimeUpdate(j, system) {
|
|
479
|
+
const lock = sqliteLock(join(j.stageDir, "worker.sqlite"), 1000);
|
|
480
|
+
if (!lock)
|
|
481
|
+
return { ok: false, phase: j.phase, message: `An updater is already working on the update to ${j.to.version}.`, journal: j };
|
|
482
|
+
const at = (phase) => UPDATE_STEPS.indexOf(phase);
|
|
483
|
+
const step = (phase, detail) => { save(j, phase, detail, system.now()); system.checkpoint?.(phase); };
|
|
484
|
+
const noun = j.kind === "update" ? "update" : "rollback";
|
|
485
|
+
const verb = j.kind === "update" ? `${j.from.version} → ${j.to.version}` : `${j.from.version} → ${j.to.version} (back)`;
|
|
486
|
+
const finish = (ok, phase, message, action, outcome, detail) => {
|
|
487
|
+
save(j, phase, message, system.now());
|
|
488
|
+
try {
|
|
489
|
+
ledger(j.databaseFile, system.now(), j.actor, action, outcome, detail);
|
|
490
|
+
}
|
|
491
|
+
catch { /* the journal still says what happened */ }
|
|
492
|
+
return { ok, phase, message, journal: j };
|
|
493
|
+
};
|
|
494
|
+
try {
|
|
495
|
+
// Scheduled: wait for the time, still cancellable.
|
|
496
|
+
if (j.phase === "scheduled" && j.at) {
|
|
497
|
+
while (system.now() < new Date(j.at)) {
|
|
498
|
+
if (cancelRequested(j))
|
|
499
|
+
return finish(true, "cancelled", `The ${noun} to ${j.to.version} was cancelled before it started. Nothing changed.`, `toolroll ${noun} cancelled`, "cancelled", verb);
|
|
500
|
+
await system.sleep(Math.min(60_000, new Date(j.at).getTime() - system.now().getTime()));
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
// Resumed past the switch: the only question left is whether the new runtime is healthy.
|
|
504
|
+
const resumedAt = at(j.phase);
|
|
505
|
+
if (j.phase === "rolling-back" || j.phase === "needs-attention")
|
|
506
|
+
return await restore(j, system, finish, verb, j.error ?? "An earlier attempt stopped.");
|
|
507
|
+
if (resumedAt < at("switching")) {
|
|
508
|
+
if (resumedAt <= at("verifying")) {
|
|
509
|
+
step("verifying", j.kind === "update" ? `Downloading and verifying Toolroll ${j.to.version}.` : `Checking the previous runtime (${j.to.version}) and its backup.`);
|
|
510
|
+
if (j.kind === "update")
|
|
511
|
+
await verify(j, system);
|
|
512
|
+
else if (fileHash(j.restoreFrom.path) !== j.restoreFrom.hash || (j.restoreFrom.codingPath && fileHash(j.restoreFrom.codingPath) !== j.restoreFrom.codingHash))
|
|
513
|
+
throw new Refusal("The update's backup changed since it was made. Nothing was changed.");
|
|
514
|
+
}
|
|
515
|
+
const unit = switchable(j, system).unit;
|
|
516
|
+
if (!unit)
|
|
517
|
+
refuseWhileWatching(j, system);
|
|
518
|
+
if (resumedAt <= at("draining")) {
|
|
519
|
+
step("draining", j.when === "now" ? "Checking that no work is running, then pausing new work." : "Pausing new work and letting running work finish. Nothing is being cancelled.");
|
|
520
|
+
await drain(j, system);
|
|
521
|
+
if (cancelRequested(j)) {
|
|
522
|
+
ungate(j);
|
|
523
|
+
return finish(true, "cancelled", `The ${noun} to ${j.to.version} was cancelled. Nothing changed; new work resumed.`, `toolroll ${noun} cancelled`, "cancelled", verb);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
// The service stops before the backup, so nothing it writes afterwards can be lost by a restore.
|
|
527
|
+
step("backing-up", unit ? "Stopping the background service, then backing up the database." : "Backing up the database.");
|
|
528
|
+
if (unit)
|
|
529
|
+
await stopService(j, system, unit);
|
|
530
|
+
refuseWhileWatching(j, system);
|
|
531
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile, { readOnly: true });
|
|
532
|
+
try {
|
|
533
|
+
assertCodingUpdateStopped(db);
|
|
534
|
+
}
|
|
535
|
+
finally {
|
|
536
|
+
db.close();
|
|
537
|
+
}
|
|
538
|
+
const backup = join(j.stageDir, existsSync(join(j.stageDir, "orders.backup.db")) ? `orders.backup.${randomUUID()}.db` : "orders.backup.db");
|
|
539
|
+
j.backupHash = await verifiedDatabaseBackup(j.databaseFile, backup, j.id, async () => { await codingBackup(j); });
|
|
540
|
+
j.backupPath = backup;
|
|
541
|
+
save(j, "backing-up", "Database and coding catalog backed up.", system.now());
|
|
542
|
+
step("rehearsing", `Rehearsing ${j.to.version} on a copy of the database.`);
|
|
543
|
+
j.rehearsal = await rehearse(j, system, j.restoreFrom?.path ?? j.backupPath);
|
|
544
|
+
}
|
|
545
|
+
if (resumedAt <= at("switching")) {
|
|
546
|
+
step("switching", `Switching to ${j.to.version}.`);
|
|
547
|
+
await switchRuntime(j, system);
|
|
548
|
+
}
|
|
549
|
+
if (resumedAt <= at("restarting")) {
|
|
550
|
+
const unit = j.switched?.unit?.path ?? null;
|
|
551
|
+
step("restarting", unit ? "Restarting the background service." : "No background service runs here; the commands now start the new version.");
|
|
552
|
+
if (unit)
|
|
553
|
+
await system.restartService(unit);
|
|
554
|
+
}
|
|
555
|
+
step("health", `Checking that ${j.to.version} is running and healthy.`);
|
|
556
|
+
const deadline = system.now().getTime() + (system.healthTimeoutMs ?? 90_000);
|
|
557
|
+
let healthy = false;
|
|
558
|
+
do {
|
|
559
|
+
healthy = await system.healthy(j);
|
|
560
|
+
if (!healthy)
|
|
561
|
+
await system.sleep(1000);
|
|
562
|
+
} while (!healthy && system.now().getTime() < deadline);
|
|
563
|
+
if (!healthy)
|
|
564
|
+
throw Error(`Toolroll ${j.to.version} did not pass its health check.`);
|
|
565
|
+
ungate(j);
|
|
566
|
+
const done = j.kind === "update"
|
|
567
|
+
? finish(true, "complete", `Toolroll ${j.to.version} is running. Your previous version (${j.from.version}) and its backup are kept; toolroll update --rollback returns to them.`, "toolroll updated", "complete", verb)
|
|
568
|
+
: finish(true, "complete", `Back on Toolroll ${j.to.version} with the database from before the update. The records made since are kept in ${basename(j.backupPath ?? "")}.`, "toolroll rolled back", "complete", verb);
|
|
569
|
+
try {
|
|
570
|
+
pruneRuntimes(j.stateDir, [j.to.dist, j.from.dist]);
|
|
571
|
+
}
|
|
572
|
+
catch { /* an old runtime left on disk is harmless */ }
|
|
573
|
+
return done;
|
|
574
|
+
}
|
|
575
|
+
catch (error) {
|
|
576
|
+
if (error.simulatedCrash)
|
|
577
|
+
throw error;
|
|
578
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
579
|
+
j.error = message.slice(0, 1500);
|
|
580
|
+
if (!j.switched) {
|
|
581
|
+
try {
|
|
582
|
+
ungate(j);
|
|
583
|
+
}
|
|
584
|
+
catch { /* no gate yet, or the database is unreadable */ }
|
|
585
|
+
// Nothing uses a refused download; the database backup is kept.
|
|
586
|
+
if (j.kind === "update")
|
|
587
|
+
rmSync(join(j.stageDir, "runtime"), { recursive: true, force: true });
|
|
588
|
+
// A service stopped for the backup starts again, unchanged.
|
|
589
|
+
let restarted = "";
|
|
590
|
+
if (j.service) {
|
|
591
|
+
try {
|
|
592
|
+
await system.restartService(j.service.unit);
|
|
593
|
+
}
|
|
594
|
+
catch (again) {
|
|
595
|
+
restarted = ` The background service did not start again: ${again.message}`;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
const refused = error instanceof Refusal;
|
|
599
|
+
return finish(false, "refused", (refused ? message : `The ${noun} stopped before anything was switched: ${message} Nothing changed; new work resumed.`) + restarted, refused ? `toolroll ${noun} refused` : `toolroll ${noun} failed`, refused ? "refused" : "failed", `${verb}: ${message}`);
|
|
600
|
+
}
|
|
601
|
+
return await restore(j, system, finish, verb, message);
|
|
602
|
+
}
|
|
603
|
+
finally {
|
|
604
|
+
lock.close();
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
/** The coding catalog, backed up inside the database's write reservation (as the desktop update does). */
|
|
608
|
+
async function codingBackup(j) {
|
|
609
|
+
const source = codingFile(j.databaseFile);
|
|
610
|
+
if (!codingCatalogExists(source))
|
|
611
|
+
return;
|
|
612
|
+
const target = join(j.stageDir, existsSync(join(j.stageDir, "coding.backup.sqlite")) ? `coding.backup.${randomUUID()}.sqlite` : "coding.backup.sqlite");
|
|
613
|
+
await backupCodingCatalog(source, target, j.id);
|
|
614
|
+
j.codingBackupPath = target;
|
|
615
|
+
j.codingBackupHash = fileHash(target);
|
|
616
|
+
}
|
|
617
|
+
/** Put back the previous runtime, then this run's own backups of the database and coding catalog, keeping what
|
|
618
|
+
* was written since in a named copy. */
|
|
619
|
+
async function restore(j, system, finish, verb, why) {
|
|
620
|
+
const noun = j.kind === "update" ? "update" : "rollback";
|
|
621
|
+
try {
|
|
622
|
+
save(j, "rolling-back", `${why} Restoring ${j.from.version} and the database backup.`, system.now());
|
|
623
|
+
system.checkpoint?.("rolling-back");
|
|
624
|
+
const unit = j.switched?.unit?.path ?? j.service?.unit ?? null;
|
|
625
|
+
if (unit)
|
|
626
|
+
await stopService(j, system, unit);
|
|
627
|
+
revertSwitch(j);
|
|
628
|
+
if (!j.backupPath || !j.backupHash)
|
|
629
|
+
throw Error("No verified backup was recorded.");
|
|
630
|
+
if (!j.restoredDatabase) {
|
|
631
|
+
// A fresh copy before EVERY attempt that is about to replace the database, not only the first: a retry
|
|
632
|
+
// after a failure earlier in the restore must not overwrite writes made since without keeping them.
|
|
633
|
+
j.keptAside = await keepAside(j);
|
|
634
|
+
save(j, "rolling-back", j.detail, system.now());
|
|
635
|
+
restoreDatabase(j, { path: j.backupPath, hash: j.backupHash, ...(j.codingBackupPath && j.codingBackupHash ? { codingPath: j.codingBackupPath, codingHash: j.codingBackupHash } : {}) });
|
|
636
|
+
j.restoredDatabase = true;
|
|
637
|
+
save(j, "rolling-back", j.detail, system.now());
|
|
638
|
+
}
|
|
639
|
+
if (unit)
|
|
640
|
+
await system.restartService(unit);
|
|
641
|
+
return finish(false, "restored", `${why} Toolroll ${j.from.version} and its database were restored. Anything written since the backup is kept in ${j.keptAside}.`, `toolroll ${noun} failed`, "restored", `${verb}: ${why}`.slice(0, 300));
|
|
642
|
+
}
|
|
643
|
+
catch (error) {
|
|
644
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
645
|
+
return finish(false, "needs-attention", `${why} Restoring the previous version also failed: ${message} Run toolroll update --resume to try again; the backup is kept at ${j.backupPath ?? "(none)"}.`, `toolroll ${noun} failed`, "needs attention", `${verb}: ${why} / ${message}`.slice(0, 300));
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
async function verify(j, system) {
|
|
649
|
+
const release = await system.release(j.to.version);
|
|
650
|
+
if (release.version !== j.to.version)
|
|
651
|
+
throw new Refusal(`The registry answered with ${release.version}, not ${j.to.version}. Nothing was changed.`);
|
|
652
|
+
if (!release.attestations)
|
|
653
|
+
throw new Refusal(`Toolroll ${j.to.version} has no npm provenance, so it cannot be verified. Nothing was changed.`);
|
|
654
|
+
const bytes = await system.download(release.tarball);
|
|
655
|
+
const sha512 = createHash("sha512").update(bytes).digest();
|
|
656
|
+
if (release.integrity !== `sha512-${sha512.toString("base64")}`)
|
|
657
|
+
throw new Refusal(`The downloaded package does not match the registry's checksum. Nothing was changed.`);
|
|
658
|
+
const provenance = checkProvenance(await system.attestations(release.attestations), j.to.version, sha512.toString("hex"));
|
|
659
|
+
const runtime = join(j.stageDir, "runtime");
|
|
660
|
+
mkdirSync(runtime, { recursive: true, mode: 0o700 });
|
|
661
|
+
j.to.dist = await system.install(runtime, release);
|
|
662
|
+
if (!j.to.dist.startsWith(runtime + sep) || !existsSync(join(j.to.dist, "bin.js")))
|
|
663
|
+
throw new Refusal("The staged runtime has no Toolroll command. Nothing was changed.");
|
|
664
|
+
j.package = { sha512: sha512.toString("hex"), ...provenance };
|
|
665
|
+
try {
|
|
666
|
+
j.notes = releaseNotes(readFileSync(join(dirname(j.to.dist), "CHANGELOG.md"), "utf8"), j.to.version);
|
|
667
|
+
}
|
|
668
|
+
catch {
|
|
669
|
+
j.notes = [];
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
async function drain(j, system) {
|
|
673
|
+
const db = new (sqlite().DatabaseSync)(j.databaseFile);
|
|
674
|
+
try {
|
|
675
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
676
|
+
if (j.when === "now") {
|
|
677
|
+
const running = runningWorkWords(db) ?? lingeringWork(db);
|
|
678
|
+
if (running)
|
|
679
|
+
throw new Refusal(`Work is running: ${running}. Nothing was changed. Use When idle to wait for it.`);
|
|
680
|
+
if (updateAdmissionPaused(db) && !updateGateOwned(db, j.id))
|
|
681
|
+
throw new Refusal("Another update owns the admission pause. Nothing was changed.");
|
|
682
|
+
installUpdateGate(db, j.id);
|
|
683
|
+
if (!freezeUpdateGate(db, j.id)) {
|
|
684
|
+
removeUpdateGate(db, j.id);
|
|
685
|
+
throw new Refusal(`Work started just now: ${runningWorkWords(db) ?? "a new run"}. Nothing was changed.`);
|
|
686
|
+
}
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
installUpdateGate(db, j.id);
|
|
690
|
+
for (;;) {
|
|
691
|
+
if (cancelRequested(j))
|
|
692
|
+
return;
|
|
693
|
+
const lingering = Object.values(activeUpdateWork(db)).every(n => n === 0) ? lingeringWork(db) : null;
|
|
694
|
+
if (!lingering && freezeUpdateGate(db, j.id))
|
|
695
|
+
return;
|
|
696
|
+
save(j, "draining", `New work is paused. Waiting for ${lingering ?? runningWorkWords(db) ?? "running work"} to finish. Nothing is being cancelled.`, system.now());
|
|
697
|
+
await system.sleep(2000);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
finally {
|
|
701
|
+
db.close();
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
/** Keep the newest release-* runtimes (and any in `keep`), at most KEEP_RUNTIMES; deploy-browser's browser-* and
|
|
705
|
+
* the rollback-* records are not this updater's to remove. */
|
|
706
|
+
export function pruneRuntimes(stateDir, keep) {
|
|
707
|
+
const root = join(stateDir, "staged-upgrades");
|
|
708
|
+
const started = (dir) => { try {
|
|
709
|
+
return String(JSON.parse(readFileSync(join(dir, "update.json"), "utf8")).startedAt ?? "");
|
|
710
|
+
}
|
|
711
|
+
catch {
|
|
712
|
+
return statSync(dir).mtime.toISOString();
|
|
713
|
+
} };
|
|
714
|
+
const releases = readdirSync(root, { withFileTypes: true }).filter(entry => entry.isDirectory() && entry.name.startsWith("release-"))
|
|
715
|
+
.map(entry => join(root, entry.name)).sort((a, b) => started(b).localeCompare(started(a)));
|
|
716
|
+
const kept = new Set(releases.filter(dir => keep.some(dist => dist !== "" && dist.startsWith(dir + sep))));
|
|
717
|
+
for (const dir of releases)
|
|
718
|
+
if (kept.size < KEEP_RUNTIMES)
|
|
719
|
+
kept.add(dir);
|
|
720
|
+
const removed = releases.filter(dir => !kept.has(dir));
|
|
721
|
+
for (const dir of removed)
|
|
722
|
+
rmSync(dir, { recursive: true, force: true });
|
|
723
|
+
return removed;
|
|
724
|
+
}
|
|
725
|
+
export function runtimeUpdateStatus(stateDir) {
|
|
726
|
+
let j = null;
|
|
727
|
+
try {
|
|
728
|
+
j = readRuntimeUpdate(stateDir);
|
|
729
|
+
}
|
|
730
|
+
catch {
|
|
731
|
+
j = null;
|
|
732
|
+
}
|
|
733
|
+
let running = false;
|
|
734
|
+
if (j && !runtimeUpdateTerminal(j.phase)) {
|
|
735
|
+
const lock = sqliteLock(join(j.stageDir, "worker.sqlite"));
|
|
736
|
+
running = lock === null;
|
|
737
|
+
lock?.close();
|
|
738
|
+
}
|
|
739
|
+
const whatsNew = j && j.kind === "update" && j.phase === "complete" && !j.seen ? { version: j.to.version, notes: j.notes ?? [] } : null;
|
|
740
|
+
return { journal: j, running, whatsNew };
|
|
741
|
+
}
|
|
742
|
+
export function markWhatsNewSeen(stateDir) {
|
|
743
|
+
const j = readRuntimeUpdate(stateDir);
|
|
744
|
+
if (!j || j.seen || j.phase !== "complete")
|
|
745
|
+
return;
|
|
746
|
+
j.seen = true;
|
|
747
|
+
durableJson(journalFile(stateDir), j);
|
|
748
|
+
durableJson(join(j.stageDir, "update.json"), j);
|
|
749
|
+
}
|
|
750
|
+
const exec = (command, args, options = {}) => spawnSync(command, args, { encoding: "utf8", timeout: options.timeout ?? 300_000, maxBuffer: 16 * 1024 * 1024, ...(options.cwd ? { cwd: options.cwd } : {}) });
|
|
751
|
+
const LABELS = ["com.toolroll.browser", "com.standing-orders.browser"];
|
|
752
|
+
export function currentRuntime(version) {
|
|
753
|
+
return { version, dist: dirname(fileURLToPath(import.meta.url)) };
|
|
754
|
+
}
|
|
755
|
+
async function json(url) {
|
|
756
|
+
const response = await fetch(url, { headers: { accept: "application/json" }, signal: AbortSignal.timeout(30_000) });
|
|
757
|
+
if (!response.ok)
|
|
758
|
+
throw new Refusal(`${url} answered ${response.status}. Nothing was changed.`);
|
|
759
|
+
return response.json();
|
|
760
|
+
}
|
|
761
|
+
function processAlive(pid) {
|
|
762
|
+
try {
|
|
763
|
+
process.kill(pid, 0);
|
|
764
|
+
return true;
|
|
765
|
+
}
|
|
766
|
+
catch (error) {
|
|
767
|
+
return error.code === "EPERM";
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
export function machineSystem(home = homedir(), env = process.env, seams = {}) {
|
|
771
|
+
const sh = seams.exec ?? exec;
|
|
772
|
+
const supervise = seams.run ?? (async (file, args, options) => (await import("./exec.js")).run(file, args, options));
|
|
773
|
+
const unitFor = (from) => {
|
|
774
|
+
for (const label of LABELS) {
|
|
775
|
+
const unit = join(home, "Library", "LaunchAgents", `${label}.plist`);
|
|
776
|
+
if (existsSync(unit) && readFileSync(unit, "utf8").includes(from.dist))
|
|
777
|
+
return unit;
|
|
778
|
+
}
|
|
779
|
+
return null;
|
|
780
|
+
};
|
|
781
|
+
const labelOf = (unit) => basename(unit, ".plist");
|
|
782
|
+
const domain = `gui/${typeof process.getuid === "function" ? process.getuid() : userInfo().uid}`;
|
|
783
|
+
const definition = (unit) => {
|
|
784
|
+
const content = readFileSync(unit, "utf8");
|
|
785
|
+
const logPath = content.match(/<key>StandardOutPath<\/key>\s*<string>([^<]+)<\/string>/)?.[1] ?? join(dirname(unit), `${labelOf(unit)}.log`);
|
|
786
|
+
return { platform: "darwin", label: labelOf(unit), unitPath: unit, unitContent: content, logPath, bin: process.execPath, entry: null };
|
|
787
|
+
};
|
|
788
|
+
return {
|
|
789
|
+
now: () => new Date(),
|
|
790
|
+
sleep: ms => new Promise(done => setTimeout(done, ms)),
|
|
791
|
+
release: async (version) => {
|
|
792
|
+
const body = await json(`${REGISTRY}/${NAME}/${version}`);
|
|
793
|
+
return { version: String(body.version ?? ""), tarball: String(body.dist?.tarball ?? ""), integrity: String(body.dist?.integrity ?? ""), attestations: body.dist?.attestations?.url ?? null };
|
|
794
|
+
},
|
|
795
|
+
download: async (url) => {
|
|
796
|
+
if (!url.startsWith(`${REGISTRY}/`))
|
|
797
|
+
throw new Refusal("The package is not hosted on the npm registry. Nothing was changed.");
|
|
798
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(120_000) });
|
|
799
|
+
if (!response.ok)
|
|
800
|
+
throw new Refusal(`The package download answered ${response.status}. Nothing was changed.`);
|
|
801
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
802
|
+
},
|
|
803
|
+
attestations: url => json(url),
|
|
804
|
+
install: async (runtime, release) => {
|
|
805
|
+
// From the registry by name, so npm's signature and attestation check covers the package itself (a local
|
|
806
|
+
// tarball is not checked), and the lockfile records exactly which bytes were installed.
|
|
807
|
+
writeFileSync(join(runtime, "package.json"), JSON.stringify({ name: `${NAME}-runtime`, private: true, dependencies: { [NAME]: release.version } }, null, 2));
|
|
808
|
+
const registry = `--registry=${REGISTRY}/`;
|
|
809
|
+
const installed = sh("npm", ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund", registry], { cwd: runtime });
|
|
810
|
+
if (installed.status !== 0)
|
|
811
|
+
throw new Refusal(`npm could not install Toolroll ${release.version}: ${(installed.stderr || installed.stdout).trim().slice(-400)} Nothing was changed.`);
|
|
812
|
+
let locked;
|
|
813
|
+
try {
|
|
814
|
+
locked = JSON.parse(readFileSync(join(runtime, "package-lock.json"), "utf8")).packages?.[`node_modules/${NAME}`];
|
|
815
|
+
}
|
|
816
|
+
catch {
|
|
817
|
+
locked = undefined;
|
|
818
|
+
}
|
|
819
|
+
if (locked?.version !== release.version || locked.integrity !== release.integrity || !String(locked.resolved ?? "").startsWith(`${REGISTRY}/`))
|
|
820
|
+
throw new Refusal(`npm installed different bytes than the verified Toolroll ${release.version}. Nothing was changed.`);
|
|
821
|
+
const signatures = sh("npm", ["audit", "signatures", registry], { cwd: runtime });
|
|
822
|
+
const output = `${signatures.stdout}\n${signatures.stderr}`;
|
|
823
|
+
if (signatures.status !== 0)
|
|
824
|
+
throw new Refusal(`npm could not verify the package signatures: ${output.trim().slice(-400)} Nothing was changed.`);
|
|
825
|
+
if (!/\b[1-9]\d* packages? ha(?:s|ve) (?:a )?verified attestations?\b/.test(output))
|
|
826
|
+
throw new Refusal(`npm did not verify Toolroll ${release.version}'s provenance attestation. Nothing was changed.`);
|
|
827
|
+
return join(runtime, "node_modules", NAME, "dist");
|
|
828
|
+
},
|
|
829
|
+
rehearse: async (dist, copy) => {
|
|
830
|
+
const script = `import(${JSON.stringify(pathToFileURL(join(dist, "store.js")).href)}).then(m => { m.openStore(${JSON.stringify(copy)}).close(); })`;
|
|
831
|
+
const done = sh(process.execPath, ["--no-warnings", "--input-type=module", "-e", script], { timeout: 600_000 });
|
|
832
|
+
if (done.status !== 0)
|
|
833
|
+
throw new Refusal(`The new version could not open a copy of the database: ${(done.stderr || "").trim().slice(-400)} Nothing was changed.`);
|
|
834
|
+
},
|
|
835
|
+
commands: () => {
|
|
836
|
+
const found = new Set();
|
|
837
|
+
for (const dir of (env["PATH"] ?? "").split(":").filter(Boolean)) {
|
|
838
|
+
for (const name of [NAME, "standing-orders"]) {
|
|
839
|
+
const path = join(dir, name);
|
|
840
|
+
try {
|
|
841
|
+
lstatSync(path);
|
|
842
|
+
found.add(path);
|
|
843
|
+
}
|
|
844
|
+
catch { /* not here */ }
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return [...found];
|
|
848
|
+
},
|
|
849
|
+
serviceUnit: from => unitFor(from),
|
|
850
|
+
servicePids: async (unit) => {
|
|
851
|
+
const printed = await supervise("launchctl", ["print", `${domain}/${labelOf(unit)}`]);
|
|
852
|
+
const pid = Number(printed.stdout.match(/\n\s*pid = (\d+)/)?.[1]);
|
|
853
|
+
return printed.code === 0 && pid > 1 ? [pid] : [];
|
|
854
|
+
},
|
|
855
|
+
stopService: async (unit) => {
|
|
856
|
+
const stopped = await stopLaunchdService(definition(unit), supervise);
|
|
857
|
+
if (!stopped.ok)
|
|
858
|
+
throw Error(`${stopped.message}. Nothing was replaced.`);
|
|
859
|
+
},
|
|
860
|
+
restartService: async (unit) => {
|
|
861
|
+
const started = await installLaunchdService(definition(unit), supervise);
|
|
862
|
+
if (!started.ok)
|
|
863
|
+
throw Error(`launchctl could not start the service: ${started.message}`);
|
|
864
|
+
},
|
|
865
|
+
processAlive: seams.alive ?? processAlive,
|
|
866
|
+
healthy: async (j) => {
|
|
867
|
+
const version = sh(process.execPath, [join(j.to.dist, "bin.js"), "--version"], { timeout: 30_000 });
|
|
868
|
+
if (version.status !== 0 || version.stdout.trim() !== j.to.version)
|
|
869
|
+
return false;
|
|
870
|
+
for (const link of j.switched?.links ?? []) {
|
|
871
|
+
try {
|
|
872
|
+
if (realpathSync(link.path) !== realpathSync(join(j.to.dist, "bin.js")))
|
|
873
|
+
return false;
|
|
874
|
+
}
|
|
875
|
+
catch {
|
|
876
|
+
return false;
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
const unit = j.switched?.unit?.path;
|
|
880
|
+
if (!unit)
|
|
881
|
+
return true;
|
|
882
|
+
const printed = await supervise("launchctl", ["print", `${domain}/${labelOf(unit)}`]);
|
|
883
|
+
const pid = Number(printed.stdout.match(/\n\s*pid = (\d+)/)?.[1]);
|
|
884
|
+
if (!(pid > 1) || !printed.stdout.includes("state = running"))
|
|
885
|
+
return false;
|
|
886
|
+
if (!sh("/bin/ps", ["-p", String(pid), "-o", "command="]).stdout.includes(j.to.dist))
|
|
887
|
+
return false;
|
|
888
|
+
const port = readFileSync(unit, "utf8").match(/<string>--port<\/string>\s*<string>(\d+)<\/string>/)?.[1];
|
|
889
|
+
if (!port)
|
|
890
|
+
return true;
|
|
891
|
+
const answered = await fetch(`http://127.0.0.1:${port}/login`, { redirect: "manual", signal: AbortSignal.timeout(5000) }).catch(() => null);
|
|
892
|
+
return answered !== null && answered.status < 500;
|
|
893
|
+
},
|
|
894
|
+
};
|
|
895
|
+
}
|
|
896
|
+
const jobUnit = (home) => join(home, "Library", "LaunchAgents", `${UPDATE_JOB_LABEL}.plist`);
|
|
897
|
+
/** The console starts the updater for one prepared journal as its own one-off
|
|
898
|
+
* launchd job, so the service it restarts is not its parent. The job resumes
|
|
899
|
+
* that id only; it does not run at login (no RunAtLoad: launchd starts it
|
|
900
|
+
* with a kickstart) and removes its definition when it finishes. Elsewhere it
|
|
901
|
+
* is a detached process. */
|
|
902
|
+
export async function launchRuntimeUpdate(args, seams = {}) {
|
|
903
|
+
const dist = args.dist ?? dirname(fileURLToPath(import.meta.url));
|
|
904
|
+
const command = [process.execPath, join(dist, "bin.js"), "update", "--resume", "--id", args.id, "--db", args.databaseFile];
|
|
905
|
+
const stateDir = dirname(args.databaseFile);
|
|
906
|
+
const log = join(stateDir, "toolroll-update.log");
|
|
907
|
+
if ((seams.platform ?? process.platform) === "darwin") {
|
|
908
|
+
const home = seams.home ?? homedir();
|
|
909
|
+
const run = seams.run ?? (await import("./exec.js")).run;
|
|
910
|
+
const unit = launchdPlist({ label: UPDATE_JOB_LABEL, command, workingDirectory: stateDir, logPath: log, pathEnv: [dirname(process.execPath), "/usr/local/bin", "/opt/homebrew/bin", "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(":"), keepAlive: false, runAtLoad: false });
|
|
911
|
+
const started = await installLaunchdService({ platform: "darwin", label: UPDATE_JOB_LABEL, bin: process.execPath, entry: join(dist, "bin.js"), logPath: log, unitPath: jobUnit(home), unitContent: unit }, run);
|
|
912
|
+
if (!started.ok)
|
|
913
|
+
throw Error(started.message);
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
const { spawn } = await import("node:child_process");
|
|
917
|
+
const out = openSync(log, "a", 0o600);
|
|
918
|
+
spawn(command[0], command.slice(1), { detached: true, stdio: ["ignore", out, out], cwd: stateDir }).unref();
|
|
919
|
+
}
|
|
920
|
+
/** The job's last act: its definition goes, so nothing can start it again. Only the definition for this id. */
|
|
921
|
+
export function retireUpdateJob(id, home = homedir()) {
|
|
922
|
+
const unit = jobUnit(home);
|
|
923
|
+
try {
|
|
924
|
+
if (readFileSync(unit, "utf8").includes(`<string>${id}</string>`))
|
|
925
|
+
rmSync(unit, { force: true });
|
|
926
|
+
}
|
|
927
|
+
catch { /* none */ }
|
|
928
|
+
}
|
|
929
|
+
export const nextScheduledAt = nextAt;
|