quoxcli 0.2.0 → 0.3.1

package/lib/server/status.js

@@ -0,0 +1,128 @@
+/**
+ * C4: `quox server status` and /server. Read-only: version, health per service, disk, ports, last update.
+ * Anything that could not be read is said as such, never filled in.
+ */
+import { readFileSync, writeFileSync, mkdirSync, appendFileSync, renameSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { readInstalled } from './install.js';
+
+const STATE_FILE = 'lifecycle-state.json';
+const HISTORY_FILE = 'history.jsonl';
+
+export function readState(root) { try { return JSON.parse(readFileSync(join(root, STATE_FILE), 'utf8')); } catch { return null; } }
+export function writeState(root, state) {
+  mkdirSync(root, { recursive: true });
+  const p = join(root, STATE_FILE); const tmp = `${p}.${process.pid}.tmp`;
+  writeFileSync(tmp, JSON.stringify(state, null, 2) + '\n', { mode: 0o600 });
+  renameSync(tmp, p);
+}
+export function appendHistory(root, event) {
+  mkdirSync(root, { recursive: true });
+  appendFileSync(join(root, HISTORY_FILE), JSON.stringify(event) + '\n', { mode: 0o600 });
+}
+export function readHistory(root) {
+  try { return readFileSync(join(root, HISTORY_FILE), 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean); } catch { return []; }
+}
+
+/** `docker compose ps --format json` is a JSON array on some versions and NDJSON on others. */
+export function parseComposePs(text) {
+  const t = String(text || '').trim();
+  if (!t) return [];
+  try { const j = JSON.parse(t); return Array.isArray(j) ? j : [j]; } catch { /* NDJSON */ }
+  return t.split('\n').map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
+}
+
+export const compose = (install, args) => ['compose', '-p', install.project, ...args];
+
+export async function composeServices(install, sys) {
+  const r = await sys.run('docker', compose(install, ['ps', '-a', '--format', 'json']), { cwd: install.repoDir });
+  if (r.code !== 0) return { ok: false, error: (r.stderr || '').trim().split('\n').pop() || `docker compose ps exited ${r.code}` };
+  return {
+    ok: true,
+    services: parseComposePs(r.stdout).map((s) => ({
+      service: s.Service || s.Name,
+      name: s.Name,
+      state: s.State || '?',
+      health: s.Health || null,
+      ports: [...new Set((s.Publishers || []).filter((p) => p.PublishedPort).map((p) => `${p.URL && p.URL !== '0.0.0.0' && p.URL !== '::' ? p.URL + ':' : ''}${p.PublishedPort}->${p.TargetPort}`))]
+    })).sort((a, b) => String(a.service).localeCompare(String(b.service)))
+  };
+}
+
+export async function diskFree(path, sys) {
+  const r = await sys.run('df', ['-Pk', path]);
+  if (r.code !== 0) return { ok: false, error: `df exited ${r.code}` };
+  const line = r.stdout.trim().split('\n').pop().split(/\s+/);
+  const availKb = Number(line[3]); const sizeKb = Number(line[1]);
+  if (!Number.isFinite(availKb)) return { ok: false, error: 'could not read df output' };
+  return { ok: true, path, freeGB: +(availKb / 1048576).toFixed(1), sizeGB: +(sizeKb / 1048576).toFixed(1), usedPct: line[4] };
+}
+
+async function getJson(url, fetchImpl, timeout = 3000) {
+  try {
+    const r = await fetchImpl(url, { signal: AbortSignal.timeout(timeout) });
+    if (!r.ok) return { ok: false, status: r.status, error: `HTTP ${r.status}` };
+    let body = null; try { body = await r.json(); } catch { /* health may be text */ }
+    return { ok: true, status: r.status, body };
+  } catch (e) { return { ok: false, error: e?.cause?.code || e?.message || 'no answer' }; }
+}
+
+/** The running build, from the collector's public /version. */
+export async function runningVersion(collectorUrl, fetchImpl = globalThis.fetch) {
+  const r = await getJson(`${collectorUrl.replace(/\/+$/, '')}/version`, fetchImpl);
+  if (!r.ok) return { ok: false, error: r.error };
+  return { ok: true, commit: r.body?.version || null, release: r.body?.release || null, stamped: r.body?.stamped ?? null };
+}
+
+export async function probeHttp(urls, fetchImpl = globalThis.fetch) {
+  return Promise.all(Object.entries(urls).map(async ([name, base]) => {
+    const t0 = Date.now();
+    const r = await getJson(`${base.replace(/\/+$/, '')}/health`, fetchImpl);
+    return { name, url: base, ok: r.ok, detail: r.ok ? `HTTP ${r.status}` : r.error, ms: Date.now() - t0 };
+  }));
+}
+
+export function defaultUrls(config) {
+  const get = (s, fb) => { try { return config?.getServiceUrl?.(s) || fb; } catch { return fb; } };
+  return { auth: get('auth', 'http://127.0.0.1:3101'), collector: get('collector', 'http://127.0.0.1:9848') };
+}
+
+/** -> the full status object. Never throws for a missing piece. */
+export async function gatherStatus({ install, sys, fetchImpl = globalThis.fetch, urls, backups }) {
+  const [installed, running, svc, disk, http] = await Promise.all([
+    readInstalled(install, sys),
+    runningVersion(urls.collector, fetchImpl),
+    composeServices(install, sys),
+    diskFree(install.root, sys),
+    probeHttp(urls, fetchImpl)
+  ]);
+  const hist = backups ? readHistory(backups) : [];
+  const lastUpdate = [...hist].reverse().find((e) => e.op === 'update' || e.op === 'rollback') || null;
+  const state = backups ? readState(backups) : null;
+  return { install, installed, running, services: svc, disk, http, lastUpdate, inFlight: state && state.status === 'running' ? state : null };
+}
+
+const svcIcon = (s) => (s.state === 'running' ? (s.health && s.health !== 'healthy' ? '⚠' : '✔') : '✖');
+
+/** Rows for the terminal and for /server (kind/key/text like the other panels). */
+export function statusRows(st) {
+  const row = (key, text) => ({ kind: 'row', key, text: String(text) });
+  const out = [{ kind: 'head', text: 'Server' }];
+  out.push(row('Install', `${st.install.repoDir} (found via ${st.install.source}, project ${st.install.project})`));
+  const i = st.installed;
+  out.push(row('Version', `${i.release || '? no VERSION file'}${i.tag ? ` (${i.tag})` : ''} · checkout ${i.commit || '?'}${i.dirty ? ' · local changes' : ''}`));
+  const rv = st.running;
+  out.push(row('Running', rv.ok ? `${rv.commit || '?'}${rv.stamped === false ? ' (unstamped build)' : ''}${rv.commit && i.commit && !String(rv.commit).startsWith(i.commit) && !String(i.commit).startsWith(rv.commit) ? ' · differs from the checkout' : ''}` : `? ${rv.error}`));
+  for (const h of st.http) out.push(row(`${h.ok ? '✔' : '✖'} ${h.name}`, `${h.url} · ${h.detail} · ${h.ms}ms`));
+  if (!st.services.ok) out.push(row('✖ Services', `? ${st.services.error}`));
+  else {
+    const up = st.services.services.filter((s) => s.state === 'running').length;
+    out.push(row('Services', `${up} of ${st.services.services.length} running`));
+    for (const s of st.services.services) out.push({ kind: s.state === 'running' && (!s.health || s.health === 'healthy') ? 'dim' : 'row', key: `  ${svcIcon(s)} ${s.service}`, text: [s.state + (s.health ? ` (${s.health})` : ''), s.ports.join(' ')].filter(Boolean).join(' · ') });
+  }
+  out.push(row('Disk', st.disk.ok ? `${st.disk.freeGB} GB free of ${st.disk.sizeGB} GB (${st.disk.usedPct} used) at ${st.disk.path}` : `? ${st.disk.error}`));
+  const lu = st.lastUpdate;
+  out.push(row('Last update', lu ? `${lu.op} ${lu.status} ${lu.at}${lu.from ? ` · ${lu.from} -> ${lu.to || '?'}` : ''}` : 'none recorded by quox server update'));
+  if (st.inFlight) out.push(row('⚠ In flight', `an update to ${st.inFlight.target} stopped after ${st.inFlight.done?.slice(-1)[0] || 'preflight'}; quox server update resumes it`));
+  return out;
+}

package/lib/server/sys.js

@@ -0,0 +1,65 @@
+/**
+ * C4 lifecycle: the one place that runs host commands (docker, docker compose, git, df, tar, openssl).
+ *
+ * Every lifecycle module takes a `sys` and never spawns on its own, so tests pass a stub and a dry run
+ * can prove it never reached a mutating command. A call marked { ro: true } is read-only and still runs
+ * in a dry run (it is how the plan learns what is there); anything else is recorded, never run, when
+ * dry. That rule lives here, once, in `gate()`.
+ */
+import { spawn as defaultSpawn } from 'node:child_process';
+import { createWriteStream, createReadStream } from 'node:fs';
+
+/** -> { run(cmd, args, opts) -> Promise<{ code, stdout, stderr }> } */
+export function createSys({ spawnImpl = defaultSpawn } = {}) {
+  return {
+    kind: 'real',
+    run(cmd, args = [], { cwd, env, stdoutFile, stdinFile, stdinText, timeoutMs = 0 } = {}) {
+      return new Promise((resolve) => {
+        let child;
+        try {
+          child = spawnImpl(cmd, args, { cwd, env: env ? { ...process.env, ...env } : process.env, stdio: [stdinFile || typeof stdinText === 'string' ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
+        } catch (e) { resolve({ code: 127, stdout: '', stderr: e.message }); return; }
+        let stdout = ''; let stderr = '';
+        const sink = stdoutFile ? createWriteStream(stdoutFile, { mode: 0o600 }) : null;
+        if (sink) child.stdout.pipe(sink); else child.stdout.on('data', (d) => { if (stdout.length < 4e6) stdout += d; });
+        child.stderr.on('data', (d) => { if (stderr.length < 1e6) stderr += d; });
+        if (stdinFile) createReadStream(stdinFile).pipe(child.stdin);
+        else if (typeof stdinText === 'string') child.stdin.end(stdinText); // secrets via stdin, never argv
+        const timer = timeoutMs ? setTimeout(() => { try { child.kill('SIGKILL'); } catch { /* gone */ } }, timeoutMs) : null;
+        child.on('error', (e) => { clearTimeout(timer); resolve({ code: 127, stdout, stderr: stderr || e.message }); });
+        child.on('close', (code, sig) => {
+          clearTimeout(timer);
+          const done = () => resolve({ code: sig ? 128 : code, stdout, stderr });
+          if (sink) sink.end(done); else done();
+        });
+      });
+    }
+  };
+}
+
+export const show = (cmd, args = []) => [cmd, ...args].map((a) => (/^[\w./:=@%+,-]+$/.test(a) ? a : `'${String(a).replace(/'/g, `'\\''`)}'`)).join(' ');
+
+/**
+ * A step runner over sys: ro calls always run; mutating calls run only when !dryRun, else they are
+ * recorded in `planned` and answered { code: 0, dry: true }. log(line) receives every command shown.
+ */
+export function gate(sys, { dryRun = false, log = () => {} } = {}) {
+  const planned = [];
+  return {
+    dryRun,
+    planned,
+    async ro(cmd, args, opts) { return sys.run(cmd, args, opts); },
+    async mut(cmd, args, opts = {}) {
+      const line = show(cmd, args) + (opts.stdoutFile ? ` > ${opts.stdoutFile}` : '') + (opts.stdinFile ? ` < ${opts.stdinFile}` : '');
+      if (dryRun) { planned.push(line); log(`[dry run] ${line}`); return { code: 0, stdout: '', stderr: '', dry: true }; }
+      log(`$ ${line}`);
+      return sys.run(cmd, args, opts);
+    },
+    /** A filesystem change (or any non-command effect): described, and only performed when !dryRun. */
+    async fx(desc, fn) {
+      if (dryRun) { planned.push(desc); log(`[dry run] ${desc}`); return undefined; }
+      log(desc);
+      return fn();
+    }
+  };
+}

package/lib/server/uninstall.js

@@ -0,0 +1,63 @@
+/**
+ * C4: `quox server uninstall`. Lists exactly what goes first; two-step confirmation (the command layer);
+ * keeps backups unless --purge, and takes a final snapshot first unless --no-backup or --purge.
+ */
+import { existsSync, readdirSync, rmSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { SIBLINGS } from './install.js';
+import { compose, composeServices } from './status.js';
+import { createSnapshot, listSnapshots, stackVolumes } from './snapshot.js';
+import { readInstalled } from './install.js';
+import { gate } from './sys.js';
+
+const DANGEROUS = (p, home) => {
+  const r = resolve(p);
+  return r === '/' || r === resolve(home) || r.split('/').filter(Boolean).length < 2;
+};
+
+/** Everything that will be removed and kept. Read-only. */
+export async function planUninstall({ install, sys, backups, purge = false, journeyFile = join(homedir(), '.config', 'quox', 'onboarding.json'), home = homedir() }) {
+  const svc = await composeServices(install, sys);
+  const vols = await stackVolumes(install, (c, a, o) => sys.run(c, a, o));
+  const imgs = await sys.run('docker', compose(install, ['config', '--images']), { cwd: install.repoDir });
+  const dirs = [install.repoDir, ...SIBLINGS.map((s) => join(install.root, s)).filter((d) => existsSync(d))];
+  const refused = dirs.filter((d) => DANGEROUS(d, home));
+  let rootToo = false;
+  try { rootToo = !DANGEROUS(install.root, home) && readdirSync(install.root).every((f) => dirs.includes(join(install.root, f))); } catch { /* unreadable */ }
+  const snaps = listSnapshots(backups);
+  const backupsInside = dirs.some((d) => resolve(backups).startsWith(resolve(d) + '/')) || (rootToo && resolve(backups).startsWith(resolve(install.root) + '/'));
+  return {
+    backupsInside,
+    containers: svc.ok ? svc.services.map((s) => s.name) : [],
+    containersError: svc.ok ? null : svc.error,
+    volumes: vols.volumes,
+    images: imgs.code === 0 ? [...new Set(imgs.stdout.split('\n').map((s) => s.trim()).filter(Boolean))] : [],
+    networks: `the ${install.project} compose networks`,
+    dirs: dirs.filter((d) => !refused.includes(d)),
+    root: rootToo ? install.root : null,
+    refused,
+    journeyFile: existsSync(journeyFile) ? journeyFile : null,
+    backups: { dir: backups, count: snaps.length, removed: purge }
+  };
+}
+
+/** Executes the plan. Confirmation is the caller's job and must have happened already. */
+export async function runUninstall({ install, sys, backups, purge = false, noBackup = false, dryRun = false, log = () => {}, plan, now }) {
+  const g = gate(sys, { dryRun, log });
+  if (plan.backupsInside && !purge) return { ok: false, step: 'plan', error: `the backups directory ${backups} sits inside the install, so removing the install would delete them. Set QUOX_SERVER_BACKUPS to a path outside it, or pass --purge.` };
+  let finalSnapshot = null;
+  if (!noBackup && !purge) {
+    const installed = await readInstalled(install, sys);
+    const snap = await createSnapshot({ install, installed, backups, g, reason: 'before uninstall', now, log });
+    if (!snap.ok) return { ok: false, step: 'backup', error: `${snap.error}. Nothing was removed; pass --no-backup to uninstall without a final snapshot.` };
+    finalSnapshot = snap.dir;
+  }
+  const r = await g.mut('docker', compose(install, ['down', '--volumes', '--rmi', 'local', '--remove-orphans']), { cwd: install.repoDir });
+  if (r.code !== 0) return { ok: false, step: 'down', error: `docker compose down failed (exit ${r.code}); the install directory was left in place` };
+  for (const d of plan.dirs) await g.fx(`remove ${d}`, () => rmSync(d, { recursive: true, force: true }));
+  if (plan.root) await g.fx(`remove ${plan.root} (now empty)`, () => rmSync(plan.root, { recursive: true, force: true }));
+  if (plan.journeyFile) await g.fx(`remove ${plan.journeyFile}`, () => rmSync(plan.journeyFile, { force: true }));
+  if (purge) await g.fx(`remove every backup in ${backups}`, () => rmSync(backups, { recursive: true, force: true }));
+  return { ok: true, dryRun, finalSnapshot, planned: g.planned };
+}

package/lib/server/update.js

@@ -0,0 +1,234 @@
+/**
+ * C4: `quox server update` and `quox server rollback`.
+ *
+ * update:  preflight -> backup -> fetch -> build -> migrate -> health, with the state written after every
+ *          step (resumable: a re-run continues after the last completed step). A failure from fetch on
+ *          rolls back automatically: before `migrate` only the code moves back (data was never touched);
+ *          from `migrate` on, the full rollback restores the pre-update snapshot.
+ * rollback: previous snapshot + previous version: checkout, build, restore DB + config, start, health.
+ *
+ * Migrations run on service boot (each service applies its own on start), so `migrate` = `up -d`, and
+ * `health` proves they finished: every service answers and the collector's /version is the new commit.
+ */
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { readInstalled, readEnvKeys, fetchAvailable, cmpSemver, parseSemver, vtag, compareUrl, releaseNotesUrl, PUBLIC_REPO, SIBLINGS } from './install.js';
+import { readState, writeState, appendHistory, compose, diskFree, runningVersion, probeHttp } from './status.js';
+import { createSnapshot, restoreSnapshot, latestSnapshot, listSnapshots } from './snapshot.js';
+import { gate } from './sys.js';
+
+export const UPDATE_STEPS = ['preflight', 'backup', 'fetch', 'build', 'migrate', 'health'];
+export const MIN_FREE_GB = 5;
+
+const sleepDefault = (ms) => new Promise((r) => setTimeout(r, ms));
+const lastLine = (s) => String(s || '').trim().split('\n').pop();
+
+/** Poll auth + collector /health, and (when known) the collector /version == expectCommit. */
+export async function waitHealthy({ urls, fetchImpl, expectCommit = null, tries = 40, intervalMs = 3000, sleep = sleepDefault, log = () => {} }) {
+  let last = '';
+  for (let i = 0; i < Math.max(1, tries); i++) {
+    const http = await probeHttp(urls, fetchImpl);
+    const bad = http.filter((h) => !h.ok);
+    if (!bad.length) {
+      if (!expectCommit) return { ok: true };
+      const v = await runningVersion(urls.collector, fetchImpl);
+      if (v.ok && v.commit && (String(v.commit).startsWith(expectCommit) || String(expectCommit).startsWith(v.commit))) return { ok: true, commit: v.commit };
+      last = v.ok ? `collector reports ${v.commit}, expected ${expectCommit}` : `collector /version: ${v.error}`;
+    } else last = bad.map((h) => `${h.name}: ${h.detail}`).join(', ');
+    log(`waiting for health: ${last}`);
+    if (i + 1 < tries) await sleep(intervalMs);
+  }
+  return { ok: false, error: last || 'not healthy' };
+}
+
+async function checkoutCode({ install, g, ref, siblings = {}, pullSiblings = false, log }) {
+  const r = await g.mut('git', ['-C', install.repoDir, 'checkout', '--detach', ref]);
+  if (r.code !== 0) return { ok: false, error: `git checkout ${ref} failed: ${lastLine(r.stderr)}` };
+  for (const s of SIBLINGS) {
+    const d = join(install.root, s);
+    if (!existsSync(join(d, '.git'))) continue;
+    const sr = pullSiblings ? await g.mut('git', ['-C', d, 'pull', '--ff-only'])
+      : siblings[s] ? await g.mut('git', ['-C', d, 'checkout', '--detach', siblings[s]]) : { code: 0 };
+    if (sr.code !== 0) log(`warning: ${s} did not move (${lastLine(sr.stderr) || `exit ${sr.code}`}); it stays where it was`);
+  }
+  return { ok: true };
+}
+
+async function composeBuild({ install, g, commit }) {
+  const gid = readEnvKeys(install.repoDir, ['DOCKER_GID']).DOCKER_GID;
+  const r = await g.mut('docker', compose(install, ['build']), { cwd: install.repoDir, env: { QUOXCORE_VERSION: commit, ...(gid ? { DOCKER_GID: gid } : {}) } });
+  return r.code === 0 ? { ok: true } : { ok: false, error: `docker compose build failed (exit ${r.code}): ${lastLine(r.stderr)}` };
+}
+
+async function composeUp({ install, g }) {
+  const r = await g.mut('docker', compose(install, ['up', '-d', '--remove-orphans']), { cwd: install.repoDir });
+  return r.code === 0 ? { ok: true } : { ok: false, error: `docker compose up -d failed (exit ${r.code}): ${lastLine(r.stderr)}` };
+}
+
+/**
+ * Plan + run an update. opts: { install, sys, backups, urls, fetchImpl, dryRun, to, force, restart, noRollback,
+ * log, now, healthTries, healthIntervalMs, sleep, repo }
+ * -> { ok, upToDate?, dryRun?, plan, planned?, failedStep?, error?, rolledBack?, rollback? }
+ */
+export async function runUpdate(opts) {
+  const { install, sys, backups, urls, fetchImpl = globalThis.fetch, dryRun = false, force = false, restart = false, noRollback = false,
+    log = () => {}, now = () => new Date(), healthTries = 40, healthIntervalMs = 3000, sleep = sleepDefault, repo = PUBLIC_REPO } = opts;
+  const g = gate(sys, { dryRun, log });
+
+  // ---- preflight (read-only) ----------------------------------------------------------------------
+  const installed = await readInstalled(install, sys);
+  let target = opts.to ? vtag(opts.to) : null;
+  if (!target) {
+    const av = await fetchAvailable({ fetchImpl, repo });
+    if (!av.ok) return { ok: false, failedStep: 'preflight', error: `could not read the available versions: ${av.error}` };
+    target = av.latest;
+  } else if (!parseSemver(target)) return { ok: false, failedStep: 'preflight', error: `--to ${opts.to} is not a version like 4.3.0` };
+  const current = installed.release ? vtag(installed.release) : null;
+  const plan = {
+    current, currentCommit: installed.commit, target,
+    changes: current ? compareUrl(current, target, repo) : null, releaseNotes: releaseNotesUrl(target, repo),
+    backups, steps: UPDATE_STEPS
+  };
+  const cmp = current ? cmpSemver(target, current) : 1;
+  if (cmp !== null && cmp <= 0 && !force) return { ok: true, upToDate: true, plan };
+
+  const problems = [];
+  const forceable = [];
+  if (installed.dirty) forceable.push(`the checkout at ${install.repoDir} has local changes (git status); commit or discard them, or pass --force`);
+  if (installed.dirty === null) problems.push(`${install.repoDir} is not a git checkout, so it cannot be moved to ${target}`);
+  const disk = await diskFree(install.root, sys);
+  if (disk.ok && disk.freeGB < MIN_FREE_GB) forceable.push(`only ${disk.freeGB} GB free at ${install.root}; an update needs at least ${MIN_FREE_GB} GB (backup + images)`);
+  const dv = await sys.run('docker', ['compose', 'version', '--short']);
+  if (dv.code !== 0) problems.push('docker compose is not available to this user');
+  plan.problems = [...problems, ...forceable];
+  if (problems.length || (forceable.length && !force)) return { ok: false, failedStep: 'preflight', error: plan.problems.join('; '), plan };
+
+  // ---- resume -------------------------------------------------------------------------------------
+  const prev = readState(backups);
+  let state = null;
+  if (prev && prev.op === 'update' && prev.status === 'running') {
+    if (prev.target !== target && !restart) {
+      return { ok: false, failedStep: 'preflight', error: `an update to ${prev.target} stopped after ${prev.done.slice(-1)[0] || 'preflight'}. Re-run without --to to resume it, or pass --restart to abandon it.`, plan };
+    }
+    if (prev.target === target) {
+      state = prev;
+      plan.resumedAfter = state.done.slice(-1)[0] || 'preflight';
+      log(`resuming the update to ${target} after ${plan.resumedAfter}`);
+    }
+  }
+  if (!state) state = { op: 'update', status: 'running', target, from: { release: installed.release, commit: installed.commit, siblings: installed.siblings }, done: ['preflight'], snapshot: null, startedAt: now().toISOString() };
+  const save = () => { if (!dryRun) writeState(backups, { ...state, updatedAt: now().toISOString() }); };
+  const mark = (s) => { if (!state.done.includes(s)) state.done.push(s); save(); };
+  const doneStep = (s) => state.done.includes(s);
+  const finish = (status, extra = {}) => {
+    state.status = status; save();
+    if (!dryRun) appendHistory(backups, { op: 'update', status, at: now().toISOString(), from: state.from.release || state.from.commit, to: target, snapshot: state.snapshot, ...extra });
+  };
+  save();
+
+  const fail = async (step, error) => {
+    log(`✖ ${step}: ${error}`);
+    if (noRollback || dryRun) { finish('failed', { failedStep: step, error }); return { ok: false, failedStep: step, error, plan, rolledBack: false }; }
+    log('rolling back automatically');
+    let rb;
+    if (['fetch', 'build'].includes(step)) {
+      const c = await checkoutCode({ install, g, ref: state.from.commit, siblings: state.from.siblings, log });
+      const h = c.ok ? await waitHealthy({ urls, fetchImpl, tries: healthTries, intervalMs: healthIntervalMs, sleep, log }) : c;
+      rb = { ok: c.ok && h.ok, scope: 'code only (data was never touched)', error: c.ok ? h.error : c.error };
+    } else {
+      rb = await runRollback({ ...opts, snapshotDir: state.snapshot, fromUpdate: true });
+      rb.scope = 'code + data from the pre-update snapshot';
+    }
+    finish(rb.ok ? 'rolled-back' : 'rollback-failed', { failedStep: step, error });
+    return { ok: false, failedStep: step, error, plan, rolledBack: rb.ok, rollback: rb };
+  };
+
+  // ---- backup -------------------------------------------------------------------------------------
+  if (!doneStep('backup')) {
+    const snap = await createSnapshot({ install, installed, backups, g, reason: `before update to ${target}`, now, log });
+    if (!snap.ok) {
+      finish('failed', { failedStep: 'backup', error: snap.error });
+      return { ok: false, failedStep: 'backup', error: `${snap.error}. Nothing was changed.`, plan, rolledBack: false };
+    }
+    state.snapshot = snap.dir; mark('backup');
+  }
+  plan.snapshot = state.snapshot;
+
+  // ---- fetch --------------------------------------------------------------------------------------
+  if (!doneStep('fetch')) {
+    const tagRef = `refs/tags/${target}`;
+    const has = await g.ro('git', ['-C', install.repoDir, 'rev-parse', '-q', '--verify', tagRef]);
+    if (has.code !== 0) {
+      const f1 = await g.mut('git', ['-C', install.repoDir, 'fetch', '--tags', 'origin']);
+      const again = dryRun ? { code: 1 } : await g.ro('git', ['-C', install.repoDir, 'rev-parse', '-q', '--verify', tagRef]);
+      if (f1.code !== 0 || again.code !== 0) {
+        const f2 = await g.mut('git', ['-C', install.repoDir, 'fetch', `https://github.com/${repo}.git`, `+${tagRef}:${tagRef}`]);
+        if (f2.code !== 0) return fail('fetch', `could not fetch ${target} from origin or ${repo}`);
+      }
+    }
+    const c = await checkoutCode({ install, g, ref: target, pullSiblings: true, log });
+    if (!c.ok) return fail('fetch', c.error);
+    mark('fetch');
+  }
+
+  // ---- build --------------------------------------------------------------------------------------
+  const rc = await g.ro('git', ['-C', install.repoDir, 'rev-parse', '--short', `${target}^{commit}`]);
+  const newCommit = rc.code === 0 && rc.stdout.trim() ? rc.stdout.trim() : null;
+  plan.targetCommit = newCommit;
+  if (!doneStep('build')) {
+    const b = await composeBuild({ install, g, commit: newCommit || target });
+    if (!b.ok) return fail('build', b.error);
+    mark('build');
+  }
+
+  // ---- migrate (on boot) --------------------------------------------------------------------------
+  if (!doneStep('migrate')) {
+    const u = await composeUp({ install, g });
+    if (!u.ok) return fail('migrate', u.error);
+    mark('migrate');
+  }
+
+  // ---- health -------------------------------------------------------------------------------------
+  if (dryRun) {
+    log(`[dry run] would wait for auth + collector /health and the collector /version to report ${newCommit || target}`);
+    return { ok: true, dryRun: true, plan, planned: g.planned };
+  }
+  const h = await waitHealthy({ urls, fetchImpl, expectCommit: newCommit, tries: healthTries, intervalMs: healthIntervalMs, sleep, log });
+  if (!h.ok) return fail('health', h.error);
+  mark('health');
+  finish('ok');
+  return { ok: true, plan };
+}
+
+/** opts: { install, sys, backups, urls, fetchImpl, dryRun, snapshotDir, log, healthTries, ... } */
+export async function runRollback(opts) {
+  const { install, sys, backups, urls, fetchImpl = globalThis.fetch, dryRun = false, log = () => {}, now = () => new Date(),
+    healthTries = 40, healthIntervalMs = 3000, sleep = sleepDefault, fromUpdate = false } = opts;
+  const g = gate(sys, { dryRun, log });
+  const snaps = listSnapshots(backups);
+  const snap = opts.snapshotDir ? snaps.find((s) => s.dir === opts.snapshotDir || s.dir.endsWith(`/${opts.snapshotDir}`)) : latestSnapshot(backups);
+  if (!snap) return { ok: false, error: opts.snapshotDir ? `no complete snapshot at ${opts.snapshotDir}` : `no snapshot in ${backups}; nothing to roll back to` };
+  const plan = { snapshot: snap.dir, release: snap.release, commit: snap.commit, createdAt: snap.createdAt };
+  const record = (status, extra = {}) => {
+    if (!dryRun && !fromUpdate) appendHistory(backups, { op: 'rollback', status, at: now().toISOString(), to: snap.release || snap.commit, snapshot: snap.dir, ...extra });
+  };
+  if (!snap.commit) return { ok: false, error: 'the snapshot does not record which commit it came from', plan };
+
+  const c = await checkoutCode({ install, g, ref: snap.commit, siblings: snap.siblings, log });
+  if (!c.ok) { record('failed', { error: c.error }); return { ok: false, step: 'checkout', error: c.error, plan }; }
+  const b = await composeBuild({ install, g, commit: snap.commit });
+  if (!b.ok) { record('failed', { error: b.error }); return { ok: false, step: 'build', error: b.error, plan }; }
+  const r = await restoreSnapshot({ install, snap, g, sleep });
+  if (!r.ok) { record('failed', { error: r.error }); return { ok: false, step: 'restore', error: r.error, plan }; }
+  const u = await composeUp({ install, g });
+  if (!u.ok) { record('failed', { error: u.error }); return { ok: false, step: 'start', error: u.error, plan }; }
+  if (dryRun) {
+    log(`[dry run] would wait for health and the collector /version to report ${snap.commit}`);
+    return { ok: true, dryRun: true, plan, planned: g.planned };
+  }
+  const h = await waitHealthy({ urls, fetchImpl, expectCommit: snap.commit, tries: healthTries, intervalMs: healthIntervalMs, sleep, log });
+  if (!h.ok) { record('failed', { error: h.error }); return { ok: false, step: 'health', error: h.error, plan }; }
+  if (!fromUpdate) writeState(backups, { op: 'rollback', status: 'ok', snapshot: snap.dir, at: now().toISOString() });
+  record('ok');
+  return { ok: true, plan };
+}