parrot-blackbox 1.0.0

package/src/daemon/daemon.js

@@ -0,0 +1,107 @@
+/**
+ * Background daemon. Runs a detached process that polls the scheduler every
+ * `pollIntervalSeconds`, and — the catch-up magic — watches for the
+ * offline→online transition so any missed backups fire immediately once WiFi
+ * returns, in order.
+ *
+ * `daemon start` spawns a detached process; `systemctl --user` integration
+ * (`schedule install`) is the recommended always-on wrapper.
+ */
+
+import fs from 'node:fs';
+import { execa } from 'execa';
+import { loadConfig, journal, readJsonSafe } from '../core/store.js';
+import { daemonLogFile, daemonPidFile, stateDir } from '../core/paths.js';
+import { isOnline } from '../util/network.js';
+import { runDueJobs } from './scheduler.js';
+
+export function readDaemonPid() {
+  const rec = readJsonSafe(daemonPidFile(), {});
+  return rec.pid && Number.isInteger(rec.pid) ? rec.pid : null;
+}
+
+export function daemonRunning() {
+  const pid = readDaemonPid();
+  if (!pid) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function startDaemon() {
+  if (daemonRunning()) return { started: false, reason: 'already running' };
+  const entry = process.argv[1];
+  fs.mkdirSync(stateDir(), { recursive: true });
+  const logFd = fs.openSync(daemonLogFile(), 'a');
+  const child = execa(process.execPath, [entry, 'daemon', 'foreground'], {
+    detached: true,
+    stdio: ['ignore', logFd, logFd],
+    env: process.env,
+  });
+  child.unref();
+  fs.writeFileSync(daemonPidFile(), JSON.stringify({ pid: child.pid, at: Date.now() }));
+  journal('daemon', `started pid=${child.pid}`);
+  return { started: true, pid: child.pid };
+}
+
+export async function stopDaemon() {
+  const pid = readDaemonPid();
+  if (!pid) {
+    fs.rmSync(daemonPidFile(), { force: true });
+    return { stopped: false, reason: 'not running' };
+  }
+  try {
+    process.kill(pid, 'SIGTERM');
+    for (let i = 0; i < 20; i += 1) {
+      if (!daemonRunning()) break;
+      await sleep(150);
+    }
+  } catch {
+    /* already gone */
+  }
+  fs.rmSync(daemonPidFile(), { force: true });
+  journal('daemon', 'stopped');
+  return { stopped: true, pid };
+}
+
+/** Long-running foreground loop (used by daemon and the systemd unit). */
+export async function daemonForeground() {
+  const ignore = () => {};
+  process.on('SIGTERM', () => process.exit(0));
+  process.on('SIGINT', () => process.exit(0));
+
+  const cfg = loadConfig();
+  const intervalMs = (cfg.daemon.pollIntervalSeconds || 60) * 1000;
+  journal('daemon', `foreground loop started (poll ${intervalMs}ms)`);
+
+  let online = await isOnline();
+  journal('daemon', online ? 'network: online' : 'network: offline');
+
+  for (;;) {
+    const sleepPromise = sleep(intervalMs);
+    try {
+      await runDueJobs({ privileged: 'noninteractive', onProgress: ignore });
+    } catch (e) {
+      journal('daemon', `run error: ${e.message}`, 'error');
+    }
+    await sleepPromise;
+
+    const nowOnline = await isOnline();
+    if (!online && nowOnline) {
+      journal('daemon', 'network came online — draining pending backups');
+      try {
+        await runDueJobs({ privileged: 'noninteractive', onProgress: ignore });
+      } catch (e) {
+        journal('daemon', `catch-up run error: ${e.message}`, 'error');
+      }
+    }
+    online = nowOnline;
+  }
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}

package/src/daemon/scheduler.js

@@ -0,0 +1,135 @@
+/**
+ * The scheduler — the crash-proof heart.
+ *
+ * On every tick (daemon poll, cron run, or manual `run`) it:
+ *   1. advances the due window (computing every missed calendar due),
+ *   2. if anything is pending but the machine is offline → defer (retry later),
+ *   3. otherwise drains pending dues OLDEST FIRST (files before snapshots),
+ *   4. prunes old generations (files + snapshots, local & cloud together),
+ *   5. persists state atomically after every job.
+ * A crash anywhere leaves the journal with an unfinished entry and the pending
+ * list intact, so the next tick simply retries. The lock prevents the daemon
+ * and a manual `force` from colliding.
+ */
+
+import { withLock } from '../core/lock.js';
+import { loadConfig, loadState, saveState, journal } from '../core/store.js';
+import { iso, clock, advancePending } from '../core/time.js';
+import { isOnline } from '../util/network.js';
+import { runFilesBackup } from '../backup/workspace.js';
+import { runSnapshotBackup, SudoDeferredError } from '../backup/snapshot.js';
+
+/**
+ * @param {object} opts
+ *  force: bool          run a file backup NOW regardless of the schedule
+ *  privileged: 'noninteractive'|'interactive'
+ *  onProgress: fn
+ * @returns {Promise<object>} {ok, deferred, report}
+ */
+export async function runDueJobs({ force = false, privileged = 'noninteractive', onProgress } = {}) {
+  return withLock(async () => {
+    const cfg = loadConfig();
+    const state = loadState();
+    const now = clock();
+    const report = [];
+
+    if (force) {
+      // Manual force: run EVERY enabled job now (default = the weekly snapshot).
+      const due = iso(now);
+      if (cfg.jobs.files.enabled) {
+        try {
+          const r = await runFilesBackup(cfg, state, { due, onProgress });
+          report.push({ type: 'files', due, ok: true, size: r.sizeBytes, pruned: r.pruned });
+        } catch (e) {
+          journal('files', `force failed due=${due}: ${e.message}`, 'error');
+          report.push({ type: 'files', due, ok: false, error: e.message });
+        }
+      }
+      if (cfg.jobs.snapshots.enabled) {
+        try {
+          const r = await runSnapshotBackup(cfg, state, { due, privileged, onProgress });
+          report.push({ type: 'snapshots', due, ok: true, snapshot: r.snapshot, size: r.manifest?.totalSize, pruned: r.pruned });
+        } catch (e) {
+          const isSudo = e instanceof SudoDeferredError;
+          journal('snapshots', `force failed due=${due}: ${e.message}`, isSudo ? 'info' : 'error');
+          report.push({ type: 'snapshots', due, ok: isSudo, deferred: isSudo, error: e.message });
+        }
+      }
+      return { ok: true, deferred: false, report };
+    }
+
+    // 1. Advance due windows for enabled jobs.
+    let advanced = false;
+    for (const type of ['files', 'snapshots']) {
+      const jc = cfg.jobs[type];
+      const js = state.jobs[type];
+      if (!jc || !jc.enabled || !js) continue;
+      const res = advancePending(js, jc, now);
+      if (res.lastDue !== js.lastDue) {
+        js.lastDue = res.lastDue;
+        advanced = true;
+      }
+      js.pending = res.pending;
+      if (res.dropped > 0) {
+        journal(type, `dropped ${res.dropped} stale overdue dues (catch-up limit ${jc.catchUpLimit})`, 'warn');
+      }
+    }
+    if (advanced) saveState(state);
+
+    const pendingFiles = (state?.jobs?.files) ? [...state.jobs.files.pending] : [];
+    const pendingSnaps = (state?.jobs?.snapshots) ? [...state.jobs.snapshots.pending] : [];
+
+    if (pendingFiles.length === 0 && pendingSnaps.length === 0) {
+      return { ok: true, deferred: false, report };
+    }
+
+    // 2. Network gate — defer everything if we are offline.
+    if (!(await isOnline())) {
+      journal('daemon', `offline — deferring ${pendingFiles.length} file + ${pendingSnaps.length} snapshot due(s)`);
+      for (const type of ['files', 'snapshots']) {
+        if (state.jobs[type]) {
+          state.jobs[type].lastStatus = 'deferred';
+          state.jobs[type].lastRunAt = iso(now);
+        }
+      }
+      saveState(state);
+      return { ok: true, deferred: true, report };
+    }
+
+    // 3. Drain pending files first (oldest → newest), then snapshots.
+    for (const due of pendingFiles) {
+      try {
+        const r = await runFilesBackup(cfg, state, { due, onProgress });
+        report.push({ type: 'files', due, ok: true, size: r.sizeBytes, pruned: r.pruned });
+      } catch (e) {
+        journal('files', `failed due=${due}: ${e.message}`, 'error');
+        state.jobs.files.lastStatus = 'error';
+        state.jobs.files.lastError = e.message;
+        state.jobs.files.lastRunAt = iso(now);
+        saveState(state);
+        report.push({ type: 'files', due, ok: false, error: e.message });
+        break; // stop draining; the rest retry on the next tick
+      }
+    }
+
+    for (const due of pendingSnaps) {
+      try {
+        const r = await runSnapshotBackup(cfg, state, { due, privileged, onProgress });
+        report.push({ type: 'snapshots', due, ok: true, snapshot: r.snapshot, size: r.manifest?.totalSize, pruned: r.pruned });
+      } catch (e) {
+        const isSudo = e instanceof SudoDeferredError;
+        journal('snapshots', `failed due=${due}: ${e.message}`, isSudo ? 'info' : 'error');
+        state.jobs.snapshots.lastStatus = isSudo ? 'deferred' : 'error';
+        state.jobs.snapshots.lastError = e.message;
+        state.jobs.snapshots.lastRunAt = iso(now);
+        saveState(state);
+        // A deferred sudo job is NOT a hard failure — it retries later when the
+        // sudo timestamp is re-armed (e.g. by an interactive `snapshot now`).
+        report.push({ type: 'snapshots', due, ok: isSudo, deferred: isSudo, snapshot: null, error: e.message });
+        if (!isSudo) break; // hard errors stop draining; soft deferrals keep going
+      }
+    }
+
+    return { ok: true, deferred: false, report };
+  });
+}

package/src/storage/accounts.js

@@ -0,0 +1,110 @@
+/**
+ * Storage accounts — the gitswitch-style account manager for the multi-cloud
+ * pool. Each account maps to ONE rclone remote (one MEGA or Google Drive
+ * login). 5 MEGA + 5 Drive accounts with ~20GiB / ~15GiB each ≈ 175GiB that the
+ * allocator can spread backups across so we are never out of space.
+ */
+
+import crypto from 'node:crypto';
+import { loadConfig, saveConfig } from '../core/store.js';
+import { aboutRemote, listRemotes, hasRclone } from './rclone.js';
+import { bytesHuman } from '../util/misc.js';
+
+const GiB = 1024 ** 3;
+
+function accountId() {
+  return crypto.randomUUID().slice(0, 8);
+}
+
+/** Provider defaults (used only when `rclone about` cannot report a quota). */
+export function defaultQuota(provider, cfg) {
+  const p = cfg.storage.providers[provider];
+  return (p?.defaultQuotaGiB ?? 20) * GiB;
+}
+
+export function providerLabel(provider) {
+  return provider === 'mega' ? 'MEGA' : provider === 'gdrive' ? 'Google Drive' : provider;
+}
+
+/**
+ * Add an account. `remote` must already exist in rclone's config (created via
+ * `rclone config` or by the wizard's interactive `rclone config` call).
+ */
+export async function addAccount({ provider = 'mega', label = '', remote = '', quotaGiB = null }) {
+  const cfg = loadConfig();
+  const remotes = await listRemotes();
+  if (!remotes.includes(remote)) {
+    return { ok: false, error: `rclone remote "${remote}" not found. Create it first with: rclone config` };
+  }
+  if (cfg.storage.accounts.some((a) => a.remote === remote)) {
+    return { ok: false, error: `an account already uses remote "${remote}"` };
+  }
+  const account = {
+    id: accountId(),
+    provider,
+    label: label || remote,
+    remote,
+    quotaGiB: quotaGiB && Number(quotaGiB) > 0 ? Number(quotaGiB) : undefined,
+    addedAt: new Date().toISOString(),
+  };
+  cfg.storage.accounts.push(account);
+  saveConfig(cfg);
+  return { ok: true, account };
+}
+
+export function removeAccount(id) {
+  const cfg = loadConfig();
+  const before = cfg.storage.accounts.length;
+  cfg.storage.accounts = cfg.storage.accounts.filter((a) => a.id !== id && a.remote !== id);
+  const removed = cfg.storage.accounts.length !== before;
+  if (removed) saveConfig(cfg);
+  return removed;
+}
+
+export function listAccounts() {
+  return loadConfig().storage.accounts || [];
+}
+
+export function hasAccounts(cfg = loadConfig()) {
+  return (cfg.storage.accounts || []).length > 0;
+}
+
+/**
+ * Refresh live quota data for every account. Falls back to the configured /
+ * provider default quota when the backend does not report usage.
+ * @returns {Promise<Array>} accounts with {total, used, free} bytes
+ */
+export async function refreshAccounts(cfg = loadConfig()) {
+  const accounts = (cfg.storage.accounts || []).map((a) => ({ ...a }));
+  const refreshed = [];
+  for (const acc of accounts) {
+    const about = await aboutRemote(acc.remote + ':');
+    const quota = acc.quotaGiB ? Number(acc.quotaGiB) * GiB : defaultQuota(acc.provider, cfg);
+    const total = about.total ?? quota;
+    const used = about.used ?? 0;
+    const free = about.free ?? Math.max(0, total - used);
+    refreshed.push({ ...acc, total, used, free, live: about.free != null });
+  }
+  return refreshed;
+}
+
+/** One-line storage pool summary (for status/doctor). */
+export function poolSummary(accounts) {
+  const total = accounts.reduce((s, a) => s + (a.total || 0), 0);
+  const used = accounts.reduce((s, a) => s + (a.used || 0), 0);
+  const free = accounts.reduce((s, a) => s + (a.free || 0), 0);
+  return {
+    accounts: accounts.length,
+    total,
+    used,
+    free,
+    text: `${accounts.length} account(s) — ${bytesHuman(total)} total / ${bytesHuman(used)} used / ${bytesHuman(free)} free`,
+  };
+}
+
+/** Spaces requirement check for setup. */
+export async function storageHealth() {
+  const has = await hasRclone();
+  if (!has) return { ok: false, reason: 'rclone is not installed' };
+  return { ok: true, reason: null };
+}

package/src/storage/allocator.js

@@ -0,0 +1,189 @@
+/**
+ * Smart storage allocator — the brain that keeps the multi-account pool
+ * (many MEGA + Google Drive logins) from ever running out of space.
+ *
+ * Strategy:
+ *   1. Every file of an artifact is placed WHOLE on a single account when it
+ *      fits. The account is chosen to minimise the resulting used-percentage
+ *      (water-filling), then most free, then oldest — spreading wear.
+ *   2. If a single file is bigger than any one account's free space, it is
+ *      split into byte-range chunks distributed across several accounts
+ *      (first-fit over free space). A manifest records the exact ranges so
+ *      restore reassembles the file byte-perfectly.
+ * The manifest is written to the cloud (`<root>/<kind>/<id>/__MANIFEST__.json`)
+ * AND mirrored locally, so restore works even from a wiped machine.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import streams from 'node:stream/promises';
+import { createWriteStream } from 'node:fs';
+import { copyToFile, mkdirRemote } from './rclone.js';
+import { bytesHuman } from '../util/misc.js';
+
+export { bytesHuman };
+export const GiB = 1024 ** 3;
+export const MANIFEST_NAME = '__MANIFEST__.json';
+
+/** Walk a dir tree returning [{rel, abs, size, isDir}]. Symlinks skipped. */
+export function walkFiles(localDir) {
+  const out = [];
+  const base = path.resolve(localDir);
+  function rec(abs, rel) {
+    for (const ent of fs.readdirSync(abs, { withFileTypes: true })) {
+      const eAbs = path.join(abs, ent.name);
+      const eRel = rel ? `${rel}/${ent.name}` : ent.name;
+      let st;
+      try {
+        st = fs.statSync(eAbs);
+      } catch {
+        continue; // dangling symlink / unreadable
+      }
+      if (ent.isSymbolicLink()) continue;
+      if (ent.isDirectory()) {
+        out.push({ rel: eRel, abs: eAbs, isDir: true, size: 0 });
+        rec(eAbs, eRel);
+      } else {
+        out.push({ rel: eRel, abs: eAbs, isDir: false, size: st.size });
+      }
+    }
+  }
+  rec(base, '');
+  return out;
+}
+
+/** Choose the account that minimises its resulting used-percentage. */
+export function chooseAccount(needed, accounts) {
+  let best = null;
+  let bestScore = Infinity;
+  for (const acc of accounts) {
+    if (acc.free < needed) continue;
+    const quota = acc.total || 0;
+    const score = quota ? (acc.used + needed) / quota : needed;
+    if (score < bestScore || (score === bestScore && (acc.free > (best?.free ?? -1)))) {
+      bestScore = score;
+      best = acc;
+    }
+  }
+  return best;
+}
+
+/**
+ * Place a whole local dir into the pool.
+ * @param {string} localDir artifact directory to place
+ * @param {object} opts {kind, id, accounts, remoteRoot, chunkSize, onProgress}
+ * @returns {Promise<object>} manifest
+ */
+export async function planAndPlace(localDir, { kind, id, accounts, remoteRoot, chunkSize, onProgress }) {
+  if (!accounts || accounts.length === 0) {
+    throw new Error('no storage accounts configured — add one with `parrot-blackbox account add`');
+  }
+  chunkSize = chunkSize || 2 * GiB;
+
+  const entries = walkFiles(localDir).filter((e) => e.isDir || e.size > 0);
+  const files = entries.filter((e) => !e.isDir).sort((a, b) => b.size - a.size);
+  const totalSize = files.reduce((s, f) => s + f.size, 0);
+
+  const pool = accounts.map((a) => ({ ...a }));
+  const consume = (account, bytes) => {
+    const a = pool.find((x) => x.id === account.id);
+    if (a) a.free -= bytes;
+  };
+
+  const basePath = `${remoteRoot}/${kind}/${id}`;
+  const manifest = {
+    schema: 1,
+    kind,
+    id,
+    createdAt: new Date().toISOString(),
+    totalSize,
+    remoteRoot,
+    entries: [],
+  };
+
+  const report = (done, text) => {
+    if (typeof onProgress === 'function') onProgress({ done, total: files.length, text });
+  };
+
+  let placed = 0;
+  for (const entry of entries) {
+    if (entry.isDir) {
+      const acc = chooseAccount(0, pool);
+      if (!acc) throw outOfSpace();
+      const rp = `${basePath}/${entry.rel}`;
+      const res = await mkdirRemote(`${acc.remote}:${rp}`);
+      if (!res.ok) throw new Error(`mkdir failed on ${acc.remote}: ${res.error}`);
+      manifest.entries.push({ rel: entry.rel, type: 'dir', size: 0, loc: [{ remote: acc.remote, path: rp }] });
+      continue;
+    }
+
+    const rel = entry.rel;
+    const destPath = `${basePath}/${rel}`;
+    const freeMax = Math.max(0, ...pool.map((a) => a.free));
+    if (entry.size <= freeMax) {
+      const acc = chooseAccount(entry.size, pool);
+      if (!acc) { report(placed, `no space for ${rel}`); throw outOfSpace(); }
+      const res = await copyToFile(entry.abs, `${acc.remote}:${destPath}`);
+      if (!res.ok) throw new Error(`upload failed for ${rel} on ${acc.remote}: ${res.error}`);
+      consume(acc, entry.size);
+      manifest.entries.push({
+        rel,
+        type: 'file',
+        size: entry.size,
+        loc: [{ remote: acc.remote, path: destPath, start: 0, end: entry.size, size: entry.size }],
+      });
+    } else {
+      // Split across accounts by byte ranges.
+      const locs = [];
+      let start = 0;
+      let partIndex = 0;
+      while (start < entry.size) {
+        const len = Math.min(chunkSize, entry.size - start);
+        const acc = chooseAccount(len, pool);
+        if (process.env.PBB_DEBUG_ALLOC === '1') {
+          console.error(`[alloc] file=${entry.rel} size=${entry.size} start=${start} len=${len} pool=${pool.map((a) => `${a.remote}:free=${a.free}`).join(',')} acc=${acc ? acc.remote : 'NONE'}`);
+        }
+        if (!acc) { report(placed, `no space for part of ${rel}`); throw outOfSpace(); }
+        const partAbs = await makePartFile(entry, start, len);
+        const partPath = `${destPath}.part-${String(partIndex).padStart(4, '0')}`;
+        const res = await copyToFile(partAbs, `${acc.remote}:${partPath}`);
+        if (!res.ok) throw new Error(`upload failed for ${rel} part on ${acc.remote}: ${res.error}`);
+        consume(acc, len);
+        locs.push({ remote: acc.remote, path: partPath, start, end: start + len, size: len });
+        start += len;
+        partIndex += 1;
+      }
+      manifest.entries.push({ rel, type: 'file', size: entry.size, split: true, loc: locs });
+      report(placed, `split ${rel} across ${locs.length} account(s)`);
+    }
+    placed += 1;
+    report(placed, `placed ${placed}/${files.length}`);
+  }
+
+  // Manifest: cloud + local mirror.
+  const manifestLocalDir = process.env.PBB_MANIFESTS_DIR || path.join(process.env.PBB_STATE_DIR || '.', 'manifests');
+  const manifestLocalPath = path.join(manifestLocalDir, `${kind}-${id}.json`);
+  fs.mkdirSync(manifestLocalDir, { recursive: true });
+  fs.writeFileSync(manifestLocalPath, JSON.stringify(manifest, null, 2));
+  const accForManifest = pool.find((a) => a.free > 0) || pool[0];
+  if (accForManifest) {
+    const res = await copyToFile(manifestLocalPath, `${accForManifest.remote}:${basePath}/${MANIFEST_NAME}`);
+    if (res.ok) manifest.account = accForManifest.remote;
+  }
+  return manifest;
+}
+
+function outOfSpace() {
+  return new Error(
+    'OUT OF SPACE — the pool has no account with enough free room. Add an account, raise a quota, or prune old backups.',
+  );
+}
+
+/** Write a byte range of a file to a temp part file; returns its path. */
+async function makePartFile(file, start, len) {
+  const dir = process.env.PBB_CHUNK_DIR || path.join(process.env.PBB_STATE_DIR || '.', 'chunks');
+  const partAbs = path.join(dir, `.part-${process.pid}-${file.rel.replace(/[^a-z0-9_.-]/gi, '_')}`);
+  fs.mkdirSync(path.dirname(partAbs), { recursive: true });
+  await streams.pipeline(fs.createReadStream(file.abs, { start, end: start + len - 1 }), createWriteStream(partAbs));
+  return partAbs;
+}

package/src/storage/archive.js

@@ -0,0 +1,138 @@
+/**
+ * Archive operations on top of the allocator: discover manifests, restore an
+ * artifact (reassembling byte-range chunks), remove artifacts, list what is
+ * stored across the pool.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import streams from 'node:stream/promises';
+import { createWriteStream } from 'node:fs';
+import { execa } from 'execa';
+import { catRemote, lsjson, purge, copyToFile } from './rclone.js';
+import { MANIFEST_NAME } from './allocator.js';
+
+export { MANIFEST_NAME };
+
+function manifestMirrorPath(kind, id) {
+  const dir = process.env.PBB_MANIFESTS_DIR || path.join(process.env.PBB_STATE_DIR || '.', 'manifests');
+  return path.join(dir, `${kind}-${id}.json`);
+}
+
+/**
+ * Find the manifest for an artifact by scanning all accounts.
+ * @returns {Promise<object|null>} {account, manifest}
+ */
+export async function discoverManifest(kind, id, accounts, remoteRoot) {
+  for (const acc of accounts || []) {
+    const remotePath = `${acc.remote}:${remoteRoot}/${kind}/${id}/${MANIFEST_NAME}`;
+    const res = await catRemote(remotePath);
+    if (res.ok) {
+      try {
+        const manifest = JSON.parse(res.stdout);
+        if (manifest.kind === kind && manifest.id === id) return { account: acc, manifest };
+      } catch {
+        /* corrupt manifest — keep scanning */
+      }
+    }
+  }
+  // Local mirror fallback.
+  try {
+    const local = JSON.parse(fs.readFileSync(manifestMirrorPath(kind, id), 'utf8'));
+    return { account: null, manifest: local };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Restore an artifact from its manifest into destDir.
+ * @returns {Promise<{files:number, bytes:number}>}
+ */
+export async function restoreArtifact(manifest, destDir, { onProgress } = {}) {
+  let files = 0;
+  let bytes = 0;
+
+  for (const entry of manifest.entries || []) {
+    const target = path.join(destDir, ...entry.rel.split('/'));
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+
+    if (entry.type === 'dir') {
+      fs.mkdirSync(target, { recursive: true });
+      continue;
+    }
+    if (entry.loc.length === 1) {
+      const { remote, path: rp } = entry.loc[0];
+      const res = await copyToFile(`${remote}:${rp}`, target);
+      if (!res.ok) throw new Error(`download failed for ${entry.rel}: ${res.error}`);
+      files += 1;
+      bytes += entry.size;
+      if (typeof onProgress === 'function') onProgress({ done: files, text: `restored ${entry.rel}` });
+      continue;
+    }
+
+    // Reassemble split file from byte-range parts.
+    const partAbs = `${target}.assembling-${process.pid}`;
+    const out = createWriteStream(partAbs);
+    const sorted = [...entry.loc].sort((a, b) => a.start - b.start);
+    for (const loc of sorted) {
+      const src = `${loc.remote}:${loc.path}`;
+      const child = execa('rclone', ['cat', src], { reject: false });
+      child.stdout.on('error', () => {});
+      await streams.pipeline(child.stdout, out, { end: false }).catch(() => {});
+      const res = await child; // wait for the process to exit and get its code
+      if (res.exitCode !== 0) throw new Error(`chunk download failed for ${entry.rel} (exit ${res.exitCode})`);
+    }
+    if (out.writableEnded === false) {
+      out.end();
+      await streams.finished(out);
+    }
+    fs.renameSync(partAbs, target);
+    files += 1;
+    bytes += entry.size;
+    if (typeof onProgress === 'function') onProgress({ done: files, text: `restored ${entry.rel}` });
+  }
+  return { files, bytes };
+}
+
+/** Purge an artifact from every account that hosts it (+ local mirror). */
+export async function removeArtifact(kind, id, accounts, remoteRoot) {
+  const removed = [];
+  for (const acc of accounts || []) {
+    const res = await purge(`${acc.remote}:${remoteRoot}/${kind}/${id}`);
+    if (res.ok) removed.push(acc.remote);
+  }
+  try {
+    fs.rmSync(manifestMirrorPath(kind, id), { force: true });
+  } catch {
+    /* best effort local cleanup */
+  }
+  return removed;
+}
+
+/**
+ * List artifacts of a kind discovered on any account (scans manifests).
+ * @returns {Promise<Array>} [{kind, id, createdAt, totalSize, account, manifest}]
+ */
+export async function listArtifacts(kind, accounts, remoteRoot, onProgress) {
+  const out = [];
+  for (const acc of accounts || []) {
+    const scanPath = `${acc.remote}:${remoteRoot}/${kind}`;
+    const res = await lsjson(scanPath, { recursive: true });
+    if (!res.ok) continue;
+    const manifests = res.entries.filter((e) => !e.IsDir && e.Path.endsWith(`/${MANIFEST_NAME}`));
+    if (typeof onProgress === 'function') onProgress({ text: `scanning ${acc.remote}…` });
+    for (const m of manifests) {
+      const id = m.Path.split('/').slice(0, -1).pop();
+      const cat = await catRemote(`${acc.remote}:${remoteRoot}/${kind}/${id}/${MANIFEST_NAME}`);
+      if (!cat.ok) continue;
+      try {
+        const manifest = JSON.parse(cat.stdout);
+        out.push({ kind, id, createdAt: manifest.createdAt, totalSize: manifest.totalSize, account: acc.remote, manifest });
+      } catch {
+        /* skip corrupt */
+      }
+    }
+  }
+  return out.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+}