slidemind 0.3.20 → 0.4.0

package/src/workspace/daemon.mjs

@@ -0,0 +1,360 @@
+// The workspace process. Run detached by lifecycle.mjs (`node daemon.mjs`) or
+// in-process for `--foreground`. Owns the descriptor for its lifetime, serves
+// the loopback API, logs to a bounded, redacted file, and drains on stop.
+//
+// Stop = refuse new mutations, wait up to 5s for in-flight local commits, let
+// registered hooks mark unsettled network work for readback, remove OUR
+// descriptor (pid + nonce match — never someone else's), close the listener.
+// The process then ends on its own: nothing here calls process.exit.
+//
+// The config dir is read ONCE, at start: the descriptor, the log, the session
+// store and the remembered port all live there for this process's lifetime,
+// whatever the environment says later (a test that restores XDG_CONFIG_HOME
+// while an in-process daemon is still draining must not write the real one).
+//
+// It looks after itself while it runs:
+//   every 30 s  the descriptor still names us — another live workspace owns it
+//               → drain and leave; missing or unreadable → write ours back;
+//   every 60 s  (detached only) newer code on disk (an npm auto-update, a
+//               release) → restart into it once idle. Sessions and the port
+//               survive, so an open page reconnects by itself.
+
+import { randomBytes, randomUUID } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { scrub } from "../bridge/secrets.mjs";
+import { readPackageJson } from "../cli/common.mjs";
+import { planPageOwner, resolveWorkspace } from "../lib/deck-paths.mjs";
+import { resolveWorkspaceAdapter } from "./adapters.mjs";
+import { autoDiscover } from "./discovery.mjs";
+import { codeStamp, compareCode, descriptorPath, lastPortPath, logPath, pidAlive, probe, readDescriptor, rotateLogs } from "./lifecycle.mjs";
+import { listDecks, workspaceConfigDir, writeJsonAtomic } from "./registry.mjs";
+import { createAuth } from "./auth.mjs";
+import { events as defaultEvents } from "./events.mjs";
+import { createRoutes } from "./routes.mjs";
+import { createServer, homePort, PROTOCOL_VERSION } from "./server.mjs";
+import { startSyncLoop } from "./sync.mjs";
+
+const DRAIN_MS = 5_000;
+// newer code this process already failed to restart into: not retried until the disk moves again
+let failedStamp = null;
+const SELF_CHECK_MS = 30_000;
+const CODE_CHECK_MS = 60_000;
+const REDACT = [
+  [/(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/g, "$1«redacted»"],
+  [/([?&](?:sig|signature|token|access_token|tempauth|code)=)[^&\s"']+/gi, "$1«redacted»"],
+  [/(sk-|eyJ)[A-Za-z0-9._-]{16,}/g, "«redacted»"],
+];
+
+/** Append one redacted line to the daemon log (`file`, default the current config dir's), rotating at the 5 MiB cap. */
+export function log(line, file = logPath()) {
+  let text = scrub(String(line));
+  for (const [re, rep] of REDACT) text = text.replace(re, rep);
+  try {
+    rotateLogs(file);
+    fs.appendFileSync(file, `${new Date().toISOString()} ${text}\n`);
+  } catch {
+    // logging never takes the daemon down
+  }
+}
+
+function readLastPort(dir) {
+  try {
+    const port = JSON.parse(fs.readFileSync(lastPortPath(dir), "utf8"))?.port;
+    return Number.isInteger(port) ? port : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Start the workspace in this process. Resolves `{ url, descriptor, stopped, stop() }`
+ * once listening; `stopped` resolves after a drain completes (stop route, SIGTERM/SIGINT,
+ * or `stop()`). Without `handle` the full API (routes.mjs) is served; `deps` are
+ * its injection points (tests). `restart()` drains, closes and re-execs a fresh
+ * detached process through lifecycle — modules are never reloaded in place.
+ * `deps.storeDir` (default the config dir; null = memory only) keeps browser
+ * sessions across restarts; `deps.watchCode` (default: only the detached
+ * child) restarts into newer code on disk; `deps.selfCheckMs` /
+ * `deps.codeCheckMs` tune the two timers.
+ */
+export async function runDaemon({ handle = null, hooks = {}, deps = {} } = {}) {
+  const configDir = workspaceConfigDir();
+  const descriptorFile = descriptorPath(configDir);
+  const logFile = logPath(configDir);
+  const dlog = (line) => log(line, logFile);
+  const bus = deps.events ?? defaultEvents;
+  const inflight = new Set();
+  const stopHooks = [];
+  const timers = [];
+  const track = (p) => {
+    inflight.add(p);
+    // the caller handles the outcome; the tracker only counts
+    p.finally(() => inflight.delete(p)).catch(() => {});
+    return p;
+  };
+  let resolveStopped;
+  const stopped = new Promise((r) => (resolveStopped = r));
+  let draining = null;
+
+  const descriptor = {
+    schemaVersion: 1,
+    protocolVersion: PROTOCOL_VERSION,
+    pid: process.pid,
+    nonce: randomUUID(),
+    url: null,
+    startedAt: new Date().toISOString(),
+    cliVersion: readPackageJson().version,
+    code: deps.codeStamp ?? codeStamp(),
+    cliBearer: randomBytes(32).toString("base64url"),
+  };
+
+  const drain = async () => {
+    if (draining) return draining;
+    draining = (async () => {
+      // /health says so from now on: nobody reuses a workspace on its way out
+      api.state.draining = true;
+      for (const t of timers) clearInterval(t);
+      dlog(`stop requested; ${inflight.size} local operation(s) in flight`);
+      let cap;
+      const timer = new Promise((r) => (cap = setTimeout(r, DRAIN_MS)));
+      await Promise.race([Promise.allSettled([...inflight]), timer]);
+      clearTimeout(cap); // a pending cap would hold the process for its full 5 s after the drain
+      for (const hook of stopHooks) {
+        try {
+          await hook();
+        } catch (err) {
+          dlog(`stop hook failed: ${err?.message ?? err}`);
+        }
+      }
+      const current = readDescriptor(configDir);
+      if (current?.pid === descriptor.pid && current?.nonce === descriptor.nonce) {
+        fs.rmSync(descriptorFile, { force: true });
+      }
+      dlog("stopped");
+    })();
+    return draining;
+  };
+  // Signal / programmatic stop: drain, then close. The stop ROUTE drains via
+  // onStop and closes itself after its response has left.
+  const shutdown = () => drain().then(() => bus.close()).then(() => api.close());
+  // `{ resume }`: when the new process cannot start, the detached workspace
+  // comes back up on the code it already has loaded rather than leave the page
+  // with nothing (the code watcher passes it; a requested restart reports).
+  const restart = async ({ resume = false } = {}) => {
+    await shutdown();
+    // an in-process workspace whose environment moved (a test restoring its env)
+    // must not start a workspace for some other config dir
+    if (workspaceConfigDir() !== configDir) return dlog("restart skipped: the config dir changed under this process");
+    const { ensureRunning } = await import("./lifecycle.mjs");
+    try {
+      const next = await ensureRunning({ note: dlog });
+      dlog(`restarted as pid ${next.pid} at ${next.url}`);
+    } catch (err) {
+      dlog(`restart failed: ${err?.message ?? err}`);
+      if (!resume) return;
+      try {
+        // it still runs the old code: say so, and the next CLI replaces it when it can
+        await runDaemon({ handle, hooks, deps: { ...deps, codeStamp: descriptor.code } });
+        dlog("resumed on the code already loaded");
+      } catch (again) {
+        dlog(`could not resume: ${again?.message ?? again}`);
+      }
+    }
+  };
+
+  let auth;
+  try {
+    auth = createAuth({ bearer: descriptor.cliBearer, storeDir: deps.storeDir === undefined ? configDir : deps.storeDir });
+  } catch (err) {
+    // an unwritable store costs persistence, never the workspace
+    dlog(`session store unavailable (${err?.code ?? err?.message ?? err}); sessions last until this process ends`);
+    auth = createAuth({ bearer: descriptor.cliBearer });
+  }
+  const scanning = new Set(); // auto-discovery job ids still running (the library page says "Scanning…")
+  const routes = handle ?? createRoutes({ auth, deps: { track, restart, events: bus, scanning: () => [...scanning], ...deps } });
+  // Load every lazily imported route module now: a background `npm install -g`
+  // that swaps the files under a running workspace must not break its next request.
+  if (typeof routes.preload === "function") {
+    try {
+      await routes.preload();
+    } catch (err) {
+      dlog(`preload failed: ${err?.message ?? err}`);
+    }
+  }
+  const lastPort = readLastPort(configDir);
+  const api = createServer({
+    bearer: descriptor.cliBearer,
+    nonce: descriptor.nonce,
+    pid: descriptor.pid,
+    pendingJobs: () => (hooks.pendingJobs ? hooks.pendingJobs() : 0) + inflight.size,
+    pages: () => bus.clientCount,
+    onStop: drain,
+    handle: routes,
+    ports: [lastPort, homePort()],
+  });
+  api.server.on("close", () => resolveStopped());
+  // The sender for every registered pack-owned deck; a stop marks what is on
+  // the wire as outcome-unknown so the next start reads back before resending.
+  if (deps.sync !== false) {
+    // a drain that applied something, or settled a crashed approval, is news for an open page
+    const observe = (p) => {
+      p.then((r) => {
+        if (!r || typeof r !== "object") return;
+        if (r.action === "applied" || r.action === "recovered" || r.action === "created") bus.invalidate({ type: "plan" });
+        if (r.approval?.ok) bus.invalidate({ type: "tasks" });
+      }).catch(() => {});
+      return track(p);
+    };
+    const sync = startSyncLoop({
+      // the registry of OUR config dir only (see the header)
+      decks: () => (workspaceConfigDir() === configDir ? listDecks({ limit: 1000 }).decks.filter((d) => !d.missing).map((d) => resolveWorkspace(d.path)) : []),
+      adapterFor: (ws) => {
+        const owner = planPageOwner(ws).owner;
+        if (!owner || owner === "core") return null;
+        const r = resolveWorkspaceAdapter(owner === "extension" ? null : owner);
+        return r.entry ? r : null;
+      },
+      track: observe,
+      log: dlog,
+      ...(deps.sync ?? {}),
+    });
+    stopHooks.push(sync.stop);
+  }
+  descriptor.url = await api.listen();
+  fs.mkdirSync(configDir, { recursive: true, mode: 0o700 });
+  writeJsonAtomic(descriptorFile, descriptor, { mode: 0o600 });
+  // The remembered port is the page's home: the browser's cookie and every link
+  // an agent already handed out name it. A start pushed off it (something held
+  // it for a moment) serves elsewhere but keeps the home, so the next start goes
+  // back; only a workspace with no home yet records where it landed.
+  const boundPort = Number(new URL(descriptor.url).port);
+  if (lastPort === null || boundPort === lastPort) {
+    try {
+      writeJsonAtomic(lastPortPath(configDir), { port: boundPort });
+    } catch {
+      // the next start just tries the preferred port first
+    }
+  } else {
+    dlog(`port ${lastPort} is taken; serving on ${boundPort} until the next start goes back to ${lastPort}`);
+  }
+  dlog(`listening on ${descriptor.url} (pid ${process.pid}, cli ${descriptor.cliVersion})`);
+  // A new CLI fills the library by itself: remembered roots + the decks root of
+  // the command that started us. Tests pass `autoDiscover:false`.
+  let discovery = null;
+  if (deps.autoDiscover !== false) {
+    discovery = autoDiscover({ version: descriptor.cliVersion, decksDir: deps.decksDir ?? process.env.SLIDEMIND_DECKS ?? null, log: dlog });
+    if (discovery) {
+      for (const id of discovery.jobIds) scanning.add(id);
+      track(
+        discovery.done.then((jobs) => {
+          scanning.clear();
+          dlog(`auto-discovery done: ${jobs.reduce((n, j) => n + j.found.length, 0)} deck(s) registered`);
+          bus.invalidate({ type: "library" });
+        }),
+      );
+    }
+  }
+
+  // Is the descriptor still ours? Another live workspace owning it means we are
+  // an orphan (a hung process that woke up after being replaced): leave. A
+  // missing or broken one is written back — the workspace is running, and every
+  // CLI finds it only through that file.
+  let checking = false;
+  const selfCheck = async () => {
+    if (draining || checking) return;
+    checking = true;
+    try {
+      const cur = readDescriptor(configDir);
+      if (!cur || cur.malformed) {
+        writeJsonAtomic(descriptorFile, descriptor, { mode: 0o600 });
+        dlog("descriptor was missing; wrote it back");
+      } else if (cur.nonce !== descriptor.nonce) {
+        const p = await probe(cur);
+        if (p.live) {
+          dlog(`another workspace (pid ${cur.pid}) owns the descriptor; stopping this one`);
+          await shutdown();
+        } else if (p.draining || (p.unreachable === "timeout" && (cur.pid === process.pid || pidAlive(cur.pid)))) {
+          // busy (a timeout is not an absence: it may be the newer workspace at work)
+          // or on its way out (it removes its own descriptor): look again next tick
+        } else if (!draining) {
+          // definitively gone: refused or reset, its pid dead, or something else answering
+          writeJsonAtomic(descriptorFile, descriptor, { mode: 0o600 });
+          dlog(`descriptor named a workspace that is gone (pid ${cur.pid}); wrote ours back`);
+        }
+      }
+    } catch (err) {
+      dlog(`self-check failed: ${err?.message ?? err}`);
+    } finally {
+      checking = false;
+    }
+  };
+  const every = (ms, fn) => {
+    const t = setInterval(fn, ms);
+    t.unref?.();
+    timers.push(t);
+  };
+  every(deps.selfCheckMs ?? SELF_CHECK_MS, selfCheck);
+  // Newer code on disk: restart into it once nothing is in flight and no agent
+  // is waiting on an answer through us. Only the detached process does this —
+  // a foreground or in-process workspace belongs to whoever runs it.
+  if (deps.watchCode ?? deps.isChild ?? false) {
+    let restarting = false;
+    let seen = null; // the newer stamp of the previous tick: an install still copying files moves it
+    every(deps.codeCheckMs ?? CODE_CHECK_MS, () => {
+      if (draining || restarting) return;
+      let disk;
+      try {
+        disk = codeStamp();
+      } catch {
+        return; // mid-install: the next tick sees the finished tree
+      }
+      if (compareCode(disk, descriptor.code) <= 0 || disk === failedStamp) return;
+      const settled = disk === seen;
+      seen = disk;
+      if (!settled) return; // the same newer tree on two ticks in a row, or wait
+      if (inflight.size > 0 || (routes.openWaits?.() ?? 0) > 0) return;
+      restarting = true;
+      failedStamp = disk; // should the new process fail to start, this process does not try the same tree again
+      dlog(`newer code on disk (${descriptor.code} → ${disk}); restarting into it`);
+      restart({ resume: true });
+    });
+  }
+
+  const onSignal = () => {
+    shutdown();
+  };
+  process.once("SIGTERM", onSignal);
+  process.once("SIGINT", onSignal);
+  stopped.then(() => {
+    for (const t of timers) clearInterval(t);
+    process.off("SIGTERM", onSignal);
+    process.off("SIGINT", onSignal);
+  });
+
+  return {
+    url: descriptor.url,
+    descriptor,
+    api,
+    track,
+    auth,
+    onStop: (fn) => stopHooks.push(fn),
+    discovery,
+    stop: shutdown,
+    restart,
+    stopped,
+    selfCheck,
+  };
+}
+
+// Child entry: `node src/workspace/daemon.mjs`. Uncaught errors go to the
+// inherited log fd; a crash leaves the descriptor behind, and the next start
+// finds it dead (health probe) and replaces it without touching any pid.
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  runDaemon({ deps: { isChild: true } }).catch((err) => {
+    log(`startup failed: ${err?.stack ?? err}`);
+    process.exitCode = 1;
+  });
+}

package/src/workspace/discovery.mjs

@@ -0,0 +1,180 @@
+// Bounded deck discovery under a configured root.
+//
+// Breadth-first, four directory reads in flight, never through a symlink,
+// never into hidden directories (that covers `.git`, `.cache`, `.slidemind`),
+// `node_modules` or cache dirs. A directory that stands as a deck (see
+// deck-paths `isDeckDir`) is registered and NOT descended into — so a deck's
+// `.slidemind/scratch` is never reached, and a pack's deck shows up exactly once.
+// Ordinary `.pptx` files are not decks: only the existing markers count.
+//
+// Job records live at `<sessionConfigDir>/workspace/jobs/<id>.json` and carry
+// no tokens. Cancelling keeps every entry already registered.
+
+import { randomUUID } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import { isDeckDir } from "../lib/deck-paths.mjs";
+import { addRoot, coded, readRegistry, recordRegistryVersion, registerDeckBestEffort, workspaceConfigDir, writeJsonAtomic } from "./registry.mjs";
+
+const CONCURRENCY = 4;
+// ponytail: fixed depth cap keeps a mistaken `discover ~` bounded; make it a root option if a real tree needs more.
+const MAX_DEPTH = 8;
+const SKIP_NAMES = new Set(["node_modules", "cache", "caches", "__pycache__", "dist", "build"]);
+
+export function jobsDir() {
+  return path.join(workspaceConfigDir(), "jobs");
+}
+
+export function jobPath(id) {
+  if (!/^[0-9a-f-]{36}$/.test(String(id))) throw coded("bad_job_id", "job ids are UUIDs");
+  return path.join(jobsDir(), `${id}.json`);
+}
+
+export function readJob(id) {
+  try {
+    return JSON.parse(fs.readFileSync(jobPath(id), "utf8"));
+  } catch (err) {
+    if (err.code === "ENOENT") return null;
+    throw err;
+  }
+}
+
+function writeJob(job) {
+  writeJsonAtomic(jobPath(job.id), job);
+  return job;
+}
+
+/** Directories the scan and the folder picker both leave out. */
+export function skip(entry) {
+  return (
+    !entry.isDirectory() ||
+    entry.isSymbolicLink() ||
+    entry.name.startsWith(".") ||
+    SKIP_NAMES.has(entry.name)
+  );
+}
+
+/**
+ * Remember `root` and scan it. Returns `{ jobId, done }`; `done` resolves to the
+ * final job record (`state` completed | cancelled | failed) and never rejects
+ * for a scan problem — those land in `warnings`. Pass an AbortSignal to cancel.
+ */
+export function startDiscovery(root, { signal = null } = {}) {
+  const job = {
+    schemaVersion: 1,
+    id: randomUUID(),
+    kind: "discovery",
+    root: path.resolve(root),
+    state: "queued",
+    found: [],
+    warnings: [],
+    startedAt: new Date().toISOString(),
+    finishedAt: null,
+  };
+  let real;
+  try {
+    real = addRoot(root);
+  } catch (err) {
+    job.state = "failed";
+    job.finishedAt = job.startedAt;
+    job.warnings.push({ path: job.root, code: err.code ?? "inaccessible", message: err.message });
+    writeJob(job);
+    return { jobId: job.id, done: Promise.resolve(job) };
+  }
+  job.root = real;
+  job.state = "running";
+  writeJob(job);
+  return { jobId: job.id, done: scan(job, signal) };
+}
+
+async function scan(job, signal) {
+  const queue = [{ dir: job.root, depth: 0 }];
+  const seen = new Set();
+  let active = 0;
+  await new Promise((resolve) => {
+    const pump = () => {
+      if (signal?.aborted) {
+        if (active === 0) resolve();
+        return;
+      }
+      while (active < CONCURRENCY && queue.length) {
+        const item = queue.shift();
+        active++;
+        visit(item).finally(() => {
+          active--;
+          pump();
+        });
+      }
+      if (active === 0 && queue.length === 0) resolve();
+    };
+    const visit = async ({ dir, depth }) => {
+      let real;
+      try {
+        real = await fs.promises.realpath(dir);
+      } catch (err) {
+        job.warnings.push({ path: dir, code: err.code ?? "inaccessible", message: err.message });
+        return;
+      }
+      if (seen.has(real)) return;
+      seen.add(real);
+      // A deck is registered and its inside is never walked: `.slidemind/` and
+      // whatever lives under it (the scratch deck) belong to this host deck.
+      if (isDeckDir(real)) {
+        const entry = registerDeckBestEffort(real);
+        if (entry) job.found.push(entry.id);
+        return;
+      }
+      if (depth >= MAX_DEPTH) return;
+      let entries;
+      try {
+        entries = await fs.promises.readdir(real, { withFileTypes: true });
+      } catch (err) {
+        job.warnings.push({ path: real, code: err.code ?? "inaccessible", message: err.message });
+        return;
+      }
+      for (const e of entries) {
+        if (!skip(e)) queue.push({ dir: path.join(real, e.name), depth: depth + 1 });
+      }
+    };
+    pump();
+  });
+  job.state = signal?.aborted ? "cancelled" : "completed";
+  job.finishedAt = new Date().toISOString();
+  return writeJob(job);
+}
+
+/**
+ * The library fills itself on a new CLI: when the registry is missing or was
+ * populated by another version, record this one and scan every remembered
+ * root plus the invoking command's decks root (`decksDir`, lifecycle passes it
+ * as SLIDEMIND_DECKS). Roots nested under another are skipped — one scan
+ * covers them. Null when the library is current or the registry is
+ * unreadable (a malformed file is a recovery case, not a rescan).
+ * Returns `{ previousVersion, roots, jobIds, done }`.
+ */
+export function autoDiscover({ version, decksDir = null, log = () => {} }) {
+  let reg;
+  try {
+    reg = readRegistry();
+  } catch (err) {
+    log(`auto-discovery skipped: ${err.message}`);
+    return null;
+  }
+  if (reg.cliVersion === version) return null;
+  const roots = new Set(reg.roots);
+  if (decksDir) {
+    try {
+      const real = fs.realpathSync(decksDir);
+      if (fs.statSync(real).isDirectory()) roots.add(real);
+    } catch {
+      // the invoking command had no decks dir yet
+    }
+  }
+  const sorted = [...roots].sort();
+  const top = sorted.filter((r) => !sorted.some((o) => o !== r && r.startsWith(o + path.sep)));
+  recordRegistryVersion(version);
+  const jobs = top.map((r) => startDiscovery(r));
+  log(`library populated by cli ${reg.cliVersion ?? "none"}, now ${version}: scanning ${top.length} root(s)`);
+  return { previousVersion: reg.cliVersion ?? null, roots: top, jobIds: jobs.map((j) => j.jobId), done: Promise.all(jobs.map((j) => j.done)) };
+}