total-recall-brain 3.24.2 → 3.25.1

package/src/cli/mesh.mjs

@@ -108,6 +108,15 @@ export default async function meshCli(argv = []) {
   const vaultRoot = path.join(brainDir, 'memory-vault');
   const command = args.shift();
 
+  // `--help` after a subcommand must never EXECUTE that subcommand. Only the
+  // bare `mesh --help` was handled, so `mesh preauthkey --help` fell straight
+  // into the preauthkey branch and MINTED A LIVE ENROLLMENT KEY for someone who
+  // had asked for usage text — a credential issued by a help flag.
+  if (args.includes('--help') || args.includes('-h')) {
+    printHelp();
+    return;
+  }
+
   try {
     if (command === 'status') {
       const info = await describeHeadscaleAvailability(brainDir);

package/src/core/server-restart.mjs

@@ -0,0 +1,130 @@
+/**
+ * server-restart — restart the brain server so freshly installed code is the
+ * code that is actually running.
+ *
+ * Updating did not restart anything. `POST /api/update/run` installed
+ * total-recall-brain@latest and returned a summary; the server kept serving the
+ * modules Node had already loaded, and the update banner cleared — so a finished
+ * update looked identical to a running one. Only the daemon had a restart
+ * control; the server had none.
+ *
+ * The hard rule here: NEVER exit unless a supervisor will bring us back.
+ * Exiting an unsupervised server is not a restart, it is an outage, so
+ * supervision is proven (the job's reported PID is ours) rather than assumed.
+ */
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const LAUNCHD_LABELS = ['com.totalrecall.brain', 'com.totalrecall.server'];
+const SYSTEMD_UNITS = ['total-recall-server', 'total-recall-server.service'];
+
+function launchdPidFor(label) {
+  try {
+    const out = execFileSync('launchctl', ['list', label], { encoding: 'utf8', stdio: 'pipe' });
+    const m = out.match(/"PID"\s*=\s*(\d+)/);
+    return m ? Number(m[1]) : null;
+  } catch {
+    return null;
+  }
+}
+
+function systemdPidFor(unit) {
+  for (const scope of [['--user'], []]) {
+    try {
+      const out = execFileSync('systemctl', [...scope, 'show', '-p', 'MainPID', '--value', unit], {
+        encoding: 'utf8',
+        stdio: 'pipe',
+      });
+      const pid = Number(String(out).trim());
+      if (pid > 0) return { pid, scope: scope.length ? 'user' : 'system' };
+    } catch {
+      /* unit not present in this scope */
+    }
+  }
+  return null;
+}
+
+/**
+ * Prove that a supervisor owns THIS process and will restart it.
+ * @returns {{supervised: boolean, kind: string|null, label: string|null, reason: string}}
+ */
+export function detectSupervisor(pid = process.pid) {
+  // An explicit opt-in for hosts running under a supervisor we cannot query
+  // (docker restart policies, pm2, runit). The operator asserts respawn.
+  if (process.env.TR_SUPERVISED === '1') {
+    return { supervised: true, kind: 'declared', label: 'TR_SUPERVISED=1', reason: 'declared by operator' };
+  }
+  if (process.platform === 'darwin') {
+    for (const label of LAUNCHD_LABELS) {
+      const jobPid = launchdPidFor(label);
+      if (jobPid === pid) {
+        return { supervised: true, kind: 'launchd', label, reason: `launchd job ${label} owns pid ${pid}` };
+      }
+    }
+    return { supervised: false, kind: null, label: null, reason: 'no launchd job reports this pid' };
+  }
+  for (const unit of SYSTEMD_UNITS) {
+    const found = systemdPidFor(unit);
+    if (found && found.pid === pid) {
+      return { supervised: true, kind: 'systemd', label: unit, reason: `systemd ${found.scope} unit ${unit} owns pid ${pid}` };
+    }
+  }
+  return { supervised: false, kind: null, label: null, reason: 'no systemd unit reports this pid' };
+}
+
+/**
+ * Schedule a graceful self-restart.
+ *
+ * Sends SIGTERM to ourselves rather than calling process.exit, so the existing
+ * shutdown path runs: the HTTP server drains, background work finishes, and the
+ * server PID lock is RELEASED. Skipping that lock release would leave the
+ * respawned instance hitting the singleton guard and exiting immediately —
+ * turning a restart into a permanent stop.
+ *
+ * The delay lets the HTTP response flush before the socket goes away.
+ *
+ * @returns {{scheduled: boolean, supervisor: object, delayMs?: number, reason?: string}}
+ */
+export function requestSelfRestart({ delayMs = 750, force = false } = {}) {
+  const supervisor = detectSupervisor();
+  if (!supervisor.supervised && !force) {
+    return {
+      scheduled: false,
+      supervisor,
+      reason:
+        'refusing to exit: no supervisor would restart this process, so exiting would take the server down permanently. '
+        + 'Run it under launchd/systemd, or set TR_SUPERVISED=1 if another supervisor guarantees respawn.',
+    };
+  }
+  const timer = setTimeout(() => {
+    process.kill(process.pid, 'SIGTERM');
+  }, delayMs);
+  timer.unref?.();
+  return { scheduled: true, supervisor, delayMs };
+}
+
+/**
+ * The version recorded in a package root's manifest, read from DISK rather than
+ * from the ESM import cache.
+ *
+ * This is how we decide whether an update actually replaced the code backing
+ * this process. `summary.updated > 0` cannot answer that: it counts registered
+ * projects, and updating some other project's node_modules is no reason to
+ * bounce this server. Comparing the manifest on disk against the version this
+ * process booted with is directly observable and right in every install shape —
+ * source tree (never changes from an npm install, so no restart), or a package
+ * installed under a consumer repo (npm replaces the directory we are running
+ * from, so restart).
+ *
+ * @param {string} rootDir package root to read
+ * @returns {string|null}
+ */
+export function packageVersionOnDisk(rootDir) {
+  try {
+    const raw = fs.readFileSync(path.join(rootDir, 'package.json'), 'utf8');
+    return JSON.parse(raw).version || null;
+  } catch {
+    return null;
+  }
+}

package/src/server/route-manifest.json

@@ -347,6 +347,10 @@
     "method": "GET",
     "path": "/api/mesh/nodes"
   },
+  {
+    "method": "POST",
+    "path": "/api/mesh/preauthkey"
+  },
   {
     "method": "GET",
     "path": "/api/network/audit"
@@ -575,6 +579,10 @@
     "method": "POST",
     "path": "/api/secrets/usage"
   },
+  {
+    "method": "POST",
+    "path": "/api/server/restart"
+  },
   {
     "method": "GET",
     "path": "/api/sessions"

package/src/server/routes/mesh.mjs

@@ -16,6 +16,7 @@ import {
   readSshConfig,
   resolveNodeAccess,
 } from '../../core/mesh-access.mjs';
+import { createHeadscalePreAuthKey, resolveHeadscaleConfig } from '../../core/headscale-client.mjs';
 import { throttledFetch } from '../../core/throttled-fetch.mjs';
 import { defaultVaultRoot } from '../../core/vfs-documents.mjs';
 import {
@@ -264,6 +265,45 @@ router.post('/api/mesh/enroll', requireAuth, requireScope('config:write'), async
   }
 });
 
+/**
+ * POST /api/mesh/preauthkey
+ * Mint an enrollment key for a NEW device (a phone, a tablet, another laptop).
+ *
+ * /api/mesh/enroll only ever enrolls THIS host, so adding a phone had no API at
+ * all and the only route in was the CLI — which a phone user does not have.
+ *
+ * Body: { reusable?: boolean, ephemeral?: boolean, ttlMinutes?: number }
+ */
+router.post('/api/mesh/preauthkey', requireAuth, requireScope('config:write'), async (req, res) => {
+  try {
+    const reusable = req.body?.reusable === true;
+    const ephemeral = req.body?.ephemeral === true;
+    // Short by default: an enrollment key is a bearer credential, and the window
+    // only has to cover walking to the device. Capped so a UI cannot mint a
+    // long-lived key by accident.
+    const requested = Number(req.body?.ttlMinutes);
+    const ttlMinutes = Number.isFinite(requested) ? Math.min(Math.max(requested, 1), 60) : 15;
+
+    const { key, expiration } = await createHeadscalePreAuthKey(
+      { reusable, ephemeral, ttlMinutes },
+      BRAIN_DIR,
+    );
+    res.json({
+      success: true,
+      key,
+      expiration,
+      reusable,
+      ephemeral,
+      ttl_minutes: ttlMinutes,
+      // Surfaced so the UI can show the exact control-server URL to type into a
+      // phone; never include the token, only the URL.
+      login_server: await resolveHeadscaleConfig(BRAIN_DIR).then((c) => c.url).catch(() => null),
+    });
+  } catch (err) {
+    res.status(500).json({ success: false, message: err.message || 'could not mint enrollment key' });
+  }
+});
+
 /**
  * This host's I/O capability profile for agent UI generation
  * (screen, touch, mic, speaker, keyboard, camera, headless, …).

package/src/server/routes/system.mjs

@@ -3,10 +3,13 @@
  *
  * GET  /api/logs/:type           — Read last 200 lines of server or daemon log
  * POST /api/diagnostics/agents   — Run upgrade --agents diagnostics
- * POST /api/diagnostics/agents   — Run upgrade --agents diagnostics
+ * GET  /api/pairing              — Reachable URLs for the mobile pairing QR
  * GET  /api/tasks/failed         — List failed tasks from the DLQ
  * POST /api/tasks/:id/retry      — Re-queue a failed task
  * GET  /api/usage                — Usage ledger + cost summary
+ * GET  /api/usage/providers      — Spend as the providers report it
+ * POST /api/daemon/restart       — Restart the Active Intelligence daemon
+ * POST /api/server/restart       — Restart this server (supervised hosts only)
  */
 
 import { Router } from 'express';
@@ -217,5 +220,39 @@ router.post('/api/daemon/restart', requireAuth, requireScope('config:write'), as
   }
 });
 
+/**
+ * POST /api/server/restart
+ * Restarts the brain server itself, so freshly installed code is the code that
+ * is actually running.
+ *
+ * `force` is deliberately NOT accepted from HTTP. Forcing an exit on a host
+ * where nothing would respawn us is not a restart, it is an outage, and an API
+ * caller must not be able to cause one. Hosts under a supervisor we cannot
+ * query (docker restart policies, pm2, runit) opt in with TR_SUPERVISED=1.
+ */
+router.post('/api/server/restart', requireAuth, requireScope('config:write'), async (_req, res) => {
+  try {
+    const { requestSelfRestart } = await import('../../core/server-restart.mjs');
+    const result = requestSelfRestart();
+    if (!result.scheduled) {
+      return res.status(409).json({
+        success: false,
+        scheduled: false,
+        supervisor: result.supervisor,
+        message: result.reason,
+      });
+    }
+    logger.info('server', `Restart requested via API — ${result.supervisor.reason}`);
+    res.json({
+      success: true,
+      scheduled: true,
+      supervisor: result.supervisor,
+      message: `Server restarting (${result.supervisor.label}). It should be back in a few seconds.`,
+    });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
 export default router;
 export { router as systemRouter };

package/src/server/routes/update.mjs

@@ -13,9 +13,35 @@ import {
   needsUpdate,
   isPackageAutoUpdateEnabled,
 } from '../../core/package-auto-update.mjs';
+import { packageVersionOnDisk, requestSelfRestart } from '../../core/server-restart.mjs';
 
 const router = Router();
 
+/**
+ * Install into the host root and WAIT for it.
+ *
+ * This used to be `detached: true` + `unref()`, which meant the endpoint
+ * answered "update complete" while npm was still writing files — the same
+ * class of lie as reporting an update finished while the old code kept
+ * running. We need the exit anyway: the restart decision below reads the
+ * manifest on disk, and reading it before npm has replaced it would see the
+ * old version and skip the restart.
+ *
+ * @returns {Promise<{ok: boolean, code?: number, error?: string}>}
+ */
+function installIntoHostRoot(latest) {
+  return new Promise((resolve) => {
+    const proc = spawn('npm', ['install', `${PACKAGE_NAME}@${latest}`, '--save'], {
+      cwd: ROOT,
+      stdio: 'ignore',
+    });
+    // spawn reports a missing binary or bad cwd as an 'error' EVENT, not a
+    // throw, and an unhandled 'error' on an EventEmitter kills the server.
+    proc.on('error', (err) => resolve({ ok: false, error: err.message }));
+    proc.on('close', (code) => resolve({ ok: code === 0, code }));
+  });
+}
+
 router.get('/api/update/check', requireAuth, async (_req, res) => {
   try {
     // Host package (this install)
@@ -75,6 +101,11 @@ router.post('/api/update/run', requireAuth, requireScope('config:write'), async
     const dryRun = Boolean(req.body?.dryRun);
     const force = req.body?.force !== false;
     const roots = Array.isArray(req.body?.roots) ? req.body.roots : undefined;
+    const restartWanted = req.body?.restart !== false;
+
+    // The version this process is actually RUNNING, captured before anything is
+    // installed over it.
+    const versionBefore = packageVersionOnDisk(ROOT);
 
     // Fire multi-project updater (awaits — install can take minutes; client should timeout high)
     const summary = await runPackageAutoUpdate({
@@ -86,17 +117,13 @@ router.post('/api/update/run', requireAuth, requireScope('config:write'), async
       save: true,
     });
 
-    // Also kick a local install in ROOT when this host is a consumer (non-source)
+    // Also install into ROOT when this host is a consumer (non-source)
     const hostInfo = inspectProjectPackage(ROOT);
+    let hostInstall = null;
     if (!hostInfo.isSourceTree && (hostInfo.declared || hostInfo.installed) && !dryRun) {
       const latest = summary.latest || (await fetchLatestNpmVersionAsync());
       if (latest && needsUpdate(hostInfo.installed, latest)) {
-        const proc = spawn('npm', ['install', `${PACKAGE_NAME}@${latest}`, '--save'], {
-          cwd: ROOT,
-          detached: true,
-          stdio: 'ignore',
-        });
-        proc.unref();
+        hostInstall = await installIntoHostRoot(latest);
       }
     }
 
@@ -114,17 +141,49 @@ router.post('/api/update/run', requireAuth, requireScope('config:write'), async
     if (failed) parts.push(`${failed} failed`);
     const detail = parts.length ? parts.join(', ') : 'no projects checked';
 
+    // Restart only when the update replaced the code backing THIS process.
+    // Some other project's node_modules changing is no reason to bounce the
+    // server, and a source tree is never touched by an npm install at all.
+    const versionAfter = packageVersionOnDisk(ROOT);
+    const selfReplaced = Boolean(versionBefore && versionAfter && versionBefore !== versionAfter);
+    let restart = {
+      scheduled: false,
+      required: selfReplaced,
+      reason: selfReplaced
+        ? 'restart not requested'
+        : 'this server is already running the installed code',
+    };
+    if (selfReplaced && !dryRun && restartWanted) {
+      const outcome = requestSelfRestart();
+      restart = {
+        scheduled: outcome.scheduled,
+        required: true,
+        supervisor: outcome.supervisor,
+        reason: outcome.scheduled
+          ? `restarting into v${versionAfter} (${outcome.supervisor.label})`
+          : outcome.reason,
+      };
+    }
+
+    const restartNote = restart.scheduled
+      ? ` Restarting into v${versionAfter} now.`
+      : restart.required
+        ? ` This server is still running v${versionBefore} — restart it to pick up v${versionAfter}.`
+        : '';
+
     res.json({
       success: ok,
       updating: !dryRun && ok,
       message: dryRun
         ? `Dry-run complete — ${detail}`
         : ok
-          ? `Package auto-update finished for registered projects (${detail}). Latest: ${summary?.latest || 'n/a'}`
+          ? `Package auto-update finished for registered projects (${detail}). Latest: ${summary?.latest || 'n/a'}.${restartNote}`
           : summary?.reason === 'npm-view-failed'
             ? 'Could not resolve latest version from npm (network or registry). Try again.'
             : `Package auto-update finished with errors (${detail}). Latest: ${summary?.latest || 'n/a'}`,
       summary,
+      host_install: hostInstall,
+      restart,
     });
   } catch (err) {
     serverError(res, err);