total-recall-brain 3.23.2 → 3.24.0

package/src/core/vault.mjs

@@ -3,9 +3,27 @@ import path from 'path';
 import matter from 'gray-matter';
 import { logger } from './logger.mjs';
 
-// Slug allowlist shared with validated-write.mjs — a slug is a filename;
-// anything outside this set risks path traversal on delete.
-const SAFE_NAME = /^[a-zA-Z0-9_-]+$/;
+// Slug allowlist shared with validated-write.mjs — a slug is a filename, so it
+// must not be able to escape its category directory.
+//
+// Dots are permitted in the interior because repository names legitimately
+// contain them — any domain-style name (`example.com`) produces a slug with a
+// dot on ingest, and the stricter rule rejected thousands of real nodes over an
+// ordinary character. What enables traversal is a path separator or `..`, and
+// neither is reachable here — separators are outside the character class, a
+// leading dot or dash is excluded by the first-character rule, and `..` is
+// rejected outright below.
+const SAFE_NAME = /^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$/;
+
+/**
+ * True when `name` is safe to use as a path segment.
+ */
+export function isSafeVaultName(name) {
+  const value = String(name);
+  if (!SAFE_NAME.test(value)) return false;
+  if (value.includes('..')) return false;
+  return true;
+}
 
 /**
  * Atomic file write using write-then-rename to prevent partial corruption.
@@ -158,7 +176,7 @@ export async function writeNode(node, vaultDir, options = {}) {
  * @returns {boolean}
  */
 export function deleteNode(slug, vaultDir) {
-  if (!SAFE_NAME.test(slug)) return false;
+  if (!isSafeVaultName(slug)) return false;
   if (!fs.existsSync(vaultDir)) return false;
   
   // Scan category directories directly instead of loading entire vault

package/src/server/auth.mjs

@@ -355,6 +355,42 @@ export function requireSandboxEnabled(req, res, next) {
   next();
 }
 
+/**
+ * Cookie options for the dashboard session.
+ *
+ * `secure` has to describe the connection that is actually in use, not the
+ * connection the config wishes for. It was previously derived from
+ * `nodeEnv !== 'development' && require_https !== false`, and since nodeEnv
+ * defaults to 'production' and the server only ever calls app.listen() — there
+ * is no HTTPS listener anywhere in it — every login over the mesh bind issued a
+ * `Secure` cookie on a plain-HTTP response. Browsers drop those, so login
+ * returned 200, no session was stored, and the next request 401'd straight back
+ * to the login page. http://localhost is exempt as a trustworthy origin, which
+ * is why this looked fine locally and only bit when reaching the dashboard by
+ * its host or mesh IP.
+ *
+ * Express resolves `req.secure` through `trust proxy`, so a deployment behind
+ * an HTTPS-terminating reverse proxy still gets `Secure` set.
+ */
+function sessionCookieOptions(req, maxAge) {
+  const config = loadSecurityConfig();
+  const isSecureConnection = req?.secure === true;
+
+  if (config.network?.require_https !== false && !isSecureConnection) {
+    logger.warn('auth',
+      'require_https is set but this request arrived over plain HTTP; issuing the session '
+      + 'cookie without Secure so login works. Terminate TLS in front of the dashboard, or '
+      + 'set network.require_https: false to silence this.');
+  }
+
+  return {
+    httpOnly: true,
+    secure: isSecureConnection,
+    maxAge,
+    sameSite: 'lax',
+  };
+}
+
 export async function loginHandler(req, res) {
   const ip = req.ip || req.socket?.remoteAddress;
   if (ip && watchdog.isIpBlocked(ip)) {
@@ -383,13 +419,9 @@ export async function loginHandler(req, res) {
 
   const ttlHours = config.dashboard?.session_ttl_hours || 24;
   const token = jwt.sign({ user: 'admin' }, JWT_SECRET, { expiresIn: `${ttlHours}h` });
-  
-  res.cookie('session', token, {
-    httpOnly: true,
-    secure: nodeEnv !== 'development' && config.network?.require_https !== false,
-    maxAge: ttlHours * 60 * 60 * 1000,
-    sameSite: 'lax'
-  });
+
+  res.cookie('session', token, sessionCookieOptions(req, ttlHours * 60 * 60 * 1000));
+
   
   const requiresReset = !!config.dashboard?.force_password_reset;
   res.json({ success: true, requiresPasswordReset: requiresReset });
@@ -475,6 +507,8 @@ export async function verifyDashboardPassword(password) {
 }
 
 export function logoutHandler(req, res) {
-  res.clearCookie('session');
+  // Attributes must match those the cookie was set with or the browser keeps it.
+  const { maxAge, ...clearOptions } = sessionCookieOptions(req, 0);
+  res.clearCookie('session', clearOptions);
   res.json({ success: true });
 }

package/src/server/health-coverage.mjs

@@ -0,0 +1,74 @@
+/**
+ * Keeps /health cheap without making it dishonest.
+ *
+ * Embedding coverage is the expensive half of the health check: for every
+ * active brain it parses every node in that brain's vault and loads that
+ * brain's embeddings index — 12MB on one machine here. Computed inside the
+ * request it made /health take 25-57s on a loaded laptop while the dashboard
+ * answered in 116ms, so the server read as dead to everything that probes it.
+ * That is the worst endpoint to make slow: /health is what the watchdog polls,
+ * and a liveness probe that times out is indistinguishable from a server that
+ * is down — which is exactly how it was misread.
+ *
+ * A cache alone would trade one wrong answer for another, because the reading
+ * it holds is a criticality signal: a brain with nodes and no vectors serves
+ * keyword results that look like real ones. So the snapshot always carries the
+ * time it was taken, and reports "no reading yet" as null rather than as clean.
+ */
+
+const DEFAULT_TTL_MS = 30_000;
+
+/**
+ * A single-flight, time-bounded cache over an expensive async computation.
+ *
+ * Never blocks a caller: a stale value is served while a refresh runs behind
+ * it, and before the first value lands callers get null. `compute` errors are
+ * swallowed rather than left to reject an unawaited promise — a failed refresh
+ * keeps the previous reading and its original timestamp, so failure looks like
+ * staleness rather than success.
+ *
+ * @param {object} opts
+ * @param {() => Promise<any>} opts.compute  the expensive work
+ * @param {number} [opts.ttlMs]              how long a reading stays fresh
+ * @param {() => number} [opts.now]          clock seam, for tests
+ */
+export function createCoverageCache({ compute, ttlMs = DEFAULT_TTL_MS, now = () => Date.now() }) {
+  let value = null;
+  let takenAt = 0;
+  let inFlight = null;
+
+  function refresh() {
+    // Single-flight: a second caller during a refresh joins it rather than
+    // starting a parallel full-vault parse.
+    if (inFlight) return inFlight;
+    inFlight = Promise.resolve()
+      .then(() => compute())
+      .then((next) => {
+        value = next;
+        takenAt = now();
+      })
+      .catch(() => {
+        // Keep the prior reading and its timestamp; a failed refresh must not
+        // be able to masquerade as a fresh clean result.
+      })
+      .finally(() => {
+        inFlight = null;
+      });
+    return inFlight;
+  }
+
+  return {
+    /** Last known reading, kicking off a background refresh when stale. */
+    snapshot() {
+      if (!value || now() - takenAt > ttlMs) refresh();
+      return value
+        ? { coverage: value, as_of: new Date(takenAt).toISOString() }
+        : { coverage: null, as_of: null };
+    },
+
+    /** Resolves once any in-progress refresh settles. Used to warm at boot. */
+    async settled() {
+      while (inFlight) await inFlight;
+    },
+  };
+}

package/src/server/index.mjs

@@ -29,6 +29,7 @@ import {
 } from './auth.mjs';
 import { logger, logEvents } from "../core/logger.mjs";
 import { readProcessCommand, entryPathHint, shouldHonorPidLock } from '../core/pid-lock.mjs';
+import { createCoverageCache } from './health-coverage.mjs';
 import { drainActiveEmbeddings } from './routes/sessions.mjs';
 import { agentDir as configAgentDir, brainDir as configBrainDir, port as configPort, host as configHost, nodeEnv } from '../core/config.mjs';
 import { getDaemonStatus, ensureDaemonRunning } from '../core/daemon-control.mjs';
@@ -172,6 +173,56 @@ app.use(cookieParser());
 
 // ─── Health Check ───────────────────────────────────────────────────────────────
 
+/**
+ * Embedding coverage is the expensive half of this endpoint — see
+ * health-coverage.mjs for why it is computed off the request path. The check
+ * itself is unchanged: a brain with nodes and no vectors still reports at
+ * critical severity, because recall silently answering from keyword matching
+ * is the failure this was written to catch.
+ */
+async function computeEmbeddingCoverage() {
+  const coverage = [];
+  try {
+    const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
+    const { getActiveBrains } = await import('../core/config.mjs');
+    const brains = getActiveBrains();
+    for (const b of Object.values(brains)) {
+      if (!b?.brainDir) continue;
+      const vaultPath = path.join(b.brainDir, 'memory-vault');
+      if (!fs.existsSync(vaultPath)) continue;
+      // Count only what the embedder actually indexes: type:memory nodes.
+      // Counting every .md under the vault compares against a denominator that
+      // includes proposals/ (optimizer tickets, never embedded by design), so a
+      // perfectly healthy brain reported "847/16701 — 5% coverage" and tripped
+      // a false degraded alarm. Ask the loader, don't count files.
+      let nodeCount = 0;
+      try {
+        const { loadNodes } = await import('../core/vault.mjs');
+        nodeCount = loadNodes(vaultPath).length;
+      } catch { /* unreadable vault */ }
+      let embedded = 0;
+      try {
+        embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
+      } catch { /* index unreadable — reported as 0, which is the honest value */ }
+      coverage.push({
+        layer: b.layer || 'unknown',
+        nodes: nodeCount,
+        embedded,
+        vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
+      });
+    }
+  } catch (err) {
+    coverage.push({ error: String(err?.message || err) });
+  }
+  return coverage;
+}
+
+const coverageCache = createCoverageCache({ compute: computeEmbeddingCoverage });
+const embeddingCoverageSnapshot = () => coverageCache.snapshot();
+
+// Warm it at boot so the first probe already has a real answer.
+embeddingCoverageSnapshot();
+
 app.get('/health', requireAuthOrLocal, async (req, res) => {
   let disk = { free: 0, total: 0 };
   try {
@@ -206,7 +257,7 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   // Check daemon status
   const daemonStatus = getDaemonStatus();
 
-  // Embedding coverage per brain.
+  // Embedding coverage per brain, served from the background cache above.
   //
   // This existed nowhere before, and its absence is exactly why a brain with
   // 2602 vault nodes and 0 embeddings looked healthy for weeks: compile
@@ -214,43 +265,13 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   // real results, and nothing anywhere printed "N of M nodes embedded".
   // Vector search silently off is a CRITICAL failure, not a cosmetic one —
   // recall answers from the wrong brain and the caller cannot tell.
-  const embeddingCoverage = [];
-  try {
-    const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
-    const { getActiveBrains } = await import('../core/config.mjs');
-    const brains = getActiveBrains();
-    for (const b of Object.values(brains)) {
-      if (!b?.brainDir) continue;
-      const vaultPath = path.join(b.brainDir, 'memory-vault');
-      if (!fs.existsSync(vaultPath)) continue;
-      // Count only what the embedder actually indexes: type:memory nodes.
-      // Counting every .md under the vault compares against a denominator that
-      // includes proposals/ (optimizer tickets, never embedded by design), so a
-      // perfectly healthy brain reported "847/16701 — 5% coverage" and tripped
-      // a false degraded alarm. Ask the loader, don't count files.
-      let nodeCount = 0;
-      try {
-        const { loadNodes } = await import('../core/vault.mjs');
-        nodeCount = loadNodes(vaultPath).length;
-      } catch { /* unreadable vault */ }
-      let embedded = 0;
-      try {
-        embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
-      } catch { /* index unreadable — reported as 0, which is the honest value */ }
-      embeddingCoverage.push({
-        layer: b.layer || 'unknown',
-        nodes: nodeCount,
-        embedded,
-        vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
-      });
-    }
-  } catch (err) {
-    embeddingCoverage.push({ error: String(err?.message || err) });
-  }
+  const { coverage: embeddingCoverage, as_of: coverageAsOf } = embeddingCoverageSnapshot();
 
   // A brain with nodes but no vectors answers every query from keyword
-  // matching. Surface it at the same severity as a dead daemon.
-  const unembeddedBrains = embeddingCoverage.filter(c => c.nodes > 0 && c.embedded === 0);
+  // matching. Surface it at the same severity as a dead daemon. Before the
+  // first reading lands there is nothing to judge, and an unknown is reported
+  // as unknown rather than quietly counted as healthy.
+  const unembeddedBrains = (embeddingCoverage || []).filter(c => c.nodes > 0 && c.embedded === 0);
 
   // Determine overall status
   const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
@@ -290,6 +311,9 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
     cli_agents: cliAgents,
     daemon: daemonStatus,
     embedding_coverage: embeddingCoverage,
+    // When this reading was taken. Null means the first computation has not
+    // landed yet, so the coverage field is unknown rather than clean.
+    embedding_coverage_as_of: coverageAsOf,
     // Explicit, greppable signal. A brain with nodes and no vectors serves
     // keyword-only results that look exactly like real ones.
     vector_search_degraded: unembeddedBrains.length > 0

package/src/server/routes/brains.mjs

@@ -218,7 +218,7 @@ router.get('/api/brains/:id/nodes', requireAuth, requireScope('ssss:read', 'memo
 router.post('/api/brains/sync', requireAuth, requireScope('memory:write'), async (req, res) => {
   try {
     const { syncAllRepos } = await import('../../core/repo-sync.mjs');
-    const result = syncAllRepos();
+    const result = await syncAllRepos();
     res.json({
       ok: true,
       totalIngested: result.totalIngested,

package/src/server/routes/proposals.mjs

@@ -90,11 +90,11 @@ proposalsRouter.post('/api/proposals/:id/apply', requireAuth, requireScope('memo
   }
 });
 
-proposalsRouter.post('/api/proposals/:id/reject', requireAuth, requireScope('memory:write'), (req, res) => {
+proposalsRouter.post('/api/proposals/:id/reject', requireAuth, requireScope('memory:write'), async (req, res) => {
   try {
     const vaultDir = resolveVaultFromQuery(req);
     const reason = req.body?.reason || 'Rejected via API.';
-    res.json(setProposalStatus(vaultDir, req.params.id, 'rejected', {
+    res.json(await setProposalStatus(vaultDir, req.params.id, 'rejected', {
       rejection_reason: reason,
       reviewed_by: req.body?.actor || 'api',
     }));
@@ -104,9 +104,9 @@ proposalsRouter.post('/api/proposals/:id/reject', requireAuth, requireScope('mem
   }
 });
 
-proposalsRouter.post('/api/proposals/:id/revert', requireAuth, requireScope('memory:write'), (req, res) => {
+proposalsRouter.post('/api/proposals/:id/revert', requireAuth, requireScope('memory:write'), async (req, res) => {
   try {
-    res.json(revertProposal(resolveVaultFromQuery(req), req.params.id));
+    res.json(await revertProposal(resolveVaultFromQuery(req), req.params.id));
   } catch (err) {
     if (/No undo snapshot|not found/i.test(err.message)) return res.status(404).json({ error: err.message });
     serverError(res, err);

package/src/server/routes/secrets.mjs

@@ -49,7 +49,7 @@ const router = express.Router();
 
 /**
  * Provider secrets are a single host SSOT — NOT per project-vault brain.
- * Selecting project:ultrachat for memory/graph must not empty the Secrets page.
+ * Selecting any project:<name> scope for memory/graph must not empty the Secrets page.
  * Repo binding lives in secret metadata (repos[]), not separate secrets.enc files.
  *
  * Override only via TR_SECRETS_BRAIN / TR_BRAIN env (deploy/ops).