total-recall-brain 3.23.2 → 3.23.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.23.2",
+  "version": "3.23.3",
   "description": "Personal AI system — database-free, SSSS-powered intelligence engine deployable to any POSIX host.",
   "type": "module",
   "workspaces": [

package/src/server/health-coverage.mjs

@@ -0,0 +1,74 @@
+/**
+ * Keeps /health cheap without making it dishonest.
+ *
+ * Embedding coverage is the expensive half of the health check: for every
+ * active brain it parses every node in that brain's vault and loads that
+ * brain's embeddings index — 12MB on one machine here. Computed inside the
+ * request it made /health take 25-57s on a loaded laptop while the dashboard
+ * answered in 116ms, so the server read as dead to everything that probes it.
+ * That is the worst endpoint to make slow: /health is what the watchdog polls,
+ * and a liveness probe that times out is indistinguishable from a server that
+ * is down — which is exactly how it was misread.
+ *
+ * A cache alone would trade one wrong answer for another, because the reading
+ * it holds is a criticality signal: a brain with nodes and no vectors serves
+ * keyword results that look like real ones. So the snapshot always carries the
+ * time it was taken, and reports "no reading yet" as null rather than as clean.
+ */
+
+const DEFAULT_TTL_MS = 30_000;
+
+/**
+ * A single-flight, time-bounded cache over an expensive async computation.
+ *
+ * Never blocks a caller: a stale value is served while a refresh runs behind
+ * it, and before the first value lands callers get null. `compute` errors are
+ * swallowed rather than left to reject an unawaited promise — a failed refresh
+ * keeps the previous reading and its original timestamp, so failure looks like
+ * staleness rather than success.
+ *
+ * @param {object} opts
+ * @param {() => Promise<any>} opts.compute  the expensive work
+ * @param {number} [opts.ttlMs]              how long a reading stays fresh
+ * @param {() => number} [opts.now]          clock seam, for tests
+ */
+export function createCoverageCache({ compute, ttlMs = DEFAULT_TTL_MS, now = () => Date.now() }) {
+  let value = null;
+  let takenAt = 0;
+  let inFlight = null;
+
+  function refresh() {
+    // Single-flight: a second caller during a refresh joins it rather than
+    // starting a parallel full-vault parse.
+    if (inFlight) return inFlight;
+    inFlight = Promise.resolve()
+      .then(() => compute())
+      .then((next) => {
+        value = next;
+        takenAt = now();
+      })
+      .catch(() => {
+        // Keep the prior reading and its timestamp; a failed refresh must not
+        // be able to masquerade as a fresh clean result.
+      })
+      .finally(() => {
+        inFlight = null;
+      });
+    return inFlight;
+  }
+
+  return {
+    /** Last known reading, kicking off a background refresh when stale. */
+    snapshot() {
+      if (!value || now() - takenAt > ttlMs) refresh();
+      return value
+        ? { coverage: value, as_of: new Date(takenAt).toISOString() }
+        : { coverage: null, as_of: null };
+    },
+
+    /** Resolves once any in-progress refresh settles. Used to warm at boot. */
+    async settled() {
+      while (inFlight) await inFlight;
+    },
+  };
+}

package/src/server/index.mjs

@@ -29,6 +29,7 @@ import {
 } from './auth.mjs';
 import { logger, logEvents } from "../core/logger.mjs";
 import { readProcessCommand, entryPathHint, shouldHonorPidLock } from '../core/pid-lock.mjs';
+import { createCoverageCache } from './health-coverage.mjs';
 import { drainActiveEmbeddings } from './routes/sessions.mjs';
 import { agentDir as configAgentDir, brainDir as configBrainDir, port as configPort, host as configHost, nodeEnv } from '../core/config.mjs';
 import { getDaemonStatus, ensureDaemonRunning } from '../core/daemon-control.mjs';
@@ -172,6 +173,56 @@ app.use(cookieParser());
 
 // ─── Health Check ───────────────────────────────────────────────────────────────
 
+/**
+ * Embedding coverage is the expensive half of this endpoint — see
+ * health-coverage.mjs for why it is computed off the request path. The check
+ * itself is unchanged: a brain with nodes and no vectors still reports at
+ * critical severity, because recall silently answering from keyword matching
+ * is the failure this was written to catch.
+ */
+async function computeEmbeddingCoverage() {
+  const coverage = [];
+  try {
+    const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
+    const { getActiveBrains } = await import('../core/config.mjs');
+    const brains = getActiveBrains();
+    for (const b of Object.values(brains)) {
+      if (!b?.brainDir) continue;
+      const vaultPath = path.join(b.brainDir, 'memory-vault');
+      if (!fs.existsSync(vaultPath)) continue;
+      // Count only what the embedder actually indexes: type:memory nodes.
+      // Counting every .md under the vault compares against a denominator that
+      // includes proposals/ (optimizer tickets, never embedded by design), so a
+      // perfectly healthy brain reported "847/16701 — 5% coverage" and tripped
+      // a false degraded alarm. Ask the loader, don't count files.
+      let nodeCount = 0;
+      try {
+        const { loadNodes } = await import('../core/vault.mjs');
+        nodeCount = loadNodes(vaultPath).length;
+      } catch { /* unreadable vault */ }
+      let embedded = 0;
+      try {
+        embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
+      } catch { /* index unreadable — reported as 0, which is the honest value */ }
+      coverage.push({
+        layer: b.layer || 'unknown',
+        nodes: nodeCount,
+        embedded,
+        vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
+      });
+    }
+  } catch (err) {
+    coverage.push({ error: String(err?.message || err) });
+  }
+  return coverage;
+}
+
+const coverageCache = createCoverageCache({ compute: computeEmbeddingCoverage });
+const embeddingCoverageSnapshot = () => coverageCache.snapshot();
+
+// Warm it at boot so the first probe already has a real answer.
+embeddingCoverageSnapshot();
+
 app.get('/health', requireAuthOrLocal, async (req, res) => {
   let disk = { free: 0, total: 0 };
   try {
@@ -206,7 +257,7 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   // Check daemon status
   const daemonStatus = getDaemonStatus();
 
-  // Embedding coverage per brain.
+  // Embedding coverage per brain, served from the background cache above.
   //
   // This existed nowhere before, and its absence is exactly why a brain with
   // 2602 vault nodes and 0 embeddings looked healthy for weeks: compile
@@ -214,43 +265,13 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   // real results, and nothing anywhere printed "N of M nodes embedded".
   // Vector search silently off is a CRITICAL failure, not a cosmetic one —
   // recall answers from the wrong brain and the caller cannot tell.
-  const embeddingCoverage = [];
-  try {
-    const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
-    const { getActiveBrains } = await import('../core/config.mjs');
-    const brains = getActiveBrains();
-    for (const b of Object.values(brains)) {
-      if (!b?.brainDir) continue;
-      const vaultPath = path.join(b.brainDir, 'memory-vault');
-      if (!fs.existsSync(vaultPath)) continue;
-      // Count only what the embedder actually indexes: type:memory nodes.
-      // Counting every .md under the vault compares against a denominator that
-      // includes proposals/ (optimizer tickets, never embedded by design), so a
-      // perfectly healthy brain reported "847/16701 — 5% coverage" and tripped
-      // a false degraded alarm. Ask the loader, don't count files.
-      let nodeCount = 0;
-      try {
-        const { loadNodes } = await import('../core/vault.mjs');
-        nodeCount = loadNodes(vaultPath).length;
-      } catch { /* unreadable vault */ }
-      let embedded = 0;
-      try {
-        embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
-      } catch { /* index unreadable — reported as 0, which is the honest value */ }
-      embeddingCoverage.push({
-        layer: b.layer || 'unknown',
-        nodes: nodeCount,
-        embedded,
-        vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
-      });
-    }
-  } catch (err) {
-    embeddingCoverage.push({ error: String(err?.message || err) });
-  }
+  const { coverage: embeddingCoverage, as_of: coverageAsOf } = embeddingCoverageSnapshot();
 
   // A brain with nodes but no vectors answers every query from keyword
-  // matching. Surface it at the same severity as a dead daemon.
-  const unembeddedBrains = embeddingCoverage.filter(c => c.nodes > 0 && c.embedded === 0);
+  // matching. Surface it at the same severity as a dead daemon. Before the
+  // first reading lands there is nothing to judge, and an unknown is reported
+  // as unknown rather than quietly counted as healthy.
+  const unembeddedBrains = (embeddingCoverage || []).filter(c => c.nodes > 0 && c.embedded === 0);
 
   // Determine overall status
   const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
@@ -290,6 +311,9 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
     cli_agents: cliAgents,
     daemon: daemonStatus,
     embedding_coverage: embeddingCoverage,
+    // When this reading was taken. Null means the first computation has not
+    // landed yet, so the coverage field is unknown rather than clean.
+    embedding_coverage_as_of: coverageAsOf,
     // Explicit, greppable signal. A brain with nodes and no vectors serves
     // keyword-only results that look exactly like real ones.
     vector_search_degraded: unembeddedBrains.length > 0