total-recall-brain 3.20.0 → 3.20.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.20.0",
+  "version": "3.20.1",
   "description": "Personal AI system — database-free, SSSS-powered intelligence engine deployable to any POSIX host.",
   "type": "module",
   "workspaces": [

package/src/cli/recall.mjs

@@ -129,6 +129,7 @@ export default async function recall(args) {
     let allResults = [];
     let isDegraded = false;
     const searchErrors = [];
+    const degradedReasons = new Map();
 
     for (const target of searchTargets) {
       const vaultDir = path.join(target.brainDir, 'memory-vault');
@@ -163,7 +164,13 @@ export default async function recall(args) {
             priority
           });
         }
-        if (results.degradedTextSearch) isDegraded = true;
+        if (results.degradedTextSearch) {
+          isDegraded = true;
+          degradedReasons.set(results.degradedReason || 'unknown', {
+            label: target.label,
+            detail: results.degradedDetail || '',
+          });
+        }
         // Tag each result with its layer
         for (const r of results) {
           r._layer = target.label;
@@ -215,11 +222,35 @@ export default async function recall(args) {
       return;
     }
     if (isDegraded) {
-      console.log('\n  ⚠️  WARNING: Embedding API unreachable or unconfigured. Cos-similarity vector search is disabled.');
-      console.log('              Running basic text keyword search instead. Configure GOOGLE_API_KEY or OPENAI_API_KEY.');
+      // This block used to print ONE message for two unrelated failures:
+      // "Embedding API unreachable or unconfigured … Configure GOOGLE_API_KEY
+      // or OPENAI_API_KEY". When the real cause was an unbuilt index, that
+      // sent the reader to check credentials that were already working, while
+      // a brain with thousands of nodes quietly answered from keyword matching
+      // and returned results from the wrong project. Name the actual cause.
+      const empty = degradedReasons.get('empty_index');
+      const failed = degradedReasons.get('embedding_failed');
+
+      console.log('\n  ⚠️  Vector search is OFF — these results are keyword-only and may miss obvious matches.');
+      if (empty) {
+        console.log(`      Cause: the ${empty.label} brain has NO embeddings indexed. The API is fine; the index was never built.`);
+        console.log('      Fix:   npx total-recall compile        (embeds any unembedded nodes for this brain)');
+      }
+      if (failed) {
+        console.log(`      Cause: embedding generation failed for the ${failed.label} brain — ${failed.detail}`);
+        console.log('      Fix:   set OPENROUTER_API_KEY (preferred), or GOOGLE_API_KEY / OPENAI_API_KEY.');
+      }
+      if (!empty && !failed) {
+        console.log('      Cause: unknown — embeddings unavailable and no reason was reported.');
+      }
     }
     const layerSuffix = searchTargets.length > 1 ? ' (merged)' : ` (${searchTargets[0].label})`;
-    console.log(`\n  🔍 Semantic search results for "${query}"${layerSuffix}:\n`);
+    // Never call it "Semantic search results" when semantic search did not run.
+    // The old header said exactly that regardless, so degraded keyword output
+    // was indistinguishable from real vector results — same header, same
+    // percentage scores. If the label doesn't change, nobody can tell.
+    const modeLabel = isDegraded ? '🔤 KEYWORD-ONLY results' : '🔍 Semantic search results';
+    console.log(`\n  ${modeLabel} for "${query}"${layerSuffix}:\n`);
     for (let i = 0; i < allResults.length; i++) {
       const match = allResults[i];
       const rank = i + 1;

package/src/core/config.mjs

@@ -285,7 +285,13 @@ export function detectProjectBrain(startDir = process.cwd()) {
     if (fs.existsSync(path.join(candidate, 'SKILL.md'))) {
       return {
         agentDir: path.join(dir, '.agent'),
-        brainDir: globalBrainDir,
+        // The project brain lives at the detected candidate — NEVER alias it to
+        // globalBrainDir. This returned `globalBrainDir` while the sibling
+        // implementation in src/cli/agent-dir.mjs returned `candidate`, so the
+        // two disagreed about where the project brain is: anything routed
+        // through this function silently operated on the GLOBAL vault while
+        // reporting itself as the 'project' layer.
+        brainDir: candidate,
         projectRoot: dir,
       };
     }

package/src/core/search.mjs

@@ -132,12 +132,22 @@ export async function semanticSearch(query, {
   const filteredSlugs = new Set(filteredNodes.map(n => n.slug));
   const results = [];
 
-  // Get query embedding
+  // Get query embedding.
+  //
+  // Falling back to lexical-only search is fine. Losing the REASON is not:
+  // "the embedding API failed" and "the index for this brain was never built"
+  // are completely different problems with completely different fixes, and
+  // this used to report both as the former. That mislabelling cost real time —
+  // festech.live's project brain had 2602 vault nodes and 0 embeddings, and
+  // every recall against it printed "Embedding API unreachable or
+  // unconfigured. Configure GOOGLE_API_KEY or OPENAI_API_KEY", which sent the
+  // reader off to check credentials that were working perfectly.
   let queryEmbedding = null;
+  let embeddingError = null;
   try {
     queryEmbedding = await getEmbedding(String(query));
   } catch (err) {
-    // Gracefully ignore embedding generation errors
+    embeddingError = String(err?.message || err);
   }
 
   const vaultIndex = loadEmbeddingsIndex(derivedDir);
@@ -279,6 +289,13 @@ export async function semanticSearch(query, {
   const finalResults = results.slice(0, k);
   if (!queryEmbedding || vaultEntries.length === 0) {
     finalResults.degradedTextSearch = true;
+    // Say WHICH failure this is. The caller renders remediation from it, and
+    // the two causes need opposite fixes: one is credentials, the other is an
+    // unbuilt index for this specific brain.
+    finalResults.degradedReason = !queryEmbedding ? 'embedding_failed' : 'empty_index';
+    finalResults.degradedDetail = !queryEmbedding
+      ? (embeddingError || 'embedding provider returned nothing')
+      : `no embeddings indexed for this brain (${derivedDir})`;
   }
 
   return finalResults;

package/src/server/index.mjs

@@ -187,8 +187,55 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   // Check daemon status
   const daemonStatus = getDaemonStatus();
 
+  // Embedding coverage per brain.
+  //
+  // This existed nowhere before, and its absence is exactly why a brain with
+  // 2602 vault nodes and 0 embeddings looked healthy for weeks: compile
+  // reported `compiled: true`, recall returned keyword hits that read like
+  // real results, and nothing anywhere printed "N of M nodes embedded".
+  // Vector search silently off is a CRITICAL failure, not a cosmetic one —
+  // recall answers from the wrong brain and the caller cannot tell.
+  const embeddingCoverage = [];
+  try {
+    const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
+    const { getActiveBrains } = await import('../core/config.mjs');
+    const brains = getActiveBrains();
+    for (const b of Object.values(brains)) {
+      if (!b?.brainDir) continue;
+      const vaultPath = path.join(b.brainDir, 'memory-vault');
+      if (!fs.existsSync(vaultPath)) continue;
+      let nodeCount = 0;
+      const walk = (d) => {
+        for (const e of fs.readdirSync(d, { withFileTypes: true })) {
+          if (e.name.startsWith('.')) continue;
+          const p = path.join(d, e.name);
+          if (e.isDirectory()) walk(p);
+          else if (e.name.endsWith('.md')) nodeCount++;
+        }
+      };
+      try { walk(vaultPath); } catch { /* unreadable vault */ }
+      let embedded = 0;
+      try {
+        embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
+      } catch { /* index unreadable — reported as 0, which is the honest value */ }
+      embeddingCoverage.push({
+        layer: b.layer || 'unknown',
+        nodes: nodeCount,
+        embedded,
+        vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
+      });
+    }
+  } catch (err) {
+    embeddingCoverage.push({ error: String(err?.message || err) });
+  }
+
+  // A brain with nodes but no vectors answers every query from keyword
+  // matching. Surface it at the same severity as a dead daemon.
+  const unembeddedBrains = embeddingCoverage.filter(c => c.nodes > 0 && c.embedded === 0);
+
   // Determine overall status
-  const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0;
+  const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
+    || unembeddedBrains.length > 0;
 
   // Check Caddy and Cloudflare status
   let caddyStatus = 'inactive';
@@ -223,6 +270,12 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
     disk,
     cli_agents: cliAgents,
     daemon: daemonStatus,
+    embedding_coverage: embeddingCoverage,
+    // Explicit, greppable signal. A brain with nodes and no vectors serves
+    // keyword-only results that look exactly like real ones.
+    vector_search_degraded: unembeddedBrains.length > 0
+      ? unembeddedBrains.map(b => `${b.layer}: ${b.embedded}/${b.nodes} embedded`)
+      : null,
     caddy: caddyStatus,
     cloudflare: cloudflareStatus,
     emergency_alerts: emergencyAlerts || null,

package/src/server/routes/vault.mjs

@@ -15,6 +15,7 @@ import {
 import { compileSurface } from '../../core/surface.mjs';
 import { getNodes, invalidate } from '../../core/vault-cache.mjs';
 import { loadEmbeddingsIndex, loadSessionEmbeddingsIndex } from '../../core/embeddings.mjs';
+import { logger } from '../../core/logger.mjs';
 
 const router = Router();
 
@@ -32,20 +33,30 @@ router.post('/api/vault/compile', requireAuth, requireScope('memory:recompile'),
       force: true,
     });
     // Incrementally embed any new vault nodes and sessions for THIS brain
-    let vaultEmbed = null, sessionEmbed = null;
+    let vaultEmbed = null, sessionEmbed = null, embedError = null;
     try {
       const { buildEmbeddingsIndex, buildSessionEmbeddingsIndex } = await import('../../core/embeddings.mjs');
       invalidate(vaultDir);
       const vaultNodes = getNodes(vaultDir);
       vaultEmbed = await buildEmbeddingsIndex(vaultNodes, paths.derivedDir);
       sessionEmbed = await buildSessionEmbeddingsIndex(paths.sessionsDir, paths.derivedDir);
-    } catch { /* embedding service may be unavailable — non-fatal */ }
+    } catch (err) {
+      // Still non-fatal — a compile that produced surfaces is worth keeping —
+      // but NEVER silent. This was a bare `catch {}`, and it meant every
+      // compile could fail to embed a single node while reporting
+      // `compiled: true`. A brain with thousands of nodes and zero vectors
+      // looked perfectly healthy, and recall silently answered from keyword
+      // matching for weeks.
+      embedError = String(err?.message || err);
+      logger.error('vault', `Embedding build FAILED during compile (surfaces still compiled): ${embedError}`, { vaultDir });
+    }
     res.json({
       compiled: true,
       elapsed_ms: Date.now() - start,
       vault_dir: vaultDir,
       vault_embeddings: vaultEmbed,
       session_embeddings: sessionEmbed,
+      embedding_error: embedError,
     });
   } catch (err) {
     serverError(res, err);