total-recall-brain 3.19.2 → 3.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "total-recall-brain",
3
- "version": "3.19.2",
3
+ "version": "3.20.1",
4
4
  "description": "Personal AI system — database-free, SSSS-powered intelligence engine deployable to any POSIX host.",
5
5
  "type": "module",
6
6
  "workspaces": [
@@ -1,30 +1,66 @@
1
1
  agents:
2
- - name: antigravity
3
- binary: antigravity
4
- flags: --sandbox=false --yolo -o json
2
+ # Google Antigravity CLI. The binary is `agy` (NOT `antigravity`) and it runs
3
+ # on the AI Ultra plan. Preferred agent for bulk work.
4
+ # agy -p "prompt" --output-format json -> {status:"SUCCESS", response:"..."}
5
+ # Docs: https://antigravity.google/docs/cli/headless
6
+ - name: agy
7
+ binary: agy
8
+ flags: --output-format json -p
5
9
  priority: 1
6
10
  enabled: true
7
11
  exec: flag
8
12
 
9
- - name: grok
10
- binary: grok
11
- # Grok Build TUI: -p single-turn, plain text for TR parser
12
- flags: --output-format plain --always-approve --permission-mode bypassPermissions
13
- priority: 2
14
- enabled: true
15
- exec: flag
16
-
13
+ # `-p` MUST be present (non-interactive requires --print) and MUST be LAST:
14
+ # `--tools` is variadic (`<tools...>`) in the shipped build, so anything after
15
+ # it — including the appended prompt — is swallowed as a tool name. Proven
16
+ # both ways on claude v2.1.201.
17
17
  - name: claude
18
18
  binary: claude
19
- flags: --output-format json --permission-mode bypassPermissions --setting-sources local --tools ""
20
- priority: 3
19
+ flags: --output-format json --permission-mode bypassPermissions --setting-sources local --tools "" -p
20
+ priority: 2
21
21
  enabled: true
22
22
  exec: flag
23
+
24
+ # `-m` is REQUIRED: ~/.codex/config.toml may pin a model the account cannot
25
+ # use (seen: gpt-5.6-sol -> HTTP 400 "not supported when using Codex with a
26
+ # ChatGPT account"). Account-supported slugs: ~/.codex/models_cache.json.
27
+ # `--full-auto` is hidden from --help but still works as a deprecated alias;
28
+ # the explicit `--sandbox` form is OpenAI's current recommendation.
23
29
  - name: codex
24
30
  binary: codex
25
- flags: --full-auto --json --skip-git-repo-check
26
- priority: 4
31
+ flags: -m gpt-5.5 --sandbox workspace-write --json --skip-git-repo-check
32
+ priority: 3
27
33
  enabled: true
28
34
  exec: subcommand
35
+
36
+ # DISABLED — `antigravity` on PATH is total-recall/bin/antigravity.mjs, a thin
37
+ # wrapper that POSTs to generativelanguage.googleapis.com with GOOGLE_API_KEY,
38
+ # i.e. the METERED Gemini API, not the Ultra plan. Verified 2026-08-01: every
39
+ # call returned 429 "You exceeded your current quota". Use `agy` above.
40
+ - name: antigravity
41
+ binary: antigravity
42
+ flags: --sandbox=false --yolo -o json
43
+ priority: 8
44
+ enabled: false
45
+ exec: flag
46
+
47
+ # DISABLED — this user has no Gemini CLI. Do not enable because a `gemini`
48
+ # binary resolves on PATH; their Google surface is `agy`.
49
+ - name: gemini
50
+ binary: gemini
51
+ flags: --sandbox=false --yolo -o json
52
+ priority: 9
53
+ enabled: false
54
+ exec: flag
55
+
56
+ # DISABLED — not installed. A stale binary may still resolve at
57
+ # ~/.grok/bin/grok; PATH presence is not proof an agent works.
58
+ - name: grok
59
+ binary: grok
60
+ flags: --output-format plain --always-approve --permission-mode bypassPermissions
61
+ priority: 10
62
+ enabled: false
63
+ exec: flag
64
+
29
65
  timeout: 300
30
66
  max_retries: 2
@@ -128,6 +128,8 @@ export default async function recall(args) {
128
128
  // Search all target brains and merge results
129
129
  let allResults = [];
130
130
  let isDegraded = false;
131
+ const searchErrors = [];
132
+ const degradedReasons = new Map();
131
133
 
132
134
  for (const target of searchTargets) {
133
135
  const vaultDir = path.join(target.brainDir, 'memory-vault');
@@ -162,14 +164,28 @@ export default async function recall(args) {
162
164
  priority
163
165
  });
164
166
  }
165
- if (results.degradedTextSearch) isDegraded = true;
167
+ if (results.degradedTextSearch) {
168
+ isDegraded = true;
169
+ degradedReasons.set(results.degradedReason || 'unknown', {
170
+ label: target.label,
171
+ detail: results.degradedDetail || '',
172
+ });
173
+ }
166
174
  // Tag each result with its layer
167
175
  for (const r of results) {
168
176
  r._layer = target.label;
169
177
  }
170
178
  allResults = allResults.concat(results);
171
- } catch {
172
- // Skip brains that can't be searched (no embeddings, etc.)
179
+ } catch (err) {
180
+ // A brain that cannot be searched is skipped — but NEVER silently.
181
+ //
182
+ // This used to be a bare `catch {}`. A missing native binding for
183
+ // better-sqlite3 ("Could not locate the bindings file") made every
184
+ // semantic search throw, and the swallow turned a hard dependency
185
+ // crash into the message "No matching memory nodes found" — so a
186
+ // fully-populated vault looked empty and the real cause was invisible.
187
+ // Degrading to lexical-only results is fine; hiding why is not.
188
+ searchErrors.push({ label: target.label, message: String(err?.message || err) });
173
189
  }
174
190
  }
175
191
 
@@ -182,16 +198,59 @@ export default async function recall(args) {
182
198
  return;
183
199
  }
184
200
 
201
+ if (searchErrors.length > 0) {
202
+ // Surface the real cause. "No matches" and "search crashed" are very
203
+ // different problems and must never look the same.
204
+ console.log('');
205
+ for (const e of searchErrors) {
206
+ console.log(` ⚠️ ${e.label} brain search failed: ${e.message.split('\n')[0]}`);
207
+ }
208
+ if (/bindings file|\.node|NODE_MODULE_VERSION|better-sqlite3/i.test(searchErrors.map(e => e.message).join(' '))) {
209
+ console.log(' A native module is missing or was built for a different Node version.');
210
+ console.log(' Fix: pnpm rebuild better-sqlite3 (or npm rebuild better-sqlite3)');
211
+ console.log(' With pnpm, the package must also be allowed to run build scripts.');
212
+ }
213
+ console.log(' Results below (if any) are lexical-only and may be incomplete.');
214
+ }
215
+
185
216
  if (allResults.length === 0) {
186
- console.log('\n 🔍 No matching memory nodes found.');
217
+ console.log(
218
+ searchErrors.length > 0
219
+ ? '\n 🔍 No results — search failed above, so this is NOT proof the vault is empty.'
220
+ : '\n 🔍 No matching memory nodes found.',
221
+ );
187
222
  return;
188
223
  }
189
224
  if (isDegraded) {
190
- console.log('\n ⚠️ WARNING: Embedding API unreachable or unconfigured. Cos-similarity vector search is disabled.');
191
- console.log(' Running basic text keyword search instead. Configure GOOGLE_API_KEY or OPENAI_API_KEY.');
225
+ // This block used to print ONE message for two unrelated failures:
226
+ // "Embedding API unreachable or unconfigured … Configure GOOGLE_API_KEY
227
+ // or OPENAI_API_KEY". When the real cause was an unbuilt index, that
228
+ // sent the reader to check credentials that were already working, while
229
+ // a brain with thousands of nodes quietly answered from keyword matching
230
+ // and returned results from the wrong project. Name the actual cause.
231
+ const empty = degradedReasons.get('empty_index');
232
+ const failed = degradedReasons.get('embedding_failed');
233
+
234
+ console.log('\n ⚠️ Vector search is OFF — these results are keyword-only and may miss obvious matches.');
235
+ if (empty) {
236
+ console.log(` Cause: the ${empty.label} brain has NO embeddings indexed. The API is fine; the index was never built.`);
237
+ console.log(' Fix: npx total-recall compile (embeds any unembedded nodes for this brain)');
238
+ }
239
+ if (failed) {
240
+ console.log(` Cause: embedding generation failed for the ${failed.label} brain — ${failed.detail}`);
241
+ console.log(' Fix: set OPENROUTER_API_KEY (preferred), or GOOGLE_API_KEY / OPENAI_API_KEY.');
242
+ }
243
+ if (!empty && !failed) {
244
+ console.log(' Cause: unknown — embeddings unavailable and no reason was reported.');
245
+ }
192
246
  }
193
247
  const layerSuffix = searchTargets.length > 1 ? ' (merged)' : ` (${searchTargets[0].label})`;
194
- console.log(`\n 🔍 Semantic search results for "${query}"${layerSuffix}:\n`);
248
+ // Never call it "Semantic search results" when semantic search did not run.
249
+ // The old header said exactly that regardless, so degraded keyword output
250
+ // was indistinguishable from real vector results — same header, same
251
+ // percentage scores. If the label doesn't change, nobody can tell.
252
+ const modeLabel = isDegraded ? '🔤 KEYWORD-ONLY results' : '🔍 Semantic search results';
253
+ console.log(`\n ${modeLabel} for "${query}"${layerSuffix}:\n`);
195
254
  for (let i = 0; i < allResults.length; i++) {
196
255
  const match = allResults[i];
197
256
  const rank = i + 1;
@@ -285,7 +285,13 @@ export function detectProjectBrain(startDir = process.cwd()) {
285
285
  if (fs.existsSync(path.join(candidate, 'SKILL.md'))) {
286
286
  return {
287
287
  agentDir: path.join(dir, '.agent'),
288
- brainDir: globalBrainDir,
288
+ // The project brain lives at the detected candidate — NEVER alias it to
289
+ // globalBrainDir. This returned `globalBrainDir` while the sibling
290
+ // implementation in src/cli/agent-dir.mjs returned `candidate`, so the
291
+ // two disagreed about where the project brain is: anything routed
292
+ // through this function silently operated on the GLOBAL vault while
293
+ // reporting itself as the 'project' layer.
294
+ brainDir: candidate,
289
295
  projectRoot: dir,
290
296
  };
291
297
  }
@@ -47,11 +47,21 @@ export function findBinaryInPath(binaryName) {
47
47
 
48
48
  // ─── Default agent registry (used when agents.yml doesn't exist) ─────────────
49
49
 
50
+ // NOTE ON FLAG ORDER AND MODEL PINS (verified by live probe 2026-08-01):
51
+ // agy — the REAL Google Antigravity CLI (AI Ultra). The binary named
52
+ // `antigravity` is a metered Gemini API wrapper, NOT this. Disabled
53
+ // below; enabling it bills an exhausted API budget (429s).
54
+ // claude — `-p` is required for non-interactive AND must be LAST, because
55
+ // `--tools` is variadic and otherwise swallows the prompt.
56
+ // codex — `-m` must be pinned; a config.toml model the account cannot use
57
+ // fails with HTTP 400 at exit code 0.
50
58
  const DEFAULT_AGENTS = [
51
- { name: 'antigravity', binary: 'antigravity', flags: '--sandbox=false --yolo -o json', priority: 1, enabled: true, exec: 'flag' },
52
- { name: 'grok', binary: 'grok', flags: '--output-format plain --always-approve --permission-mode bypassPermissions', priority: 2, enabled: true, exec: 'flag' },
53
- { name: 'claude', binary: 'claude', flags: '--output-format json --permission-mode bypassPermissions --setting-sources local --tools ""', priority: 3, enabled: true, exec: 'flag' },
54
- { name: 'codex', binary: 'codex', flags: '--sandbox workspace-write --json --skip-git-repo-check', priority: 4, enabled: true, exec: 'subcommand' },
59
+ { name: 'agy', binary: 'agy', flags: '--output-format json -p', priority: 1, enabled: true, exec: 'flag' },
60
+ { name: 'claude', binary: 'claude', flags: '--output-format json --permission-mode bypassPermissions --setting-sources local --tools "" -p', priority: 2, enabled: true, exec: 'flag' },
61
+ { name: 'codex', binary: 'codex', flags: '-m gpt-5.5 --sandbox workspace-write --json --skip-git-repo-check', priority: 3, enabled: true, exec: 'subcommand' },
62
+ { name: 'antigravity', binary: 'antigravity', flags: '--sandbox=false --yolo -o json', priority: 8, enabled: false, exec: 'flag' },
63
+ { name: 'gemini', binary: 'gemini', flags: '--sandbox=false --yolo -o json', priority: 9, enabled: false, exec: 'flag' },
64
+ { name: 'grok', binary: 'grok', flags: '--output-format plain --always-approve --permission-mode bypassPermissions', priority: 10, enabled: false, exec: 'flag' },
55
65
  ];
56
66
 
57
67
  /**
@@ -299,29 +309,65 @@ export async function callLocalRuntime(prompt, system, config) {
299
309
  }
300
310
  } catch {}
301
311
 
302
- // Try agents in priority order with retry budget
303
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
304
- const agent = attempt === 0 ? resolveAgent(config) : resolveAgent({
305
- ...config,
306
- agents: agents.filter(a => a.name !== lastError?.agentName),
307
- });
312
+ // Try agents in priority order, never re-trying one that already failed.
313
+ //
314
+ // The old filter excluded only `lastError.agentName` — the MOST RECENT
315
+ // failure — so a chain of three could hand work back to an agent that had
316
+ // already failed while a healthy one further down was never reached. Track
317
+ // every failure instead.
318
+ //
319
+ // The iteration cap is `agents.length + maxRetries` so that each enabled
320
+ // agent gets a turn even when the retry budget is smaller than the chain;
321
+ // the loop exits as soon as the untried pool is empty.
322
+ const failedAgents = new Set();
323
+ const maxAttempts = agents.length + maxRetries;
324
+
325
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
326
+ const remaining = agents.filter(a => !failedAgents.has(a.name));
327
+ const agent = remaining.length > 0
328
+ ? resolveAgent({ ...config, agents: remaining })
329
+ : null;
308
330
 
309
331
  if (!agent) {
310
- throw new Error('No CLI agents available. Install antigravity, grok, claude, or codex.');
332
+ if (failedAgents.size > 0) {
333
+ throw new Error(
334
+ `All CLI agents failed (tried: ${[...failedAgents].join(', ')}). ` +
335
+ `Last error: ${lastError?.error || 'unknown'}`,
336
+ );
337
+ }
338
+ throw new Error(
339
+ 'No CLI agents available. Install agy (Google Antigravity), claude, or codex.',
340
+ );
311
341
  }
312
342
 
343
+ // The prompt goes LAST, after every flag. Pushing it first (the old
344
+ // behaviour) broke the registry's own flag sets: it emitted a duplicate
345
+ // `-p` for agents whose flags already carry one, and it defeats the
346
+ // ordering that keeps a variadic flag (claude's `--tools <tools...>`) from
347
+ // swallowing the prompt as one of its values.
313
348
  const args = [];
314
349
  if (agent.exec === 'subcommand') {
315
- args.push('exec', fullPrompt);
316
- } else {
317
- args.push('-p', fullPrompt);
350
+ args.push('exec');
318
351
  }
319
352
  if (agent.flags) {
320
- args.push(...agent.flags.split(/\s+/).filter(Boolean));
353
+ // spawnSync does not go through a shell, so quotes in the registry are
354
+ // literal characters: `--tools ""` would pass a two-character string
355
+ // rather than an empty one. Strip matched surrounding quotes per token.
356
+ args.push(
357
+ ...agent.flags
358
+ .split(/\s+/)
359
+ .filter(Boolean)
360
+ .map(tok => tok.replace(/^(['"])(.*)\1$/, '$2')),
361
+ );
321
362
  }
322
363
  if (agent.model) {
323
364
  args.push('-m', agent.model);
324
365
  }
366
+ // `exec: flag` agents need an explicit `-p` only if their flags omit it.
367
+ if (agent.exec !== 'subcommand' && !args.includes('-p') && !args.includes('--print')) {
368
+ args.push('-p');
369
+ }
370
+ args.push(fullPrompt);
325
371
 
326
372
  const cmd = `${agent.binaryPath} ${args.map(a => a.includes(' ') || a.includes('\n') ? `"${a.replace(/"/g, '\\"')}"` : a).join(' ')}`;
327
373
 
@@ -373,11 +419,27 @@ export async function callLocalRuntime(prompt, system, config) {
373
419
  cwd: process.cwd(),
374
420
  });
375
421
 
376
- if (result.status === 0) {
422
+ const softFailure = detectAgentFailure(result.stdout || '');
423
+
424
+ if (result.status === 0 && !softFailure) {
377
425
  const output = result.stdout?.trim() || '';
378
426
  return parseAgentOutput(output);
379
427
  }
380
428
 
429
+ if (softFailure) {
430
+ // Exit code 0 but the payload says the run failed. Auth and model-config
431
+ // failures are not transient, so record and move to the next agent
432
+ // instead of burning the remaining retries on the same broken agent.
433
+ logger.error({
434
+ subsystem: 'runtime',
435
+ message: `${agent.name} reported failure despite exit 0`,
436
+ error: softFailure.slice(0, 500),
437
+ });
438
+ lastError = { agentName: agent.name, error: softFailure.slice(0, 500) };
439
+ failedAgents.add(agent.name);
440
+ continue;
441
+ }
442
+
381
443
  const err = result.stderr || result.stdout || 'Unknown error';
382
444
  logger.error({
383
445
  subsystem: 'runtime',
@@ -385,9 +447,13 @@ export async function callLocalRuntime(prompt, system, config) {
385
447
  error: err.slice(0, 500),
386
448
  });
387
449
  lastError = { agentName: agent.name, error: err.slice(0, 500) };
450
+ failedAgents.add(agent.name);
388
451
  }
389
452
 
390
- throw new Error(`All CLI agents failed. Last error: ${lastError?.error || 'unknown'}`);
453
+ throw new Error(
454
+ `All CLI agents failed (tried: ${[...failedAgents].join(', ') || 'none'}). ` +
455
+ `Last error: ${lastError?.error || 'unknown'}`,
456
+ );
391
457
  }
392
458
 
393
459
  /**
@@ -406,6 +472,57 @@ export async function callLocalRuntimeRaw(messages, config, _tools = undefined)
406
472
  /**
407
473
  * Parse JSON or text output from a CLI agent into plain text.
408
474
  */
475
+ /**
476
+ * Control-frame `type` values in codex's `--json` event stream. None of them
477
+ * carry the model's answer.
478
+ */
479
+ const STREAM_CONTROL_TYPES = new Set([
480
+ 'thread.started', 'turn.started', 'turn.completed', 'turn.failed',
481
+ 'item.started', 'item.updated', 'item.completed', 'error',
482
+ ]);
483
+
484
+ /**
485
+ * Detects a failure the exit code did not report.
486
+ *
487
+ * CLI agents exit 0 on failed runs and describe the failure only in the
488
+ * payload (verified 2026-08-01):
489
+ * claude → {"subtype":"success","is_error":true,"result":"Not logged in …"}
490
+ * codex → {"type":"turn.failed","error":{"message":"…"}}
491
+ * agy → {"status":"ERROR"|"CANCELED","error":"…"}
492
+ *
493
+ * Without this, `status === 0` accepts the error text as the model's answer —
494
+ * e.g. "Not logged in · Please run /login" gets stored as a memory node.
495
+ *
496
+ * @param {string} output raw stdout
497
+ * @returns {string|null} failure reason, or null if the run looks clean
498
+ */
499
+ function detectAgentFailure(output) {
500
+ if (!output) return null;
501
+ for (const line of output.split('\n')) {
502
+ const t = line.trim();
503
+ if (!t.startsWith('{')) continue;
504
+ let parsed;
505
+ try {
506
+ parsed = JSON.parse(t);
507
+ } catch {
508
+ continue;
509
+ }
510
+ if (parsed.is_error === true) {
511
+ return typeof parsed.result === 'string' ? parsed.result : 'is_error: true';
512
+ }
513
+ if (parsed.type === 'turn.failed' || parsed.type === 'error') {
514
+ const msg = parsed.error?.message ?? parsed.message;
515
+ return typeof msg === 'string' ? msg : String(parsed.type);
516
+ }
517
+ if (typeof parsed.status === 'string' && parsed.status.toUpperCase() !== 'SUCCESS') {
518
+ return typeof parsed.error === 'string'
519
+ ? `${parsed.status}: ${parsed.error}`
520
+ : `status=${parsed.status}`;
521
+ }
522
+ }
523
+ return null;
524
+ }
525
+
409
526
  function parseAgentOutput(output) {
410
527
  try {
411
528
  const parsed = JSON.parse(output);
@@ -418,7 +535,34 @@ function parseAgentOutput(output) {
418
535
  return last.content || last.text || last.response || JSON.stringify(last);
419
536
  }
420
537
  } catch {
421
- // Not JSON — return raw text
538
+ // Not a single JSON document. It may still be a JSONL event stream —
539
+ // codex `--json` emits one frame per line, so a whole-output JSON.parse
540
+ // always throws and the raw stream used to be returned as the "answer".
541
+ const answers = [];
542
+ for (const line of output.split('\n')) {
543
+ const t = line.trim();
544
+ if (!t.startsWith('{')) continue;
545
+ let frame;
546
+ try {
547
+ frame = JSON.parse(t);
548
+ } catch {
549
+ continue;
550
+ }
551
+ // codex: the reply is item.text of the `agent_message` item.
552
+ if (frame.item?.type === 'agent_message' && typeof frame.item.text === 'string') {
553
+ answers.push(frame.item.text);
554
+ continue;
555
+ }
556
+ if (STREAM_CONTROL_TYPES.has(frame.type)) continue;
557
+ for (const key of ['response', 'result', 'content', 'text']) {
558
+ if (typeof frame[key] === 'string') {
559
+ answers.push(frame[key]);
560
+ break;
561
+ }
562
+ }
563
+ }
564
+ // Last message wins: earlier frames are partial or superseded.
565
+ if (answers.length > 0) return answers[answers.length - 1];
422
566
  }
423
567
  return output;
424
568
  }
@@ -132,12 +132,22 @@ export async function semanticSearch(query, {
132
132
  const filteredSlugs = new Set(filteredNodes.map(n => n.slug));
133
133
  const results = [];
134
134
 
135
- // Get query embedding
135
+ // Get query embedding.
136
+ //
137
+ // Falling back to lexical-only search is fine. Losing the REASON is not:
138
+ // "the embedding API failed" and "the index for this brain was never built"
139
+ // are completely different problems with completely different fixes, and
140
+ // this used to report both as the former. That mislabelling cost real time —
141
+ // festech.live's project brain had 2602 vault nodes and 0 embeddings, and
142
+ // every recall against it printed "Embedding API unreachable or
143
+ // unconfigured. Configure GOOGLE_API_KEY or OPENAI_API_KEY", which sent the
144
+ // reader off to check credentials that were working perfectly.
136
145
  let queryEmbedding = null;
146
+ let embeddingError = null;
137
147
  try {
138
148
  queryEmbedding = await getEmbedding(String(query));
139
149
  } catch (err) {
140
- // Gracefully ignore embedding generation errors
150
+ embeddingError = String(err?.message || err);
141
151
  }
142
152
 
143
153
  const vaultIndex = loadEmbeddingsIndex(derivedDir);
@@ -279,6 +289,13 @@ export async function semanticSearch(query, {
279
289
  const finalResults = results.slice(0, k);
280
290
  if (!queryEmbedding || vaultEntries.length === 0) {
281
291
  finalResults.degradedTextSearch = true;
292
+ // Say WHICH failure this is. The caller renders remediation from it, and
293
+ // the two causes need opposite fixes: one is credentials, the other is an
294
+ // unbuilt index for this specific brain.
295
+ finalResults.degradedReason = !queryEmbedding ? 'embedding_failed' : 'empty_index';
296
+ finalResults.degradedDetail = !queryEmbedding
297
+ ? (embeddingError || 'embedding provider returned nothing')
298
+ : `no embeddings indexed for this brain (${derivedDir})`;
282
299
  }
283
300
 
284
301
  return finalResults;
@@ -552,6 +552,23 @@ export function resetGateStats() {
552
552
  * Reset internal rate-limit/concurrency state (for testing only).
553
553
  */
554
554
  export function resetGateStateForTests() {
555
+ // Tear down policy watchers FIRST. They are module-level singletons that
556
+ // outlive an individual test, and their callbacks call applyPolicy() +
557
+ // drainQueue() — so a watcher registered by an earlier test can fire in the
558
+ // middle of a later one, silently rewriting blockedDomains/domainMinInterval
559
+ // and draining the queue. That made the firewall-reject and minInterval
560
+ // tests fail intermittently under full-suite load while passing in
561
+ // isolation. Closing them here makes each test start from a quiet state.
562
+ for (const w of [policyWatcher, parentWatcher]) {
563
+ try {
564
+ w?.close();
565
+ } catch {
566
+ // already closed / never opened
567
+ }
568
+ }
569
+ policyWatcher = null;
570
+ parentWatcher = null;
571
+
555
572
  gateGeneration += 1;
556
573
  globalInFlight = 0;
557
574
  domainInFlight.clear();
@@ -187,8 +187,55 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
187
187
  // Check daemon status
188
188
  const daemonStatus = getDaemonStatus();
189
189
 
190
+ // Embedding coverage per brain.
191
+ //
192
+ // This existed nowhere before, and its absence is exactly why a brain with
193
+ // 2602 vault nodes and 0 embeddings looked healthy for weeks: compile
194
+ // reported `compiled: true`, recall returned keyword hits that read like
195
+ // real results, and nothing anywhere printed "N of M nodes embedded".
196
+ // Vector search silently off is a CRITICAL failure, not a cosmetic one —
197
+ // recall answers from the wrong brain and the caller cannot tell.
198
+ const embeddingCoverage = [];
199
+ try {
200
+ const { loadEmbeddingsIndex } = await import('../core/embeddings.mjs');
201
+ const { getActiveBrains } = await import('../core/config.mjs');
202
+ const brains = getActiveBrains();
203
+ for (const b of Object.values(brains)) {
204
+ if (!b?.brainDir) continue;
205
+ const vaultPath = path.join(b.brainDir, 'memory-vault');
206
+ if (!fs.existsSync(vaultPath)) continue;
207
+ let nodeCount = 0;
208
+ const walk = (d) => {
209
+ for (const e of fs.readdirSync(d, { withFileTypes: true })) {
210
+ if (e.name.startsWith('.')) continue;
211
+ const p = path.join(d, e.name);
212
+ if (e.isDirectory()) walk(p);
213
+ else if (e.name.endsWith('.md')) nodeCount++;
214
+ }
215
+ };
216
+ try { walk(vaultPath); } catch { /* unreadable vault */ }
217
+ let embedded = 0;
218
+ try {
219
+ embedded = Object.keys(loadEmbeddingsIndex(path.join(b.brainDir, 'memory-derived'))).length;
220
+ } catch { /* index unreadable — reported as 0, which is the honest value */ }
221
+ embeddingCoverage.push({
222
+ layer: b.layer || 'unknown',
223
+ nodes: nodeCount,
224
+ embedded,
225
+ vector_search: embedded > 0 ? 'on' : 'OFF — keyword-only',
226
+ });
227
+ }
228
+ } catch (err) {
229
+ embeddingCoverage.push({ error: String(err?.message || err) });
230
+ }
231
+
232
+ // A brain with nodes but no vectors answers every query from keyword
233
+ // matching. Surface it at the same severity as a dead daemon.
234
+ const unembeddedBrains = embeddingCoverage.filter(c => c.nodes > 0 && c.embedded === 0);
235
+
190
236
  // Determine overall status
191
- const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0;
237
+ const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0
238
+ || unembeddedBrains.length > 0;
192
239
 
193
240
  // Check Caddy and Cloudflare status
194
241
  let caddyStatus = 'inactive';
@@ -223,6 +270,12 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
223
270
  disk,
224
271
  cli_agents: cliAgents,
225
272
  daemon: daemonStatus,
273
+ embedding_coverage: embeddingCoverage,
274
+ // Explicit, greppable signal. A brain with nodes and no vectors serves
275
+ // keyword-only results that look exactly like real ones.
276
+ vector_search_degraded: unembeddedBrains.length > 0
277
+ ? unembeddedBrains.map(b => `${b.layer}: ${b.embedded}/${b.nodes} embedded`)
278
+ : null,
226
279
  caddy: caddyStatus,
227
280
  cloudflare: cloudflareStatus,
228
281
  emergency_alerts: emergencyAlerts || null,
@@ -15,6 +15,7 @@ import {
15
15
  import { compileSurface } from '../../core/surface.mjs';
16
16
  import { getNodes, invalidate } from '../../core/vault-cache.mjs';
17
17
  import { loadEmbeddingsIndex, loadSessionEmbeddingsIndex } from '../../core/embeddings.mjs';
18
+ import { logger } from '../../core/logger.mjs';
18
19
 
19
20
  const router = Router();
20
21
 
@@ -32,20 +33,30 @@ router.post('/api/vault/compile', requireAuth, requireScope('memory:recompile'),
32
33
  force: true,
33
34
  });
34
35
  // Incrementally embed any new vault nodes and sessions for THIS brain
35
- let vaultEmbed = null, sessionEmbed = null;
36
+ let vaultEmbed = null, sessionEmbed = null, embedError = null;
36
37
  try {
37
38
  const { buildEmbeddingsIndex, buildSessionEmbeddingsIndex } = await import('../../core/embeddings.mjs');
38
39
  invalidate(vaultDir);
39
40
  const vaultNodes = getNodes(vaultDir);
40
41
  vaultEmbed = await buildEmbeddingsIndex(vaultNodes, paths.derivedDir);
41
42
  sessionEmbed = await buildSessionEmbeddingsIndex(paths.sessionsDir, paths.derivedDir);
42
- } catch { /* embedding service may be unavailable — non-fatal */ }
43
+ } catch (err) {
44
+ // Still non-fatal — a compile that produced surfaces is worth keeping —
45
+ // but NEVER silent. This was a bare `catch {}`, and it meant every
46
+ // compile could fail to embed a single node while reporting
47
+ // `compiled: true`. A brain with thousands of nodes and zero vectors
48
+ // looked perfectly healthy, and recall silently answered from keyword
49
+ // matching for weeks.
50
+ embedError = String(err?.message || err);
51
+ logger.error('vault', `Embedding build FAILED during compile (surfaces still compiled): ${embedError}`, { vaultDir });
52
+ }
43
53
  res.json({
44
54
  compiled: true,
45
55
  elapsed_ms: Date.now() - start,
46
56
  vault_dir: vaultDir,
47
57
  vault_embeddings: vaultEmbed,
48
58
  session_embeddings: sessionEmbed,
59
+ embedding_error: embedError,
49
60
  });
50
61
  } catch (err) {
51
62
  serverError(res, err);