total-recall-brain 3.2.3 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +1 -1
  2. package/extension/background.js +14 -2
  3. package/extension/content-script.js +18 -1
  4. package/extension/lib/brain-client.js +86 -52
  5. package/extension/lib/preconfigured.js +5 -0
  6. package/extension/manifest.json +1 -1
  7. package/extension/options/options.html +1 -0
  8. package/extension/options/options.js +12 -6
  9. package/extension/popup/popup.html +1 -0
  10. package/extension/popup/popup.js +29 -4
  11. package/extension/sidepanel/sidepanel.html +1 -0
  12. package/extension/sidepanel/sidepanel.js +24 -6
  13. package/frontend/index.html +8 -0
  14. package/frontend/src/api.ts +1 -2
  15. package/frontend/src/pages/ChatPage.tsx +9 -6
  16. package/frontend/src/pages/IntegrationsPage.tsx +144 -3
  17. package/package.json +2 -2
  18. package/scaffold/.agent/skills/total-recall/SKILL.md +55 -6
  19. package/scaffold/.agent/skills/total-recall/memory-vault/invariants/operating-instructions.md +2 -2
  20. package/scaffold/.agent/skills/total-recall/skills/cli-agents/agents.yml +1 -6
  21. package/scaffold/.agent/skills/total-recall/skills/ssss/SKILL.md +19 -0
  22. package/scaffold/.agent/skills/total-recall/skills/ssss/references/ssss-spec.md +33 -5
  23. package/src/cli/forget.mjs +18 -12
  24. package/src/cli/ingest/index.mjs +3 -2
  25. package/src/cli/init.mjs +30 -0
  26. package/src/cli/map.mjs +2 -2
  27. package/src/cli/rebuild.mjs +1 -1
  28. package/src/cli/remember.mjs +21 -11
  29. package/src/cli/setup.mjs +10 -1
  30. package/src/cli/share.mjs +7 -2
  31. package/src/core/append-log.mjs +131 -0
  32. package/src/core/clarity-rewriter.mjs +10 -6
  33. package/src/core/conclusion-writer.mjs +3 -2
  34. package/src/core/config.mjs +6 -0
  35. package/src/core/conflict-detector.mjs +4 -2
  36. package/src/core/context-compiler.mjs +412 -0
  37. package/src/core/daemon-loop.mjs +15 -0
  38. package/src/core/dream.mjs +4 -4
  39. package/src/core/drift-detector.mjs +3 -3
  40. package/src/core/embeddings.mjs +60 -10
  41. package/src/core/fact-seeker.mjs +70 -44
  42. package/src/core/import-rules.mjs +3 -2
  43. package/src/core/inference-engine.mjs +5 -4
  44. package/src/core/migrate.mjs +4 -3
  45. package/src/core/optimizer.mjs +6 -5
  46. package/src/core/parallel-context.mjs +288 -0
  47. package/src/core/post-mortem.mjs +4 -3
  48. package/src/core/research-queue.mjs +4 -3
  49. package/src/core/runtime.mjs +2 -2
  50. package/src/core/sandbox.mjs +56 -26
  51. package/src/core/scheduler.mjs +6 -5
  52. package/src/core/schema.mjs +2 -2
  53. package/src/core/search.mjs +167 -101
  54. package/src/core/surface.mjs +203 -53
  55. package/src/core/validated-write.mjs +100 -0
  56. package/src/core/vault-cache.mjs +46 -18
  57. package/src/core/vault.mjs +17 -29
  58. package/src/core/vector-field.mjs +659 -0
  59. package/src/core/vector-store.mjs +85 -0
  60. package/src/server/api.mjs +4 -3
  61. package/src/server/index.mjs +13 -61
  62. package/src/server/rest.mjs +238 -29
  63. package/src/server/route-manifest.json +344 -77
  64. package/src/server/routes/memory.mjs +85 -39
  65. package/src/server/routes/sessions.mjs +6 -2
  66. package/src/server/routes/share.mjs +9 -2
@@ -4,7 +4,8 @@ import matter from 'gray-matter';
4
4
  import crypto from 'crypto';
5
5
  import os from 'os';
6
6
  import { callLocalRuntime, cleanAndParseJSON } from './runtime.mjs';
7
- import { loadNodes, writeNode, atomicWrite, safeStringify } from './vault.mjs';
7
+ import { writeNode, atomicWrite, safeStringify } from './vault.mjs';
8
+ import { getNodes } from './vault-cache.mjs';
8
9
  import { logger } from './logger.mjs';
9
10
  import { addToQueue } from './research-queue.mjs';
10
11
  import { persistTaskToDisk } from './scheduler.mjs';
@@ -285,62 +286,87 @@ async function gatherFromSources(topicEntry, researchConfig) {
285
286
  const useGithub = (suggested_sources || []).includes('github') ||
286
287
  (tags || []).some(t => ['code', 'library', 'framework', 'open-source'].includes(t));
287
288
 
289
+ const tasks = [];
290
+
288
291
  // 1. Web search — Brave → Serper fallback, automatically selected
289
292
  if (availability.available.includes('brave-search') || availability.available.includes('serper')) {
290
- try {
291
- const webResults = await webSearch(topic, researchConfig, 5);
292
- results.push(...webResults);
293
- logger.info({ subsystem: 'fact-seeker', message: `Web search: ${webResults.length} results for "${topic}"` });
294
- } catch (err) {
295
- errors.push({ source: 'web-search', error: err.message });
296
- logger.info({ subsystem: 'fact-seeker', message: `Web search failed: ${err.message}` });
297
- }
293
+ tasks.push((async () => {
294
+ try {
295
+ const webResults = await webSearch(topic, researchConfig, 5);
296
+ logger.info({ subsystem: 'fact-seeker', message: `Web search: ${webResults.length} results for "${topic}"` });
297
+ return { success: true, results: webResults };
298
+ } catch (err) {
299
+ logger.info({ subsystem: 'fact-seeker', message: `Web search failed: ${err.message}` });
300
+ return { success: false, source: 'web-search', error: err.message };
301
+ }
302
+ })());
298
303
  }
299
304
 
300
305
  // 2. DuckDuckGo Instant Answers (always available, good for definitions/facts)
301
- try {
302
- const ddgResult = await duckduckgoInstant(topic, researchConfig);
303
- if (ddgResult) results.push(ddgResult);
304
- } catch (err) {
305
- errors.push({ source: 'duckduckgo', error: err.message });
306
- }
306
+ tasks.push((async () => {
307
+ try {
308
+ const ddgResult = await duckduckgoInstant(topic, researchConfig);
309
+ return { success: true, results: ddgResult ? [ddgResult] : [] };
310
+ } catch (err) {
311
+ return { success: false, source: 'duckduckgo', error: err.message };
312
+ }
313
+ })());
307
314
 
308
315
  // 3. Wikipedia (always available, high-quality structured knowledge)
309
- try {
310
- const wikiResult = await wikipediaFetch(topic, researchConfig);
311
- if (wikiResult) results.push(wikiResult);
312
- } catch (err) {
313
- errors.push({ source: 'wikipedia', error: err.message });
314
- }
315
-
316
- // 4. arXiv (for academic/research topics)
317
- if (useArxiv) {
316
+ tasks.push((async () => {
318
317
  try {
319
- const arxivResults = await arxivSearch(topic, researchConfig, 3);
320
- results.push(...arxivResults);
321
- logger.info({ subsystem: 'fact-seeker', message: `arXiv: ${arxivResults.length} papers for "${topic}"` });
318
+ const wikiResult = await wikipediaFetch(topic, researchConfig);
319
+ return { success: true, results: wikiResult ? [wikiResult] : [] };
322
320
  } catch (err) {
323
- errors.push({ source: 'arxiv', error: err.message });
321
+ return { success: false, source: 'wikipedia', error: err.message };
324
322
  }
323
+ })());
324
+
325
+ // 4. arXiv (for academic/research topics)
326
+ if (useArxiv) {
327
+ tasks.push((async () => {
328
+ try {
329
+ const arxivResults = await arxivSearch(topic, researchConfig, 3);
330
+ logger.info({ subsystem: 'fact-seeker', message: `arXiv: ${arxivResults.length} papers for "${topic}"` });
331
+ return { success: true, results: arxivResults };
332
+ } catch (err) {
333
+ return { success: false, source: 'arxiv', error: err.message };
334
+ }
335
+ })());
325
336
  }
326
337
 
327
338
  // 5. npm (for JS/Node topics)
328
339
  if (useNpm) {
329
- try {
330
- const npmResults = await npmSearch(topic, researchConfig, 3);
331
- results.push(...npmResults);
332
- } catch (err) {
333
- errors.push({ source: 'npm', error: err.message });
334
- }
340
+ tasks.push((async () => {
341
+ try {
342
+ const npmResults = await npmSearch(topic, researchConfig, 3);
343
+ return { success: true, results: npmResults };
344
+ } catch (err) {
345
+ return { success: false, source: 'npm', error: err.message };
346
+ }
347
+ })());
335
348
  }
336
349
 
337
350
  // 6. GitHub (for code/library topics)
338
351
  if (useGithub) {
339
- try {
340
- const ghResults = await githubSearch(topic, researchConfig, 'repositories', 3);
341
- results.push(...ghResults);
342
- } catch (err) {
343
- errors.push({ source: 'github', error: err.message });
352
+ tasks.push((async () => {
353
+ try {
354
+ const ghResults = await githubSearch(topic, researchConfig, 'repositories', 3);
355
+ return { success: true, results: ghResults };
356
+ } catch (err) {
357
+ return { success: false, source: 'github', error: err.message };
358
+ }
359
+ })());
360
+ }
361
+
362
+ // Await all tasks concurrently in parallel
363
+ const taskResults = await Promise.all(tasks);
364
+
365
+ for (const res of taskResults) {
366
+ if (res.success) {
367
+ results.push(...res.results);
368
+ } else {
369
+ errors.push({ source: res.source, error: res.error });
344
370
  }
345
371
  }
346
372
 
@@ -730,7 +756,7 @@ Output ONLY valid JSON.`;
730
756
  * Evaluates coverage, temporal awareness, source diversity, and limitations.
731
757
  */
732
758
  export async function runSelfDiagnosis({ vaultDir, runtimeConfig }) {
733
- const nodes = loadNodes(vaultDir);
759
+ const nodes = getNodes(vaultDir);
734
760
  const activeNodes = nodes.filter(n => n.status === 'active');
735
761
  const agenda = loadAgenda();
736
762
  const availability = checkSourceAvailability(loadResearchConfig());
@@ -994,7 +1020,7 @@ async function pushDeliberationConclusion(conclusions = []) {
994
1020
  * Phase 2/5: Deliberation
995
1021
  */
996
1022
  export async function runResearchDeliberationCycle({ vaultDir, nodeSlug, topic, runtimeConfig }) {
997
- const nodes = loadNodes(vaultDir);
1023
+ const nodes = getNodes(vaultDir);
998
1024
  const targetNode = nodes.find(n => n.slug === nodeSlug);
999
1025
  if (!targetNode) {
1000
1026
  return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
@@ -1309,7 +1335,7 @@ Output ONLY valid JSON.`;
1309
1335
  * Phase 3/5: Refinement & Clarity Auditing
1310
1336
  */
1311
1337
  export async function runResearchImprovementCycle({ vaultDir, nodeSlug, topic, runtimeConfig }) {
1312
- const nodes = loadNodes(vaultDir);
1338
+ const nodes = getNodes(vaultDir);
1313
1339
  const targetNode = nodes.find(n => n.slug === nodeSlug);
1314
1340
  if (!targetNode) {
1315
1341
  return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
@@ -1384,7 +1410,7 @@ export async function runResearchMonitoringCycle({
1384
1410
  derivedDir,
1385
1411
  instructionsFile
1386
1412
  }) {
1387
- const nodes = loadNodes(vaultDir);
1413
+ const nodes = getNodes(vaultDir);
1388
1414
  const targetNode = nodes.find(n => n.slug === nodeSlug);
1389
1415
  if (!targetNode) {
1390
1416
  return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
@@ -1490,7 +1516,7 @@ Output ONLY valid JSON.`;
1490
1516
  * Phase 5/5: Domain Tangents Expansion
1491
1517
  */
1492
1518
  export async function runResearchExpansionCycle({ vaultDir, nodeSlug, topic, runtimeConfig }) {
1493
- const nodes = loadNodes(vaultDir);
1519
+ const nodes = getNodes(vaultDir);
1494
1520
  const targetNode = nodes.find(n => n.slug === nodeSlug);
1495
1521
  if (!targetNode) {
1496
1522
  return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
@@ -14,7 +14,8 @@ import fs from 'node:fs';
14
14
  import path from 'node:path';
15
15
  import os from 'node:os';
16
16
  import crypto from 'node:crypto';
17
- import { writeNode, loadNodes } from './vault.mjs';
17
+ import { writeNode } from './vault.mjs';
18
+ import { getNodes } from './vault-cache.mjs';
18
19
  import { agentDir, brainDir } from './config.mjs';
19
20
 
20
21
  const AGENT_DIR = agentDir;
@@ -75,7 +76,7 @@ export function detectRuleFiles(dirs) {
75
76
  // Load existing vault nodes to detect already-imported files
76
77
  let existingSlugs = new Set();
77
78
  try {
78
- existingSlugs = new Set(loadNodes(VAULT_DIR).map(n => n.slug));
79
+ existingSlugs = new Set(getNodes(VAULT_DIR).map(n => n.slug));
79
80
  } catch { /* vault may not exist yet */ }
80
81
 
81
82
  for (const dir of searchDirs) {
@@ -3,7 +3,8 @@ import path from 'path';
3
3
  import matter from 'gray-matter';
4
4
  import crypto from 'crypto';
5
5
  import { callLocalRuntime, cleanAndParseJSON } from './runtime.mjs';
6
- import { loadNodes, atomicWrite, safeStringify } from './vault.mjs';
6
+ import { atomicWrite, safeStringify } from './vault.mjs';
7
+ import { getNodes } from './vault-cache.mjs';
7
8
  import { logger } from './logger.mjs';
8
9
  import { brainDir } from './config.mjs';
9
10
 
@@ -83,7 +84,7 @@ function buildInferencePrompt(nodes) {
83
84
  * @returns {{ conclusions: object[], contradictions: object[], merge_candidates: object[], error? }}
84
85
  */
85
86
  export async function runInferenceTask(slugs, { vaultDir, inboxDir, runtimeConfig }) {
86
- const allNodes = loadNodes(vaultDir);
87
+ const allNodes = getNodes(vaultDir);
87
88
  const nodes = allNodes.filter(n => slugs.includes(n.slug) && n.status === 'active');
88
89
 
89
90
  if (nodes.length < 2) {
@@ -154,7 +155,7 @@ function writeConclusionNode(inboxDir, conclusion, sourceSlugList, vaultDir) {
154
155
 
155
156
  if (vaultDir) {
156
157
  try {
157
- const allNodes = loadNodes(vaultDir);
158
+ const allNodes = getNodes(vaultDir);
158
159
  citations = sourceSlugList.map(s => {
159
160
  const matchingNode = allNodes.find(n => n.slug === s);
160
161
  const cat = matchingNode ? matchingNode.category : 'facts';
@@ -312,7 +313,7 @@ Output valid JSON:
312
313
  * @param {object} opts.runtimeConfig
313
314
  */
314
315
  export async function runSynthesisTask(slugA, slugB, { vaultDir, inboxDir, runtimeConfig }) {
315
- const allNodes = loadNodes(vaultDir);
316
+ const allNodes = getNodes(vaultDir);
316
317
  const nodeA = allNodes.find(n => n.slug === slugA);
317
318
  const nodeB = allNodes.find(n => n.slug === slugB);
318
319
 
@@ -1,7 +1,8 @@
1
1
  import fs from 'fs';
2
2
  import path from 'path';
3
3
  import crypto from 'crypto';
4
- import { loadNodes, writeNode, atomicWrite } from './vault.mjs';
4
+ import { writeNode, atomicWrite } from './vault.mjs';
5
+ import { getNodes } from './vault-cache.mjs';
5
6
  import { createSnapshot } from './snapshot.mjs';
6
7
  import { SSSS_SCHEMAS } from './schema.mjs';
7
8
  import { logger } from './logger.mjs';
@@ -33,7 +34,7 @@ export async function runMigration(vaultDir, fromVersion, toVersion, migrationFn
33
34
  };
34
35
 
35
36
  try {
36
- const nodes = loadNodes(vaultDir);
37
+ const nodes = getNodes(vaultDir);
37
38
  logger.info('migration', `Loaded ${nodes.length} nodes for migration processing.`);
38
39
 
39
40
  // 2. Execute Migration
@@ -78,7 +79,7 @@ export async function runMigration(vaultDir, fromVersion, toVersion, migrationFn
78
79
  export async function testMigration(vaultDir, migrationFn, targetSchemaType) {
79
80
  logger.info('migration', 'Running migration harness in isolated context...');
80
81
 
81
- const nodes = loadNodes(vaultDir).slice(0, 50); // Sample a subset
82
+ const nodes = getNodes(vaultDir).slice(0, 50); // Sample a subset
82
83
 
83
84
  try {
84
85
  const testMigrated = await migrationFn(nodes);
@@ -2,7 +2,8 @@ import fs from 'fs';
2
2
  import path from 'path';
3
3
  import crypto from 'crypto';
4
4
  import matter from 'gray-matter';
5
- import { loadNodes, writeNode } from './vault.mjs';
5
+ import { writeNode } from './vault.mjs';
6
+ import { getNodes } from './vault-cache.mjs';
6
7
  import { logger } from './logger.mjs';
7
8
  import { brainDir } from './config.mjs';
8
9
 
@@ -31,7 +32,7 @@ export function createProposal(category, summary, targetPath = null, rationale =
31
32
  * Scans the vault for duplicate or contradictory memories and generates a cleanup proposal.
32
33
  */
33
34
  export async function generateMemoryCleanupProposals(vaultDir) {
34
- const nodes = loadNodes(vaultDir);
35
+ const nodes = getNodes(vaultDir);
35
36
  const proposals = [];
36
37
 
37
38
  // Simple heuristic: find nodes with the same predicate and object but different subjects, or duplicate subjects.
@@ -163,7 +164,7 @@ export async function generateWorkflowRepairProposals(vaultDir) {
163
164
  */
164
165
  export async function generateModelRoutingProposals(vaultDir) {
165
166
  const proposals = [];
166
- const nodes = loadNodes(vaultDir);
167
+ const nodes = getNodes(vaultDir);
167
168
 
168
169
  // Find high-importance facts with low confidence — likely needed frontier model
169
170
  const underconfident = nodes.filter(n =>
@@ -187,7 +188,7 @@ export async function generateModelRoutingProposals(vaultDir) {
187
188
  }
188
189
 
189
190
  export async function generateStaleKnowledgeRefreshProposals(vaultDir) {
190
- const nodes = loadNodes(vaultDir);
191
+ const nodes = getNodes(vaultDir);
191
192
  const proposals = [];
192
193
 
193
194
  const now = Date.now();
@@ -272,7 +273,7 @@ Output ONLY valid JSON.`;
272
273
  */
273
274
  export async function runSmartDecay(vaultDir, runtimeConfig, ageThresholdDays = 60) {
274
275
  const { callLocalRuntime } = await import('./runtime.mjs');
275
- const nodes = loadNodes(vaultDir);
276
+ const nodes = getNodes(vaultDir);
276
277
  const now = Date.now();
277
278
  const ageThresholdMs = ageThresholdDays * 24 * 60 * 60 * 1000;
278
279
  const proposals = [];
@@ -0,0 +1,288 @@
1
+ /**
2
+ * src/core/parallel-context.mjs
3
+ *
4
+ * Parallel Flash Context Streaming Engine
5
+ *
6
+ * 1 node = 1 stream. Massive GPU-style parallelism.
7
+ * Every node in the vault gets its own concurrent micro-Flash call.
8
+ * Each call is tiny (~100 tokens in, ~30 tokens out).
9
+ * Results stream back as they complete.
10
+ *
11
+ * Cost: 1000 nodes × ~150 tokens = 150K tokens = $0.04
12
+ * Speed: All calls fire simultaneously, bounded only by API rate limits.
13
+ *
14
+ * Architecture:
15
+ * 1. SHATTER — every node becomes its own stream
16
+ * 2. FLOOD — fire all streams concurrently (rate-limited)
17
+ * 3. COLLECT — results arrive as futures resolve
18
+ * 4. RANK — sort by score, pack into budget
19
+ * 5. DELIVER — compressed context ready for inference
20
+ */
21
+
22
+ import { getNodes } from './vault-cache.mjs';
23
+ import { googleApiKey } from './config.mjs';
24
+ import { logger } from './logger.mjs';
25
+ import { inferMemoryLayer } from './memory-layers.mjs';
26
+
27
+ // ─── Configuration ──────────────────────────────────────────────────────────
28
+
29
+ const FLASH_MODEL = 'gemini-3.1-flash-lite';
30
+ const FLASH_API_URL = 'https://generativelanguage.googleapis.com/v1beta/models';
31
+ const MAX_CONCURRENCY = 50; // concurrent HTTP connections
32
+ const FLASH_TIMEOUT_MS = 6000; // per-call timeout
33
+ const DEFAULT_BUDGET_TOKENS = 8000;
34
+ const CHARS_PER_TOKEN = 4;
35
+
36
+ // ─── Flash Micro-Call ───────────────────────────────────────────────────────
37
+
38
+ /**
39
+ * Single micro-call to Flash. Score ONE node against the query.
40
+ * Prompt is minimal — ~100 tokens in, ~30 tokens out.
41
+ */
42
+ async function scoreOne(query, node, apiKey) {
43
+ const body = (node.body || node.content || '').slice(0, 400);
44
+ const prompt = `Score relevance 0-100 and extract the key bit.
45
+ Q: "${query}"
46
+ NODE [${node.slug}] ${node.category} imp:${node.importance || 3}: ${node.title || ''} — ${body}
47
+ Reply EXACTLY: SCORE|KEY_BIT (one line, no explanation)`;
48
+
49
+ const url = `${FLASH_API_URL}/${FLASH_MODEL}:generateContent?key=${apiKey}`;
50
+
51
+ const controller = new AbortController();
52
+ const timeout = setTimeout(() => controller.abort(), FLASH_TIMEOUT_MS);
53
+
54
+ try {
55
+ const response = await fetch(url, {
56
+ method: 'POST',
57
+ headers: { 'Content-Type': 'application/json' },
58
+ signal: controller.signal,
59
+ body: JSON.stringify({
60
+ contents: [{ parts: [{ text: prompt }] }],
61
+ generationConfig: { maxOutputTokens: 128, temperature: 0.0 },
62
+ }),
63
+ });
64
+
65
+ if (!response.ok) {
66
+ // Rate limited — return fallback score
67
+ if (response.status === 429) {
68
+ return { slug: node.slug, score: node.priority === 'absolute' ? 90 : 25, compressed: body.slice(0, 150), rateLimited: true };
69
+ }
70
+ throw new Error(`${response.status}`);
71
+ }
72
+
73
+ const data = await response.json();
74
+ const text = (data.candidates?.[0]?.content?.parts?.[0]?.text || '').trim();
75
+
76
+ // Parse "SCORE|KEY_BIT"
77
+ const pipeIdx = text.indexOf('|');
78
+ if (pipeIdx > 0) {
79
+ const score = parseInt(text.slice(0, pipeIdx).trim(), 10);
80
+ const compressed = text.slice(pipeIdx + 1).trim();
81
+ if (!isNaN(score)) {
82
+ return {
83
+ slug: node.slug,
84
+ score: Math.min(Math.max(score, 0), 100),
85
+ compressed: compressed.slice(0, 300),
86
+ title: node.title,
87
+ category: node.category,
88
+ layer: inferMemoryLayer(node),
89
+ };
90
+ }
91
+ }
92
+
93
+ // Parsing failed — fallback
94
+ return { slug: node.slug, score: 30, compressed: body.slice(0, 150), title: node.title, category: node.category, parseFail: true };
95
+ } catch (err) {
96
+ // Timeout or network error — fallback
97
+ return {
98
+ slug: node.slug,
99
+ score: node.priority === 'absolute' ? 90 : 20,
100
+ compressed: body.slice(0, 150),
101
+ title: node.title,
102
+ category: node.category,
103
+ error: err.name === 'AbortError' ? 'timeout' : err.message,
104
+ };
105
+ } finally {
106
+ clearTimeout(timeout);
107
+ }
108
+ }
109
+
110
+ // ─── Flood Dispatcher ───────────────────────────────────────────────────────
111
+
112
+ /**
113
+ * Fire all streams concurrently with bounded concurrency.
114
+ * Uses a semaphore pattern — N workers pulling from a shared queue.
115
+ */
116
+ async function flood(items, fn, concurrency) {
117
+ const results = new Array(items.length);
118
+ let cursor = 0;
119
+ let completed = 0;
120
+
121
+ async function worker() {
122
+ while (cursor < items.length) {
123
+ const i = cursor++;
124
+ results[i] = await fn(items[i], i);
125
+ completed++;
126
+ }
127
+ }
128
+
129
+ const workers = Array.from(
130
+ { length: Math.min(concurrency, items.length) },
131
+ () => worker()
132
+ );
133
+ await Promise.all(workers);
134
+ return results;
135
+ }
136
+
137
+ // ─── Main Entry Point ───────────────────────────────────────────────────────
138
+
139
+ /**
140
+ * Stream context from the entire vault — 1 node = 1 stream.
141
+ *
142
+ * @param {object} opts
143
+ * @param {string} opts.query - Query to score against
144
+ * @param {string} opts.vaultDir - Vault directory
145
+ * @param {number} [opts.budgetTokens=8000] - Token budget
146
+ * @param {number} [opts.concurrency=50] - Max concurrent streams
147
+ * @param {number} [opts.minScore=20] - Minimum score to include
148
+ * @returns {Promise<{ context: string, stats: object, scored: array }>}
149
+ */
150
+ export async function streamParallelContext({
151
+ query,
152
+ vaultDir,
153
+ budgetTokens = DEFAULT_BUDGET_TOKENS,
154
+ concurrency = MAX_CONCURRENCY,
155
+ minScore = 20,
156
+ } = {}) {
157
+ const startTime = Date.now();
158
+ const allNodes = getNodes(vaultDir).filter(n => n.status === 'active');
159
+ const totalNodes = allNodes.length;
160
+
161
+ if (!query || totalNodes === 0) {
162
+ return {
163
+ context: '',
164
+ stats: { total_nodes: totalNodes, streams: 0, compile_ms: 0 },
165
+ scored: [],
166
+ };
167
+ }
168
+
169
+ const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || googleApiKey;
170
+ if (!apiKey) throw new Error('No Gemini API key available for Flash streaming');
171
+
172
+ // ── Tier 0: Invariants (instant, no LLM) ──
173
+ const invariants = allNodes.filter(n => n.priority === 'absolute');
174
+ const tier0 = invariants.map(n => ({
175
+ slug: n.slug,
176
+ score: 100,
177
+ compressed: (n.body || n.content || '').slice(0, 500),
178
+ title: n.title,
179
+ category: n.category,
180
+ layer: 'conscious',
181
+ tier: 0,
182
+ }));
183
+
184
+ // ── Tier 1: FLOOD — every non-invariant node gets its own stream ──
185
+ const scorable = allNodes.filter(n => n.priority !== 'absolute');
186
+ const effectiveConcurrency = Math.min(concurrency, scorable.length);
187
+
188
+ logger.info('parallel-context', `FLOOD: ${scorable.length} streams × ${effectiveConcurrency} concurrent (${totalNodes} total, ${invariants.length} invariants pre-loaded)`);
189
+
190
+ const tier1 = await flood(
191
+ scorable,
192
+ (node) => scoreOne(query, node, apiKey),
193
+ effectiveConcurrency,
194
+ );
195
+
196
+ const floodMs = Date.now() - startTime;
197
+
198
+ // ── MERGE: combine tiers, rank by score ──
199
+ const allScored = [...tier0, ...tier1].sort((a, b) => b.score - a.score);
200
+ const filtered = allScored.filter(r => r.score >= minScore);
201
+
202
+ // ── PACK: assemble into token budget ──
203
+ let tokenCount = 0;
204
+ const included = [];
205
+
206
+ for (const item of filtered) {
207
+ const text = `- **${item.title || item.slug}** [${item.category}]: ${item.compressed}`;
208
+ const tokens = Math.ceil(text.length / CHARS_PER_TOKEN);
209
+ if (tokenCount + tokens > budgetTokens) break;
210
+ included.push({ ...item, text, tokens });
211
+ tokenCount += tokens;
212
+ }
213
+
214
+ // ── Format by category ──
215
+ const sections = new Map();
216
+ for (const item of included) {
217
+ const section = item.category || 'general';
218
+ if (!sections.has(section)) sections.set(section, []);
219
+ sections.get(section).push(item);
220
+ }
221
+
222
+ let context = '';
223
+ for (const [section, items] of sections) {
224
+ const label = section.charAt(0).toUpperCase() + section.slice(1);
225
+ context += `## ${label}\n\n`;
226
+ for (const item of items) {
227
+ context += `${item.text}\n`;
228
+ }
229
+ context += '\n';
230
+ }
231
+
232
+ const totalMs = Date.now() - startTime;
233
+ const rateLimited = allScored.filter(r => r.rateLimited).length;
234
+ const errors = allScored.filter(r => r.error).length;
235
+ const parseFails = allScored.filter(r => r.parseFail).length;
236
+
237
+ const stats = {
238
+ total_nodes: totalNodes,
239
+ streams: scorable.length,
240
+ invariants_preloaded: invariants.length,
241
+ concurrency: effectiveConcurrency,
242
+ flood_ms: floodMs,
243
+ compile_ms: totalMs,
244
+ scored: allScored.length,
245
+ above_threshold: filtered.length,
246
+ included: included.length,
247
+ tokens_used: tokenCount,
248
+ tokens_budget: budgetTokens,
249
+ rate_limited: rateLimited,
250
+ errors,
251
+ parse_fails: parseFails,
252
+ avg_score: included.length > 0 ? Math.round(included.reduce((s, i) => s + i.score, 0) / included.length) : 0,
253
+ throughput_nodes_per_sec: Math.round((scorable.length / (floodMs / 1000)) * 10) / 10,
254
+ };
255
+
256
+ logger.info('parallel-context', `DELIVERED: ${included.length} nodes (${tokenCount} tok) from ${totalNodes} total in ${totalMs}ms | ${stats.throughput_nodes_per_sec} nodes/s | ${rateLimited} rate-limited | ${errors} errors`);
257
+
258
+ return { context: context.trim(), stats, scored: allScored };
259
+ }
260
+
261
+ /**
262
+ * Quick health check — verify Flash API connectivity.
263
+ */
264
+ export async function checkFlashHealth() {
265
+ const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || googleApiKey;
266
+ if (!apiKey) return { status: 'unhealthy', error: 'No API key' };
267
+
268
+ const url = `${FLASH_API_URL}/${FLASH_MODEL}:generateContent?key=${apiKey}`;
269
+
270
+ try {
271
+ const start = Date.now();
272
+ const response = await fetch(url, {
273
+ method: 'POST',
274
+ headers: { 'Content-Type': 'application/json' },
275
+ body: JSON.stringify({
276
+ contents: [{ parts: [{ text: 'Reply with exactly: OK' }] }],
277
+ generationConfig: { maxOutputTokens: 8, temperature: 0 },
278
+ }),
279
+ });
280
+
281
+ if (!response.ok) throw new Error(`API ${response.status}`);
282
+ const data = await response.json();
283
+ const text = data.candidates?.[0]?.content?.parts?.[0]?.text || '';
284
+ return { status: 'healthy', response: text.trim().slice(0, 20), latency_ms: Date.now() - start, model: FLASH_MODEL };
285
+ } catch (err) {
286
+ return { status: 'unhealthy', error: err.message, model: FLASH_MODEL };
287
+ }
288
+ }
@@ -3,7 +3,8 @@ import path from 'path';
3
3
  import matter from 'gray-matter';
4
4
  import crypto from 'crypto';
5
5
  import { callLocalRuntime, loadRuntimeConfig } from './runtime.mjs';
6
- import { atomicWrite, loadNodes, safeStringify } from './vault.mjs';
6
+ import { atomicWrite, safeStringify } from './vault.mjs';
7
+ import { getNodes } from './vault-cache.mjs';
7
8
  import { logger } from './logger.mjs';
8
9
  import { brainDir } from './config.mjs';
9
10
 
@@ -115,7 +116,7 @@ export async function runPostMortem(sessionPath, { vaultDir, inboxDir, runtimeCo
115
116
  }
116
117
 
117
118
  // Load active rules for violation checking
118
- const nodes = loadNodes(vaultDir);
119
+ const nodes = getNodes(vaultDir);
119
120
  const rules = nodes.filter(n =>
120
121
  n.status === 'active' &&
121
122
  n.priority === 'absolute' &&
@@ -324,7 +325,7 @@ export async function runComplianceAudit(sessionPath, { vaultDir, runtimeConfig
324
325
  return { compliant: true, violations: [], skipped: 'too-short' };
325
326
  }
326
327
 
327
- const nodes = loadNodes(vaultDir);
328
+ const nodes = getNodes(vaultDir);
328
329
  const rules = nodes.filter(n =>
329
330
  n.status === 'active' &&
330
331
  (n.modality === 'must' || n.modality === 'must_not')
@@ -13,7 +13,8 @@ import fs from 'node:fs';
13
13
  import path from 'node:path';
14
14
  import crypto from 'node:crypto';
15
15
  import matter from 'gray-matter';
16
- import { loadNodes, writeNode, deleteNode } from './vault.mjs';
16
+ import { writeNode, deleteNode } from './vault.mjs';
17
+ import { getNodes } from './vault-cache.mjs';
17
18
  import { brainDir } from './config.mjs';
18
19
 
19
20
  const getBrainDir = () => {
@@ -235,7 +236,7 @@ export function loadQueue(overrideBrainDir) {
235
236
  let node = null;
236
237
  if (item.node_slug) {
237
238
  try {
238
- const nodes = loadNodes(vaultDir);
239
+ const nodes = getNodes(vaultDir);
239
240
  node = nodes.find(n => n.slug === item.node_slug);
240
241
  if (!node) {
241
242
  const inboxFile = path.join(effectiveBrainDir, 'memory-inbox', 'pending', `${item.node_slug}.md`);
@@ -384,7 +385,7 @@ export function updateQueueItem(id, patch = {}, overrideBrainDir) {
384
385
  let node = null;
385
386
  if (item.node_slug) {
386
387
  try {
387
- const nodes = loadNodes(vaultDir);
388
+ const nodes = getNodes(vaultDir);
388
389
  node = nodes.find(n => n.slug === item.node_slug);
389
390
 
390
391
  if (!node) {
@@ -43,7 +43,6 @@ export function findBinaryInPath(binaryName) {
43
43
 
44
44
  const DEFAULT_AGENTS = [
45
45
  { name: 'antigravity', binary: 'antigravity', flags: '--sandbox=false --yolo -o json', priority: 1, enabled: true, exec: 'flag' },
46
- { name: 'gemini', binary: 'gemini', flags: '--sandbox=false --yolo -o json', priority: 2, enabled: true, exec: 'flag' },
47
46
  { name: 'claude', binary: 'claude', flags: '--output-format json --permission-mode bypassPermissions', priority: 3, enabled: true, exec: 'flag' },
48
47
  { name: 'codex', binary: 'codex', flags: '--full-auto --json', priority: 4, enabled: true, exec: 'subcommand' },
49
48
  ];
@@ -290,7 +289,8 @@ export async function callLocalRuntime(prompt, system, config) {
290
289
  maxBuffer: 10 * 1024 * 1024,
291
290
  env: {
292
291
  ...process.env,
293
- GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || googleApiKey,
292
+ GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY || googleApiKey,
293
+ GEMINI_API_KEY: process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || googleApiKey,
294
294
  TAVILY_API_KEY: process.env.TAVILY_API_KEY || tavilyApiKey,
295
295
  BRAVE_API_KEY: process.env.BRAVE_API_KEY || braveApiKey,
296
296
  EXA_API_KEY: process.env.EXA_API_KEY || exaApiKey,