total-recall-brain 3.2.3 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/extension/background.js +14 -2
- package/extension/content-script.js +18 -1
- package/extension/lib/brain-client.js +86 -52
- package/extension/lib/preconfigured.js +5 -0
- package/extension/manifest.json +1 -1
- package/extension/options/options.html +1 -0
- package/extension/options/options.js +12 -6
- package/extension/popup/popup.html +1 -0
- package/extension/popup/popup.js +29 -4
- package/extension/sidepanel/sidepanel.html +1 -0
- package/extension/sidepanel/sidepanel.js +24 -6
- package/frontend/index.html +8 -0
- package/frontend/src/api.ts +1 -2
- package/frontend/src/pages/ChatPage.tsx +9 -6
- package/frontend/src/pages/IntegrationsPage.tsx +144 -3
- package/package.json +2 -2
- package/scaffold/.agent/skills/total-recall/SKILL.md +55 -6
- package/scaffold/.agent/skills/total-recall/memory-vault/invariants/operating-instructions.md +2 -2
- package/scaffold/.agent/skills/total-recall/skills/cli-agents/agents.yml +1 -6
- package/scaffold/.agent/skills/total-recall/skills/ssss/SKILL.md +19 -0
- package/scaffold/.agent/skills/total-recall/skills/ssss/references/ssss-spec.md +33 -5
- package/src/cli/forget.mjs +18 -12
- package/src/cli/ingest/index.mjs +3 -2
- package/src/cli/init.mjs +30 -0
- package/src/cli/map.mjs +2 -2
- package/src/cli/rebuild.mjs +1 -1
- package/src/cli/remember.mjs +21 -11
- package/src/cli/setup.mjs +10 -1
- package/src/cli/share.mjs +7 -2
- package/src/core/append-log.mjs +131 -0
- package/src/core/clarity-rewriter.mjs +10 -6
- package/src/core/conclusion-writer.mjs +3 -2
- package/src/core/config.mjs +6 -0
- package/src/core/conflict-detector.mjs +4 -2
- package/src/core/context-compiler.mjs +412 -0
- package/src/core/daemon-loop.mjs +15 -0
- package/src/core/dream.mjs +4 -4
- package/src/core/drift-detector.mjs +3 -3
- package/src/core/embeddings.mjs +60 -10
- package/src/core/fact-seeker.mjs +70 -44
- package/src/core/import-rules.mjs +3 -2
- package/src/core/inference-engine.mjs +5 -4
- package/src/core/migrate.mjs +4 -3
- package/src/core/optimizer.mjs +6 -5
- package/src/core/parallel-context.mjs +288 -0
- package/src/core/post-mortem.mjs +4 -3
- package/src/core/research-queue.mjs +4 -3
- package/src/core/runtime.mjs +2 -2
- package/src/core/sandbox.mjs +56 -26
- package/src/core/scheduler.mjs +6 -5
- package/src/core/schema.mjs +2 -2
- package/src/core/search.mjs +167 -101
- package/src/core/surface.mjs +203 -53
- package/src/core/validated-write.mjs +100 -0
- package/src/core/vault-cache.mjs +46 -18
- package/src/core/vault.mjs +17 -29
- package/src/core/vector-field.mjs +659 -0
- package/src/core/vector-store.mjs +85 -0
- package/src/server/api.mjs +4 -3
- package/src/server/index.mjs +13 -61
- package/src/server/rest.mjs +238 -29
- package/src/server/route-manifest.json +344 -77
- package/src/server/routes/memory.mjs +85 -39
- package/src/server/routes/sessions.mjs +6 -2
- package/src/server/routes/share.mjs +9 -2
package/src/core/fact-seeker.mjs
CHANGED
|
@@ -4,7 +4,8 @@ import matter from 'gray-matter';
|
|
|
4
4
|
import crypto from 'crypto';
|
|
5
5
|
import os from 'os';
|
|
6
6
|
import { callLocalRuntime, cleanAndParseJSON } from './runtime.mjs';
|
|
7
|
-
import {
|
|
7
|
+
import { writeNode, atomicWrite, safeStringify } from './vault.mjs';
|
|
8
|
+
import { getNodes } from './vault-cache.mjs';
|
|
8
9
|
import { logger } from './logger.mjs';
|
|
9
10
|
import { addToQueue } from './research-queue.mjs';
|
|
10
11
|
import { persistTaskToDisk } from './scheduler.mjs';
|
|
@@ -285,62 +286,87 @@ async function gatherFromSources(topicEntry, researchConfig) {
|
|
|
285
286
|
const useGithub = (suggested_sources || []).includes('github') ||
|
|
286
287
|
(tags || []).some(t => ['code', 'library', 'framework', 'open-source'].includes(t));
|
|
287
288
|
|
|
289
|
+
const tasks = [];
|
|
290
|
+
|
|
288
291
|
// 1. Web search — Brave → Serper fallback, automatically selected
|
|
289
292
|
if (availability.available.includes('brave-search') || availability.available.includes('serper')) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
293
|
+
tasks.push((async () => {
|
|
294
|
+
try {
|
|
295
|
+
const webResults = await webSearch(topic, researchConfig, 5);
|
|
296
|
+
logger.info({ subsystem: 'fact-seeker', message: `Web search: ${webResults.length} results for "${topic}"` });
|
|
297
|
+
return { success: true, results: webResults };
|
|
298
|
+
} catch (err) {
|
|
299
|
+
logger.info({ subsystem: 'fact-seeker', message: `Web search failed: ${err.message}` });
|
|
300
|
+
return { success: false, source: 'web-search', error: err.message };
|
|
301
|
+
}
|
|
302
|
+
})());
|
|
298
303
|
}
|
|
299
304
|
|
|
300
305
|
// 2. DuckDuckGo Instant Answers (always available, good for definitions/facts)
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
306
|
+
tasks.push((async () => {
|
|
307
|
+
try {
|
|
308
|
+
const ddgResult = await duckduckgoInstant(topic, researchConfig);
|
|
309
|
+
return { success: true, results: ddgResult ? [ddgResult] : [] };
|
|
310
|
+
} catch (err) {
|
|
311
|
+
return { success: false, source: 'duckduckgo', error: err.message };
|
|
312
|
+
}
|
|
313
|
+
})());
|
|
307
314
|
|
|
308
315
|
// 3. Wikipedia (always available, high-quality structured knowledge)
|
|
309
|
-
|
|
310
|
-
const wikiResult = await wikipediaFetch(topic, researchConfig);
|
|
311
|
-
if (wikiResult) results.push(wikiResult);
|
|
312
|
-
} catch (err) {
|
|
313
|
-
errors.push({ source: 'wikipedia', error: err.message });
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
// 4. arXiv (for academic/research topics)
|
|
317
|
-
if (useArxiv) {
|
|
316
|
+
tasks.push((async () => {
|
|
318
317
|
try {
|
|
319
|
-
const
|
|
320
|
-
results
|
|
321
|
-
logger.info({ subsystem: 'fact-seeker', message: `arXiv: ${arxivResults.length} papers for "${topic}"` });
|
|
318
|
+
const wikiResult = await wikipediaFetch(topic, researchConfig);
|
|
319
|
+
return { success: true, results: wikiResult ? [wikiResult] : [] };
|
|
322
320
|
} catch (err) {
|
|
323
|
-
|
|
321
|
+
return { success: false, source: 'wikipedia', error: err.message };
|
|
324
322
|
}
|
|
323
|
+
})());
|
|
324
|
+
|
|
325
|
+
// 4. arXiv (for academic/research topics)
|
|
326
|
+
if (useArxiv) {
|
|
327
|
+
tasks.push((async () => {
|
|
328
|
+
try {
|
|
329
|
+
const arxivResults = await arxivSearch(topic, researchConfig, 3);
|
|
330
|
+
logger.info({ subsystem: 'fact-seeker', message: `arXiv: ${arxivResults.length} papers for "${topic}"` });
|
|
331
|
+
return { success: true, results: arxivResults };
|
|
332
|
+
} catch (err) {
|
|
333
|
+
return { success: false, source: 'arxiv', error: err.message };
|
|
334
|
+
}
|
|
335
|
+
})());
|
|
325
336
|
}
|
|
326
337
|
|
|
327
338
|
// 5. npm (for JS/Node topics)
|
|
328
339
|
if (useNpm) {
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
340
|
+
tasks.push((async () => {
|
|
341
|
+
try {
|
|
342
|
+
const npmResults = await npmSearch(topic, researchConfig, 3);
|
|
343
|
+
return { success: true, results: npmResults };
|
|
344
|
+
} catch (err) {
|
|
345
|
+
return { success: false, source: 'npm', error: err.message };
|
|
346
|
+
}
|
|
347
|
+
})());
|
|
335
348
|
}
|
|
336
349
|
|
|
337
350
|
// 6. GitHub (for code/library topics)
|
|
338
351
|
if (useGithub) {
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
352
|
+
tasks.push((async () => {
|
|
353
|
+
try {
|
|
354
|
+
const ghResults = await githubSearch(topic, researchConfig, 'repositories', 3);
|
|
355
|
+
return { success: true, results: ghResults };
|
|
356
|
+
} catch (err) {
|
|
357
|
+
return { success: false, source: 'github', error: err.message };
|
|
358
|
+
}
|
|
359
|
+
})());
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// Await all tasks concurrently in parallel
|
|
363
|
+
const taskResults = await Promise.all(tasks);
|
|
364
|
+
|
|
365
|
+
for (const res of taskResults) {
|
|
366
|
+
if (res.success) {
|
|
367
|
+
results.push(...res.results);
|
|
368
|
+
} else {
|
|
369
|
+
errors.push({ source: res.source, error: res.error });
|
|
344
370
|
}
|
|
345
371
|
}
|
|
346
372
|
|
|
@@ -730,7 +756,7 @@ Output ONLY valid JSON.`;
|
|
|
730
756
|
* Evaluates coverage, temporal awareness, source diversity, and limitations.
|
|
731
757
|
*/
|
|
732
758
|
export async function runSelfDiagnosis({ vaultDir, runtimeConfig }) {
|
|
733
|
-
const nodes =
|
|
759
|
+
const nodes = getNodes(vaultDir);
|
|
734
760
|
const activeNodes = nodes.filter(n => n.status === 'active');
|
|
735
761
|
const agenda = loadAgenda();
|
|
736
762
|
const availability = checkSourceAvailability(loadResearchConfig());
|
|
@@ -994,7 +1020,7 @@ async function pushDeliberationConclusion(conclusions = []) {
|
|
|
994
1020
|
* Phase 2/5: Deliberation
|
|
995
1021
|
*/
|
|
996
1022
|
export async function runResearchDeliberationCycle({ vaultDir, nodeSlug, topic, runtimeConfig }) {
|
|
997
|
-
const nodes =
|
|
1023
|
+
const nodes = getNodes(vaultDir);
|
|
998
1024
|
const targetNode = nodes.find(n => n.slug === nodeSlug);
|
|
999
1025
|
if (!targetNode) {
|
|
1000
1026
|
return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
|
|
@@ -1309,7 +1335,7 @@ Output ONLY valid JSON.`;
|
|
|
1309
1335
|
* Phase 3/5: Refinement & Clarity Auditing
|
|
1310
1336
|
*/
|
|
1311
1337
|
export async function runResearchImprovementCycle({ vaultDir, nodeSlug, topic, runtimeConfig }) {
|
|
1312
|
-
const nodes =
|
|
1338
|
+
const nodes = getNodes(vaultDir);
|
|
1313
1339
|
const targetNode = nodes.find(n => n.slug === nodeSlug);
|
|
1314
1340
|
if (!targetNode) {
|
|
1315
1341
|
return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
|
|
@@ -1384,7 +1410,7 @@ export async function runResearchMonitoringCycle({
|
|
|
1384
1410
|
derivedDir,
|
|
1385
1411
|
instructionsFile
|
|
1386
1412
|
}) {
|
|
1387
|
-
const nodes =
|
|
1413
|
+
const nodes = getNodes(vaultDir);
|
|
1388
1414
|
const targetNode = nodes.find(n => n.slug === nodeSlug);
|
|
1389
1415
|
if (!targetNode) {
|
|
1390
1416
|
return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
|
|
@@ -1490,7 +1516,7 @@ Output ONLY valid JSON.`;
|
|
|
1490
1516
|
* Phase 5/5: Domain Tangents Expansion
|
|
1491
1517
|
*/
|
|
1492
1518
|
export async function runResearchExpansionCycle({ vaultDir, nodeSlug, topic, runtimeConfig }) {
|
|
1493
|
-
const nodes =
|
|
1519
|
+
const nodes = getNodes(vaultDir);
|
|
1494
1520
|
const targetNode = nodes.find(n => n.slug === nodeSlug);
|
|
1495
1521
|
if (!targetNode) {
|
|
1496
1522
|
return { success: false, error: `Target node not found for slug: ${nodeSlug}` };
|
|
@@ -14,7 +14,8 @@ import fs from 'node:fs';
|
|
|
14
14
|
import path from 'node:path';
|
|
15
15
|
import os from 'node:os';
|
|
16
16
|
import crypto from 'node:crypto';
|
|
17
|
-
import { writeNode
|
|
17
|
+
import { writeNode } from './vault.mjs';
|
|
18
|
+
import { getNodes } from './vault-cache.mjs';
|
|
18
19
|
import { agentDir, brainDir } from './config.mjs';
|
|
19
20
|
|
|
20
21
|
const AGENT_DIR = agentDir;
|
|
@@ -75,7 +76,7 @@ export function detectRuleFiles(dirs) {
|
|
|
75
76
|
// Load existing vault nodes to detect already-imported files
|
|
76
77
|
let existingSlugs = new Set();
|
|
77
78
|
try {
|
|
78
|
-
existingSlugs = new Set(
|
|
79
|
+
existingSlugs = new Set(getNodes(VAULT_DIR).map(n => n.slug));
|
|
79
80
|
} catch { /* vault may not exist yet */ }
|
|
80
81
|
|
|
81
82
|
for (const dir of searchDirs) {
|
|
@@ -3,7 +3,8 @@ import path from 'path';
|
|
|
3
3
|
import matter from 'gray-matter';
|
|
4
4
|
import crypto from 'crypto';
|
|
5
5
|
import { callLocalRuntime, cleanAndParseJSON } from './runtime.mjs';
|
|
6
|
-
import {
|
|
6
|
+
import { atomicWrite, safeStringify } from './vault.mjs';
|
|
7
|
+
import { getNodes } from './vault-cache.mjs';
|
|
7
8
|
import { logger } from './logger.mjs';
|
|
8
9
|
import { brainDir } from './config.mjs';
|
|
9
10
|
|
|
@@ -83,7 +84,7 @@ function buildInferencePrompt(nodes) {
|
|
|
83
84
|
* @returns {{ conclusions: object[], contradictions: object[], merge_candidates: object[], error? }}
|
|
84
85
|
*/
|
|
85
86
|
export async function runInferenceTask(slugs, { vaultDir, inboxDir, runtimeConfig }) {
|
|
86
|
-
const allNodes =
|
|
87
|
+
const allNodes = getNodes(vaultDir);
|
|
87
88
|
const nodes = allNodes.filter(n => slugs.includes(n.slug) && n.status === 'active');
|
|
88
89
|
|
|
89
90
|
if (nodes.length < 2) {
|
|
@@ -154,7 +155,7 @@ function writeConclusionNode(inboxDir, conclusion, sourceSlugList, vaultDir) {
|
|
|
154
155
|
|
|
155
156
|
if (vaultDir) {
|
|
156
157
|
try {
|
|
157
|
-
const allNodes =
|
|
158
|
+
const allNodes = getNodes(vaultDir);
|
|
158
159
|
citations = sourceSlugList.map(s => {
|
|
159
160
|
const matchingNode = allNodes.find(n => n.slug === s);
|
|
160
161
|
const cat = matchingNode ? matchingNode.category : 'facts';
|
|
@@ -312,7 +313,7 @@ Output valid JSON:
|
|
|
312
313
|
* @param {object} opts.runtimeConfig
|
|
313
314
|
*/
|
|
314
315
|
export async function runSynthesisTask(slugA, slugB, { vaultDir, inboxDir, runtimeConfig }) {
|
|
315
|
-
const allNodes =
|
|
316
|
+
const allNodes = getNodes(vaultDir);
|
|
316
317
|
const nodeA = allNodes.find(n => n.slug === slugA);
|
|
317
318
|
const nodeB = allNodes.find(n => n.slug === slugB);
|
|
318
319
|
|
package/src/core/migrate.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import crypto from 'crypto';
|
|
4
|
-
import {
|
|
4
|
+
import { writeNode, atomicWrite } from './vault.mjs';
|
|
5
|
+
import { getNodes } from './vault-cache.mjs';
|
|
5
6
|
import { createSnapshot } from './snapshot.mjs';
|
|
6
7
|
import { SSSS_SCHEMAS } from './schema.mjs';
|
|
7
8
|
import { logger } from './logger.mjs';
|
|
@@ -33,7 +34,7 @@ export async function runMigration(vaultDir, fromVersion, toVersion, migrationFn
|
|
|
33
34
|
};
|
|
34
35
|
|
|
35
36
|
try {
|
|
36
|
-
const nodes =
|
|
37
|
+
const nodes = getNodes(vaultDir);
|
|
37
38
|
logger.info('migration', `Loaded ${nodes.length} nodes for migration processing.`);
|
|
38
39
|
|
|
39
40
|
// 2. Execute Migration
|
|
@@ -78,7 +79,7 @@ export async function runMigration(vaultDir, fromVersion, toVersion, migrationFn
|
|
|
78
79
|
export async function testMigration(vaultDir, migrationFn, targetSchemaType) {
|
|
79
80
|
logger.info('migration', 'Running migration harness in isolated context...');
|
|
80
81
|
|
|
81
|
-
const nodes =
|
|
82
|
+
const nodes = getNodes(vaultDir).slice(0, 50); // Sample a subset
|
|
82
83
|
|
|
83
84
|
try {
|
|
84
85
|
const testMigrated = await migrationFn(nodes);
|
package/src/core/optimizer.mjs
CHANGED
|
@@ -2,7 +2,8 @@ import fs from 'fs';
|
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import crypto from 'crypto';
|
|
4
4
|
import matter from 'gray-matter';
|
|
5
|
-
import {
|
|
5
|
+
import { writeNode } from './vault.mjs';
|
|
6
|
+
import { getNodes } from './vault-cache.mjs';
|
|
6
7
|
import { logger } from './logger.mjs';
|
|
7
8
|
import { brainDir } from './config.mjs';
|
|
8
9
|
|
|
@@ -31,7 +32,7 @@ export function createProposal(category, summary, targetPath = null, rationale =
|
|
|
31
32
|
* Scans the vault for duplicate or contradictory memories and generates a cleanup proposal.
|
|
32
33
|
*/
|
|
33
34
|
export async function generateMemoryCleanupProposals(vaultDir) {
|
|
34
|
-
const nodes =
|
|
35
|
+
const nodes = getNodes(vaultDir);
|
|
35
36
|
const proposals = [];
|
|
36
37
|
|
|
37
38
|
// Simple heuristic: find nodes with the same predicate and object but different subjects, or duplicate subjects.
|
|
@@ -163,7 +164,7 @@ export async function generateWorkflowRepairProposals(vaultDir) {
|
|
|
163
164
|
*/
|
|
164
165
|
export async function generateModelRoutingProposals(vaultDir) {
|
|
165
166
|
const proposals = [];
|
|
166
|
-
const nodes =
|
|
167
|
+
const nodes = getNodes(vaultDir);
|
|
167
168
|
|
|
168
169
|
// Find high-importance facts with low confidence — likely needed frontier model
|
|
169
170
|
const underconfident = nodes.filter(n =>
|
|
@@ -187,7 +188,7 @@ export async function generateModelRoutingProposals(vaultDir) {
|
|
|
187
188
|
}
|
|
188
189
|
|
|
189
190
|
export async function generateStaleKnowledgeRefreshProposals(vaultDir) {
|
|
190
|
-
const nodes =
|
|
191
|
+
const nodes = getNodes(vaultDir);
|
|
191
192
|
const proposals = [];
|
|
192
193
|
|
|
193
194
|
const now = Date.now();
|
|
@@ -272,7 +273,7 @@ Output ONLY valid JSON.`;
|
|
|
272
273
|
*/
|
|
273
274
|
export async function runSmartDecay(vaultDir, runtimeConfig, ageThresholdDays = 60) {
|
|
274
275
|
const { callLocalRuntime } = await import('./runtime.mjs');
|
|
275
|
-
const nodes =
|
|
276
|
+
const nodes = getNodes(vaultDir);
|
|
276
277
|
const now = Date.now();
|
|
277
278
|
const ageThresholdMs = ageThresholdDays * 24 * 60 * 60 * 1000;
|
|
278
279
|
const proposals = [];
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/core/parallel-context.mjs
|
|
3
|
+
*
|
|
4
|
+
* Parallel Flash Context Streaming Engine
|
|
5
|
+
*
|
|
6
|
+
* 1 node = 1 stream. Massive GPU-style parallelism.
|
|
7
|
+
* Every node in the vault gets its own concurrent micro-Flash call.
|
|
8
|
+
* Each call is tiny (~100 tokens in, ~30 tokens out).
|
|
9
|
+
* Results stream back as they complete.
|
|
10
|
+
*
|
|
11
|
+
* Cost: 1000 nodes × ~150 tokens = 150K tokens = $0.04
|
|
12
|
+
* Speed: All calls fire simultaneously, bounded only by API rate limits.
|
|
13
|
+
*
|
|
14
|
+
* Architecture:
|
|
15
|
+
* 1. SHATTER — every node becomes its own stream
|
|
16
|
+
* 2. FLOOD — fire all streams concurrently (rate-limited)
|
|
17
|
+
* 3. COLLECT — results arrive as futures resolve
|
|
18
|
+
* 4. RANK — sort by score, pack into budget
|
|
19
|
+
* 5. DELIVER — compressed context ready for inference
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { getNodes } from './vault-cache.mjs';
|
|
23
|
+
import { googleApiKey } from './config.mjs';
|
|
24
|
+
import { logger } from './logger.mjs';
|
|
25
|
+
import { inferMemoryLayer } from './memory-layers.mjs';
|
|
26
|
+
|
|
27
|
+
// ─── Configuration ──────────────────────────────────────────────────────────
|
|
28
|
+
|
|
29
|
+
const FLASH_MODEL = 'gemini-3.1-flash-lite';
|
|
30
|
+
const FLASH_API_URL = 'https://generativelanguage.googleapis.com/v1beta/models';
|
|
31
|
+
const MAX_CONCURRENCY = 50; // concurrent HTTP connections
|
|
32
|
+
const FLASH_TIMEOUT_MS = 6000; // per-call timeout
|
|
33
|
+
const DEFAULT_BUDGET_TOKENS = 8000;
|
|
34
|
+
const CHARS_PER_TOKEN = 4;
|
|
35
|
+
|
|
36
|
+
// ─── Flash Micro-Call ───────────────────────────────────────────────────────
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Single micro-call to Flash. Score ONE node against the query.
|
|
40
|
+
* Prompt is minimal — ~100 tokens in, ~30 tokens out.
|
|
41
|
+
*/
|
|
42
|
+
async function scoreOne(query, node, apiKey) {
|
|
43
|
+
const body = (node.body || node.content || '').slice(0, 400);
|
|
44
|
+
const prompt = `Score relevance 0-100 and extract the key bit.
|
|
45
|
+
Q: "${query}"
|
|
46
|
+
NODE [${node.slug}] ${node.category} imp:${node.importance || 3}: ${node.title || ''} — ${body}
|
|
47
|
+
Reply EXACTLY: SCORE|KEY_BIT (one line, no explanation)`;
|
|
48
|
+
|
|
49
|
+
const url = `${FLASH_API_URL}/${FLASH_MODEL}:generateContent?key=${apiKey}`;
|
|
50
|
+
|
|
51
|
+
const controller = new AbortController();
|
|
52
|
+
const timeout = setTimeout(() => controller.abort(), FLASH_TIMEOUT_MS);
|
|
53
|
+
|
|
54
|
+
try {
|
|
55
|
+
const response = await fetch(url, {
|
|
56
|
+
method: 'POST',
|
|
57
|
+
headers: { 'Content-Type': 'application/json' },
|
|
58
|
+
signal: controller.signal,
|
|
59
|
+
body: JSON.stringify({
|
|
60
|
+
contents: [{ parts: [{ text: prompt }] }],
|
|
61
|
+
generationConfig: { maxOutputTokens: 128, temperature: 0.0 },
|
|
62
|
+
}),
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
if (!response.ok) {
|
|
66
|
+
// Rate limited — return fallback score
|
|
67
|
+
if (response.status === 429) {
|
|
68
|
+
return { slug: node.slug, score: node.priority === 'absolute' ? 90 : 25, compressed: body.slice(0, 150), rateLimited: true };
|
|
69
|
+
}
|
|
70
|
+
throw new Error(`${response.status}`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const data = await response.json();
|
|
74
|
+
const text = (data.candidates?.[0]?.content?.parts?.[0]?.text || '').trim();
|
|
75
|
+
|
|
76
|
+
// Parse "SCORE|KEY_BIT"
|
|
77
|
+
const pipeIdx = text.indexOf('|');
|
|
78
|
+
if (pipeIdx > 0) {
|
|
79
|
+
const score = parseInt(text.slice(0, pipeIdx).trim(), 10);
|
|
80
|
+
const compressed = text.slice(pipeIdx + 1).trim();
|
|
81
|
+
if (!isNaN(score)) {
|
|
82
|
+
return {
|
|
83
|
+
slug: node.slug,
|
|
84
|
+
score: Math.min(Math.max(score, 0), 100),
|
|
85
|
+
compressed: compressed.slice(0, 300),
|
|
86
|
+
title: node.title,
|
|
87
|
+
category: node.category,
|
|
88
|
+
layer: inferMemoryLayer(node),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Parsing failed — fallback
|
|
94
|
+
return { slug: node.slug, score: 30, compressed: body.slice(0, 150), title: node.title, category: node.category, parseFail: true };
|
|
95
|
+
} catch (err) {
|
|
96
|
+
// Timeout or network error — fallback
|
|
97
|
+
return {
|
|
98
|
+
slug: node.slug,
|
|
99
|
+
score: node.priority === 'absolute' ? 90 : 20,
|
|
100
|
+
compressed: body.slice(0, 150),
|
|
101
|
+
title: node.title,
|
|
102
|
+
category: node.category,
|
|
103
|
+
error: err.name === 'AbortError' ? 'timeout' : err.message,
|
|
104
|
+
};
|
|
105
|
+
} finally {
|
|
106
|
+
clearTimeout(timeout);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// ─── Flood Dispatcher ───────────────────────────────────────────────────────
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Fire all streams concurrently with bounded concurrency.
|
|
114
|
+
* Uses a semaphore pattern — N workers pulling from a shared queue.
|
|
115
|
+
*/
|
|
116
|
+
async function flood(items, fn, concurrency) {
|
|
117
|
+
const results = new Array(items.length);
|
|
118
|
+
let cursor = 0;
|
|
119
|
+
let completed = 0;
|
|
120
|
+
|
|
121
|
+
async function worker() {
|
|
122
|
+
while (cursor < items.length) {
|
|
123
|
+
const i = cursor++;
|
|
124
|
+
results[i] = await fn(items[i], i);
|
|
125
|
+
completed++;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const workers = Array.from(
|
|
130
|
+
{ length: Math.min(concurrency, items.length) },
|
|
131
|
+
() => worker()
|
|
132
|
+
);
|
|
133
|
+
await Promise.all(workers);
|
|
134
|
+
return results;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// ─── Main Entry Point ───────────────────────────────────────────────────────
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Stream context from the entire vault — 1 node = 1 stream.
|
|
141
|
+
*
|
|
142
|
+
* @param {object} opts
|
|
143
|
+
* @param {string} opts.query - Query to score against
|
|
144
|
+
* @param {string} opts.vaultDir - Vault directory
|
|
145
|
+
* @param {number} [opts.budgetTokens=8000] - Token budget
|
|
146
|
+
* @param {number} [opts.concurrency=50] - Max concurrent streams
|
|
147
|
+
* @param {number} [opts.minScore=20] - Minimum score to include
|
|
148
|
+
* @returns {Promise<{ context: string, stats: object, scored: array }>}
|
|
149
|
+
*/
|
|
150
|
+
export async function streamParallelContext({
|
|
151
|
+
query,
|
|
152
|
+
vaultDir,
|
|
153
|
+
budgetTokens = DEFAULT_BUDGET_TOKENS,
|
|
154
|
+
concurrency = MAX_CONCURRENCY,
|
|
155
|
+
minScore = 20,
|
|
156
|
+
} = {}) {
|
|
157
|
+
const startTime = Date.now();
|
|
158
|
+
const allNodes = getNodes(vaultDir).filter(n => n.status === 'active');
|
|
159
|
+
const totalNodes = allNodes.length;
|
|
160
|
+
|
|
161
|
+
if (!query || totalNodes === 0) {
|
|
162
|
+
return {
|
|
163
|
+
context: '',
|
|
164
|
+
stats: { total_nodes: totalNodes, streams: 0, compile_ms: 0 },
|
|
165
|
+
scored: [],
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || googleApiKey;
|
|
170
|
+
if (!apiKey) throw new Error('No Gemini API key available for Flash streaming');
|
|
171
|
+
|
|
172
|
+
// ── Tier 0: Invariants (instant, no LLM) ──
|
|
173
|
+
const invariants = allNodes.filter(n => n.priority === 'absolute');
|
|
174
|
+
const tier0 = invariants.map(n => ({
|
|
175
|
+
slug: n.slug,
|
|
176
|
+
score: 100,
|
|
177
|
+
compressed: (n.body || n.content || '').slice(0, 500),
|
|
178
|
+
title: n.title,
|
|
179
|
+
category: n.category,
|
|
180
|
+
layer: 'conscious',
|
|
181
|
+
tier: 0,
|
|
182
|
+
}));
|
|
183
|
+
|
|
184
|
+
// ── Tier 1: FLOOD — every non-invariant node gets its own stream ──
|
|
185
|
+
const scorable = allNodes.filter(n => n.priority !== 'absolute');
|
|
186
|
+
const effectiveConcurrency = Math.min(concurrency, scorable.length);
|
|
187
|
+
|
|
188
|
+
logger.info('parallel-context', `FLOOD: ${scorable.length} streams × ${effectiveConcurrency} concurrent (${totalNodes} total, ${invariants.length} invariants pre-loaded)`);
|
|
189
|
+
|
|
190
|
+
const tier1 = await flood(
|
|
191
|
+
scorable,
|
|
192
|
+
(node) => scoreOne(query, node, apiKey),
|
|
193
|
+
effectiveConcurrency,
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
const floodMs = Date.now() - startTime;
|
|
197
|
+
|
|
198
|
+
// ── MERGE: combine tiers, rank by score ──
|
|
199
|
+
const allScored = [...tier0, ...tier1].sort((a, b) => b.score - a.score);
|
|
200
|
+
const filtered = allScored.filter(r => r.score >= minScore);
|
|
201
|
+
|
|
202
|
+
// ── PACK: assemble into token budget ──
|
|
203
|
+
let tokenCount = 0;
|
|
204
|
+
const included = [];
|
|
205
|
+
|
|
206
|
+
for (const item of filtered) {
|
|
207
|
+
const text = `- **${item.title || item.slug}** [${item.category}]: ${item.compressed}`;
|
|
208
|
+
const tokens = Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
209
|
+
if (tokenCount + tokens > budgetTokens) break;
|
|
210
|
+
included.push({ ...item, text, tokens });
|
|
211
|
+
tokenCount += tokens;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// ── Format by category ──
|
|
215
|
+
const sections = new Map();
|
|
216
|
+
for (const item of included) {
|
|
217
|
+
const section = item.category || 'general';
|
|
218
|
+
if (!sections.has(section)) sections.set(section, []);
|
|
219
|
+
sections.get(section).push(item);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
let context = '';
|
|
223
|
+
for (const [section, items] of sections) {
|
|
224
|
+
const label = section.charAt(0).toUpperCase() + section.slice(1);
|
|
225
|
+
context += `## ${label}\n\n`;
|
|
226
|
+
for (const item of items) {
|
|
227
|
+
context += `${item.text}\n`;
|
|
228
|
+
}
|
|
229
|
+
context += '\n';
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const totalMs = Date.now() - startTime;
|
|
233
|
+
const rateLimited = allScored.filter(r => r.rateLimited).length;
|
|
234
|
+
const errors = allScored.filter(r => r.error).length;
|
|
235
|
+
const parseFails = allScored.filter(r => r.parseFail).length;
|
|
236
|
+
|
|
237
|
+
const stats = {
|
|
238
|
+
total_nodes: totalNodes,
|
|
239
|
+
streams: scorable.length,
|
|
240
|
+
invariants_preloaded: invariants.length,
|
|
241
|
+
concurrency: effectiveConcurrency,
|
|
242
|
+
flood_ms: floodMs,
|
|
243
|
+
compile_ms: totalMs,
|
|
244
|
+
scored: allScored.length,
|
|
245
|
+
above_threshold: filtered.length,
|
|
246
|
+
included: included.length,
|
|
247
|
+
tokens_used: tokenCount,
|
|
248
|
+
tokens_budget: budgetTokens,
|
|
249
|
+
rate_limited: rateLimited,
|
|
250
|
+
errors,
|
|
251
|
+
parse_fails: parseFails,
|
|
252
|
+
avg_score: included.length > 0 ? Math.round(included.reduce((s, i) => s + i.score, 0) / included.length) : 0,
|
|
253
|
+
throughput_nodes_per_sec: Math.round((scorable.length / (floodMs / 1000)) * 10) / 10,
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
logger.info('parallel-context', `DELIVERED: ${included.length} nodes (${tokenCount} tok) from ${totalNodes} total in ${totalMs}ms | ${stats.throughput_nodes_per_sec} nodes/s | ${rateLimited} rate-limited | ${errors} errors`);
|
|
257
|
+
|
|
258
|
+
return { context: context.trim(), stats, scored: allScored };
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Quick health check — verify Flash API connectivity.
|
|
263
|
+
*/
|
|
264
|
+
export async function checkFlashHealth() {
|
|
265
|
+
const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || googleApiKey;
|
|
266
|
+
if (!apiKey) return { status: 'unhealthy', error: 'No API key' };
|
|
267
|
+
|
|
268
|
+
const url = `${FLASH_API_URL}/${FLASH_MODEL}:generateContent?key=${apiKey}`;
|
|
269
|
+
|
|
270
|
+
try {
|
|
271
|
+
const start = Date.now();
|
|
272
|
+
const response = await fetch(url, {
|
|
273
|
+
method: 'POST',
|
|
274
|
+
headers: { 'Content-Type': 'application/json' },
|
|
275
|
+
body: JSON.stringify({
|
|
276
|
+
contents: [{ parts: [{ text: 'Reply with exactly: OK' }] }],
|
|
277
|
+
generationConfig: { maxOutputTokens: 8, temperature: 0 },
|
|
278
|
+
}),
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
if (!response.ok) throw new Error(`API ${response.status}`);
|
|
282
|
+
const data = await response.json();
|
|
283
|
+
const text = data.candidates?.[0]?.content?.parts?.[0]?.text || '';
|
|
284
|
+
return { status: 'healthy', response: text.trim().slice(0, 20), latency_ms: Date.now() - start, model: FLASH_MODEL };
|
|
285
|
+
} catch (err) {
|
|
286
|
+
return { status: 'unhealthy', error: err.message, model: FLASH_MODEL };
|
|
287
|
+
}
|
|
288
|
+
}
|
package/src/core/post-mortem.mjs
CHANGED
|
@@ -3,7 +3,8 @@ import path from 'path';
|
|
|
3
3
|
import matter from 'gray-matter';
|
|
4
4
|
import crypto from 'crypto';
|
|
5
5
|
import { callLocalRuntime, loadRuntimeConfig } from './runtime.mjs';
|
|
6
|
-
import { atomicWrite,
|
|
6
|
+
import { atomicWrite, safeStringify } from './vault.mjs';
|
|
7
|
+
import { getNodes } from './vault-cache.mjs';
|
|
7
8
|
import { logger } from './logger.mjs';
|
|
8
9
|
import { brainDir } from './config.mjs';
|
|
9
10
|
|
|
@@ -115,7 +116,7 @@ export async function runPostMortem(sessionPath, { vaultDir, inboxDir, runtimeCo
|
|
|
115
116
|
}
|
|
116
117
|
|
|
117
118
|
// Load active rules for violation checking
|
|
118
|
-
const nodes =
|
|
119
|
+
const nodes = getNodes(vaultDir);
|
|
119
120
|
const rules = nodes.filter(n =>
|
|
120
121
|
n.status === 'active' &&
|
|
121
122
|
n.priority === 'absolute' &&
|
|
@@ -324,7 +325,7 @@ export async function runComplianceAudit(sessionPath, { vaultDir, runtimeConfig
|
|
|
324
325
|
return { compliant: true, violations: [], skipped: 'too-short' };
|
|
325
326
|
}
|
|
326
327
|
|
|
327
|
-
const nodes =
|
|
328
|
+
const nodes = getNodes(vaultDir);
|
|
328
329
|
const rules = nodes.filter(n =>
|
|
329
330
|
n.status === 'active' &&
|
|
330
331
|
(n.modality === 'must' || n.modality === 'must_not')
|
|
@@ -13,7 +13,8 @@ import fs from 'node:fs';
|
|
|
13
13
|
import path from 'node:path';
|
|
14
14
|
import crypto from 'node:crypto';
|
|
15
15
|
import matter from 'gray-matter';
|
|
16
|
-
import {
|
|
16
|
+
import { writeNode, deleteNode } from './vault.mjs';
|
|
17
|
+
import { getNodes } from './vault-cache.mjs';
|
|
17
18
|
import { brainDir } from './config.mjs';
|
|
18
19
|
|
|
19
20
|
const getBrainDir = () => {
|
|
@@ -235,7 +236,7 @@ export function loadQueue(overrideBrainDir) {
|
|
|
235
236
|
let node = null;
|
|
236
237
|
if (item.node_slug) {
|
|
237
238
|
try {
|
|
238
|
-
const nodes =
|
|
239
|
+
const nodes = getNodes(vaultDir);
|
|
239
240
|
node = nodes.find(n => n.slug === item.node_slug);
|
|
240
241
|
if (!node) {
|
|
241
242
|
const inboxFile = path.join(effectiveBrainDir, 'memory-inbox', 'pending', `${item.node_slug}.md`);
|
|
@@ -384,7 +385,7 @@ export function updateQueueItem(id, patch = {}, overrideBrainDir) {
|
|
|
384
385
|
let node = null;
|
|
385
386
|
if (item.node_slug) {
|
|
386
387
|
try {
|
|
387
|
-
const nodes =
|
|
388
|
+
const nodes = getNodes(vaultDir);
|
|
388
389
|
node = nodes.find(n => n.slug === item.node_slug);
|
|
389
390
|
|
|
390
391
|
if (!node) {
|
package/src/core/runtime.mjs
CHANGED
|
@@ -43,7 +43,6 @@ export function findBinaryInPath(binaryName) {
|
|
|
43
43
|
|
|
44
44
|
const DEFAULT_AGENTS = [
|
|
45
45
|
{ name: 'antigravity', binary: 'antigravity', flags: '--sandbox=false --yolo -o json', priority: 1, enabled: true, exec: 'flag' },
|
|
46
|
-
{ name: 'gemini', binary: 'gemini', flags: '--sandbox=false --yolo -o json', priority: 2, enabled: true, exec: 'flag' },
|
|
47
46
|
{ name: 'claude', binary: 'claude', flags: '--output-format json --permission-mode bypassPermissions', priority: 3, enabled: true, exec: 'flag' },
|
|
48
47
|
{ name: 'codex', binary: 'codex', flags: '--full-auto --json', priority: 4, enabled: true, exec: 'subcommand' },
|
|
49
48
|
];
|
|
@@ -290,7 +289,8 @@ export async function callLocalRuntime(prompt, system, config) {
|
|
|
290
289
|
maxBuffer: 10 * 1024 * 1024,
|
|
291
290
|
env: {
|
|
292
291
|
...process.env,
|
|
293
|
-
GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || googleApiKey,
|
|
292
|
+
GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY || googleApiKey,
|
|
293
|
+
GEMINI_API_KEY: process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || googleApiKey,
|
|
294
294
|
TAVILY_API_KEY: process.env.TAVILY_API_KEY || tavilyApiKey,
|
|
295
295
|
BRAVE_API_KEY: process.env.BRAVE_API_KEY || braveApiKey,
|
|
296
296
|
EXA_API_KEY: process.env.EXA_API_KEY || exaApiKey,
|