total-recall-brain 3.28.2 → 3.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/total-recall.mjs +24 -1
- package/docs/ARCHITECTURE.md +26 -18
- package/extension/background.js +259 -84
- package/extension/content-script.css +7 -13
- package/extension/content-script.js +252 -477
- package/extension/icons/icon-128.png +0 -0
- package/extension/icons/icon-16.png +0 -0
- package/extension/icons/icon-32.png +0 -0
- package/extension/icons/icon-48.png +0 -0
- package/extension/icons/mark.svg +6 -0
- package/extension/lib/brain-client.js +160 -73
- package/extension/lib/tokens.css +187 -0
- package/extension/lib/ui-helpers.js +229 -0
- package/extension/lib/ui-helpers.spec.mjs +94 -0
- package/extension/manifest.json +20 -5
- package/extension/options/options.css +64 -218
- package/extension/options/options.html +91 -43
- package/extension/options/options.js +208 -85
- package/extension/sidepanel/sidepanel.css +268 -970
- package/extension/sidepanel/sidepanel.html +200 -166
- package/extension/sidepanel/sidepanel.js +788 -942
- package/frontend/dist/assets/index-DGMNZi_1.js +5742 -0
- package/frontend/dist/index.html +1 -1
- package/frontend/src/api/plugins.ts +124 -92
- package/frontend/src/components/ResearchAgendaTab.spec.tsx +11 -0
- package/frontend/src/components/ResearchAgendaTab.tsx +19 -14
- package/frontend/src/pages/PluginsPage.spec.tsx +87 -16
- package/frontend/src/pages/PluginsPage.tsx +454 -1218
- package/package.json +2 -1
- package/plugins/code-quality/README.md +25 -0
- package/plugins/code-quality/cli.mjs +25 -0
- package/plugins/code-quality/plugin.json +17 -0
- package/plugins/git-sentinel/README.md +27 -0
- package/{scaffold/.agent/plugins → plugins}/git-sentinel/cli.mjs +8 -5
- package/plugins/git-sentinel/plugin.json +18 -0
- package/plugins/system-monitor/README.md +26 -0
- package/{scaffold/.agent/plugins → plugins}/system-monitor/cli.mjs +5 -4
- package/{scaffold/.agent/plugins → plugins}/system-monitor/generator.mjs +4 -5
- package/plugins/system-monitor/plugin.json +29 -0
- package/scaffold/.agent/skills/meta-harness/SKILL.md +4 -4
- package/scaffold/.agent/skills/plugins/SKILL.md +53 -0
- package/scaffold/.agent/skills/total-recall/SKILL.md +22 -6
- package/scaffold/.agent/skills/total-recall/scripts/brain-state.json +1 -0
- package/src/cli/agent.mjs +1 -1
- package/src/cli/app/index.mjs +493 -0
- package/src/cli/brain.mjs +33 -3
- package/src/cli/command.mjs +80 -41
- package/src/cli/connect.mjs +10 -1
- package/src/cli/harness.mjs +1 -1
- package/src/cli/init.mjs +3 -1
- package/src/cli/mesh.mjs +118 -12
- package/src/cli/plugin/create.mjs +144 -25
- package/src/cli/plugin/index.mjs +83 -53
- package/src/cli/plugin/install.mjs +10 -127
- package/src/cli/plugin/list.mjs +44 -44
- package/src/cli/plugin/remove.mjs +23 -30
- package/src/cli/plugin/scaffold-cmd.mjs +30 -0
- package/src/cli/plugin/search.mjs +108 -23
- package/src/cli/rebuild.mjs +37 -1
- package/src/cli/recall.mjs +17 -2
- package/src/cli/research.mjs +4 -6
- package/src/cli/share.mjs +2 -0
- package/src/core/app-deploy/apply.mjs +386 -0
- package/src/core/app-deploy/design-tokens.mjs +217 -0
- package/src/core/app-deploy/plan.mjs +249 -0
- package/src/core/app-deploy/resolve.mjs +295 -0
- package/src/core/app-deploy/source.mjs +337 -0
- package/src/core/app-deploy/standalone.mjs +248 -0
- package/src/core/app-deploy/upgrade.mjs +256 -0
- package/src/core/app-deploy/verify.mjs +213 -0
- package/src/core/bound-hosts.mjs +1 -1
- package/src/core/brain-registry.mjs +146 -0
- package/src/core/brain-state.json +1 -0
- package/src/core/daemon-loop.mjs +30 -26
- package/src/core/dream.mjs +5 -17
- package/src/core/fact-seeker.mjs +53 -368
- package/src/core/lan-discovery.mjs +3 -3
- package/src/core/mesh-access-sync.mjs +134 -0
- package/src/core/mesh-access.mjs +18 -3
- package/src/core/mesh-discover.mjs +173 -0
- package/src/core/mesh.mjs +131 -32
- package/src/core/optimizer.mjs +4 -41
- package/src/core/plugin-bundle.mjs +167 -0
- package/src/core/plugin-context.mjs +8 -13
- package/src/core/plugin-loader.mjs +272 -68
- package/src/core/plugin-peers.mjs +163 -0
- package/src/core/plugin-public.mjs +85 -0
- package/src/core/plugin-runner-child.mjs +24 -0
- package/src/core/plugin-runner.mjs +102 -0
- package/src/core/plugin-store.mjs +384 -0
- package/src/core/plugin-tasks.mjs +212 -0
- package/src/core/post-mortem.mjs +40 -9
- package/src/core/research-gate.mjs +182 -0
- package/src/core/research-queue.mjs +47 -7
- package/src/core/research-surface.mjs +211 -0
- package/src/core/research.mjs +25 -32
- package/src/core/scheduler.mjs +67 -153
- package/src/core/schema.mjs +129 -0
- package/src/core/search.mjs +12 -1
- package/src/core/secret-integration-research.mjs +12 -8
- package/src/core/secrets-store.mjs +76 -2
- package/src/core/session-watcher.mjs +6 -0
- package/src/core/surface.mjs +64 -3
- package/src/core/task-executors.mjs +8 -45
- package/src/core/vault-cache.mjs +70 -4
- package/src/server/api.mjs +21 -0
- package/src/server/rest.mjs +2 -0
- package/src/server/route-manifest.json +21 -5
- package/src/server/routes/collab.mjs +38 -11
- package/src/server/routes/extension.mjs +3 -2
- package/src/server/routes/memory.mjs +24 -2
- package/src/server/routes/mesh.mjs +7 -7
- package/src/server/routes/plugins-mesh.mjs +49 -0
- package/src/server/routes/plugins.mjs +91 -426
- package/src/server/routes/research.mjs +5 -2
- package/src/server/routes/share.mjs +2 -0
- package/src/server/tools.mjs +29 -2
- package/extension/icons/README.md +0 -27
- package/extension/lib/preconfigured.js +0 -5
- package/extension/popup/popup.css +0 -227
- package/extension/popup/popup.html +0 -50
- package/extension/popup/popup.js +0 -175
- package/frontend/dist/assets/index-U7o-qU99.js +0 -5742
- package/scaffold/.agent/plugins/.gitkeep +0 -0
- package/scaffold/.agent/plugins/code-quality/README.md +0 -16
- package/scaffold/.agent/plugins/code-quality/cli.mjs +0 -15
- package/scaffold/.agent/plugins/code-quality/plugin.json +0 -34
- package/scaffold/.agent/plugins/git-sentinel/README.md +0 -16
- package/scaffold/.agent/plugins/git-sentinel/plugin.json +0 -34
- package/scaffold/.agent/plugins/meta-harness/README.md +0 -16
- package/scaffold/.agent/plugins/meta-harness/cli.mjs +0 -34
- package/scaffold/.agent/plugins/meta-harness/plugin.json +0 -34
- package/scaffold/.agent/plugins/system-monitor/README.md +0 -16
- package/scaffold/.agent/plugins/system-monitor/plugin.json +0 -38
package/src/core/surface.mjs
CHANGED
|
@@ -11,6 +11,9 @@ import {
|
|
|
11
11
|
inferMemoryLayer,
|
|
12
12
|
} from './memory-layers.mjs';
|
|
13
13
|
import { assemblePluginContexts } from './plugin-context.mjs';
|
|
14
|
+
import { selectResearchBriefs, formatResearchBriefs } from './research-surface.mjs';
|
|
15
|
+
import { loadQueue } from './research-queue.mjs';
|
|
16
|
+
import { brainDir as globalBrainDir } from './config.mjs';
|
|
14
17
|
|
|
15
18
|
/**
|
|
16
19
|
* Extract [[slug]] wikilink references and relative Markdown link targets from body text.
|
|
@@ -307,6 +310,26 @@ async function compactNode(node, derivedDir, force = false) {
|
|
|
307
310
|
return compacted;
|
|
308
311
|
}
|
|
309
312
|
|
|
313
|
+
/**
|
|
314
|
+
* Finished background research for the current project, plus research the user
|
|
315
|
+
* asked for recently, as a compact brief section (see research-surface.mjs).
|
|
316
|
+
* Reports live in the global brain, so nodes are looked up there when the
|
|
317
|
+
* caller's node set (e.g. a project vault) does not contain them.
|
|
318
|
+
*/
|
|
319
|
+
export function buildResearchSection(nodes = [], { projectRoot, queueItems, researchNodes, now } = {}) {
|
|
320
|
+
try {
|
|
321
|
+
const items = queueItems || loadQueue();
|
|
322
|
+
if (!items.some((i) => i.status === 'done' && i.node_slug)) return '';
|
|
323
|
+
const project = path.basename(projectRoot || process.cwd());
|
|
324
|
+
const pool = researchNodes || [...nodes, ...getNodes(path.join(globalBrainDir, 'memory-vault'))];
|
|
325
|
+
const section = formatResearchBriefs(selectResearchBriefs({ queueItems: items, nodes: pool, project, now }));
|
|
326
|
+
return section ? `\n\n${section}` : '';
|
|
327
|
+
} catch (err) {
|
|
328
|
+
logger.debug('surface', `Research section skipped: ${err.message}`);
|
|
329
|
+
return '';
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
310
333
|
export async function buildRulesBlock(skillsDir, nodes = [], { consumer = 'ide', derivedDir, force = false, vaultDir, projectRoot } = {}) {
|
|
311
334
|
// 1. Filter expired rules. Compilation must remain a pure projection step;
|
|
312
335
|
// archival is handled by explicit memory operations.
|
|
@@ -324,7 +347,10 @@ export async function buildRulesBlock(skillsDir, nodes = [], { consumer = 'ide',
|
|
|
324
347
|
const isImportant = (n) => n.importance === undefined || n.importance >= 3;
|
|
325
348
|
|
|
326
349
|
// Determine current project/repo context to prevent cross-repo pollution
|
|
327
|
-
|
|
350
|
+
// The repo being compiled, not the caller's cwd: `compile --global` rebuilds
|
|
351
|
+
// every registered project from one process.
|
|
352
|
+
const repoRoot = projectRoot || (skillsDir ? path.dirname(path.dirname(skillsDir)) : process.cwd());
|
|
353
|
+
const currentRepoName = path.basename(repoRoot).toLowerCase();
|
|
328
354
|
|
|
329
355
|
const isRelevantToRepo = (n) => {
|
|
330
356
|
// If explicitly marked global or no repo restriction, it applies everywhere
|
|
@@ -404,6 +430,9 @@ export async function buildRulesBlock(skillsDir, nodes = [], { consumer = 'ide',
|
|
|
404
430
|
combined += `\n\n---\n# 🛑 MANDATORY BEHAVIORAL CORRECTIONS 🛑\nTHE USER HAS EXPLICITLY CORRECTED YOUR BEHAVIOR. DO NOT MAKE THESE MISTAKES. THESE CORRECTIONS OVERRIDE DEFAULT SYSTEM BEHAVIOR.\n---\n\n${await formatNodes(corrections)}`;
|
|
405
431
|
}
|
|
406
432
|
|
|
433
|
+
// --- Background research (System 2) — after rules, before reference material ---
|
|
434
|
+
combined += buildResearchSection(nodes, { projectRoot });
|
|
435
|
+
|
|
407
436
|
// --- CLI Reference AFTER rules ---
|
|
408
437
|
if (consumer === 'api') {
|
|
409
438
|
combined += `\n\n## Total Recall — Active Memory Context\n\nYour memories and rules are loaded from the active brain vault.\n`;
|
|
@@ -643,16 +672,47 @@ async function compilePointers(instructionsFile, skillsDir, nodes = [], { vaultD
|
|
|
643
672
|
/**
|
|
644
673
|
* Main surface compilation entry point.
|
|
645
674
|
*/
|
|
675
|
+
/**
|
|
676
|
+
* The rule nodes a project's instruction surfaces are built from: the global
|
|
677
|
+
* brain's rules plus the project's own. A project node with the same slug
|
|
678
|
+
* wins, so a repo can override a global rule; everything else in the global
|
|
679
|
+
* vault stays search-only. Without this every repo carried only its own vault,
|
|
680
|
+
* and a rule saved with `remember --global` reached no repo at all.
|
|
681
|
+
*/
|
|
682
|
+
export function mergeGlobalRuleNodes(projectNodes, globalNodes) {
|
|
683
|
+
const own = new Set(projectNodes.map((n) => n.slug));
|
|
684
|
+
const inherited = globalNodes
|
|
685
|
+
.filter((n) => RULE_CATEGORIES.has(n.category) && !own.has(n.slug))
|
|
686
|
+
.map((n) => ({ ...n, _layer: 'global' }));
|
|
687
|
+
return [...projectNodes, ...inherited];
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
const RULE_CATEGORIES = new Set(['invariants', 'preferences', 'anti-patterns']);
|
|
691
|
+
|
|
692
|
+
function globalVaultFor(vaultDir) {
|
|
693
|
+
const globalVault = path.join(globalBrainDir, 'memory-vault');
|
|
694
|
+
return path.resolve(globalVault) === path.resolve(vaultDir) ? null : globalVault;
|
|
695
|
+
}
|
|
696
|
+
|
|
646
697
|
export async function compileSurface({ vaultDir, skillsDir, derivedDir, instructionsFile, force = false }) {
|
|
647
698
|
const nodes = getNodes(vaultDir);
|
|
699
|
+
const globalVault = globalVaultFor(vaultDir);
|
|
700
|
+
const ruleNodes = globalVault && fs.existsSync(globalVault)
|
|
701
|
+
? mergeGlobalRuleNodes(nodes, getNodes(globalVault))
|
|
702
|
+
: nodes;
|
|
648
703
|
|
|
649
704
|
// ── Incremental compilation: vault content hash check ──
|
|
705
|
+
// The global rules hash lives in its own file: vault-hash.txt is served by
|
|
706
|
+
// /api/vault/hash and must keep describing this vault alone.
|
|
650
707
|
const hashFile = path.join(derivedDir, 'vault-hash.txt');
|
|
651
708
|
const currentHash = computeVaultHash(vaultDir);
|
|
709
|
+
const globalHashFile = path.join(derivedDir, 'global-rules-hash.txt');
|
|
710
|
+
const globalHash = globalVault && fs.existsSync(globalVault) ? computeVaultHash(globalVault) : '';
|
|
652
711
|
|
|
653
712
|
if (!force && fs.existsSync(hashFile)) {
|
|
654
713
|
const storedHash = fs.readFileSync(hashFile, 'utf8').trim();
|
|
655
|
-
|
|
714
|
+
const storedGlobal = fs.existsSync(globalHashFile) ? fs.readFileSync(globalHashFile, 'utf8').trim() : '';
|
|
715
|
+
if (storedHash === currentHash && storedGlobal === globalHash) {
|
|
656
716
|
return {
|
|
657
717
|
nodesProcessed: nodes.length,
|
|
658
718
|
skillsInjected: 0,
|
|
@@ -665,7 +725,7 @@ export async function compileSurface({ vaultDir, skillsDir, derivedDir, instruct
|
|
|
665
725
|
}
|
|
666
726
|
|
|
667
727
|
// 1. Write pointer and active rules to all instruction shims
|
|
668
|
-
const skillsInjected = await compilePointers(instructionsFile, skillsDir,
|
|
728
|
+
const skillsInjected = await compilePointers(instructionsFile, skillsDir, ruleNodes, { vaultDir, derivedDir, force });
|
|
669
729
|
|
|
670
730
|
// 2. Build derived indexes (powers semantic search API)
|
|
671
731
|
if (!fs.existsSync(derivedDir)) {
|
|
@@ -730,6 +790,7 @@ export async function compileSurface({ vaultDir, skillsDir, derivedDir, instruct
|
|
|
730
790
|
|
|
731
791
|
// 5. Write vault hash + projection manifest
|
|
732
792
|
atomicWrite(hashFile, currentHash);
|
|
793
|
+
atomicWrite(globalHashFile, globalHash);
|
|
733
794
|
writeProjectionManifest(derivedDir, currentHash);
|
|
734
795
|
|
|
735
796
|
// 6. Generate live OKF Index and Log
|
|
@@ -369,6 +369,14 @@ async function runResearchLegacy(task, ctx) {
|
|
|
369
369
|
String(task.slug || '').includes('fact-seeker') ||
|
|
370
370
|
String(task.slug || '').includes('knowledge-acquisition')
|
|
371
371
|
) {
|
|
372
|
+
// Research needs a topic somebody chose. A task without one used to pull
|
|
373
|
+
// "the next agenda topic" — which is how self-spawned tangents got
|
|
374
|
+
// researched — so it is now a no-op (RESEARCH_SYSTEM2).
|
|
375
|
+
const topic = String(task.target || '').trim();
|
|
376
|
+
if (!topic || topic === 'global') {
|
|
377
|
+
return { success: true, output: 'No research topic on this task — nothing to research.', executor: 'research' };
|
|
378
|
+
}
|
|
379
|
+
|
|
372
380
|
// Prefer deep multi-source research when this came from the research queue
|
|
373
381
|
const deep =
|
|
374
382
|
String(task.slug || '').startsWith('research-acquisition-') ||
|
|
@@ -497,51 +505,6 @@ async function runResearchLegacy(task, ctx) {
|
|
|
497
505
|
};
|
|
498
506
|
}
|
|
499
507
|
|
|
500
|
-
if (String(task.slug || '').startsWith('research-monitoring-')) {
|
|
501
|
-
const missing = requireNode('Monitoring');
|
|
502
|
-
if (missing) return missing;
|
|
503
|
-
const { runResearchMonitoringCycle } = await import('./fact-seeker.mjs');
|
|
504
|
-
const result = await runResearchMonitoringCycle({
|
|
505
|
-
vaultDir: VAULT_DIR,
|
|
506
|
-
nodeSlug: task._node_slug,
|
|
507
|
-
topic: task.title || task.target || 'Unknown Topic',
|
|
508
|
-
runtimeConfig,
|
|
509
|
-
skillsDir: SKILLS_DIR,
|
|
510
|
-
derivedDir: DERIVED_DIR,
|
|
511
|
-
instructionsFile: INSTRUCTIONS_FILE,
|
|
512
|
-
});
|
|
513
|
-
if (result?.success === false) {
|
|
514
|
-
return { success: false, error: result.error || 'Monitoring failed', executor: 'research' };
|
|
515
|
-
}
|
|
516
|
-
return {
|
|
517
|
-
success: true,
|
|
518
|
-
output: result?.output || 'Monitoring complete',
|
|
519
|
-
factSlug: result?.factSlug || task._node_slug,
|
|
520
|
-
executor: 'research',
|
|
521
|
-
};
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
if (String(task.slug || '').startsWith('research-expansion-')) {
|
|
525
|
-
const missing = requireNode('Expansion');
|
|
526
|
-
if (missing) return missing;
|
|
527
|
-
const { runResearchExpansionCycle } = await import('./fact-seeker.mjs');
|
|
528
|
-
const result = await runResearchExpansionCycle({
|
|
529
|
-
vaultDir: VAULT_DIR,
|
|
530
|
-
nodeSlug: task._node_slug,
|
|
531
|
-
topic: task.title || task.target || 'Unknown Topic',
|
|
532
|
-
runtimeConfig,
|
|
533
|
-
});
|
|
534
|
-
if (result?.success === false) {
|
|
535
|
-
return { success: false, error: result.error || 'Expansion failed', executor: 'research' };
|
|
536
|
-
}
|
|
537
|
-
return {
|
|
538
|
-
success: true,
|
|
539
|
-
output: result?.output || 'Expansion complete',
|
|
540
|
-
factSlug: result?.factSlug || task._node_slug,
|
|
541
|
-
executor: 'research',
|
|
542
|
-
};
|
|
543
|
-
}
|
|
544
|
-
|
|
545
508
|
if (String(task.slug || '').includes('validate')) {
|
|
546
509
|
const { runConclusionWriter } = await import('./conclusion-writer.mjs');
|
|
547
510
|
const result = await runConclusionWriter({
|
package/src/core/vault-cache.mjs
CHANGED
|
@@ -60,6 +60,75 @@ export function invalidate(vaultDir) {
|
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Watches a directory tree without keeping the process alive.
|
|
65
|
+
*
|
|
66
|
+
* macOS and Windows implement `fs.watch({ recursive: true })` natively as one
|
|
67
|
+
* handle, so `unref()` works. On Linux, Node emulates it with one inotify
|
|
68
|
+
* watcher per subdirectory and `unref()` on the returned object does not reach
|
|
69
|
+
* them, so every CLI command that touched the vault cache hung after printing
|
|
70
|
+
* its result. There, each directory gets its own non-recursive, unref'd
|
|
71
|
+
* watcher instead; directories created later are picked up as they appear.
|
|
72
|
+
*
|
|
73
|
+
* @param {string} root - Directory to watch
|
|
74
|
+
* @param {(eventType: string, filename: string) => void} onChange - Receives paths relative to root
|
|
75
|
+
* @returns {{ close: () => void }}
|
|
76
|
+
*/
|
|
77
|
+
function watchTree(root, onChange) {
|
|
78
|
+
if (process.platform === 'darwin' || process.platform === 'win32') {
|
|
79
|
+
const watcher = fs.watch(root, { recursive: true }, onChange);
|
|
80
|
+
watcher.unref();
|
|
81
|
+
return watcher;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const watchers = new Map();
|
|
85
|
+
const watchDir = (dir) => {
|
|
86
|
+
if (watchers.has(dir)) return;
|
|
87
|
+
let watcher;
|
|
88
|
+
try {
|
|
89
|
+
watcher = fs.watch(dir, (eventType, filename) => {
|
|
90
|
+
if (!filename) return;
|
|
91
|
+
const full = path.join(dir, String(filename));
|
|
92
|
+
if (eventType === 'rename') {
|
|
93
|
+
try {
|
|
94
|
+
if (fs.statSync(full).isDirectory()) walk(full);
|
|
95
|
+
} catch {
|
|
96
|
+
// Removed; its watcher closes itself on error.
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
onChange(eventType, path.relative(root, full));
|
|
100
|
+
});
|
|
101
|
+
} catch {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
watcher.on('error', () => {
|
|
105
|
+
watcher.close();
|
|
106
|
+
watchers.delete(dir);
|
|
107
|
+
});
|
|
108
|
+
watcher.unref();
|
|
109
|
+
watchers.set(dir, watcher);
|
|
110
|
+
};
|
|
111
|
+
const walk = (dir) => {
|
|
112
|
+
watchDir(dir);
|
|
113
|
+
let entries = [];
|
|
114
|
+
try {
|
|
115
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
116
|
+
} catch {
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
for (const item of entries) {
|
|
120
|
+
if (item.isDirectory()) walk(path.join(dir, item.name));
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
walk(root);
|
|
124
|
+
return {
|
|
125
|
+
close() {
|
|
126
|
+
for (const watcher of watchers.values()) watcher.close();
|
|
127
|
+
watchers.clear();
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
63
132
|
/**
|
|
64
133
|
* Starts a filesystem watcher on the vault directory to automatically invalidate the cache
|
|
65
134
|
* when external edits occur.
|
|
@@ -82,15 +151,12 @@ function startWatcher(vaultDir) {
|
|
|
82
151
|
|
|
83
152
|
logger.info('Starting filesystem watcher for vault cache', { vaultDir: resolvedDir });
|
|
84
153
|
try {
|
|
85
|
-
|
|
154
|
+
entry.watcher = watchTree(resolvedDir, (eventType, filename) => {
|
|
86
155
|
if (filename && filename.endsWith('.md')) {
|
|
87
156
|
logger.info(`Vault filesystem change detected (${eventType}: ${filename}), invalidating cache`);
|
|
88
157
|
invalidate(resolvedDir);
|
|
89
158
|
}
|
|
90
159
|
});
|
|
91
|
-
// Unref the watcher so it does not prevent Node from exiting cleanly
|
|
92
|
-
watcher.unref();
|
|
93
|
-
entry.watcher = watcher;
|
|
94
160
|
} catch (err) {
|
|
95
161
|
logger.warn('Failed to start vault fs watcher, cache will rely on explicit invalidations', { error: err.message });
|
|
96
162
|
}
|
package/src/server/api.mjs
CHANGED
|
@@ -436,6 +436,7 @@ COMPUTER USE TOOLS (desktop/X11 — use to control apps or the full desktop):
|
|
|
436
436
|
- 'computer_scroll': Scroll up or down at screen coordinates.
|
|
437
437
|
|
|
438
438
|
CODE / MEMORY:
|
|
439
|
+
- 'queue_research': Queue deep BACKGROUND research when the user asks you to research a topic in depth or later. Findings arrive in memory, not in this reply. Never use it on your own initiative; use 'search_web' for answers now.
|
|
439
440
|
- 'execute_code': Run Node.js to call APIs, process data, or perform calculations.
|
|
440
441
|
- 'update_design': Write markdown to DESIGN.md when asked to create a UI or document.
|
|
441
442
|
|
|
@@ -645,6 +646,26 @@ ${interviewTask}`;
|
|
|
645
646
|
}
|
|
646
647
|
}
|
|
647
648
|
|
|
649
|
+
// System 2 grounding: background research relevant to what the user just
|
|
650
|
+
// asked. One embedding lookup, no LLM call; failure never blocks the reply.
|
|
651
|
+
try {
|
|
652
|
+
const lastUser = [...messages].reverse().find((m) => m.role === 'user');
|
|
653
|
+
const lastText = typeof lastUser?.content === 'string' ? lastUser.content : '';
|
|
654
|
+
if (lastText) {
|
|
655
|
+
const { findRelevantResearch, formatChatResearch } = await import('../core/research-surface.mjs');
|
|
656
|
+
const found = await findRelevantResearch(lastText, {
|
|
657
|
+
vaultDir: VAULT_DIR,
|
|
658
|
+
derivedDir: path.join(path.dirname(VAULT_DIR), 'memory-derived'),
|
|
659
|
+
});
|
|
660
|
+
if (found.length) {
|
|
661
|
+
baseSystemPrompt += formatChatResearch(found);
|
|
662
|
+
logger.info('api', `Research grounding: ${found.map((f) => f.slug).join(', ')}`);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
} catch (err) {
|
|
666
|
+
logger.warn('api', `Research grounding skipped: ${err.message}`);
|
|
667
|
+
}
|
|
668
|
+
|
|
648
669
|
let currentMessages = [...messages];
|
|
649
670
|
if (currentMessages.length > 0 && currentMessages[0].role === 'system') {
|
|
650
671
|
currentMessages[0].content = `${baseSystemPrompt}\n\n${currentMessages[0].content}`;
|
package/src/server/rest.mjs
CHANGED
|
@@ -125,6 +125,7 @@ import { rulesRouter } from './routes/rules.mjs';
|
|
|
125
125
|
import notificationsRouter from './routes/notifications.mjs';
|
|
126
126
|
import meshPresenceRouter from './routes/mesh-presence.mjs';
|
|
127
127
|
import pluginsRouter from './routes/plugins.mjs';
|
|
128
|
+
import pluginsMeshRouter from './routes/plugins-mesh.mjs';
|
|
128
129
|
// ollamaUrl removed — CLI agents replace Ollama
|
|
129
130
|
import {
|
|
130
131
|
AGENT_DIR,
|
|
@@ -243,6 +244,7 @@ router.use(rulesRouter);
|
|
|
243
244
|
router.use(notificationsRouter);
|
|
244
245
|
router.use(meshPresenceRouter);
|
|
245
246
|
router.use(pluginsRouter);
|
|
247
|
+
router.use(pluginsMeshRouter);
|
|
246
248
|
|
|
247
249
|
export { router as restRouter };
|
|
248
250
|
export default router;
|
|
@@ -351,6 +351,14 @@
|
|
|
351
351
|
"method": "GET",
|
|
352
352
|
"path": "/api/mesh/nodes"
|
|
353
353
|
},
|
|
354
|
+
{
|
|
355
|
+
"method": "GET",
|
|
356
|
+
"path": "/api/mesh/plugins"
|
|
357
|
+
},
|
|
358
|
+
{
|
|
359
|
+
"method": "GET",
|
|
360
|
+
"path": "/api/mesh/plugins/:id/bundle"
|
|
361
|
+
},
|
|
354
362
|
{
|
|
355
363
|
"method": "POST",
|
|
356
364
|
"path": "/api/mesh/preauthkey"
|
|
@@ -451,10 +459,6 @@
|
|
|
451
459
|
"method": "GET",
|
|
452
460
|
"path": "/api/plugins/:id"
|
|
453
461
|
},
|
|
454
|
-
{
|
|
455
|
-
"method": "POST",
|
|
456
|
-
"path": "/api/plugins/:id/rate"
|
|
457
|
-
},
|
|
458
462
|
{
|
|
459
463
|
"method": "GET",
|
|
460
464
|
"path": "/api/plugins/:id/readme"
|
|
@@ -463,14 +467,22 @@
|
|
|
463
467
|
"method": "POST",
|
|
464
468
|
"path": "/api/plugins/:id/run"
|
|
465
469
|
},
|
|
470
|
+
{
|
|
471
|
+
"method": "POST",
|
|
472
|
+
"path": "/api/plugins/:id/share"
|
|
473
|
+
},
|
|
466
474
|
{
|
|
467
475
|
"method": "GET",
|
|
468
|
-
"path": "/api/plugins/
|
|
476
|
+
"path": "/api/plugins/available"
|
|
469
477
|
},
|
|
470
478
|
{
|
|
471
479
|
"method": "POST",
|
|
472
480
|
"path": "/api/plugins/install"
|
|
473
481
|
},
|
|
482
|
+
{
|
|
483
|
+
"method": "GET",
|
|
484
|
+
"path": "/api/plugins/peers"
|
|
485
|
+
},
|
|
474
486
|
{
|
|
475
487
|
"method": "GET",
|
|
476
488
|
"path": "/api/proposals"
|
|
@@ -495,6 +507,10 @@
|
|
|
495
507
|
"method": "GET",
|
|
496
508
|
"path": "/api/proposals/stale"
|
|
497
509
|
},
|
|
510
|
+
{
|
|
511
|
+
"method": "GET",
|
|
512
|
+
"path": "/api/public/plugins/:id/bundle"
|
|
513
|
+
},
|
|
498
514
|
{
|
|
499
515
|
"method": "GET",
|
|
500
516
|
"path": "/api/research"
|
|
@@ -7,14 +7,29 @@ import jwt from 'jsonwebtoken';
|
|
|
7
7
|
import { WebSocket } from 'ws';
|
|
8
8
|
import { brainDir as configBrainDir } from '../../core/config.mjs';
|
|
9
9
|
|
|
10
|
-
const JWT_SECRET = process.env.JWT_SECRET || 'total-recall-collab-secret-key-1234';
|
|
11
|
-
|
|
12
10
|
// Setup database paths in brainDir
|
|
13
11
|
const DATA_DIR = path.join(configBrainDir, 'collab');
|
|
14
12
|
if (!fs.existsSync(DATA_DIR)) {
|
|
15
13
|
fs.mkdirSync(DATA_DIR, { recursive: true });
|
|
16
14
|
}
|
|
17
15
|
|
|
16
|
+
// Signing secret: JWT_SECRET from the environment, otherwise a random secret
|
|
17
|
+
// generated once and persisted beside the collab store. There is deliberately no
|
|
18
|
+
// hardcoded fallback — a secret that ships in public source lets anyone forge a
|
|
19
|
+
// token for any username.
|
|
20
|
+
const JWT_SECRET_FILE = path.join(DATA_DIR, '.jwt-secret');
|
|
21
|
+
export function resolveJwtSecret(env = process.env, secretFile = JWT_SECRET_FILE) {
|
|
22
|
+
if (env.JWT_SECRET) return env.JWT_SECRET;
|
|
23
|
+
try {
|
|
24
|
+
const existing = fs.readFileSync(secretFile, 'utf8').trim();
|
|
25
|
+
if (existing.length >= 32) return existing;
|
|
26
|
+
} catch { /* not created yet */ }
|
|
27
|
+
const generated = crypto.randomBytes(48).toString('base64url');
|
|
28
|
+
fs.writeFileSync(secretFile, generated, { encoding: 'utf8', mode: 0o600 });
|
|
29
|
+
return generated;
|
|
30
|
+
}
|
|
31
|
+
const JWT_SECRET = resolveJwtSecret();
|
|
32
|
+
|
|
18
33
|
const USERS_FILE = path.join(DATA_DIR, 'users.json');
|
|
19
34
|
const GROUPS_FILE = path.join(DATA_DIR, 'groups.json');
|
|
20
35
|
const ANNOTATIONS_FILE = path.join(DATA_DIR, 'annotations.json');
|
|
@@ -203,7 +218,7 @@ collabRouter.post('/api/collab/annotations', requireCollabAuth, (req, res) => {
|
|
|
203
218
|
return res.status(403).json({ error: 'You are not a member of this group' });
|
|
204
219
|
}
|
|
205
220
|
const note = addAnnotation(url, groupCode, req.user.username, text, excerpt);
|
|
206
|
-
broadcastToUrl(url, { type: 'ANNOTATION_ADDED', annotation: note });
|
|
221
|
+
broadcastToUrl(url, { type: 'ANNOTATION_ADDED', annotation: note }, { audienceGroups: [groupCode] });
|
|
207
222
|
res.json(note);
|
|
208
223
|
} catch (err) {
|
|
209
224
|
res.status(500).json({ error: err.message });
|
|
@@ -240,7 +255,7 @@ export function handleCollabUpgrade(request, socket, head, wss) {
|
|
|
240
255
|
type: 'USER_JOINED',
|
|
241
256
|
username: clientInfo.username,
|
|
242
257
|
url: cleanUrl
|
|
243
|
-
}, ws);
|
|
258
|
+
}, { excludeWs: ws, audienceGroups: groupCodesOf(clientInfo.username) });
|
|
244
259
|
}
|
|
245
260
|
|
|
246
261
|
if (data.type === 'CHAT_MESSAGE') {
|
|
@@ -250,7 +265,7 @@ export function handleCollabUpgrade(request, socket, head, wss) {
|
|
|
250
265
|
username: clientInfo.username,
|
|
251
266
|
text: data.text,
|
|
252
267
|
created_at: new Date().toISOString()
|
|
253
|
-
});
|
|
268
|
+
}, { audienceGroups: groupCodesOf(clientInfo.username) });
|
|
254
269
|
}
|
|
255
270
|
} catch (err) {
|
|
256
271
|
ws.send(JSON.stringify({ type: 'ERROR', error: 'Invalid frame' }));
|
|
@@ -263,7 +278,7 @@ export function handleCollabUpgrade(request, socket, head, wss) {
|
|
|
263
278
|
broadcastToUrl(clientInfo.currentUrl, {
|
|
264
279
|
type: 'USER_LEFT',
|
|
265
280
|
username: clientInfo.username
|
|
266
|
-
});
|
|
281
|
+
}, { excludeWs: ws, audienceGroups: groupCodesOf(clientInfo.username) });
|
|
267
282
|
}
|
|
268
283
|
clients.delete(ws);
|
|
269
284
|
});
|
|
@@ -274,13 +289,25 @@ export function handleCollabUpgrade(request, socket, head, wss) {
|
|
|
274
289
|
}
|
|
275
290
|
}
|
|
276
291
|
|
|
277
|
-
function
|
|
292
|
+
function groupCodesOf(username) {
|
|
293
|
+
return getUserGroups(username).map(g => g.code);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Send to sockets viewing `url` that share at least one of `audienceGroups`.
|
|
298
|
+
* Presence and chat are group-private: two unrelated groups reading the same
|
|
299
|
+
* page must never see each other's messages. An empty audience reaches no one.
|
|
300
|
+
*/
|
|
301
|
+
export function broadcastToUrl(url, messageObj, { excludeWs = null, audienceGroups = [] } = {}, registry = clients) {
|
|
278
302
|
const cleanUrl = url.split('#')[0];
|
|
279
303
|
const payload = JSON.stringify(messageObj);
|
|
304
|
+
const audience = new Set(audienceGroups);
|
|
305
|
+
if (audience.size === 0) return;
|
|
306
|
+
const groups = loadGroups();
|
|
280
307
|
|
|
281
|
-
for (const [ws, info] of
|
|
282
|
-
if (ws
|
|
283
|
-
|
|
284
|
-
|
|
308
|
+
for (const [ws, info] of registry.entries()) {
|
|
309
|
+
if (ws === excludeWs || info.currentUrl !== cleanUrl || ws.readyState !== WebSocket.OPEN) continue;
|
|
310
|
+
const shares = groups.some(g => audience.has(g.code) && g.members.includes(info.username));
|
|
311
|
+
if (shares) ws.send(payload);
|
|
285
312
|
}
|
|
286
313
|
}
|
|
@@ -34,14 +34,15 @@ router.get('/api/extension/download', requireAuth, requireScope('config:read'),
|
|
|
34
34
|
res.setHeader('Content-Disposition', 'attachment; filename="total-recall-extension.zip"');
|
|
35
35
|
|
|
36
36
|
// Use zip if available, fall back to tar
|
|
37
|
-
|
|
37
|
+
// Unit specs live beside the extension sources but are not part of the product.
|
|
38
|
+
const zip = spawn('zip', ['-r', '-', '.', '-x', '*.spec.mjs'], { cwd: extDir, stdio: ['ignore', 'pipe', 'ignore'] });
|
|
38
39
|
zip.stdout.pipe(res);
|
|
39
40
|
zip.on('error', () => {
|
|
40
41
|
// zip not available — try tar
|
|
41
42
|
if (!res.headersSent) {
|
|
42
43
|
res.setHeader('Content-Type', 'application/gzip');
|
|
43
44
|
res.setHeader('Content-Disposition', 'attachment; filename="total-recall-extension.tar.gz"');
|
|
44
|
-
const tar = spawn('tar', ['czf', '-', '-C', path.dirname(extDir), 'extension'], { stdio: ['ignore', 'pipe', 'ignore'] });
|
|
45
|
+
const tar = spawn('tar', ['czf', '-', '--exclude=*.spec.mjs', '-C', path.dirname(extDir), 'extension'], { stdio: ['ignore', 'pipe', 'ignore'] });
|
|
45
46
|
tar.stdout.pipe(res);
|
|
46
47
|
tar.on('error', err => { if (!res.headersSent) serverError(res, err); });
|
|
47
48
|
tar.on('close', () => { if (!res.writableEnded) res.end(); });
|
|
@@ -119,6 +119,17 @@ const PASSTHROUGH_FIELDS = [
|
|
|
119
119
|
'project',
|
|
120
120
|
];
|
|
121
121
|
|
|
122
|
+
function stripInternalFields(result) {
|
|
123
|
+
const clean = {};
|
|
124
|
+
for (const [key, value] of Object.entries(result)) {
|
|
125
|
+
if (!key.startsWith('_')) clean[key] = value;
|
|
126
|
+
}
|
|
127
|
+
if (clean.type !== 'session' && clean.content === undefined && typeof clean.body === 'string') {
|
|
128
|
+
clean.content = clean.body;
|
|
129
|
+
}
|
|
130
|
+
return clean;
|
|
131
|
+
}
|
|
132
|
+
|
|
122
133
|
router.get('/api/memory', requireAuth, requireScope('memory:read'), (req, res) => {
|
|
123
134
|
try {
|
|
124
135
|
const vaultDirs = resolveAllVaultsFromQuery(req);
|
|
@@ -137,7 +148,7 @@ router.get('/api/memory', requireAuth, requireScope('memory:read'), (req, res) =
|
|
|
137
148
|
}
|
|
138
149
|
}
|
|
139
150
|
}
|
|
140
|
-
const { q, category, tag, tags, status, limit = '200', offset = '0' } = req.query;
|
|
151
|
+
const { q, category, tag, tags, status, sort, limit = '200', offset = '0' } = req.query;
|
|
141
152
|
|
|
142
153
|
if (q) {
|
|
143
154
|
const query = String(q).toLowerCase();
|
|
@@ -163,6 +174,15 @@ router.get('/api/memory', requireAuth, requireScope('memory:read'), (req, res) =
|
|
|
163
174
|
}
|
|
164
175
|
}
|
|
165
176
|
|
|
177
|
+
// sort=recent (newest created first) / sort=updated (newest update first), so a
|
|
178
|
+
// paged client sees the latest nodes instead of vault walk order. `recent` keys
|
|
179
|
+
// on creation because dream consolidation re-stamps `updated` on old nodes.
|
|
180
|
+
if (sort === 'recent' || sort === 'updated') {
|
|
181
|
+
const field = sort === 'recent' ? 'created' : 'updated';
|
|
182
|
+
const ts = (n) => Date.parse(n[field] || n.created || '') || 0;
|
|
183
|
+
list.sort((a, b) => ts(b) - ts(a));
|
|
184
|
+
}
|
|
185
|
+
|
|
166
186
|
const total = list.length;
|
|
167
187
|
const off = Math.max(0, parseInt(offset, 10) || 0);
|
|
168
188
|
const lim = Math.min(500, Math.max(1, parseInt(limit, 10) || 200));
|
|
@@ -456,7 +476,9 @@ router.post('/api/memory/search/semantic', requireAuth, requireScope('memory:rea
|
|
|
456
476
|
}
|
|
457
477
|
|
|
458
478
|
merged.sort((a, b) => (b.score || 0) - (a.score || 0));
|
|
459
|
-
|
|
479
|
+
// Never expose absolute vault paths (_filePath, _brainVault, …) over the API;
|
|
480
|
+
// mirror sanitizeNode's `content` so clients read one field for the text.
|
|
481
|
+
const results = merged.slice(0, k).map(stripInternalFields);
|
|
460
482
|
|
|
461
483
|
if (!anyIndex || results.length === 0) {
|
|
462
484
|
return res.status(503).json({
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
getMeshPeers,
|
|
7
7
|
getMeshHostname,
|
|
8
8
|
listEnrichedMeshNodes,
|
|
9
|
+
meshVaultRoot,
|
|
9
10
|
listMeshNodeEntities,
|
|
10
11
|
attachSelfInterfaces,
|
|
11
12
|
normalizeHostname,
|
|
@@ -26,7 +27,6 @@ import {
|
|
|
26
27
|
// headscale rejects anything else with a 500; validate up front instead.
|
|
27
28
|
const AUTH_REQUEST_PREFIX = 'hskey-authreq-';
|
|
28
29
|
import { throttledFetch } from '../../core/throttled-fetch.mjs';
|
|
29
|
-
import { defaultVaultRoot } from '../../core/vfs-documents.mjs';
|
|
30
30
|
import {
|
|
31
31
|
listLocalInterfaces,
|
|
32
32
|
summarizeInterfacesForEntity,
|
|
@@ -101,7 +101,7 @@ router.get('/api/mesh/leader', requireAuth, requireScope('config:read'), async (
|
|
|
101
101
|
* Self node includes live interface kinds (wifi/ethernet/vpn_overlay/…).
|
|
102
102
|
*/
|
|
103
103
|
router.get('/api/mesh/nodes', requireAuth, requireScope('config:read'), async (req, res) => {
|
|
104
|
-
const vaultRoot =
|
|
104
|
+
const vaultRoot = meshVaultRoot();
|
|
105
105
|
let nodes = listEnrichedMeshNodes(vaultRoot);
|
|
106
106
|
try {
|
|
107
107
|
nodes = attachSelfInterfaces(nodes, summarizeInterfacesForEntity());
|
|
@@ -169,7 +169,7 @@ router.post('/api/mesh/access', requireAuth, requireScope('config:write'), async
|
|
|
169
169
|
}
|
|
170
170
|
|
|
171
171
|
try {
|
|
172
|
-
const result = await setMeshNodeAccess(target, patch, { vaultRoot:
|
|
172
|
+
const result = await setMeshNodeAccess(target, patch, { vaultRoot: meshVaultRoot() });
|
|
173
173
|
if (!result.written) {
|
|
174
174
|
const notFound = result.reason === 'node-not-found';
|
|
175
175
|
res.status(notFound ? 404 : 500).json({
|
|
@@ -194,7 +194,7 @@ router.post('/api/mesh/access', requireAuth, requireScope('config:write'), async
|
|
|
194
194
|
*/
|
|
195
195
|
router.get('/api/mesh/access/proposals', requireAuth, requireScope('config:read'), async (_req, res) => {
|
|
196
196
|
try {
|
|
197
|
-
const nodes = listEnrichedMeshNodes(
|
|
197
|
+
const nodes = listEnrichedMeshNodes(meshVaultRoot());
|
|
198
198
|
res.json({
|
|
199
199
|
proposals: proposeAccessFromSshConfig(nodes, readSshConfig()),
|
|
200
200
|
missing_access: nodes
|
|
@@ -210,7 +210,7 @@ router.get('/api/mesh/access/proposals', requireAuth, requireScope('config:read'
|
|
|
210
210
|
/** Apply every proposal from this host's ssh config to the node entities. */
|
|
211
211
|
router.post('/api/mesh/access/import', requireAuth, requireScope('config:write'), async (_req, res) => {
|
|
212
212
|
try {
|
|
213
|
-
const vaultRoot =
|
|
213
|
+
const vaultRoot = meshVaultRoot();
|
|
214
214
|
const proposals = proposeAccessFromSshConfig(listEnrichedMeshNodes(vaultRoot), readSshConfig());
|
|
215
215
|
const results = [];
|
|
216
216
|
for (const proposal of proposals) {
|
|
@@ -434,7 +434,7 @@ router.delete('/api/mesh/watch', requireAuth, requireScope('config:write'), asyn
|
|
|
434
434
|
router.get('/api/mesh/io', requireAuth, requireScope('config:read'), async (_req, res) => {
|
|
435
435
|
try {
|
|
436
436
|
const live = detectDeviceIo();
|
|
437
|
-
const entities = listMeshNodeEntities(
|
|
437
|
+
const entities = listMeshNodeEntities(meshVaultRoot());
|
|
438
438
|
const selfHost = normalizeHostname(getMeshHostname());
|
|
439
439
|
const ent = entities.find(
|
|
440
440
|
(e) => normalizeHostname(e.hostname) === selfHost,
|
|
@@ -498,7 +498,7 @@ router.post('/api/mesh/lan/register', requireAuth, requireScope('config:write'),
|
|
|
498
498
|
maxProbes: limit,
|
|
499
499
|
});
|
|
500
500
|
const registration = await registerLanMeshNodes(snapshot.hosts, {
|
|
501
|
-
vaultRoot:
|
|
501
|
+
vaultRoot: meshVaultRoot(),
|
|
502
502
|
onlyReachable: true,
|
|
503
503
|
});
|
|
504
504
|
res.json({
|