total-recall-brain 3.2.3 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +1 -1
  2. package/extension/background.js +14 -2
  3. package/extension/content-script.js +18 -1
  4. package/extension/lib/brain-client.js +86 -52
  5. package/extension/lib/preconfigured.js +5 -0
  6. package/extension/manifest.json +1 -1
  7. package/extension/options/options.html +1 -0
  8. package/extension/options/options.js +12 -6
  9. package/extension/popup/popup.html +1 -0
  10. package/extension/popup/popup.js +29 -4
  11. package/extension/sidepanel/sidepanel.html +1 -0
  12. package/extension/sidepanel/sidepanel.js +24 -6
  13. package/frontend/index.html +8 -0
  14. package/frontend/src/api.ts +1 -2
  15. package/frontend/src/pages/ChatPage.tsx +9 -6
  16. package/frontend/src/pages/IntegrationsPage.tsx +144 -3
  17. package/package.json +2 -2
  18. package/scaffold/.agent/skills/total-recall/SKILL.md +55 -6
  19. package/scaffold/.agent/skills/total-recall/memory-vault/invariants/operating-instructions.md +2 -2
  20. package/scaffold/.agent/skills/total-recall/skills/cli-agents/agents.yml +1 -6
  21. package/scaffold/.agent/skills/total-recall/skills/ssss/SKILL.md +19 -0
  22. package/scaffold/.agent/skills/total-recall/skills/ssss/references/ssss-spec.md +33 -5
  23. package/src/cli/forget.mjs +18 -12
  24. package/src/cli/ingest/index.mjs +3 -2
  25. package/src/cli/init.mjs +30 -0
  26. package/src/cli/map.mjs +2 -2
  27. package/src/cli/rebuild.mjs +1 -1
  28. package/src/cli/remember.mjs +21 -11
  29. package/src/cli/setup.mjs +10 -1
  30. package/src/cli/share.mjs +7 -2
  31. package/src/core/append-log.mjs +131 -0
  32. package/src/core/clarity-rewriter.mjs +10 -6
  33. package/src/core/conclusion-writer.mjs +3 -2
  34. package/src/core/config.mjs +6 -0
  35. package/src/core/conflict-detector.mjs +4 -2
  36. package/src/core/context-compiler.mjs +412 -0
  37. package/src/core/daemon-loop.mjs +15 -0
  38. package/src/core/dream.mjs +4 -4
  39. package/src/core/drift-detector.mjs +3 -3
  40. package/src/core/embeddings.mjs +60 -10
  41. package/src/core/fact-seeker.mjs +70 -44
  42. package/src/core/import-rules.mjs +3 -2
  43. package/src/core/inference-engine.mjs +5 -4
  44. package/src/core/migrate.mjs +4 -3
  45. package/src/core/optimizer.mjs +6 -5
  46. package/src/core/parallel-context.mjs +288 -0
  47. package/src/core/post-mortem.mjs +4 -3
  48. package/src/core/research-queue.mjs +4 -3
  49. package/src/core/runtime.mjs +2 -2
  50. package/src/core/sandbox.mjs +56 -26
  51. package/src/core/scheduler.mjs +6 -5
  52. package/src/core/schema.mjs +2 -2
  53. package/src/core/search.mjs +167 -101
  54. package/src/core/surface.mjs +203 -53
  55. package/src/core/validated-write.mjs +100 -0
  56. package/src/core/vault-cache.mjs +46 -18
  57. package/src/core/vault.mjs +17 -29
  58. package/src/core/vector-field.mjs +659 -0
  59. package/src/core/vector-store.mjs +85 -0
  60. package/src/server/api.mjs +4 -3
  61. package/src/server/index.mjs +13 -61
  62. package/src/server/rest.mjs +238 -29
  63. package/src/server/route-manifest.json +344 -77
  64. package/src/server/routes/memory.mjs +85 -39
  65. package/src/server/routes/sessions.mjs +6 -2
  66. package/src/server/routes/share.mjs +9 -2
@@ -0,0 +1,85 @@
1
+ /**
2
+ * src/core/vector-store.mjs
3
+ *
4
+ * Pluggable vector index interface. Defaults to in-memory cosine similarity scanning,
5
+ * but structured to allow drop-in SQLite-VSS or HNSWLib index engines.
6
+ */
7
+
8
+ export class VectorStore {
9
+ constructor() {
10
+ this.vectors = new Map(); // slug -> { embedding: number[], metadata?: any }
11
+ }
12
+
13
+ /**
14
+ * Load vector index from standard index payload.
15
+ * @param {Record<string, { embedding: number[], chunks?: Array<{ embedding: number[] }> }>} indexData
16
+ */
17
+ load(indexData) {
18
+ this.vectors.clear();
19
+ if (!indexData) return;
20
+ for (const [slug, entry] of Object.entries(indexData)) {
21
+ if (entry && Array.isArray(entry.embedding)) {
22
+ this.vectors.set(slug, {
23
+ embedding: entry.embedding,
24
+ chunks: entry.chunks || []
25
+ });
26
+ }
27
+ }
28
+ }
29
+
30
+ /**
31
+ * Add or update a single vector.
32
+ */
33
+ add(slug, embedding, chunks = []) {
34
+ if (Array.isArray(embedding)) {
35
+ this.vectors.set(slug, { embedding, chunks });
36
+ }
37
+ }
38
+
39
+ /**
40
+ * Delete a single vector.
41
+ */
42
+ delete(slug) {
43
+ this.vectors.delete(slug);
44
+ }
45
+
46
+ /**
47
+ * Perform cosine-similarity based k-nearest neighbors (k-NN) search.
48
+ * Compares query against parent embedding and all child chunk embeddings,
49
+ * matching on the maximum similarity.
50
+ *
51
+ * @param {number[]} queryEmbedding - The query vector
52
+ * @param {number} topK - Maximum results
53
+ * @param {Set<string>|null} filterSet - Optional allow-list of slugs
54
+ * @param {Function} cosineSimilarityFn - Cosine similarity function
55
+ * @returns {Array<{ slug: string, similarity: number }>} Sorted matches
56
+ */
57
+ search(queryEmbedding, topK = 5, filterSet = null, cosineSimilarityFn) {
58
+ if (!queryEmbedding || this.vectors.size === 0) return [];
59
+
60
+ const results = [];
61
+ for (const [slug, data] of this.vectors.entries()) {
62
+ if (filterSet && !filterSet.has(slug)) continue;
63
+
64
+ let maxSimilarity = 0;
65
+ if (Array.isArray(data.embedding)) {
66
+ maxSimilarity = cosineSimilarityFn(queryEmbedding, data.embedding);
67
+ }
68
+
69
+ for (const chunk of data.chunks || []) {
70
+ if (Array.isArray(chunk.embedding)) {
71
+ const sim = cosineSimilarityFn(queryEmbedding, chunk.embedding);
72
+ if (sim > maxSimilarity) {
73
+ maxSimilarity = sim;
74
+ }
75
+ }
76
+ }
77
+
78
+ results.push({ slug, similarity: maxSimilarity });
79
+ }
80
+
81
+ return results
82
+ .sort((a, b) => b.similarity - a.similarity)
83
+ .slice(0, topK);
84
+ }
85
+ }
@@ -8,7 +8,8 @@ import { requireAuth, requireScope, loginHandler, logoutHandler, changePasswordH
8
8
  import { logger } from '../core/logger.mjs';
9
9
  import { synthesize as synthesizeTts, isTtsEnabled, TtsNotConfiguredError } from '../core/tts.mjs';
10
10
  import { KNOWN_SCOPES, loadKeys, issueKey, revokeKey } from './keys.mjs';
11
- import { loadNodes, writeNode } from '../core/vault.mjs';
11
+ import { writeNode } from '../core/vault.mjs';
12
+ import { getNodes } from '../core/vault-cache.mjs';
12
13
  import { compileSurface } from '../core/surface.mjs';
13
14
  import { runInSandbox } from '../core/sandbox.mjs';
14
15
  import { resolveAgentDir } from '../cli/agent-dir.mjs';
@@ -599,7 +600,7 @@ ${interviewTask}`;
599
600
  if (Array.isArray(groundingNodes) && groundingNodes.length > 0) {
600
601
  try {
601
602
  const groundingVaultDir = resolveVaultDir(brainId);
602
- const allNodes = loadNodes(groundingVaultDir);
603
+ const allNodes = getNodes(groundingVaultDir);
603
604
  let groundingPrompt = '\n\n=== ACTIVE GROUNDING BRAIN NODES ===\nThe user has explicitly selected the following brain memory nodes as context for this conversation. Integrate their contents into your knowledge base and refer to them to inform your answers:';
604
605
  let groundedAny = false;
605
606
  for (const slug of groundingNodes) {
@@ -864,7 +865,7 @@ apiRouter.delete('/v1/chat/threads/:id', requireAuth, requireScope('chat:write')
864
865
  apiRouter.get('/v1/chat/suggestions', requireAuth, requireScope('chat:read'), (req, res) => {
865
866
  try {
866
867
  const suggestionsVaultDir = resolveVaultDir(req.query?.brain);
867
- const allNodes = loadNodes(suggestionsVaultDir);
868
+ const allNodes = getNodes(suggestionsVaultDir);
868
869
  // Find active or draft memory nodes that represent facts, concepts, or research
869
870
  const candidates = allNodes.filter(n => n.status !== 'archived' && n.category !== 'preferences');
870
871
 
@@ -17,6 +17,7 @@ import cookieParser from 'cookie-parser';
17
17
  import path from 'node:path';
18
18
  import { fileURLToPath } from 'node:url';
19
19
  import zlib from 'node:zlib';
20
+ import fs from 'node:fs';
20
21
  import {
21
22
  apiRateLimiter,
22
23
  corsOptions,
@@ -34,6 +35,13 @@ import { getDaemonStatus, ensureDaemonRunning } from '../core/daemon-control.mjs
34
35
 
35
36
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
36
37
  const ROOT = path.join(__dirname, '..', '..');
38
+ const PACKAGE_VERSION = (() => {
39
+ try {
40
+ return JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8')).version || 'unknown';
41
+ } catch {
42
+ return 'unknown';
43
+ }
44
+ })();
37
45
 
38
46
  // ─── Watchdog ───────────────────────────────────────────────────────────────────
39
47
  // Attach circuit-breaker log monitor before any subsystem can emit events.
@@ -91,8 +99,6 @@ app.use(express.json({ limit: '50mb' }));
91
99
  app.use(express.urlencoded({ limit: '50mb', extended: true }));
92
100
  app.use(cookieParser());
93
101
 
94
- import fs from 'node:fs';
95
-
96
102
  // ─── Health Check ───────────────────────────────────────────────────────────────
97
103
 
98
104
  app.get('/health', requireAuthOrLocal, async (req, res) => {
@@ -134,7 +140,7 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
134
140
 
135
141
  res.json({
136
142
  status: hasCriticalIssue ? 'degraded' : 'healthy',
137
- version: '3.0.0',
143
+ version: PACKAGE_VERSION,
138
144
  uptime_seconds: Math.floor(process.uptime()),
139
145
  timestamp: new Date().toISOString(),
140
146
  disk,
@@ -205,60 +211,7 @@ try {
205
211
  logger.info('server', 'API routes mounted at /v1/chat/completions');
206
212
  }
207
213
  } catch (err) {
208
- // api.mjs may still be in standalone mode — mount it directly
209
- logger.warn('server', 'API router not exported as middleware, loading standalone routes...');
210
- const { callLocalRuntimeRaw, loadRuntimeConfig } = await import('../core/runtime.mjs');
211
- const os = await import('node:os');
212
-
213
- app.post('/v1/chat/completions', async (req, res) => {
214
- try {
215
- const authHeader = req.headers.authorization;
216
- if (!authHeader || !authHeader.startsWith('Bearer ')) {
217
- return res.status(401).json({ error: 'Missing or invalid Authorization header' });
218
- }
219
-
220
- const configPath = path.join(configBrainDir, 'config', 'runtime.yml');
221
- const config = loadRuntimeConfig(configPath);
222
- const { messages, model, temperature } = req.body;
223
-
224
- if (!messages || messages.length === 0) {
225
- return res.status(400).json({ error: 'Messages array is required' });
226
- }
227
-
228
- const activeConfig = {
229
- ...config,
230
- model: model || config.model,
231
- temperature: temperature || config.temperature
232
- };
233
-
234
- if (model && activeConfig.agents) {
235
- const targetAgentName = model.toLowerCase();
236
- const idx = activeConfig.agents.findIndex(a => a.name === targetAgentName);
237
- if (idx >= 0) {
238
- activeConfig.agents = activeConfig.agents.map(a => ({ ...a }));
239
- activeConfig.agents[idx].priority = 0;
240
- activeConfig.agents.sort((a, b) => a.priority - b.priority);
241
- }
242
- }
243
-
244
- const response = await callLocalRuntimeRaw(messages, activeConfig);
245
-
246
- res.json({
247
- id: `chatcmpl-${Date.now()}`,
248
- object: 'chat.completion',
249
- created: Math.floor(Date.now() / 1000),
250
- model: model || config.model,
251
- choices: [{
252
- index: 0,
253
- message: response,
254
- finish_reason: 'stop'
255
- }]
256
- });
257
- } catch (error) {
258
- logger.error('api', 'API Error', { error: error.message });
259
- res.status(500).json({ error: error.message });
260
- }
261
- });
214
+ logger.error('server', `API router failed to load; /v1/chat/completions left unmounted: ${err.message}`);
262
215
  }
263
216
 
264
217
 
@@ -271,7 +224,7 @@ const frontendDist = path.join(ROOT, 'frontend', 'dist');
271
224
  if (!fs.existsSync(path.join(frontendDist, 'index.html'))) {
272
225
  const frontendDir = path.join(ROOT, 'frontend');
273
226
  if (fs.existsSync(path.join(frontendDir, 'package.json'))) {
274
- log.info('Frontend not built. Building automatically...');
227
+ logger.info('server', 'Frontend not built. Building automatically...');
275
228
  try {
276
229
  const { execSync } = await import('node:child_process');
277
230
  execSync('npm install --no-audit --no-fund 2>/dev/null && npm run build', {
@@ -279,9 +232,9 @@ if (!fs.existsSync(path.join(frontendDist, 'index.html'))) {
279
232
  stdio: 'pipe',
280
233
  timeout: 120_000,
281
234
  });
282
- log.info('Frontend build complete.');
235
+ logger.info('server', 'Frontend build complete.');
283
236
  } catch (err) {
284
- log.warn(`Frontend auto-build failed: ${err.message?.split('\\n')[0] || 'unknown error'}. Dashboard will show API info instead.`);
237
+ logger.warn('server', `Frontend auto-build failed: ${err.message?.split('\\n')[0] || 'unknown error'}. Dashboard will show API info instead.`);
285
238
  }
286
239
  }
287
240
  }
@@ -865,4 +818,3 @@ async function handleShutdown(signal) {
865
818
 
866
819
  process.on('SIGTERM', () => handleShutdown('SIGTERM'));
867
820
  process.on('SIGINT', () => handleShutdown('SIGINT'));
868
-
@@ -52,10 +52,11 @@ import crypto from 'node:crypto';
52
52
  import { spawn, spawnSync } from 'node:child_process';
53
53
  import matter from 'gray-matter';
54
54
  import { fileURLToPath } from 'node:url';
55
+ import yaml from 'yaml';
55
56
  import { loadRuntimeConfig } from '../core/runtime.mjs';
56
57
 
57
58
 
58
- import { loadNodes, writeNode, deleteNode, walkMd, safeStringify } from '../core/vault.mjs';
59
+ import { writeNode, deleteNode, safeStringify } from '../core/vault.mjs';
59
60
  import { getNodes, invalidate } from '../core/vault-cache.mjs';
60
61
  import { compileSurface } from '../core/surface.mjs';
61
62
  import { runInSandbox } from '../core/sandbox.mjs';
@@ -231,6 +232,161 @@ router.post('/api/vault/compile', requireAuth, requireScope('memory:recompile'),
231
232
  }
232
233
  });
233
234
 
235
+ /**
236
+ * POST /api/vault/compact
237
+ * Compact all active append-only log files (dedup, remove tombstones).
238
+ */
239
+ router.post('/api/vault/compact', requireAuth, requireScope('memory:write'), async (req, res) => {
240
+ try {
241
+ const { compactAppendLogs } = await import('../core/append-log.mjs');
242
+ const result = compactAppendLogs();
243
+ res.json({ compacted: true, ...result });
244
+ } catch (err) {
245
+ serverError(res, err);
246
+ }
247
+ });
248
+
249
+ /**
250
+ * POST /api/context
251
+ * Dynamic streaming context compilation.
252
+ * Assembles a query-aware, temporally-scored, budget-constrained context
253
+ * document from the vault in real-time. Every token earns its place.
254
+ *
255
+ * Body: { query: string, budget?: object, momentum_slugs?: string[] }
256
+ * Returns: { context: string, stats: object }
257
+ */
258
+ router.post('/api/context', requireAuth, requireScope('memory:read'), async (req, res) => {
259
+ try {
260
+ const { compileContext } = await import('../core/context-compiler.mjs');
261
+ const { query, budget, momentum_slugs } = req.body || {};
262
+ const result = await compileContext({
263
+ query: query || '',
264
+ vaultDir: VAULT_DIR,
265
+ derivedDir: DERIVED_DIR,
266
+ budget: budget || {},
267
+ consumer: 'api',
268
+ momentumSlugs: momentum_slugs || [],
269
+ });
270
+ res.json(result);
271
+ } catch (err) {
272
+ serverError(res, err);
273
+ }
274
+ });
275
+
276
+ /**
277
+ * GET /api/context/preview
278
+ * Lightweight context preview — returns temporally-scored candidates
279
+ * and budget allocation without computing embeddings.
280
+ */
281
+ router.get('/api/context/preview', requireAuth, requireScope('memory:read'), async (req, res) => {
282
+ try {
283
+ const { previewContext } = await import('../core/context-compiler.mjs');
284
+ const result = previewContext({ vaultDir: VAULT_DIR });
285
+ res.json(result);
286
+ } catch (err) {
287
+ serverError(res, err);
288
+ }
289
+ });
290
+
291
+ /**
292
+ * POST /api/context/stream
293
+ * Massively parallel context streaming via Flash fan-out.
294
+ * Chunks the entire vault, dispatches parallel Flash calls to score
295
+ * every node against the query simultaneously, merges into budget.
296
+ *
297
+ * Body: { query: string, budget_tokens?: number, batch_size?: number,
298
+ * concurrency?: number, min_score?: number }
299
+ * Returns: { context: string, stats: object, scored: array }
300
+ */
301
+ router.post('/api/context/stream', requireAuth, requireScope('memory:read'), async (req, res) => {
302
+ try {
303
+ const { streamParallelContext } = await import('../core/parallel-context.mjs');
304
+ const { query, budget_tokens, batch_size, concurrency, min_score } = req.body || {};
305
+ if (!query) return badRequest(res, 'query is required');
306
+ const result = await streamParallelContext({
307
+ query,
308
+ vaultDir: VAULT_DIR,
309
+ budgetTokens: budget_tokens,
310
+ batchSize: batch_size,
311
+ concurrency,
312
+ minScore: min_score,
313
+ });
314
+ res.json(result);
315
+ } catch (err) {
316
+ serverError(res, err);
317
+ }
318
+ });
319
+
320
+ /**
321
+ * GET /api/context/flash/health
322
+ * Check Flash API connectivity for parallel context streaming.
323
+ */
324
+ router.get('/api/context/flash/health', requireAuth, requireScope('memory:read'), async (req, res) => {
325
+ try {
326
+ const { checkFlashHealth } = await import('../core/parallel-context.mjs');
327
+ const result = await checkFlashHealth();
328
+ res.json(result);
329
+ } catch (err) {
330
+ serverError(res, err);
331
+ }
332
+ });
333
+
334
+ // ─── Vector Field ─────────────────────────────────────────────────────────────
335
+
336
+ /**
337
+ * POST /api/field/compile
338
+ * Compile the full vector field: embed all nodes, compute N×N covariance
339
+ * matrix, track velocities, persist to disk.
340
+ */
341
+ router.post('/api/field/compile', requireAuth, requireScope('memory:recompile'), async (req, res) => {
342
+ try {
343
+ const { compileField } = await import('../core/vector-field.mjs');
344
+ const result = await compileField({ vaultDir: VAULT_DIR, derivedDir: DERIVED_DIR });
345
+ res.json({ compiled: true, ...result.meta });
346
+ } catch (err) {
347
+ serverError(res, err);
348
+ }
349
+ });
350
+
351
+ /**
352
+ * POST /api/field/sample
353
+ * Sample the vector field at a query point.
354
+ * Returns ranked nodes with direct similarity, entanglement boost, and velocity.
355
+ *
356
+ * Body: { query: string, top_k?: number, entanglement_boost?: number, velocity_weight?: number }
357
+ */
358
+ router.post('/api/field/sample', requireAuth, requireScope('memory:read'), async (req, res) => {
359
+ try {
360
+ const { sampleField } = await import('../core/vector-field.mjs');
361
+ const { query, top_k, entanglement_boost, velocity_weight } = req.body || {};
362
+ if (!query) return badRequest(res, 'query is required');
363
+ const result = await sampleField({
364
+ query,
365
+ topK: top_k,
366
+ entanglementBoost: entanglement_boost,
367
+ velocityWeight: velocity_weight,
368
+ derivedDir: DERIVED_DIR,
369
+ });
370
+ res.json(result);
371
+ } catch (err) {
372
+ serverError(res, err);
373
+ }
374
+ });
375
+
376
+ /**
377
+ * GET /api/field/stats
378
+ * Vector field statistics: point count, velocity distribution,
379
+ * strongest couplings, compilation metadata.
380
+ */
381
+ router.get('/api/field/stats', requireAuth, requireScope('memory:read'), async (req, res) => {
382
+ try {
383
+ const { fieldStats } = await import('../core/vector-field.mjs');
384
+ res.json(fieldStats(DERIVED_DIR));
385
+ } catch (err) {
386
+ serverError(res, err);
387
+ }
388
+ });
389
+
234
390
  /**
235
391
  * POST /api/dream
236
392
  */
@@ -253,24 +409,39 @@ router.post('/api/dream', requireAuth, requireScope('memory:recompile'), async (
253
409
  }
254
410
  });
255
411
 
412
+ /**
413
+ * GET /api/vault/hash
414
+ * Lightweight endpoint for PWA cache invalidation.
415
+ * Returns only the vault content hash — no expensive I/O.
416
+ */
417
+ router.get('/api/vault/hash', requireAuth, requireScope('memory:read'), (req, res) => {
418
+ try {
419
+ const hashFile = path.join(DERIVED_DIR, 'vault-hash.txt');
420
+ const hash = fs.existsSync(hashFile) ? fs.readFileSync(hashFile, 'utf8').trim() : null;
421
+ res.json({ vault_hash: hash });
422
+ } catch (err) {
423
+ serverError(res, err);
424
+ }
425
+ });
426
+
256
427
  /**
257
428
  * GET /api/vault/status
258
429
  */
259
430
  router.get('/api/vault/status', requireAuth, requireScope('memory:read'), async (req, res) => {
260
431
  try {
261
- const nodeCount = fs.existsSync(VAULT_DIR) ? walkMd(VAULT_DIR).length : 0;
262
- const skillCount = fs.existsSync(SKILLS_DIR) ? fs.readdirSync(SKILLS_DIR).filter(d =>
263
- fs.existsSync(path.join(SKILLS_DIR, d, 'SKILL.md'))).length : 0;
432
+ // Use vault-cache instead of scanning every .md file from disk
433
+ const nodeCount = getNodes(VAULT_DIR).length;
434
+ const skillCount = fs.existsSync(SKILLS_DIR) ? fs.readdirSync(SKILLS_DIR, { withFileTypes: true }).filter(d =>
435
+ d.isDirectory() && fs.existsSync(path.join(SKILLS_DIR, d.name, 'SKILL.md'))).length : 0;
264
436
  const lastCompile = fs.existsSync(INSTRUCTIONS)
265
437
  ? fs.statSync(INSTRUCTIONS).mtime.toISOString()
266
438
  : null;
267
439
  const derivedFiles = fs.existsSync(DERIVED_DIR)
268
440
  ? fs.readdirSync(DERIVED_DIR).length : 0;
269
441
 
270
- const vaultEmbedCount = fs.existsSync(path.join(DERIVED_DIR, 'embeddings.json'))
271
- ? Object.keys(JSON.parse(fs.readFileSync(path.join(DERIVED_DIR, 'embeddings.json'), 'utf8') || '{}')).length : 0;
272
- const sessionEmbedCount = fs.existsSync(path.join(DERIVED_DIR, 'session-embeddings.json'))
273
- ? Object.keys(JSON.parse(fs.readFileSync(path.join(DERIVED_DIR, 'session-embeddings.json'), 'utf8') || '{}')).length : 0;
442
+ // Use mtime-cached loaders instead of raw readFile + JSON.parse
443
+ const vaultEmbedCount = Object.keys(loadEmbeddingsIndex(DERIVED_DIR)).length;
444
+ const sessionEmbedCount = Object.keys(loadSessionEmbeddingsIndex(DERIVED_DIR)).length;
274
445
 
275
446
  // CLI agent availability (best-effort)
276
447
  const { findBinaryInPath } = await import('../core/runtime.mjs');
@@ -471,6 +642,7 @@ router.get('/api', (req, res) => {
471
642
  },
472
643
  vault: {
473
644
  'POST /api/vault/compile': 'Recompile SSSS surface (INSTRUCTIONS.md)',
645
+ 'POST /api/vault/compact': 'Compact all active append-only log files',
474
646
  'GET /api/vault/status': 'Node count, skill count, last compile time',
475
647
  },
476
648
  keys: {
@@ -728,12 +900,7 @@ router.get('/api/brain/export', requireAuth, requireScope('brain:export'), async
728
900
 
729
901
  // ─── Chrome Extension Download ───────────────────────────────────────────────
730
902
 
731
- /**
732
- * GET /api/extension/download
733
- * Streams the Chrome extension directory as a .zip file for easy installation.
734
- * No auth required — the extension itself is not sensitive.
735
- */
736
- router.get('/api/extension/download', async (_req, res) => {
903
+ router.get('/api/extension/download', requireAuth, requireScope('config:read'), async (_req, res) => {
737
904
  try {
738
905
  // Extension lives at <package-root>/extension/
739
906
  const extDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../extension');
@@ -741,6 +908,9 @@ router.get('/api/extension/download', async (_req, res) => {
741
908
  return res.status(404).json({ error: 'Chrome extension not found in this installation.' });
742
909
  }
743
910
 
911
+ // Never inject PATs into packaged extension source. Pair from the extension
912
+ // options page so secrets stay in extension-local storage.
913
+
744
914
  res.setHeader('Content-Type', 'application/zip');
745
915
  res.setHeader('Content-Disposition', 'attachment; filename="total-recall-extension.zip"');
746
916
 
@@ -758,15 +928,19 @@ router.get('/api/extension/download', async (_req, res) => {
758
928
  tar.on('close', () => { if (!res.writableEnded) res.end(); });
759
929
  }
760
930
  });
761
- zip.on('close', code => { if (code !== 0 && !res.writableEnded) res.end(); });
762
- } catch (err) { serverError(res, err); }
931
+ zip.on('close', code => {
932
+ if (code !== 0 && !res.writableEnded) res.end();
933
+ });
934
+ } catch (err) {
935
+ serverError(res, err);
936
+ }
763
937
  });
764
938
 
765
939
  /**
766
940
  * GET /api/extension/status
767
941
  * Returns whether the extension is available (packaged) and connected (has sent captures).
768
942
  */
769
- router.get('/api/extension/status', async (_req, res) => {
943
+ router.get('/api/extension/status', requireAuth, requireScope('config:read'), async (_req, res) => {
770
944
  try {
771
945
  const extDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../extension');
772
946
  const available = fs.existsSync(path.join(extDir, 'manifest.json'));
@@ -1066,7 +1240,7 @@ router.put("/api/scripts/:name", requireAuth, requireScope("files:write"), (req,
1066
1240
  }
1067
1241
  });
1068
1242
 
1069
- router.post("/api/scripts/:name/run", requireAuth, requireScope("sandbox:run"), (req, res) => {
1243
+ router.post("/api/scripts/:name/run", sandboxRateLimiter(), requireAuth, requireScope("sandbox:run"), requireSandboxEnabled, async (req, res) => {
1070
1244
  try {
1071
1245
  const { name } = req.params;
1072
1246
  if (name.includes("..") || name.includes("/") || name.includes("\\")) {
@@ -1077,15 +1251,21 @@ router.post("/api/scripts/:name/run", requireAuth, requireScope("sandbox:run"),
1077
1251
  return res.status(404).json({ error: `Script "${name}" not found` });
1078
1252
  }
1079
1253
 
1080
- const isPython = name.endsWith(".py");
1081
- const isShell = name.endsWith(".sh");
1082
- const runner = isPython ? "python3" : isShell ? "bash" : "node";
1254
+ const isNodeScript = name.endsWith(".js") || name.endsWith(".mjs") || name.endsWith(".cjs");
1255
+ if (!isNodeScript) {
1256
+ return res.status(400).json({
1257
+ error: "Only Node.js scripts can be run through the sandbox endpoint."
1258
+ });
1259
+ }
1083
1260
 
1084
- const result = spawnSync(runner, [scriptPath], { encoding: "utf8", timeout: 10000 });
1261
+ const result = await runInSandbox(scriptPath, 10000, {
1262
+ allowNetwork: req.body?.allowNetwork === true
1263
+ });
1085
1264
  res.json({
1086
- success: result.status === 0,
1087
- output: result.stdout || result.stderr || "(no output)",
1088
- exitCode: result.status
1265
+ success: result.success,
1266
+ output: result.output || "(no output)",
1267
+ exitCode: result.code ?? null,
1268
+ signal: result.signal ?? null
1089
1269
  });
1090
1270
  } catch (err) {
1091
1271
  serverError(res, err);
@@ -1212,7 +1392,13 @@ router.get('/api/config-json', requireAuth, requireScope('config:read'), (req, r
1212
1392
  } catch {}
1213
1393
  }
1214
1394
 
1215
- res.json({ security, budget, brain });
1395
+ const safeBrain = { ...brain };
1396
+ if (safeBrain.token) {
1397
+ safeBrain.has_token = true;
1398
+ delete safeBrain.token;
1399
+ }
1400
+
1401
+ res.json({ security, budget, brain: safeBrain });
1216
1402
  } catch (err) { serverError(res, err); }
1217
1403
  });
1218
1404
 
@@ -1234,7 +1420,18 @@ router.post('/api/config-json', requireAuth, requireScope('config:write'), (req,
1234
1420
  fs.writeFileSync(budgetPath, yaml.stringify(budget), { encoding: 'utf8', mode: 0o600 });
1235
1421
  }
1236
1422
  if (brain) {
1237
- fs.writeFileSync(brainPath, JSON.stringify(brain, null, 2), { encoding: 'utf8', mode: 0o600 });
1423
+ let existingBrain = {};
1424
+ if (fs.existsSync(brainPath)) {
1425
+ try {
1426
+ existingBrain = JSON.parse(fs.readFileSync(brainPath, 'utf8')) || {};
1427
+ } catch {}
1428
+ }
1429
+ const nextBrain = { ...existingBrain, ...brain };
1430
+ if ((brain.token === undefined || brain.token === '') && existingBrain.token) {
1431
+ nextBrain.token = existingBrain.token;
1432
+ }
1433
+ delete nextBrain.has_token;
1434
+ fs.writeFileSync(brainPath, JSON.stringify(nextBrain, null, 2), { encoding: 'utf8', mode: 0o600 });
1238
1435
  }
1239
1436
 
1240
1437
  res.json({ success: true });
@@ -1510,7 +1707,7 @@ router.get('/api/brains', requireAuth, requireScope('ssss:read'), (req, res) =>
1510
1707
  let globalNodeCount = 0;
1511
1708
  if (fs.existsSync(globalVaultDir)) {
1512
1709
  try {
1513
- globalNodeCount = loadNodes(globalVaultDir).length;
1710
+ globalNodeCount = getNodes(globalVaultDir).length;
1514
1711
  } catch {
1515
1712
  // Count .md files manually
1516
1713
  const countMd = (dir) => {
@@ -1611,7 +1808,19 @@ router.get('/api/brains/:id/nodes', requireAuth, requireScope('ssss:read', 'memo
1611
1808
  return res.json({ nodes: [], brain_id: brainId });
1612
1809
  }
1613
1810
 
1614
- const nodes = loadNodes(vaultDir);
1811
+ let nodes = getNodes(vaultDir);
1812
+ const { q, category, status, tag } = req.query;
1813
+ if (q) {
1814
+ const query = String(q).toLowerCase();
1815
+ nodes = nodes.filter(n =>
1816
+ [n.slug, n.title, n.category, (n.tags || []).join(' '), n.body]
1817
+ .join(' ').toLowerCase().includes(query)
1818
+ );
1819
+ }
1820
+ if (category) nodes = nodes.filter(n => n.category === category);
1821
+ if (status) nodes = nodes.filter(n => n.status === status);
1822
+ if (tag) nodes = nodes.filter(n => (n.tags || []).includes(tag));
1823
+
1615
1824
  res.json({ nodes, brain_id: brainId, count: nodes.length });
1616
1825
  } catch (err) { serverError(res, err); }
1617
1826
  });