total-recall-brain 3.2.3 → 3.4.0

package/src/core/sandbox.mjs

@@ -1,10 +1,52 @@
-import { spawn } from 'child_process';
+import { spawn, spawnSync } from 'child_process';
 import path from 'path';
 import fs from 'fs';
 import os from 'os';
 import crypto from "node:crypto";
 import { logger } from './logger.mjs';
 
+let macSandboxProbe = null;
+
+function escapeSandboxPath(value) {
+  return String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+}
+
+function buildMacSandboxProfile({ allowNetwork }) {
+  return [
+    '(version 1)',
+    '(allow default)',
+    '(deny file-write*)',
+    '(allow file-write*',
+    `  (subpath "/private/tmp")`,
+    `  (subpath "/tmp")`,
+    `  (subpath "${escapeSandboxPath(process.cwd())}")`,
+    `  (subpath "${escapeSandboxPath(os.tmpdir())}")`,
+    `)`,
+    allowNetwork ? '' : '(deny network-outbound)'
+  ].filter(Boolean).join('\n');
+}
+
+function canUseMacSandbox(profile) {
+  if (process.platform !== 'darwin') return false;
+  if (macSandboxProbe !== null) return macSandboxProbe;
+  const result = spawnSync('sandbox-exec', [
+    '-p',
+    profile,
+    process.execPath,
+    '--no-warnings',
+    '-e',
+    'process.exit(0)'
+  ], {
+    encoding: 'utf8',
+    timeout: 1500
+  });
+  macSandboxProbe = result.status === 0;
+  if (!macSandboxProbe) {
+    logger.warn('sandbox', 'macOS sandbox-exec probe failed; falling back to Node process limits.');
+  }
+  return macSandboxProbe;
+}
+
 /**
  * Command Sanitizer / Whitelist Execution Validator
  * Blocks high-risk actions (destructive deletes, reverse shells, shell pipes)
@@ -62,33 +104,21 @@ export async function runInSandbox(scriptPath, timeoutMs = 5000, options = {}) {
   safeEnv.NODE_OPTIONS = '--experimental-vm-modules';
 
   // OS-specific sandboxing wrappers
-  let cmd = 'node';
+  let cmd = process.execPath;
   let args = ['--no-warnings', scriptPath];
   let tempSbProfilePath = null;
 
   try {
     if (process.platform === 'darwin') {
       // macOS sandbox-exec integration
-      const profile = [
-        '(version 1)',
-        '(allow default)',
-        '(deny file-write*)',
-        '(allow file-write*',
-        `  (subpath "/private/tmp")`,
-        `  (subpath "/tmp")`,
-        `  (subpath "${process.cwd()}")`,
-        `  (subpath "${os.tmpdir()}")`,
-        `)`,
-        allowNetwork 
-          ? '' 
-          : '(deny network-outbound)'
-      ].filter(Boolean).join('\n');
-
-      tempSbProfilePath = path.join(os.tmpdir(), `tr-sandbox-${Date.now()}-${crypto.randomBytes(4).toString("hex")}.sb`);
-      fs.writeFileSync(tempSbProfilePath, profile, 'utf8');
-
-      cmd = 'sandbox-exec';
-      args = ['-f', tempSbProfilePath, 'node', '--no-warnings', scriptPath];
+      const profile = buildMacSandboxProfile({ allowNetwork });
+      if (canUseMacSandbox(profile)) {
+        tempSbProfilePath = path.join(os.tmpdir(), `tr-sandbox-${Date.now()}-${crypto.randomBytes(4).toString("hex")}.sb`);
+        fs.writeFileSync(tempSbProfilePath, profile, 'utf8');
+
+        cmd = 'sandbox-exec';
+        args = ['-f', tempSbProfilePath, process.execPath, '--no-warnings', scriptPath];
+      }
     } else if (process.platform === 'linux') {
       // Linux namespace isolation check via unshare
       const hasUnshare = fs.existsSync('/usr/bin/unshare') || fs.existsSync('/bin/unshare');
@@ -98,7 +128,7 @@ export async function runInSandbox(scriptPath, timeoutMs = 5000, options = {}) {
         if (!allowNetwork) {
           unshareArgs.push('--net');
         }
-        args = [...unshareArgs, 'node', '--no-warnings', scriptPath];
+        args = [...unshareArgs, process.execPath, '--no-warnings', scriptPath];
       }
     }
   } catch (err) {
@@ -117,14 +147,14 @@ export async function runInSandbox(scriptPath, timeoutMs = 5000, options = {}) {
     proc.stdout.on('data', (data) => { stdout += data.toString(); });
     proc.stderr.on('data', (data) => { stderr += data.toString(); });
 
-    proc.on('close', (code) => {
+    proc.on('close', (code, signal) => {
       if (tempSbProfilePath && fs.existsSync(tempSbProfilePath)) {
         try { fs.unlinkSync(tempSbProfilePath); } catch {}
       }
       if (code === 0) {
-        resolve({ success: true, output: stdout });
+        resolve({ success: true, output: stdout, code, signal });
       } else {
-        resolve({ success: false, output: stderr || stdout, code });
+        resolve({ success: false, output: stderr || stdout, code, signal });
       }
     });
 

package/src/core/scheduler.mjs

@@ -2,7 +2,8 @@ import fs from 'fs';
 import path from 'path';
 import matter from 'gray-matter';
 import crypto from 'crypto';
-import { loadNodes, atomicWrite, safeStringify } from './vault.mjs';
+import { atomicWrite, safeStringify } from './vault.mjs';
+import { getNodes } from './vault-cache.mjs';
 import { logger } from './logger.mjs';
 import { loadQueue, updateQueueItem } from './research-queue.mjs';
 
@@ -243,7 +244,7 @@ function generateProactiveResearchTask() {
  * Pick a random active memory node and create a clarity review task.
  */
 function generateClarityReviewTask(vaultDir) {
-  const nodes = loadNodes(vaultDir).filter((n) => n.status === 'active');
+  const nodes = getNodes(vaultDir).filter((n) => n.status === 'active');
   if (nodes.length === 0) {
     return makeFallbackTask('memory-maintenance', 'No active nodes to review');
   }
@@ -266,7 +267,7 @@ function generateClarityReviewTask(vaultDir) {
  * Pick the oldest fact node and create a staleness check task.
  */
 function generateStalenessCheckTask(vaultDir) {
-  const nodes = loadNodes(vaultDir)
+  const nodes = getNodes(vaultDir)
     .filter((n) => n.status === 'active' && n.category === 'facts')
     .sort((a, b) => {
       const aDate = new Date(a.last_accessed || a.updated || 0).getTime();
@@ -276,7 +277,7 @@ function generateStalenessCheckTask(vaultDir) {
 
   if (nodes.length === 0) {
     // No facts? Pick any old node
-    const allNodes = loadNodes(vaultDir).filter((n) => n.status === 'active');
+    const allNodes = getNodes(vaultDir).filter((n) => n.status === 'active');
     if (allNodes.length === 0) {
       return makeFallbackTask('research-acquisition', 'No facts to verify');
     }
@@ -314,7 +315,7 @@ function generateStalenessCheckTask(vaultDir) {
  * Pick a cluster of related nodes and create an inference task.
  */
 function generateInferenceTask(vaultDir) {
-  const nodes = loadNodes(vaultDir).filter((n) => n.status === 'active');
+  const nodes = getNodes(vaultDir).filter((n) => n.status === 'active');
   if (nodes.length < 3) {
     return makeFallbackTask('system2-deliberation', 'Not enough nodes for inference');
   }

package/src/core/schema.mjs

@@ -43,7 +43,7 @@ export const MemoryNodeSchema = z.object({
   routes_to_skills: z.array(z.string()),
   sentiment_polarity: z.enum(['directive_must', 'directive_must_not', 'descriptive', 'preference']),
   sentiment_target: z.string(),
-  modality: z.enum(['must', 'must_not', 'should', 'should_not']),
+  modality: z.enum(['must', 'must_not', 'should', 'should_not', 'descriptive', 'preference']),
   subject: z.string().regex(/^[a-zA-Z0-9_\s.-]+$/),
   predicate: z.string().regex(/^[a-zA-Z0-9_\s.-]+$/),
   object: z.string(),
@@ -86,7 +86,7 @@ export const MemoryNodeSchema = z.object({
   // Temporal rule expiration
   expires_at: ssssDatetimeNullable().optional(),
   // Absolute Invariant Extensions
-  priority: z.literal('absolute').optional(),
+  priority: z.enum(['absolute', 'high', 'normal', 'low']).optional(),
   immutable: z.boolean().optional()
 });
 

package/src/core/search.mjs

@@ -7,13 +7,70 @@
  * MCP and REST are thin wrappers; all logic lives here.
  */
 
-import { loadNodes } from './vault.mjs';
+import { getNodes } from './vault-cache.mjs';
 import {
   getEmbedding,
   cosineSimilarity,
   loadEmbeddingsIndex,
   loadSessionEmbeddingsIndex,
 } from './embeddings.mjs';
+import { inferMemoryLayer, memoryLayerRoutingWeight } from './memory-layers.mjs';
+import { VectorStore } from './vector-store.mjs';
+
+/**
+ * Compute a temporal relevance boost for a vault node.
+ * Blends recency, confidence, access frequency, importance,
+ * memory layer, and priority into a multiplier applied to
+ * cosine similarity scores.
+ *
+ * Range: ~0.2 (cold/stale/research-tier) to ~2.0 (hot/absolute/conscious)
+ * Neutral (no temporal data): 1.0 (pure cosine similarity)
+ *
+ * @param {object|null} node - The vault node with frontmatter fields
+ * @returns {number} Temporal boost multiplier
+ */
+function computeTemporalBoost(node) {
+  if (!node) return 1.0;
+  const now = Date.now();
+
+  // ── Recency: exponential decay from last_accessed ──
+  // Uses the node's own half_life_days if set, otherwise 7 days default
+  let recency = 1.0;
+  const lastAccessed = node.last_accessed || node.updated || node.created;
+  if (lastAccessed) {
+    const ageMs = now - new Date(lastAccessed).getTime();
+    const halfLifeDays = node.decay?.half_life_days || 7;
+    const halfLifeMs = halfLifeDays * 24 * 60 * 60 * 1000;
+    recency = Math.pow(0.5, ageMs / halfLifeMs);
+    recency = Math.max(recency, 0.05); // floor: never fully invisible
+  }
+
+  // ── Confidence: linear weight ──
+  // confidence 1.0 → 1.0, confidence 0.5 → 0.75, confidence 0 → 0.5
+  const confidence = typeof node.confidence === 'number' ? node.confidence : 1.0;
+  const confidenceWeight = 0.5 + (confidence * 0.5);
+
+  // ── Access frequency: log scale (from decay.access_count) ──
+  // 0 accesses → 1.0, 10 accesses → 1.23, 100 accesses → 1.46
+  const accessCount = node.decay?.access_count ?? 0;
+  const frequencySignal = 1.0 + (Math.log10(accessCount + 1) * 0.2);
+
+  // ── Importance tier: linear boost ──
+  // importance 1 → 0.8, 3 → 1.0, 5 → 1.2
+  const importance = typeof node.importance === 'number' ? node.importance : 3;
+  const importanceBoost = 0.8 + ((importance - 1) * 0.1);
+
+  // ── Priority override ──
+  // absolute priority nodes get a fixed boost to ensure they rank highly
+  const priorityBoost = node.priority === 'absolute' ? 1.3 : 1.0;
+
+  // ── Memory layer routing weight ──
+  // conscious=1.25, system2=1.0, research=0.75
+  const layer = inferMemoryLayer(node);
+  const layerWeight = memoryLayerRoutingWeight(layer);
+
+  return recency * confidenceWeight * frequencySignal * importanceBoost * priorityBoost * layerWeight;
+}
 
 /**
  * Run a semantic search across vault nodes and/or session history.
@@ -44,125 +101,134 @@ export async function semanticSearch(query, {
   priority = null
 } = {}) {
   const k = Math.min(Number(top_k) || 5, 20);
+
+  // 1. Load nodes and apply filters
+  const allNodes = getNodes(vaultDir);
+  let filteredNodes = allNodes;
+
+  if (category) {
+    const catLower = category.toLowerCase();
+    filteredNodes = filteredNodes.filter(n => n.category && n.category.toLowerCase() === catLower);
+  }
+  if (tags && tags.length > 0) {
+    filteredNodes = filteredNodes.filter(n => n.tags && tags.some(t => n.tags.includes(t)));
+  }
+  if (modality) {
+    const modLower = modality.toLowerCase();
+    filteredNodes = filteredNodes.filter(n => n.modality && n.modality.toLowerCase() === modLower);
+  }
+  if (importance !== null && importance !== undefined) {
+    const minImp = parseInt(importance, 10);
+    filteredNodes = filteredNodes.filter(n => {
+      const val = parseInt(n.importance, 10);
+      return !isNaN(val) && val >= minImp;
+    });
+  }
+  if (priority) {
+    const prioLower = priority.toLowerCase();
+    filteredNodes = filteredNodes.filter(n => n.priority && n.priority.toLowerCase() === prioLower);
+  }
+
+  const filteredSlugs = new Set(filteredNodes.map(n => n.slug));
+  const results = [];
+
+  // Get query embedding
   let queryEmbedding = null;
   try {
     queryEmbedding = await getEmbedding(String(query));
   } catch (err) {
-    // Gracefully ignore embedding generation errors to allow text search fallback
+    // Gracefully ignore embedding generation errors
   }
-  const results = [];
 
-  // If metadata filters are active, session log search is automatically bypassed
-  const hasMetaFilters = category || (tags && tags.length > 0) || modality || importance !== null || priority;
-  const activeIncludeSessions = hasMetaFilters ? false : includeSessions;
-
-  // ── Vault nodes ────────────────────────────────────────────────────────────
   const vaultIndex = loadEmbeddingsIndex(derivedDir);
   const vaultEntries = Object.entries(vaultIndex);
 
+  // 2. Perform Semantic Search Channel (if embedding & index available)
+  let semanticRanked = [];
   if (queryEmbedding && vaultEntries.length > 0) {
-    let filteredNodes = loadNodes(vaultDir);
-
-    if (category) {
-      const catLower = category.toLowerCase();
-      filteredNodes = filteredNodes.filter(n => n.category && n.category.toLowerCase() === catLower);
-    }
-    if (tags && tags.length > 0) {
-      filteredNodes = filteredNodes.filter(n => n.tags && tags.some(t => n.tags.includes(t)));
-    }
-    if (modality) {
-      const modLower = modality.toLowerCase();
-      filteredNodes = filteredNodes.filter(n => n.modality && n.modality.toLowerCase() === modLower);
-    }
-    if (importance !== null && importance !== undefined) {
-      const minImp = parseInt(importance, 10);
-      filteredNodes = filteredNodes.filter(n => {
-        const val = parseInt(n.importance, 10);
-        return !isNaN(val) && val >= minImp;
-      });
-    }
-    if (priority) {
-      const prioLower = priority.toLowerCase();
-      filteredNodes = filteredNodes.filter(n => n.priority && n.priority.toLowerCase() === prioLower);
-    }
+    const store = new VectorStore();
+    store.load(vaultIndex);
+    const matches = store.search(queryEmbedding, filteredNodes.length, filteredSlugs, cosineSimilarity);
+    
+    semanticRanked = matches.map(({ slug, similarity }) => {
+      const node = filteredNodes.find(n => n.slug === slug);
+      const temporal = computeTemporalBoost(node);
+      return { slug, score: similarity * temporal };
+    });
+  }
 
-    const filteredSlugs = new Set(filteredNodes.map(n => n.slug));
+  // 3. Perform Lexical Search Channel (Keyword matcher)
+  const qLower = String(query).toLowerCase();
+  let lexicalRanked = [];
+  for (const node of filteredNodes) {
+    let rawLexicalScore = 0;
+    const slug = String(node.slug || '').toLowerCase();
+    const title = String(node.title || '').toLowerCase();
+    const body = String(node.body || '').toLowerCase();
+    const nTags = Array.isArray(node.tags) ? node.tags.map(t => String(t).toLowerCase()) : [];
 
-    const scored = vaultEntries
-      .filter(([slug]) => filteredSlugs.has(slug))
-      .map(([slug, { embedding }]) => ({ slug, score: cosineSimilarity(queryEmbedding, embedding) }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, k);
+    if (slug === qLower || title === qLower) {
+      rawLexicalScore = 1.0;
+    } else if (slug.includes(qLower) || title.includes(qLower)) {
+      rawLexicalScore = 0.8;
+    } else if (nTags.some(t => t.includes(qLower))) {
+      rawLexicalScore = 0.7;
+    } else if (body.includes(qLower)) {
+      rawLexicalScore = 0.5;
+    }
 
-    for (const { slug, score } of scored) {
-      const node = filteredNodes.find(n => n.slug === slug);
-      if (node) {
-        results.push({ ...node, type: 'vault', score: Math.round(score * 1000) / 1000 });
-      }
+    if (rawLexicalScore > 0) {
+      const temporal = computeTemporalBoost(node);
+      lexicalRanked.push({ slug: node.slug, score: rawLexicalScore * temporal });
     }
-  } else {
-    // ── Text Search Fallback for Vault Nodes ──
-    try {
-      let filteredNodes = loadNodes(vaultDir);
+  }
+  lexicalRanked.sort((a, b) => b.score - a.score);
 
-      if (category) {
-        const catLower = category.toLowerCase();
-        filteredNodes = filteredNodes.filter(n => n.category && n.category.toLowerCase() === catLower);
-      }
-      if (tags && tags.length > 0) {
-        filteredNodes = filteredNodes.filter(n => n.tags && tags.some(t => n.tags.includes(t)));
-      }
-      if (modality) {
-        const modLower = modality.toLowerCase();
-        filteredNodes = filteredNodes.filter(n => n.modality && n.modality.toLowerCase() === modLower);
-      }
-      if (importance !== null && importance !== undefined) {
-        const minImp = parseInt(importance, 10);
-        filteredNodes = filteredNodes.filter(n => {
-          const val = parseInt(n.importance, 10);
-          return !isNaN(val) && val >= minImp;
-        });
-      }
-      if (priority) {
-        const prioLower = priority.toLowerCase();
-        filteredNodes = filteredNodes.filter(n => n.priority && n.priority.toLowerCase() === prioLower);
-      }
+  // 4. Blend Rankings via Reciprocal Rank Fusion (RRF)
+  let blendedRanked = [];
+  if (semanticRanked.length > 0) {
+    const rrfMap = new Map();
+    const RRF_K = 60; // Standard RRF parameter constant
 
-      const qLower = String(query).toLowerCase();
-      const scoredText = [];
-
-      for (const node of filteredNodes) {
-        let score = 0;
-        const slug = String(node.slug || '').toLowerCase();
-        const title = String(node.title || '').toLowerCase();
-        const body = String(node.body || '').toLowerCase();
-        const nTags = Array.isArray(node.tags) ? node.tags.map(t => String(t).toLowerCase()) : [];
-
-        if (slug === qLower || title === qLower) {
-          score = 0.9;
-        } else if (slug.includes(qLower) || title.includes(qLower)) {
-          score = 0.7;
-        } else if (nTags.some(t => t.includes(qLower))) {
-          score = 0.6;
-        } else if (body.includes(qLower)) {
-          score = 0.5;
-        }
+    // Index rank positions (1-indexed)
+    semanticRanked.forEach((item, index) => {
+      rrfMap.set(item.slug, 1 / (RRF_K + (index + 1)));
+    });
 
-        if (score > 0) {
-          scoredText.push({ node, score });
-        }
-      }
+    lexicalRanked.forEach((item, index) => {
+      const semScore = rrfMap.get(item.slug) || 0;
+      rrfMap.set(item.slug, semScore + (1 / (RRF_K + (index + 1))));
+    });
 
-      scoredText.sort((a, b) => b.score - a.score);
-      for (const { node, score } of scoredText.slice(0, k)) {
-        results.push({ ...node, type: 'vault', score });
-      }
-    } catch {
-      // Gracefully ignore filesystem reading issues
+    // Convert back, score, and sort
+    blendedRanked = Array.from(rrfMap.entries())
+      .map(([slug, rrfScore]) => {
+        const node = filteredNodes.find(n => n.slug === slug);
+        // Normalize RRF score to [0, 1] relative to theoretical max of 2/60 (rank 1 in both)
+        const maxPossibleRrf = 2 / RRF_K;
+        const normalizedScore = rrfScore / maxPossibleRrf;
+        return { node, score: Math.round(normalizedScore * 1000) / 1000 };
+      })
+      .sort((a, b) => b.score - a.score);
+  } else {
+    // If semantic search is unavailable, fallback entirely to lexical ranking
+    blendedRanked = lexicalRanked.map(item => {
+      const node = filteredNodes.find(n => n.slug === item.slug);
+      return { node, score: Math.round(item.score * 1000) / 1000 };
+    });
+  }
+
+  // Slice top results
+  for (const { node, score } of blendedRanked.slice(0, k)) {
+    if (node) {
+      results.push({ ...node, type: 'vault', score });
     }
   }
 
-  // ── Sessions ───────────────────────────────────────────────────────────────
+  // 5. Sessions Search Channel
+  const hasMetaFilters = category || (tags && tags.length > 0) || modality || importance !== null || priority;
+  const activeIncludeSessions = hasMetaFilters ? false : includeSessions;
+
   if (activeIncludeSessions) {
     const sessionIndex = loadSessionEmbeddingsIndex(derivedDir);
     const sessionEntries = Object.entries(sessionIndex);
@@ -184,9 +250,8 @@ export async function semanticSearch(query, {
         results.push({ type: 'session', key, session_id, snippet, chunk, total_chunks, score: Math.round(score * 1000) / 1000 });
       }
     } else {
-      // ── Text Search Fallback for Sessions ──
+      // Lexical Fallback for Sessions
       try {
-        const qLower = String(query).toLowerCase();
         const scoredText = [];
         for (const [key, entry] of sessionEntries) {
           const snippet = String(entry.snippet || '').toLowerCase();
@@ -215,5 +280,6 @@ export async function semanticSearch(query, {
   if (!queryEmbedding || vaultEntries.length === 0) {
     finalResults.degradedTextSearch = true;
   }
+
   return finalResults;
 }