total-recall-brain 3.1.0

package/src/core/inference-engine.mjs

@@ -0,0 +1,367 @@
+import fs from 'fs';
+import path from 'path';
+import matter from 'gray-matter';
+import crypto from 'crypto';
+import { callLocalRuntime, cleanAndParseJSON } from './runtime.mjs';
+import { loadNodes, atomicWrite, safeStringify } from './vault.mjs';
+import { logger } from './logger.mjs';
+import { brainDir } from './config.mjs';
+
+/**
+ * Total Recall System 2 Inference Engine
+ *
+ * Reads clusters of related memory nodes and asks the local LLM to:
+ *   1. Draw higher-level conclusions
+ *   2. Identify implicit patterns that should be explicit rules
+ *   3. Find subtle contradictions that Jaccard wouldn't catch
+ *
+ * Produces new nodes with category: decisions or concepts,
+ * x_memory_layer: system2, status: draft in the inbox.
+ * All outputs are proposals — never direct mutations.
+ */
+
+// ─── Prompts ────────────────────────────────────────────────────────────────────
+
+const INFERENCE_SYSTEM = `You are a System 2 Reasoning Engine for a sovereign AI memory system called Total Recall.
+
+You will be given a cluster of related memory nodes. Your job is to reason deeply about them and produce higher-level conclusions.
+
+Output valid JSON matching this schema:
+{
+  "conclusions": [
+    {
+      "title": "string",
+      "body": "string",
+      "category": "decisions" | "concepts" | "patterns",
+      "confidence": 0.0-1.0,
+      "reasoning": "string"
+    }
+  ],
+  "contradictions": [
+    {
+      "node_a": "slug string",
+      "node_b": "slug string",
+      "description": "string"
+    }
+  ],
+  "merge_candidates": [
+    {
+      "node_a": "slug string",
+      "node_b": "slug string",
+      "reason": "string"
+    }
+  ]
+}
+
+Guidelines:
+- Only include conclusions that are genuinely new — not just summaries of existing nodes.
+- Confidence reflects how well the evidence supports the conclusion.
+- "contradictions" are only real logical contradictions, not just different topics.
+- "merge_candidates" are nodes that say the same thing differently.
+- Output ONLY valid JSON.`;
+
+function buildInferencePrompt(nodes) {
+  const nodeTexts = nodes.map(n => [
+    `### [${n.slug}] ${n.title}`,
+    `Category: ${n.category} | Confidence: ${n.confidence} | Layer: ${n.x_memory_layer || 'inferred'}`,
+    (n.body || '').slice(0, 800),
+  ].join('\n')).join('\n\n');
+
+  return `Analyze these ${nodes.length} related memory nodes and reason about them:\n\n${nodeTexts}\n\nDraw higher-level conclusions, identify contradictions, and flag merge candidates.`;
+}
+
+// ─── Inference Execution ────────────────────────────────────────────────────────
+
+/**
+ * Run inference over a set of node slugs.
+ *
+ * @param {string[]} slugs     Node slugs to analyze
+ * @param {object} opts
+ * @param {string} opts.vaultDir     Path to memory vault
+ * @param {string} opts.inboxDir     Path to memory inbox
+ * @param {object} opts.runtimeConfig Local LLM config
+ * @returns {{ conclusions: object[], contradictions: object[], merge_candidates: object[], error? }}
+ */
+export async function runInferenceTask(slugs, { vaultDir, inboxDir, runtimeConfig }) {
+  const allNodes = loadNodes(vaultDir);
+  const nodes = allNodes.filter(n => slugs.includes(n.slug) && n.status === 'active');
+
+  if (nodes.length < 2) {
+    return { conclusions: [], contradictions: [], merge_candidates: [], skipped: 'too-few-nodes' };
+  }
+
+  const prompt = buildInferencePrompt(nodes);
+
+  let result;
+  try {
+    const rawResponse = await callLocalRuntime(prompt, INFERENCE_SYSTEM, runtimeConfig);
+    const jsonMatch = rawResponse.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) throw new Error('No JSON in response');
+    result = cleanAndParseJSON(jsonMatch[0]);
+  } catch (err) {
+    logger.info({
+      subsystem: 'inference-engine',
+      message: `Inference failed for slugs [${slugs.join(', ')}]: ${err.message}`,
+    });
+    return { conclusions: [], contradictions: [], merge_candidates: [], error: err.message };
+  }
+
+  const conclusions = Array.isArray(result.conclusions) ? result.conclusions : [];
+  const contradictions = Array.isArray(result.contradictions) ? result.contradictions : [];
+  const mergeCandidates = Array.isArray(result.merge_candidates) ? result.merge_candidates : [];
+
+  // Write conclusions as draft nodes in the inbox
+  if (!fs.existsSync(inboxDir)) {
+    fs.mkdirSync(inboxDir, { recursive: true });
+  }
+
+  for (const conclusion of conclusions) {
+    writeConclusionNode(inboxDir, conclusion, slugs, vaultDir);
+  }
+
+  // Write contradictions as conflict records
+  if (contradictions.length > 0) {
+    const conflictsDir = path.join(brainDir, 'memory-inbox', 'conflicts');
+    for (const contradiction of contradictions) {
+      writeContradictionRecord(conflictsDir, contradiction);
+    }
+  }
+
+  // Write merge proposals
+  if (mergeCandidates.length > 0) {
+    const proposalsDir = path.join(vaultDir, 'proposals');
+    for (const candidate of mergeCandidates) {
+      writeMergeProposal(proposalsDir, candidate);
+    }
+  }
+
+  logger.info({
+    subsystem: 'inference-engine',
+    message: `Inference complete: ${conclusions.length} conclusions, ${contradictions.length} contradictions, ${mergeCandidates.length} merge candidates`,
+  });
+
+  return { conclusions, contradictions, merge_candidates: mergeCandidates };
+}
+
+// ─── Node Writers ───────────────────────────────────────────────────────────────
+
+function writeConclusionNode(inboxDir, conclusion, sourceSlugList, vaultDir) {
+  const slug = `inference-${crypto.randomBytes(4).toString('hex')}`;
+  const now = new Date().toISOString();
+
+  let latestPublished = null;
+  let citations = null;
+  
+  if (vaultDir) {
+    try {
+      const allNodes = loadNodes(vaultDir);
+      citations = sourceSlugList.map(s => {
+        const matchingNode = allNodes.find(n => n.slug === s);
+        const cat = matchingNode ? matchingNode.category : 'facts';
+        const published = matchingNode ? (matchingNode.updated || matchingNode.created || now) : now;
+        if (!latestPublished || new Date(published) > new Date(latestPublished)) {
+          latestPublished = published;
+        }
+        return {
+          source: 'memory-vault',
+          title: matchingNode ? matchingNode.title : `Memory Node: ${s}`,
+          url: `file://${path.join(vaultDir, cat, s + '.md')}`,
+          published,
+          relevance: 1.0,
+          accessed: now
+        };
+      });
+    } catch (err) {
+      logger.info({
+        subsystem: 'inference-engine',
+        message: `Failed to build citations for conclusion node: ${err.message}`
+      });
+    }
+  }
+
+  if (!citations) {
+    citations = sourceSlugList.map(s => ({
+      source: 'memory-vault',
+      title: `Memory Node: ${s}`,
+      url: `memory://${s}`,
+      published: now,
+      relevance: 1.0,
+      accessed: now
+    }));
+  }
+
+  const temporalContext = latestPublished || now;
+
+  const frontmatter = {
+    type: 'memory',
+    slug,
+    category: conclusion.category || 'concepts',
+    title: conclusion.title || 'Inferred Conclusion',
+    status: 'draft',
+    confidence: Math.min(1.0, Math.max(0.0, Number(conclusion.confidence) || 0.7)),
+    importance: 3,
+    created: now,
+    updated: now,
+    last_accessed: now,
+    source: {
+      type: 'dream-cycle',
+      session_id: `inference-${Date.now().toString(36)}`,
+      agent: 'inference-engine',
+      evidence_count: sourceSlugList.length,
+    },
+    supersedes: [],
+    superseded_by: null,
+    contradicts: [],
+    tags: ['inference', 'system2', 'auto-generated'],
+    related: sourceSlugList,
+    routes_to_skills: [],
+    sentiment_polarity: 'descriptive',
+    sentiment_target: 'knowledge',
+    modality: 'should',
+    subject: 'agent',
+    predicate: 'conclude',
+    object: 'knowledge',
+    decay: { half_life_days: 180, access_count: 1 },
+    schema_version: 2,
+    x_memory_layer: 'system2',
+    x_reasoning: (conclusion.reasoning || '').slice(0, 500),
+    x_temporal_context: temporalContext,
+    x_citations: citations,
+  };
+
+  const body = conclusion.body || '';
+  atomicWrite(path.join(inboxDir, `${slug}.md`), safeStringify(body, frontmatter));
+}
+
+function writeContradictionRecord(conflictsDir, contradiction) {
+  if (!fs.existsSync(conflictsDir)) {
+    fs.mkdirSync(conflictsDir, { recursive: true });
+  }
+
+  const slug = `contradiction-${crypto.randomBytes(4).toString('hex')}`;
+  const now = new Date().toISOString();
+
+  const data = {
+    type: 'conflict',
+    slug,
+    status: 'detected',
+    detected_at: now,
+    detected_by: 'inference-engine',
+    node_a: contradiction.node_a,
+    node_b: contradiction.node_b,
+    description: contradiction.description || 'Logical contradiction detected by inference engine',
+  };
+
+  atomicWrite(
+    path.join(conflictsDir, `${slug}.md`),
+    safeStringify('', data),
+  );
+}
+
+function writeMergeProposal(proposalsDir, candidate) {
+  if (!fs.existsSync(proposalsDir)) {
+    fs.mkdirSync(proposalsDir, { recursive: true });
+  }
+
+  const slug = `merge-proposal-${crypto.randomBytes(4).toString('hex')}`;
+  const now = new Date().toISOString();
+
+  const data = {
+    type: 'proposal',
+    slug,
+    category: 'memory-cleanup',
+    status: 'draft',
+    proposed_by: 'inference-engine',
+    proposed_at: now,
+    target_path: `${candidate.node_a},${candidate.node_b}`,
+    summary: `Merge ${candidate.node_a} and ${candidate.node_b}`,
+    rationale: candidate.reason || 'Nodes appear to express the same concept differently.',
+  };
+
+  atomicWrite(
+    path.join(proposalsDir, `${slug}.md`),
+    safeStringify('', data),
+  );
+}
+
+// ─── Memory Synthesizer ─────────────────────────────────────────────────────────
+
+const SYNTHESIS_SYSTEM = `You are a Memory Synthesizer for a sovereign AI memory system.
+
+Given two memory nodes that may be similar, determine:
+1. Are they saying the same thing? → "MERGE"
+2. Are they complementary? → "LINK"
+3. Are they actually about different things? → "KEEP"
+
+Output valid JSON:
+{
+  "verdict": "MERGE" | "LINK" | "KEEP",
+  "reason": "string",
+  "merged_title": "string (only if MERGE)",
+  "merged_body": "string (only if MERGE)"
+}`;
+
+/**
+ * Compare two nodes semantically and suggest merge/link/keep.
+ *
+ * @param {string} slugA   First node slug
+ * @param {string} slugB   Second node slug
+ * @param {object} opts
+ * @param {string} opts.vaultDir
+ * @param {string} opts.inboxDir
+ * @param {object} opts.runtimeConfig
+ */
+export async function runSynthesisTask(slugA, slugB, { vaultDir, inboxDir, runtimeConfig }) {
+  const allNodes = loadNodes(vaultDir);
+  const nodeA = allNodes.find(n => n.slug === slugA);
+  const nodeB = allNodes.find(n => n.slug === slugB);
+
+  if (!nodeA || !nodeB) {
+    return { verdict: 'KEEP', error: 'One or both nodes not found' };
+  }
+
+  const prompt = [
+    `## Node A: ${nodeA.title}`,
+    (nodeA.body || '').slice(0, 1000),
+    '',
+    `## Node B: ${nodeB.title}`,
+    (nodeB.body || '').slice(0, 1000),
+    '',
+    'Should these nodes be MERGED, LINKED, or KEPT separate?',
+  ].join('\n');
+
+  try {
+    const rawResponse = await callLocalRuntime(prompt, SYNTHESIS_SYSTEM, runtimeConfig);
+    const jsonMatch = rawResponse.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) throw new Error('No JSON in response');
+    const result = cleanAndParseJSON(jsonMatch[0]);
+
+    if (result.verdict === 'MERGE' && result.merged_title) {
+      writeConclusionNode(inboxDir, {
+        title: result.merged_title,
+        body: result.merged_body || '',
+        category: nodeA.category,
+        confidence: Math.max(nodeA.confidence || 0, nodeB.confidence || 0),
+        reasoning: `Merged from ${slugA} and ${slugB}: ${result.reason}`,
+      }, [slugA, slugB], vaultDir);
+
+      logger.info({
+        subsystem: 'inference-engine',
+        message: `MERGE proposed: ${slugA} + ${slugB} → ${result.merged_title}`,
+      });
+    } else if (result.verdict === 'LINK') {
+      logger.info({
+        subsystem: 'inference-engine',
+        message: `LINK suggested: ${slugA} ↔ ${slugB}`,
+      });
+    }
+
+    return result;
+  } catch (err) {
+    logger.info({
+      subsystem: 'inference-engine',
+      message: `Synthesis failed for ${slugA}/${slugB}: ${err.message}`,
+    });
+    return { verdict: 'KEEP', error: err.message };
+  }
+}

package/src/core/logger.mjs

@@ -0,0 +1,75 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { EventEmitter } from 'node:events';
+import { brainDir } from './config.mjs';
+
+const BRAIN_DIR = brainDir;
+const LOG_DIR = path.join(BRAIN_DIR, 'logs');
+
+if (!fs.existsSync(LOG_DIR)) {
+  fs.mkdirSync(LOG_DIR, { recursive: true });
+}
+
+const getLogFile = () => {
+  const d = new Date();
+  return path.join(LOG_DIR, `system-${d.toISOString().split('T')[0]}.jsonl`);
+};
+
+export const logEvents = new EventEmitter();
+logEvents.setMaxListeners(50);
+
+export const logger = {
+  log: (subsystemOrEntry, level, message, meta = {}) => {
+    let entry;
+    if (typeof subsystemOrEntry === 'object' && subsystemOrEntry !== null) {
+      entry = {
+        timestamp: new Date().toISOString(),
+        level: level || subsystemOrEntry.level || 'info',
+        subsystem: subsystemOrEntry.subsystem || 'unknown',
+        message: subsystemOrEntry.message || '',
+        ...subsystemOrEntry
+      };
+    } else if (typeof subsystemOrEntry === 'string' && message === undefined) {
+      entry = {
+        timestamp: new Date().toISOString(),
+        level: level || 'info',
+        subsystem: 'system',
+        message: subsystemOrEntry,
+        ...meta
+      };
+    } else {
+      entry = {
+        timestamp: new Date().toISOString(),
+        level: level || 'info',
+        subsystem: subsystemOrEntry || 'unknown',
+        message: message || '',
+        ...meta
+      };
+    }
+
+    const line = JSON.stringify(entry) + '\n';
+
+    // Server rule: stderr only
+    console.error(line.trim());
+
+    try {
+      fs.appendFileSync(getLogFile(), line);
+    } catch (e) {
+      // If we fail to write to log file, we still printed to stderr
+    }
+
+    // Emit so watchdog and other in-process monitors can react without tailing the file.
+    try {
+      logEvents.emit('log', entry);
+    } catch (_) {
+      // Listeners must not break the logger.
+    }
+  },
+  info: (subsystem, message, meta) => logger.log(subsystem, 'info', message, meta),
+  warn: (subsystem, message, meta) => logger.log(subsystem, 'warn', message, meta),
+  error: (subsystem, message, meta) => logger.log(subsystem, 'error', message, meta),
+  debug: (subsystem, message, meta) => logger.log(subsystem, 'debug', message, meta)
+};
+
+export { LOG_DIR, getLogFile };

package/src/core/memory-layers.mjs

@@ -0,0 +1,96 @@
+export const MEMORY_LAYERS = Object.freeze(['conscious', 'system2', 'research']);
+
+export const DEFAULT_MEMORY_LAYER = 'system2';
+
+const LAYER_ALIASES = new Map([
+  ['conscious', 'conscious'],
+  ['awareness', 'conscious'],
+  ['working-memory', 'conscious'],
+  ['working_memory', 'conscious'],
+  ['working memory', 'conscious'],
+  ['hot', 'conscious'],
+  ['system2', 'system2'],
+  ['system-2', 'system2'],
+  ['system_2', 'system2'],
+  ['system 2', 'system2'],
+  ['deliberation', 'system2'],
+  ['reasoning', 'system2'],
+  ['research', 'research'],
+  ['knowledge-acquisition', 'research'],
+  ['knowledge_acquisition', 'research'],
+  ['knowledge acquisition', 'research'],
+  ['acquisition', 'research'],
+]);
+
+const CONSCIOUS_CATEGORIES = new Set(['invariants', 'patterns', 'anti-patterns', 'preferences']);
+const SYSTEM2_CATEGORIES = new Set(['decisions', 'concepts', 'proposals']);
+const RESEARCH_CATEGORIES = new Set(['facts', 'research', 'lore']);
+const SYSTEM2_TAGS = new Set(['system2', 'system-2', 'deliberation', 'reasoning', 'synthesis', 'validation']);
+const RESEARCH_TAGS = new Set(['research', 'web', 'knowledge-acquisition', 'stale-knowledge-refresh']);
+const RESEARCH_SOURCES = new Set(['web-search', 'deep-research', 'research', 'external-citation']);
+
+export function normalizeMemoryLayer(layer) {
+  if (typeof layer !== 'string') return null;
+  const trimmed = layer.trim().toLowerCase();
+  if (!trimmed) return null;
+  return LAYER_ALIASES.get(trimmed) || null;
+}
+
+function normalizedTags(node) {
+  if (!Array.isArray(node?.tags)) return new Set();
+  return new Set(node.tags.map((tag) => String(tag).trim().toLowerCase()).filter(Boolean));
+}
+
+function hasAny(tags, candidates) {
+  for (const tag of tags) {
+    if (candidates.has(tag)) return true;
+  }
+  return false;
+}
+
+export function inferMemoryLayer(node = {}) {
+  const explicit = normalizeMemoryLayer(
+    node.x_memory_layer || node.memory_layer || node.cognitive_layer
+  );
+  if (explicit) return explicit;
+
+  const category = String(node.category || '').trim().toLowerCase();
+  const tags = normalizedTags(node);
+  const sourceType = String(node.source?.type || '').trim().toLowerCase();
+
+  if (hasAny(tags, RESEARCH_TAGS) || RESEARCH_SOURCES.has(sourceType)) return 'research';
+  if (hasAny(tags, SYSTEM2_TAGS)) return 'system2';
+  if (node.priority === 'absolute') return 'conscious';
+  if (CONSCIOUS_CATEGORIES.has(category)) return 'conscious';
+  if (SYSTEM2_CATEGORIES.has(category)) return 'system2';
+  if (RESEARCH_CATEGORIES.has(category)) return 'research';
+
+  return DEFAULT_MEMORY_LAYER;
+}
+
+export function memoryLayerRoutingWeight(layer) {
+  switch (normalizeMemoryLayer(layer) || DEFAULT_MEMORY_LAYER) {
+    case 'conscious':
+      return 1.25;
+    case 'research':
+      return 0.75;
+    case 'system2':
+    default:
+      return 1;
+  }
+}
+
+export function buildMemoryLayerIndex(nodes) {
+  return nodes.map((node) => ({
+    v: 1,
+    slug: node.slug,
+    layer: inferMemoryLayer(node),
+    category: node.category,
+    status: node.status,
+    confidence: node.confidence,
+    importance: node.importance,
+    title: node.title,
+    updated: node.updated || null,
+    routes_to_skills: Array.isArray(node.routes_to_skills) ? node.routes_to_skills : [],
+  }));
+}

package/src/core/migrate.mjs

@@ -0,0 +1,116 @@
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { loadNodes, writeNode, atomicWrite } from './vault.mjs';
+import { createSnapshot } from './snapshot.mjs';
+import { SSSS_SCHEMAS } from './schema.mjs';
+import { logger } from './logger.mjs';
+
+/**
+ * Executes a schema migration.
+ * Because a migration is destructive, it MUST:
+ * 1. Take a safety snapshot.
+ * 2. Execute the migration script over the vault nodes.
+ * 3. Write a `type: migration` record tracking the result.
+ */
+export async function runMigration(vaultDir, fromVersion, toVersion, migrationFn, description) {
+  logger.info('migration', `Initiating migration from v${fromVersion} to v${toVersion}`);
+  logger.info('migration', `Description: ${description}`);
+
+  // 1. Safety Snapshot
+  logger.info('migration', 'Creating pre-migration safety snapshot...');
+  const snapshotId = await createSnapshot(vaultDir);
+  logger.info('migration', `Safety snapshot created: ${snapshotId}`);
+
+  const migrationId = `mig_${crypto.randomBytes(6).toString('hex')}`;
+  const migrationRecord = {
+    type: 'migration',
+    migration_id: migrationId,
+    from_version: fromVersion,
+    to_version: toVersion,
+    status: 'pending',
+    description,
+  };
+
+  try {
+    const nodes = loadNodes(vaultDir);
+    logger.info('migration', `Loaded ${nodes.length} nodes for migration processing.`);
+
+    // 2. Execute Migration
+    const migratedNodes = await migrationFn(nodes);
+
+    logger.info('migration', `Writing ${migratedNodes.length} migrated nodes back to vault...`);
+    // Note: in a real system we'd diff and only write changed, but migration often touches all.
+    for (const node of migratedNodes) {
+      // In a real migration we might delete old files if slug changes, but this assumes in-place update
+      writeNode(node, vaultDir);
+    }
+
+    migrationRecord.status = 'applied';
+    migrationRecord.applied_at = new Date().toISOString();
+    logger.info('migration', `Migration to v${toVersion} completed successfully.`);
+
+  } catch (err) {
+    logger.error('migration', `Migration failed: ${err.message}`);
+    migrationRecord.status = 'failed';
+    migrationRecord.applied_at = new Date().toISOString();
+    
+    // Auto-rollback on failure could be triggered here.
+    logger.error('migration', `WARNING: Vault may be in an inconsistent state. Please run 'total-recall snapshot rollback ${snapshotId}' immediately to recover.`);
+    throw err;
+  } finally {
+    // 3. Record the migration event
+    const migrationsDir = path.join(vaultDir, 'system', 'migrations');
+    if (!fs.existsSync(migrationsDir)) fs.mkdirSync(migrationsDir, { recursive: true });
+    
+    // Ensure slug doesn't conflict
+    const slug = `migration-v${toVersion}-${migrationId}`;
+    writeNode({ ...migrationRecord, slug }, migrationsDir);
+  }
+
+  return { migrationId, status: migrationRecord.status };
+}
+
+/**
+ * Migration Test Harness
+ * Simulates a migration in-memory to verify it produces valid schema shapes.
+ */
+export async function testMigration(vaultDir, migrationFn, targetSchemaType) {
+  logger.info('migration', 'Running migration harness in isolated context...');
+  
+  const nodes = loadNodes(vaultDir).slice(0, 50); // Sample a subset
+  
+  try {
+    const testMigrated = await migrationFn(nodes);
+    
+    let validCount = 0;
+    let invalidCount = 0;
+
+    const schema = SSSS_SCHEMAS[targetSchemaType];
+    
+    for (const node of testMigrated) {
+      if (node.type === targetSchemaType) {
+        if (schema) {
+          const res = schema.safeParse(node);
+          if (res.success) {
+            validCount++;
+          } else {
+            invalidCount++;
+            logger.error('migration', `Schema violation on node ${node.slug}`, { errors: res.error.errors });
+          }
+        }
+      }
+    }
+
+    if (invalidCount === 0) {
+      logger.info('migration', `Harness passed. ${validCount} target nodes successfully conform to new schema.`);
+      return true;
+    } else {
+      logger.error('migration', `Harness failed. ${invalidCount} nodes failed schema validation.`);
+      return false;
+    }
+  } catch (err) {
+    logger.error('migration', `Harness crashed during execution: ${err.message}`);
+    return false;
+  }
+}