total-recall-brain 3.1.0

package/src/cli/integration-dispatcher.mjs

@@ -0,0 +1,128 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import matter from 'gray-matter';
+import { agentDir } from '../core/config.mjs';
+import { logger } from '../core/logger.mjs';
+
+/**
+ * Dynamic SSSS-driven Integration Dispatcher.
+ * Exposes arbitrary third-party APIs as first-class CLI commands.
+ *
+ * @param {string} serviceName - e.g. "github", "slack", "linear"
+ * @param {string} actionName - e.g. "list_issues", "post_message"
+ * @param {object} cliArgs - Map of CLI arguments passed (e.g. { owner: "...", repo: "..." })
+ */
+export default async function dispatchIntegrationCommand(serviceName, actionName, cliArgs) {
+  const resolvedAgentDir = process.env.AGENT_DIR || agentDir;
+  const localFilePath = path.join(resolvedAgentDir, 'skills', 'total-recall', 'integrations', `${serviceName}.md`);
+  if (!fs.existsSync(localFilePath)) {
+    throw new Error(`Integration "${serviceName}" is not configured in the VFS.`);
+  }
+
+  // 1. Read and parse SSSS YAML frontmatter
+  let parsed;
+  try {
+    parsed = matter(fs.readFileSync(localFilePath, 'utf8'));
+  } catch (err) {
+    throw new Error(`Failed to parse integration SSSS frontmatter: ${err.message}`);
+  }
+
+  const schema = parsed.data || {};
+  if (schema.type !== 'integration') {
+    throw new Error(`Invalid SSSS node type in "${serviceName}.md". Expected "integration", found "${schema.type}".`);
+  }
+
+  const endpoints = schema.endpoints || {};
+  const endpointTemplate = endpoints[actionName];
+  if (!endpointTemplate) {
+    throw new Error(`Action "${actionName}" not found for service "${serviceName}". Available actions: ${Object.keys(endpoints).join(', ')}`);
+  }
+
+  // 2. Resolve Method and Path template (e.g. "GET /repos/{owner}/{repo}/issues")
+  const parts = endpointTemplate.trim().split(/\s+/);
+  if (parts.length !== 2) {
+    throw new Error(`Malformed endpoint definition: "${endpointTemplate}". Expected format: "METHOD /path".`);
+  }
+  const [method, pathTemplate] = parts;
+
+  // 3. Bind Path variables (e.g. {owner} -> val)
+  let finalPath = pathTemplate;
+  const remainingArgs = { ...cliArgs };
+  
+  const placeholders = pathTemplate.match(/\{([a-zA-Z0-9_-]+)\}/g) || [];
+  for (const placeholder of placeholders) {
+    const key = placeholder.slice(1, -1);
+    const value = cliArgs[key];
+    if (value === undefined) {
+      throw new Error(`Missing required path variable: --${key}`);
+    }
+    finalPath = finalPath.replace(placeholder, encodeURIComponent(String(value)));
+    delete remainingArgs[key]; // consumed by the path template
+  }
+
+  // 4. Resolve Auth Credential from Env
+  let token = '';
+  const envVars = schema.env_vars || [];
+  for (const envVar of envVars) {
+    if (process.env[envVar]) {
+      token = process.env[envVar];
+      break;
+    }
+  }
+
+  // 5. Populate and sign headers
+  const headers = {};
+  if (schema.headers) {
+    for (const [key, val] of Object.entries(schema.headers)) {
+      if (typeof val === 'string') {
+        // Substitute tokens dynamically
+        headers[key] = val
+          .replace('<GITHUB_TOKEN>', token)
+          .replace('<SLACK_BOT_TOKEN>', token)
+          .replace('<LINEAR_API_KEY>', token)
+          .replace('<API_KEY>', token);
+      } else {
+        headers[key] = String(val);
+      }
+    }
+  }
+
+  // 6. Build final URL and fetch configuration
+  const apiBaseUrl = (schema.api_base_url || '').replace(/\/$/, '');
+  const url = new URL(`${apiBaseUrl}${finalPath}`);
+  const fetchOpts = {
+    method,
+    headers,
+    signal: AbortSignal.timeout(30000)
+  };
+
+  if (method === 'GET' || method === 'DELETE') {
+    for (const [key, val] of Object.entries(remainingArgs)) {
+      if (val !== undefined && val !== null) {
+        url.searchParams.append(key, String(val));
+      }
+    }
+  } else {
+    // POST, PUT, PATCH
+    headers['Content-Type'] ??= 'application/json';
+    fetchOpts.body = JSON.stringify(remainingArgs);
+  }
+
+  logger.info({
+    subsystem: 'integration-dispatcher',
+    message: `Executing ${method} request to ${url.toString()}`
+  });
+
+  const res = await fetch(url.toString(), fetchOpts);
+  const responseText = await res.text();
+
+  if (!res.ok) {
+    throw new Error(`HTTP Error ${res.status} (${res.statusText}): ${responseText.slice(0, 1000)}`);
+  }
+
+  try {
+    return JSON.parse(responseText);
+  } catch {
+    return responseText;
+  }
+}

package/src/cli/lint.mjs

@@ -0,0 +1,166 @@
+/**
+ * total-recall lint
+ *
+ * Validate all vault nodes against SSSS schema v2.
+ * Reports invalid frontmatter, missing required fields, and type errors.
+ *
+ * Usage:
+ *   npx total-recall lint [options]
+ *
+ * Options:
+ *   --vault <path>     Override vault directory (default: ~/.agent/memory-vault)
+ *   --strict           Treat warnings as errors (exit 1 on any issue)
+ *   --json             Output results as JSONL
+ *   --help             Show this help
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { resolveAgentDir, resolveBrainDir } from './agent-dir.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const AGENT_DIR = resolveAgentDir();
+const BRAIN_DIR = resolveBrainDir();
+
+function parseArgs(args) {
+  const opts = { vault: null, strict: false, json: false, help: false };
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case '--vault': opts.vault = args[++i]; break;
+      case '--strict': opts.strict = true; break;
+      case '--json': opts.json = true; break;
+      case '--help': case '-h': opts.help = true; break;
+    }
+  }
+  return opts;
+}
+
+function printHelp() {
+  console.log(`
+  total-recall lint — Validate vault nodes against SSSS schema v2
+
+  Scans all .md files in the memory vault and validates their YAML
+  frontmatter against the Zod schemas defined in src/core/schema.mjs.
+
+  Usage: total-recall lint [options]
+
+  Options:
+    --vault <path>     Override vault directory (default: ~/.agent/memory-vault)
+    --strict           Treat warnings as errors (exit 1 on any issue)
+    --json             Output results as JSONL
+    --help, -h         Show this help
+
+  Exit codes:
+    0  All nodes valid
+    1  Validation errors found
+`);
+}
+
+export default async function lint(args) {
+  const opts = parseArgs(args);
+  if (opts.help) { printHelp(); return; }
+
+  const vaultDir = opts.vault || path.join(BRAIN_DIR, 'memory-vault');
+
+  if (!opts.json) {
+    console.error(`\n  🔍 Linting vault: ${vaultDir}\n`);
+  }
+
+  let matter;
+  try {
+    matter = (await import('gray-matter')).default;
+  } catch {
+    console.error('  ❌ gray-matter not installed. Run: npm install gray-matter');
+    process.exit(1);
+  }
+
+  const { MemoryNodeSchema, TaskSchema, ConflictRecordSchema } = await import('../core/schema.mjs');
+  const { walkMd } = await import('../core/vault.mjs');
+
+  const files = walkMd(vaultDir);
+  let errors = 0;
+  let warnings = 0;
+  let valid = 0;
+
+  const schemaMap = {
+    memory: MemoryNodeSchema,
+    task: TaskSchema,
+    conflict: ConflictRecordSchema,
+  };
+
+  for (const file of files) {
+    const rel = path.relative(vaultDir, file);
+    try {
+      const raw = fs.readFileSync(file, 'utf8');
+      const { data } = matter(raw);
+
+      if (!data.type) {
+        if (opts.json) {
+          console.log(JSON.stringify({ file: rel, level: 'warning', message: 'Missing type field in frontmatter' }));
+        } else {
+          console.error(`  ⚠️  ${rel}: Missing 'type' field in frontmatter`);
+        }
+        warnings++;
+        continue;
+      }
+
+      if (data.type === 'query') {
+        valid++;
+        continue;
+      }
+
+      const schema = schemaMap[data.type];
+      if (!schema) {
+        if (opts.json) {
+          console.log(JSON.stringify({ file: rel, level: 'warning', message: `Unknown type: ${data.type}` }));
+        } else {
+          console.error(`  ⚠️  ${rel}: Unknown type '${data.type}' — no schema to validate against`);
+        }
+        warnings++;
+        continue;
+      }
+
+      const result = schema.safeParse(data);
+      if (!result.success) {
+        const issues = result.error.issues.map(i => `${i.path.join('.')}: ${i.message}`);
+        if (opts.json) {
+          console.log(JSON.stringify({ file: rel, level: 'error', type: data.type, issues }));
+        } else {
+          console.error(`  ❌ ${rel} (${data.type}):`);
+          for (const issue of issues) {
+            console.error(`       ${issue}`);
+          }
+        }
+        errors++;
+      } else {
+        valid++;
+      }
+    } catch (err) {
+      if (opts.json) {
+        console.log(JSON.stringify({ file: rel, level: 'error', message: `Parse error: ${err.message}` }));
+      } else {
+        console.error(`  ❌ ${rel}: Parse error — ${err.message}`);
+      }
+      errors++;
+    }
+  }
+
+  if (!opts.json) {
+    console.error(`\n  ── Results ──`);
+    console.error(`     Files scanned:  ${files.length}`);
+    console.error(`     Valid:          ${valid}`);
+    console.error(`     Warnings:       ${warnings}`);
+    console.error(`     Errors:         ${errors}`);
+
+    if (errors === 0 && (warnings === 0 || !opts.strict)) {
+      console.error(`\n  ✅ All nodes valid\n`);
+    } else {
+      console.error(`\n  ❌ Validation issues found\n`);
+    }
+  }
+
+  if (errors > 0 || (opts.strict && warnings > 0)) {
+    process.exit(1);
+  }
+}

package/src/cli/map.mjs

@@ -0,0 +1,194 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { resolveBrainDir, parseLayerFlag, getBothBrains } from './agent-dir.mjs';
+import { loadNodes } from '../core/vault.mjs';
+
+function printHelp() {
+  console.log(`
+  total-recall map — Visualize memory vault structures and relationships
+
+  Usage: total-recall map [options]
+
+  Options:
+    --global                   Visualize global brain only
+    --project                  Visualize project brain only
+    --category, -cat <name>    Visualize only nodes within a specific category
+    --relations, -r            Visualize node-to-node relationships map
+
+  Examples:
+    npx total-recall map
+    npx total-recall map --project
+    npx total-recall map --category decisions
+    npx total-recall map --relations
+  `);
+}
+
+export default async function map(args) {
+  const { layer, remainingArgs } = parseLayerFlag(args);
+
+  let categoryFilter = null;
+  let showRelations = false;
+
+  for (let i = 0; i < remainingArgs.length; i++) {
+    const arg = remainingArgs[i];
+    const val = remainingArgs[i + 1];
+    if (arg === '--help' || arg === '-h') {
+      printHelp();
+      return;
+    } else if (arg === '--category' || arg === '-cat') {
+      if (val) {
+        categoryFilter = val.trim().toLowerCase();
+        i++;
+      }
+    } else if (arg === '--relations' || arg === '-r') {
+      showRelations = true;
+    }
+  }
+
+  const brains = getBothBrains();
+  const layersToProcess = [];
+
+  if ((layer === 'global' || layer === 'auto') && brains.global) {
+    layersToProcess.push({ name: 'global', path: brains.global.brainDir });
+  }
+  if ((layer === 'project' || layer === 'auto') && brains.project) {
+    layersToProcess.push({ name: 'project', path: brains.project.brainDir });
+  }
+
+  if (layersToProcess.length === 0) {
+    console.error('  ⚠️  No active brains resolved to visualize. Run total-recall init first.');
+    process.exit(1);
+  }
+
+  // Load nodes for each active layer
+  const loadedLayers = [];
+  let totalNodesCount = 0;
+
+  for (const l of layersToProcess) {
+    const vaultDir = path.join(l.path, 'memory-vault');
+    if (!fs.existsSync(vaultDir)) continue;
+
+    const nodes = loadNodes(vaultDir).map(n => ({ ...n, _layer: l.name }));
+    loadedLayers.push({ name: l.name, nodes });
+    totalNodesCount += nodes.length;
+  }
+
+  console.log(`\n  🧠 Total Recall Knowledge Map (Total: ${totalNodesCount} nodes)\n`);
+
+  if (showRelations) {
+    renderRelationsMap(loadedLayers);
+  } else {
+    renderCategoryTree(loadedLayers, categoryFilter);
+  }
+}
+
+function renderCategoryTree(loadedLayers, categoryFilter) {
+  const allCategories = [
+    'invariants',
+    'preferences',
+    'anti-patterns',
+    'patterns',
+    'decisions',
+    'concepts',
+    'facts',
+    'lore'
+  ];
+
+  for (const layer of loadedLayers) {
+    const layerEmoji = layer.name === 'global' ? '🌐' : '📁';
+    const layerLabel = layer.name === 'global' ? 'Global Brain' : 'Project Brain';
+    console.log(`  ${layerEmoji} ${layerLabel} (${layer.nodes.length} nodes)`);
+
+    // Group by category
+    const grouped = {};
+    for (const c of allCategories) grouped[c] = [];
+    for (const n of layer.nodes) {
+      const cat = (n.category || 'facts').toLowerCase();
+      if (!grouped[cat]) grouped[cat] = [];
+      grouped[cat].push(n);
+    }
+
+    const categoriesToRender = categoryFilter 
+      ? allCategories.filter(c => c.includes(categoryFilter)) 
+      : allCategories;
+
+    const activeCategories = categoriesToRender.filter(cat => grouped[cat].length > 0);
+
+    if (activeCategories.length === 0) {
+      console.log(`     └── (No nodes matching category filter)`);
+      console.log('');
+      continue;
+    }
+
+    for (let i = 0; i < activeCategories.length; i++) {
+      const cat = activeCategories[i];
+      const isLastCat = i === activeCategories.length - 1;
+      const catPrefix = isLastCat ? '  └──' : '  ├──';
+      const nodes = grouped[cat];
+
+      console.log(`     ${catPrefix} 📁 ${cat} (${nodes.length})`);
+
+      for (let j = 0; j < nodes.length; j++) {
+        const node = nodes[j];
+        const isLastNode = j === nodes.length - 1;
+        const nodePrefix = isLastCat 
+          ? (isLastNode ? '       └──' : '       ├──') 
+          : (isLastNode ? '  │    └──' : '  │    ├──');
+
+        const importanceVal = parseInt(node.importance, 10) || 3;
+        const stars = '⭐'.repeat(Math.max(1, Math.min(5, importanceVal)));
+        const modalityTag = node.modality ? ` [${node.modality.toUpperCase()}]` : '';
+
+        console.log(`     ${nodePrefix} 📄 [${node.slug}] — ${node.title} (${stars}${modalityTag})`);
+      }
+    }
+    console.log('');
+  }
+}
+
+function renderRelationsMap(loadedLayers) {
+  // Build lookup map
+  const allNodes = {};
+  for (const layer of loadedLayers) {
+    for (const node of layer.nodes) {
+      allNodes[node.slug] = node;
+    }
+  }
+
+  console.log('  🔗 Semantic Node Relations Map:\n');
+  let hasRelations = false;
+
+  for (const layer of loadedLayers) {
+    const layerEmoji = layer.name === 'global' ? '🌐' : '📁';
+    console.log(`  ${layerEmoji} ${layer.name.toUpperCase()} LAYER RELATIONS:`);
+
+    let layerHasRelations = false;
+    for (const node of layer.nodes) {
+      const related = Array.isArray(node.related) ? node.related : [];
+      if (related.length === 0) continue;
+
+      layerHasRelations = true;
+      hasRelations = true;
+      console.log(`    📄 [${node.slug}] "${node.title}"`);
+      
+      for (let i = 0; i < related.length; i++) {
+        const targetSlug = related[i];
+        const isLast = i === related.length - 1;
+        const branch = isLast ? '    └──' : '    ├──';
+        const targetNode = allNodes[targetSlug];
+
+        if (targetNode) {
+          const targetLayer = targetNode._layer === node._layer ? '' : ` [${targetNode._layer}]`;
+          console.log(`      ${branch} ──► [${targetSlug}] "${targetNode.title}"${targetLayer}`);
+        } else {
+          console.log(`      ${branch} ──► [${targetSlug}] (unresolved reference)`);
+        }
+      }
+      console.log('');
+    }
+
+    if (!layerHasRelations) {
+      console.log('    (No outbound relations mapped)\n');
+    }
+  }
+}

package/src/cli/migrate.mjs

@@ -0,0 +1,87 @@
+import path from 'path';
+import { runMigration, testMigration } from '../core/migrate.mjs';
+import { brainDir } from '../core/config.mjs';
+
+const VAULT_DIR = path.join(brainDir, 'memory-vault');
+
+export default async function migrateCommand(args) {
+  if (args[0] === '--help' || args[0] === '-h') {
+    console.log(`
+  total-recall migrate <test|apply> [options]
+
+  Run backwards-incompatible SSSS schema migrations.
+  Because migrations are destructive, 'apply' will automatically create a pre-migration safety snapshot.
+
+  Commands:
+    test   <script.mjs>   Run migration test harness (in-memory, no writes)
+    apply  <script.mjs>   Apply the migration over the active memory vault
+
+  Options:
+    --from <n>   Starting schema version
+    --to <n>     Target schema version
+    --desc "..." Description of the migration
+
+  Examples:
+    total-recall migrate test ./upgrade-v2-v3.mjs --to 3
+    total-recall migrate apply ./upgrade-v2-v3.mjs --from 2 --to 3 --desc "Add confidence decay to memories"
+`);
+    process.exit(0);
+  }
+
+  const action = args[0];
+  const scriptPath = args[1];
+
+  if (!['test', 'apply'].includes(action)) {
+    console.error(`Invalid action. Use 'test' or 'apply'.`);
+    process.exit(1);
+  }
+
+  if (!scriptPath) {
+    console.error(`Please specify a migration script (e.g. ./upgrade.mjs).`);
+    process.exit(1);
+  }
+
+  // Parse args
+  let fromVersion = 0;
+  let toVersion = 0;
+  let desc = 'Schema migration';
+
+  for (let i = 2; i < args.length; i++) {
+    if (args[i] === '--from') fromVersion = parseInt(args[++i], 10);
+    else if (args[i] === '--to') toVersion = parseInt(args[++i], 10);
+    else if (args[i] === '--desc') desc = args[++i];
+  }
+
+  const absoluteScriptPath = path.resolve(process.cwd(), scriptPath);
+  let migrationModule;
+  try {
+    migrationModule = await import(absoluteScriptPath);
+  } catch (err) {
+    console.error(`Failed to load migration script at ${absoluteScriptPath}: ${err.message}`);
+    process.exit(1);
+  }
+
+  const migrationFn = migrationModule.default || migrationModule.migrate;
+  if (typeof migrationFn !== 'function') {
+    console.error(`Migration script must export a default function or 'migrate' function.`);
+    process.exit(1);
+  }
+
+  if (action === 'test') {
+    const passed = await testMigration(VAULT_DIR, migrationFn, 'memory'); // Assuming 'memory' is the primary target
+    process.exit(passed ? 0 : 1);
+  }
+
+  if (action === 'apply') {
+    if (!fromVersion || !toVersion) {
+      console.error(`Both --from and --to versions are required for apply.`);
+      process.exit(1);
+    }
+
+    try {
+      await runMigration(VAULT_DIR, fromVersion, toVersion, migrationFn, desc);
+    } catch (err) {
+      process.exit(1);
+    }
+  }
+}

package/src/cli/rebuild.mjs

@@ -0,0 +1,88 @@
+import fs from 'fs';
+import path from 'path';
+import { compileSurface } from '../core/surface.mjs';
+import { detectIndexDrift } from '../core/drift-detector.mjs';
+import { resolveAgentDir, resolveBrainDir } from './agent-dir.mjs';
+import { logger } from '../core/logger.mjs';
+
+/**
+ * SSSS Projection Rebuild Command
+ *
+ * Implements the rebuild operation required by §10 of the SSSS spec.
+ * Discards derived indexes and rebuilds them deterministically from
+ * the canonical vault state.
+ */
+
+export async function runRebuild(options = {}) {
+  const agentDir = resolveAgentDir();
+  const brainDir = resolveBrainDir();
+  const vaultDir = path.join(brainDir, 'memory-vault');
+  const skillsDir = path.join(agentDir, 'skills');
+  const derivedDir = path.join(brainDir, 'memory-derived');
+  const instructionsFile = path.join(agentDir, 'INSTRUCTIONS.md');
+
+
+
+  console.log('🔄 SSSS Projection Rebuild');
+  console.log('==========================');
+  console.log(`Vault:    ${vaultDir}`);
+  console.log(`Derived:  ${derivedDir}`);
+  console.log('--------------------------');
+
+  if (options.check) {
+    console.log('Detecting drift...');
+    const drift = detectIndexDrift(vaultDir, derivedDir);
+    if (!drift.drifted) {
+      console.log('✅ Indexes are fully synchronized. No drift detected.');
+      return 0;
+    } else {
+      console.log('⚠️ Drift detected between canonical vault and derived indexes:');
+      if (drift.missing_in_index.length > 0) {
+        console.log(`   - ${drift.missing_in_index.length} nodes missing in index`);
+      }
+      if (drift.stale_in_index.length > 0) {
+        console.log(`   - ${drift.stale_in_index.length} nodes out of sync in index`);
+      }
+      if (drift.missing_in_vault.length > 0) {
+        console.log(`   - ${drift.missing_in_vault.length} ghost records in index (deleted from vault)`);
+      }
+      console.log('\nRun `total-recall rebuild` (without --check) to repair.');
+      return 1;
+    }
+  }
+
+  // Perform full rebuild
+  console.log('🗑️  Discarding existing derived indexes...');
+  if (fs.existsSync(derivedDir)) {
+    fs.rmSync(derivedDir, { recursive: true, force: true });
+  }
+
+  console.log('🏗️  Recompiling surface from canonical vault...');
+  try {
+    const stats = await compileSurface({ vaultDir, skillsDir, derivedDir, instructionsFile });
+    console.log(`✅ Processed ${stats.nodesProcessed} canonical memory nodes.`);
+    console.log(`✅ Injected memory into ${stats.skillsInjected} skill files.`);
+    console.log(`✅ Rebuilt graph-index.jsonl, memory-layers.jsonl, and skill-routes.jsonl.`);
+    
+    // Verify
+    const drift = detectIndexDrift(vaultDir, derivedDir);
+    if (!drift.drifted) {
+      console.log('✅ Post-build verification passed: 0 drift.');
+      logger.info('rebuild', `Rebuild completed successfully. Processed ${stats.nodesProcessed} nodes.`);
+      return 0;
+    } else {
+      console.error('❌ Post-build verification failed! Drift remains.');
+      return 1;
+    }
+  } catch (err) {
+    console.error(`❌ Rebuild failed: ${err.message}`);
+    logger.error('rebuild', `Rebuild failed: ${err.message}`);
+    return 1;
+  }
+}
+
+export default async function cli(args) {
+  const options = { check: args.includes('--check') };
+  const code = await runRebuild(options);
+  process.exit(code);
+}