total-recall-brain 3.1.0

package/src/core/dream.mjs

@@ -0,0 +1,347 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { loadNodes, writeNode, atomicWrite, walkMd } from './vault.mjs';
+import { logger } from './logger.mjs';
+import { agentDir, brainDir } from './config.mjs';
+import { loadQueue } from './research-queue.mjs';
+
+/**
+ * Write a daily dream-cycle summary to memory-vault/daily/YYYY-MM-DD.md.
+ * SSSS node natively; Obsidian Daily Notes plugin reads these files directly.
+ */
+function writeDailyNote(vaultDir, summaryLines) {
+  const today = new Date().toISOString().slice(0, 10);
+  const now = new Date().toISOString();
+  const dailyDir = path.join(vaultDir, 'daily');
+  if (!fs.existsSync(dailyDir)) fs.mkdirSync(dailyDir, { recursive: true });
+
+  const filePath = path.join(dailyDir, `${today}.md`);
+
+  if (fs.existsSync(filePath)) {
+    // Append this run's summary to the existing note
+    const existing = fs.readFileSync(filePath, 'utf8');
+    const runBlock = `\n## Dream Cycle – ${now}\n\n${summaryLines.map(l => `- ${l}`).join('\n')}\n`;
+    atomicWrite(filePath, existing.trimEnd() + runBlock);
+    return;
+  }
+
+  const frontmatter = [
+    '---',
+    'type: memory',
+    `slug: "daily-${today}"`,
+    'category: daily',
+    `title: "Daily Note: ${today}"`,
+    'status: active',
+    'confidence: 1.0',
+    'importance: 2',
+    `created: "${now}"`,
+    `updated: "${now}"`,
+    `last_accessed: "${now}"`,
+    'source:',
+    '  type: dream-cycle',
+    `  session_id: "dream-${today}"`,
+    '  evidence_count: 1',
+    'supersedes: []',
+    'superseded_by: null',
+    'contradicts: []',
+    'tags: [daily, dream-cycle]',
+    'related: []',
+    'routes_to_skills: []',
+    'sentiment_polarity: descriptive',
+    'sentiment_target: system',
+    'modality: should',
+    'subject: system',
+    'predicate: ran',
+    'object: dream-cycle',
+    'decay:',
+    '  half_life_days: 30',
+    '  access_count: 0',
+    'schema_version: 2',
+    '---',
+    ''
+  ].join('\n');
+
+  const body = [
+    `# Daily Note: ${today}`,
+    '',
+    `## Dream Cycle – ${now}`,
+    '',
+    ...summaryLines.map(l => `- ${l}`),
+    ''
+  ].join('\n');
+
+  atomicWrite(filePath, frontmatter + body);
+}
+import { detectConflicts, quarantineConflict } from './steering.mjs';
+import { compileSurface } from './surface.mjs';
+
+/**
+ * Phase 5: Automatic Storage & Memory Pruning.
+ * Keeps the VFS local directories clean of old logs, draft files,
+ * expired proposals, and temporary IDE session transcripts.
+ */
+export function autoPruneStorage(brainDir, vaultDir, conflictsDir) {
+  const logsDir = path.join(brainDir, 'logs');
+  const proposalsDir = path.join(vaultDir, 'proposals');
+  const inboxDir = path.join(brainDir, 'memory-inbox');
+
+  const logMaxAgeMs = 3 * 24 * 60 * 60 * 1000;      // Keep logs and proposals for 3 days
+  const draftMaxAgeMs = 24 * 60 * 60 * 1000;        // Aggressively clear background drafts (like RLHF files) after 24 hours
+  const now = Date.now();
+
+  // Retrieve node slugs of active research projects so we never prune their drafts in progress
+  const activeResearchSlugs = new Set();
+  try {
+    const queue = loadQueue();
+    for (const item of queue) {
+      if (item.status === 'pending' || item.status === 'in_progress') {
+        if (item.node_slug) {
+          activeResearchSlugs.add(item.node_slug);
+        }
+      }
+    }
+  } catch (err) {
+    logger.warn('dream', `Failed to load active research queue during pruning: ${err.message}`);
+  }
+
+  const pruneDir = (dir, maxAge) => {
+    if (!fs.existsSync(dir)) return;
+    try {
+      const files = fs.readdirSync(dir);
+      for (const file of files) {
+        const fullPath = path.join(dir, file);
+        const stat = fs.statSync(fullPath);
+        if (stat.isFile()) {
+          // Exempt active research drafts from pruning
+          const baseName = path.basename(file, '.md');
+          if (activeResearchSlugs.has(baseName)) {
+            continue;
+          }
+          if (now - stat.mtimeMs > maxAge) {
+            fs.unlinkSync(fullPath);
+          }
+        } else if (stat.isDirectory()) {
+          pruneDir(fullPath, maxAge);
+          if (fs.readdirSync(fullPath).length === 0) {
+            fs.rmdirSync(fullPath);
+          }
+        }
+      }
+    } catch {}
+  };
+
+  pruneDir(logsDir, logMaxAgeMs);
+  pruneDir(proposalsDir, logMaxAgeMs);
+  pruneDir(inboxDir, draftMaxAgeMs); // Keeps the local dev inbox fully clean of automatic research noise
+  if (conflictsDir) pruneDir(conflictsDir, draftMaxAgeMs);
+
+  // Automatically prune transient planning files in every active/historical conversation folder,
+  // while permanently preserving the conversations/threads themselves (.system_generated)
+  try {
+    const homeDir = process.env._TR_TEST_HOME_DIR || os.homedir();
+    const agAppDir = path.join(homeDir, '.gemini', 'antigravity');
+    const agBrainDir = path.join(agAppDir, 'brain');
+    if (fs.existsSync(agBrainDir)) {
+      const convs = fs.readdirSync(agBrainDir);
+      for (const convId of convs) {
+        const convPath = path.join(agBrainDir, convId);
+        const convStat = fs.statSync(convPath);
+        if (convStat.isDirectory()) {
+          const rootFiles = fs.readdirSync(convPath);
+          for (const item of rootFiles) {
+            const itemPath = path.join(convPath, item);
+            const itemStat = fs.statSync(itemPath);
+            if (itemStat.isFile()) {
+              const isTransientFile = item.endsWith('.md') || item.endsWith('.json');
+              if (isTransientFile && now - itemStat.mtimeMs > draftMaxAgeMs) {
+                fs.unlinkSync(itemPath);
+              }
+            }
+          }
+        }
+      }
+    }
+  } catch (err) {
+    logger.warn('dream', `Failed to prune Antigravity transient conversation folders: ${err.message}`);
+  }
+}
+
+import { 
+  generateMemoryCleanupProposals, 
+  generateStaleKnowledgeRefreshProposals,
+  evaluateProposalGate
+} from './optimizer.mjs';
+
+const DREAM_PROMOTION_THRESHOLD = 0.7;
+
+/**
+ * Phase 1: Light Sleep. Scan for modifications.
+ */
+export function scanModifiedVault(vaultDir, sinceHours = 24) {
+  const cutoff = Date.now() - sinceHours * 60 * 60 * 1000;
+  return walkMd(vaultDir).filter(fp => {
+    try {
+      return fs.statSync(fp).mtimeMs >= cutoff;
+    } catch {
+      return false;
+    }
+  });
+}
+
+/**
+ * Phase 2: REM Sleep. Score nodes, check conflicts, promote/decay.
+ */
+export function evaluateCandidates(candidates, existingNodes, conflictsDir) {
+  const promoted = [];
+  const allConflicts = [];
+
+  for (const candidate of candidates) {
+    const conflicts = detectConflicts(candidate, existingNodes);
+
+    if (conflicts.length > 0) {
+      for (const c of conflicts) {
+        quarantineConflict(c, conflictsDir);
+      }
+      allConflicts.push(...conflicts);
+      continue;
+    }
+
+    const score = (candidate.evidence_count || 1) * 0.2 + (candidate.importance || 1) * 0.1;
+    if (score >= DREAM_PROMOTION_THRESHOLD || candidate.confidence >= 0.8) {
+      const node = {
+        ...candidate,
+        status: 'active',
+        confidence: Math.min(1.0, (candidate.confidence || 0.5) + 0.1),
+      };
+      promoted.push(node);
+      existingNodes.push(node);
+    }
+  }
+
+  return { promoted, conflicted: allConflicts };
+}
+
+/**
+ * Main Dream Cycle execution.
+ */
+export async function runDreamCycle({
+  vaultDir, skillsDir, derivedDir, conflictsDir, instructionsFile
+}) {
+  const sessionsDir = path.join(brainDir, 'sessions');
+
+  logger.info('dream', 'PHASE 0 — Session Ingestion (IDE Conversation Logs)');
+  try {
+    const { scanAndIngest } = await import('./session-watcher.mjs');
+    const ingestResult = scanAndIngest(sessionsDir);
+    logger.info('dream', `Ingested: ${ingestResult.ingested} new sessions`);
+  } catch (err) {
+    logger.warn('dream', `Session ingestion skipped: ${err.message}`);
+  }
+
+  logger.info('dream', 'PHASE 1 — Light Sleep (Scan)');
+  const modified = scanModifiedVault(vaultDir);
+  logger.info('dream', `Modified vault files: ${modified.length}`);
+
+  // In a real system, we'd extract candidates from sessions. For now, we simulate.
+  const candidates = []; 
+
+  logger.info('dream', 'PHASE 2 — REM (Pattern Recognition)');
+  const existingNodes = loadNodes(vaultDir);
+  
+  if (candidates.length > 0) {
+    const { promoted, conflicted } = evaluateCandidates(candidates, existingNodes, conflictsDir);
+    logger.info('dream', `Promoted: ${promoted.length} | Conflicts: ${conflicted.length}`);
+    for (const node of promoted) {
+      writeNode(node, vaultDir);
+    }
+  } else {
+    logger.info('dream', 'No new candidates to evaluate.');
+  }
+
+  logger.info('dream', 'PHASE 3 — Deep Sleep (Recompile Surface)');
+  try {
+    await compileSurface({ vaultDir, skillsDir, derivedDir, instructionsFile });
+    logger.info('dream', 'Surface recompiled successfully');
+  } catch (err) {
+    logger.error('dream', `Deep Sleep compile failed: ${err.message}`);
+  }
+
+  logger.info('dream', 'PHASE 4 — Lucid Dreaming (Optimizer Proposals)');
+  let proposalCount = 0;
+  try {
+    const proposalsDir = path.join(vaultDir, 'proposals');
+    if (!fs.existsSync(proposalsDir)) fs.mkdirSync(proposalsDir, { recursive: true });
+
+    const cleanupProposals = await generateMemoryCleanupProposals(vaultDir);
+    const staleProposals = await generateStaleKnowledgeRefreshProposals(vaultDir);
+    const allProposals = [...cleanupProposals, ...staleProposals];
+
+    if (allProposals.length > 0) {
+      logger.info('dream', `Generated ${allProposals.length} optimization proposals.`);
+      for (const p of allProposals) {
+        const passed = await evaluateProposalGate(p, null);
+        logger.info('dream', `Proposal [${p.category}]: ${p.summary} -> ${passed ? 'ACCEPTED' : 'REJECTED'}`);
+        writeNode(p, proposalsDir);
+        if (passed) proposalCount++;
+      }
+    } else {
+      logger.info('dream', 'No new proposals generated.');
+    }
+  } catch (err) {
+    logger.error('dream', `Optimizer failed: ${err.message}`);
+  }
+
+  // Write daily note summary (native SSSS node; Obsidian Daily Notes reads it directly)
+  try {
+    const existingNodes = loadNodes(vaultDir);
+    writeDailyNote(vaultDir, [
+      `Modified vault files scanned: ${modified.length}`,
+      `Active nodes: ${existingNodes.filter(n => n.status === 'active').length}`,
+      `Proposals accepted: ${proposalCount}`,
+    ]);
+  } catch (err) {
+    logger.error('dream', `Daily note write failed: ${err.message}`);
+  }
+
+  logger.info('dream', 'PHASE 5 — Automatic Storage & Memory Pruning');
+  try {
+    autoPruneStorage(brainDir, vaultDir, conflictsDir);
+    logger.info('dream', 'Automated VFS storage pruning completed successfully');
+  } catch (err) {
+    logger.error('dream', `Storage pruning failed: ${err.message}`);
+  }
+
+  return { status: 'success' };
+}
+
+// ─── Standalone Daemon Execution ────────────────────────────────────────────────
+
+import { fileURLToPath } from 'url';
+import os from 'os';
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  const vaultDir = path.join(brainDir, 'memory-vault');
+  const skillsDir = path.join(agentDir, 'skills');
+  const derivedDir = path.join(brainDir, 'memory-derived');
+  const conflictsDir = path.join(brainDir, 'memory-inbox', 'conflicts');
+  const instructionsFile = path.join(agentDir, 'INSTRUCTIONS.md');
+
+  logger.info('dream', 'Total Recall Dream Cycle Daemon Started');
+
+  async function daemonLoop() {
+    while (true) {
+      try {
+        await runDreamCycle({
+          vaultDir, skillsDir, derivedDir, conflictsDir, instructionsFile
+        });
+      } catch (err) {
+        logger.error('dream', 'Dream cycle iteration failed', { err: err.message });
+      }
+      // Sleep for 60 seconds before next cycle
+      await new Promise(resolve => setTimeout(resolve, 60000));
+    }
+  }
+
+  daemonLoop();
+}

package/src/core/drift-detector.mjs

@@ -0,0 +1,143 @@
+import fs from 'fs';
+import path from 'path';
+import crypto from 'node:crypto';
+import { loadNodes } from './vault.mjs';
+
+/**
+ * SSSS Drift Detector
+ *
+ * Implements §10 of the SSSS spec (Derived Artifacts).
+ * Detects discrepancies between the canonical Markdown vault and
+ * derived disposable projections (e.g., graph-index.jsonl).
+ */
+
+/**
+ * Compare two simple objects for equality on a specific set of keys.
+ */
+function propertiesMatch(a, b, keys) {
+  for (const key of keys) {
+    if (a[key] !== b[key]) return false;
+  }
+  return true;
+}
+
+/**
+ * Scan for drift between the vault and the graph index.
+ * 
+ * @param {string} vaultDir - Path to the memory-vault directory.
+ * @param {string} derivedDir - Path to the memory-derived directory.
+ * @returns {{ drifted: boolean, missing_in_index: string[], stale_in_index: string[], missing_in_vault: string[] }}
+ */
+export function detectIndexDrift(vaultDir, derivedDir, globalVaultDir = undefined) {
+  const result = {
+    drifted: false,
+    missing_in_index: [],
+    stale_in_index: [],
+    missing_in_vault: []
+  };
+
+  const graphIndexPath = path.join(derivedDir, 'graph-index.jsonl');
+  
+  if (!fs.existsSync(graphIndexPath)) {
+    // No index means total drift
+    result.drifted = true;
+    return result;
+  }
+
+  // Load canonical nodes
+  const nodes = loadNodes(vaultDir);
+  if (globalVaultDir && fs.existsSync(globalVaultDir)) {
+    try {
+      const globalNodes = loadNodes(globalVaultDir);
+      nodes.push(...globalNodes);
+    } catch (err) {
+      // Ignore or log global vault load failure
+    }
+  }
+  const canonicalMap = new Map();
+  for (const node of nodes) {
+    canonicalMap.set(node.slug, node);
+  }
+
+  // Load derived index
+  const indexMap = new Map();
+  try {
+    const lines = fs.readFileSync(graphIndexPath, 'utf8').trim().split('\n');
+    for (const line of lines) {
+      if (!line) continue;
+      const entry = JSON.parse(line);
+      indexMap.set(entry.slug, entry);
+    }
+  } catch (err) {
+    // Malformed index is total drift
+    result.drifted = true;
+    return result;
+  }
+
+  const keysToCheck = ['title', 'category', 'status', 'confidence'];
+
+  // Check for missing or stale entries in the index
+  for (const [slug, canonicalNode] of canonicalMap.entries()) {
+    const indexEntry = indexMap.get(slug);
+    if (!indexEntry) {
+      result.missing_in_index.push(slug);
+      result.drifted = true;
+    } else {
+      if (!propertiesMatch(canonicalNode, indexEntry, keysToCheck)) {
+        result.stale_in_index.push(slug);
+        result.drifted = true;
+      }
+    }
+  }
+
+  // Check for ghost entries in the index (exist in index but not in vault)
+  for (const slug of indexMap.keys()) {
+    if (!canonicalMap.has(slug)) {
+      result.missing_in_vault.push(slug);
+      result.drifted = true;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Verify event log append-only invariant.
+ * Checks that the event log file is well-formed JSONL and strictly temporally ordered.
+ */
+export function checkEventLogIntegrity(eventLogPath) {
+  if (!fs.existsSync(eventLogPath)) {
+    return { ok: true, reason: 'Log does not exist' };
+  }
+
+  try {
+    const lines = fs.readFileSync(eventLogPath, 'utf8').trim().split('\n');
+    let lastTs = null;
+    let lineNum = 0;
+    
+    for (const line of lines) {
+      lineNum++;
+      if (!line) continue;
+      const event = JSON.parse(line);
+      
+      if (!event.ts) {
+        return { ok: false, error: `Missing timestamp on line ${lineNum}` };
+      }
+      
+      const currentTs = new Date(event.ts).getTime();
+      if (isNaN(currentTs)) {
+        return { ok: false, error: `Invalid timestamp '${event.ts}' on line ${lineNum}` };
+      }
+      
+      if (lastTs !== null && currentTs < lastTs) {
+        return { ok: false, error: `Temporal violation on line ${lineNum}: ${event.ts} is before previous event.` };
+      }
+      
+      lastTs = currentTs;
+    }
+    
+    return { ok: true, events_checked: lineNum };
+  } catch (err) {
+    return { ok: false, error: `Parse failure: ${err.message}` };
+  }
+}