total-recall-brain 3.1.0

package/src/core/vault.mjs

@@ -0,0 +1,273 @@
+import fs from 'fs';
+import path from 'path';
+import matter from 'gray-matter';
+import { logger } from './logger.mjs';
+
+/**
+ * Atomic file write using write-then-rename to prevent partial corruption.
+ */
+export function atomicWrite(filePath, content) {
+  const tmpPath = `${filePath}.tmp.${process.pid}`;
+  fs.writeFileSync(tmpPath, content, 'utf8');
+  fs.renameSync(tmpPath, filePath);
+}
+
+/**
+ * Deep-strip undefined values from an object so js-yaml (via gray-matter)
+ * never encounters `[object Undefined]` and crashes the process.
+ */
+function stripUndefined(obj) {
+  if (obj === null || obj === undefined || typeof obj !== 'object') return obj;
+  if (Array.isArray(obj)) return obj.map(stripUndefined);
+  const clean = {};
+  for (const [k, v] of Object.entries(obj)) {
+    if (v !== undefined) {
+      clean[k] = typeof v === 'object' ? stripUndefined(v) : v;
+    }
+  }
+  return clean;
+}
+
+/**
+ * Safe wrapper around matter.stringify that prevents js-yaml crashes.
+ * Use this instead of matter.stringify() everywhere in the codebase.
+ *
+ * @param {string} content  The markdown body
+ * @param {object} data     The YAML frontmatter object
+ * @returns {string}        The serialized markdown with frontmatter
+ */
+export function safeStringify(content, data) {
+  if (data && data.type === 'memory') {
+    const now = new Date().toISOString();
+    if (!data.x_temporal_context) {
+      data.x_temporal_context = data.updated || data.created || now;
+    }
+    if (!data.x_citations) {
+      data.x_citations = [{
+        source: data.source?.type || 'unknown',
+        title: data.title || 'Untitled Memory',
+        url: data.source?.session_id ? `session://${data.source.session_id}` : 'unknown',
+        published: data.x_temporal_context,
+        relevance: 1.0,
+        accessed: now
+      }];
+    }
+  }
+  return matter.stringify(content ?? '', stripUndefined(data ?? {}));
+}
+
+/**
+ * Walk directory recursively and find all .md files.
+ */
+export function walkMd(dir) {
+  let results = [];
+  if (!fs.existsSync(dir)) return results;
+  
+  const list = fs.readdirSync(dir);
+  for (const file of list) {
+    const fullPath = path.join(dir, file);
+    const stat = fs.statSync(fullPath);
+    if (stat && stat.isDirectory()) {
+      results = results.concat(walkMd(fullPath));
+    } else if (fullPath.endsWith('.md')) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+/**
+ * Load all memory nodes from the vault.
+ */
+export function loadNodes(vaultDir) {
+  const nodes = [];
+  const files = walkMd(vaultDir);
+  for (const file of files) {
+    try {
+      const raw = fs.readFileSync(file, 'utf8');
+      const { data, content } = matter(raw);
+      if (data.type === 'memory') {
+        nodes.push({ ...data, body: content.trim() });
+      }
+    } catch (err) {
+      logger.warn('vault', `Failed to parse ${file}`, { err: err.message });
+    }
+  }
+  return nodes;
+}
+
+/**
+ * Load merged memory nodes from both global and project vaults.
+ * Project nodes override global nodes with the same slug.
+ * Each node is tagged with `_layer: 'global' | 'project'`.
+ *
+ * @param {string} globalVaultDir - Path to global memory-vault/
+ * @param {string|null} projectVaultDir - Path to project memory-vault/ (null if no project brain)
+ * @returns {object[]} Merged array of nodes
+ */
+export function loadMergedNodes(globalVaultDir, projectVaultDir) {
+  // Load global nodes
+  const globalNodes = loadNodes(globalVaultDir).map(n => ({ ...n, _layer: 'global' }));
+
+  if (!projectVaultDir || !fs.existsSync(projectVaultDir)) {
+    return globalNodes;
+  }
+
+  // Load project nodes
+  const projectNodes = loadNodes(projectVaultDir).map(n => ({ ...n, _layer: 'project' }));
+
+  // Merge: project wins on slug conflict
+  const nodeMap = new Map();
+  for (const node of globalNodes) {
+    nodeMap.set(node.slug, node);
+  }
+  for (const node of projectNodes) {
+    nodeMap.set(node.slug, node); // project overrides global
+  }
+
+  return Array.from(nodeMap.values());
+}
+
+
+/**
+ * Write a memory node to the vault.
+ */
+const SAFE_NAME = /^[a-zA-Z0-9_-]+$/;
+
+export function writeNode(node, vaultDir) {
+  if (!SAFE_NAME.test(node.slug)) throw new Error(`Invalid slug: ${node.slug}`);
+  if (!SAFE_NAME.test(node.category)) throw new Error(`Invalid category: ${node.category}`);
+  const targetDir = path.join(vaultDir, node.category);
+  const filePath = path.join(targetDir, `${node.slug}.md`);
+  
+  const dirPath = path.dirname(filePath);
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+  
+  const { body, ...frontmatter } = node;
+  
+  if (frontmatter.type === 'memory') {
+    const now = new Date().toISOString();
+    if (!frontmatter.x_temporal_context) {
+      frontmatter.x_temporal_context = frontmatter.updated || frontmatter.created || now;
+    }
+    if (!frontmatter.x_citations) {
+      frontmatter.x_citations = [{
+        source: frontmatter.source?.type || 'unknown',
+        title: frontmatter.title || 'Untitled Memory',
+        url: frontmatter.source?.session_id ? `session://${frontmatter.source.session_id}` : 'unknown',
+        published: frontmatter.x_temporal_context,
+        relevance: 1.0,
+        accessed: now
+      }];
+    }
+  }
+  
+  const raw = safeStringify(body || '', frontmatter);
+  atomicWrite(filePath, raw);
+}
+
+/**
+ * Delete a memory node by slug.
+ * Finds the node via loadNodes, then removes its .md file.
+ * Returns true if deleted, false if the node was not found.
+ *
+ * @param {string} slug
+ * @param {string} vaultDir
+ * @returns {boolean}
+ */
+export function deleteNode(slug, vaultDir) {
+  const nodes = loadNodes(vaultDir);
+  const node = nodes.find(n => n.slug === slug);
+  if (!node) return false;
+  const filePath = path.join(vaultDir, node.category, `${node.slug}.md`);
+  if (fs.existsSync(filePath)) {
+    fs.rmSync(filePath);
+    return true;
+  }
+  return false;
+}
+
+
+/**
+ * Creates a fully populated memory node from an API payload.
+ */
+export function createMemoryNode({ slug, title, category, content }) {
+  const now = new Date().toISOString();
+  return {
+    type: 'memory',
+    slug,
+    title,
+    category,
+    body: content,
+    status: 'active',
+    confidence: 1.0,
+    importance: 3,
+    created: now,
+    updated: now,
+    last_accessed: now,
+    source: {
+      type: 'api-client',
+      session_id: 'api-external',
+      evidence_count: 1
+    },
+    supersedes: [],
+    superseded_by: null,
+    contradicts: [],
+    tags: [],
+    related: [],
+    routes_to_skills: [],
+    sentiment_polarity: 'descriptive',
+    sentiment_target: 'api-external',
+    modality: 'should',
+    subject: 'api-external',
+    predicate: 'provided_information',
+    object: 'api-external',
+    decay: {
+      half_life_days: 30,
+      access_count: 1
+    },
+    schema_version: 2,
+    x_temporal_context: now,
+    x_citations: [{
+      source: 'api-client',
+      title: title || 'External API Ingest',
+      url: 'api://external',
+      published: now,
+      relevance: 1.0,
+      accessed: now
+    }]
+  };
+}
+
+/**
+ * Load all skill manifests from the skills directory.
+ */
+export function loadSkills(skillsDir) {
+  const skills = [];
+  if (!fs.existsSync(skillsDir)) return skills;
+
+  for (const skillName of fs.readdirSync(skillsDir)) {
+    const skillPath = path.join(skillsDir, skillName, 'SKILL.md');
+    if (fs.existsSync(skillPath)) {
+      try {
+        const raw = fs.readFileSync(skillPath, 'utf8');
+        const { data, content } = matter(raw);
+        // Accept either `type: skill` or files that simply have a `name` field
+        // (which is the convention for all real SKILL.md files)
+        if (data.type === 'skill' || data.name) {
+          skills.push({
+            ...data,
+            name: data.name || skillName,
+            body: content.trim(),
+            filepath: skillPath,
+          });
+        }
+      } catch (err) {
+        logger.warn('vault', `Failed to parse ${skillPath}`, { err: err.message });
+      }
+    }
+  }
+  return skills;
+}

package/src/core/watchdog.mjs

@@ -0,0 +1,236 @@
+import fs from 'fs';
+import path from 'path';
+import readline from 'node:readline';
+import { logger, logEvents, LOG_DIR, getLogFile } from './logger.mjs';
+import { brainDir } from './config.mjs';
+
+const QUARANTINE_FILE = path.join(brainDir, 'config', 'quarantine.json');
+
+let state = {
+  sandboxFailures: 0,
+  authFailures: {},
+  tokenSpikes: 0,
+  latencyBaseline: 1000,
+  latencySamples: [],
+  blockedIps: new Set(),
+  writeHalt: false
+};
+
+const saveQuarantine = () => {
+  try {
+    fs.mkdirSync(path.dirname(QUARANTINE_FILE), { recursive: true });
+    fs.writeFileSync(QUARANTINE_FILE, JSON.stringify({ blockedIps: Array.from(state.blockedIps) }), 'utf8');
+  } catch (e) {
+    logger.error('watchdog', `Failed to save quarantine state: ${e.message}`);
+  }
+};
+
+const loadQuarantine = () => {
+  if (fs.existsSync(QUARANTINE_FILE)) {
+    try {
+      const data = JSON.parse(fs.readFileSync(QUARANTINE_FILE, 'utf8'));
+      if (data.blockedIps) {
+        state.blockedIps = new Set(data.blockedIps);
+      }
+    } catch (e) {
+      logger.error('watchdog', `Failed to load quarantine state: ${e.message}`);
+    }
+  }
+};
+loadQuarantine();
+
+export const watchdog = {
+  recordSandboxFailure: () => {
+    state.sandboxFailures++;
+    logger.warn('watchdog', `Sandbox failure recorded. Count: ${state.sandboxFailures}`);
+    if (state.sandboxFailures >= 3) {
+      logger.error('watchdog', 'Sandbox circuit breaker triggered (≥3 failures). Quarantining sandbox.');
+    }
+  },
+  resetSandboxFailures: () => {
+    state.sandboxFailures = 0;
+  },
+  isSandboxQuarantined: () => state.sandboxFailures >= 3,
+
+  recordAuthFailure: (ip) => {
+    state.authFailures[ip] = (state.authFailures[ip] || 0) + 1;
+    logger.warn('watchdog', `Auth failure for IP ${ip}. Count: ${state.authFailures[ip]}`);
+    if (state.authFailures[ip] >= 9999) {
+      if (!state.blockedIps.has(ip)) {
+        logger.error('watchdog', `Auth lockout triggered for IP ${ip}. Blocking.`);
+        state.blockedIps.add(ip);
+        saveQuarantine();
+      }
+    }
+  },
+  resetAuthFailures: (ip) => {
+    if (state.authFailures[ip]) {
+      state.authFailures[ip] = 0;
+    }
+  },
+  isIpBlocked: (ip) => state.blockedIps.has(ip),
+
+  recordTokens: (count) => {
+    if (count > 8000) {
+      state.tokenSpikes++;
+      logger.warn('watchdog', `Token spike detected: ${count}. Count: ${state.tokenSpikes}`);
+      if (state.tokenSpikes >= 3) {
+        logger.error('watchdog', 'Exfiltration monitor triggered. Suspending routing.');
+      }
+    } else {
+      state.tokenSpikes = 0;
+    }
+  },
+  isRoutingSuspended: () => state.tokenSpikes >= 3,
+
+  recordLatency: (ms) => {
+    if (state.latencySamples.length > 10) state.latencySamples.shift();
+    state.latencySamples.push(ms);
+    const avg = state.latencySamples.reduce((a, b) => a + b, 0) / state.latencySamples.length;
+    state.latencyBaseline = avg;
+
+    if (ms > state.latencyBaseline * 2 && state.latencyBaseline > 500) {
+      logger.warn('watchdog', `Latency anomaly triggered: ${ms}ms vs baseline ${state.latencyBaseline.toFixed(0)}ms. Flushing KV cache.`);
+    }
+  },
+
+  checkDiskSpace: () => {
+    try {
+      const stat = fs.statfsSync(brainDir);
+      const percentFree = stat.bavail / stat.blocks;
+      const percentUsed = 1 - percentFree;
+
+      if (percentUsed > 0.95) {
+        if (!state.writeHalt) {
+          logger.error('watchdog', 'Disk space >95% used. Halting writes.');
+          state.writeHalt = true;
+        }
+      } else if (percentUsed > 0.80) {
+        logger.warn('watchdog', 'Disk space >80% used. Log rotation advised.');
+        state.writeHalt = false;
+      } else {
+        state.writeHalt = false;
+      }
+    } catch (e) {
+      logger.error('watchdog', `Disk check failed: ${e.message}`);
+    }
+  },
+  isWriteHalted: () => state.writeHalt
+};
+
+// ─── Log event monitor (PRD §9.3) ───────────────────────────────────────────
+//
+// Subscribe to in-process log events and automatically fire circuit breakers
+// when subsystems report errors. This satisfies "watchdog wired to log file
+// monitor" — every logger.* call emits a `log` event that arrives here.
+
+const LOG_PATTERNS = [
+  // Sandbox: any error from sandbox subsystem → record failure
+  {
+    match: (e) => e.subsystem === 'sandbox' && e.level === 'error',
+    handler: () => watchdog.recordSandboxFailure()
+  },
+  // Auth: failed login from auth subsystem → record by IP
+  {
+    match: (e) => e.subsystem === 'auth' && e.level === 'warn' && /failed|invalid|denied/i.test(e.message || ''),
+    handler: (e) => {
+      if (e.ip) watchdog.recordAuthFailure(e.ip);
+    }
+  },
+  // Latency: any subsystem can record latency via meta
+  {
+    match: (e) => typeof e.latency_ms === 'number',
+    handler: (e) => watchdog.recordLatency(e.latency_ms)
+  },
+  // Tokens: any subsystem can record token usage via meta
+  {
+    match: (e) => typeof e.tokens === 'number',
+    handler: (e) => watchdog.recordTokens(e.tokens)
+  }
+];
+
+function onLogEntry(entry) {
+  // Never react to our own emissions to avoid recursion / amplification.
+  if (entry.subsystem === 'watchdog') return;
+  for (const { match, handler } of LOG_PATTERNS) {
+    try {
+      if (match(entry)) handler(entry);
+    } catch (err) {
+      // Don't let a broken handler crash the daemon.
+    }
+  }
+}
+
+let monitorAttached = false;
+export function attachLogMonitor() {
+  if (monitorAttached) return;
+  monitorAttached = true;
+  logEvents.on('log', onLogEntry);
+  logger.info('watchdog', 'Log event monitor attached. Circuit breakers wired.');
+}
+
+export function detachLogMonitor() {
+  if (!monitorAttached) return;
+  logEvents.off('log', onLogEntry);
+  monitorAttached = false;
+}
+
+// ─── Log file tail (out-of-process daemon scenario) ─────────────────────────
+//
+// When the daemon runs in a different process from the server, in-process
+// events won't reach it. `attachLogTail()` polls today's log file and feeds
+// each new JSONL line through the same pattern matcher.
+
+let tailAttached = false;
+let tailTimer = null;
+let tailState = { file: null, offset: 0 };
+
+async function pumpTail() {
+  const file = getLogFile();
+  try {
+    if (file !== tailState.file) {
+      tailState = { file, offset: 0 };
+    }
+    if (!fs.existsSync(file)) return;
+    const stat = fs.statSync(file);
+    if (stat.size <= tailState.offset) return;
+    const stream = fs.createReadStream(file, { start: tailState.offset, end: stat.size });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    for await (const line of rl) {
+      if (!line.trim()) continue;
+      try {
+        const entry = JSON.parse(line);
+        onLogEntry(entry);
+      } catch (_) {
+        // Skip malformed lines.
+      }
+    }
+    tailState.offset = stat.size;
+  } catch (err) {
+    logger.error('watchdog', `Log tail error: ${err.message}`);
+  }
+}
+
+export function attachLogTail(intervalMs = 2000) {
+  if (tailAttached) return;
+  tailAttached = true;
+  tailState = { file: getLogFile(), offset: 0 };
+  try {
+    if (fs.existsSync(tailState.file)) {
+      tailState.offset = fs.statSync(tailState.file).size;
+    }
+  } catch (_) {}
+  tailTimer = setInterval(() => { pumpTail(); }, intervalMs);
+  tailTimer.unref?.();
+  logger.info('watchdog', `Log file tail attached (every ${intervalMs}ms).`);
+}
+
+export function detachLogTail() {
+  if (!tailAttached) return;
+  if (tailTimer) clearInterval(tailTimer);
+  tailTimer = null;
+  tailAttached = false;
+}
+
+// Periodic disk check
+setInterval(watchdog.checkDiskSpace, 60000).unref();