total-recall-brain 3.1.0

package/src/core/conclusion-writer.mjs

@@ -0,0 +1,259 @@
+import fs from 'fs';
+import path from 'path';
+import matter from 'gray-matter';
+import { callLocalRuntime } from './runtime.mjs';
+import { atomicWrite, safeStringify } from './vault.mjs';
+import { logger } from './logger.mjs';
+
+/**
+ * Total Recall Conclusion Writer — Research → Active Validation Gate
+ *
+ * All research-derived facts arrive as status: draft in .agent/memory-inbox/pending/.
+ * Before promoting any draft to status: active, the Conclusion Writer asks the local LLM:
+ *   1. Is this fact self-consistent?
+ *   2. Does it contradict any existing active node?
+ *   3. Is the confidence appropriate given the source?
+ *
+ * If it passes, the node is promoted (status: active, moved to vault).
+ * If it fails, it gets quarantined with a reason record.
+ *
+ * This prevents research hallucinations from polluting working memory.
+ */
+
+const VALIDATION_SYSTEM = `You are a Fact Validation Gate for a sovereign AI memory system called Total Recall.
+
+You will be given a draft memory node and a list of related active nodes. Determine if the draft should be promoted to active memory.
+
+Output valid JSON:
+{
+  "verdict": "APPROVE" | "REJECT" | "NEEDS_REVISION",
+  "confidence_adjustment": -0.3 to +0.1,
+  "reasoning": "string",
+  "revision_instructions": "string (only if NEEDS_REVISION)"
+}
+
+Criteria for REJECT:
+- Factual claim that contradicts an active node with higher confidence
+- Circular or empty content (no actionable information)
+- Obvious hallucination (specific version numbers without any evidence)
+- Duplicate of an existing node
+
+Criteria for NEEDS_REVISION:
+- Directionally correct but confidence is too high for the evidence
+- Title is misleading
+- Body needs tightening
+
+Criteria for APPROVE:
+- Adds genuine new knowledge
+- Does not contradict existing high-confidence nodes
+- Confidence is appropriate for the evidence
+
+Output ONLY valid JSON.`;
+
+function buildValidationPrompt(draftNode, relatedActiveNodes) {
+  const draftText = [
+    `## Draft Node: ${draftNode.slug}`,
+    `Title: "${draftNode.title}"`,
+    `Category: ${draftNode.category} | Confidence: ${draftNode.confidence}`,
+    `Source: ${draftNode.source?.type || 'unknown'} / ${draftNode.source?.agent || 'unknown'}`,
+    '',
+    '## Body',
+    (draftNode.body || '').slice(0, 1200),
+  ].join('\n');
+
+  const relatedText = relatedActiveNodes.length > 0
+    ? '\n\n## Related Active Nodes\n' + relatedActiveNodes.map(n => [
+        `### [${n.slug}] ${n.title} (confidence: ${n.confidence})`,
+        (n.body || '').slice(0, 400),
+      ].join('\n')).join('\n\n')
+    : '\n\n## Related Active Nodes\n(none)';
+
+  return draftText + relatedText + '\n\nShould this draft be promoted to active memory?';
+}
+
+/**
+ * Find active nodes related to a draft (same category or shared tags).
+ */
+function findRelatedNodes(draftNode, allActiveNodes) {
+  return allActiveNodes
+    .filter(n => {
+      if (n.category !== draftNode.category) return false;
+      const draftTags = new Set(draftNode.tags || []);
+      const sharedTags = (n.tags || []).filter(t => draftTags.has(t));
+      return sharedTags.length > 0;
+    })
+    .slice(0, 5); // Cap at 5 to keep prompt size manageable
+}
+
+/**
+ * Run the validation gate on a single draft node.
+ *
+ * @param {object} draftNode      Parsed draft node object (includes _filepath)
+ * @param {object[]} allActiveNodes All currently active nodes from vault
+ * @param {object} opts
+ * @param {string} opts.vaultDir     Path to memory vault
+ * @param {string} opts.quarantineDir Path to quarantine directory
+ * @param {object} opts.runtimeConfig Local LLM config
+ * @returns {{ verdict: string, promoted: boolean, error?: string }}
+ */
+export async function validateDraftNode(draftNode, allActiveNodes, { vaultDir, quarantineDir, runtimeConfig }) {
+  const relatedNodes = findRelatedNodes(draftNode, allActiveNodes);
+  const prompt = buildValidationPrompt(draftNode, relatedNodes);
+
+  let result;
+  try {
+    const rawResponse = await callLocalRuntime(prompt, VALIDATION_SYSTEM, runtimeConfig);
+    const jsonMatch = rawResponse.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) throw new Error('No JSON in response');
+    result = JSON.parse(jsonMatch[0]);
+  } catch (err) {
+    logger.info({
+      subsystem: 'conclusion-writer',
+      message: `Validation failed for ${draftNode.slug}: ${err.message}`,
+    });
+    return { verdict: 'SKIP', promoted: false, error: err.message };
+  }
+
+  const verdict = result.verdict || 'REJECT';
+  const confidenceAdj = Number(result.confidence_adjustment) || 0;
+
+  if (verdict === 'APPROVE') {
+    await promoteToVault(draftNode, confidenceAdj, vaultDir);
+    logger.info({
+      subsystem: 'conclusion-writer',
+      message: `APPROVED: ${draftNode.slug} → ${draftNode.category}/`,
+    });
+    return { verdict: 'APPROVE', promoted: true };
+  }
+
+  if (verdict === 'NEEDS_REVISION') {
+    await applyRevision(draftNode, confidenceAdj, result.revision_instructions);
+    logger.info({
+      subsystem: 'conclusion-writer',
+      message: `NEEDS_REVISION: ${draftNode.slug} — revised in-place`,
+    });
+    return { verdict: 'NEEDS_REVISION', promoted: false };
+  }
+
+  // REJECT → quarantine
+  await quarantineDraft(draftNode, result.reasoning || 'Rejected by validation gate', quarantineDir);
+  logger.info({
+    subsystem: 'conclusion-writer',
+    message: `REJECTED: ${draftNode.slug} — ${result.reasoning}`,
+  });
+  return { verdict: 'REJECT', promoted: false };
+}
+
+async function promoteToVault(draftNode, confidenceAdj, vaultDir) {
+  if (!draftNode._filepath) return;
+
+  const raw = fs.readFileSync(draftNode._filepath, 'utf8');
+  const { data, content } = matter(raw);
+
+  // Promote
+  data.status = 'active';
+  data.confidence = Math.min(1.0, Math.max(0.1, (data.confidence || 0.5) + confidenceAdj));
+  data.updated = new Date().toISOString();
+  data.x_promoted_at = new Date().toISOString();
+
+  // Compute target path in vault
+  const targetDir = path.join(vaultDir, data.category || 'concepts');
+  if (!fs.existsSync(targetDir)) {
+    fs.mkdirSync(targetDir, { recursive: true });
+  }
+
+  const targetPath = path.join(targetDir, `${data.slug}.md`);
+  atomicWrite(targetPath, safeStringify(content, data));
+
+  // Remove from inbox
+  try { fs.unlinkSync(draftNode._filepath); } catch { /* ignore */ }
+}
+
+async function applyRevision(draftNode, confidenceAdj, revisionInstructions) {
+  if (!draftNode._filepath || !revisionInstructions) return;
+
+  const raw = fs.readFileSync(draftNode._filepath, 'utf8');
+  const { data, content } = matter(raw);
+
+  data.confidence = Math.min(1.0, Math.max(0.1, (data.confidence || 0.5) + confidenceAdj));
+  data.x_revision_requested = revisionInstructions;
+  data.updated = new Date().toISOString();
+
+  atomicWrite(draftNode._filepath, safeStringify(content, data));
+}
+
+async function quarantineDraft(draftNode, reason, quarantineDir) {
+  if (!fs.existsSync(quarantineDir)) {
+    fs.mkdirSync(quarantineDir, { recursive: true });
+  }
+
+  if (!draftNode._filepath) return;
+
+  const raw = fs.readFileSync(draftNode._filepath, 'utf8');
+  const { data, content } = matter(raw);
+  data.status = 'quarantined';
+  data.x_quarantine_reason = reason;
+  data.x_quarantined_at = new Date().toISOString();
+
+  const targetPath = path.join(quarantineDir, `${data.slug}.md`);
+  atomicWrite(targetPath, safeStringify(content, data));
+
+  try { fs.unlinkSync(draftNode._filepath); } catch { /* ignore */ }
+}
+
+/**
+ * Run the validation gate over all draft nodes in the inbox.
+ *
+ * @param {object} opts
+ * @param {string} opts.inboxDir      Path to pending inbox
+ * @param {string} opts.vaultDir      Path to memory vault
+ * @param {string} opts.quarantineDir Path to quarantine
+ * @param {object} opts.runtimeConfig
+ * @returns {{ processed: number, approved: number, rejected: number }}
+ */
+export async function runConclusionWriter({ inboxDir, vaultDir, quarantineDir, runtimeConfig }) {
+  if (!fs.existsSync(inboxDir)) {
+    return { processed: 0, approved: 0, rejected: 0, skipped: 'no-inbox' };
+  }
+
+  const files = fs.readdirSync(inboxDir).filter(f => f.endsWith('.md'));
+  if (files.length === 0) {
+    return { processed: 0, approved: 0, rejected: 0, skipped: 'empty-inbox' };
+  }
+
+  // Load all active vault nodes for comparison
+  const { loadNodes } = await import('./vault.mjs');
+  const allActiveNodes = loadNodes(vaultDir).filter(n => n.status === 'active');
+
+  let approved = 0;
+  let rejected = 0;
+
+  // Process up to 10 drafts per run to avoid excessive LLM calls
+  const batch = files.slice(0, 10);
+
+  for (const file of batch) {
+    const filepath = path.join(inboxDir, file);
+    try {
+      const raw = fs.readFileSync(filepath, 'utf8');
+      const { data, content } = matter(raw);
+      if (!data.slug || data.status !== 'draft') continue;
+
+      const draftNode = { ...data, body: content.trim(), _filepath: filepath };
+      const result = await validateDraftNode(draftNode, allActiveNodes, {
+        vaultDir,
+        quarantineDir,
+        runtimeConfig,
+      });
+
+      if (result.promoted) approved++;
+      else if (result.verdict === 'REJECT') rejected++;
+    } catch (err) {
+      logger.info({
+        subsystem: 'conclusion-writer',
+        message: `Error processing ${file}: ${err.message}`,
+      });
+    }
+  }
+
+  return { processed: batch.length, approved, rejected };
+}

package/src/core/config.mjs

@@ -0,0 +1,277 @@
+import { z } from 'zod';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+
+// ─── Robust PATH-Expansion for LaunchAgents & background processes ─────────────
+try {
+  const HOME = os.homedir();
+  const commonPaths = [
+    '/opt/homebrew/bin',
+    '/usr/local/bin',
+    path.join(HOME, '.local/bin'),
+    path.join(HOME, '.nvm/versions/node/v24.12.0/bin'),
+    path.join(HOME, '.npm-global/bin'),
+    path.join(HOME, 'bin'),
+    '/usr/bin',
+    '/bin',
+    '/usr/sbin',
+    '/sbin'
+  ];
+
+  const nvmVersionsDir = path.join(HOME, '.nvm', 'versions', 'node');
+  if (fs.existsSync(nvmVersionsDir)) {
+    const versions = fs.readdirSync(nvmVersionsDir);
+    for (const v of versions) {
+      commonPaths.unshift(path.join(nvmVersionsDir, v, 'bin'));
+    }
+  }
+
+  const activePaths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [];
+  const uniquePaths = Array.from(new Set([...activePaths, ...commonPaths])).filter(p => fs.existsSync(p));
+  process.env.PATH = uniquePaths.join(path.delimiter);
+} catch (e) {
+  // Silent fallback
+}
+
+
+/**
+ * Central Configuration Schema
+ * Uses Zod to validate and parse environment variables into a strongly typed system.
+ */
+const configSchema = z.object({
+  agentDir: z.preprocess(
+    (val) => val || process.env._TR_TEST_AGENT_DIR || path.join(os.homedir(), '.agent'),
+    z.string()
+  ),
+  cliAgent: z.string().optional(),
+  cliModel: z.string().optional(),
+  cliTimeout: z.preprocess((val) => val === undefined || val === '' ? 300 : parseInt(String(val), 10), z.number().int().default(300)),
+  googleApiKey: z.string().optional(),
+  embedModel: z.string().default('gemini-embedding-2'),
+  searxngBaseUrl: z.string().optional(),
+  braveApiKey: z.string().optional(),
+  braveSearchApiKey: z.string().optional(),
+  exaApiKey: z.string().optional(),
+  githubToken: z.string().optional(),
+  serperApiKey: z.string().optional(),
+  tavilyApiKey: z.string().optional(),
+  dailySearchLimit: z.preprocess((val) => val === undefined || val === '' ? 50 : parseInt(String(val), 10), z.number().int().default(50)),
+  researchCooldownMs: z.preprocess((val) => val === undefined || val === '' ? 3600000 : parseInt(String(val), 10), z.number().int().default(3600000)),
+  sessionSecret: z.string().optional(),
+  nodeEnv: z.string().default('production'),
+  port: z.preprocess((val) => val === undefined || val === '' ? 3000 : parseInt(String(val), 10), z.number().int().default(3000)),
+  host: z.string().default('127.0.0.1'),
+  display: z.string().optional(),
+  totalRecallToken: z.string().optional(),
+  trBrain: z.string().optional(),
+  trPat: z.string().optional(),
+  xdgConfigHome: z.preprocess(
+    (val) => val || path.join(os.homedir(), '.config'),
+    z.string()
+  )
+});
+
+// Resolve brainDir paths
+const resolvedAgentDir = process.env.AGENT_DIR || process.env._TR_TEST_AGENT_DIR || path.join(os.homedir(), '.agent');
+const tempGlobalBrainDir = path.join(os.homedir(), '.agent', 'skills', 'total-recall');
+
+// Auto-detect project brain if any
+let projectBrainDir = null;
+if (!process.env._TR_TEST_AGENT_DIR) {
+  let dir = process.cwd();
+  const homeDir = os.homedir();
+  while (dir !== path.dirname(dir)) {
+    if (dir === homeDir) break;
+    const candidate = path.join(dir, '.agent', 'skills', 'total-recall');
+    if (fs.existsSync(path.join(candidate, 'SKILL.md'))) {
+      projectBrainDir = candidate;
+      break;
+    }
+    dir = path.dirname(dir);
+  }
+}
+
+// Load secrets from project or global
+let secrets = {};
+try {
+  const pathsToCheck = [];
+  if (projectBrainDir) {
+    pathsToCheck.push(path.join(projectBrainDir, 'config', 'secrets.enc'));
+  }
+  pathsToCheck.push(path.join(tempGlobalBrainDir, 'config', 'secrets.enc'));
+  pathsToCheck.push(path.join(resolvedAgentDir, 'skills', 'total-recall', 'config', 'secrets.enc'));
+
+  for (const p of pathsToCheck) {
+    if (fs.existsSync(p)) {
+      try {
+        const parsed = JSON.parse(fs.readFileSync(p, 'utf8') || '{}');
+        if (Object.keys(parsed).length > 0) {
+          secrets = parsed;
+          break; // Found valid secrets!
+        }
+      } catch {}
+    }
+  }
+} catch (e) {
+  // Ignore
+}
+
+// Capture raw configuration values from process.env and secrets.enc
+const rawConfig = {
+  agentDir: process.env.AGENT_DIR,
+  cliAgent: process.env.TR_CLI_AGENT,
+  cliModel: process.env.TR_CLI_MODEL,
+  cliTimeout: process.env.TR_CLI_TIMEOUT,
+  googleApiKey: process.env.GOOGLE_API_KEY || secrets.google_api_key,
+  embedModel: process.env.TR_EMBED_MODEL,
+  searxngBaseUrl: process.env.SEARXNG_BASE_URL,
+  braveApiKey: process.env.BRAVE_SEARCH_API_KEY || process.env.BRAVE_API_KEY || secrets.brave_api_key,
+  braveSearchApiKey: process.env.BRAVE_SEARCH_API_KEY || secrets.brave_api_key,
+  exaApiKey: process.env.EXA_API_KEY || secrets.exa_api_key,
+  githubToken: process.env.GITHUB_TOKEN || secrets.github_token,
+  serperApiKey: process.env.SERPER_API_KEY || secrets.serper_api_key,
+  tavilyApiKey: process.env.TAVILY_API_KEY || secrets.tavily_api_key,
+  dailySearchLimit: process.env.TR_DAILY_SEARCH_LIMIT,
+  researchCooldownMs: process.env.RESEARCH_COOLDOWN_MS,
+  sessionSecret: process.env.SESSION_SECRET,
+  nodeEnv: process.env.NODE_ENV,
+  port: process.env.PORT,
+  host: process.env.HOST,
+  display: process.env.DISPLAY,
+  totalRecallToken: process.env.TOTAL_RECALL_TOKEN,
+  trBrain: process.env.TR_BRAIN,
+  trPat: process.env.TR_PAT,
+  xdgConfigHome: process.env.XDG_CONFIG_HOME
+};
+
+let validated;
+try {
+  validated = configSchema.parse(rawConfig);
+} catch (err) {
+  const issues = err.issues.map(i => `${i.path.join('.')}: ${i.message}`).join(', ');
+  throw new Error(`Configuration validation failed: ${issues}`);
+}
+
+const config = Object.freeze(validated);
+
+export default config;
+
+/**
+ * brainDir — The Total Recall meta-skill directory.
+ * This IS the user's brain. ALL user data (memory-vault, config, sessions,
+ * scheduler, memory-derived, memory-inbox, logs, .backups) lives inside here.
+ * Backup backs up this one directory and gets everything.
+ */
+export const brainDir = path.join(config.agentDir, 'skills', 'total-recall');
+
+export const {
+  agentDir,
+  cliAgent,
+  cliModel,
+  cliTimeout,
+  googleApiKey,
+  embedModel,
+  searxngBaseUrl,
+  braveApiKey,
+  braveSearchApiKey,
+  exaApiKey,
+  githubToken,
+  serperApiKey,
+  tavilyApiKey,
+  dailySearchLimit,
+  researchCooldownMs,
+  sessionSecret,
+  nodeEnv,
+  port,
+  host,
+  display,
+  totalRecallToken,
+  trBrain,
+  trPat,
+  xdgConfigHome
+} = config;
+
+export function getEnvVar(name) {
+  return process.env[name];
+}
+
+// ─── Layered Brain Resolution ───────────────────────────────────────────────
+
+/**
+ * Global brain — always at ~/.agent/skills/total-recall/
+ * Holds identity: universal preferences, invariants, corrections, coding principles.
+ */
+export const globalAgentDir = path.join(os.homedir(), '.agent');
+export const globalBrainDir = path.join(globalAgentDir, 'skills', 'total-recall');
+
+/**
+ * Detect a project-level brain by walking up from startDir looking for
+ * .agent/skills/total-recall/. Skips the home directory (that's the global brain).
+ *
+ * @param {string} [startDir=process.cwd()] - Directory to start searching from
+ * @returns {{ agentDir: string, brainDir: string, projectRoot: string } | null}
+ */
+export function detectProjectBrain(startDir = process.cwd()) {
+  // In test mode, skip project detection
+  if (process.env._TR_TEST_AGENT_DIR) return null;
+
+  let dir = startDir;
+  const homeDir = os.homedir();
+  while (dir !== path.dirname(dir)) {
+    if (dir === homeDir) break; // don't detect global as project
+    const candidate = path.join(dir, '.agent', 'skills', 'total-recall');
+    if (fs.existsSync(path.join(candidate, 'SKILL.md'))) {
+      return {
+        agentDir: path.join(dir, '.agent'),
+        brainDir: candidate,
+        projectRoot: dir,
+      };
+    }
+    dir = path.dirname(dir);
+  }
+  return null;
+}
+
+/**
+ * Get both brain layers for the current context.
+ *
+ * @returns {{ global: { agentDir: string, brainDir: string, layer: 'global' },
+ *             project: { agentDir: string, brainDir: string, projectRoot: string, layer: 'project' } | null }}
+ */
+export function getActiveBrains() {
+  const global = { agentDir: globalAgentDir, brainDir: globalBrainDir, layer: 'global' };
+  const project = detectProjectBrain();
+  return {
+    global,
+    project: project ? { ...project, layer: 'project' } : null,
+  };
+}
+
+/**
+ * Resolve which brain to use for a given operation.
+ *
+ * @param {'global' | 'project' | 'auto'} layer - Explicit layer choice or auto-detect
+ * @param {string} [category] - Memory category (used for auto-detect heuristic)
+ * @returns {{ agentDir: string, brainDir: string, layer: 'global' | 'project' }}
+ */
+export function resolveBrainLayer(layer = 'auto', category) {
+  const { global, project } = getActiveBrains();
+
+  if (layer === 'global') return global;
+  if (layer === 'project') {
+    if (!project) {
+      throw new Error('No project brain found. Run `npx total-recall init --project` to create one.');
+    }
+    return project;
+  }
+
+  // Auto-detect by category heuristic
+  if (project && category) {
+    const projectCategories = new Set(['fact', 'concept', 'pattern', 'anti-pattern', 'decision']);
+    if (projectCategories.has(category)) return project;
+  }
+
+  // Default: project if exists, else global
+  return project || global;
+}