total-recall-brain 3.1.0

package/src/core/daemon-control.mjs

@@ -0,0 +1,158 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { logger } from './logger.mjs';
+import { agentDir, brainDir } from './config.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(__dirname, '..', '..');
+
+const PID_FILE = path.join(brainDir, 'logs', 'daemon.pid');
+const LOG_FILE = path.join(brainDir, 'logs', 'daemon.log');
+
+/**
+ * Reads the daemon PID file and checks if the process is actually running.
+ * @returns {number|null} The active PID, or null if not running/invalid.
+ */
+export function readPid() {
+  try {
+    if (!fs.existsSync(PID_FILE)) return null;
+    const pid = parseInt(fs.readFileSync(PID_FILE, 'utf8').trim(), 10);
+    if (isNaN(pid) || pid <= 0) return null;
+    
+    // Check if the process is alive using signal 0
+    try {
+      process.kill(pid, 0);
+      return pid;
+    } catch {
+      return null;
+    }
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Returns the status of the daemon process.
+ * @returns {'running'|'dead'|'not_started'}
+ */
+export function getDaemonStatus() {
+  try {
+    const pid = readPid();
+    if (pid) {
+      return 'running';
+    }
+    
+    // If process is not running but PID file exists, it's considered dead
+    if (fs.existsSync(PID_FILE)) {
+      return 'dead';
+    }
+    
+    return 'not_started';
+  } catch {
+    return 'not_started';
+  }
+}
+
+/**
+ * Detaches the daemon script in the background and writes its PID to file.
+ * @returns {number} The PID of the newly spawned daemon.
+ */
+export function startDaemon() {
+  const existingPid = readPid();
+  if (existingPid) {
+    logger.warn('daemon-control', `Daemon already running under PID ${existingPid}.`);
+    return existingPid;
+  }
+
+  const dreamScript = path.join(ROOT, 'src', 'core', 'daemon-loop.mjs');
+  if (!fs.existsSync(dreamScript)) {
+    // Fallback to dream.mjs for backward compatibility
+    const fallback = path.join(ROOT, 'src', 'core', 'dream.mjs');
+    if (fs.existsSync(fallback)) {
+      logger.warn('daemon-control', 'daemon-loop.mjs not found, falling back to dream.mjs');
+    } else {
+      throw new Error(`Daemon script not found at ${dreamScript}`);
+    }
+  }
+
+  const logsDir = path.dirname(LOG_FILE);
+  if (!fs.existsSync(logsDir)) {
+    fs.mkdirSync(logsDir, { recursive: true });
+  }
+
+  const logFd = fs.openSync(LOG_FILE, 'a');
+  
+  // Use spawn to launch in a detached background state
+  const child = spawn(process.execPath, [dreamScript], {
+    detached: true,
+    stdio: ['ignore', logFd, logFd],
+    cwd: ROOT,
+    env: {
+      ...process.env,
+      NODE_ENV: 'production',
+      AGENT_DIR: agentDir,
+    },
+  });
+
+  child.unref();
+  
+  // Write the PID file
+  fs.writeFileSync(PID_FILE, String(child.pid), 'utf8');
+  fs.closeSync(logFd);
+
+  logger.info('daemon-control', `Active Intelligence Daemon started detached (PID ${child.pid}).`);
+  return child.pid;
+}
+
+/**
+ * Stops the running background daemon.
+ * @returns {boolean} True if successfully stopped, false otherwise.
+ */
+export function stopDaemon() {
+  const pid = readPid();
+  if (!pid) {
+    // Clear PID file in case it was a stale/dead record
+    try {
+      if (fs.existsSync(PID_FILE)) fs.unlinkSync(PID_FILE);
+    } catch {}
+    return false;
+  }
+
+  try {
+    process.kill(pid, 'SIGTERM');
+    logger.info('daemon-control', `Sent SIGTERM to daemon PID ${pid}.`);
+    
+    // Wait briefly and verify cleanup
+    try {
+      if (fs.existsSync(PID_FILE)) fs.unlinkSync(PID_FILE);
+    } catch {}
+    return true;
+  } catch (err) {
+    logger.error('daemon-control', `Failed to terminate daemon PID ${pid}: ${err.message}`);
+    return false;
+  }
+}
+
+/**
+ * Ensures that the daemon is in a running state. Auto-starts if not.
+ * @returns {number|null} The PID of the daemon.
+ */
+export async function ensureDaemonRunning() {
+  const status = getDaemonStatus();
+  if (status === 'running') {
+    return readPid();
+  }
+  
+  logger.info('daemon-control', `Daemon status is '${status}'. Initiating auto-start...`);
+  try {
+    const pid = startDaemon();
+    logger.info('daemon-control', `Auto-start successful. Daemon is now running on PID ${pid}.`);
+    return pid;
+  } catch (err) {
+    logger.error('daemon-control', `Failed to auto-start background daemon: ${err.message}`);
+    throw err;
+  }
+}

package/src/core/daemon-loop.mjs

@@ -0,0 +1,364 @@
+/**
+ * Total Recall Active Intelligence Daemon Loop — v2 (Deterministic)
+ *
+ * A deterministic automation engine that runs crons, manages the research
+ * queue, ingests sessions, and maintains the skill folder. No local LLM required.
+ *
+ * Phases per tick:
+ *   1. Ingest new IDE conversation logs
+ *   2. Load/refresh task queue from disk
+ *   3. Dispatch next task (deterministic only — no LLM tasks)
+ *   4. Periodically recompile surfaces
+ *
+ * This script is spawned by `total-recall daemon start` and runs until killed.
+ */
+
+import path from 'path';
+import fs from 'fs';
+import { createScheduler, updateTaskStatus } from './scheduler.mjs';
+import { scanAndIngest } from './session-watcher.mjs';
+import { logger } from './logger.mjs';
+import { updateQueueItem, loadQueue } from './research-queue.mjs';
+import { execFileSync } from "node:child_process";
+import { agentDir, brainDir } from './config.mjs';
+
+const AGENT_DIR = agentDir;
+const BRAIN_DIR = brainDir;
+const VAULT_DIR = path.join(BRAIN_DIR, 'memory-vault');
+const SKILLS_DIR = path.join(AGENT_DIR, 'skills');
+const DERIVED_DIR = path.join(BRAIN_DIR, 'memory-derived');
+const CONFLICTS_DIR = path.join(BRAIN_DIR, 'memory-inbox', 'conflicts');
+const SESSIONS_DIR = path.join(BRAIN_DIR, 'sessions');
+const QUEUE_DIR = path.join(BRAIN_DIR, 'scheduler', 'queue');
+const INSTRUCTIONS_FILE = path.join(BRAIN_DIR, 'INSTRUCTIONS.md');
+
+// How often to recompile surfaces (every N task ticks)
+const RECOMPILE_EVERY_N_TASKS = 20;
+
+// Pause between task iterations (ms)
+const TASK_SLEEP_MS = 10000;
+
+// ─── Categories that require LLM (skipped in deterministic mode) ────────────────
+
+const LLM_REQUIRED_CATEGORIES = new Set([
+  'system2-deliberation',
+  'conscious-enforcement',
+  'cutoff-audit',
+]);
+
+/**
+ * Check if a task requires an LLM to execute.
+ * These tasks are skipped — the daemon is deterministic.
+ */
+function requiresLLM(task) {
+  if (LLM_REQUIRED_CATEGORIES.has(task.category)) return true;
+
+  const slug = task.slug;
+  if (slug.startsWith('research-deliberation-')) return true;
+  if (slug.startsWith('research-improvement-')) return true;
+  if (slug.startsWith('research-monitoring-')) return true;
+  if (slug.startsWith('research-expansion-')) return true;
+  if (slug.includes('self-diagnosis')) return true;
+  if (slug.includes('clarity-review')) return true;
+  if (slug.includes('staleness-check')) return true;
+  if (slug.includes('inference')) return true;
+  if (slug.includes('post-mortem')) return true;
+  if (slug.includes('compliance')) return true;
+
+  return false;
+}
+
+/**
+ * Dispatch a task to the appropriate engine.
+ * Only deterministic (non-LLM) tasks are executed.
+ */
+async function dispatchTask(task) {
+  const category = task.category;
+
+  try {
+    switch (category) {
+      case 'memory-maintenance':
+        return await runMaintenanceTask(task);
+
+      default:
+        // All other categories (research, deliberation, inference, etc.)
+        // require LLM and are handled by IDE agents, not the daemon.
+        logger.info({
+          subsystem: 'daemon-loop',
+          message: `Skipping task (not deterministic): ${task.slug} [${category}]`,
+        });
+        return { success: true, output: 'Skipped — handled by IDE agents' };
+    }
+  } catch (err) {
+    logger.info({
+      subsystem: 'daemon-loop',
+      message: `Task ${task.slug} failed: ${err.message}`,
+    });
+    return { success: false, error: err.message };
+  }
+}
+
+// ─── Research Engine (Web Fetching Only — No LLM Synthesis) ─────────────────────
+
+async function runResearchTask(task) {
+  const inboxDir = path.join(BRAIN_DIR, 'memory-inbox', 'pending');
+
+  // Knowledge acquisition — fetch from web, save raw results
+  if (task.slug.startsWith('research-acquisition-') || task.category === 'proactive-research' || task.slug.includes('fact-seeker') || task.slug.includes('knowledge-acquisition')) {
+    const { runKnowledgeAcquisitionCycle } = await import('./fact-seeker.mjs');
+    const forceTopic = task.target || null;
+    const result = await runKnowledgeAcquisitionCycle({
+      vaultDir: VAULT_DIR,
+      inboxDir,
+      queueDir: QUEUE_DIR,
+      forceTopic,
+      skillsDir: SKILLS_DIR,
+      derivedDir: DERIVED_DIR,
+      instructionsFile: INSTRUCTIONS_FILE,
+    });
+    if (result.skipped) return { success: true, output: `Knowledge acquisition: ${result.skipped}` };
+    const surfaceNote = result.surfaced ? ' [SURFACED]' : ' [staged]';
+    return {
+      success: true,
+      output: `Researched "${result.topic}": ${result.sources?.join(', ')} | confidence: ${result.confidence || 'n/a'} | slug: ${result.factSlug}${surfaceNote}`,
+      factSlug: result.factSlug,
+    };
+  }
+
+  // Conclusion writer — validate pending drafts (deterministic checks)
+  if (task.slug.includes('validate') || task.category === 'research-acquisition') {
+    const { runConclusionWriter } = await import('./conclusion-writer.mjs');
+    const result = await runConclusionWriter({
+      inboxDir,
+      vaultDir: VAULT_DIR,
+      quarantineDir: path.join(BRAIN_DIR, 'memory-inbox', 'quarantine'),
+    });
+    if (result.skipped) return { success: true, output: `Conclusion writer: ${result.skipped}` };
+    return {
+      success: true,
+      output: `Validated ${result.processed} drafts: ${result.approved} approved, ${result.rejected} rejected`,
+    };
+  }
+
+  return { success: true, output: 'No-op research task' };
+}
+
+// ─── Maintenance Engine (Deterministic Only) ────────────────────────────────────
+
+async function runMaintenanceTask(task) {
+  // Lease vacuuming — purely filesystem, no LLM
+  try {
+    const leasesDir = path.join(BRAIN_DIR, 'leases');
+    if (fs.existsSync(leasesDir)) {
+      const workspaces = fs.readdirSync(leasesDir);
+      for (const ws of workspaces) {
+        const wsDir = path.join(leasesDir, ws);
+        if (!fs.statSync(wsDir).isDirectory()) continue;
+        const files = fs.readdirSync(wsDir);
+        for (const file of files) {
+          if (!file.endsWith('.lease.json')) continue;
+          const fp = path.join(wsDir, file);
+          try {
+            const lease = JSON.parse(fs.readFileSync(fp, 'utf8'));
+            if (new Date(lease.expires_at) < new Date()) {
+              fs.unlinkSync(fp);
+              logger.info({ subsystem: 'daemon-loop', message: `Vacuumed expired lease: ${fp}` });
+            }
+          } catch {
+            try { fs.unlinkSync(fp); } catch {}
+          }
+        }
+      }
+    }
+  } catch (err) {
+    logger.info({ subsystem: 'daemon-loop', message: `Lease vacuuming failed: ${err.message}` });
+  }
+
+  return { success: true, output: 'Maintenance complete' };
+}
+
+// ─── Interrupt Writer ───────────────────────────────────────────────────────────
+
+/**
+ * Write an interrupt to the SKILL.md file so the agent picks it up on next turn.
+ * Replaces the old interrupts/pending.md approach.
+ */
+export function writeInterrupt(message) {
+  const skillMdPath = path.join(SKILLS_DIR, 'total-recall', 'SKILL.md');
+  if (!fs.existsSync(skillMdPath)) return;
+
+  const content = fs.readFileSync(skillMdPath, 'utf8');
+  const marker = '<!-- daemon writes below this line, agent clears after reading -->';
+  const idx = content.indexOf(marker);
+  if (idx === -1) return;
+
+  const insertPoint = idx + marker.length;
+  const timestamp = new Date().toISOString();
+  const interrupt = `\n🔔 **[${timestamp}]** ${message}`;
+
+  const updated = content.slice(0, insertPoint) + interrupt + content.slice(insertPoint);
+  fs.writeFileSync(skillMdPath, updated);
+
+  logger.info({ subsystem: 'daemon-loop', message: `Interrupt written to SKILL.md: ${message}` });
+}
+
+// ─── Main Daemon Loop ───────────────────────────────────────────────────────────
+
+let taskCount = 0;
+let running = true;
+
+process.on('SIGTERM', () => {
+  logger.info({ subsystem: 'daemon-loop', message: 'SIGTERM received — shutting down gracefully' });
+  running = false;
+});
+
+process.on('SIGINT', () => {
+  logger.info({ subsystem: 'daemon-loop', message: 'SIGINT received — shutting down gracefully' });
+  running = false;
+});
+
+async function main() {
+  logger.info({
+    subsystem: 'daemon-loop',
+    message: `Deterministic Daemon starting. Vault: ${VAULT_DIR}`,
+  });
+
+  // Initial session ingest
+  try {
+    const ingestResult = scanAndIngest(SESSIONS_DIR);
+    logger.info({
+      subsystem: 'daemon-loop',
+      message: `Boot ingest: ${ingestResult.ingested} new sessions`,
+    });
+  } catch (err) {
+    logger.info({ subsystem: 'daemon-loop', message: `Boot ingest failed: ${err.message}` });
+  }
+
+  while (running) {
+    try {
+      // Periodically recompile surfaces
+      if (taskCount > 0 && taskCount % RECOMPILE_EVERY_N_TASKS === 0) {
+        logger.info({ subsystem: 'daemon-loop', message: 'Running scheduled surface recompile...' });
+        try {
+          const { compileSurface } = await import('./surface.mjs');
+          await compileSurface({
+            vaultDir: VAULT_DIR,
+            skillsDir: SKILLS_DIR,
+            derivedDir: DERIVED_DIR,
+            instructionsFile: INSTRUCTIONS_FILE,
+          });
+        } catch (err) {
+          logger.info({ subsystem: 'daemon-loop', message: `Surface recompile error: ${err.message}` });
+        }
+      }
+
+      // Refresh scheduler from disk on each iteration
+      const scheduler = createScheduler({
+        queueDir: QUEUE_DIR,
+        vaultDir: VAULT_DIR,
+        sessionsDir: SESSIONS_DIR,
+      });
+
+      const { task, source } = scheduler.next();
+      taskCount++;
+
+      logger.info({
+        subsystem: 'daemon-loop',
+        message: `Task #${taskCount} [${source}] ${task.category}: ${task.slug}`,
+      });
+
+      // Mark in-progress
+      if ((source === 'explicit' || source === 'idle') && task._filepath) {
+        try {
+          updateTaskStatus(task, 'in-progress', QUEUE_DIR);
+        } catch (statusErr) {
+          logger.info({
+            subsystem: 'daemon-loop',
+            message: `Failed to mark task in-progress (non-fatal): ${statusErr.message}`,
+          });
+        }
+      }
+      if (task._research_id) {
+        try {
+          updateQueueItem(task._research_id, { status: 'in_progress' });
+        } catch (err) {
+          logger.error({
+            subsystem: 'daemon-loop',
+            message: `Failed to update research queue status: ${err.message}`,
+          });
+        }
+      }
+
+      const result = await dispatchTask(task);
+
+      // Mark complete
+      if ((source === 'explicit' || source === 'idle') && task._filepath) {
+        try {
+          updateTaskStatus(task, result.success ? 'completed' : 'failed', QUEUE_DIR);
+        } catch (statusErr) {
+          logger.info({
+            subsystem: 'daemon-loop',
+            message: `Failed to mark task complete (non-fatal): ${statusErr.message}`,
+          });
+        }
+      }
+      if (task._research_id) {
+        try {
+          const patch = {
+            status: result.success ? 'pending' : 'failed',
+            notes: result.output || result.error || null,
+          };
+          if (result.factSlug) {
+            patch.node_slug = result.factSlug;
+          }
+          // Only advance through acquisition phase — skip LLM phases
+          if (result.success && task._research_phase === 'acquisition' && result.factSlug) {
+            // Research stays in acquisition — no LLM phases to advance to
+            patch.status = 'done';
+          }
+          updateQueueItem(task._research_id, patch);
+        } catch (err) {
+          logger.error({
+            subsystem: 'daemon-loop',
+            message: `Failed to update research queue status: ${err.message}`,
+          });
+        }
+      }
+
+      logger.info({
+        subsystem: 'daemon-loop',
+        message: `Task #${taskCount} done: ${result.output || result.error || 'ok'}`,
+      });
+
+      // Throttle to protect local resources
+      await new Promise(r => setTimeout(r, TASK_SLEEP_MS));
+
+    } catch (loopErr) {
+      // Individual task failures must NEVER kill the daemon.
+      logger.info({
+        subsystem: 'daemon-loop',
+        message: `Task loop iteration crashed (non-fatal, continuing): ${loopErr.message}`,
+      });
+      await new Promise(r => setTimeout(r, 1000));
+    }
+  }
+
+  logger.info({ subsystem: 'daemon-loop', message: 'Deterministic Daemon stopped.' });
+}
+
+main().catch(async (err) => {
+  logger.info({ subsystem: 'daemon-loop', message: `Fatal error: ${err.message}` });
+
+  try {
+    const { writeEmergencyAlert } = await import('./emergency-alerts.mjs');
+    writeEmergencyAlert(
+      `The Daemon has CRASHED with a fatal error: ${err.message}. ` +
+      `No background automation is running. ` +
+      `Restart with: node bin/total-recall.mjs daemon start`
+    );
+  } catch {
+    // If even the alert system fails, we still exit
+  }
+
+  process.exit(1);
+});