total-recall-brain 3.6.4 → 3.6.6

package/src/core/daemon-loop.mjs

@@ -38,61 +38,81 @@ const RECOMPILE_EVERY_N_TASKS = 20;
 // Pause between task iterations (ms)
 const TASK_SLEEP_MS = 10000;
 
-// ─── Categories that require LLM (skipped in deterministic mode) ────────────────
+// ─── Tasks are no longer skipped to conserve resources. We use CLI agents! ───
 
-const LLM_REQUIRED_CATEGORIES = new Set([
-  'system2-deliberation',
-  'conscious-enforcement',
-  'cutoff-audit',
-]);
-
-/**
- * Check if a task requires an LLM to execute.
- * These tasks are skipped — the daemon is deterministic.
- */
-function requiresLLM(task) {
-  if (LLM_REQUIRED_CATEGORIES.has(task.category)) return true;
-
-  const slug = task.slug;
-  if (slug.startsWith('research-deliberation-')) return true;
-  if (slug.startsWith('research-improvement-')) return true;
-  if (slug.startsWith('research-monitoring-')) return true;
-  if (slug.startsWith('research-expansion-')) return true;
-  if (slug.includes('self-diagnosis')) return true;
-  if (slug.includes('clarity-review')) return true;
-  if (slug.includes('staleness-check')) return true;
-  if (slug.includes('inference')) return true;
-  if (slug.includes('post-mortem')) return true;
-  if (slug.includes('compliance')) return true;
-
-  return false;
-}
-
-/**
- * Dispatch a task to the appropriate engine.
- * Only deterministic (non-LLM) tasks are executed.
- */
 async function dispatchTask(task) {
   const category = task.category;
 
   try {
-    switch (category) {
-      case 'memory-maintenance':
-        return await runMaintenanceTask(task);
-
-      case 'proactive-research':
-      case 'research-acquisition':
-        return await runResearchTask(task);
-
-      default:
-        // All other categories (research, deliberation, inference, etc.)
-        // require LLM and are handled by IDE agents, not the daemon.
-        logger.info({
-          subsystem: 'daemon-loop',
-          message: `Skipping task (not deterministic): ${task.slug} [${category}]`,
-        });
-        return { success: true, output: 'Skipped — handled by IDE agents' };
+    const { loadRuntimeConfig } = await import('./runtime.mjs');
+    const runtimeConfig = await loadRuntimeConfig();
+
+    // ─── Research & Proactive Tasks (LLM + Fetching) ───
+    if (task.slug.startsWith('research-') || category === 'proactive-research' || category === 'research-acquisition') {
+      if (task.slug.startsWith('staleness-check-')) {
+         const { runStalenessCheck } = await import('./clarity-rewriter.mjs');
+         const result = await runStalenessCheck(task.target, { vaultDir: VAULT_DIR, queueDir: QUEUE_DIR, runtimeConfig });
+         return { success: true, output: `Staleness check complete: ${result.verdict} (conf: ${result.confidence})` };
+      }
+      return await runResearchTask(task, runtimeConfig);
+    }
+    
+    // ─── System 2 Inference ───
+    if (category === 'system2-deliberation') {
+      const { runInferenceTask } = await import('./inference-engine.mjs');
+      const slugs = task.target ? task.target.split(',') : [];
+      const result = await runInferenceTask(slugs, {
+        vaultDir: VAULT_DIR,
+        inboxDir: path.join(BRAIN_DIR, 'memory-inbox', 'pending'),
+        runtimeConfig
+      });
+      return { success: true, output: `Inference complete: ${result.conclusions?.length || 0} conclusions` };
+    }
+
+    // ─── Cutoff Audit ───
+    if (category === 'cutoff-audit' || task.slug.startsWith('cutoff-audit-')) {
+      const { runCutoffAudit } = await import('./clarity-rewriter.mjs');
+      const result = await runCutoffAudit({ vaultDir: VAULT_DIR, queueDir: QUEUE_DIR, runtimeConfig });
+      return { success: true, output: `Cutoff audit complete: ${result.audited} audited, ${result.flagged} flagged, ${result.critical} critical` };
+    }
+
+    // ─── Self Diagnosis & Conscious Enforcement ───
+    if (category === 'conscious-enforcement' || task.slug.includes('self-diagnosis')) {
+      if (task.slug.startsWith('post-mortem-')) {
+        const { runPostMortem } = await import('./post-mortem.mjs');
+        const sessionPath = path.join(brainDir, 'sessions', task.target);
+        const result = await runPostMortem(sessionPath, { vaultDir: VAULT_DIR, inboxDir: path.join(BRAIN_DIR, 'memory-inbox', 'pending'), runtimeConfig });
+        return { success: true, output: `Post-mortem complete: ${result.nodesCreated || 0} nodes extracted` };
+      }
+
+      if (task.slug.includes('self-diagnosis')) {
+        const { runSelfDiagnosis } = await import('./fact-seeker.mjs');
+        await runSelfDiagnosis({ vaultDir: VAULT_DIR, runtimeConfig });
+        return { success: true, output: `Self-diagnosis complete` };
+      }
+      return { success: true, output: `Conscious enforcement handled` };
     }
+
+    // ─── Memory Maintenance ───
+    if (category === 'memory-maintenance') {
+      if (task.slug.startsWith('clarity-review-')) {
+        const { runClarityReview } = await import('./clarity-rewriter.mjs');
+        const result = await runClarityReview(task.target, { vaultDir: VAULT_DIR, inboxDir: path.join(BRAIN_DIR, 'memory-inbox', 'pending'), runtimeConfig });
+        return { success: true, output: `Clarity review complete: rewrote=${result.rewrote}` };
+      }
+      if (task.slug.startsWith('memory-compaction-')) {
+        const { runMemoryCompaction } = await import('./fact-seeker.mjs');
+        const result = await runMemoryCompaction({ vaultDir: VAULT_DIR, inboxDir: path.join(BRAIN_DIR, 'memory-inbox', 'pending'), runtimeConfig });
+        return { success: true, output: `Memory compaction complete. Consolidated nodes: ${result.consolidatedCount || 0}` };
+      }
+      return await runMaintenanceTask(task);
+    }
+
+    logger.info({
+      subsystem: 'daemon-loop',
+      message: `Skipping unhandled task category: ${task.slug} [${category}]`,
+    });
+    return { success: true, output: 'Skipped unhandled task' };
   } catch (err) {
     logger.info({
       subsystem: 'daemon-loop',
@@ -104,10 +124,10 @@ async function dispatchTask(task) {
 
 // ─── Research Engine (Web Fetching Only — No LLM Synthesis) ─────────────────────
 
-async function runResearchTask(task) {
+async function runResearchTask(task, runtimeConfig) {
   const inboxDir = path.join(BRAIN_DIR, 'memory-inbox', 'pending');
 
-  // Knowledge acquisition — fetch from web, save raw results
+  // Phase 1: Knowledge acquisition
   if (task.slug.startsWith('research-acquisition-') || task.category === 'proactive-research' || task.slug.includes('fact-seeker') || task.slug.includes('knowledge-acquisition')) {
     const { runKnowledgeAcquisitionCycle } = await import('./fact-seeker.mjs');
     const forceTopic = task.target || null;
@@ -119,6 +139,7 @@ async function runResearchTask(task) {
       skillsDir: SKILLS_DIR,
       derivedDir: DERIVED_DIR,
       instructionsFile: INSTRUCTIONS_FILE,
+      runtimeConfig
     });
     if (result.skipped) return { success: true, output: `Knowledge acquisition: ${result.skipped}` };
     const surfaceNote = result.surfaced ? ' [SURFACED]' : ' [staged]';
@@ -129,8 +150,59 @@ async function runResearchTask(task) {
     };
   }
 
+  // Phase 2: Deliberation
+  if (task.slug.startsWith('research-deliberation-')) {
+    const { runResearchDeliberationCycle } = await import('./fact-seeker.mjs');
+    const result = await runResearchDeliberationCycle({
+      vaultDir: VAULT_DIR,
+      nodeSlug: task._node_slug || 'pending',
+      topic: task.title || task.target || 'Unknown Topic',
+      runtimeConfig
+    });
+    return { success: true, output: `Deliberation complete: ${result.slug || 'done'}` };
+  }
+
+  // Phase 3: Improvement
+  if (task.slug.startsWith('research-improvement-')) {
+    const { runResearchImprovementCycle } = await import('./fact-seeker.mjs');
+    const result = await runResearchImprovementCycle({
+      vaultDir: VAULT_DIR,
+      nodeSlug: task._node_slug || 'pending',
+      topic: task.title || task.target || 'Unknown Topic',
+      runtimeConfig
+    });
+    return { success: true, output: `Improvement complete` };
+  }
+
+  // Phase 4: Monitoring
+  if (task.slug.startsWith('research-monitoring-')) {
+    const { runResearchMonitoringCycle } = await import('./fact-seeker.mjs');
+    const result = await runResearchMonitoringCycle({
+      vaultDir: VAULT_DIR,
+      nodeSlug: task._node_slug || 'pending',
+      topic: task.title || task.target || 'Unknown Topic',
+      runtimeConfig,
+      skillsDir: SKILLS_DIR,
+      derivedDir: DERIVED_DIR,
+      instructionsFile: INSTRUCTIONS_FILE
+    });
+    return { success: true, output: `Monitoring complete` };
+  }
+
+  // Phase 5: Expansion
+  if (task.slug.startsWith('research-expansion-')) {
+    const { runResearchExpansionCycle } = await import('./fact-seeker.mjs');
+    const result = await runResearchExpansionCycle({
+      vaultDir: VAULT_DIR,
+      nodeSlug: task._node_slug || 'pending',
+      topic: task.title || task.target || 'Unknown Topic',
+      runtimeConfig
+    });
+    return { success: true, output: `Expansion complete` };
+  }
+
   // Conclusion writer — validate pending drafts (deterministic checks)
-  if (task.slug.includes('validate') || task.category === 'research-acquisition') {
+  if (task.slug.includes('validate')) {
     const { runConclusionWriter } = await import('./conclusion-writer.mjs');
     const result = await runConclusionWriter({
       inboxDir,
@@ -309,7 +381,21 @@ async function main() {
       // Mark complete
       if ((source === 'explicit' || source === 'idle') && task._filepath) {
         try {
-          updateTaskStatus(task, result.success ? 'completed' : 'failed', QUEUE_DIR);
+          if (result.skippedLLM) {
+            updateTaskStatus(task, 'pending', QUEUE_DIR);
+          } else if (result.success) {
+            updateTaskStatus(task, 'completed', QUEUE_DIR);
+          } else {
+            // Dead Letter Queue / Retry Logic
+            task.retry_count = (task.retry_count || 0) + 1;
+            if (task.retry_count >= 3) {
+              logger.info({ subsystem: 'daemon-loop', message: `Task ${task.slug} failed 3 times. Moving to DLQ (failed).` });
+              updateTaskStatus(task, 'failed', QUEUE_DIR);
+            } else {
+              logger.info({ subsystem: 'daemon-loop', message: `Task ${task.slug} failed. Retry ${task.retry_count}/3. Requeueing.` });
+              updateTaskStatus(task, 'pending', QUEUE_DIR);
+            }
+          }
         } catch (statusErr) {
           logger.info({
             subsystem: 'daemon-loop',
@@ -321,15 +407,33 @@ async function main() {
         try {
           const patch = {
             status: result.success ? 'pending' : 'failed',
-            notes: result.output || result.error || null,
+            // Append the output rather than completely overwriting the original rationale
+            notes: result.output ? `Phase output: ${result.output}` : (result.error || null),
           };
           if (result.factSlug) {
             patch.node_slug = result.factSlug;
           }
-          // Only advance through acquisition phase — skip LLM phases
-          if (result.success && task._research_phase === 'acquisition' && result.factSlug) {
-            // Research stays in acquisition — no LLM phases to advance to
-            patch.status = 'done';
+          // If the task was skipped because it requires an LLM, DO NOT complete the research project.
+          if (result.skippedLLM) {
+            patch.status = 'pending';
+          } else if (result.success && result.factSlug) {
+            if (task._research_phase === 'acquisition') {
+              patch.status = 'pending';
+              patch.research_phase = 'deliberation';
+              patch.completed_at = new Date().toISOString();
+            } else if (task._research_phase === 'deliberation') {
+              patch.status = 'pending';
+              patch.research_phase = 'improvement';
+            } else if (task._research_phase === 'improvement') {
+              patch.status = 'pending';
+              patch.research_phase = 'monitoring';
+            } else if (task._research_phase === 'monitoring') {
+              patch.status = 'pending';
+              patch.research_phase = 'expansion';
+            } else if (task._research_phase === 'expansion') {
+              patch.status = 'done';
+              patch.completed_at = new Date().toISOString();
+            }
           }
           updateQueueItem(task._research_id, patch);
         } catch (err) {

package/src/core/fact-seeker.mjs

@@ -529,7 +529,8 @@ export function buildMultiNoteGraph(topic, synthesis, parsedSupportingNotes, sta
     return { slug, body: bodyLines.join('\n'), frontmatter };
   });
 
-  const masterSlug = `fact-master-${crypto.randomBytes(5).toString('hex')}`;
+  const topicHash = crypto.createHash('md5').update(topic).digest('hex').slice(0, 10);
+  const masterSlug = `fact-master-${topicHash}`;
 
   // Bidirectionally link the master node and supporting notes
   const masterRelated = supportingNodes.map(n => n.slug);
@@ -658,7 +659,8 @@ async function writeAndSurfaceImmediately(topic, synthesis, sourceResults, {
   const topResults = sourceResults.slice(0, 4);
   for (const r of topResults) {
     const summary = await synthesizeSupportingNote(topic, r, runtimeConfig);
-    const slug = `fact-note-${crypto.randomBytes(5).toString('hex')}`;
+    const noteHash = crypto.createHash('md5').update(r.url || r.title || r.snippet).digest('hex').slice(0, 10);
+    const slug = `fact-note-${noteHash}`;
     parsedSupportingNotes.push({ result: r, summary, slug });
   }
 
@@ -700,7 +702,8 @@ async function writeDraftToInbox(topic, synthesis, sourceResults, inboxDir, runt
   const topResults = sourceResults.slice(0, 4);
   for (const r of topResults) {
     const summary = await synthesizeSupportingNote(topic, r, runtimeConfig);
-    const slug = `fact-note-${crypto.randomBytes(5).toString('hex')}`;
+    const noteHash = crypto.createHash('md5').update(r.url || r.title || r.snippet).digest('hex').slice(0, 10);
+    const slug = `fact-note-${noteHash}`;
     parsedSupportingNotes.push({ result: r, summary, slug });
   }
 

package/src/core/research-queue.mjs

@@ -325,6 +325,14 @@ export function listQueue({ status, query, limit = 100, offset = 0, brainDir: ov
  */
 export function addToQueue({ topic, priority = 'medium', notes, brainDir: overrideBrainDir } = {}) {
   if (!topic) throw new Error('topic is required');
+
+  const items = loadQueue(overrideBrainDir);
+  const normalizedTopic = String(topic).trim().toLowerCase();
+  const existing = items.find(i => i.topic.trim().toLowerCase() === normalizedTopic);
+  if (existing) {
+    return existing;
+  }
+
   const item = {
     id:           crypto.randomUUID(),
     topic:        String(topic),
@@ -338,7 +346,6 @@ export function addToQueue({ topic, priority = 'medium', notes, brainDir: overri
     completed_at: null,
   };
   item.summary = compileResearchProjectSummary(item, null);
-  const items = loadQueue(overrideBrainDir);
   items.unshift(item);
   saveQueue(items, overrideBrainDir);
 

package/src/core/scheduler.mjs

@@ -210,6 +210,7 @@ export function generateIdleTask({ vaultDir, sessionsDir }) {
     () => generateInferenceTask(vaultDir),
     () => generatePostMortemTask(sessionsDir),
     () => generateClarityReviewTask(vaultDir),
+    () => generateMemoryCompactionTask(vaultDir),
   ];
 
   const strategy = cleanStrategies[_idleCycleCounter % cleanStrategies.length];
@@ -248,18 +249,34 @@ function generateClarityReviewTask(vaultDir) {
   if (nodes.length === 0) {
     return makeFallbackTask('memory-maintenance', 'No active nodes to review');
   }
-
-  const target = nodes[crypto.randomInt(0, nodes.length)];
+  const target = nodes[Math.floor(Math.random() * nodes.length)];
   return {
     type: 'task',
-    slug: `clarity-review-${target.slug}-${Date.now().toString(36)}`,
+    slug: `clarity-review-${crypto.randomBytes(4).toString('hex')}`,
     priority: 30,
     category: 'memory-maintenance',
     target: target.slug,
     status: 'pending',
     created_by: 'scheduler-idle',
-    reason: `Idle task: review "${target.title}" for clarity and actionability.`,
-    body: `## Objective\nReview memory node "${target.slug}" and evaluate:\n1. Is the title descriptive enough for a 7-item skill injection?\n2. Is the body actionable by an AI agent?\n3. Does the frontmatter accurately reflect the content?\n\nIf improvements are needed, generate a rewrite proposal.`,
+    reason: `Idle task: selected active node ${target.slug} for clarity review.`,
+    body: `Analyze node ${target.slug} for clarity, deduplication, formatting, and structural integrity. Rewrite via SSSS strict compliance if needed.`,
+  };
+}
+
+/**
+ * Generate a memory compaction task to fuse highly overlapping fragmented nodes.
+ */
+function generateMemoryCompactionTask(vaultDir) {
+  return {
+    type: 'task',
+    slug: `memory-compaction-${crypto.randomBytes(4).toString('hex')}`,
+    priority: 35,
+    category: 'memory-maintenance',
+    target: 'global',
+    status: 'pending',
+    created_by: 'scheduler-idle',
+    reason: `Idle task: scan vault for fragmented nodes to merge into comprehensive master documents.`,
+    body: `Scan for highly overlapping or fragmented memory nodes and fuse them into comprehensive master nodes, archiving the fragments.`,
   };
 }
 
@@ -523,6 +540,7 @@ export function createScheduler({ queueDir, vaultDir, sessionsDir }) {
         body,
         _research_id: item.id,
         _research_phase: phase,
+        _node_slug: item.node_slug,
       });
     }
     if (researchItems.length > 0) {

package/src/core/schema.mjs

@@ -644,6 +644,28 @@ export const PhoneNumberSchema = z.object({
   user_id: z.string().optional().nullable(),
 });
 
+export const DomainSchema = z.object({
+  type: z.literal('domain'),
+  domain_name: z.string(),
+  registrar: z.string(),
+  status: z.string(),
+  payment_status: z.string(),
+  workspace_id: z.string().optional().nullable(),
+  user_id: z.string().optional().nullable(),
+  base_price_usd: z.number().optional().nullable(),
+  purchase_price_usd: z.number().optional().nullable(),
+  annual_renewal_usd: z.number().optional().nullable(),
+  markup_amount_usd: z.number().optional().nullable(),
+  markup_rate: z.number().optional().nullable(),
+  expires_at: z.string().optional().nullable(),
+  payment_method: z.string().optional().nullable(),
+  vercel_domain_id: z.string().optional().nullable(),
+  stripe_payment_intent_id: z.string().optional().nullable(),
+  stripe_checkout_session_id: z.string().optional().nullable(),
+  credits_charged: z.number().optional().nullable(),
+  credit_transaction_id: z.string().optional().nullable(),
+});
+
 export const WorkspaceSchema = z.object({
   type: z.literal('workspace'),
   name: z.string(),
@@ -804,6 +826,7 @@ export const SSSS_SCHEMAS = {
   workspace_transfer: WorkspaceTransferSchema,
   extension: ExtensionSchema,
   phone_number: PhoneNumberSchema,
+  domain: DomainSchema,
   workspace: WorkspaceSchema,
   commerce_catalog: CommerceCatalogSchema,
   product: ProductSchema,

package/src/server/index.mjs

@@ -138,6 +138,23 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
   // Determine overall status
   const hasCriticalIssue = emergencyAlerts.length > 0 || daemonStatus === 'dead' || cliAgents.length === 0;
 
+  // Check Caddy and Cloudflare status
+  let caddyStatus = 'inactive';
+  let cloudflareStatus = 'inactive';
+  try {
+    const { exec } = await import('node:child_process');
+    const util = await import('node:util');
+    const execAsync = util.promisify(exec);
+    try {
+      await execAsync('pgrep caddy');
+      caddyStatus = 'active';
+    } catch { }
+    try {
+      await execAsync('pgrep cloudflared');
+      cloudflareStatus = 'active';
+    } catch { }
+  } catch { }
+
   res.json({
     status: hasCriticalIssue ? 'degraded' : 'healthy',
     version: PACKAGE_VERSION,
@@ -146,6 +163,8 @@ app.get('/health', requireAuthOrLocal, async (req, res) => {
     disk,
     cli_agents: cliAgents,
     daemon: daemonStatus,
+    caddy: caddyStatus,
+    cloudflare: cloudflareStatus,
     emergency_alerts: emergencyAlerts || null,
     vfs: {
       exists: vaultExists,