total-recall-brain 3.1.0

package/src/core/tts.mjs

@@ -0,0 +1,142 @@
+/**
+ * Total Recall — Text-to-Speech (Kokoro-82M)
+ *
+ * Kokoro-82M is a tiny (~82M parameter) high-quality multilingual TTS model.
+ * The reference implementation runs in Python; the recommended deployment is
+ * `kokoro-fastapi` or a similar HTTP wrapper running on the host.
+ *
+ * This module is the kernel-side façade. It does NOT bundle Python. Instead
+ * it speaks HTTP to a local Kokoro service whose URL lives in voice.yml.
+ * Operators wire Kokoro into systemd; the OS just routes through it.
+ *
+ * Configuration (~/.agent/config/voice.yml):
+ *   tts:
+ *     engine: kokoro            # or "disabled"
+ *     endpoint: http://127.0.0.1:8880/v1/audio/speech
+ *     voice: af_heart           # any Kokoro voice id
+ *     format: wav               # wav | mp3 | flac | opus | pcm
+ *     speed: 1.0
+ *
+ * Behavior:
+ *   - When `engine: disabled` or no config exists → `synthesize()` throws
+ *     `TtsNotConfiguredError`. The HTTP layer surfaces this as 503 so the
+ *     frontend can fall back to browser speechSynthesis.
+ *   - When configured → POSTs to `endpoint` using the OpenAI-compatible
+ *     `/v1/audio/speech` contract (Kokoro-fastapi default) and streams the
+ *     audio buffer back.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'yaml';
+import { logger } from './logger.mjs';
+import { brainDir } from './config.mjs';
+
+const VOICE_CONFIG = path.join(brainDir, 'config', 'voice.yml');
+
+const DEFAULT_CONFIG = {
+  tts: {
+    engine: 'disabled',
+    endpoint: 'http://127.0.0.1:8880/v1/audio/speech',
+    voice: 'af_heart',
+    format: 'wav',
+    speed: 1.0
+  }
+};
+
+export class TtsNotConfiguredError extends Error {
+  constructor(message = 'Kokoro TTS is not configured. See ~/.agent/config/voice.yml.') {
+    super(message);
+    this.name = 'TtsNotConfiguredError';
+    this.code = 'TTS_NOT_CONFIGURED';
+  }
+}
+
+export function loadVoiceConfig(file = VOICE_CONFIG) {
+  if (!fs.existsSync(file)) return DEFAULT_CONFIG;
+  try {
+    const parsed = yaml.parse(fs.readFileSync(file, 'utf8')) || {};
+    return { tts: { ...DEFAULT_CONFIG.tts, ...(parsed.tts || {}) } };
+  } catch (err) {
+    logger.error('tts', `Failed to parse voice.yml: ${err.message}`);
+    return DEFAULT_CONFIG;
+  }
+}
+
+const MIME_BY_FORMAT = {
+  wav: 'audio/wav',
+  mp3: 'audio/mpeg',
+  flac: 'audio/flac',
+  opus: 'audio/opus',
+  pcm: 'audio/L16'
+};
+
+/**
+ * Synthesize `text` to audio bytes using the configured Kokoro endpoint.
+ *
+ * @returns {Promise<{ buffer: Buffer, mimeType: string, format: string }>}
+ * @throws  {TtsNotConfiguredError} when Kokoro is disabled or unreachable.
+ */
+export async function synthesize(text, opts = {}) {
+  if (typeof text !== 'string' || !text.trim()) {
+    throw new Error('synthesize(): text must be a non-empty string');
+  }
+
+  const cfg = loadVoiceConfig().tts;
+  if (cfg.engine === 'disabled' || !cfg.endpoint) {
+    throw new TtsNotConfiguredError();
+  }
+
+  const voice = opts.voice || cfg.voice;
+  const format = opts.format || cfg.format;
+  const speed = opts.speed ?? cfg.speed;
+
+  const body = {
+    model: 'kokoro',
+    input: text,
+    voice,
+    response_format: format,
+    speed
+  };
+
+  const start = Date.now();
+  let res;
+  try {
+    res = await fetch(cfg.endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body)
+    });
+  } catch (err) {
+    // Kokoro service down or unreachable — treat as not-configured so the
+    // frontend can fall back rather than displaying a crash.
+    logger.warn('tts', `Kokoro endpoint unreachable: ${err.message}`);
+    throw new TtsNotConfiguredError(`Kokoro endpoint unreachable: ${err.message}`);
+  }
+
+  const latency_ms = Date.now() - start;
+
+  if (!res.ok) {
+    const detail = await res.text().catch(() => '');
+    logger.error('tts', `Kokoro returned ${res.status}: ${detail.slice(0, 200)}`, { latency_ms });
+    throw new Error(`Kokoro TTS failed: HTTP ${res.status}`);
+  }
+
+  const arrayBuffer = await res.arrayBuffer();
+  const buffer = Buffer.from(arrayBuffer);
+  logger.info('tts', `synthesized ${buffer.length} bytes (${format}, voice=${voice})`, {
+    latency_ms,
+    bytes: buffer.length
+  });
+
+  return {
+    buffer,
+    mimeType: MIME_BY_FORMAT[format] || 'application/octet-stream',
+    format
+  };
+}
+
+export function isTtsEnabled() {
+  const cfg = loadVoiceConfig().tts;
+  return cfg.engine !== 'disabled' && !!cfg.endpoint;
+}

package/src/core/usage-tracker.mjs

@@ -0,0 +1,349 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import yaml from 'yaml';
+import { logger } from './logger.mjs';
+import { brainDir } from './config.mjs';
+
+// Standard pricing constants ($ per million tokens)
+export const PRICING = {
+  claude: {
+    input: 3.00 / 1000000,
+    output: 15.00 / 1000000,
+    cacheRead: 0.30 / 1000000,
+    cacheCreation: 3.75 / 1000000
+  },
+  gemini: {
+    input: 0.075 / 1000000,
+    output: 0.30 / 1000000
+  },
+  codex: {
+    input: 2.50 / 1000000,
+    output: 10.00 / 1000000
+  }
+};
+
+/**
+ * Recursively find files in a directory.
+ */
+function walkDir(dir, callback) {
+  if (!fs.existsSync(dir)) return;
+  try {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walkDir(fullPath, callback);
+      } else if (entry.isFile()) {
+        callback(fullPath);
+      }
+    }
+  } catch (err) {
+    logger.warn('usage-tracker', `Error walking directory ${dir}: ${err.message}`);
+  }
+}
+
+/**
+ * Calculates local token and dollar cost usage across Gemini, Claude, and Codex.
+ * Tracks usage in the last 24 hours (daily) and last 7 days (weekly).
+ *
+ * @returns {object} Combined daily and weekly metrics and breakdown per agent.
+ */
+export function calculateCurrentCost() {
+  const now = new Date();
+  
+  const dailyLimitMs = 24 * 60 * 60 * 1000;
+  const weeklyLimitMs = 7 * 24 * 60 * 60 * 1000;
+  
+  // Format local date string as YYYY-MM-DD
+  const year = now.getFullYear();
+  const month = String(now.getMonth() + 1).padStart(2, '0');
+  const day = String(now.getDate()).padStart(2, '0');
+  const todayStr = `${year}-${month}-${day}`;
+
+  const gemini = {
+    dailyInput: 0,
+    dailyOutput: 0,
+    weeklyInput: 0,
+    weeklyOutput: 0
+  };
+
+  const claude = {
+    dailyCost: 0,
+    weeklyCost: 0
+  };
+
+  const codex = {
+    dailyInput: 0,
+    dailyOutput: 0,
+    weeklyInput: 0,
+    weeklyOutput: 0
+  };
+
+  // 1. Gemini Ingestion
+  const geminiTmpBase = path.join(os.homedir(), '.gemini', 'tmp');
+  if (fs.existsSync(geminiTmpBase)) {
+    try {
+      const dirs = fs.readdirSync(geminiTmpBase).filter(d => {
+        const full = path.join(geminiTmpBase, d, 'chats');
+        try {
+          return fs.existsSync(full) && fs.statSync(full).isDirectory();
+        } catch {
+          return false;
+        }
+      });
+
+      for (const dir of dirs) {
+        const chatsDir = path.join(geminiTmpBase, dir, 'chats');
+        try {
+          const files = fs.readdirSync(chatsDir).filter(f => f.endsWith('.jsonl'));
+          for (const file of files) {
+            const filePath = path.join(chatsDir, file);
+            try {
+              const stat = fs.statSync(filePath);
+              const ageMs = now.getTime() - stat.mtime.getTime();
+              
+              // Skip files older than 7 days entirely to avoid reading large logs
+              if (ageMs > weeklyLimitMs) continue;
+
+              const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
+              for (const line of lines) {
+                try {
+                  const entry = JSON.parse(line);
+                  if (entry.tokens) {
+                    const timestamp = entry.timestamp ? new Date(entry.timestamp) : stat.mtime;
+                    const entryAgeMs = now.getTime() - timestamp.getTime();
+                    
+                    const input = entry.tokens.input || 0;
+                    const output = entry.tokens.output || 0;
+
+                    if (entryAgeMs <= dailyLimitMs) {
+                      gemini.dailyInput += input;
+                      gemini.dailyOutput += output;
+                    }
+                    if (entryAgeMs <= weeklyLimitMs) {
+                      gemini.weeklyInput += input;
+                      gemini.weeklyOutput += output;
+                    }
+                  }
+                } catch { /* malformed line */ }
+              }
+            } catch (err) { /* unreadable file */ }
+          }
+        } catch { /* unreadable directory */ }
+      }
+    } catch (err) {
+      logger.warn('usage-tracker', `Failed to read Gemini tmp chats: ${err.message}`);
+    }
+  }
+
+  // 2. Claude Ingestion
+  const statsFile = path.join(os.homedir(), '.claude', 'stats-cache.json');
+  if (fs.existsSync(statsFile)) {
+    try {
+      const data = JSON.parse(fs.readFileSync(statsFile, 'utf8'));
+      const modelStats = {};
+      const fallbackCostPerToken = 5.00 / 1000000; // conservative blended rate
+
+      // Calculate historical cost per token for each model
+      const usageSrc = data.modelUsage || data.models;
+      if (usageSrc) {
+        for (const [model, stats] of Object.entries(usageSrc)) {
+          const input = stats.inputTokens || 0;
+          const output = stats.outputTokens || 0;
+          const cacheRead = stats.cacheReadInputTokens || 0;
+          const cacheCreation = stats.cacheCreationInputTokens || 0;
+          
+          const totalTokens = input + output + cacheRead + cacheCreation;
+          if (totalTokens > 0) {
+            const totalCost = (input * PRICING.claude.input) +
+                              (output * PRICING.claude.output) +
+                              (cacheRead * PRICING.claude.cacheRead) +
+                              (cacheCreation * PRICING.claude.cacheCreation);
+            modelStats[model] = {
+              costPerToken: totalCost / totalTokens
+            };
+          }
+        }
+      }
+
+      // Parse daily activity
+      if (Array.isArray(data.dailyModelTokens)) {
+        for (const entry of data.dailyModelTokens) {
+          if (!entry.date) continue;
+          
+          const entryDate = new Date(`${entry.date}T12:00:00`); // parse cleanly in local/system time
+          const diffMs = now.getTime() - entryDate.getTime();
+          const isToday = entry.date === todayStr;
+          const isThisWeek = diffMs <= weeklyLimitMs;
+
+          if (isToday || isThisWeek) {
+            let entryCost = 0;
+            if (entry.tokensByModel) {
+              for (const [model, tokens] of Object.entries(entry.tokensByModel)) {
+                const costPerToken = modelStats[model]?.costPerToken || fallbackCostPerToken;
+                entryCost += tokens * costPerToken;
+              }
+            }
+            if (isToday) {
+              claude.dailyCost += entryCost;
+            }
+            if (isThisWeek) {
+              claude.weeklyCost += entryCost;
+            }
+          }
+        }
+      }
+    } catch (err) {
+      logger.warn('usage-tracker', `Failed to read Claude stats-cache.json: ${err.message}`);
+    }
+  }
+
+  // 3. Codex Ingestion
+  const codexSessionsDir = path.join(os.homedir(), '.codex', 'sessions');
+  if (fs.existsSync(codexSessionsDir)) {
+    try {
+      walkDir(codexSessionsDir, (filePath) => {
+        if (!filePath.endsWith('.jsonl')) return;
+        try {
+          const stat = fs.statSync(filePath);
+          const ageMs = now.getTime() - stat.mtime.getTime();
+          if (ageMs > weeklyLimitMs) return;
+
+          const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
+          for (const line of lines) {
+            try {
+              const entry = JSON.parse(line);
+              if (entry.usage) {
+                const timestamp = entry.timestamp ? new Date(entry.timestamp) : stat.mtime;
+                const entryAgeMs = now.getTime() - timestamp.getTime();
+                
+                const input = entry.usage.prompt_tokens || entry.usage.input_tokens || 0;
+                const output = entry.usage.completion_tokens || entry.usage.output_tokens || 0;
+
+                if (entryAgeMs <= dailyLimitMs) {
+                  codex.dailyInput += input;
+                  codex.dailyOutput += output;
+                }
+                if (entryAgeMs <= weeklyLimitMs) {
+                  codex.weeklyInput += input;
+                  codex.weeklyOutput += output;
+                }
+              }
+            } catch { /* line error */ }
+          }
+        } catch { /* file error */ }
+      });
+    } catch (err) {
+      logger.warn('usage-tracker', `Failed to scan Codex sessions: ${err.message}`);
+    }
+  }
+
+  // Cost conversion
+  const geminiDailyCost = (gemini.dailyInput * PRICING.gemini.input) + (gemini.dailyOutput * PRICING.gemini.output);
+  const geminiWeeklyCost = (gemini.weeklyInput * PRICING.gemini.input) + (gemini.weeklyOutput * PRICING.gemini.output);
+  
+  const codexDailyCost = (codex.dailyInput * PRICING.codex.input) + (codex.dailyOutput * PRICING.codex.output);
+  const codexWeeklyCost = (codex.weeklyInput * PRICING.codex.input) + (codex.weeklyOutput * PRICING.codex.output);
+
+  const dailyUsd = geminiDailyCost + claude.dailyCost + codexDailyCost;
+  const weeklyUsd = geminiWeeklyCost + claude.weeklyCost + codexWeeklyCost;
+
+  return {
+    timestamp: now.toISOString(),
+    dailyUsd,
+    weeklyUsd,
+    breakdown: {
+      gemini: {
+        dailyUsd: geminiDailyCost,
+        weeklyUsd: geminiWeeklyCost,
+        dailyTokens: gemini.dailyInput + gemini.dailyOutput,
+        weeklyTokens: gemini.weeklyInput + gemini.weeklyOutput
+      },
+      claude: {
+        dailyUsd: claude.dailyCost,
+        weeklyUsd: claude.weeklyCost
+      },
+      codex: {
+        dailyUsd: codexDailyCost,
+        weeklyUsd: codexWeeklyCost,
+        dailyTokens: codex.dailyInput + codex.dailyOutput,
+        weeklyTokens: codex.weeklyInput + codex.weeklyOutput
+      }
+    }
+  };
+}
+
+/**
+ * Safely load and parse budget.yml config.
+ *
+ * @param {string} configPath
+ * @returns {object} Budget config structure.
+ */
+export function loadBudgetConfig(configPath) {
+  if (!fs.existsSync(configPath)) return {};
+  try {
+    const raw = fs.readFileSync(configPath, 'utf8');
+    return yaml.parse(raw) || {};
+  } catch (err) {
+    logger.warn('usage-tracker', `Failed to parse budget config at ${configPath}: ${err.message}`);
+    return {};
+  }
+}
+
+/**
+ * Checks if the current token/dollar spend exceeds safety limits in budget.yml.
+ * Throws a budget exhaustion error if caps are breached.
+ *
+ * @param {string} configPath - Path to the budget.yml config file.
+ */
+export function checkBudgetSafety(configPath) {
+  try {
+    let resolvedPath = configPath;
+    if (!resolvedPath || !fs.existsSync(resolvedPath)) {
+      // Fallback 1: workspace root config
+      const workspacePath = path.join(process.cwd(), 'config', 'budget.yml');
+      if (fs.existsSync(workspacePath)) {
+        resolvedPath = workspacePath;
+      } else {
+        // Fallback 2: agentDir config
+        const agentHomePath = path.join(brainDir, 'config', 'budget.yml');
+        if (fs.existsSync(agentHomePath)) {
+          resolvedPath = agentHomePath;
+        }
+      }
+    }
+
+    if (!resolvedPath || !fs.existsSync(resolvedPath)) {
+      logger.warn('usage-tracker', `Budget config not found, bypassing safety check.`);
+      return;
+    }
+
+    const config = loadBudgetConfig(resolvedPath);
+    if (!config.budget || config.budget.enabled === false) {
+      return;
+    }
+
+    const { daily_cap_usd, weekly_cap_usd } = config.budget;
+    const usage = calculateCurrentCost();
+
+    if (daily_cap_usd !== undefined && usage.dailyUsd > daily_cap_usd) {
+      const errorMsg = `Budget safety limit reached! Daily spend of $${usage.dailyUsd.toFixed(4)} exceeds the cap of $${daily_cap_usd.toFixed(4)}. Dispatch blocked.`;
+      logger.error('usage-tracker', errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    if (weekly_cap_usd !== undefined && usage.weeklyUsd > weekly_cap_usd) {
+      const errorMsg = `Budget safety limit reached! Weekly spend of $${usage.weeklyUsd.toFixed(4)} exceeds the cap of $${weekly_cap_usd.toFixed(4)}. Dispatch blocked.`;
+      logger.error('usage-tracker', errorMsg);
+      throw new Error(errorMsg);
+    }
+
+    logger.info('usage-tracker', `Spend within limits. Daily: $${usage.dailyUsd.toFixed(4)} / $${daily_cap_usd?.toFixed(4) || 'N/A'}. Weekly: $${usage.weeklyUsd.toFixed(4)} / $${weekly_cap_usd?.toFixed(4) || 'N/A'}.`);
+  } catch (err) {
+    if (err.message.includes('Budget safety limit reached')) {
+      throw err;
+    }
+    logger.warn('usage-tracker', `Error executing safety check: ${err.message}. Safeguard bypassed.`);
+  }
+}

package/src/core/vault-cache.mjs

@@ -0,0 +1,76 @@
+import fs from 'fs';
+import { loadNodes } from './vault.mjs';
+import { logger } from './logger.mjs';
+
+let cachedNodes = null;
+let watcher = null;
+let currentVaultDir = null;
+
+/**
+ * Retrieves memory nodes from the in-memory cache if fresh, otherwise loads them from disk.
+ * Automatically wires a file watcher on first load to track external changes.
+ *
+ * @param {string} vaultDir - Path to the memory vault directory
+ * @returns {Array} List of memory node objects
+ */
+export function getNodes(vaultDir) {
+  currentVaultDir = vaultDir;
+  if (cachedNodes !== null) {
+    logger.debug('Vault cache hit');
+    return cachedNodes;
+  }
+  logger.info('Vault cache miss, loading nodes from disk', { vaultDir });
+  cachedNodes = loadNodes(vaultDir);
+  startWatcher(vaultDir);
+  return cachedNodes;
+}
+
+/**
+ * Invalidates the in-memory vault cache, forcing the next read to load from disk.
+ */
+export function invalidate() {
+  logger.info('Vault cache invalidated');
+  cachedNodes = null;
+}
+
+/**
+ * Starts a filesystem watcher on the vault directory to automatically invalidate the cache
+ * when external edits occur.
+ *
+ * @param {string} vaultDir - Path to the vault directory to watch
+ */
+function startWatcher(vaultDir) {
+  if (watcher) return;
+  if (!fs.existsSync(vaultDir)) {
+    try {
+      fs.mkdirSync(vaultDir, { recursive: true });
+    } catch (err) {
+      logger.error('Failed to create vault directory for watcher', { error: err.message });
+      return;
+    }
+  }
+
+  logger.info('Starting filesystem watcher for vault cache', { vaultDir });
+  try {
+    watcher = fs.watch(vaultDir, { recursive: true }, (eventType, filename) => {
+      if (filename && filename.endsWith('.md')) {
+        logger.info(`Vault filesystem change detected (${eventType}: ${filename}), invalidating cache`);
+        invalidate();
+      }
+    });
+    // Unref the watcher so it does not prevent Node from exiting cleanly
+    watcher.unref();
+  } catch (err) {
+    logger.warn('Failed to start vault fs watcher, cache will rely on explicit invalidations', { error: err.message });
+  }
+}
+
+/**
+ * Explicitly boots the filesystem watcher for the vault directory.
+ *
+ * @param {string} vaultDir - Path to the vault directory to watch
+ */
+export function start(vaultDir) {
+  currentVaultDir = vaultDir;
+  startWatcher(vaultDir);
+}