total-recall-brain 3.26.0 → 3.28.0

package/src/core/evolving-context.mjs

@@ -0,0 +1,171 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  getCachedSection,
+  loadSectionCache,
+  saveSectionCache,
+  computeSectionHash,
+  manageContextBudget
+} from './context-cache.mjs';
+
+/**
+ * Compile evolving context blocks across multi-domain SSSS nodes and plugins.
+ * Supports deep token windows up to 500,000 tokens with section-level caching.
+ *
+ * @param {object} options
+ * @param {string} [options.derivedDir] - Target directory for memory-derived artifacts
+ * @param {Array<object>} [options.nodes] - SSSS memory nodes
+ * @param {Array<object>} [options.plugins] - Active plugin descriptors
+ * @param {number} [options.maxChars=500000] - Token/character budget ceiling
+ * @param {number} [options.slidingWindowLimit=50] - Number of recent historical items to retain
+ * @returns {Promise<{ content: string, cacheHits: number, cacheMisses: number, totalSections: number }>}
+ */
+export async function compileEvolvingContext(options = {}) {
+  const {
+    derivedDir,
+    nodes = [],
+    plugins = [],
+    maxChars = 500_000,
+    slidingWindowLimit = 50
+  } = options;
+
+  let cacheHits = 0;
+  let cacheMisses = 0;
+
+  const activeNodes = nodes.filter((n) => n && n.status !== 'deprecated' && n.status !== 'archived');
+
+  // Partition nodes by domain
+  const userProjects = activeNodes.filter((n) => n.category === 'user-projects');
+  const benchmarks = activeNodes.filter((n) => n.category === 'benchmarks');
+  const researchNodes = activeNodes.filter((n) => n.category === 'research' || n.category === 'frontier-capabilities');
+
+  // Section 1: User Projects & Active Blockers (Priority 100 - unevictable)
+  const projectsSection = await getCachedSection({
+    derivedDir,
+    sectionKey: 'user_projects',
+    nodes: userProjects,
+    generatorFn: async () => {
+      if (userProjects.length === 0) return '';
+      const lines = ['### 1. Grounded User Science & Active Blockers\n'];
+      for (const p of userProjects) {
+        lines.push(`* **Project**: \`${p.title || p.slug}\` (Path: \`${p.source?.repo_path || 'local'}\`)`);
+        if (p.description) lines.push(`  * **Mission**: ${p.description}`);
+        if (Array.isArray(p.enables) && p.enables.length > 0) {
+          lines.push(`  * **Required Capabilities**: ${p.enables.join(', ')}`);
+        }
+      }
+      return lines.join('\n');
+    }
+  });
+  if (projectsSection.cached) cacheHits++; else cacheMisses++;
+
+  // Section 2: Benchmark Ledger (Priority 90)
+  const benchmarkSection = await getCachedSection({
+    derivedDir,
+    sectionKey: 'benchmark_ledger',
+    nodes: benchmarks,
+    generatorFn: async () => {
+      if (benchmarks.length === 0) return '';
+      const lines = [
+        '### 2. State-of-the-Art Benchmark Ledger & Pareto Frontiers\n',
+        '| Domain / Metric | Current World Record | Baseline | Verification Tier | Source / Lab |',
+        '|---|---|---|---|---|'
+      ];
+      for (const b of benchmarks) {
+        const src = b.source || {};
+        const metricCode = src.metric_name || b.slug;
+        const metricDisplay = b.title && metricCode && b.title !== metricCode
+          ? `${b.title} (\`${metricCode}\`)`
+          : `\`${metricCode || b.title}\``;
+        const current = src.current_record ? `**${src.current_record} ${src.unit || ''}**`.trim() : '—';
+        const baseline = src.previous_record ? `${src.previous_record} ${src.unit || ''}`.trim() : '—';
+        const tier = src.verification_status || 'unverified';
+        const ref = src.doi ? `[${src.doi}](https://doi.org/${src.doi})` : (src.lab || 'empirical');
+        lines.push(`| ${metricDisplay} | ${current} | ${baseline} | ${tier} | ${ref} |`);
+      }
+      return lines.join('\n');
+    }
+  });
+  if (benchmarkSection.cached) cacheHits++; else cacheMisses++;
+
+  // Section 3: Verified Capability Breakthroughs (Priority 80)
+  const breakthroughsSection = await getCachedSection({
+    derivedDir,
+    sectionKey: 'breakthroughs',
+    nodes: researchNodes,
+    generatorFn: async () => {
+      if (researchNodes.length === 0) return '';
+      const lines = ['### 3. Verified Capability Breakthroughs (What Is Now Possible)\n'];
+      // Apply sliding window to prioritize the most recent breakthroughs
+      const sorted = [...researchNodes].sort((a, b) => new Date(b.timestamp || 0).getTime() - new Date(a.timestamp || 0).getTime());
+      const windowed = sorted.slice(0, slidingWindowLimit);
+
+      for (const r of windowed) {
+        const src = r.source || {};
+        const tier = src.epistemic_tier ? `[Tier ${src.epistemic_tier}] ` : '';
+        const modality = src.empirical_modality ? `(\`${src.empirical_modality}\`)` : '';
+        lines.push(`#### ${tier}${r.title || r.slug} ${modality}`);
+        if (r.description) lines.push(`> **Capability**: ${r.description}\n`);
+        if (src.doi) lines.push(`* **Primary DOI**: https://doi.org/${src.doi}`);
+        if (Array.isArray(r.enables) && r.enables.length > 0) {
+          lines.push(`* **Unlocks Subgraph**: ${r.enables.join(', ')}`);
+        }
+        lines.push('');
+      }
+      return lines.join('\n');
+    }
+  });
+  if (breakthroughsSection.cached) cacheHits++; else cacheMisses++;
+
+  // Section 4: Dynamic Plugin Injections
+  const pluginSections = [];
+  for (const plug of plugins) {
+    const plugNodes = activeNodes.filter(n => Array.isArray(plug.categories) && plug.categories.includes(n.category));
+    const pSec = await getCachedSection({
+      derivedDir,
+      sectionKey: `plugin_${plug.id}`,
+      nodes: plugNodes,
+      extraInput: JSON.stringify(plug.manifest || {}),
+      generatorFn: async () => {
+        const lines = [`### Plugin: ${plug.name || plug.id}\n`];
+        if (plug.description) lines.push(`> ${plug.description}\n`);
+        for (const pn of plugNodes) {
+          lines.push(`- **${pn.title || pn.slug}**: ${pn.body || pn.description || ''}`);
+        }
+        return lines.join('\n');
+      }
+    });
+    if (pSec.cached) cacheHits++; else cacheMisses++;
+    pluginSections.push({ priority: 60, content: pSec.content });
+  }
+
+  // Manage total context budget
+  const allSections = [
+    { priority: 100, content: projectsSection.content, allowTruncation: false },
+    { priority: 90, content: benchmarkSection.content },
+    { priority: 80, content: breakthroughsSection.content },
+    ...pluginSections
+  ].filter(s => s.content && s.content.trim().length > 0);
+
+  const assembledContent = manageContextBudget(allSections, { maxChars });
+
+  // Persist to memory-derived/evolving-context.md if derivedDir is provided
+  if (derivedDir) {
+    try {
+      if (!fs.existsSync(derivedDir)) {
+        fs.mkdirSync(derivedDir, { recursive: true });
+      }
+      const targetPath = path.join(derivedDir, 'evolving-context.md');
+      fs.writeFileSync(targetPath, assembledContent, 'utf8');
+    } catch {
+      // Best-effort write
+    }
+  }
+
+  return {
+    content: assembledContent,
+    cacheHits,
+    cacheMisses,
+    totalSections: allSections.length
+  };
+}

package/src/core/mesh-activity.mjs

@@ -0,0 +1,140 @@
+import os from 'node:os';
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { getMeshIp, getMeshHostname } from './mesh.mjs';
+
+/**
+ * Measure user idle time in seconds on the local operating system.
+ * macOS uses IOHIDSystem (HIDIdleTime nanoseconds).
+ * Linux falls back to w/proc.
+ * @returns {number} Idle duration in seconds
+ */
+export function getLocalIdleSeconds() {
+  const platform = process.platform;
+
+  if (platform === 'darwin') {
+    try {
+      const res = spawnSync('ioreg', ['-c', 'IOHIDSystem'], {
+        encoding: 'utf8',
+        timeout: 1000,
+        stdio: ['ignore', 'pipe', 'ignore']
+      });
+      if (res.status === 0 && res.stdout) {
+        const match = res.stdout.match(/"HIDIdleTime"\s*=\s*(\d+)/);
+        if (match && match[1]) {
+          const nanos = BigInt(match[1]);
+          const secs = Number(nanos / 1_000_000_000n);
+          return Math.max(0, secs);
+        }
+      }
+    } catch {
+      // Fallback
+    }
+  }
+
+  // Linux or fallback: check w / who idle time
+  try {
+    const res = spawnSync('w', ['-u'], {
+      encoding: 'utf8',
+      timeout: 1000,
+      stdio: ['ignore', 'pipe', 'ignore']
+    });
+    if (res.status === 0 && res.stdout) {
+      const lines = res.stdout.trim().split('\n').slice(2);
+      if (lines.length > 0) {
+        // Just report 0 if user is logged into an active TTY session
+        return 0;
+      }
+    }
+  } catch {}
+
+  return 0;
+}
+
+/**
+ * Identify the active editor or developer surface on the local machine.
+ * @returns {string} E.g. 'antigravity', 'cursor', 'claude', 'vscode', 'terminal'
+ */
+export function detectActiveSurface() {
+  try {
+    const res = spawnSync('ps', ['-ax', '-o', 'comm'], {
+      encoding: 'utf8',
+      timeout: 1500,
+      stdio: ['ignore', 'pipe', 'ignore']
+    });
+
+    if (res.status === 0 && res.stdout) {
+      const procs = res.stdout.toLowerCase();
+      if (procs.includes('antigravity')) return 'antigravity';
+      if (procs.includes('cursor')) return 'cursor';
+      if (procs.includes('code') || procs.includes('electron')) return 'vscode';
+      if (procs.includes('claude')) return 'claude';
+    }
+  } catch {}
+
+  return 'terminal';
+}
+
+/**
+ * Generate a complete presence heartbeat for the local node.
+ * @returns {object} Presence record
+ */
+export function getLocalPresence() {
+  const idleSecs = getLocalIdleSeconds();
+  const isActive = idleSecs < 300; // Active if interacted within last 5 minutes
+  const now = Date.now();
+  const lastInteraction = now - (idleSecs * 1000);
+
+  let hostname = 'unknown-node';
+  try {
+    hostname = getMeshHostname() || os.hostname();
+  } catch {
+    hostname = os.hostname();
+  }
+
+  let meshIp = null;
+  try {
+    meshIp = getMeshIp();
+  } catch {}
+
+  return {
+    node_id: hostname,
+    mesh_ip: meshIp,
+    user_active: isActive,
+    idle_seconds: idleSecs,
+    last_interaction: lastInteraction,
+    active_surface: detectActiveSurface(),
+    timestamp: new Date(now).toISOString()
+  };
+}
+
+/**
+ * "Follow the User" dynamic dispatch resolver.
+ * Evaluates presence records across mesh nodes to determine the user's currently active device.
+ * @param {Array<object>} presenceList - Array of presence records from cluster nodes
+ * @returns {object} The most actively interacted node
+ */
+export function resolveActiveDevice(presenceList = []) {
+  if (!Array.isArray(presenceList) || presenceList.length === 0) {
+    return {
+      node_id: 'local',
+      mesh_ip: null,
+      user_active: true,
+      active_surface: 'local'
+    };
+  }
+
+  // Filter for active nodes
+  const activeNodes = presenceList.filter(p => p && p.user_active);
+
+  if (activeNodes.length === 0) {
+    // If no nodes are actively interacted with within threshold, select the most recently interacted
+    const sorted = [...presenceList].sort((a, b) => (b.last_interaction || 0) - (a.last_interaction || 0));
+    return sorted[0] || { node_id: 'local', mesh_ip: null, user_active: false };
+  }
+
+  // Return the active node with the most recent interaction timestamp
+  activeNodes.sort((a, b) => (b.last_interaction || 0) - (a.last_interaction || 0));
+  return activeNodes[0];
+}

package/src/core/mesh.mjs

@@ -1,4 +1,4 @@
-import { spawnSync } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
 import path from 'node:path';
 import crypto from 'node:crypto';
 import {
@@ -8,6 +8,7 @@ import {
 } from './vfs-documents.mjs';
 import { logger } from './logger.mjs';
 import { STATUS_TIMEOUT_MS, resolveTailscaleBinary } from './tailscale-cli.mjs';
+import { buildSshArgs, resolveNodeAccess } from './mesh-access.mjs';
 
 const CACHE_MS = 2_000;
 
@@ -355,6 +356,67 @@ export async function setMeshNodeAccess(nameOrAddress, accessPatch, options = {}
   return { written: !!result?.success, path: vfsPath, access: merged, result };
 }
 
+/**
+ * Ensure every live mesh peer has an entity document.
+ *
+ * `patchOwnMeshNode()` describes only the machine it runs on, so a peer that
+ * will never run a brain can never be described: a phone cannot run one, a
+ * control-plane droplet may not, and a machine that is simply switched off
+ * never gets the chance. On a five-node tailnet that left three nodes with no
+ * entity at all — including the always-on host, which is the one an agent is
+ * most likely to need access details for.
+ *
+ * Offline peers are deliberately skipped rather than created. A node that has
+ * not been seen in weeks is usually a dead registration, and minting an entity
+ * for it re-legitimises something the operator may be trying to retire. They
+ * are reported instead, so the choice stays with the person.
+ *
+ * Creation goes through `setMeshNodeAccess`, so it is the same Operation
+ * Contract path as every other entity write — never a raw file write.
+ *
+ * Discovery and the writer are injectable so this is testable without a live
+ * tailnet or a real vault; production callers pass neither.
+ *
+ * @param {{vaultRoot?: string, includeOffline?: boolean, peers?: object[],
+ *          entities?: object[], write?: Function}} [options]
+ * @returns {Promise<{created: string[], skipped_offline: string[], existing: number}>}
+ */
+export async function ensureMeshNodeEntities(options = {}) {
+  const vaultRoot = options.vaultRoot || defaultVaultRoot();
+  const peers = options.peers || getMeshPeers({ includeSelf: true });
+  const entities = options.entities || listMeshNodeEntities(vaultRoot);
+  const write = options.write || ((host) => setMeshNodeAccess(host, {}, { vaultRoot }));
+
+  const knownKeys = new Set(entities.map((e) => meshNodeKey(e.hostname)).filter(Boolean));
+  const knownIps = new Set(entities.map((e) => e.ip).filter(Boolean).map(String));
+
+  const created = [];
+  const skippedOffline = [];
+
+  for (const peer of peers) {
+    if (!peer.hostname) continue;
+    // Match on either identity: the same host can be recorded under a bare or
+    // suffixed name, and creating a second document for it is the duplicate
+    // problem meshNodeKey exists to prevent.
+    if (knownKeys.has(meshNodeKey(peer.hostname))) continue;
+    if (peer.ip && knownIps.has(String(peer.ip))) continue;
+
+    if (!peer.online && !options.includeOffline) {
+      skippedOffline.push(peer.hostname);
+      continue;
+    }
+
+    const res = await write(peer.hostname);
+    if (res.written) {
+      created.push(peer.hostname);
+      knownKeys.add(meshNodeKey(peer.hostname));
+      if (peer.ip) knownIps.add(String(peer.ip));
+    }
+  }
+
+  return { created, skipped_offline: skippedOffline, existing: entities.length };
+}
+
 /**
  * Upsert **this** node's mesh_node entity from live discovery.
  * Does not hardcode device names — slug and fields come from Tailscale self.
@@ -507,3 +569,114 @@ export async function patchOwnMeshNode(options = {}) {
     result,
   };
 }
+
+/**
+ * Execute a command on a remote mesh node over SSH.
+ *
+ * @param {string} target - Node hostname or IP
+ * @param {string} command - Shell command to execute
+ * @param {object} options
+ * @param {string} [options.vaultRoot]
+ * @param {number} [options.timeoutMs=60000]
+ * @param {boolean} [options.loginShell=true]
+ * @returns {Promise<{ node: string, hostname: string, ip: string, exitCode: number, success: boolean, stdout: string, stderr: string }>}
+ */
+export async function execMeshCommand(target, command, options = {}) {
+  const { vaultRoot = defaultVaultRoot(), timeoutMs = 60_000, loginShell = true } = options;
+  const node = findMeshNode(target, vaultRoot);
+  if (!node) {
+    throw new Error(`No mesh node matches "${target}".`);
+  }
+  const resolved = resolveNodeAccess(node);
+  if (!resolved.complete) {
+    throw new Error(
+      `No login account recorded for ${node.hostname}. Run 'total-recall mesh access ${target} --user <login>'`,
+    );
+  }
+
+  const standardPath = 'export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:$HOME/.local/bin:$PATH";';
+  const remoteCommand = `${standardPath} ${command}`;
+  const sshArgs = buildSshArgs(resolved, {
+    command: remoteCommand,
+    extraOptions: ['BatchMode=yes', 'ConnectTimeout=10'],
+  });
+
+  return new Promise((resolve, reject) => {
+    const proc = spawn('ssh', sshArgs);
+    let stdout = '';
+    let stderr = '';
+    let timedOut = false;
+
+    const timer = setTimeout(() => {
+      timedOut = true;
+      proc.kill('SIGTERM');
+      reject(new Error(`Command on "${target}" timed out after ${timeoutMs}ms.`));
+    }, timeoutMs);
+
+    proc.stdout.on('data', (d) => { stdout += d.toString(); });
+    proc.stderr.on('data', (d) => { stderr += d.toString(); });
+
+    proc.on('close', (code) => {
+      clearTimeout(timer);
+      if (timedOut) return;
+      resolve({
+        node: target,
+        hostname: node.hostname,
+        ip: node.ip,
+        exitCode: code ?? 1,
+        success: code === 0,
+        stdout: stdout.trim(),
+        stderr: stderr.trim(),
+      });
+    });
+
+    proc.on('error', (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+  });
+}
+
+/**
+ * Resolves the primary or Cloudflare Tunnel fallback URL for mesh control plane access.
+ * Returns { url, source, is_fallback }
+ *
+ * @param {object} [options]
+ * @param {string} [options.url]
+ * @param {string} [options.loginServer]
+ * @returns {{ url: string|null, source: string, is_fallback: boolean }}
+ */
+export function resolveMeshControlUrl(options = {}) {
+  const explicit = options.url || options.loginServer;
+  if (explicit && String(explicit).trim()) {
+    const clean = String(explicit).trim();
+    return {
+      url: clean.endsWith('/') ? clean.slice(0, -1) : clean,
+      source: 'explicit',
+      is_fallback: false
+    };
+  }
+
+  const envPrimary = process.env.TR_LOGIN_SERVER || process.env.TR_HEADSCALE_URL;
+  if (envPrimary && envPrimary.trim()) {
+    const clean = envPrimary.trim();
+    return {
+      url: clean.endsWith('/') ? clean.slice(0, -1) : clean,
+      source: 'env_primary',
+      is_fallback: false
+    };
+  }
+
+  const envTunnel = process.env.TR_CLOUDFLARE_TUNNEL_URL || process.env.CF_TUNNEL_URL;
+  if (envTunnel && envTunnel.trim()) {
+    const clean = envTunnel.trim();
+    return {
+      url: clean.endsWith('/') ? clean.slice(0, -1) : clean,
+      source: 'cloudflare_tunnel_fallback',
+      is_fallback: true
+    };
+  }
+
+  return { url: null, source: 'none', is_fallback: false };
+}
+