total-recall-brain 3.13.0 → 3.13.2

package/src/core/remote-vault-sync.mjs

@@ -0,0 +1,194 @@
+/**
+ * Optional remote vault content sync (generic).
+ * All endpoints/paths come from config / env — no product or host-app assumptions.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { spawnSync } from 'child_process';
+import http from 'http';
+import https from 'https';
+import { createHash } from 'crypto';
+import { createEngine } from '@ssss/cli/engine';
+import { remoteVaultSync } from './config.mjs';
+import { logger } from './logger.mjs';
+
+function fetchExport(url, token, outStream) {
+  return new Promise((resolve, reject) => {
+    const client = url.startsWith('https') ? https : http;
+    const req = client.get(url, { headers: { Authorization: `Bearer ${token}` } }, (res) => {
+      if (res.statusCode !== 200) {
+        let errStr = '';
+        res.on('data', (c) => (errStr += String(c)));
+        res.on('end', () => reject(new Error(`HTTP ${res.statusCode}: ${errStr}`)));
+        return;
+      }
+      res.pipe(outStream);
+      outStream.on('finish', () => resolve(true));
+      outStream.on('error', reject);
+    });
+    req.on('error', reject);
+  });
+}
+
+function pruneAssets(assetsDir, keepCount) {
+  if (!fs.existsSync(assetsDir)) return;
+  const files = fs
+    .readdirSync(assetsDir)
+    .filter((f) => f.endsWith('.tar.gz'))
+    .map((f) => ({ name: f, time: fs.statSync(path.join(assetsDir, f)).mtimeMs }))
+    .sort((a, b) => b.time - a.time);
+
+  for (let i = keepCount; i < files.length; i++) {
+    try {
+      fs.unlinkSync(path.join(assetsDir, files[i].name));
+    } catch {
+      // ignore
+    }
+  }
+}
+
+function remoteRegistryDir() {
+  if (process.env.TR_REMOTE_REGISTRY_DIR) return process.env.TR_REMOTE_REGISTRY_DIR;
+  if (remoteVaultSync.registryDir) return remoteVaultSync.registryDir;
+  throw new Error(
+    'TR_REMOTE_REGISTRY_DIR is required for remote vault content sync (SSSS registry path for imports).',
+  );
+}
+
+/**
+ * Replay bundle documents through the SSSS engine with an explicit registry.
+ */
+export function importRemoteBundle(bundle, vaultDir, registryDir) {
+  const files = Array.isArray(bundle?.files) ? bundle.files : [];
+  if (files.length === 0) {
+    return { imported: 0 };
+  }
+  if (!registryDir) {
+    throw new Error('TR_REMOTE_REGISTRY_DIR is required when the bundle contains files to import.');
+  }
+  const engine = createEngine({ registryDir });
+  const failures = [];
+  for (const file of files) {
+    const result = engine.processOperation(
+      {
+        type: 'operation',
+        idempotency_key: createHash('sha256')
+          .update(file.path)
+          .update('\n')
+          .update(file.content)
+          .digest('hex'),
+        workspace_id: process.env.TR_REMOTE_WORKSPACE_ID || 'remote',
+        path: file.path,
+        content: file.content,
+        actor: { role: 'system' },
+      },
+      vaultDir,
+    );
+    if (!result.success) {
+      failures.push(
+        `${file.path}: ${(result.validation?.errors || ['unknown operation failure']).join('; ')}`,
+      );
+    }
+  }
+  if (failures.length) throw new Error(`Bundle import failed: ${failures.join(' | ')}`);
+  return { imported: files.length };
+}
+
+/** @deprecated alias */
+export const importPortfolioBundle = importRemoteBundle;
+
+export async function runSync() {
+  if (!remoteVaultSync.enabled && process.env.TR_REMOTE_VAULT_SYNC !== '1') return;
+
+  if (!remoteVaultSync.baseUrl) {
+    logger.error({
+      subsystem: 'remote-vault-sync',
+      message: 'TR_REMOTE_VAULT_URL / remoteVaultSync.baseUrl is not configured — skipping',
+    });
+    return;
+  }
+
+  const tenantDir = path.dirname(remoteVaultSync.vaultDir);
+  const statusFile = path.join(tenantDir, 'sync-status.json');
+
+  if (!fs.existsSync(tenantDir)) {
+    fs.mkdirSync(tenantDir, { recursive: true });
+  }
+
+  const token = process.env[remoteVaultSync.tokenRef];
+  if (!token) {
+    const errStr = `Missing ${remoteVaultSync.tokenRef} environment variable`;
+    logger.error({ subsystem: 'remote-vault-sync', message: errStr });
+    fs.writeFileSync(
+      statusFile,
+      JSON.stringify({ lastRunAt: new Date().toISOString(), ok: false, error: errStr }, null, 2),
+    );
+    return;
+  }
+
+  try {
+    const bundleUrl = `${remoteVaultSync.baseUrl.replace(/\/+$/, '')}/api/admin/export-bundle`;
+    const tmpBundle = path.join(tenantDir, `tmp-bundle-${process.pid}-${Date.now()}.json`);
+    const bundleStream = fs.createWriteStream(tmpBundle);
+    await fetchExport(bundleUrl, token, bundleStream);
+
+    const registryDir = remoteRegistryDir();
+    const ssssCmd = path.join(process.cwd(), 'node_modules', '.bin', 'ssss');
+    let validateCmd = 'ssss';
+    if (fs.existsSync(ssssCmd)) {
+      validateCmd = ssssCmd;
+    }
+    const valRes = spawnSync(validateCmd, ['validate', tmpBundle, '--registry', registryDir], {
+      encoding: 'utf8',
+    });
+    if (valRes.status !== 0) {
+      throw new Error(`Bundle validation failed: ${valRes.stderr}`);
+    }
+
+    const bundleContent = JSON.parse(fs.readFileSync(tmpBundle, 'utf8'));
+    const nodeCounts =
+      bundleContent.manifest?.primitive_inventory || bundleContent.primitive_inventory || {};
+
+    if (!fs.existsSync(remoteVaultSync.vaultDir)) {
+      fs.mkdirSync(remoteVaultSync.vaultDir, { recursive: true });
+    }
+    importRemoteBundle(bundleContent, remoteVaultSync.vaultDir, registryDir);
+
+    const assetsUrl = `${remoteVaultSync.baseUrl.replace(/\/+$/, '')}/api/admin/export-assets`;
+    if (!fs.existsSync(remoteVaultSync.assetsDir)) {
+      fs.mkdirSync(remoteVaultSync.assetsDir, { recursive: true });
+    }
+    const assetFile = path.join(
+      remoteVaultSync.assetsDir,
+      `assets-${new Date().toISOString().replace(/[:.]/g, '-')}.tar.gz`,
+    );
+    const assetStream = fs.createWriteStream(assetFile);
+    await fetchExport(assetsUrl, token, assetStream);
+
+    pruneAssets(remoteVaultSync.assetsDir, remoteVaultSync.keepAssets);
+
+    fs.unlinkSync(tmpBundle);
+
+    fs.writeFileSync(
+      statusFile,
+      JSON.stringify({
+        lastRunAt: new Date().toISOString(),
+        ok: true,
+        nodeCounts,
+      }, null, 2),
+    );
+
+    logger.info({ subsystem: 'remote-vault-sync', message: 'Remote vault sync completed successfully' });
+  } catch (err) {
+    logger.error({ subsystem: 'remote-vault-sync', message: `Sync failed: ${err.message}` });
+    try {
+      fs.writeFileSync(
+        statusFile,
+        JSON.stringify({ lastRunAt: new Date().toISOString(), ok: false, error: err.message }, null, 2),
+      );
+    } catch {
+      // ignore
+    }
+  }
+}

package/src/core/research-queue.mjs

@@ -112,7 +112,7 @@ export function compileResearchProjectSummary(item, node) {
  * Sync a research project summary node to the SSSS memory vault.
  * Creates/updates a .md node in memory-vault/facts/ named research-project-<id>.md.
  */
-export function syncResearchProjectNode(item) {
+export async function syncResearchProjectNode(item) {
   if (!item) return;
   const vaultDir = path.join(getBrainDir(), 'memory-vault');
 
@@ -210,7 +210,7 @@ export function syncResearchProjectNode(item) {
   };
 
   try {
-    writeNode(node, vaultDir);
+    await writeNode(node, vaultDir);
   } catch (e) {
     // Log or handle error gracefully
   }
@@ -257,7 +257,7 @@ export function loadQueue(overrideBrainDir) {
       const summaryPath = path.join(vaultDir, 'facts', `research-project-${item.id}.md`);
       if (!fs.existsSync(summaryPath)) {
         try {
-          syncResearchProjectNode(item);
+          void syncResearchProjectNode(item);
         } catch {}
       }
     } else {
@@ -349,10 +349,8 @@ export function addToQueue({ topic, priority = 'medium', notes, brainDir: overri
   items.unshift(item);
   saveQueue(items, overrideBrainDir);
 
-  // Sync initial pending project summary node to the vault
-  try {
-    syncResearchProjectNode(item);
-  } catch {}
+  // Sync initial pending project summary node to the vault (async, non-blocking)
+  void syncResearchProjectNode(item).catch(() => {});
 
   return item;
 }
@@ -412,10 +410,8 @@ export function updateQueueItem(id, patch = {}, overrideBrainDir) {
   items[idx] = item;
   saveQueue(items, overrideBrainDir);
 
-  // Sync updated research project summary node to the vault
-  try {
-    syncResearchProjectNode(item);
-  } catch {}
+  // Sync updated research project summary node to the vault (async, non-blocking)
+  void syncResearchProjectNode(item).catch(() => {});
 
   return item;
 }

package/src/core/runtime.mjs

@@ -14,7 +14,13 @@ import { validateCommand } from './sandbox.mjs';
  */
 export function findBinaryInPath(binaryName) {
   const pathEnv = process.env.PATH || '';
-  const dirs = pathEnv.split(path.delimiter);
+  const extra = [
+    path.join(os.homedir(), '.local', 'bin'),
+    path.join(os.homedir(), '.grok', 'bin'),
+    '/opt/homebrew/bin',
+    '/usr/local/bin',
+  ];
+  const dirs = [...new Set([...pathEnv.split(path.delimiter), ...extra].filter(Boolean))];
   for (const dir of dirs) {
     const fullPath = path.join(dir, binaryName);
     try {
@@ -35,14 +41,15 @@ export function findBinaryInPath(binaryName) {
  * Total Recall Runtime — CLI Agent Dispatch
  *
  * All reasoning tasks are dispatched to headless CLI agents.
- * Agent registry lives in the tr-cli-agents skill:
- *   .agent/skills/total-recall/skills/tr-cli-agents/agents.yml
+ * Agent registry lives in the agents module (not an IDE skill):
+ *   .agent/skills/total-recall/modules/agents/agents.yml
  */
 
 // ─── Default agent registry (used when agents.yml doesn't exist) ─────────────
 
 const DEFAULT_AGENTS = [
   { name: 'antigravity', binary: 'antigravity', flags: '--sandbox=false --yolo -o json', priority: 1, enabled: true, exec: 'flag' },
+  { name: 'grok',        binary: 'grok',        flags: '--output-format plain --always-approve --permission-mode bypassPermissions', priority: 2, enabled: true, exec: 'flag' },
   { name: 'claude',      binary: 'claude',      flags: '--output-format json --permission-mode bypassPermissions --setting-sources local --tools ""', priority: 3, enabled: true, exec: 'flag' },
   { name: 'codex',       binary: 'codex',       flags: '--full-auto --json --skip-git-repo-check', priority: 4, enabled: true, exec: 'subcommand' },
 ];
@@ -51,14 +58,13 @@ const DEFAULT_AGENTS = [
  * Resolve the path to the cli-agents config file.
  */
 function getAgentsConfigPath() {
-  // The cli-agents skill was namespaced to tr-cli-agents; check the new name
-  // first, then fall back to the legacy name for repos initialized before the
-  // rename, in both the nested (meta-skill) and standalone locations.
+  // Prefer modules/agents (TR_CORE_FOCUS). Fall back to legacy nested-skill paths.
   const candidates = [
+    path.join(agentDir, 'skills', 'total-recall', 'modules', 'agents', 'agents.yml'),
     path.join(agentDir, 'skills', 'total-recall', 'skills', 'tr-cli-agents', 'agents.yml'),
     path.join(agentDir, 'skills', 'total-recall', 'skills', 'cli-agents', 'agents.yml'),
     path.join(agentDir, 'skills', 'tr-cli-agents', 'agents.yml'),
-    path.join(agentDir, 'skills', 'cli-agents', 'agents.yml')
+    path.join(agentDir, 'skills', 'cli-agents', 'agents.yml'),
   ];
   for (const p of candidates) {
     if (fs.existsSync(p)) return p;
@@ -200,7 +206,7 @@ export async function checkRuntimeHealth(config) {
   if (available.length === 0) {
     return {
       status: 'degraded',
-      reason: 'No CLI agents found. Install antigravity, claude, or codex.',
+      reason: 'No CLI agents found. Install antigravity, grok, claude, or codex.',
       available: [],
     };
   }
@@ -290,7 +296,7 @@ export async function callLocalRuntime(prompt, system, config) {
     });
 
     if (!agent) {
-      throw new Error('No CLI agents available. Install antigravity, claude, or codex.');
+      throw new Error('No CLI agents available. Install antigravity, grok, claude, or codex.');
     }
 
     const modelFlag = agent.model ? `-m ${agent.model}` : '';
@@ -299,6 +305,7 @@ export async function callLocalRuntime(prompt, system, config) {
     if (agent.exec === 'subcommand') {
       cmd = `${agent.binaryPath} exec "${escapeShell(fullPrompt)}" ${agent.flags} ${modelFlag}`.trim();
     } else {
+      // flag-style CLIs (antigravity, grok, claude, gemini): -p / --single prompt
       cmd = `${agent.binaryPath} -p "${escapeShell(fullPrompt)}" ${agent.flags} ${modelFlag}`.trim();
     }
 
@@ -311,12 +318,29 @@ export async function callLocalRuntime(prompt, system, config) {
     // Pre-flight command execution validation
     validateCommand(cmd);
 
+    const xaiKey =
+      process.env.XAI_API_KEY ||
+      process.env.GROK_API_KEY ||
+      dynamicSecrets.XAI_API_KEY ||
+      dynamicSecrets.xai_api_key ||
+      dynamicSecrets.GROK_API_KEY ||
+      dynamicSecrets.grok_api_key ||
+      '';
+
     const spawnEnv = {
       ...process.env,
+      // Ensure common CLI install dirs are visible (e.g. ~/.local/bin/grok)
+      PATH: [
+        path.join(os.homedir(), '.local', 'bin'),
+        path.join(os.homedir(), '.grok', 'bin'),
+        process.env.PATH || '',
+      ].filter(Boolean).join(path.delimiter),
       GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY || dynamicSecrets.google_api_key || googleApiKey,
       GEMINI_API_KEY: process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || dynamicSecrets.google_api_key || googleApiKey,
       ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || dynamicSecrets.anthropic_api_key,
       OPENAI_API_KEY: process.env.OPENAI_API_KEY || dynamicSecrets.openai_api_key,
+      XAI_API_KEY: xaiKey || process.env.XAI_API_KEY,
+      GROK_API_KEY: xaiKey || process.env.GROK_API_KEY,
       TAVILY_API_KEY: process.env.TAVILY_API_KEY || dynamicSecrets.tavily_api_key || tavilyApiKey,
       BRAVE_API_KEY: process.env.BRAVE_API_KEY || dynamicSecrets.brave_api_key || braveApiKey,
       EXA_API_KEY: process.env.EXA_API_KEY || dynamicSecrets.exa_api_key || exaApiKey,

package/src/core/scheduler.mjs

@@ -3,6 +3,7 @@ import path from 'path';
 import matter from 'gray-matter';
 import crypto from 'crypto';
 import { atomicWrite, safeStringify } from './vault.mjs';
+import config from './config.mjs';
 import { getNodes } from './vault-cache.mjs';
 import { logger } from './logger.mjs';
 import { loadQueue, updateQueueItem } from './research-queue.mjs';
@@ -124,7 +125,7 @@ export function loadPendingTasks(queueDir) {
 /**
  * Update a task's status on disk.
  */
-export function updateTaskStatus(task, newStatus, queueDir) {
+export function updateTaskStatus(task, newStatus, queueDir, lastError = null) {
   const filepath = task._filepath || path.join(queueDir, `${task.slug}.md`);
   if (!fs.existsSync(filepath)) return;
 
@@ -134,6 +135,9 @@ export function updateTaskStatus(task, newStatus, queueDir) {
   if (newStatus === 'completed') {
     data.completed_at = new Date().toISOString();
   }
+  if (lastError) {
+    data.last_error = lastError;
+  }
 
   // Also update in-memory object properties so they stay in sync
   task.status = newStatus;
@@ -252,7 +256,7 @@ function generateClarityReviewTask(vaultDir) {
   const target = nodes[Math.floor(Math.random() * nodes.length)];
   return {
     type: 'task',
-    slug: `clarity-review-${crypto.randomBytes(4).toString('hex')}`,
+    slug: `clarity-review-${crypto.createHash('md5').update(target.slug).digest('hex').slice(0, 8)}`,
     priority: 30,
     category: 'memory-maintenance',
     target: target.slug,
@@ -269,7 +273,7 @@ function generateClarityReviewTask(vaultDir) {
 function generateMemoryCompactionTask(vaultDir) {
   return {
     type: 'task',
-    slug: `memory-compaction-${crypto.randomBytes(4).toString('hex')}`,
+    slug: `memory-compaction-${new Date().toISOString().slice(0, 10)}`,
     priority: 35,
     category: 'memory-maintenance',
     target: 'global',
@@ -459,6 +463,41 @@ export function createScheduler({ queueDir, vaultDir, sessionsDir }) {
     queue.enqueue(task);
   }
 
+  // Optional remote vault sync job (env-configured only)
+  try {
+    const { remoteVaultSync } = config;
+    if (remoteVaultSync.enabled) {
+      const statusFile = path.join(path.dirname(remoteVaultSync.vaultDir), 'sync-status.json');
+      let lastRunMs = 0;
+      if (fs.existsSync(statusFile)) {
+        try {
+          const status = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
+          lastRunMs = new Date(status.lastRunAt || 0).getTime();
+        } catch {}
+      }
+
+      const SYNC_COOLDOWN_MS = remoteVaultSync.intervalMinutes * 60000;
+      if (Date.now() - lastRunMs >= SYNC_COOLDOWN_MS) {
+        queue.enqueue({
+          type: 'task',
+          slug: `remote-vault-sync-${Date.now().toString(36)}`,
+          priority: 90,
+          category: 'memory-maintenance',
+          status: 'pending',
+          created_by: 'scheduler',
+          reason: 'Scheduled remote vault sync.',
+          body: 'Run remote vault content synchronization.',
+          _is_remote_vault_sync: true,
+        });
+      }
+    }
+  } catch (err) {
+    logger.error({
+      subsystem: 'scheduler',
+      message: `Failed to load remote vault sync: ${err.message}`,
+    });
+  }
+
   // Load pending research queue tasks
   try {
     const COOLDOWN_MS = process.env.RESEARCH_COOLDOWN_MS
@@ -574,12 +613,27 @@ export function createScheduler({ queueDir, vaultDir, sessionsDir }) {
     },
 
     /**
-     * Get the next task. If the queue is empty, generate an idle task.
+     * Get the next task.
+     *
+     * Default: only explicit (and research-queue) tasks — never invent work.
+     * Idle generation requires TR_IDLE_TASKS=1 or opts.allowIdle === true.
+     *
+     * @param {{ allowIdle?: boolean }} [opts]
+     * @returns {{ task: object|null, source: 'explicit'|'idle'|'empty' }}
      */
-    next() {
+    next(opts = {}) {
       const explicit = queue.dequeue();
       if (explicit) return { task: explicit, source: 'explicit' };
 
+      const allowIdle =
+        opts.allowIdle === true ||
+        process.env.TR_IDLE_TASKS === '1' ||
+        process.env.TR_IDLE_TASKS === 'true';
+
+      if (!allowIdle) {
+        return { task: null, source: 'empty' };
+      }
+
       const idle = generateIdleTask({ vaultDir, sessionsDir });
       persistTaskToDisk(idle, queueDir);
       return { task: idle, source: 'idle' };

package/src/core/schema.mjs

@@ -272,6 +272,19 @@ export const SchemaProposalSchema = z.object({
   reviewed_by: z.string().nullable().optional(),
 });
 
+/** SSSS 0.9 meta-primitive for runtime-authored definitions (package core). */
+export const PrimitiveSchema = z.object({
+  type: z.literal('primitive'),
+  title: z.string(),
+  description: z.string(),
+  timestamp: ssssDatetime(),
+  primitive_id: z.string().optional(),
+  namespace: z.string().optional(),
+  name: z.string().optional(),
+  version: z.union([z.number(), z.string()]).optional(),
+  portability: z.enum(['structural', 'tenant_private', 'resource_bound']).optional(),
+}).passthrough();
+
 export const MigrationSchema = z.object({
   type: z.literal('migration'),
   migration_id: z.string(),
@@ -875,6 +888,7 @@ export const SSSS_SCHEMAS = {
   model: ModelSchema,
   conversation: ConversationSchema,
   run: RunSchema,
+  primitive: PrimitiveSchema,
   proposal: ProposalSchema,
   'schema-proposal': SchemaProposalSchema,
   migration: MigrationSchema,

package/src/core/search.mjs

@@ -2,7 +2,7 @@
  * src/core/search.mjs
  *
  * Canonical semantic search logic — searches vault nodes and session history
- * by meaning via Ollama embeddings. Called by both the MCP layer and REST API.
+ * by meaning via embedding vector similarity. Called by both the MCP layer and REST API.
  *
  * MCP and REST are thin wrappers; all logic lives here.
  */