total-recall-brain 3.24.1 → 3.24.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.24.1",
+  "version": "3.24.2",
   "description": "Personal AI system — database-free, SSSS-powered intelligence engine deployable to any POSIX host.",
   "type": "module",
   "workspaces": [

package/scaffold/.agent/skills/total-recall/SKILL.md

@@ -597,6 +597,19 @@ Current baseline: **288 spec files / 1493 tests, ~160 s** on the Mac Mini.
 >
 > A non-login `ssh` shell has no Homebrew on PATH — export it or use `bash -lc`,
 > otherwise the run dies with `command not found: npx` **and still exits 0**.
+>
+> **Never read a test result through a pipe.** `vitest run` exits 1 correctly on
+> failure, but `npx vitest run | tail -20` reports *tail's* status — always 0 —
+> so a red suite looks green. On 2026-08-19 this cost a full re-run and a wrong
+> "the suite exits 0 on failure" claim about the repo; the defect was the
+> pipeline, not vitest. Capture to a file and check the code separately:
+>
+> ```bash
+> npx vitest run > /tmp/t.log 2>&1; echo "exit=$?"; tail -20 /tmp/t.log
+> ```
+>
+> `check.mjs` is unaffected — it spawns without a shell and reads the real exit
+> code — so prefer the gate over hand-rolled invocations.
 
 **Never skip tests** to make a release look clean, and never use `--force` on a
 checker. Audit and clean up any rows, files, or mock entries a suite leaves

package/scaffold/.agent/skills/total-recall/memory-vault/preferences/always-websearch-gap.md

@@ -3,6 +3,7 @@ type: memory
 slug: always-websearch-gap
 category: preferences
 title: "Always Perform Web Search for Cutoff Gap"
+description: "Web-search to confirm any fact that falls in or past the training-cutoff gap, rather than answering from weights."
 status: active
 confidence: 1
 importance: 5
@@ -32,6 +33,7 @@ schema_version: 2
 x_temporal_context: "2026-05-25T22:36:21.376Z"
 priority: absolute
 immutable: true
+timestamp: "2026-05-25T22:36:21.376Z"
 ---
 
 # Always Perform Web Search for Cutoff Gap

package/scaffold/.agent/skills/total-recall/memory-vault/preferences/topic-research-sop.md

@@ -3,6 +3,7 @@ type: memory
 slug: topic-research-sop
 category: preferences
 title: "Standard Operating Procedure for Topic Research"
+description: "How the agent handles a research request: queue it to the background daemon, keep one living scratch document, and compile so it is searchable."
 status: active
 confidence: 1
 importance: 5
@@ -32,6 +33,7 @@ schema_version: 2
 x_temporal_context: "2026-05-25T22:36:21.376Z"
 priority: absolute
 immutable: true
+timestamp: "2026-05-25T22:36:21.376Z"
 ---
 
 # Standard Operating Procedure for Topic Research

package/scaffold/.agent/skills/total-recall/scripts/sync-repo.mjs

@@ -1,83 +1,188 @@
+#!/usr/bin/env node
+/**
+ * sync-repo — refresh this brain's skill definitions from the upstream templates.
+ *
+ * Replaces a stub that printed "Synced and verified skill: <name>", "Merging core
+ * invariants non-destructively" and "Sync Completed Successfully!" without
+ * fetching, merging or writing anything — while the skill's Core Directive #4
+ * told agents to run it to keep skills current. It also probed
+ * <agent>/memory-vault, which is not where a brain lives, so on a correct
+ * install it aborted with a misleading "run init first".
+ *
+ * What it actually does now:
+ *   - resolves a real template source (env override, installed package,
+ *     running-from-source checkout, or a shallow clone of upstream)
+ *   - copies skill files, PRESERVING each destination's compiled
+ *     "<!-- BEGIN INJECTED MEMORY -->" block, which is per-repo and must not be
+ *     overwritten by a template
+ *   - never touches memory-vault/ — that is user memory, not a template
+ *   - reports the files it really changed, and exits non-zero if it changed nothing
+ *     because it could not find a source
+ *
+ * Usage: node sync-repo.mjs [--dry-run] [--source <dir>]
+ */
 import fs from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
+import { execFileSync } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
 
-// Resolve agent paths
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const DRY = process.argv.includes('--dry-run');
+const srcFlag = process.argv.indexOf('--source');
+const SRC_OVERRIDE = srcFlag !== -1 ? process.argv[srcFlag + 1] : process.env.TR_SCAFFOLD_DIR;
+
 const home = os.homedir();
 const AGENT_DIR = process.env.AGENT_DIR || path.join(home, '.agent');
-const VAULT_DIR = path.join(AGENT_DIR, 'memory-vault');
+// A brain is <agent>/skills/total-recall — memory-vault, config and the rest all
+// live INSIDE it. The old code looked for <agent>/memory-vault and never found one.
+const BRAIN_DIR = path.join(AGENT_DIR, 'skills', 'total-recall');
 const SKILLS_DIR = path.join(AGENT_DIR, 'skills');
 
-console.log('🔄 Starting Total Recall Upstream Skill Synchronization...');
-console.log(`📂 Local Agent Directory: ${AGENT_DIR}`);
+const INJECT_START = '<!-- BEGIN INJECTED MEMORY';
+const INJECT_END = '<!-- END INJECTED MEMORY -->';
+
+function log(m) { console.log(m); }
+function fail(m) { console.error(`❌ ${m}`); process.exit(1); }
+
+/** Locate real template files. Returns { dir, origin } or null. */
+function resolveTemplateSource() {
+  if (SRC_OVERRIDE) {
+    if (!fs.existsSync(SRC_OVERRIDE)) fail(`--source path does not exist: ${SRC_OVERRIDE}`);
+    return { dir: SRC_OVERRIDE, origin: `explicit: ${SRC_OVERRIDE}` };
+  }
+  // Walk up looking for the package root rather than counting '..' segments:
+  // this script legitimately runs from two different depths (a brain at
+  // <agent>/skills/total-recall/scripts, and the template copy at
+  // <repo>/scaffold/.agent/skills/total-recall/scripts). A fixed relative path
+  // is right for one and silently wrong for the other — which resolved to the
+  // INSTALLED (older) package and would have reverted newer local templates.
+  const fromCheckout = (() => {
+    let dir = __dirname;
+    for (let i = 0; i < 8; i += 1) {
+      const pkg = path.join(dir, 'package.json');
+      const scaffold = path.join(dir, 'scaffold', '.agent', 'skills');
+      if (fs.existsSync(pkg) && fs.existsSync(scaffold)) {
+        try {
+          if (JSON.parse(fs.readFileSync(pkg, 'utf8')).name === 'total-recall-brain') return scaffold;
+        } catch { /* keep walking */ }
+      }
+      const up = path.dirname(dir);
+      if (up === dir) break;
+      dir = up;
+    }
+    return null;
+  })();
 
-// 1. Resolve upstream config
-let upstreamUrl = 'https://github.com/gregiteen/total-recall';
-try {
-  const brainConfigPath = path.join(AGENT_DIR, 'config', 'brain.json');
-  if (fs.existsSync(brainConfigPath)) {
-    const config = JSON.parse(fs.readFileSync(brainConfigPath, 'utf8'));
-    if (config.upstream) upstreamUrl = config.upstream;
+  const candidates = [
+    [fromCheckout, 'source checkout'],
+    [path.join(process.cwd(), 'node_modules', 'total-recall-brain', 'scaffold', '.agent', 'skills'), 'installed package'],
+    [path.join(home, 'node_modules', 'total-recall-brain', 'scaffold', '.agent', 'skills'), 'installed package (home)'],
+  ];
+  for (const [dir, origin] of candidates) {
+    if (dir && fs.existsSync(dir) && fs.statSync(dir).isDirectory()) return { dir, origin };
   }
-} catch {
-  // Use default
+  return null;
 }
-console.log(`🌐 Upstream Repository URL: ${upstreamUrl}`);
 
-// 2. Perform synchronization simulation / stub logic with absolute robustness
-async function runSync() {
+/** Last resort: shallow-clone upstream into a temp dir. */
+function cloneUpstream() {
+  let upstream = 'https://github.com/gregiteen/total-recall';
   try {
-    // Check if network is available / validate environment
-    console.log('🔎 Auditing local skill installations...');
-    
-    if (!fs.existsSync(VAULT_DIR) || !fs.existsSync(SKILLS_DIR)) {
-      throw new Error('Total Recall directories are missing. Run "npx total-recall init" first.');
+    const cfg = path.join(BRAIN_DIR, 'config', 'brain.json');
+    if (fs.existsSync(cfg)) {
+      const parsed = JSON.parse(fs.readFileSync(cfg, 'utf8'));
+      if (parsed.upstream) upstream = parsed.upstream;
     }
+  } catch { /* default */ }
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-sync-'));
+  log(`🌐 No local templates — shallow-cloning ${upstream}`);
+  try {
+    execFileSync('git', ['clone', '--depth', '1', '--quiet', upstream, tmp], { stdio: 'pipe' });
+  } catch (err) {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    fail(`could not clone upstream (${err.message.split('\n')[0]}). Pass --source <dir> to sync offline.`);
+  }
+  const dir = path.join(tmp, 'scaffold', '.agent', 'skills');
+  if (!fs.existsSync(dir)) {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    fail('upstream clone has no scaffold/.agent/skills — cannot sync.');
+  }
+  return { dir, origin: `clone of ${upstream}`, cleanup: tmp };
+}
 
-    const localSkills = fs.readdirSync(SKILLS_DIR);
-    console.log(`✅ Found ${localSkills.length} local skill packages: ${localSkills.join(', ')}`);
-
-    // In a real implementation, this would git fetch or fetch zip bundle from upstreamUrl.
-    // For local resilience, we perform a non-destructive merge of scaffolded presets.
-    console.log('📥 Fetching upstream skill templates...');
-    
-    // Simulate fetching core templates (e.g. ssss, project-management, total-recall itself)
-    const templates = ['ssss', 'project-management', 'total-recall'];
-    let updatedCount = 0;
-    
-    for (const t of templates) {
-      console.log(`📦 Synced and verified skill: ${t}`);
-      updatedCount++;
+/** Copy src over dst, carrying dst's compiled injected-memory block across. */
+function writePreservingInjectedBlock(srcFile, dstFile) {
+  let next = fs.readFileSync(srcFile, 'utf8');
+  if (fs.existsSync(dstFile)) {
+    const cur = fs.readFileSync(dstFile, 'utf8');
+    const s = cur.indexOf(INJECT_START);
+    const e = cur.indexOf(INJECT_END);
+    if (s !== -1 && e !== -1 && e > s) {
+      const block = cur.slice(s, e + INJECT_END.length);
+      const ns = next.indexOf(INJECT_START);
+      const ne = next.indexOf(INJECT_END);
+      next = (ns !== -1 && ne !== -1 && ne > ns)
+        ? next.slice(0, ns) + block + next.slice(ne + INJECT_END.length)
+        : `${next.replace(/\s*$/, '')}\n\n${block}\n`;
     }
+    if (cur === next) return 'identical';
+  }
+  if (!DRY) {
+    fs.mkdirSync(path.dirname(dstFile), { recursive: true });
+    fs.writeFileSync(dstFile, next);
+  }
+  return 'updated';
+}
 
-    console.log('📝 Merging core invariants non-destructively...');
-    console.log('🛡️ Preserving custom user-authored memory nodes inside memory-vault/');
-
-    // Trigger local surface compile to apply all changes
-    console.log('⚙️ Re-compiling system memory surfaces...');
-    try {
-      const { compileSurface } = await import('../../../src/core/surface.mjs');
-      const instructionsFile = path.join(process.cwd(), 'INSTRUCTIONS.md');
-      const derivedDir = path.join(AGENT_DIR, 'memory-derived');
-      
-      const result = await compileSurface({
-        vaultDir: VAULT_DIR,
-        skillsDir: SKILLS_DIR,
-        derivedDir,
-        instructionsFile
-      });
-      
-      console.log(`🚀 Compilation Complete: Processed ${result.nodesProcessed} nodes, injected ${result.skillsInjected} skills.`);
-    } catch (err) {
-      console.warn(`⚠️ Compilation skipped or ran into error: ${err.message}`);
-    }
+function walk(dir) {
+  const out = [];
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) {
+      // memory-vault is the user's own memory. A template must never overwrite it.
+      if (e.name === 'memory-vault' || e.name === 'memory-derived' || e.name === 'sessions') continue;
+      if (e.name === '.git' || e.name === 'node_modules') continue;
+      out.push(...walk(p));
+    } else out.push(p);
+  }
+  return out;
+}
 
-    console.log('🎉 Total Recall Skill Sync Completed Successfully!');
-  } catch (error) {
-    console.error(`❌ Sync Failed: ${error.message}`);
-    process.exit(1);
+log('🔄 Total Recall — syncing skill definitions from upstream templates');
+log(`📂 Brain: ${BRAIN_DIR}`);
+if (DRY) log('🔍 DRY RUN — nothing will be written');
+
+if (!fs.existsSync(BRAIN_DIR)) fail(`no brain at ${BRAIN_DIR}. Run "npx total-recall init" first.`);
+
+let source = resolveTemplateSource();
+if (!source) source = cloneUpstream();
+log(`📦 Template source: ${source.origin}`);
+
+const skills = fs.readdirSync(source.dir, { withFileTypes: true })
+  .filter((e) => e.isDirectory()).map((e) => e.name);
+if (skills.length === 0) fail(`template source has no skills: ${source.dir}`);
+
+let updated = 0, identical = 0, created = 0;
+const touched = [];
+for (const skill of skills) {
+  const from = path.join(source.dir, skill);
+  const to = path.join(SKILLS_DIR, skill);
+  const isNew = !fs.existsSync(to);
+  for (const f of walk(from)) {
+    const rel = path.relative(from, f);
+    const res = writePreservingInjectedBlock(f, path.join(to, rel));
+    if (res === 'updated') { updated++; touched.push(`${skill}/${rel}`); } else identical++;
   }
+  if (isNew) created++;
 }
 
-runSync();
+if (source.cleanup) fs.rmSync(source.cleanup, { recursive: true, force: true });
+
+log('');
+log(`${DRY ? '🔍' : '✅'} ${updated} file(s) ${DRY ? 'would change' : 'updated'}, ${identical} already current, ${created} new skill(s)`);
+for (const t of touched.slice(0, 20)) log(`   ${DRY ? '~' : '↳'} ${t}`);
+if (touched.length > 20) log(`   … and ${touched.length - 20} more`);
+if (updated === 0) log('   Nothing to do — every skill file already matches the templates.');
+log('');
+log(DRY ? 'Re-run without --dry-run to apply.' : 'Run "npx total-recall compile" to rebuild the instruction surfaces.');

package/src/cli/backup.mjs

@@ -116,6 +116,33 @@ function printHelp() {
 `);
 }
 
+function ensureBrainGitignore(SKILL_DIR) {
+  const gitignorePath = path.join(SKILL_DIR, '.gitignore');
+  const entries = [
+    'memory-derived/', 'sessions/', '*.backup', 'logs/', 'node_modules/',
+    'memory-vault/.events/', 'browser-profile/', '.snapshots/', 'daemon.pid',
+  ];
+  let existing = '';
+  if (fs.existsSync(gitignorePath)) existing = fs.readFileSync(gitignorePath, 'utf8');
+  const missing = entries.filter((e) => !existing.split('\n').some((l) => l.trim() === e));
+  if (missing.length === 0) return 0;
+  fs.appendFileSync(
+    gitignorePath,
+    '\n# Auto-added by total-recall backup — runtime/derived/sensitive, never versioned\n'
+      + missing.join('\n') + '\n',
+  );
+  console.error(`  \u2705 .gitignore: added ${missing.length} protected entr${missing.length === 1 ? 'y' : 'ies'}`);
+  // Anything already tracked must also be untracked, or the ignore is inert.
+  for (const e of missing) {
+    const tracked = git(SKILL_DIR, 'ls-files', e.replace(/\/$/, ''));
+    if (tracked.ok && tracked.stdout) {
+      git(SKILL_DIR, 'rm', '-r', '--cached', '-f', '-q', e.replace(/\/$/, ''));
+      console.error(`  \u2705 untracked ${e} (kept on disk)`);
+    }
+  }
+  return missing.length;
+}
+
 async function pushGitBackup(remote, layer = 'auto') {
   const SKILL_DIR = getSkillDir(layer);
   if (!commandExists('git')) {
@@ -129,6 +156,19 @@ async function pushGitBackup(remote, layer = 'auto') {
     process.exit(1);
   }
 
+  // A brain is memory, not runtime state. Three classes of file have each
+  // broken a real backup by being versioned:
+  //   memory-vault/.events/  append-only logs; default.jsonl reached 306MB and
+  //                          GitHub hard-rejects >100MB, silently killing every
+  //                          global backup for 5 weeks (2026-07-14 -> 08-19).
+  //   browser-profile/       the rotation Chromium profile — live Stripe/GitHub
+  //                          cookies and Login Data. Was tracked and one
+  //                          successful push from landing on GitHub.
+  //   .snapshots/            ~46MB vault tarballs stored inside the vault.
+  // Enforced on every push, not just --install, so brains created before this
+  // rule are corrected on their next backup.
+  ensureBrainGitignore(SKILL_DIR);
+
   // Initialise the skill folder as a git repo if needed
   const gitDir = path.join(SKILL_DIR, '.git');
   if (!fs.existsSync(gitDir)) {
@@ -170,6 +210,29 @@ async function pushGitBackup(remote, layer = 'auto') {
   const stamp = new Date().toISOString().replace('T', ' ').slice(0, 19);
 
   if (!status.stdout) {
+    // A clean working tree does NOT mean the remote is current. Two scheduled
+    // jobs pointing the shared "backup" remote at different URLs is enough to
+    // break this: whichever ran second re-pointed the remote, found the tree
+    // clean, and returned "nothing to push" — so that destination silently
+    // froze for months while every run reported success. Only skip when the
+    // remote actually has our HEAD.
+    const head = git(SKILL_DIR, 'rev-parse', 'HEAD');
+    const localSha = head.ok ? (head.stdout || '').trim() : '';
+    const remoteHead = git(SKILL_DIR, 'ls-remote', remote, 'main');
+    const remoteSha = remoteHead.ok ? (remoteHead.stdout || '').split(/\s+/)[0] : '';
+    // No local commit to compare, or the remote already has it: genuinely
+    // nothing to do. Only a KNOWN mismatch justifies pushing a clean tree.
+    if (!localSha || remoteSha === localSha) {
+      console.error(`  ✅ Brain unchanged — nothing to push (${stamp})`);
+      return;
+    }
+    if (remoteSha) {
+      console.error(`  📤 Working tree clean but ${remote} is behind — pushing existing commits...`);
+      const p = git(SKILL_DIR, 'push', remote, 'HEAD:main');
+      if (p.ok) console.error(`  ✅ Brain pushed to backup (${stamp})`);
+      else console.error(`  ❌ Push failed: ${p.stderr || p.stdout}`);
+      return;
+    }
     console.error(`  ✅ Brain unchanged — nothing to push (${stamp})`);
     return;
   }

package/src/server/route-manifest.json

@@ -103,6 +103,10 @@
     "method": "POST",
     "path": "/api/context/stream"
   },
+  {
+    "method": "POST",
+    "path": "/api/daemon/restart"
+  },
   {
     "method": "GET",
     "path": "/api/dashboard/instructions"