total-recall-brain 3.24.0 → 3.24.2

package/scaffold/.agent/skills/total-recall/scripts/sync-repo.mjs

@@ -1,83 +1,188 @@
+#!/usr/bin/env node
+/**
+ * sync-repo — refresh this brain's skill definitions from the upstream templates.
+ *
+ * Replaces a stub that printed "Synced and verified skill: <name>", "Merging core
+ * invariants non-destructively" and "Sync Completed Successfully!" without
+ * fetching, merging or writing anything — while the skill's Core Directive #4
+ * told agents to run it to keep skills current. It also probed
+ * <agent>/memory-vault, which is not where a brain lives, so on a correct
+ * install it aborted with a misleading "run init first".
+ *
+ * What it actually does now:
+ *   - resolves a real template source (env override, installed package,
+ *     running-from-source checkout, or a shallow clone of upstream)
+ *   - copies skill files, PRESERVING each destination's compiled
+ *     "<!-- BEGIN INJECTED MEMORY -->" block, which is per-repo and must not be
+ *     overwritten by a template
+ *   - never touches memory-vault/ — that is user memory, not a template
+ *   - reports the files it really changed, and exits non-zero if it changed nothing
+ *     because it could not find a source
+ *
+ * Usage: node sync-repo.mjs [--dry-run] [--source <dir>]
+ */
 import fs from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
+import { execFileSync } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
 
-// Resolve agent paths
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const DRY = process.argv.includes('--dry-run');
+const srcFlag = process.argv.indexOf('--source');
+const SRC_OVERRIDE = srcFlag !== -1 ? process.argv[srcFlag + 1] : process.env.TR_SCAFFOLD_DIR;
+
 const home = os.homedir();
 const AGENT_DIR = process.env.AGENT_DIR || path.join(home, '.agent');
-const VAULT_DIR = path.join(AGENT_DIR, 'memory-vault');
+// A brain is <agent>/skills/total-recall — memory-vault, config and the rest all
+// live INSIDE it. The old code looked for <agent>/memory-vault and never found one.
+const BRAIN_DIR = path.join(AGENT_DIR, 'skills', 'total-recall');
 const SKILLS_DIR = path.join(AGENT_DIR, 'skills');
 
-console.log('🔄 Starting Total Recall Upstream Skill Synchronization...');
-console.log(`📂 Local Agent Directory: ${AGENT_DIR}`);
+const INJECT_START = '<!-- BEGIN INJECTED MEMORY';
+const INJECT_END = '<!-- END INJECTED MEMORY -->';
+
+function log(m) { console.log(m); }
+function fail(m) { console.error(`❌ ${m}`); process.exit(1); }
+
+/** Locate real template files. Returns { dir, origin } or null. */
+function resolveTemplateSource() {
+  if (SRC_OVERRIDE) {
+    if (!fs.existsSync(SRC_OVERRIDE)) fail(`--source path does not exist: ${SRC_OVERRIDE}`);
+    return { dir: SRC_OVERRIDE, origin: `explicit: ${SRC_OVERRIDE}` };
+  }
+  // Walk up looking for the package root rather than counting '..' segments:
+  // this script legitimately runs from two different depths (a brain at
+  // <agent>/skills/total-recall/scripts, and the template copy at
+  // <repo>/scaffold/.agent/skills/total-recall/scripts). A fixed relative path
+  // is right for one and silently wrong for the other — which resolved to the
+  // INSTALLED (older) package and would have reverted newer local templates.
+  const fromCheckout = (() => {
+    let dir = __dirname;
+    for (let i = 0; i < 8; i += 1) {
+      const pkg = path.join(dir, 'package.json');
+      const scaffold = path.join(dir, 'scaffold', '.agent', 'skills');
+      if (fs.existsSync(pkg) && fs.existsSync(scaffold)) {
+        try {
+          if (JSON.parse(fs.readFileSync(pkg, 'utf8')).name === 'total-recall-brain') return scaffold;
+        } catch { /* keep walking */ }
+      }
+      const up = path.dirname(dir);
+      if (up === dir) break;
+      dir = up;
+    }
+    return null;
+  })();
 
-// 1. Resolve upstream config
-let upstreamUrl = 'https://github.com/gregiteen/total-recall';
-try {
-  const brainConfigPath = path.join(AGENT_DIR, 'config', 'brain.json');
-  if (fs.existsSync(brainConfigPath)) {
-    const config = JSON.parse(fs.readFileSync(brainConfigPath, 'utf8'));
-    if (config.upstream) upstreamUrl = config.upstream;
+  const candidates = [
+    [fromCheckout, 'source checkout'],
+    [path.join(process.cwd(), 'node_modules', 'total-recall-brain', 'scaffold', '.agent', 'skills'), 'installed package'],
+    [path.join(home, 'node_modules', 'total-recall-brain', 'scaffold', '.agent', 'skills'), 'installed package (home)'],
+  ];
+  for (const [dir, origin] of candidates) {
+    if (dir && fs.existsSync(dir) && fs.statSync(dir).isDirectory()) return { dir, origin };
   }
-} catch {
-  // Use default
+  return null;
 }
-console.log(`🌐 Upstream Repository URL: ${upstreamUrl}`);
 
-// 2. Perform synchronization simulation / stub logic with absolute robustness
-async function runSync() {
+/** Last resort: shallow-clone upstream into a temp dir. */
+function cloneUpstream() {
+  let upstream = 'https://github.com/gregiteen/total-recall';
   try {
-    // Check if network is available / validate environment
-    console.log('🔎 Auditing local skill installations...');
-    
-    if (!fs.existsSync(VAULT_DIR) || !fs.existsSync(SKILLS_DIR)) {
-      throw new Error('Total Recall directories are missing. Run "npx total-recall init" first.');
+    const cfg = path.join(BRAIN_DIR, 'config', 'brain.json');
+    if (fs.existsSync(cfg)) {
+      const parsed = JSON.parse(fs.readFileSync(cfg, 'utf8'));
+      if (parsed.upstream) upstream = parsed.upstream;
     }
+  } catch { /* default */ }
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'tr-sync-'));
+  log(`🌐 No local templates — shallow-cloning ${upstream}`);
+  try {
+    execFileSync('git', ['clone', '--depth', '1', '--quiet', upstream, tmp], { stdio: 'pipe' });
+  } catch (err) {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    fail(`could not clone upstream (${err.message.split('\n')[0]}). Pass --source <dir> to sync offline.`);
+  }
+  const dir = path.join(tmp, 'scaffold', '.agent', 'skills');
+  if (!fs.existsSync(dir)) {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    fail('upstream clone has no scaffold/.agent/skills — cannot sync.');
+  }
+  return { dir, origin: `clone of ${upstream}`, cleanup: tmp };
+}
 
-    const localSkills = fs.readdirSync(SKILLS_DIR);
-    console.log(`✅ Found ${localSkills.length} local skill packages: ${localSkills.join(', ')}`);
-
-    // In a real implementation, this would git fetch or fetch zip bundle from upstreamUrl.
-    // For local resilience, we perform a non-destructive merge of scaffolded presets.
-    console.log('📥 Fetching upstream skill templates...');
-    
-    // Simulate fetching core templates (e.g. ssss, project-management, total-recall itself)
-    const templates = ['ssss', 'project-management', 'total-recall'];
-    let updatedCount = 0;
-    
-    for (const t of templates) {
-      console.log(`📦 Synced and verified skill: ${t}`);
-      updatedCount++;
+/** Copy src over dst, carrying dst's compiled injected-memory block across. */
+function writePreservingInjectedBlock(srcFile, dstFile) {
+  let next = fs.readFileSync(srcFile, 'utf8');
+  if (fs.existsSync(dstFile)) {
+    const cur = fs.readFileSync(dstFile, 'utf8');
+    const s = cur.indexOf(INJECT_START);
+    const e = cur.indexOf(INJECT_END);
+    if (s !== -1 && e !== -1 && e > s) {
+      const block = cur.slice(s, e + INJECT_END.length);
+      const ns = next.indexOf(INJECT_START);
+      const ne = next.indexOf(INJECT_END);
+      next = (ns !== -1 && ne !== -1 && ne > ns)
+        ? next.slice(0, ns) + block + next.slice(ne + INJECT_END.length)
+        : `${next.replace(/\s*$/, '')}\n\n${block}\n`;
     }
+    if (cur === next) return 'identical';
+  }
+  if (!DRY) {
+    fs.mkdirSync(path.dirname(dstFile), { recursive: true });
+    fs.writeFileSync(dstFile, next);
+  }
+  return 'updated';
+}
 
-    console.log('📝 Merging core invariants non-destructively...');
-    console.log('🛡️ Preserving custom user-authored memory nodes inside memory-vault/');
-
-    // Trigger local surface compile to apply all changes
-    console.log('⚙️ Re-compiling system memory surfaces...');
-    try {
-      const { compileSurface } = await import('../../../src/core/surface.mjs');
-      const instructionsFile = path.join(process.cwd(), 'INSTRUCTIONS.md');
-      const derivedDir = path.join(AGENT_DIR, 'memory-derived');
-      
-      const result = await compileSurface({
-        vaultDir: VAULT_DIR,
-        skillsDir: SKILLS_DIR,
-        derivedDir,
-        instructionsFile
-      });
-      
-      console.log(`🚀 Compilation Complete: Processed ${result.nodesProcessed} nodes, injected ${result.skillsInjected} skills.`);
-    } catch (err) {
-      console.warn(`⚠️ Compilation skipped or ran into error: ${err.message}`);
-    }
+function walk(dir) {
+  const out = [];
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) {
+      // memory-vault is the user's own memory. A template must never overwrite it.
+      if (e.name === 'memory-vault' || e.name === 'memory-derived' || e.name === 'sessions') continue;
+      if (e.name === '.git' || e.name === 'node_modules') continue;
+      out.push(...walk(p));
+    } else out.push(p);
+  }
+  return out;
+}
 
-    console.log('🎉 Total Recall Skill Sync Completed Successfully!');
-  } catch (error) {
-    console.error(`❌ Sync Failed: ${error.message}`);
-    process.exit(1);
+log('🔄 Total Recall — syncing skill definitions from upstream templates');
+log(`📂 Brain: ${BRAIN_DIR}`);
+if (DRY) log('🔍 DRY RUN — nothing will be written');
+
+if (!fs.existsSync(BRAIN_DIR)) fail(`no brain at ${BRAIN_DIR}. Run "npx total-recall init" first.`);
+
+let source = resolveTemplateSource();
+if (!source) source = cloneUpstream();
+log(`📦 Template source: ${source.origin}`);
+
+const skills = fs.readdirSync(source.dir, { withFileTypes: true })
+  .filter((e) => e.isDirectory()).map((e) => e.name);
+if (skills.length === 0) fail(`template source has no skills: ${source.dir}`);
+
+let updated = 0, identical = 0, created = 0;
+const touched = [];
+for (const skill of skills) {
+  const from = path.join(source.dir, skill);
+  const to = path.join(SKILLS_DIR, skill);
+  const isNew = !fs.existsSync(to);
+  for (const f of walk(from)) {
+    const rel = path.relative(from, f);
+    const res = writePreservingInjectedBlock(f, path.join(to, rel));
+    if (res === 'updated') { updated++; touched.push(`${skill}/${rel}`); } else identical++;
   }
+  if (isNew) created++;
 }
 
-runSync();
+if (source.cleanup) fs.rmSync(source.cleanup, { recursive: true, force: true });
+
+log('');
+log(`${DRY ? '🔍' : '✅'} ${updated} file(s) ${DRY ? 'would change' : 'updated'}, ${identical} already current, ${created} new skill(s)`);
+for (const t of touched.slice(0, 20)) log(`   ${DRY ? '~' : '↳'} ${t}`);
+if (touched.length > 20) log(`   … and ${touched.length - 20} more`);
+if (updated === 0) log('   Nothing to do — every skill file already matches the templates.');
+log('');
+log(DRY ? 'Re-run without --dry-run to apply.' : 'Run "npx total-recall compile" to rebuild the instruction surfaces.');

package/src/cli/backup.mjs

@@ -116,6 +116,33 @@ function printHelp() {
 `);
 }
 
+function ensureBrainGitignore(SKILL_DIR) {
+  const gitignorePath = path.join(SKILL_DIR, '.gitignore');
+  const entries = [
+    'memory-derived/', 'sessions/', '*.backup', 'logs/', 'node_modules/',
+    'memory-vault/.events/', 'browser-profile/', '.snapshots/', 'daemon.pid',
+  ];
+  let existing = '';
+  if (fs.existsSync(gitignorePath)) existing = fs.readFileSync(gitignorePath, 'utf8');
+  const missing = entries.filter((e) => !existing.split('\n').some((l) => l.trim() === e));
+  if (missing.length === 0) return 0;
+  fs.appendFileSync(
+    gitignorePath,
+    '\n# Auto-added by total-recall backup — runtime/derived/sensitive, never versioned\n'
+      + missing.join('\n') + '\n',
+  );
+  console.error(`  \u2705 .gitignore: added ${missing.length} protected entr${missing.length === 1 ? 'y' : 'ies'}`);
+  // Anything already tracked must also be untracked, or the ignore is inert.
+  for (const e of missing) {
+    const tracked = git(SKILL_DIR, 'ls-files', e.replace(/\/$/, ''));
+    if (tracked.ok && tracked.stdout) {
+      git(SKILL_DIR, 'rm', '-r', '--cached', '-f', '-q', e.replace(/\/$/, ''));
+      console.error(`  \u2705 untracked ${e} (kept on disk)`);
+    }
+  }
+  return missing.length;
+}
+
 async function pushGitBackup(remote, layer = 'auto') {
   const SKILL_DIR = getSkillDir(layer);
   if (!commandExists('git')) {
@@ -129,6 +156,19 @@ async function pushGitBackup(remote, layer = 'auto') {
     process.exit(1);
   }
 
+  // A brain is memory, not runtime state. Three classes of file have each
+  // broken a real backup by being versioned:
+  //   memory-vault/.events/  append-only logs; default.jsonl reached 306MB and
+  //                          GitHub hard-rejects >100MB, silently killing every
+  //                          global backup for 5 weeks (2026-07-14 -> 08-19).
+  //   browser-profile/       the rotation Chromium profile — live Stripe/GitHub
+  //                          cookies and Login Data. Was tracked and one
+  //                          successful push from landing on GitHub.
+  //   .snapshots/            ~46MB vault tarballs stored inside the vault.
+  // Enforced on every push, not just --install, so brains created before this
+  // rule are corrected on their next backup.
+  ensureBrainGitignore(SKILL_DIR);
+
   // Initialise the skill folder as a git repo if needed
   const gitDir = path.join(SKILL_DIR, '.git');
   if (!fs.existsSync(gitDir)) {
@@ -170,6 +210,29 @@ async function pushGitBackup(remote, layer = 'auto') {
   const stamp = new Date().toISOString().replace('T', ' ').slice(0, 19);
 
   if (!status.stdout) {
+    // A clean working tree does NOT mean the remote is current. Two scheduled
+    // jobs pointing the shared "backup" remote at different URLs is enough to
+    // break this: whichever ran second re-pointed the remote, found the tree
+    // clean, and returned "nothing to push" — so that destination silently
+    // froze for months while every run reported success. Only skip when the
+    // remote actually has our HEAD.
+    const head = git(SKILL_DIR, 'rev-parse', 'HEAD');
+    const localSha = head.ok ? (head.stdout || '').trim() : '';
+    const remoteHead = git(SKILL_DIR, 'ls-remote', remote, 'main');
+    const remoteSha = remoteHead.ok ? (remoteHead.stdout || '').split(/\s+/)[0] : '';
+    // No local commit to compare, or the remote already has it: genuinely
+    // nothing to do. Only a KNOWN mismatch justifies pushing a clean tree.
+    if (!localSha || remoteSha === localSha) {
+      console.error(`  ✅ Brain unchanged — nothing to push (${stamp})`);
+      return;
+    }
+    if (remoteSha) {
+      console.error(`  📤 Working tree clean but ${remote} is behind — pushing existing commits...`);
+      const p = git(SKILL_DIR, 'push', remote, 'HEAD:main');
+      if (p.ok) console.error(`  ✅ Brain pushed to backup (${stamp})`);
+      else console.error(`  ❌ Push failed: ${p.stderr || p.stdout}`);
+      return;
+    }
     console.error(`  ✅ Brain unchanged — nothing to push (${stamp})`);
     return;
   }

package/src/core/package-auto-update.mjs

@@ -169,20 +169,24 @@ export function inspectProjectPackage(projectRoot) {
   } catch {
     return { declared: null, installed: null, isSourceTree: false, packageName: null };
   }
+  const isSourceTree =
+    pkg.name === PACKAGE_NAME &&
+    fs.existsSync(path.join(projectRoot, 'src', 'server', 'index.mjs'));
   const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
   const declared = deps[PACKAGE_NAME] || null;
   let installed = null;
-  const nm = path.join(projectRoot, 'node_modules', PACKAGE_NAME, 'package.json');
-  if (fs.existsSync(nm)) {
-    try {
-      installed = JSON.parse(fs.readFileSync(nm, 'utf8')).version || null;
-    } catch {
-      installed = null;
+  if (isSourceTree) {
+    installed = pkg.version || null;
+  } else {
+    const nm = path.join(projectRoot, 'node_modules', PACKAGE_NAME, 'package.json');
+    if (fs.existsSync(nm)) {
+      try {
+        installed = JSON.parse(fs.readFileSync(nm, 'utf8')).version || null;
+      } catch {
+        installed = null;
+      }
     }
   }
-  const isSourceTree =
-    pkg.name === PACKAGE_NAME &&
-    fs.existsSync(path.join(projectRoot, 'src', 'server', 'index.mjs'));
   return { declared, installed, isSourceTree, packageName: pkg.name || null };
 }
 

package/src/core/surface.mjs

@@ -321,38 +321,67 @@ export async function buildRulesBlock(skillsDir, nodes = [], { consumer = 'ide',
   // Note: Only invariants, preferences, and anti-patterns are included in instructions,
   // enforcing Category Partitioning (concepts, decisions, facts are search-only).
   const isImportant = (n) => n.importance === undefined || n.importance >= 3;
-  const invariants = nodes.filter(n => n.category === 'invariants' && n.status === 'active' && !isExpired(n) && isImportant(n));
-  const preferences = nodes.filter(n => n.category === 'preferences' && n.status === 'active' && !isExpired(n) && isImportant(n));
-  const corrections = nodes.filter(n => n.category === 'anti-patterns' && n.status === 'active' && !isExpired(n) && isImportant(n));
+  
+  // Determine current project/repo context to prevent cross-repo pollution
+  const currentRepoName = path.basename(process.cwd()).toLowerCase();
+  
+  const isRelevantToRepo = (n) => {
+    // If explicitly marked global or no repo restriction, it applies everywhere
+    if (n.scope === 'global' || n._layer === 'global') {
+      // If the node text explicitly binds to specific other repos, skip it
+      if (Array.isArray(n.repos) && n.repos.length > 0) {
+        return n.repos.some(r => r.toLowerCase() === currentRepoName);
+      }
+      return true;
+    }
+    // If tagged with specific repos or project name
+    if (Array.isArray(n.repos) && n.repos.length > 0) {
+      return n.repos.some(r => r.toLowerCase() === currentRepoName);
+    }
+    if (n.project && typeof n.project === 'string') {
+      return n.project.toLowerCase() === currentRepoName;
+    }
+    return true;
+  };
 
-  // Deduplicate nodes by compacted content hash (OKF non-destructive merge pattern).
-  // When two nodes produce identical compacted text, keep the one with higher importance.
-  const deduplicateNodes = (list) => {
+  const rawInvariants = nodes.filter(n => n.category === 'invariants' && n.status === 'active' && !isExpired(n) && isImportant(n) && isRelevantToRepo(n));
+  const rawPreferences = nodes.filter(n => n.category === 'preferences' && n.status === 'active' && !isExpired(n) && isImportant(n) && isRelevantToRepo(n));
+  const rawCorrections = nodes.filter(n => n.category === 'anti-patterns' && n.status === 'active' && !isExpired(n) && isImportant(n) && isRelevantToRepo(n));
+
+  // Deduplicate nodes by normalized content key and rank by priority/importance
+  const deduplicateAndRankNodes = (list, maxCount = 15) => {
     const seen = new Map();
     const deduped = [];
-    for (const n of list) {
-      const key = (n.body || n.content || '').trim().toLowerCase().substring(0, 200);
-      if (seen.has(key)) {
-        const existing = seen.get(key);
-        if ((n.importance || 3) > (existing.importance || 3)) {
-          // Replace with higher-importance node
-          const idx = deduped.indexOf(existing);
-          if (idx !== -1) deduped[idx] = n;
-          seen.set(key, n);
-        }
-        // else skip the lower-importance duplicate
-      } else {
-        seen.set(key, n);
+    
+    // Sort by importance (descending) and priority (absolute first)
+    const sorted = [...list].sort((a, b) => {
+      const aAbs = a.priority === 'absolute' ? 10 : 0;
+      const bAbs = b.priority === 'absolute' ? 10 : 0;
+      const aScore = (a.importance || 3) + aAbs;
+      const bScore = (b.importance || 3) + bAbs;
+      return bScore - aScore;
+    });
+
+    for (const n of sorted) {
+      // Normalize content to catch paraphrased/duplicate directives
+      const rawText = (n.body || n.content || n.title || '').trim().toLowerCase();
+      const normKey = rawText.replace(/[^a-z0-9]/g, '').substring(0, 120);
+      
+      if (!seen.has(normKey)) {
+        seen.set(normKey, n);
         deduped.push(n);
       }
     }
-    return deduped;
+    return deduped.slice(0, maxCount);
   };
 
+  const invariants = deduplicateAndRankNodes(rawInvariants, 15);
+  const preferences = deduplicateAndRankNodes(rawPreferences, 10);
+  const corrections = deduplicateAndRankNodes(rawCorrections, 15);
+
   const formatNodes = async (list) => {
-    const unique = deduplicateNodes(list);
     const formatted = [];
-    for (const n of unique) {
+    for (const n of list) {
       const snippet = await compactNode(n, derivedDir, force);
       formatted.push(snippet.startsWith('-') ? snippet : `- ${snippet}`);
     }

package/src/server/route-manifest.json

@@ -103,6 +103,10 @@
     "method": "POST",
     "path": "/api/context/stream"
   },
+  {
+    "method": "POST",
+    "path": "/api/daemon/restart"
+  },
   {
     "method": "GET",
     "path": "/api/dashboard/instructions"

package/src/server/routes/system.mjs

@@ -202,5 +202,20 @@ router.get('/api/usage/providers', requireAuth, async (req, res) => {
   } catch (err) { serverError(res, err); }
 });
 
+/**
+ * POST /api/daemon/restart
+ * Restarts the Active Intelligence background daemon process.
+ */
+router.post('/api/daemon/restart', requireAuth, requireScope('config:write'), async (_req, res) => {
+  try {
+    const { stopDaemon, startDaemon } = await import('../../core/daemon-control.mjs');
+    stopDaemon();
+    const pid = startDaemon();
+    res.json({ success: true, message: `Daemon restarted successfully (PID ${pid})`, pid });
+  } catch (err) {
+    serverError(res, err);
+  }
+});
+
 export default router;
 export { router as systemRouter };

package/src/server/routes/update.mjs

@@ -45,7 +45,7 @@ router.get('/api/update/check', requireAuth, async (_req, res) => {
 
     const consumersBehind = projects.filter((p) => p.update_available).length;
     const updateAvailable =
-      Boolean(latest && current && current !== latest) || consumersBehind > 0;
+      Boolean(latest && current && needsUpdate(current, latest)) || consumersBehind > 0;
 
     res.json({
       package: PACKAGE_NAME,