total-recall-brain 3.41.4 → 3.41.5

package/src/core/context-compactor.mjs

@@ -0,0 +1,143 @@
+/** Budgeted task capsules: rank rules, keep verbatim text, and account for every rule left out. */
+
+import { estimateTokens } from './context-policy.mjs';
+import { cosineSimilarity } from './embeddings.mjs';
+
+const DUPLICATE_SIMILARITY = 0.92;
+const REFERENCE_SHARE = 0.4;
+const TOP_FULL_SHARE = 0.3;
+const SUMMARY_MAX_CHARS = 200;
+const SUMMARY_MIN_CHARS = 80;
+// A period here does not end a sentence.
+const SENTENCE_END = /(?<!\be\.g)(?<!\bi\.e)(?<!\bvs)[.!?](?=\s|$)/g;
+
+/** The leading sentences of a rule, verbatim. Never rewords, because rewording can change a constraint. */
+export function summarizeRule(text) {
+  const flat = String(text || '').replace(/^#+\s.*$/gm, ' ').replace(/\s+/g, ' ').trim();
+  if (flat.length <= SUMMARY_MAX_CHARS) return flat;
+  let end = 0;
+  for (const match of flat.matchAll(SENTENCE_END)) {
+    const candidate = match.index + 1;
+    if (candidate > SUMMARY_MAX_CHARS) break;
+    end = candidate;
+    if (end >= SUMMARY_MIN_CHARS) return flat.slice(0, end);
+  }
+  if (end) return flat.slice(0, end);
+  const cut = flat.lastIndexOf(' ', SUMMARY_MAX_CHARS);
+  return `${flat.slice(0, cut > 0 ? cut : SUMMARY_MAX_CHARS)}…`;
+}
+
+/** Relevance alone orders the text; ties fall to importance, then slug, so output is stable. */
+function rank(items) {
+  return [...items].sort((a, b) =>
+    (b.score || 0) - (a.score || 0) ||
+    (b.importance || 3) - (a.importance || 3) ||
+    a.slug.localeCompare(b.slug));
+}
+
+function findDuplicates(ranked) {
+  const duplicateOf = new Map();
+  const normalize = text => String(text).toLowerCase().replace(/\s+/g, ' ').trim();
+  ranked.forEach((item, i) => {
+    if (item.priority === 'absolute') return;
+    for (const earlier of ranked.slice(0, i)) {
+      if (duplicateOf.has(earlier.slug)) continue;
+      const same = normalize(item.body) === normalize(earlier.body) ||
+        (item.vector && earlier.vector && cosineSimilarity(item.vector, earlier.vector) >= DUPLICATE_SIMILARITY);
+      if (same) { duplicateOf.set(item.slug, earlier.slug); return; }
+    }
+  });
+  return duplicateOf;
+}
+
+/**
+ * Assign each rule a tier so the printed output fits `budget`, measuring the real
+ * render after every change. `header(counts, tokens)` supplies the status lines.
+ */
+export function compactRules(items, { budget, expand = [], header, complete = false }) {
+  if (!Number.isFinite(budget) || budget < 1) throw new Error('Compact budget must be a positive token estimate');
+  const ranked = rank(items);
+  const known = new Set(ranked.map(item => item.slug));
+  const unknown = expand.filter(slug => !known.has(slug));
+  if (unknown.length) throw new Error(`Cannot expand rules not in this capsule: ${unknown.join(', ')}`);
+  const duplicateOf = findDuplicates(ranked.filter(item => !expand.includes(item.slug)));
+  const tiers = new Map(ranked.map(item => [item.slug, 'omitted']));
+
+  const render = () => {
+    const of = tier => ranked.filter(item => tiers.get(item.slug) === tier);
+    const full = of('full'), summary = of('summary'), reference = of('reference'), duplicate = of('duplicate');
+    const counts = { rules: ranked.length, full: full.length, summarized: summary.length, referenced: reference.length,
+      duplicate: duplicate.length, omitted: of('omitted').length };
+    const blocks = [
+      ...full.map(item => item.text.trim()),
+      ...summary.map(item => `[${item.slug}] ${summarizeRule(item.body)}`),
+    ];
+    if (reference.length) blocks.push(`Referenced: ${reference.map(item => item.slug).join(', ')}`);
+    if (duplicate.length) blocks.push(`Duplicates: ${duplicate.map(item => `${item.slug} (of ${duplicateOf.get(item.slug)})`).join(', ')}`);
+    const body = blocks.join('\n\n---\n\n');
+    // The token count sits inside the text it counts, so iterate to a stable value.
+    let tokens = estimateTokens(body);
+    let context = '';
+    for (let i = 0; i < 5; i++) {
+      context = `${header(counts, tokens)}${body ? `\n${body}` : ''}`;
+      const measured = estimateTokens(`${context}\n`);
+      if (measured === tokens) break;
+      tokens = measured;
+    }
+    return { context, counts, tokens };
+  };
+  const used = () => render().tokens;
+  const tryTier = (item, tier, limit = budget) => {
+    const previous = tiers.get(item.slug);
+    tiers.set(item.slug, tier);
+    if (used() <= limit) return true;
+    tiers.set(item.slug, previous);
+    return false;
+  };
+
+  // A complete capsule measures its own, higher floor below, so a retry at the reported number succeeds.
+  const floor = used();
+  if (!complete && floor > budget) {
+    const error = new Error(`Budget ${budget} cannot hold the capsule header; the minimum is ${floor}`);
+    error.requiredBudget = floor;
+    throw error;
+  }
+
+  for (const slug of expand) {
+    if (!tryTier(ranked.find(item => item.slug === slug), 'full')) {
+      const error = new Error(`Budget ${budget} cannot hold expanded rule ${slug}; raise --budget or expand fewer rules`);
+      throw error;
+    }
+  }
+  // A complete capsule accounts for every rule before ranking spends any budget, so
+  // ranking decides how much text a rule gets and never whether it applies.
+  if (complete) {
+    for (const item of ranked) {
+      if (tiers.get(item.slug) !== 'omitted') continue;
+      tiers.set(item.slug, duplicateOf.has(item.slug) ? 'duplicate' : item.priority === 'absolute' ? 'summary' : 'reference');
+    }
+    const completeFloor = used();
+    if (completeFloor > budget) {
+      const error = new Error(`Budget ${budget} cannot hold every rule; the minimum is ${completeFloor}`);
+      error.requiredBudget = completeFloor;
+      throw error;
+    }
+  }
+  const candidates = ranked.filter(item => !duplicateOf.has(item.slug) && tiers.get(item.slug) !== 'full');
+
+  // Slugs alone must not consume a small budget: the top rules need their text.
+  const referenceLimit = used() + Math.floor((budget - used()) * REFERENCE_SHARE);
+  // Absolute rules are named first, so a small budget still shows they exist.
+  const unplaced = candidates.filter(item => tiers.get(item.slug) === 'omitted');
+  const namedFirst = [...unplaced.filter(item => item.priority === 'absolute'), ...unplaced.filter(item => item.priority !== 'absolute')];
+  for (const item of namedFirst) if (!tryTier(item, 'reference', referenceLimit)) break;
+  const topLimit = used() + Math.floor((budget - used()) * TOP_FULL_SHARE);
+  for (const item of candidates) tryTier(item, 'full', topLimit);
+  for (const item of candidates) if (tiers.get(item.slug) !== 'full') tryTier(item, 'summary');
+  for (const item of candidates) if (tiers.get(item.slug) !== 'full') tryTier(item, 'full');
+  for (const item of namedFirst) if (tiers.get(item.slug) === 'omitted') tryTier(item, 'reference');
+  for (const slug of duplicateOf.keys()) tryTier(ranked.find(item => item.slug === slug), 'duplicate');
+
+  const { context, counts, tokens } = render();
+  return { context, counts, tokens, tiers: Object.fromEntries(tiers) };
+}

package/src/core/preflight-gate.mjs

@@ -6,7 +6,7 @@
  * Rules, in order:
  *   1. Tools that only display or discover (`meta`, `ui`) are always open.
  *   2. Reads of skill files are open: reading them is how the preflight is earned.
- *   3. A single plain Total Recall preflight, context, recall or help command is
+ *   3. A single plain Total Recall preflight, context, compact, recall or help command is
  *      open, so the gate can always be satisfied or repaired. Chaining, piping,
  *      substitution and redirects are not plain, so the exemption cannot carry
  *      other work and a capsule cannot be truncated through a pipe.
@@ -21,7 +21,7 @@ export const TOOL_CLASSES = ['shell', 'read', 'write', 'search', 'web', 'mcp', '
 export const SWEEP_CONSULT_MS = 15 * 60 * 1000;
 
 const SKILL_PATH = /\/\.(agent|claude)\/skills\//;
-const SATISFYING = /^(cd\s+\S+\s*&&\s*)?(npx\s+)?(total-recall|node\s+\S*bin\/total-recall\.mjs)\s+(tr-preflight|preflight|context|recall|help|--help)(\s+[^;&|`<>$\n]*)?$/;
+const SATISFYING = /^(cd\s+\S+\s*&&\s*)?(npx\s+)?(total-recall|node\s+\S*bin\/total-recall\.mjs)\s+(tr-preflight|preflight|context|compact|recall|help|--help)(\s+[^;&|`<>$\n]*)?$/;
 
 /** True when the whole command is one plain Total Recall invocation that satisfies or repairs the gate. */
 export function isSatisfyingCommand(command) {

package/src/core/preflight-receipt.mjs

@@ -98,7 +98,7 @@ export function acknowledge({ root, task = '', actions = 'read,edit,test', maxAg
   const context = run(['context', task || '(no task given)', '--action', actions], root);
   if (!/ready:true/.test(context.out)) {
     return { ok: false, problems: ['total-recall context did not return ready:true',
-      context.code === 2 ? 'exit 2: capsule overflow; raise --budget or curate applicability, then retry' : `context exit=${context.code ?? '?'}`] };
+      context.code === 2 ? 'exit 2: capsule overflow; the budget cannot hold every rule even compacted, so retry with the required_budget it reports' : `context exit=${context.code ?? '?'}`] };
   }
   const version = (context.out.match(/version:([0-9a-f]+)/) || [])[1] || null;
   const rules = run(['rules', 'verify'], root);

package/src/core/surface.mjs

@@ -427,7 +427,7 @@ Inspect all core capabilities, installed plugins, and commands.
 
 2. RUN TOTAL RECALL CONTEXT BEFORE ACTING:
 \`total-recall context "task description" --action <actions>\`
-Read the complete returned capsule and require ready:true. Unknown rule applicability is conservatively required. If it exits 2, stop: the required set exceeds its budget; inspect overflow and explicitly raise the budget or curate applicability before action.
+Read the returned capsule and require ready:true. Unknown rule applicability is conservatively required. Over budget it compacts automatically and every listed rule still applies: read a referenced rule with \`total-recall compact "task" --expand <slug>\` before relying on it. If it exits 2, retry with the required_budget it reports.
 
 Total Recall is the agent's working memory, toolset, and single source of truth across all decisions. Refresh at every changed task, action or project, and after a memory/skill edit. Do not infer applicability from semantic similarity. CLI retrieval is local and works without the server. Skills have compact entrypoints; load only relevant references. Canonical vault writes use the validated CLI.
 

package/src/server/invoke-durability.mjs

@@ -0,0 +1,39 @@
+/**
+ * The restricted write child runs under Node's permission model, which disables
+ * fsync. Durability is therefore the unconfined parent's job: before a write is
+ * reported as successful, it flushes every file the child changed and the
+ * directories that hold them.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+// File timestamps can round down, so a change made at the start of the run still counts.
+const MTIME_SLACK_MS = 2000;
+
+function flush(target) {
+  const fd = fs.openSync(target, 'r');
+  try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
+}
+
+/** Flush files under `roots` modified since `sinceMs`, then their directories, and return how many files were flushed. */
+export function syncChangedFiles(roots, sinceMs) {
+  let files = 0;
+  const walk = dir => {
+    let changed = false;
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (entry.isSymbolicLink()) continue;
+      const target = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (walk(target)) changed = true;
+      } else if (entry.isFile() && fs.statSync(target).mtimeMs >= sinceMs - MTIME_SLACK_MS) {
+        flush(target);
+        files++;
+        changed = true;
+      }
+    }
+    if (changed) flush(dir);
+    return changed;
+  };
+  for (const root of roots) if (fs.existsSync(root)) walk(root);
+  return files;
+}

package/src/server/invoke-write-child.mjs

@@ -1,4 +1,29 @@
 /** Structured trusted mutation entrypoint. Never accepts executable code. */
+import fs from 'node:fs';
+
+// Node.js Permission Model disables fsync/fsyncSync APIs, so this child cannot
+// flush its own writes. The unconfined parent flushes them before reporting
+// success (invoke-durability.mjs); without that step these writes would not survive a crash.
+if (fs.fsyncSync) {
+  const origFsyncSync = fs.fsyncSync;
+  fs.fsyncSync = (fd) => {
+    try { return origFsyncSync(fd); }
+    catch (err) {
+      if (err.message && err.message.includes('fsync API is disabled when Permission Model is enabled')) return;
+      throw err;
+    }
+  };
+}
+if (fs.fsync) {
+  const origFsync = fs.fsync;
+  fs.fsync = (fd, cb) => {
+    origFsync(fd, (err) => {
+      if (err && err.message && err.message.includes('fsync API is disabled when Permission Model is enabled')) return cb?.(null);
+      return cb?.(err);
+    });
+  };
+}
+
 import { createConfinedFact } from '../core/confined-memory.mjs';
 
 try {

package/src/server/routes/invoke.mjs

@@ -15,6 +15,7 @@ import { composedCommandDirectories } from '../../core/composed-command-resoluti
 import { invokeAuthority, authorizeInvoke, operationAllowed } from '../invoke-policy.mjs';
 import { restrictedBrainTargets } from '../invoke-targets.mjs';
 import { INVOKE_LIMITS, InvokeError, executeInvokeChild, requireSuccessfulRun, invokeEnvironment, absoluteTarget } from '../invoke-execution.mjs';
+import { syncChangedFiles } from '../invoke-durability.mjs';
 
 const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
 // Share across router instances in this server; do not race projection refreshes.
@@ -214,6 +215,7 @@ const HANDLERS = {
     const { category, title, content, tags, scope, timeoutMs } = params;
     if (context.authority.mode === 'restricted') {
       const brain = context.selectedBrain;
+      const startedAt = Date.now();
       const run = await context.run({ entry: path.join(root, 'src/server/invoke-write-child.mjs'),
         cwd: root, env: invokeEnvironment(false), timeoutMs, nodeArgs: ['--max-old-space-size=128', '--permission',
           `--allow-fs-read=${path.join(root, 'src')}`, `--allow-fs-read=${path.join(root, 'node_modules')}`,
@@ -231,6 +233,11 @@ const HANDLERS = {
           result || { committed: 'unknown', indexed: false });
       }
       if (!result?.committed || !result?.indexed) throw new InvokeError(502, 'invalid-owner-output', 'Mutation returned invalid output.', { committed: 'unknown', indexed: false });
+      try {
+        syncChangedFiles([path.join(brain, 'memory-vault'), path.join(brain, 'memory-derived')], startedAt);
+      } catch {
+        throw new InvokeError(502, 'durability-unconfirmed', 'Fact was written but could not be flushed to disk; check it before retrying.', result);
+      }
       return result;
     }
     if (!content || typeof content !== 'string') throw new Error('ssss.write needs content.');