total-recall-brain 3.33.0 → 3.33.1

package/docs/ARCHITECTURE.md

@@ -334,7 +334,13 @@ Connected instruction files contain a small routing bootstrap. Before action,
 Activation uses existing validated memory tags, not semantic inference or storage
 scope. Unknown applicability stays required; oversized required sets report
 not-ready status. Contributions are admitted whole under complete-render token
-estimates. User-authored text outside generated blocks remains intact.
+estimates. The 4,000-token default counts the complete serialized response. CLI
+output defaults to text; JSON, supporting knowledge and diagnostics are explicit
+options. A local `context:policy` decision can assign concise directives/actions
+to exact canonical source fingerprints. Invalid, conflicting, expired or stale
+entries fall back to the full original rules. Canonical bodies stay unchanged and
+private curation is excluded from scaffolds. Static surfaces use the same policy.
+User-authored text outside generated blocks remains intact.
 
 `recall` defaults to local retrieval. Compilation builds `local-search.json` as a
 disposable full-text posting/locator projection; matching documents are hydrated

package/docs/reference/cli-reference.md

@@ -90,22 +90,35 @@ Autonomously learn and save a new memory node to the vault.
 ---
 
 ### `context`
-Assemble required rules and supporting knowledge for the current task before action.
+Assemble applicable required rules for the current task before action.
 
 ```sh
-total-recall context "task description" --action edit,test --format json
+total-recall context "task description" --action edit,test
 ```
 
-The CLI defaults to 16,000 estimated tokens for conservative uncurated rule sets;
-`--budget 4000` selects a smaller explicit budget. The API defaults to 4,000.
+CLI and API default to 4,000 estimated tokens. The CLI defaults to compact text;
+`--format json` returns a compact machine-readable response. Supporting documents
+are opt-in (`--knowledge`; API `include_knowledge: true`). Diagnostic inventories
+require `--debug --format json` (API `debug: true`), within the same budget.
 Unknown applicability remains required. `ready: false` and exit 2 mean the full
-required set exceeds the budget; never act from a truncated capsule. All headings
-and separators count. Refresh on task/action/project/rule/skill changes.
+complete output exceeds the budget; never act from a truncated capsule. Status,
+metadata, JSON escaping, headings and separators all count. Refresh on task/action/project/rule/skill changes.
 Validated memory tags `context:universal` and `context:action:<action>` curate
 activation separately from modality. Unknown actions activate rules conservatively.
 Supported actions: read, edit, test, build, publish, deploy, secrets, network,
 memory, skills and project. The local CLI works without a server.
 
+For explicit local curation, save one project `decision` through the CLI with
+tag `context:policy`. Its JSON body contains `rules`, keyed by `project:<slug>`
+or `global:<slug>`. Each entry has `source_hash` (from debug `curation_sources`),
+`actions` (supported actions or `universal`) and a manually verified `directive`.
+Keep every operative constraint; move history and examples to the original,
+retrievable rule. A deliberate local exclusion uses `enabled: false` and a
+nonempty `reason`. Original bodies and global rules remain intact.
+Malformed, conflicting, expired or source-mismatched curation falls back to the
+canonical rule. Never generate applicability from semantic ranking. Curation
+is private runtime state and must not be included in scaffolds or releases.
+
 ### `recall`
 Search local metadata and indexed full text across enabled memory layers.
 - **What it does**: Searches a disposable local index and reads only selected Markdown documents. Semantic enrichment is opt-in and exact hits bypass providers.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total-recall-brain",
-  "version": "3.33.0",
+  "version": "3.33.1",
   "description": "Personal AI system — database-free, SSSS-powered intelligence engine deployable to any POSIX host.",
   "type": "module",
   "workspaces": [

package/src/cli/context.mjs

@@ -2,20 +2,23 @@ import path from 'node:path';
 import { getBothBrains, parseLayerFlag } from './agent-dir.mjs';
 import { getNodes } from '../core/vault-cache.mjs';
 import { mergeGlobalRuleNodes, legacyRuleContributions } from '../core/surface.mjs';
-import { compileContext } from '../core/context-compiler.mjs';
+import { compileContext, capsuleResponse, renderCapsule } from '../core/context-compiler.mjs';
 import { isBrainEnabled } from '../core/brain-registry.mjs';
 import { surfaceInputsHash } from '../core/command-surface.mjs';
 
 export default async function context(args = []) {
   if (!args.length || args.includes('--help')) {
-    console.log('Usage: total-recall context "task" [--action edit,test,publish] [--budget 16000] [--format json|text] [--project|--global]\nLocal only. Unknown applicability is required. A required-set overflow exits 2: do not act until a complete capsule fits. Refresh on task/action/project or policy version changes.');
+    console.log('Usage: total-recall context "task" [--action edit,test,publish] [--budget 4000] [--format text|json] [--knowledge] [--debug] [--project|--global]\nLocal rules only by default. Unknown applicability is required. Overflow exits 2; do not act until the complete output fits. --knowledge adds supporting documents; --debug exposes inventories within the same budget. Refresh on task/action/project or policy changes.');
     return;
   }
   const { layer, remainingArgs } = parseLayerFlag(args);
   const query = remainingArgs[0];
-  let actions = [], budget = 16000, format = 'json';
+  let actions = [], budget = 4000, format = 'text', includeKnowledge = false, debug = false;
   for (let i = 1; i < remainingArgs.length; i++) {
-    const flag = remainingArgs[i], value = remainingArgs[++i];
+    const flag = remainingArgs[i];
+    if (flag === '--knowledge') { includeKnowledge = true; continue; }
+    if (flag === '--debug') { debug = true; continue; }
+    const value = remainingArgs[++i];
     if (flag === '--action') actions = value?.split(',').filter(Boolean) || [];
     else if (flag === '--budget') budget = Number(value);
     else if (flag === '--format' && ['json', 'text'].includes(value)) format = value;
@@ -28,8 +31,10 @@ export default async function context(args = []) {
   const nodes = own ? mergeGlobalRuleNodes(getNodes(path.join(own, 'memory-vault')), global ? getNodes(path.join(global, 'memory-vault')) : []) :
     getNodes(path.join(global, 'memory-vault')).map(n => ({ ...n, _layer: 'global' }));
   const skillsDir = path.dirname(own || global);
-  const result = await compileContext({ query, actions, nodes, budget: { total: budget }, projectRoot: process.cwd(),
+  const result = await compileContext({ query, actions, nodes, includeKnowledge, budget: { total: budget }, projectRoot: process.cwd(),
     contributions: legacyRuleContributions(skillsDir), versionInputs: surfaceInputsHash({ skillsDir }) });
-  console.log(format === 'json' ? JSON.stringify(result, null, 2) : `${result.ready ? 'READY' : 'NOT READY: required instructions exceed budget'}\n${result.context}`);
-  if (!result.ready) process.exitCode = 2;
+  if (debug && format === 'text') throw new Error('--debug requires --format json');
+  const response = capsuleResponse(result, { total: budget, debug, format });
+  console.log(renderCapsule(response, format));
+  if (!response.ready) process.exitCode = 2;
 }

package/src/core/context-compiler.mjs

@@ -2,7 +2,7 @@
 
 import { getNodes } from './vault-cache.mjs';
 import crypto from 'node:crypto';
-import { selectRules, assembleContext } from './context-policy.mjs';
+import { selectRules, assembleContext, curatedRules, ruleFingerprint, RULE_CATEGORIES } from './context-policy.mjs';
 import {
   getEmbedding,
   cosineSimilarity,
@@ -69,21 +69,21 @@ function temporalScore(node) {
 export async function compileContext({
   query = '', vaultDir, derivedDir, budget = {}, consumer = 'api',
   momentumSlugs = [], actions = [], projectRoot = process.cwd(), semantic = false,
-  nodes, contributions = [], versionInputs = '',
+  nodes, contributions = [], versionInputs = '', includeKnowledge = false,
 } = {}) {
   const startTime = Date.now();
   const b = { ...DEFAULT_BUDGET, total: 4000, ...budget };
   const allNodes = (nodes || getNodes(vaultDir)).map(n => ({ ...n }));
-  const rules = selectRules(allNodes, { actions, projectRoot });
+  const rules = selectRules(curatedRules(allNodes), { actions, projectRoot });
   const required = rules.map(n => ({ id: `${n._layer || 'project'}:${n.slug}`, required: true,
-    text: `## ${n.title || n.slug} [${n.slug}]\n\n${(n.body || n.content || '').trim()}` }));
+    text: n._directive ? `[${n.slug}] ${n._directive}` : `## ${n.title || n.slug} [${n.slug}]\n\n${(n.body || n.content || '').trim()}` }));
   let queryEmbedding = null, embeddingsIndex = {};
-  if (semantic && query) {
+  if (includeKnowledge && semantic && query) {
     try { queryEmbedding = await getEmbedding(String(query)); embeddingsIndex = loadEmbeddingsIndex(derivedDir); } catch { /* local ranking remains usable */ }
   }
   const terms = String(query).toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || [];
   const selectedIds = new Set(rules.map(n => n.slug));
-  const candidates = allNodes.filter(n => n.status === 'active' && !selectedIds.has(n.slug) &&
+  const candidates = (includeKnowledge ? allNodes : []).filter(n => n.status === 'active' && !n.tags?.includes('context:policy') && !selectedIds.has(n.slug) &&
     !['invariants', 'preferences', 'anti-patterns'].includes(n.category)).map(n => {
     const text = `${n.title || ''} ${n.body || n.content || ''}`.toLowerCase();
     const overlap = terms.filter(term => text.includes(term)).length;
@@ -98,10 +98,13 @@ export async function compileContext({
   result.stats.compile_ms = Date.now() - startTime;
   result.stats.consumer = consumer;
   result.stats.actions = actions;
+  result.stats.curation_sources = allNodes.filter(n => RULE_CATEGORIES.has(n.category)).map(n => ({
+    id: `${n._layer || 'project'}:${n.slug}`, source_hash: ruleFingerprint(n),
+  }));
   result.stats.unresolved_contradictions = rules.flatMap(node => (node.contradicts || [])
     .filter(slug => rules.some(other => other.slug === slug)).map(slug => [node.slug, slug]));
   result.stats.version = crypto.createHash('sha256').update(JSON.stringify({
-    policy: 1, projectRoot, actions, versionInputs, nodes: allNodes, capsule: result.stats.version,
+    policy: 2, projectRoot, actions, includeKnowledge, versionInputs, nodes: allNodes, capsule: result.stats.version,
   })).digest('hex');
   result.stats.mode = queryEmbedding ? 'semantic' : 'local';
   result.stats.slots = { invariants: { nodes: rules.filter(n => n.category === 'invariants').length,
@@ -110,6 +113,35 @@ export async function compileContext({
   return result;
 }
 
+export function renderCapsule(result, format = 'json') {
+  return format === 'text' ? `ready:${result.ready} tokens:${result.stats.total_tokens} version:${result.stats.version}\n${result.context}` : JSON.stringify(result);
+}
+
+/** Budget the actual wire representation, including metadata, escaping and status. */
+export function capsuleResponse(result, { total = 4000, debug = false, format = 'json' } = {}) {
+  if (!Number.isFinite(total) || total < 1) throw new Error('Context total must be a positive finite token estimate');
+  const stats = debug ? { ...result.stats } : {
+    version: result.stats.version, mode: result.stats.mode,
+    token_measurement: 'estimated_chars_divided_by_four',
+    required_count: result.stats.required_ids.length, body_tokens: estimateTokens(result.context),
+  };
+  const response = { ready: result.ready, context: result.context, stats };
+  stats.budget = total;
+  // Counts affect serialization themselves. Iterate to a stable representation.
+  for (let i = 0; i < 20; i++) {
+    const previous = renderCapsule(response, format);
+    stats.total_tokens = estimateTokens(previous + '\n');
+    if (debug) {
+      stats.budget_used = stats.total_tokens;
+      stats.budget_remaining = Math.max(0, total - stats.total_tokens);
+    }
+    stats.overflow_tokens = Math.max(0, stats.total_tokens - total);
+    response.ready = result.ready && stats.overflow_tokens === 0;
+    if (renderCapsule(response, format) === previous) return response;
+  }
+  throw new Error('Context output accounting did not converge');
+}
+
 /**
  * Lightweight variant that returns pre-scored candidates without embedding lookup.
  * Used for fast context previews and budget estimation.

package/src/core/context-policy.mjs

@@ -5,13 +5,40 @@ export const RULE_CATEGORIES = new Set(['invariants', 'preferences', 'anti-patte
 export const CONTEXT_ACTIONS = new Set(['read', 'edit', 'test', 'build', 'publish', 'deploy', 'secrets', 'network', 'memory', 'skills', 'project']);
 export const estimateTokens = text => Math.ceil(String(text || '').length / 4);
 
+/** Curation is local, explicit and invalidated by any canonical rule change. */
+export const ruleFingerprint = node => crypto.createHash('sha256').update(JSON.stringify(
+  Object.fromEntries(Object.entries(node).filter(([key]) => !key.startsWith('_')).sort(([a], [b]) => a.localeCompare(b))),
+)).digest('hex');
+
+export function curatedRules(nodes, now = Date.now()) {
+  const policies = nodes.filter(n => n._layer !== 'global' && n.category === 'decisions' && n.status === 'active' &&
+    !n.superseded_by && !(n.expires_at && Date.parse(n.expires_at) <= now) && n.tags?.includes('context:policy'));
+  // Multiple policies are ambiguous, never pick a winner by order or relevance.
+  if (policies.length !== 1) return nodes;
+  let rules;
+  try { rules = JSON.parse(policies[0].body || '').rules; } catch { return nodes; }
+  if (!rules || typeof rules !== 'object' || Array.isArray(rules)) return nodes;
+  return nodes.map(node => {
+    const entry = rules[`${node._layer || 'project'}:${node.slug}`];
+    if (!RULE_CATEGORIES.has(node.category) || !entry || entry.source_hash !== ruleFingerprint(node)) return node;
+    if (entry.enabled === false && typeof entry.reason === 'string' && entry.reason.trim()) return { ...node, _contextExcluded: true };
+    if (entry.enabled !== undefined && entry.enabled !== true) return node;
+    if (
+      !Array.isArray(entry.actions) || !entry.actions.length || entry.actions.some(a => a !== 'universal' && !CONTEXT_ACTIONS.has(a)) ||
+      typeof entry.directive !== 'string' || !entry.directive.trim()) return node;
+    const tags = (node.tags || []).filter(t => !t.startsWith('context:action:') && t !== 'context:universal');
+    return { ...node, tags: [...tags, ...entry.actions.map(a => a === 'universal' ? 'context:universal' : `context:action:${a}`)],
+      _directive: entry.directive.trim() };
+  });
+}
+
 /** Applicability is explicit validated memory tags, independent of obligation.
  * Unknown rules remain required in task capsules. No inference from prose.
  */
 export function selectRules(nodes, { actions = [], projectRoot = process.cwd(), bootstrap = false, now = Date.now() } = {}) {
   const repo = path.basename(projectRoot).toLowerCase();
   const applicable = nodes.filter(node => {
-    if (!RULE_CATEGORIES.has(node.category) || node.status !== 'active' || node.superseded_by) return false;
+    if (!RULE_CATEGORIES.has(node.category) || node.status !== 'active' || node.superseded_by || node._contextExcluded) return false;
     if (node.expires_at && Date.parse(node.expires_at) <= now) return false;
     if (node.repos?.length && !node.repos.some(r => String(r).toLowerCase() === repo)) return false;
     if (node.project && String(node.project).toLowerCase() !== repo) return false;

package/src/core/surface.mjs

@@ -18,7 +18,7 @@ import { brainDir as globalBrainDir, globalAgentDir } from './config.mjs';
 import { listSurfaceCommands, buildCommandsSection, surfaceInputsHash } from './command-surface.mjs';
 import { fileURLToPath } from 'url';
 import { buildLocalSearchIndex } from './fast-recall.mjs';
-import { selectRules, assembleContext } from './context-policy.mjs';
+import { selectRules, assembleContext, curatedRules } from './context-policy.mjs';
 
 // Long-lived processes (server, daemon, vault watcher) import this module once.
 // If the rule builder is edited or upgraded after they start, their in-memory
@@ -395,14 +395,14 @@ export async function buildRulesBlock(skillsDir, nodes = [], {
   consumer = 'ide', derivedDir, vaultDir, projectRoot, actions = [], bootstrap = false, total = bootstrap ? 1000 : 4000,
 } = {}) {
   const root = projectRoot || (skillsDir ? path.dirname(path.dirname(skillsDir)) : process.cwd());
-  const rules = selectRules(nodes, { actions, projectRoot: root, bootstrap });
+  const rules = selectRules(curatedRules(nodes), { actions, projectRoot: root, bootstrap });
   const contributions = [];
   const counts = category => rules.filter(n => n.category === category).length;
   const header = bootstrap ? `## Total Recall task routing\n\nBefore taking action, run \`total-recall context "task description" --action <actions>\` using the installed CLI (in a source checkout: \`node bin/total-recall.mjs context ...\`). Read the complete returned capsule and require ready:true. Unknown rule applicability is conservatively required. If it exits 2, stop: the required set exceeds its budget; inspect overflow and explicitly raise the budget or curate applicability before action.\n\nRefresh at every changed task, action or project, and after a memory/skill edit. Do not infer applicability from semantic similarity. CLI retrieval is local and works without the server. Skills have compact entrypoints; load only relevant references. Canonical vault writes use the validated CLI.\n\nDo not act from this bootstrap alone.\n` :
     `## Active Rules: ${counts('invariants')} invariants, ${counts('preferences')} preferences, ${counts('anti-patterns')} corrections`;
   contributions.push({ id: 'routing', text: header, required: true });
   for (const node of rules) contributions.push({ id: `${node._layer || 'project'}:${node.slug}`, required: true,
-    text: `### ${node.title || node.slug} [${node.slug}]\n\n${node.body || node.content || ''}` });
+    text: node._directive ? `[${node.slug}] ${node._directive}` : `### ${node.title || node.slug} [${node.slug}]\n\n${node.body || node.content || ''}` });
   if (!bootstrap) {
     if (skillsDir) {
       contributions.push(...legacyRuleContributions(skillsDir));

package/src/server/routes/context.mjs

@@ -16,13 +16,15 @@ const router = Router();
 
 router.post('/api/context', requireAuth, requireScope('memory:read'), async (req, res) => {
   try {
-    const { compileContext } = await import('../../core/context-compiler.mjs');
-    const { query, budget, momentum_slugs, actions } = req.body || {};
+    const { compileContext, capsuleResponse } = await import('../../core/context-compiler.mjs');
+    const { query, budget, momentum_slugs, actions, include_knowledge, debug } = req.body || {};
+    if (include_knowledge !== undefined && typeof include_knowledge !== 'boolean') return badRequest(res, 'include_knowledge must be boolean');
+    if (debug !== undefined && typeof debug !== 'boolean') return badRequest(res, 'debug must be boolean');
     if (actions !== undefined && (!Array.isArray(actions) || actions.some(action => typeof action !== 'string'))) return badRequest(res, 'actions must be an array of strings');
     const vaultDir = resolveVaultFromQuery(req, { strict: true });
     const { derivedDir, skillsDir } = pathsForVault(vaultDir);
     const globalVault = path.join(globalBrainDir, 'memory-vault');
-    const nodes = path.resolve(vaultDir) === path.resolve(globalVault) ? getNodes(vaultDir) :
+    const nodes = path.resolve(vaultDir) === path.resolve(globalVault) ? getNodes(vaultDir).map(n => ({ ...n, _layer: 'global' })) :
       mergeGlobalRuleNodes(getNodes(vaultDir), getNodes(globalVault));
     const result = await compileContext({
       query: query || '',
@@ -32,12 +34,13 @@ router.post('/api/context', requireAuth, requireScope('memory:read'), async (req
       consumer: 'api',
       momentumSlugs: momentum_slugs || [],
       actions: actions || [],
+      includeKnowledge: include_knowledge === true,
       nodes,
       projectRoot: path.dirname(path.dirname(skillsDir)),
       contributions: legacyRuleContributions(skillsDir),
       versionInputs: surfaceInputsHash({ skillsDir }),
     });
-    res.json(result);
+    res.json(capsuleResponse(result, { total: budget?.total ?? 4000, debug: debug === true }));
   } catch (err) {
     serverError(res, err);
   }