total-recall-brain 3.2.3 → 3.4.0

package/src/cli/share.mjs

@@ -1,6 +1,7 @@
 import crypto from 'node:crypto';
 import { parseLayerFlag, resolveBrainDir, getBothBrains, defaultLayerForCategory } from './agent-dir.mjs';
-import { writeNode, createMemoryNode } from '../core/vault.mjs';
+import { createMemoryNode } from '../core/vault.mjs';
+import { writeNodeValidated } from '../core/validated-write.mjs';
 import { addToQueue } from '../core/research-queue.mjs';
 import { compileSurface } from '../core/surface.mjs';
 import path from 'node:path';
@@ -160,7 +161,11 @@ export default async function share(args) {
     }];
   }
 
-  writeNode(node, vaultDir);
+  const vaultResult = writeNodeValidated(node, vaultDir);
+  if (!vaultResult.success) {
+    console.error(`  ❌ Validation failed: ${vaultResult.validation.errors.join('; ')}`);
+    process.exit(1);
+  }
   const layerLabel = layer === 'project' ? '[project]' : '[global]';
   console.log(`  ✅ Saved as ${category || 'facts'} ${layerLabel}: ${category || 'facts'}/${slug}.md`);
 

package/src/core/append-log.mjs

@@ -0,0 +1,131 @@
+/**
+ * Append-only JSONL log with last-write-wins semantics and compaction.
+ *
+ * Write: O(1) — fs.appendFileSync one line
+ * Read: O(n) on cold load, cached after first read
+ * Compact: Rewrites only latest version of each key
+ *
+ * File format: One JSON object per line, each with a mandatory `_key` field.
+ * Later entries for the same `_key` shadow earlier ones.
+ * Tombstones: { "_key": "slug", "_deleted": true }
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { atomicWrite } from './vault.mjs';
+import { logger } from './logger.mjs';
+
+/** @type {Map<string, AppendLog>} Registry of all active AppendLog instances */
+const activeAppendLogs = new Map();
+
+export class AppendLog {
+  #filePath;
+  #cache; // Map<string, object> — latest value per key
+  #dirty; // number of appended lines since last compact
+  #compactThreshold;
+
+  constructor(filePath, { compactThreshold = 500 } = {}) {
+    this.#filePath = filePath;
+    this.#cache = null;
+    this.#dirty = 0;
+    this.#compactThreshold = compactThreshold;
+    activeAppendLogs.set(filePath, this);
+  }
+
+  /** Read the full index. Returns Map<key, value>. Cached after first call. */
+  entries() {
+    if (this.#cache) return this.#cache;
+    this.#cache = new Map();
+    if (!fs.existsSync(this.#filePath)) return this.#cache;
+    const raw = fs.readFileSync(this.#filePath, 'utf8');
+    for (const line of raw.split('\n')) {
+      if (!line.trim()) continue;
+      try {
+        const entry = JSON.parse(line);
+        if (entry._deleted) this.#cache.delete(entry._key);
+        else this.#cache.set(entry._key, entry);
+      } catch { /* skip corrupt lines */ }
+    }
+    return this.#cache;
+  }
+
+  /** Get a single entry by key. */
+  get(key) {
+    return this.entries().get(key) ?? null;
+  }
+
+  /** Append a single entry. O(1) I/O. */
+  append(key, value) {
+    const dir = path.dirname(this.#filePath);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    const entry = { _key: key, ...value };
+    fs.appendFileSync(this.#filePath, JSON.stringify(entry) + '\n');
+    // Update in-memory cache
+    this.entries().set(key, entry);
+    this.#dirty++;
+    if (this.#dirty >= this.#compactThreshold) {
+      this.compact();
+    }
+  }
+
+  /** Write a tombstone for a key. O(1) I/O. */
+  remove(key) {
+    const dir = path.dirname(this.#filePath);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    fs.appendFileSync(this.#filePath, JSON.stringify({ _key: key, _deleted: true }) + '\n');
+    this.entries().delete(key);
+    this.#dirty++;
+  }
+
+  /** Number of live entries. */
+  get size() {
+    return this.entries().size;
+  }
+
+  /** Convert to plain object (for JSON.stringify compatibility). */
+  toObject() {
+    const obj = {};
+    for (const [key, value] of this.entries()) {
+      const { _key, ...rest } = value;
+      obj[key] = rest;
+    }
+    return obj;
+  }
+
+  /** Compact: rewrite file with only latest versions. */
+  compact() {
+    const entries = this.entries();
+    const lines = [];
+    for (const [, value] of entries) {
+      lines.push(JSON.stringify(value));
+    }
+    atomicWrite(this.#filePath, lines.join('\n') + '\n');
+    this.#dirty = 0;
+    logger.info('append-log', `Compacted ${this.#filePath}: ${lines.length} entries`);
+  }
+
+  /** Invalidate the in-memory cache (force reload on next read). */
+  invalidate() {
+    this.#cache = null;
+    this.#dirty = 0;
+  }
+}
+
+/**
+ * Compact all active AppendLog instances.
+ * Returns summary with number of logs compacted and total entries.
+ */
+export function compactAppendLogs() {
+  let logsCompacted = 0;
+  let totalEntries = 0;
+  for (const [filePath, log] of activeAppendLogs) {
+    try {
+      log.compact();
+      logsCompacted++;
+      totalEntries += log.size;
+    } catch (err) {
+      logger.warn('append-log', `Failed to compact ${filePath}: ${err.message}`);
+    }
+  }
+  return { logs_compacted: logsCompacted, total_entries: totalEntries };
+}

package/src/core/clarity-rewriter.mjs

@@ -3,7 +3,8 @@ import path from 'path';
 import matter from 'gray-matter';
 import crypto from 'crypto';
 import { callLocalRuntime, cleanAndParseJSON } from './runtime.mjs';
-import { loadNodes, atomicWrite, safeStringify } from './vault.mjs';
+import { atomicWrite, safeStringify } from './vault.mjs';
+import { getNodes, invalidate } from './vault-cache.mjs';
 import { logger } from './logger.mjs';
 
 /**
@@ -69,7 +70,7 @@ function buildClarityPrompt(node) {
  * @returns {{ rewrote: boolean, qualityScore?: number, error?: string }}
  */
 export async function runClarityReview(slug, { vaultDir, inboxDir, runtimeConfig }) {
-  const allNodes = loadNodes(vaultDir);
+  const allNodes = getNodes(vaultDir);
   const node = allNodes.find(n => n.slug === slug);
 
   if (!node) {
@@ -192,7 +193,7 @@ function buildStalenessPrompt(node) {
  * @returns {{ verdict: string, confidence: number, error?: string }}
  */
 export async function runStalenessCheck(slug, { vaultDir, queueDir, runtimeConfig }) {
-  const allNodes = loadNodes(vaultDir);
+  const allNodes = getNodes(vaultDir);
   const node = allNodes.find(n => n.slug === slug);
 
   if (!node) {
@@ -304,7 +305,7 @@ Output ONLY valid JSON. Maximum 5 gaps.`;
  * @returns {{ gaps: object[], error?: string }}
  */
 export async function runFactSeeker({ vaultDir, queueDir, runtimeConfig }) {
-  const allNodes = loadNodes(vaultDir);
+  const allNodes = getNodes(vaultDir);
   const activeNodes = allNodes.filter(n => n.status === 'active');
 
   if (activeNodes.length === 0) {
@@ -446,7 +447,7 @@ function isHighDriftNode(node) {
  * @returns {{ audited: number, flagged: number, critical: number }}
  */
 export async function runCutoffAudit({ vaultDir, queueDir, runtimeConfig }) {
-  const allNodes = loadNodes(vaultDir);
+  const allNodes = getNodes(vaultDir);
   const now = Date.now();
   const verifiedCutoff = 30 * 24 * 60 * 60 * 1000; // 30 days
 
@@ -574,6 +575,7 @@ async function stampVerified(node) {
     data.x_cutoff_verified_at = new Date().toISOString();
     data.x_cutoff_risk = 'none';
     atomicWrite(node._filepath, safeStringify(content, data));
+    invalidate(path.dirname(node._filepath));
   } catch (err) {
     logger.debug('clarity-rewriter: stampCutoffRiskNone failed', { err: err.message });
   }
@@ -591,6 +593,7 @@ async function stampCutoffRisk(node, riskLevel, claimCount) {
     const confidencePenalty = riskLevel === 'critical' ? 0.3 : riskLevel === 'high' ? 0.2 : 0.1;
     data.confidence = Math.max(0.1, (data.confidence || 0.5) - confidencePenalty);
     atomicWrite(node._filepath, safeStringify(content, data));
+    invalidate(path.dirname(node._filepath));
   } catch (err) {
     logger.debug('clarity-rewriter: stampCutoffRisk failed', { err: err.message });
   }
@@ -635,7 +638,7 @@ Output ONLY valid JSON.`;
  * @returns {{ correctionSlug: string, verdict: string, error?: string }}
  */
 export async function writeCorrection(originalSlug, researchEvidence, { vaultDir, inboxDir, runtimeConfig }) {
-  const allNodes = loadNodes(vaultDir);
+  const allNodes = getNodes(vaultDir);
   const original = allNodes.find(n => n.slug === originalSlug);
 
   if (!original) {
@@ -764,6 +767,7 @@ async function stampNodeForCorrection(node, correctionSlug, disposition) {
     data.confidence = Math.max(0.05, (data.confidence || 0.5) * 0.3);
     data.updated = new Date().toISOString();
     atomicWrite(node._filepath, safeStringify(content, data));
+    invalidate(path.dirname(node._filepath));
   } catch (err) {
     logger.debug('clarity-rewriter: stampNodeForCorrection failed', { err: err.message });
   }

package/src/core/conclusion-writer.mjs

@@ -3,6 +3,7 @@ import path from 'path';
 import matter from 'gray-matter';
 import { callLocalRuntime } from './runtime.mjs';
 import { atomicWrite, safeStringify } from './vault.mjs';
+import { getNodes, invalidate } from './vault-cache.mjs';
 import { logger } from './logger.mjs';
 
 /**
@@ -164,6 +165,7 @@ async function promoteToVault(draftNode, confidenceAdj, vaultDir) {
 
   const targetPath = path.join(targetDir, `${data.slug}.md`);
   atomicWrite(targetPath, safeStringify(content, data));
+  invalidate(vaultDir);
 
   // Remove from inbox
   try { fs.unlinkSync(draftNode._filepath); } catch { /* ignore */ }
@@ -222,8 +224,7 @@ export async function runConclusionWriter({ inboxDir, vaultDir, quarantineDir, r
   }
 
   // Load all active vault nodes for comparison
-  const { loadNodes } = await import('./vault.mjs');
-  const allActiveNodes = loadNodes(vaultDir).filter(n => n.status === 'active');
+  const allActiveNodes = getNodes(vaultDir).filter(n => n.status === 'active');
 
   let approved = 0;
   let rejected = 0;

package/src/core/config.mjs

@@ -96,6 +96,12 @@ if (!process.env._TR_TEST_AGENT_DIR) {
 let secrets = {};
 try {
   const pathsToCheck = [];
+  // 1. Prioritize workspace-local root secrets
+  pathsToCheck.push(path.join(process.cwd(), '.agent', 'secrets.enc'));
+  if (resolvedAgentDir) {
+    pathsToCheck.push(path.join(resolvedAgentDir, 'secrets.enc'));
+  }
+  // 2. Fallbacks
   if (projectBrainDir) {
     pathsToCheck.push(path.join(projectBrainDir, 'config', 'secrets.enc'));
   }

package/src/core/conflict-detector.mjs

@@ -2,7 +2,8 @@ import fs from 'fs';
 import path from 'path';
 import matter from 'gray-matter';
 import crypto from 'node:crypto';
-import { atomicWrite, walkMd, loadNodes, writeNode, safeStringify } from './vault.mjs';
+import { atomicWrite, walkMd, writeNode, safeStringify } from './vault.mjs';
+import { getNodes, invalidate } from './vault-cache.mjs';
 import { logger } from './logger.mjs';
 
 /**
@@ -158,7 +159,7 @@ export function detectSemanticConflicts(candidate, existingNodes, thresholds = {
  * @returns {object[]} Array of conflict records (deduplicated by pair).
  */
 export function scanVaultForConflicts(vaultDir, thresholds = {}) {
-  const nodes = loadNodes(vaultDir).filter(n => n.status === 'active');
+  const nodes = getNodes(vaultDir).filter(n => n.status === 'active');
   const seen = new Set();
   const conflicts = [];
 
@@ -420,6 +421,7 @@ export function applyAutoResolution(conflict, resolution, vaultDir) {
     try {
       writeNode(loserNode, vaultDir);
       writeNode(winnerNode, vaultDir);
+      invalidate(vaultDir);
     } catch (err) {
       logger.error('conflict-detector', `Failed to apply auto-resolution: ${err.message}`);
     }