shieldcortex 4.35.0 → 4.37.0

package/scripts/lib/recall-defence.mjs

@@ -0,0 +1,254 @@
+/**
+ * Recall-boundary defence shim (Feature #1).
+ *
+ * The read hooks (prompt-recall, session-start) used to inject recalled memory
+ * VERBATIM into the model prompt with no defence. This module sits between the
+ * SQL SELECT and the formatter: it filters rows by trust/sensitivity (via the
+ * dead-no-more filterByTrust) and re-scans surviving content for injection /
+ * credentials / encoded payloads, withholding (not deleting) anything bad so a
+ * poisoned or RESTRICTED row never reaches the model.
+ *
+ * `defendRecallRows` is PURE + dependency-injected so it unit-tests with no dist
+ * build and no DB. `loadRecallDefence` / `emitRecallAudit` (below) wire the real
+ * dist modules for the hooks and are exercised by the integration test.
+ */
+
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const RESTRICTED_MARKER = '[REDACTED - RESTRICTED]';
+
+let _recallDefenceCache = null;
+let _recallDefenceCacheKey = null;
+
+/**
+ * Lazy-load the built dist defence modules the read hooks need. Returns null if
+ * the dist build is missing/incomplete → the caller MUST fail OPEN (leave recall
+ * unchanged), because blanking recall in an un-built dev workspace would break
+ * the product. Imports only LEAF detector modules (never dist/defence/pipeline.js)
+ * to stay inside the hook's <500ms budget; caches across invocations.
+ *
+ * @param {string} [distRootOverride] test seam — point at an empty dir to assert fail-open.
+ */
+export async function loadRecallDefence(distRootOverride) {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const distRoot = distRootOverride ?? resolve(here, '..', '..', 'dist');
+  if (_recallDefenceCache && _recallDefenceCacheKey === distRoot) return _recallDefenceCache;
+
+  try {
+    const [trustMod, firewallMod, credMod, auditMod, initMod, sanitiseMod] = await Promise.all([
+      import(pathToFileURL(resolve(distRoot, 'defence', 'trust', 'recall-filter.js')).href),
+      import(pathToFileURL(resolve(distRoot, 'defence', 'firewall', 'index.js')).href),
+      import(pathToFileURL(resolve(distRoot, 'defence', 'credential-leak', 'index.js')).href),
+      import(pathToFileURL(resolve(distRoot, 'defence', 'audit', 'logger.js')).href),
+      import(pathToFileURL(resolve(distRoot, 'database', 'init.js')).href),
+      import(pathToFileURL(resolve(distRoot, 'defence', 'input-sanitisation', 'index.js')).href).catch(() => ({})),
+    ]);
+
+    if (
+      typeof trustMod.filterByTrust !== 'function' ||
+      typeof firewallMod.detectInstructions !== 'function' ||
+      typeof firewallMod.detectEncoding !== 'function' ||
+      typeof credMod.scanForCredentials !== 'function' ||
+      typeof auditMod.logAudit !== 'function' ||
+      typeof initMod.initDatabase !== 'function'
+    ) {
+      return null;
+    }
+
+    _recallDefenceCache = {
+      filterByTrust: trustMod.filterByTrust,
+      // Optional — strips zero-width/RTL/control bytes before scanning so a
+      // hidden injection can't dodge the regex detectors; defendRecallRows guards.
+      sanitiseInput: sanitiseMod.sanitiseInput,
+      detectInstructions: firewallMod.detectInstructions,
+      detectEncoding: firewallMod.detectEncoding,
+      // Optional — older dist builds may not export it; defendRecallRows guards.
+      detectMarkdownImageExfil: firewallMod.detectMarkdownImageExfil,
+      scanForCredentials: credMod.scanForCredentials,
+      logAudit: auditMod.logAudit,
+      initDatabase: initMod.initDatabase,
+      isDatabaseInitialized: initMod.isDatabaseInitialized,
+      getDatabase: initMod.getDatabase,
+      closeDatabase: initMod.closeDatabase,
+    };
+    _recallDefenceCacheKey = distRoot;
+    return _recallDefenceCache;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Point the dist DB singleton (used by logAudit) at the hook's DB so withhold
+ * audit rows are visible across connections. Only needed when ≥1 row is withheld
+ * — keep it out of the common all-clear path. Best-effort.
+ */
+export function ensureRecallAuditDb(defence, dbPath) {
+  if (!defence || !dbPath || dbPath === ':memory:') return;
+  try {
+    if (defence.isDatabaseInitialized && defence.isDatabaseInitialized()) {
+      const current = defence.getDatabase();
+      if (current && current.name === dbPath) return;
+      if (defence.closeDatabase) defence.closeDatabase();
+    }
+    defence.initDatabase(dbPath);
+  } catch {
+    // logAudit no-ops gracefully if the singleton isn't initialised.
+  }
+}
+
+/**
+ * Write a defence_audit row recording a withheld/redacted recall. Best-effort —
+ * never throws into the hook (a recall must not fail because audit failed).
+ */
+export function emitRecallAudit(logAudit, { memoryId, action, layer, reason, project } = {}) {
+  try {
+    logAudit({
+      memory_id: typeof memoryId === 'number' ? memoryId : null,
+      project: project ?? null,
+      timestamp: new Date().toISOString(),
+      source_type: 'hook',
+      source_identifier: 'recall-defence',
+      trust_score: 0,
+      sensitivity_level: 'INTERNAL',
+      // No 'READ' firewall result exists; encode the withhold as BLOCK (dropped)
+      // / QUARANTINE (redacted) with the detail in `reason`.
+      firewall_result: action === 'redacted' ? 'QUARANTINE' : 'BLOCK',
+      anomaly_score: 0,
+      threat_indicators: JSON.stringify([`recall:${layer ?? 'unknown'}`]),
+      blocked_patterns: '[]',
+      reason: `recall-withheld: ${reason ?? layer ?? 'policy'}`,
+      fragmentation_score: null,
+      pipeline_duration_ms: 0,
+    });
+  } catch {
+    // best-effort
+  }
+}
+
+function parseMetadata(meta) {
+  if (meta == null) return {};
+  if (typeof meta === 'object') return meta;
+  if (typeof meta === 'string') {
+    try {
+      return JSON.parse(meta);
+    } catch {
+      return {};
+    }
+  }
+  return {};
+}
+
+/**
+ * Filter recalled rows through the trust/sensitivity + content defence layers.
+ *
+ * @param {Array<object>} rows  raw recalled rows (better-sqlite3 rows — NOT mutated)
+ * @param {{ minTrust?: number, project?: string, reviewedPinnedBypass?: boolean }} opts
+ * @param {{ filterByTrust, detectInstructions, scanForCredentials, detectEncoding }} deps
+ * @returns {{ kept: object[], actions: Array<{id:any, action:'allowed'|'dropped'|'redacted', layer:string|null, reason:string|null}> }}
+ */
+export function defendRecallRows(rows, opts = {}, deps) {
+  const minTrust = typeof opts.minTrust === 'number' ? opts.minTrust : 0;
+  const reviewedPinnedBypass = opts.reviewedPinnedBypass !== false; // default ON
+  const actions = [];
+
+  // Shallow copies — never mutate the better-sqlite3 rows (reused by the dedupe
+  // ring + telemetry). Coalesce undefined trust → 1.0 (column DEFAULT 1.0) so a
+  // legacy un-migrated row isn't dropped as trust 0 by filterByTrust.
+  const copies = rows.map((r) => ({
+    ...r,
+    trust_score: r.trust_score ?? 1.0,
+    metadata: parseMetadata(r.metadata),
+  }));
+
+  // Trust + sensitivity: drops quarantined/below-minTrust, redacts RESTRICTED.
+  const trusted = deps.filterByTrust(copies, minTrust, opts.project);
+  const trustedIds = new Set(trusted.map((r) => r.id));
+  for (const c of copies) {
+    if (!trustedIds.has(c.id)) {
+      actions.push({ id: c.id, action: 'dropped', layer: 'trust', reason: `trust<${minTrust} or quarantined` });
+    }
+  }
+
+  const kept = [];
+  for (const row of trusted) {
+    // Already redacted by the trust layer — keep the masked row, don't re-scan
+    // the marker.
+    if (row.content === RESTRICTED_MARKER) {
+      actions.push({ id: row.id, action: 'redacted', layer: 'restricted', reason: 'RESTRICTED content redacted on recall' });
+      kept.push(row);
+      continue;
+    }
+
+    // Reviewed/pinned bypass: a human-reviewed or pinned memory skips the
+    // content detectors (trust/RESTRICTED above still applied) so a legitimately
+    // reviewed security note isn't re-suppressed at read time.
+    if (reviewedPinnedBypass && (row.reviewed_at != null || row.pinned)) {
+      actions.push({ id: row.id, action: 'allowed', layer: 'bypass', reason: 'reviewed/pinned — content scan skipped' });
+      kept.push(row);
+      continue;
+    }
+
+    const content = typeof row.content === 'string' ? row.content : '';
+    // Sanitise (strip zero-width / RTL / control bytes) BEFORE scanning — the
+    // write path does this, so an injection hidden behind zero-width chars
+    // otherwise dodges the read-path regex detectors. Scan the sanitised form;
+    // the original (benign zero-width is harmless) is what gets injected.
+    const scanContent = deps.sanitiseInput ? (deps.sanitiseInput(content)?.sanitised ?? content) : content;
+
+    const instr = deps.detectInstructions(scanContent);
+    if (instr && instr.detected) {
+      actions.push({ id: row.id, action: 'dropped', layer: 'instruction', reason: `instruction:${(instr.patterns ?? []).join(',')}` });
+      continue;
+    }
+
+    // Mirror the WRITE path: drop only on a BLOCKING credential finding, not a
+    // warned/logged one. A benign high-entropy hash / cache key is stored
+    // (write blocks only on action==='blocked'), so recall must not be stricter
+    // or it silently withholds legitimate notes.
+    const cred = deps.scanForCredentials(scanContent);
+    const credBlocked = !!cred && Array.isArray(cred.findings) && cred.findings.some((f) => f && f.action === 'blocked');
+    if (credBlocked) {
+      const blocked = cred.findings.filter((f) => f && f.action === 'blocked');
+      actions.push({ id: row.id, action: 'dropped', layer: 'credential', reason: `credential:${blocked.length}` });
+      continue;
+    }
+
+    // Decode-and-rescan: a bare encoding flag is NOT a drop (base64 hashes are
+    // common) — only drop if a DECODED snippet itself trips a detector.
+    const enc = deps.detectEncoding(scanContent);
+    if (enc && enc.detected) {
+      let malicious = false;
+      for (const snippet of enc.decodedSnippets ?? []) {
+        const di = deps.detectInstructions(snippet);
+        const dc = deps.scanForCredentials(snippet);
+        const dcBlocked = !!dc && Array.isArray(dc.findings) && dc.findings.some((f) => f && f.action === 'blocked');
+        if ((di && di.detected) || dcBlocked) {
+          malicious = true;
+          break;
+        }
+      }
+      if (malicious) {
+        actions.push({ id: row.id, action: 'dropped', layer: 'encoding', reason: `encoding-payload:${(enc.encodingTypes ?? []).join(',')}` });
+        continue;
+      }
+    }
+
+    // Markdown-image exfil: a stored ![alt](url?d=<smuggled>) is a click-free
+    // data-leak shape the write firewall catches but the read path didn't.
+    // detectMarkdownImageExfil only flags data-bearing image URLs (low FP).
+    if (deps.detectMarkdownImageExfil) {
+      const mdImg = deps.detectMarkdownImageExfil(scanContent);
+      if (mdImg && mdImg.detected) {
+        actions.push({ id: row.id, action: 'dropped', layer: 'markdown-image-exfil', reason: `markdown-image-exfil:${(mdImg.urls ?? []).length}` });
+        continue;
+      }
+    }
+
+    actions.push({ id: row.id, action: 'allowed', layer: null, reason: null });
+    kept.push(row);
+  }
+
+  return { kept, actions };
+}

package/scripts/lib/save-memory.mjs

@@ -76,7 +76,11 @@ export async function saveAutoExtractedMemory(db, memory, project, opts = {}) {
   const decision = result.firewall.result;
 
   if (decision === 'ALLOW') {
-    insertMemoryRow(db, memory, project, sourceIdentifier);
+    // Persist the COMPUTED trust + sensitivity from the scan — not the schema
+    // DEFAULT (trust 1.0 / INTERNAL). The INSERT used to omit these columns, so
+    // every hook-captured memory was over-trusted at 1.0, undercutting the
+    // recall shim's trust filter.
+    insertMemoryRow(db, memory, project, sourceIdentifier, result.trust?.score, result.sensitivity?.level);
     return;
   }
 
@@ -99,7 +103,7 @@ export async function saveAutoExtractedMemory(db, memory, project, opts = {}) {
 
 // ==================== Internal: writes ====================
 
-function insertMemoryRow(db, memory, project, sourceIdentifier) {
+function insertMemoryRow(db, memory, project, sourceIdentifier, trustScore, sensitivityLevel) {
   const timestamp = new Date().toISOString();
 
   // Cross-call, CROSS-PATH exact-title dedup: the hook fires repeatedly (per
@@ -153,9 +157,10 @@ function insertMemoryRow(db, memory, project, sourceIdentifier) {
     INSERT INTO memories (
       uuid, title, content, type, category, salience, tags, project,
       memory_purpose, source, source_kind, capture_method,
+      trust_score, sensitivity_level,
       created_at, last_accessed
     )
-    VALUES (?, ?, ?, 'short_term', ?, ?, ?, ?, ?, ?, 'hook', 'auto', ?, ?)
+    VALUES (?, ?, ?, 'short_term', ?, ?, ?, ?, ?, ?, 'hook', 'auto', ?, ?, ?, ?)
   `).run(
     randomUUID(),
     memory.title,
@@ -166,6 +171,10 @@ function insertMemoryRow(db, memory, project, sourceIdentifier) {
     project || null,
     memory.memoryPurpose ?? 'project',
     `hook:${sourceIdentifier}`,
+    // Computed by the scan above (hook source → 0.8). Fall back to the schema
+    // defaults only if the pipeline somehow returned no trust/sensitivity.
+    typeof trustScore === 'number' ? trustScore : 1.0,
+    sensitivityLevel ?? 'INTERNAL',
     timestamp,
     timestamp,
   );

package/scripts/prompt-recall-hook.mjs

@@ -24,6 +24,7 @@ import { computeEffectiveSalience } from './lib/salience.mjs';
 import { writeRecallLog } from './lib/recall-log.mjs';
 import { recordHookInvocation } from './lib/telemetry.mjs';
 import { filterByRelevance, extractQueryTerms } from './lib/recall-relevance.mjs';
+import { defendRecallRows, loadRecallDefence, ensureRecallAuditDb, emitRecallAudit } from './lib/recall-defence.mjs';
 
 // ==================== CONFIG ====================
 
@@ -135,6 +136,7 @@ function recallRelevant(db, project, prompt) {
         SELECT
           m.id, m.title, m.content, m.category, m.salience, fts.rank,
           m.pinned, m.access_count, m.last_accessed,
+          m.trust_score, m.sensitivity_level, m.metadata, m.reviewed_at,
           COALESCE(m.downvote_count, 0) AS downvote_count
         FROM memories m
         JOIN memories_fts fts ON m.id = fts.rowid
@@ -184,6 +186,7 @@ function recallRelevant(db, project, prompt) {
         SELECT
           id, title, content, category, salience,
           pinned, access_count, last_accessed,
+          trust_score, sensitivity_level, metadata, reviewed_at,
           COALESCE(downvote_count, 0) AS downvote_count
         FROM memories
         WHERE category = ?
@@ -517,6 +520,36 @@ process.stdin.on('end', async () => {
       }
     }
 
+    // ── RECALL-BOUNDARY DEFENCE (Feature #1) ─────────────────────────────
+    // Filter poisoned / RESTRICTED / credential-bearing rows OUT of the recalled
+    // set before they're formatted into the prompt. Runs after dedupe (acts on
+    // the final injected set) and before format + ring + telemetry (so those see
+    // only safe rows). FAIL-OPEN: a missing dist build leaves recall unchanged.
+    if (config.recallDefence !== false && memories.length > 0) {
+      try {
+        const defence = await loadRecallDefence();
+        if (defence) {
+          const minTrust = typeof config.recallDefenceMinTrust === 'number' ? config.recallDefenceMinTrust : 0;
+          const { kept: safe, actions } = defendRecallRows(memories, { minTrust, project }, defence);
+          const withheld = actions.filter((a) => a.action !== 'allowed');
+          if (withheld.length > 0) {
+            console.error(
+              `[shieldcortex] recall-defence withheld ${withheld.length} memory row(s): ` +
+                withheld.map((w) => `#${w.id}:${w.layer}`).join(', '),
+            );
+            // Only touch a writable audit connection when something was withheld.
+            ensureRecallAuditDb(defence, dbPath);
+            for (const a of withheld) {
+              emitRecallAudit(defence.logAudit, { memoryId: a.id, action: a.action, layer: a.layer, reason: a.reason, project });
+            }
+          }
+          memories = safe;
+        }
+      } catch (e) {
+        console.error('[shieldcortex] recall-defence skipped:', e?.message ?? e);
+      }
+    }
+
     const context = formatRecallContext(memories);
 
     // Update the session ring with the hashes of what we just injected.

package/scripts/session-start-hook.mjs

@@ -25,6 +25,7 @@ import { homedir } from 'os';
 import { deriveProjectKey } from './lib/project-key.mjs';
 import { truncatePreservingWords } from './lib/truncate.mjs';
 import { orderByEffectiveSalience } from './lib/session-context.mjs';
+import { defendRecallRows, loadRecallDefence, ensureRecallAuditDb, emitRecallAudit } from './lib/recall-defence.mjs';
 
 const NEW_DB_DIR = join(homedir(), '.shieldcortex');
 const LEGACY_DB_DIR = join(homedir(), '.claude-cortex');
@@ -94,7 +95,8 @@ function getProjectContext(db, project) {
   // computeEffectiveSalience.
   const candidates = db.prepare(`
     SELECT id, title, content, category, type, salience, tags, created_at,
-           pinned, access_count, last_accessed, COALESCE(downvote_count, 0) AS downvote_count
+           pinned, access_count, last_accessed, trust_score, sensitivity_level, metadata, reviewed_at,
+           COALESCE(downvote_count, 0) AS downvote_count
     FROM memories
     WHERE (project = ? OR project IS NULL)
       AND salience >= ?
@@ -111,7 +113,8 @@ function getProjectContext(db, project) {
     const excludeIds = memories.map(m => m.id);
     const placeholders = excludeIds.length > 0 ? excludeIds.map(() => '?').join(',') : '0';
     const recent = db.prepare(`
-      SELECT id, title, content, category, type, salience, tags, created_at
+      SELECT id, title, content, category, type, salience, tags, created_at,
+             pinned, trust_score, sensitivity_level, metadata, reviewed_at
       FROM memories
       WHERE (project = ? OR project IS NULL)
         AND id NOT IN (${placeholders})
@@ -208,7 +211,7 @@ process.stdin.on('readable', () => {
   }
 });
 
-process.stdin.on('end', () => {
+process.stdin.on('end', async () => {
   try {
     const hookData = JSON.parse(input || '{}');
     const source = typeof hookData.source === 'string' ? hookData.source : 'startup';
@@ -239,8 +242,37 @@ process.stdin.on('end', () => {
     } else {
       const db = new Database(DB_PATH, { readonly: true, timeout: 5000 });
       memories = getProjectContext(db, project);
-      context = formatContext(memories, project);
       db.close();
+
+      // ── RECALL-BOUNDARY DEFENCE (Feature #1) ───────────────────────────
+      // Same shim as prompt-recall: drop poisoned / RESTRICTED / credential
+      // rows from the session preamble before they're formatted. FAIL-OPEN if
+      // the dist build is missing (recall preamble unchanged).
+      if (config.recallDefence !== false && memories.length > 0) {
+        try {
+          const defence = await loadRecallDefence();
+          if (defence) {
+            const minTrust = typeof config.recallDefenceMinTrust === 'number' ? config.recallDefenceMinTrust : 0;
+            const { kept: safe, actions } = defendRecallRows(memories, { minTrust, project }, defence);
+            const withheld = actions.filter((a) => a.action !== 'allowed');
+            if (withheld.length > 0) {
+              console.error(
+                `[shieldcortex] recall-defence (session-start) withheld ${withheld.length} memory row(s): ` +
+                  withheld.map((w) => `#${w.id}:${w.layer}`).join(', '),
+              );
+              ensureRecallAuditDb(defence, DB_PATH);
+              for (const a of withheld) {
+                emitRecallAudit(defence.logAudit, { memoryId: a.id, action: a.action, layer: a.layer, reason: a.reason, project });
+              }
+            }
+            memories = safe;
+          }
+        } catch (e) {
+          console.error('[shieldcortex] recall-defence skipped:', e?.message ?? e);
+        }
+      }
+
+      context = formatContext(memories, project);
     }
 
     if (context) {