shieldcortex 4.24.3 → 4.25.1

package/scripts/pre-compact-hook.mjs

@@ -22,6 +22,7 @@ import { getAutoMemoryConfig } from './lib/auto-memory-config.mjs';
 import { recordHookInvocation } from './lib/telemetry.mjs';
 import { deriveProjectKey } from './lib/project-key.mjs';
 import { recordSessionEvent } from './lib/session-capture.mjs';
+import { writePrecompactLog } from './lib/precompact-log.mjs';
 import {
   extractMemorableSegments,
   processSegments,
@@ -183,9 +184,12 @@ process.stdin.on('end', async () => {
     console.error(`[auto-extract] Dynamic threshold: ${dynamicThreshold.toFixed(2)}`);
 
     // Only attempt extraction if we have conversation content
+    let precompactCandidates = [];
+    let rawSegmentCount = 0;
     if (conversationText && conversationText.length > 100) {
       // Extract memorable segments
       const segments = extractMemorableSegments(conversationText);
+      rawSegmentCount = segments.length;
       const processedSegments = processSegments(segments, dynamicThreshold, {
         maxMemories: MAX_AUTO_MEMORIES,
         categoryThresholds: PRE_COMPACT_CATEGORY_THRESHOLDS,
@@ -194,19 +198,47 @@ process.stdin.on('end', async () => {
 
       // Save auto-extracted memories
       for (const memory of processedSegments) {
+        // v4.25.0: collect candidate metadata for the precompact ring buffer.
+        // Operators read this via `shieldcortex inspect last-precompact`
+        // when diagnosing why extraction did/didn't pick a particular line.
+        const candidate = {
+          extractorType: memory.extractorType,
+          category: memory.category,
+          memoryPurpose: memory.memoryPurpose,
+          title: memory.title,
+          salience: memory.salience,
+          frequencyBoost: memory.frequencyBoost ?? 0,
+          saved: false,
+          error: null,
+        };
         try {
           await saveMemory(db, memory, project);
           autoExtractedCount++;
+          candidate.saved = true;
           const boostInfo = memory.frequencyBoost > 0 ? ` +${memory.frequencyBoost.toFixed(2)} boost` : '';
           console.error(`[auto-extract] Saved: ${memory.title} (salience: ${memory.salience.toFixed(2)}${boostInfo}, category: ${memory.category})`);
         } catch (err) {
+          candidate.error = err.message;
           console.error(`[auto-extract] Failed to save "${memory.title}": ${err.message}`);
         }
+        precompactCandidates.push(candidate);
       }
     } else {
       notes = 'no-content';
     }
 
+    // v4.25.0: write the run to the rolling ring buffer at
+    // ~/.shieldcortex/precompact-log/. Best-effort — never throw.
+    try {
+      writePrecompactLog({
+        thresholdUsed: dynamicThreshold,
+        contextFullnessPct: maxMemories > 0 ? Math.round((totalMemories / maxMemories) * 100) : null,
+        totalMemories,
+        rawSegmentCount,
+        candidates: precompactCandidates,
+      });
+    } catch { /* best-effort */ }
+
     console.error(`[shieldcortex] Pre-compact complete: ${autoExtractedCount} memories auto-extracted`);
 
     outputReminder(autoExtractedCount, dynamicThreshold);

package/scripts/prompt-recall-hook.mjs

@@ -19,6 +19,8 @@ import { sanitisePromptForRecall } from './lib/prompt-sanitiser.mjs';
 import { recordSessionEvent } from './lib/session-capture.mjs';
 import { truncatePreservingWords } from './lib/truncate.mjs';
 import { compareRecallResults } from './lib/recall-rank.mjs';
+import { computeEffectiveSalience } from './lib/salience.mjs';
+import { writeRecallLog } from './lib/recall-log.mjs';
 
 // ==================== CONFIG ====================
 
@@ -80,8 +82,15 @@ function recallRelevant(db, project, prompt) {
   const ftsQuery = escapeFts5(prompt);
   if (ftsQuery.trim()) {
     try {
+      // v4.25.0: project pinned/access_count/last_accessed/downvote_count
+      // so compareRecallResults can compute effective salience without a
+      // second query. COALESCE on downvote_count for DBs that pre-date the
+      // 4.25 migration (the prompt-recall-hook runs against legacy DBs too).
       const ftsRows = db.prepare(`
-        SELECT m.id, m.title, m.content, m.category, m.salience, fts.rank
+        SELECT
+          m.id, m.title, m.content, m.category, m.salience, fts.rank,
+          m.pinned, m.access_count, m.last_accessed,
+          COALESCE(m.downvote_count, 0) AS downvote_count
         FROM memories m
         JOIN memories_fts fts ON m.id = fts.rowid
         WHERE memories_fts MATCH ?
@@ -95,6 +104,10 @@ function recallRelevant(db, project, prompt) {
       for (const row of ftsRows) {
         if (!seen.has(row.id)) {
           seen.add(row.id);
+          // v4.25.1: tag with source so the recall ring buffer can show
+          // operators which path surfaced a given candidate. _-prefixed so
+          // it doesn't collide with any future SELECT column.
+          row._source = 'fts';
           results.push(row);
         }
       }
@@ -112,8 +125,14 @@ function recallRelevant(db, project, prompt) {
 
   if (categoryBoost && results.length < MAX_RESULTS) {
     try {
+      // v4.25.0: project the salience-formula inputs (same as the FTS
+      // SELECT above) so compareRecallResults can compute effective
+      // salience for rows that came in via this fallback path.
       const catRows = db.prepare(`
-        SELECT id, title, content, category, salience
+        SELECT
+          id, title, content, category, salience,
+          pinned, access_count, last_accessed,
+          COALESCE(downvote_count, 0) AS downvote_count
         FROM memories
         WHERE category = ?
           AND (project = ? OR project IS NULL OR scope = 'global')
@@ -126,6 +145,7 @@ function recallRelevant(db, project, prompt) {
       for (const row of catRows) {
         if (!seen.has(row.id)) {
           seen.add(row.id);
+          row._source = 'category-boost';
           results.push(row);
         }
       }
@@ -136,7 +156,10 @@ function recallRelevant(db, project, prompt) {
   // sort discarded the relevance signal from the FTS5 query above — high-
   // salience-but-off-topic memories bubbled to the top of the recall preamble.
   results.sort(compareRecallResults);
-  return results.slice(0, MAX_RESULTS);
+  // v4.25.1: return the full sorted set alongside the sliced top-N so the
+  // recall ring buffer can log which candidates were considered but didn't
+  // make the cut. The top-N is the existing public contract.
+  return { topN: results.slice(0, MAX_RESULTS), fullSet: results };
 }
 
 // ==================== DEDUPE STATE ====================
@@ -179,6 +202,69 @@ function hashContent(s) {
   return createHash('sha256').update(s).digest('hex').slice(0, 16);
 }
 
+// ==================== RECALL LOG (v4.25.1) ====================
+//
+// Per-run dump of "what prompt fired, which memories were considered, which
+// were injected, which were dropped, and why" to ~/.shieldcortex/recall-log/.
+// Operators read this via `shieldcortex inspect last-recall`. Best-effort —
+// never throws into the caller. Designed to gather diagnostic data so we
+// can scope v4.26.x recall-quality fixes from real fleet usage.
+
+const RECALL_LOG_PROMPT_CAP = 200;
+
+function buildLogCandidates({ fullSet, topN, injected, dedupHashes }) {
+  const injectedIds = new Set(injected.map((m) => m.id));
+  const topNIds = new Set(topN.map((m) => m.id));
+  return fullSet.map((row) => {
+    const wasInjected = injectedIds.has(row.id);
+    let dropReason = null;
+    if (!wasInjected) {
+      if (topNIds.has(row.id) && dedupHashes.has(hashContent(row.content))) {
+        dropReason = 'dedupe';
+      } else if (!topNIds.has(row.id)) {
+        dropReason = 'outside_top_n';
+      } else {
+        // In top-N, not deduped, not injected — early exit on empty memories
+        // before the dedupe filter ran. Mark as not-injected-no-reason so the
+        // CLI can distinguish from an active drop.
+        dropReason = 'not_injected';
+      }
+    }
+    return {
+      id: row.id ?? null,
+      title: row.title ?? null,
+      category: row.category ?? null,
+      memoryPurpose: row.memory_purpose ?? null,
+      salience: typeof row.salience === 'number' ? row.salience : null,
+      ftsRank: typeof row.rank === 'number' ? row.rank : null,
+      source: row._source ?? null,
+      effectiveSalience: computeEffectiveSalience(row),
+      injected: wasInjected,
+      dropReason,
+    };
+  });
+}
+
+function logRecallRun({ prompt, sessionId, project, fullSet, topN, injected, dedupHashes, context }) {
+  try {
+    const promptCapped = prompt.length > RECALL_LOG_PROMPT_CAP
+      ? prompt.slice(0, RECALL_LOG_PROMPT_CAP) + '…'
+      : prompt;
+    writeRecallLog({
+      prompt: promptCapped,
+      promptHash: hashContent(prompt),
+      sessionId: sessionId ?? null,
+      project: project ?? null,
+      minSalience: MIN_SALIENCE,
+      candidates: buildLogCandidates({ fullSet, topN, injected, dedupHashes }),
+      injectedCount: injected.length,
+      finalContextChars: context ? context.length : 0,
+    });
+  } catch {
+    // Best-effort — recall must not block on log write failures.
+  }
+}
+
 // ==================== FORMAT ====================
 
 function formatRecallContext(memories) {
@@ -267,10 +353,26 @@ process.stdin.on('end', () => {
     }
 
     const db = new Database(dbPath, { readonly: true, timeout: 2000 });
-    let memories = recallRelevant(db, project, prompt);
+    const { topN, fullSet } = recallRelevant(db, project, prompt);
     db.close();
+    let memories = topN;
 
     if (memories.length === 0) {
+      // v4.25.1: still log no-candidate runs ONLY when the FTS step actually
+      // ran (fullSet has something filtered out before the slice). Pure
+      // empties (no FTS match at all) skip the log — saves disk + signal.
+      if (fullSet.length > 0) {
+        logRecallRun({
+          prompt,
+          sessionId,
+          project,
+          fullSet,
+          topN: [],
+          injected: [],
+          dedupHashes: new Set(),
+          context: null,
+        });
+      }
       process.exit(0);
     }
 
@@ -279,13 +381,26 @@ process.stdin.on('end', () => {
     // "near-identical recall in adjacent turns" failure Jarvis flagged.
     let dedupState = null;
     let sessionRing = [];
+    let dedupHashes = new Set();
     if (sessionId) {
       dedupState = loadDedupState();
       const entry = dedupState[sessionId] ?? { ring: [], touched: Date.now() };
       sessionRing = Array.isArray(entry.ring) ? entry.ring : [];
-      const recentHashes = new Set(sessionRing);
-      memories = memories.filter((m) => !recentHashes.has(hashContent(m.content)));
+      dedupHashes = new Set(sessionRing);
+      memories = memories.filter((m) => !dedupHashes.has(hashContent(m.content)));
       if (memories.length === 0) {
+        // v4.25.1: log even when every top-N candidate was dedupe-suppressed.
+        // This is the most interesting case for diagnosing recall noise.
+        logRecallRun({
+          prompt,
+          sessionId,
+          project,
+          fullSet,
+          topN,
+          injected: [],
+          dedupHashes,
+          context: null,
+        });
         process.exit(0);
       }
     }
@@ -321,6 +436,18 @@ process.stdin.on('end', () => {
       },
     };
 
+    // v4.25.1: log the recall run (best-effort, never blocks output)
+    logRecallRun({
+      prompt,
+      sessionId,
+      project,
+      fullSet,
+      topN,
+      injected: memories,
+      dedupHashes,
+      context,
+    });
+
     console.log(JSON.stringify(output));
     console.error(`[shieldcortex] Proactive recall: ${memories.length} memories for "${project}"`);