shieldcortex 4.24.3 → 4.25.0

package/scripts/lib/extract-memorable-segments.mjs

@@ -23,6 +23,37 @@ export const BASE_THRESHOLD = 0.35;
 // trust band (0.5–0.7) used elsewhere in the pipeline.
 export const AUTO_EXTRACT_SALIENCE_CAP = 0.6;
 
+// v4.25.0: deterministic taxonomy. The extractor already knows *why* a
+// segment matched (extractorType) — derive memory_purpose and category
+// from that signal instead of re-scanning the captured text. Keyword-
+// based suggestCategory() stays as a fallback for any extractorType not
+// in the map (defensive, not currently used by any in-tree extractor).
+//
+// memory_purpose follows the global taxonomy (src/memory/types.ts):
+//   user      — who the human is
+//   feedback  — corrections / instructions the human gave the agent
+//   project   — facts about the project (decisions, architecture, fixes)
+//   reference — reusable knowledge (learnings, external references)
+export const EXTRACTOR_TO_PURPOSE = {
+  preference: 'feedback',
+  decision: 'project',
+  architecture: 'project',
+  'error-fix': 'project',
+  learning: 'reference',
+  'important-note': 'project',
+};
+
+// Pinning category by extractorType eliminates the "important-note tagged
+// as 'error' because the text mentioned a bug" failure mode.
+export const EXTRACTOR_TO_CATEGORY = {
+  preference: 'preference',
+  decision: 'context',
+  architecture: 'architecture',
+  'error-fix': 'error',
+  learning: 'learning',
+  'important-note': 'note',
+};
+
 // Default category thresholds (session-end / lighter hooks).
 // Pre-compact hook uses tighter thresholds (raised +0.10 in v4.11.0) and
 // passes them via processSegments({ categoryThresholds: ... }).
@@ -232,7 +263,7 @@ export function getExtractionThreshold(category, dynamicThreshold, categoryThres
 // newline, capping at 200 chars for safety, and optionally consumes the
 // terminator so the captured group ends cleanly. v4.24.3.
 
-const FULL_EXTRACTORS = [
+export const FULL_EXTRACTORS = [
   {
     name: 'decision',
     titlePrefix: 'Decision: ',
@@ -306,7 +337,7 @@ const FULL_EXTRACTORS = [
 // no important-note, fewer learning + preference patterns). Preserve the
 // pre-refactor surface so its behaviour does not change with this move.
 // Same sentence-bounded capture form as FULL_EXTRACTORS (v4.24.3).
-const STOP_HOOK_EXTRACTORS = [
+export const STOP_HOOK_EXTRACTORS = [
   FULL_EXTRACTORS[0], // decision
   {
     name: 'error-fix',
@@ -598,11 +629,17 @@ export function processSegments(segments, dynamicThreshold = BASE_THRESHOLD, opt
     if (!isDupe) {
       const text = seg.title + ' ' + seg.content;
       const baseSalience = calculateSalience(text);
-      const category = suggestCategory(text);
+      // v4.25.0: extractorType drives category and memory_purpose. Fall
+      // back to keyword-based suggestCategory only when extractorType is
+      // unknown (defensive; no in-tree extractor produces a name outside
+      // EXTRACTOR_TO_CATEGORY).
+      const category = EXTRACTOR_TO_CATEGORY[seg.extractorType] ?? suggestCategory(text);
+      const memoryPurpose = EXTRACTOR_TO_PURPOSE[seg.extractorType] ?? 'project';
       unique.push({
         ...seg,
         baseSalience,
         category,
+        memoryPurpose,
         tags: extractTags(text, hookTag, seg.extractorType),
       });
     }

package/scripts/lib/precompact-log.mjs

@@ -0,0 +1,142 @@
+/**
+ * Precompact ring buffer (v4.25.0).
+ *
+ * Each precompact hook run writes a debug log of "what did the extractor
+ * propose this time, and which proposals actually landed in `memories`?"
+ * Operators can read it via `shieldcortex inspect last-precompact` when
+ * tuning thresholds or chasing weird auto-extract behaviour.
+ *
+ * Ring buffer at `~/.shieldcortex/precompact-log/{0..9}.json` — index 0 is
+ * the most recent. On every write we rotate (delete 9, rename 8→9, ..., 0→1)
+ * and atomically write the new run to index 0 via a temp-file + rename so a
+ * crash mid-rotation never leaves a torn JSON file.
+ *
+ * Size cap: 10 files × ~5KB each = ~50KB. Negligible disk impact.
+ *
+ * Designed to be best-effort — never throws into the caller. The hook
+ * absolutely must not block compaction on a missing ~/.shieldcortex
+ * directory or a transient EBUSY.
+ */
+
+import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+
+const RING_SIZE = 10;
+
+// Resolved per-call so tests can swap HOME between cases. Read
+// `process.env.HOME` directly rather than going through `os.homedir()`
+// because jest's ESM VM loader doesn't reliably reflect HOME mutations
+// through homedir(), causing test-suite cross-contamination.
+function logDir() {
+  return join(process.env.HOME || homedir(), '.shieldcortex', 'precompact-log');
+}
+
+function logPath(index) {
+  return join(logDir(), `${index}.json`);
+}
+
+function ensureDir() {
+  const dir = logDir();
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+}
+
+/**
+ * Rotate the ring buffer one slot. Oldest entry (index 9) is unlinked,
+ * remaining entries shift up by one. Safe to call when slots are missing.
+ */
+function rotate() {
+  try {
+    unlinkSync(logPath(RING_SIZE - 1));
+  } catch {
+    // No file at the oldest slot — fine.
+  }
+  for (let i = RING_SIZE - 2; i >= 0; i--) {
+    const from = logPath(i);
+    const to = logPath(i + 1);
+    if (existsSync(from)) {
+      try {
+        renameSync(from, to);
+      } catch {
+        // Best-effort — skip if EBUSY/EACCES.
+      }
+    }
+  }
+}
+
+/**
+ * Atomically write a precompact run entry to slot 0 (the newest slot).
+ * Rotates the ring beforehand.
+ *
+ * @param {{
+ *   ranAt?: string,
+ *   thresholdUsed?: number,
+ *   contextFullnessPct?: number,
+ *   totalMemories?: number,
+ *   candidates: Array<{
+ *     extractorType?: string,
+ *     category?: string,
+ *     memoryPurpose?: string,
+ *     title?: string,
+ *     salience?: number,
+ *     saved?: boolean,
+ *     memoryId?: number | null,
+ *     dropReason?: string | null,
+ *   }>,
+ * }} entry
+ */
+export function writePrecompactLog(entry) {
+  try {
+    ensureDir();
+    rotate();
+    const payload = {
+      ranAt: entry.ranAt ?? new Date().toISOString(),
+      thresholdUsed: entry.thresholdUsed ?? null,
+      contextFullnessPct: entry.contextFullnessPct ?? null,
+      totalMemories: entry.totalMemories ?? null,
+      candidates: Array.isArray(entry.candidates) ? entry.candidates : [],
+    };
+    const target = logPath(0);
+    const tmp = `${target}.tmp`;
+    writeFileSync(tmp, JSON.stringify(payload, null, 2), 'utf8');
+    renameSync(tmp, target);
+  } catch {
+    // Best-effort. The hook continues without a log entry rather than
+    // blocking compaction on a disk error.
+  }
+}
+
+/**
+ * Read a precompact log entry by ring index.
+ *
+ * @param {number} index — 0 = newest, RING_SIZE-1 = oldest
+ * @returns {object | null} parsed entry, or null if missing/corrupt
+ */
+export function readPrecompactLog(index = 0) {
+  try {
+    const raw = readFileSync(logPath(index), 'utf8');
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read all present precompact log entries, newest first. Missing slots are
+ * skipped (no nulls in the returned array).
+ *
+ * @returns {Array<{ index: number, entry: object }>}
+ */
+export function listPrecompactLogs() {
+  const out = [];
+  for (let i = 0; i < RING_SIZE; i++) {
+    const entry = readPrecompactLog(i);
+    if (entry) out.push({ index: i, entry });
+  }
+  return out;
+}
+
+export const PRECOMPACT_RING_SIZE = RING_SIZE;
+export { logDir as getPrecompactLogDir };

package/scripts/lib/recall-rank.mjs

@@ -1,14 +1,12 @@
 /**
- * Recall result ranking helpers (v4.23.0).
+ * Recall result ranking helpers.
  *
- * Field context (edith, jarvis 2026-05-24): the UserPromptSubmit recall hook
- * was filtering candidates by FTS5 keyword match but then doing a final sort
- * by RAW salience, discarding the relevance signal. Result: high-salience-
- * but-off-topic memories bubbled to the top of the per-prompt recall preamble
- * ("Decision: that path for the rest of this conversation"-style fragments).
+ * v4.23.0 — FTS rank primary, raw salience tiebreaker (fixed: high-salience-
+ * but-off-topic memories bubbling to the top of the per-prompt recall preamble).
  *
- * v4.23.0 picks the simplest of the three candidate approaches (A in the
- * v4.22.0 plan): **FTS rank is primary, raw salience is the tiebreaker**.
+ * v4.25.0 — tiebreaker now uses *effective* salience instead of raw:
+ *   recency × access × pin × downvote_penalty. See scripts/lib/salience.mjs.
+ *   The DB column stays untouched — this is a read-time computation only.
  *
  * SQLite FTS5 BM25 ranks are negative numbers — more negative = more
  * relevant. So `a.rank - b.rank` (ascending) puts the most-relevant result
@@ -16,21 +14,36 @@
  * field; they sort below all FTS results.
  */
 
+import { computeEffectiveSalience } from './salience.mjs';
+
 /**
  * Sort comparator for memory recall results.
  *
  * Behaviour:
  * - Both rows have `rank` (FTS results): lower rank wins (more relevant)
  * - Only one has `rank`: that one wins (FTS beats category-only fallback)
- * - Neither has `rank` (or tied): higher salience wins
+ * - Neither has `rank` (or tied): higher *effective* salience wins
  *
  * Stable behaviour for equal-rank-equal-salience: original order preserved
  * (Array.prototype.sort in modern engines is stable).
  *
- * @param {{ rank?: number, salience?: number }} a
- * @param {{ rank?: number, salience?: number }} b
+ * @param {{ rank?: number, salience?: number, last_accessed?: string, access_count?: number, pinned?: number, downvote_count?: number }} a
+ * @param {{ rank?: number, salience?: number, last_accessed?: string, access_count?: number, pinned?: number, downvote_count?: number }} b
  * @returns {number} negative if a comes first, positive if b comes first
  */
+// A row carries the v4.25 salience-formula inputs only when the SELECT
+// projected pinned/access_count/last_accessed/downvote_count. Older call
+// sites and recall-rank tests pass rows with just {id, rank, salience} —
+// for those, fall back to raw salience comparison (v4.23 behaviour).
+function hasSalienceFormulaInputs(row) {
+  return (
+    row.access_count !== undefined ||
+    row.last_accessed !== undefined ||
+    row.pinned !== undefined ||
+    row.downvote_count !== undefined
+  );
+}
+
 export function compareRecallResults(a, b) {
   const aHasRank = typeof a.rank === 'number' && Number.isFinite(a.rank);
   const bHasRank = typeof b.rank === 'number' && Number.isFinite(b.rank);
@@ -41,6 +54,10 @@ export function compareRecallResults(a, b) {
   } else if (aHasRank !== bHasRank) {
     return aHasRank ? -1 : 1;
   }
-  // Neither has rank (both from category fallback), or tied rank → salience
+  // v4.25.0: effective salience if both rows carry the formula inputs.
+  // Otherwise (legacy or test rows) preserve v4.23 raw-salience behaviour.
+  if (hasSalienceFormulaInputs(a) && hasSalienceFormulaInputs(b)) {
+    return computeEffectiveSalience(b) - computeEffectiveSalience(a);
+  }
   return (b.salience ?? 0) - (a.salience ?? 0);
 }

package/scripts/lib/salience.mjs

@@ -0,0 +1,83 @@
+/**
+ * Effective salience computation (v4.25.0).
+ *
+ * Pre-4.25 the recall hook ranked tied FTS results by raw `salience` only.
+ * That ignored real signal: a memory that was last accessed 3 months ago
+ * carries less weight than a memory accessed yesterday; a pinned memory
+ * deserves more weight than an identical unpinned one; a memory the user
+ * downvoted via `shieldcortex memory downvote <id>` should be demoted.
+ *
+ * Formula:
+ *   effective = base × recency × access × pin × downvote_penalty
+ *
+ *   recency           = exp(-Δt_days / halfLifeDays)        // decay
+ *   access            = log(1 + access_count) / log(1 + accessNorm)
+ *   pin               = pinned ? pinBoost : 1
+ *   downvote_penalty  = max(0.1, 1 - downvoteDecay × downvote_count)
+ *
+ * All constants are env-var-tunable (no recompile needed for field tuning).
+ *
+ * The function returns the *multiplier-adjusted* salience. The base salience
+ * column in SQLite stays untouched — this is a read-time computation.
+ *
+ * @param {{
+ *   salience?: number,
+ *   last_accessed?: string | null,
+ *   access_count?: number | null,
+ *   pinned?: number | boolean | null,
+ *   downvote_count?: number | null
+ * }} memory
+ * @param {{
+ *   halfLifeDays?: number,
+ *   accessNorm?: number,
+ *   pinBoost?: number,
+ *   downvoteDecay?: number,
+ *   now?: number,
+ * }} [opts]
+ * @returns {number}
+ */
+// Resolve a numeric tuning constant from opts (explicit override), then
+// SHIELDCORTEX_* env var, then the documented default. Returns the default
+// if either of the first two is NaN — we never want a typo'd env var to
+// silently zero out a multiplier.
+function pickNumber(override, envName, fallback) {
+  if (typeof override === 'number' && Number.isFinite(override)) return override;
+  const fromEnv = Number(process.env[envName]);
+  if (Number.isFinite(fromEnv) && fromEnv !== 0) return fromEnv;
+  return fallback;
+}
+
+export function computeEffectiveSalience(memory, opts = {}) {
+  const halfLifeDays = pickNumber(opts.halfLifeDays, 'SHIELDCORTEX_SALIENCE_HALF_LIFE_DAYS', 14);
+  const accessNorm = pickNumber(opts.accessNorm, 'SHIELDCORTEX_SALIENCE_ACCESS_NORM', 10);
+  const pinBoost = pickNumber(opts.pinBoost, 'SHIELDCORTEX_SALIENCE_PIN_BOOST', 1.5);
+  const downvoteDecay = pickNumber(opts.downvoteDecay, 'SHIELDCORTEX_SALIENCE_DOWNVOTE_DECAY', 0.3);
+  const now = opts.now ?? Date.now();
+
+  const base = typeof memory.salience === 'number' ? memory.salience : 0;
+
+  // Recency: exp(-Δt_days / halfLife). last_accessed missing → assume "now"
+  // (a brand-new memory should not be penalised for never having been read).
+  let recency = 1;
+  if (memory.last_accessed) {
+    const lastMs = Date.parse(memory.last_accessed);
+    if (Number.isFinite(lastMs)) {
+      const deltaDays = Math.max(0, (now - lastMs) / 86_400_000);
+      recency = Math.exp(-deltaDays / halfLifeDays);
+    }
+  }
+
+  // Access: log-scaled, normalised so access_count=accessNorm produces ~1.0.
+  const accessCount = Math.max(0, Number(memory.access_count) || 0);
+  const access = Math.log1p(accessCount) / Math.log1p(accessNorm);
+
+  // Pin: SQLite stores boolean as 0/1.
+  const pin = memory.pinned ? pinBoost : 1;
+
+  // Downvote: each downvote lops downvoteDecay off the multiplier, floored
+  // at 0.1 so a 4×-downvoted memory can still surface when truly relevant.
+  const downvotes = Math.max(0, Number(memory.downvote_count) || 0);
+  const downvotePenalty = Math.max(0.1, 1 - downvoteDecay * downvotes);
+
+  return base * recency * access * pin * downvotePenalty;
+}

package/scripts/lib/save-memory.mjs

@@ -20,7 +20,7 @@ import { fileURLToPath, pathToFileURL } from 'url';
  * Async: the pipeline lives in dist/ as ESM and is loaded via dynamic import.
  *
  * @param {import('better-sqlite3').Database} db
- * @param {{ title: string, content: string, category: string, salience: number, tags: string[] }} memory
+ * @param {{ title: string, content: string, category: string, salience: number, tags: string[], memoryPurpose?: string }} memory
  * @param {string|null} [project]
  * @param {{ source?: string }} [opts] — `source` identifies the calling hook
  *   ('session-end-hook' | 'pre-compact-hook' | 'stop-hook' | 'hook').
@@ -50,7 +50,7 @@ export async function saveAutoExtractedMemory(db, memory, project, opts = {}) {
   const decision = result.firewall.result;
 
   if (decision === 'ALLOW') {
-    insertMemoryRow(db, memory, project);
+    insertMemoryRow(db, memory, project, sourceIdentifier);
     return;
   }
 
@@ -73,11 +73,21 @@ export async function saveAutoExtractedMemory(db, memory, project, opts = {}) {
 
 // ==================== Internal: writes ====================
 
-function insertMemoryRow(db, memory, project) {
+function insertMemoryRow(db, memory, project, sourceIdentifier) {
   const timestamp = new Date().toISOString();
+  // v4.25.0: pre-4.25 the INSERT omitted memory_purpose / source / source_kind
+  // / capture_method, so every hook write looked identical to a user-typed
+  // memory at the SQL level. The hook already knows which hook is calling
+  // (sourceIdentifier) and the extractor already computed memoryPurpose —
+  // thread both through so `shieldcortex inspect last-precompact` and any
+  // memory-source SQL query can distinguish hook writes from user writes.
   db.prepare(`
-    INSERT INTO memories (uuid, title, content, type, category, salience, tags, project, created_at, last_accessed)
-    VALUES (?, ?, ?, 'short_term', ?, ?, ?, ?, ?, ?)
+    INSERT INTO memories (
+      uuid, title, content, type, category, salience, tags, project,
+      memory_purpose, source, source_kind, capture_method,
+      created_at, last_accessed
+    )
+    VALUES (?, ?, ?, 'short_term', ?, ?, ?, ?, ?, ?, 'hook', 'auto', ?, ?)
   `).run(
     randomUUID(),
     memory.title,
@@ -86,6 +96,8 @@ function insertMemoryRow(db, memory, project) {
     memory.salience,
     JSON.stringify(memory.tags),
     project || null,
+    memory.memoryPurpose ?? 'project',
+    `hook:${sourceIdentifier}`,
     timestamp,
     timestamp,
   );

package/scripts/pre-compact-hook.mjs

@@ -22,6 +22,7 @@ import { getAutoMemoryConfig } from './lib/auto-memory-config.mjs';
 import { recordHookInvocation } from './lib/telemetry.mjs';
 import { deriveProjectKey } from './lib/project-key.mjs';
 import { recordSessionEvent } from './lib/session-capture.mjs';
+import { writePrecompactLog } from './lib/precompact-log.mjs';
 import {
   extractMemorableSegments,
   processSegments,
@@ -183,9 +184,12 @@ process.stdin.on('end', async () => {
     console.error(`[auto-extract] Dynamic threshold: ${dynamicThreshold.toFixed(2)}`);
 
     // Only attempt extraction if we have conversation content
+    let precompactCandidates = [];
+    let rawSegmentCount = 0;
     if (conversationText && conversationText.length > 100) {
       // Extract memorable segments
       const segments = extractMemorableSegments(conversationText);
+      rawSegmentCount = segments.length;
       const processedSegments = processSegments(segments, dynamicThreshold, {
         maxMemories: MAX_AUTO_MEMORIES,
         categoryThresholds: PRE_COMPACT_CATEGORY_THRESHOLDS,
@@ -194,19 +198,47 @@ process.stdin.on('end', async () => {
 
       // Save auto-extracted memories
       for (const memory of processedSegments) {
+        // v4.25.0: collect candidate metadata for the precompact ring buffer.
+        // Operators read this via `shieldcortex inspect last-precompact`
+        // when diagnosing why extraction did/didn't pick a particular line.
+        const candidate = {
+          extractorType: memory.extractorType,
+          category: memory.category,
+          memoryPurpose: memory.memoryPurpose,
+          title: memory.title,
+          salience: memory.salience,
+          frequencyBoost: memory.frequencyBoost ?? 0,
+          saved: false,
+          error: null,
+        };
         try {
           await saveMemory(db, memory, project);
           autoExtractedCount++;
+          candidate.saved = true;
           const boostInfo = memory.frequencyBoost > 0 ? ` +${memory.frequencyBoost.toFixed(2)} boost` : '';
           console.error(`[auto-extract] Saved: ${memory.title} (salience: ${memory.salience.toFixed(2)}${boostInfo}, category: ${memory.category})`);
         } catch (err) {
+          candidate.error = err.message;
           console.error(`[auto-extract] Failed to save "${memory.title}": ${err.message}`);
         }
+        precompactCandidates.push(candidate);
       }
     } else {
       notes = 'no-content';
     }
 
+    // v4.25.0: write the run to the rolling ring buffer at
+    // ~/.shieldcortex/precompact-log/. Best-effort — never throw.
+    try {
+      writePrecompactLog({
+        thresholdUsed: dynamicThreshold,
+        contextFullnessPct: maxMemories > 0 ? Math.round((totalMemories / maxMemories) * 100) : null,
+        totalMemories,
+        rawSegmentCount,
+        candidates: precompactCandidates,
+      });
+    } catch { /* best-effort */ }
+
     console.error(`[shieldcortex] Pre-compact complete: ${autoExtractedCount} memories auto-extracted`);
 
     outputReminder(autoExtractedCount, dynamicThreshold);

package/scripts/prompt-recall-hook.mjs

@@ -80,8 +80,15 @@ function recallRelevant(db, project, prompt) {
   const ftsQuery = escapeFts5(prompt);
   if (ftsQuery.trim()) {
     try {
+      // v4.25.0: project pinned/access_count/last_accessed/downvote_count
+      // so compareRecallResults can compute effective salience without a
+      // second query. COALESCE on downvote_count for DBs that pre-date the
+      // 4.25 migration (the prompt-recall-hook runs against legacy DBs too).
       const ftsRows = db.prepare(`
-        SELECT m.id, m.title, m.content, m.category, m.salience, fts.rank
+        SELECT
+          m.id, m.title, m.content, m.category, m.salience, fts.rank,
+          m.pinned, m.access_count, m.last_accessed,
+          COALESCE(m.downvote_count, 0) AS downvote_count
         FROM memories m
         JOIN memories_fts fts ON m.id = fts.rowid
         WHERE memories_fts MATCH ?
@@ -112,8 +119,14 @@ function recallRelevant(db, project, prompt) {
 
   if (categoryBoost && results.length < MAX_RESULTS) {
     try {
+      // v4.25.0: project the salience-formula inputs (same as the FTS
+      // SELECT above) so compareRecallResults can compute effective
+      // salience for rows that came in via this fallback path.
       const catRows = db.prepare(`
-        SELECT id, title, content, category, salience
+        SELECT
+          id, title, content, category, salience,
+          pinned, access_count, last_accessed,
+          COALESCE(downvote_count, 0) AS downvote_count
         FROM memories
         WHERE category = ?
           AND (project = ? OR project IS NULL OR scope = 'global')