shieldcortex 4.28.1 → 4.29.0

package/scripts/lib/session-context.mjs

@@ -0,0 +1,30 @@
+/**
+ * Session-start context ordering helpers.
+ *
+ * The boot preamble used to order high-priority memories by raw
+ * `salience DESC, last_accessed DESC`. Because ~80% of rows sit at raw
+ * salience=1.0 (the "wall"), that ordering was an 80% tie and effectively
+ * random. Ranking by *effective* salience (recency × access × pin ×
+ * downvote_penalty — see scripts/lib/salience.mjs) lets fresh/relevant
+ * memories rank above stale 1.0 ones.
+ *
+ * Kept pure + side-effect-free so it's unit-testable without importing the
+ * hook entrypoint (which runs a stdin/process.exit IIFE at module load).
+ */
+
+import { computeEffectiveSalience } from './salience.mjs';
+
+/**
+ * Sort memories by effective salience, highest first. Returns a new array;
+ * the input is not mutated. `opts` is passed through to
+ * computeEffectiveSalience so tests can inject a deterministic `now`.
+ *
+ * @param {Array<object>} memories
+ * @param {object} [opts] - forwarded to computeEffectiveSalience (e.g. { now })
+ * @returns {Array<object>}
+ */
+export function orderByEffectiveSalience(memories, opts = {}) {
+  return [...memories].sort(
+    (a, b) => computeEffectiveSalience(b, opts) - computeEffectiveSalience(a, opts),
+  );
+}

package/scripts/postinstall.mjs

@@ -174,21 +174,40 @@ if (isGlobal && !isCI) {
       console.warn('[shieldcortex] To manually install after confirming OpenClaw support:');
       console.warn('[shieldcortex]   shieldcortex openclaw install --no-plugins');
     } else if (state.pluginInstalled) {
-      // Bug #15: Plugin exists from a previous install — auto-copy new files
+      // Bug #15 + Task 6b: a previous install left the plugin (and possibly the
+      // hook) on disk. Both are installed by file-copy, so a package bump
+      // leaves them stale until something re-copies them.
       console.log('');
-      console.log('[shieldcortex] Existing plugin detected. Copying updated plugin files...');
-      const copied = autoCopyPlugin(state.pluginDir, cliPath);
-      if (copied) {
-        console.log('[shieldcortex] Plugin files updated. Run full refresh for hook updates:');
-        console.log('[shieldcortex]   shieldcortex openclaw install');
-      } else {
-        // Fall back to full refresh
-        console.log('[shieldcortex] Auto-copy failed. Running full OpenClaw refresh...');
+      if (state.hookInstalled) {
+        // Hook present → run the full refresh ONLY. `shieldcortex openclaw
+        // install` re-copies the hook (copyHookFiles) AND re-installs/registers
+        // the plugin in one idempotent pass, so a standalone autoCopyPlugin()
+        // here would copy the plugin a second time for no benefit. The full
+        // refresh is the single guarantee that keeps the file-copied hook from
+        // going stale on a plugin-present update.
+        console.log('[shieldcortex] Refreshing OpenClaw hook + plugin to latest version...');
         const ok = refreshOpenClawInstall(cliPath);
         if (!ok) {
-          console.warn('[shieldcortex] ⚠  OpenClaw auto-refresh failed (non-fatal).');
+          console.warn('[shieldcortex] ⚠  OpenClaw hook + plugin refresh failed (non-fatal).');
           console.warn('[shieldcortex] Run manually: shieldcortex openclaw install');
         }
+      } else {
+        // Plugin-only (no hook) → just freshen the plugin files in place, with
+        // a full-refresh fallback if the in-place copy fails.
+        console.log('[shieldcortex] Existing plugin detected. Copying updated plugin files...');
+        const copied = autoCopyPlugin(state.pluginDir, cliPath);
+        if (copied) {
+          console.log('[shieldcortex] Plugin files updated.');
+          console.log('[shieldcortex] No hook installed. Run full setup for hook + memory:');
+          console.log('[shieldcortex]   shieldcortex openclaw install');
+        } else {
+          console.warn('[shieldcortex] ⚠  Plugin auto-copy failed — running full refresh.');
+          const ok = refreshOpenClawInstall(cliPath);
+          if (!ok) {
+            console.warn('[shieldcortex] ⚠  OpenClaw auto-refresh failed (non-fatal).');
+            console.warn('[shieldcortex] Run manually: shieldcortex openclaw install');
+          }
+        }
       }
     } else if (state.hookInstalled) {
       // Hook exists but no plugin — run full refresh

package/scripts/prompt-recall-hook.mjs

@@ -22,6 +22,7 @@ import { truncatePreservingWords } from './lib/truncate.mjs';
 import { compareRecallResults } from './lib/recall-rank.mjs';
 import { computeEffectiveSalience } from './lib/salience.mjs';
 import { writeRecallLog } from './lib/recall-log.mjs';
+import { filterByRelevance, extractQueryTerms } from './lib/recall-relevance.mjs';
 
 // ==================== CONFIG ====================
 
@@ -37,6 +38,47 @@ const MIN_SALIENCE = 0.2;
 const DEDUP_RING_SIZE = 5;
 const DEDUP_TTL_MS = 60 * 60 * 1000; // forget session rings after 1h idle
 
+// ── P4 recall relevance gate (B9) ────────────────────────────────────────
+// Term-coverage is the PRIMARY gate; relative BM25 floor is secondary. SHADOW
+// MODE is the default: we compute the gate and LOG what it would drop, but
+// inject the original set unchanged until SHIELDCORTEX_RECALL_ENFORCE=1. This
+// lets us tune thresholds from real fleet recall-logs before trading recall
+// noise for amnesia. All thresholds are env-tunable (pickNumber pattern,
+// mirrors scripts/lib/salience.mjs).
+
+// Resolve a numeric tuning constant from a SHIELDCORTEX_* env var, falling back
+// to the documented default. Returns the default for a NaN/empty env var so a
+// typo can't silently zero out a threshold. (Same contract as salience.mjs's
+// pickNumber, minus the explicit-override arg — the hook has no overrides.)
+function pickNumber(envName, fallback) {
+  const fromEnv = Number(process.env[envName]);
+  if (Number.isFinite(fromEnv) && process.env[envName] !== undefined && process.env[envName] !== '') {
+    return fromEnv;
+  }
+  return fallback;
+}
+
+// Truthy env flag: "1", "true", "yes", "on" (case-insensitive). Unset = false.
+function pickBool(envName) {
+  const v = (process.env[envName] || '').trim().toLowerCase();
+  return v === '1' || v === 'true' || v === 'yes' || v === 'on';
+}
+
+const RECALL_ENFORCE = pickBool('SHIELDCORTEX_RECALL_ENFORCE');
+const RECALL_MIN_TERMS = pickNumber('SHIELDCORTEX_RECALL_MIN_TERMS', 2);
+const RECALL_REL_FACTOR = pickNumber('SHIELDCORTEX_RECALL_REL_FACTOR', 0.35);
+// Absolute BM25 floor is OFF by default (null) — it has no safe default across
+// corpus sizes: on a small/new FTS index real bm25 ranks are tiny (~-1e-6), so
+// any fixed floor would drop even a perfect full-coverage match the moment an
+// operator sets SHIELDCORTEX_RECALL_ENFORCE=1. The relative floor + term-
+// coverage do the gating; tune the absolute floor from fleet recall-logs if a
+// real corpus shows it's needed. Read the env only if present; absent → null.
+const RECALL_MAX_BM25 =
+  process.env.SHIELDCORTEX_RECALL_MAX_BM25 !== undefined &&
+  process.env.SHIELDCORTEX_RECALL_MAX_BM25 !== ''
+    ? pickNumber('SHIELDCORTEX_RECALL_MAX_BM25', null)
+    : null;
+
 function loadConfig() {
   try {
     if (existsSync(CONFIG_PATH)) {
@@ -62,14 +104,14 @@ function getDbPath() {
 
 // ==================== FTS5 QUERY ====================
 
+// Build the FTS5 MATCH expression from the same distinct terms the relevance
+// gate scores against. extractQueryTerms (recall-relevance.mjs) is the single
+// source of truth for "which words count" — strip FTS5 operators, drop boolean
+// keywords, keep words longer than two chars, cap at six. Keeping both in sync
+// is load-bearing: the gate must score the SAME terms the FTS query OR-joined.
 function escapeFts5(query) {
-  return query
-    .replace(/[*(){}[\]<>~^"]/g, ' ')
-    .replace(/\b(AND|OR|NOT|NEAR)\b/gi, '')
-    .split(/\s+/)
-    .filter(w => w.length > 2)
-    .map(w => `"${w}"`)
-    .slice(0, 6)
+  return extractQueryTerms(query)
+    .map((w) => `"${w}"`)
     .join(' OR ');
 }
 
@@ -78,6 +120,7 @@ function escapeFts5(query) {
 function recallRelevant(db, project, prompt) {
   const results = [];
   const seen = new Set();
+  let ftsHits = 0;
 
   // 1. FTS5 text search
   const ftsQuery = escapeFts5(prompt);
@@ -110,6 +153,7 @@ function recallRelevant(db, project, prompt) {
           // it doesn't collide with any future SELECT column.
           row._source = 'fts';
           results.push(row);
+          ftsHits += 1;
         }
       }
     } catch {
@@ -124,7 +168,13 @@ function recallRelevant(db, project, prompt) {
   else if (/\b(deploy|release|publish|ship)\b/.test(promptLower)) categoryBoost = 'architecture';
   else if (/\b(prefer|style|convention|format)\b/.test(promptLower)) categoryBoost = 'preference';
 
-  if (categoryBoost && results.length < MAX_RESULTS) {
+  // P4 (B9 item 6): only fire the category-boost fallback when the FTS path
+  // found at least one on-topic hit. The fallback has ZERO query relevance —
+  // it matches purely on category + salience — so firing it whenever the
+  // result set is short was a back-door for off-topic high-salience rows on a
+  // prompt that didn't match ANY memory. Requiring an FTS hit means the prompt
+  // is genuinely on-topic for this project before we top up from its category.
+  if (categoryBoost && ftsHits > 0 && results.length < MAX_RESULTS) {
     try {
       // v4.25.0: project the salience-formula inputs (same as the FTS
       // SELECT above) so compareRecallResults can compute effective
@@ -160,7 +210,16 @@ function recallRelevant(db, project, prompt) {
   // v4.25.1: return the full sorted set alongside the sliced top-N so the
   // recall ring buffer can log which candidates were considered but didn't
   // make the cut. The top-N is the existing public contract.
-  return { topN: results.slice(0, MAX_RESULTS), fullSet: results };
+  //
+  // P4: also surface the distinct query terms (the same set escapeFts5 OR-
+  // joined) and the FTS hit count so the caller can run the relevance gate
+  // and the dedupe step against a consistent term list.
+  return {
+    topN: results.slice(0, MAX_RESULTS),
+    fullSet: results,
+    queryTerms: extractQueryTerms(prompt),
+    ftsHits,
+  };
 }
 
 // ==================== DEDUPE STATE ====================
@@ -213,13 +272,26 @@ function hashContent(s) {
 
 const RECALL_LOG_PROMPT_CAP = 200;
 
-function buildLogCandidates({ fullSet, topN, injected, dedupHashes }) {
+function buildLogCandidates({ fullSet, topN, injected, dedupHashes, relevanceDrops }) {
   const injectedIds = new Set(injected.map((m) => m.id));
   const topNIds = new Set(topN.map((m) => m.id));
+  // P4: id → 'below_term_coverage' | 'below_relevance_floor'. Populated even
+  // in shadow mode (where these rows ARE still injected) so the recall-log
+  // surfaces "considered but below floor" to `shieldcortex inspect last-recall`
+  // for threshold tuning before enforcement. Map is empty when the gate found
+  // nothing to drop.
+  const relDrops = relevanceDrops instanceof Map ? relevanceDrops : new Map();
   return fullSet.map((row) => {
     const wasInjected = injectedIds.has(row.id);
     let dropReason = null;
-    if (!wasInjected) {
+    // The relevance-gate reason takes precedence for a row it flagged: it's the
+    // most specific "why this is noise" signal, and in shadow mode it's the
+    // ONLY drop signal (the row is still injected, so the dedupe/top-N branches
+    // below wouldn't fire). When enforcing, a gate-dropped row is also not
+    // injected, so this is still the correct reason.
+    if (relDrops.has(row.id)) {
+      dropReason = relDrops.get(row.id);
+    } else if (!wasInjected) {
       if (topNIds.has(row.id) && dedupHashes.has(hashContent(row.content))) {
         dropReason = 'dedupe';
       } else if (!topNIds.has(row.id)) {
@@ -246,7 +318,7 @@ function buildLogCandidates({ fullSet, topN, injected, dedupHashes }) {
   });
 }
 
-function logRecallRun({ prompt, sessionId, project, fullSet, topN, injected, dedupHashes, context }) {
+function logRecallRun({ prompt, sessionId, project, fullSet, topN, injected, dedupHashes, context, relevanceDrops }) {
   try {
     const promptCapped = prompt.length > RECALL_LOG_PROMPT_CAP
       ? prompt.slice(0, RECALL_LOG_PROMPT_CAP) + '…'
@@ -257,7 +329,7 @@ function logRecallRun({ prompt, sessionId, project, fullSet, topN, injected, ded
       sessionId: sessionId ?? null,
       project: project ?? null,
       minSalience: MIN_SALIENCE,
-      candidates: buildLogCandidates({ fullSet, topN, injected, dedupHashes }),
+      candidates: buildLogCandidates({ fullSet, topN, injected, dedupHashes, relevanceDrops }),
       injectedCount: injected.length,
       finalContextChars: context ? context.length : 0,
     });
@@ -362,14 +434,42 @@ process.stdin.on('end', async () => {
     }
 
     const db = new Database(dbPath, { readonly: true, timeout: 2000 });
-    const { topN, fullSet } = recallRelevant(db, project, prompt);
+    const { topN, fullSet, queryTerms } = recallRelevant(db, project, prompt);
     db.close();
-    let memories = topN;
+
+    // ── P4 recall relevance gate (B9) ───────────────────────────────────
+    // Run the term-coverage + relative-BM25 gate over the would-be-injected
+    // top-N. SHADOW MODE (default): record the would-drops in the recall log
+    // but inject the ORIGINAL top-N unchanged. ENFORCE mode
+    // (SHIELDCORTEX_RECALL_ENFORCE=1): inject only the survivors. Either way
+    // the dropped rows STAY in fullSet so `inspect last-recall` shows them as
+    // "considered but below floor".
+    const { kept, dropped } = filterByRelevance(topN, {
+      queryTerms,
+      minTermMatches: RECALL_MIN_TERMS,
+      relFactor: RECALL_REL_FACTOR,
+      maxBm25: RECALL_MAX_BM25,
+    });
+    const relevanceDrops = new Map(dropped.map((d) => [d.row.id, d.reason]));
+    if (dropped.length > 0) {
+      // Visible in the hook's stderr diagnostics; the structured detail lands
+      // in the recall log via relevanceDrops.
+      console.error(
+        `[shieldcortex] Recall relevance gate (${RECALL_ENFORCE ? 'ENFORCE' : 'SHADOW'}): ` +
+          `${dropped.length}/${topN.length} would-drop ` +
+          `(${dropped.map((d) => `#${d.row.id}:${d.reason}`).join(', ')})`,
+      );
+    }
+    // SHADOW = inject original top-N; ENFORCE = inject only survivors.
+    let memories = RECALL_ENFORCE ? kept : topN;
 
     if (memories.length === 0) {
       // v4.25.1: still log no-candidate runs ONLY when the FTS step actually
       // ran (fullSet has something filtered out before the slice). Pure
       // empties (no FTS match at all) skip the log — saves disk + signal.
+      // P4: in ENFORCE mode the gate can empty the set (the "weather query →
+      // inject nothing" case) even though fullSet has rows — those are logged
+      // with their relevance dropReason. The empty-path logger still fires.
       if (fullSet.length > 0) {
         logRecallRun({
           prompt,
@@ -380,6 +480,7 @@ process.stdin.on('end', async () => {
           injected: [],
           dedupHashes: new Set(),
           context: null,
+          relevanceDrops,
         });
       }
       process.exit(0);
@@ -409,6 +510,7 @@ process.stdin.on('end', async () => {
           injected: [],
           dedupHashes,
           context: null,
+          relevanceDrops,
         });
         process.exit(0);
       }
@@ -455,6 +557,7 @@ process.stdin.on('end', async () => {
       injected: memories,
       dedupHashes,
       context,
+      relevanceDrops,
     });
 
     console.log(JSON.stringify(output));

package/scripts/session-start-hook.mjs

@@ -24,6 +24,7 @@ import { join } from 'path';
 import { homedir } from 'os';
 import { deriveProjectKey } from './lib/project-key.mjs';
 import { truncatePreservingWords } from './lib/truncate.mjs';
+import { orderByEffectiveSalience } from './lib/session-context.mjs';
 
 const NEW_DB_DIR = join(homedir(), '.shieldcortex');
 const LEGACY_DB_DIR = join(homedir(), '.claude-cortex');
@@ -84,19 +85,29 @@ You have access to a persistent memory system. Use it proactively:
 function getProjectContext(db, project) {
   const memories = [];
 
-  const highPriority = db.prepare(`
-    SELECT id, title, content, category, type, salience, tags, created_at
+  // Over-fetch a candidate pool (raw salience is a stable pre-filter), then
+  // rank by *effective* salience in JS and slice. Widening the SQL LIMIT past
+  // MAX gives the effective-salience sort room to promote a fresher/more-active
+  // row over a higher raw-salience one *within the candidate window* — it is a
+  // pragmatic widening, not a guarantee that every deep-tail row is reconsidered.
+  // The projected pinned/access_count/last_accessed/downvote_count columns feed
+  // computeEffectiveSalience.
+  const candidates = db.prepare(`
+    SELECT id, title, content, category, type, salience, tags, created_at,
+           pinned, access_count, last_accessed, COALESCE(downvote_count, 0) AS downvote_count
     FROM memories
     WHERE (project = ? OR project IS NULL)
       AND salience >= ?
       AND type IN ('long_term', 'episodic')
     ORDER BY salience DESC, last_accessed DESC
     LIMIT ?
-  `).all(project, MIN_SALIENCE_THRESHOLD, MAX_CONTEXT_MEMORIES);
+  `).all(project, MIN_SALIENCE_THRESHOLD, MAX_CONTEXT_MEMORIES * 8);
 
-  memories.push(...highPriority);
+  const ranked = orderByEffectiveSalience(candidates).slice(0, MAX_CONTEXT_MEMORIES);
 
-  if (memories.length < 5) {
+  memories.push(...ranked);
+
+  if (memories.length < MAX_CONTEXT_MEMORIES) {
     const excludeIds = memories.map(m => m.id);
     const placeholders = excludeIds.length > 0 ? excludeIds.map(() => '?').join(',') : '0';
     const recent = db.prepare(`
@@ -106,7 +117,7 @@ function getProjectContext(db, project) {
         AND id NOT IN (${placeholders})
       ORDER BY created_at DESC
       LIMIT ?
-    `).all(project, ...excludeIds, 5 - memories.length);
+    `).all(project, ...excludeIds, MAX_CONTEXT_MEMORIES - memories.length);
 
     memories.push(...recent);
   }