shieldcortex 4.14.11 → 4.16.0

package/dist/memory/ranker/rrf.js

@@ -0,0 +1,54 @@
+/**
+ * Reciprocal Rank Fusion (RRF).
+ *
+ * Combines ordered rank lists from multiple retrievers (e.g. BM25, vector,
+ * graph) into a single fused ranking. Robust to heterogeneous score scales:
+ * the only thing that matters is each retriever's rank order.
+ *
+ *   score(d) = Σ_i  w_i / (k + rank_i(d))
+ *
+ * - rank_i(d) is the 1-indexed position of d in retriever i's list.
+ * - w_i is the optional weight for retriever i (default 1).
+ * - k is a smoothing constant. Cormack et al. (2009) recommend 60.
+ *
+ * The function returns the union of ids across all rank lists, sorted by
+ * fused score descending. Ties break by id ascending for stable output.
+ */
+const DEFAULT_K = 60;
+export function reciprocalRankFusion(rankLists, options = {}) {
+    const k = options.k ?? DEFAULT_K;
+    const accumulator = new Map();
+    for (const list of rankLists) {
+        if (list.ids.length === 0)
+            continue;
+        const weight = list.weight ?? 1;
+        const seen = new Set();
+        list.ids.forEach((id, idx) => {
+            if (seen.has(id))
+                return; // dedupe within a single rank list, keep first
+            seen.add(id);
+            const rank = idx + 1;
+            const weighted = weight / (k + rank);
+            const entry = accumulator.get(id);
+            const contribution = { name: list.name, rank, weighted };
+            if (entry) {
+                entry.score += weighted;
+                entry.contributions.push(contribution);
+            }
+            else {
+                accumulator.set(id, { score: weighted, contributions: [contribution] });
+            }
+        });
+    }
+    const results = Array.from(accumulator.entries()).map(([id, { score, contributions }]) => ({
+        id,
+        score,
+        contributions,
+    }));
+    results.sort((a, b) => {
+        if (b.score !== a.score)
+            return b.score - a.score;
+        return a.id - b.id; // stable tie-break by id ascending
+    });
+    return results;
+}

package/dist/memory/search-recall.js

@@ -34,6 +34,9 @@ import { reinforceFromSearch, enrichMemory } from './lifecycle.js';
 // Cyclic import — see header. rowToMemory and logAccessDenial live in store.ts;
 // both are only invoked inside function bodies below, never at module load.
 import { rowToMemory, logAccessDenial } from './store.js';
+import { runHybridRanker } from './ranker/index.js';
+import { graphRankFromQuery } from './ranker/graph-rank.js';
+import { getRankerConfig } from '../cloud/config.js';
 let searchCount = 0;
 async function searchMemoriesInternal(options, config, source, execution) {
     if (++searchCount % 100 === 0) {
@@ -124,72 +127,45 @@ async function searchMemoriesInternal(options, config, source, execution) {
         vectorResults,
         query: options.query,
     };
-    const results = rows.map((row) => {
-        const memory = rowToMemory(row);
-        const decayedScore = calculateDecayedScore(memory, config);
-        memory.decayedScore = decayedScore;
+    // FTS-derived per-row scores (legacy: used directly; RRF: kept for explanations)
+    const ftsScores = new Map();
+    for (const row of rows) {
+        const id = row.id;
         const rawRank = row.rank;
-        const ftsScore = rawRank ? 1 / (1 + Math.abs(rawRank)) : 0.3;
-        const hoursSinceAccess = (Date.now() - new Date(memory.lastAccessed).getTime()) / (1000 * 60 * 60);
-        const recencyBoost = hoursSinceAccess < 1 ? 0.1 : (hoursSinceAccess < 24 ? 0.05 : 0);
-        const categoryBoost = detectedCategory && memory.category === detectedCategory ? 0.1 : 0;
-        const linkBoost = calculateLinkBoost(memory.id, db);
-        const tagBoost = calculateTagScore(queryTags, memory.tags);
-        const activationBoost = getActivationBoost(memory.id);
-        const vectorSimilarity = vectorResults.get(memory.id) || 0;
-        const vectorBoost = vectorSimilarity * 0.3;
-        const priorityBoost = calculatePriority(memory) * 0.05;
-        const contradictionCount = db.prepare(`SELECT COUNT(*) as count FROM memory_links WHERE relationship = 'contradicts' AND (source_id = ? OR target_id = ?)`).get(memory.id, memory.id).count;
-        const contradictionPenalty = Math.min(0.12, contradictionCount * 0.03);
-        const eligibilityReasons = [];
-        if (memory.status === 'archived')
-            eligibilityReasons.push('Archived memories are excluded from normal recall');
-        if (memory.status === 'suppressed')
-            eligibilityReasons.push('Suppressed memories are excluded from normal recall');
-        if (memory.cloudExcluded)
-            eligibilityReasons.push('Excluded from cloud sync');
-        if (memory.trustScore < 0.7)
-            eligibilityReasons.push(`Low trust source (${memory.trustScore.toFixed(2)})`);
-        if (contradictionCount > 0)
-            eligibilityReasons.push(`${contradictionCount} contradiction link${contradictionCount === 1 ? '' : 's'} attached`);
-        const relevanceScore = (ftsScore * 0.25 +
-            vectorBoost +
-            decayedScore * 0.2 +
-            priorityBoost +
-            recencyBoost + categoryBoost + linkBoost + tagBoost + activationBoost -
-            contradictionPenalty);
-        const result = {
-            memory,
-            relevanceScore,
-            recallEligibility: {
-                eligible: eligibilityReasons.length === 0,
-                reasons: eligibilityReasons,
-            },
-        };
-        if (execution.includeExplanation) {
-            result.explanation = buildSearchExplanation(memory, scoringContext, {
-                ftsScore,
-                vectorSimilarity,
-                vectorBoost,
-                decayedScore,
-                priorityBoost,
-                recencyBoost,
-                categoryBoost,
-                linkBoost,
-                tagBoost,
-                activationBoost,
-                contradictionPenalty,
-                finalScore: relevanceScore,
-            });
-            if (result.explanation) {
-                result.explanation.eligibility = result.recallEligibility;
-            }
-        }
-        return result;
-    });
-    const sortedResults = results
-        .filter((result) => options.includeDecayed || result.memory.decayedScore >= config.salienceThreshold)
-        .sort((a, b) => b.relevanceScore - a.relevanceScore);
+        ftsScores.set(id, rawRank ? 1 / (1 + Math.abs(rawRank)) : 0.3);
+    }
+    // Resolve ranker engine. Honour caller-supplied config.ranker if present
+    // (tests and explicit callers can pin engine), otherwise fall back to the
+    // env/file resolver in cloud/config.
+    const rankerConfig = config.ranker ?? getRankerConfig();
+    let sortedResults;
+    if (rankerConfig.engine === 'rrf') {
+        sortedResults = scoreWithRrf({
+            rows,
+            ftsScores,
+            vectorResults,
+            query: options.query,
+            options,
+            db,
+            config,
+            rankerConfig,
+            limit,
+            includeExplanation: execution.includeExplanation,
+        });
+    }
+    else {
+        sortedResults = scoreLegacy({
+            rows,
+            vectorResults,
+            detectedCategory,
+            queryTags,
+            db,
+            config,
+            includeDecayed: options.includeDecayed ?? false,
+            includeExplanation: execution.includeExplanation,
+            scoringContext,
+        });
+    }
     if (execution.enableSideEffects) {
         const topResults = sortedResults.slice(0, 5);
         for (const result of topResults) {
@@ -266,6 +242,161 @@ async function searchMemoriesInternal(options, config, source, execution) {
     }
     return finalResults;
 }
+function scoreLegacy(input) {
+    const { rows, vectorResults, detectedCategory, queryTags, db, config, includeDecayed, includeExplanation, scoringContext, } = input;
+    const results = rows.map((row) => {
+        const memory = rowToMemory(row);
+        const decayedScore = calculateDecayedScore(memory, config);
+        memory.decayedScore = decayedScore;
+        const rawRank = row.rank;
+        const ftsScore = rawRank ? 1 / (1 + Math.abs(rawRank)) : 0.3;
+        const hoursSinceAccess = (Date.now() - new Date(memory.lastAccessed).getTime()) / (1000 * 60 * 60);
+        const recencyBoost = hoursSinceAccess < 1 ? 0.1 : (hoursSinceAccess < 24 ? 0.05 : 0);
+        const categoryBoost = detectedCategory && memory.category === detectedCategory ? 0.1 : 0;
+        const linkBoost = calculateLinkBoost(memory.id, db);
+        const tagBoost = calculateTagScore(queryTags, memory.tags);
+        const activationBoost = getActivationBoost(memory.id);
+        const vectorSimilarity = vectorResults.get(memory.id) || 0;
+        const vectorBoost = vectorSimilarity * 0.3;
+        const priorityBoost = calculatePriority(memory) * 0.05;
+        const contradictionCount = db.prepare(`SELECT COUNT(*) as count FROM memory_links WHERE relationship = 'contradicts' AND (source_id = ? OR target_id = ?)`).get(memory.id, memory.id).count;
+        const contradictionPenalty = Math.min(0.12, contradictionCount * 0.03);
+        const eligibilityReasons = [];
+        if (memory.status === 'archived')
+            eligibilityReasons.push('Archived memories are excluded from normal recall');
+        if (memory.status === 'suppressed')
+            eligibilityReasons.push('Suppressed memories are excluded from normal recall');
+        if (memory.cloudExcluded)
+            eligibilityReasons.push('Excluded from cloud sync');
+        if (memory.trustScore < 0.7)
+            eligibilityReasons.push(`Low trust source (${memory.trustScore.toFixed(2)})`);
+        if (contradictionCount > 0)
+            eligibilityReasons.push(`${contradictionCount} contradiction link${contradictionCount === 1 ? '' : 's'} attached`);
+        const relevanceScore = (ftsScore * 0.25 +
+            vectorBoost +
+            decayedScore * 0.2 +
+            priorityBoost +
+            recencyBoost + categoryBoost + linkBoost + tagBoost + activationBoost -
+            contradictionPenalty);
+        const result = {
+            memory,
+            relevanceScore,
+            recallEligibility: {
+                eligible: eligibilityReasons.length === 0,
+                reasons: eligibilityReasons,
+            },
+        };
+        if (includeExplanation) {
+            result.explanation = buildSearchExplanation(memory, scoringContext, {
+                ftsScore,
+                vectorSimilarity,
+                vectorBoost,
+                decayedScore,
+                priorityBoost,
+                recencyBoost,
+                categoryBoost,
+                linkBoost,
+                tagBoost,
+                activationBoost,
+                contradictionPenalty,
+                finalScore: relevanceScore,
+            });
+            if (result.explanation) {
+                result.explanation.eligibility = result.recallEligibility;
+            }
+        }
+        return result;
+    });
+    return results
+        .filter((result) => includeDecayed || result.memory.decayedScore >= config.salienceThreshold)
+        .sort((a, b) => b.relevanceScore - a.relevanceScore);
+}
+/**
+ * RRF retrieval path: build per-retriever rank lists from the FTS rows
+ * already produced by the SQL prelude, the cosine-similarity vector hits,
+ * and a fresh `graphRankFromQuery` call. Hand them to `runHybridRanker`,
+ * which fuses with RRF and applies post-fusion multiplicative boosts.
+ *
+ * Vector and graph rank lists may surface ids the FTS WHERE clause would
+ * have filtered out (category/type/status/tags/minSalience). Those are
+ * dropped at memory-map build time so the same filters apply across all
+ * three retrievers, keeping search behaviour predictable.
+ */
+function scoreWithRrf(input) {
+    const { rows, ftsScores, vectorResults, query, options, db, config, rankerConfig, limit, includeExplanation, } = input;
+    // Memory map starts with FTS rows (already passed all SQL filters).
+    const memoryMap = new Map();
+    const ftsIds = [];
+    for (const row of rows) {
+        const memory = rowToMemory(row);
+        memoryMap.set(memory.id, memory);
+        ftsIds.push(memory.id);
+    }
+    // Vector and graph rank lists may contain ids the FTS WHERE would have
+    // filtered out — fetch + filter to enforce the same constraints.
+    const considerCandidate = (id) => {
+        if (memoryMap.has(id))
+            return true;
+        const row = db.prepare('SELECT * FROM memories WHERE id = ?').get(id);
+        if (!row)
+            return false;
+        const memory = rowToMemory(row);
+        if (!candidatePassesFilters(memory, options))
+            return false;
+        memoryMap.set(memory.id, memory);
+        return true;
+    };
+    const vectorIds = Array.from(vectorResults.entries())
+        .sort((a, b) => b[1] - a[1])
+        .map(([id]) => id)
+        .filter(considerCandidate);
+    const graphLimit = Math.max(limit * 3, 50);
+    const graphResults = query && query.trim()
+        ? graphRankFromQuery(query, db, { project: options.project, limit: graphLimit })
+        : [];
+    const graphIds = graphResults.map((r) => r.memoryId).filter(considerCandidate);
+    return runHybridRanker({
+        rankLists: [
+            { name: 'fts', ids: ftsIds, weight: rankerConfig.weights.fts },
+            { name: 'vector', ids: vectorIds, weight: rankerConfig.weights.vector },
+            { name: 'graph', ids: graphIds, weight: rankerConfig.weights.graph },
+        ],
+        memories: memoryMap,
+        vectorSimilarities: vectorResults,
+        ftsScores,
+        query,
+        db,
+        config,
+        rrfK: rankerConfig.rrfK,
+        includeExplanation,
+        includeDecayed: options.includeDecayed ?? false,
+        limit,
+    });
+}
+/**
+ * Mirror the WHERE-clause filters from the FTS SQL so that vector/graph
+ * candidates not returned by FTS are held to the same standard. Project
+ * scoping is already handled at the source (`vectorSearch` and
+ * `graphRankFromQuery`) so it isn't duplicated here.
+ */
+function candidatePassesFilters(memory, options) {
+    if (options.category && memory.category !== options.category)
+        return false;
+    if (options.type && memory.type !== options.type)
+        return false;
+    if (!options.includeArchived && memory.status === 'archived')
+        return false;
+    if (!options.includeSuppressed && memory.status === 'suppressed')
+        return false;
+    if (options.minSalience !== undefined && memory.salience < options.minSalience)
+        return false;
+    if (options.tags && options.tags.length > 0) {
+        const hasTag = options.tags.some((tag) => memory.tags.includes(tag));
+        if (!hasTag)
+            return false;
+    }
+    return true;
+}
 /**
  * Search memories using full-text search, vector similarity, and filters
  * Now uses hybrid search combining FTS5 keywords with semantic vector matching

package/dist/memory/types.d.ts

@@ -141,6 +141,22 @@ export interface SalienceFactors {
     hasCodeReference: boolean;
     emotionalMarkers: boolean;
 }
+/**
+ * Hybrid retrieval engine selection. RRF is the v4.15 default; `legacy`
+ * keeps the pre-v4.15 weighted-sum scoring as a one-release safety belt.
+ */
+export type RankerEngine = 'rrf' | 'legacy';
+export interface RankerWeights {
+    fts: number;
+    vector: number;
+    graph: number;
+}
+export interface RankerConfig {
+    engine: RankerEngine;
+    /** RRF smoothing constant. Cormack et al. (2009) recommend 60. */
+    rrfK: number;
+    weights: RankerWeights;
+}
 export interface MemoryConfig {
     dbPath: string;
     decayRate: number;
@@ -150,7 +166,9 @@ export interface MemoryConfig {
     maxShortTermMemories: number;
     maxLongTermMemories: number;
     autoConsolidateHours: number;
+    ranker: RankerConfig;
 }
+export declare const DEFAULT_RANKER_CONFIG: RankerConfig;
 export declare const DEFAULT_CONFIG: MemoryConfig;
 /**
  * Category-specific deletion thresholds

package/dist/memory/types.js

@@ -1,6 +1,11 @@
 /**
  * Core type definitions for the ShieldCortex memory system
  */
+export const DEFAULT_RANKER_CONFIG = {
+    engine: 'rrf',
+    rrfK: 60,
+    weights: { fts: 0.4, vector: 0.6, graph: 0.3 },
+};
 export const DEFAULT_CONFIG = {
     dbPath: '~/.shieldcortex/memories.db', // Note: actual path auto-detected with legacy fallback
     decayRate: 0.995,
@@ -10,6 +15,7 @@ export const DEFAULT_CONFIG = {
     maxShortTermMemories: 100,
     maxLongTermMemories: 1000,
     autoConsolidateHours: 4,
+    ranker: DEFAULT_RANKER_CONFIG,
 };
 /**
  * Category-specific deletion thresholds

package/hooks/openclaw/cortex-memory/HOOK.md

@@ -64,6 +64,43 @@ Say any of these phrases to trigger an instant save to Cortex memory:
 
 Content after the trigger phrase is extracted and saved as the memory content.
 
+## Defence Audit Guarantees
+
+Every byte that lands in `memories` from the auto-extract path passes the
+6-layer defence pipeline first. The hook write path is no longer the
+bypass it once was:
+
+- **ALLOW** → row inserted into `memories`; a corresponding row appears in
+  `defence_audit` with `source_type = 'hook'` and the hook's identifier
+  (`session-end-hook` / `pre-compact-hook` / `stop-hook`).
+- **QUARANTINE** → row inserted into `quarantine` (not `memories`), linked
+  to the audit row via `audit_id`. Visible in the dashboard for review.
+- **BLOCK** → dropped. The audit row written by the pipeline carries the
+  block reason; nothing reaches `memories`.
+- **Pipeline error** → dropped + a synthetic audit row with reason
+  `pipeline_error: <msg>`. Never silently lose data.
+
+Built-in firewall rules covering instruction injection, hidden
+instruction, imperative tool-call directives ("call X tool now"), command
+injection, and credential leaks (AWS / JWT / private keys) are seeded
+into `firewall_rules` on first run with `built_in = 1`. They are
+evaluated on every tier (the Pro `custom_firewall_rules` gate applies
+only to user-added rules) and excluded from the user-facing 25-rule cap.
+
+The chunker also rejects malformed candidates *before* they reach the
+write path: imperative tool-calls, bare-imperative starts ("commit
+secrets" with the negation dropped), email-body bleed, and path-label
+fragments. Auto-extracted memories are now capped at salience 0.6
+(reserved 1.0 for LLM-rated future paths).
+
+To audit an existing database for malformed rows accumulated before this
+fix:
+
+```bash
+shieldcortex memories purge --malformed --dry-run    # preview
+shieldcortex memories purge --malformed --execute    # delete (writes a backup first)
+```
+
 ## Auto-Memory
 
 Auto-memory extraction is enabled by default. ShieldCortex complements your existing memory system by capturing decisions, fixes, and learnings with built-in deduplication to avoid noise.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shieldcortex",
-  "version": "4.14.11",
+  "version": "4.16.0",
   "description": "Trustworthy memory and security for AI agents. Recall debugging, review queue, OpenClaw session capture, and memory poisoning defence for Claude Code, Codex, OpenClaw, LangChain, and MCP agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -57,6 +57,8 @@
     "test": "node scripts/run-jest.mjs",
     "test:watch": "node scripts/run-jest.mjs --watch",
     "test:coverage": "node scripts/run-jest.mjs --coverage",
+    "bench": "tsx benchmark/longmemeval/run.ts",
+    "bench:smoke": "SHIELDCORTEX_SKIP_EMBEDDINGS=1 tsx benchmark/longmemeval/run.ts --quiet",
     "audit:security": "npm audit --audit-level=moderate",
     "prepack": "node scripts/ensure-bin-executable.mjs",
     "prepublishOnly": "npm run build"

package/plugins/openclaw/dist/openclaw.plugin.json

@@ -1,6 +1,6 @@
 {
   "id": "shieldcortex-realtime",
-  "version": "4.14.11",
+  "version": "4.16.0",
   "name": "ShieldCortex Real-time Scanner",
   "description": "Real-time defence scanning on LLM input, memory extraction on LLM output, and active tool call interception with approval gating.",
   "kind": null,