shieldcortex 2.16.2 → 2.17.0

package/plugins/openclaw/README.md

@@ -7,7 +7,7 @@ Real-time defence scanning and memory extraction for OpenClaw v2026.2.15+.
 | Hook | Action |
 |------|--------|
 | `llm_input` | Scans prompts + history through ShieldCortex defence pipeline. Logs threats, writes audit log, optionally syncs to cloud. **Fire-and-forget.** |
-| `llm_output` | Extracts decisions, fixes, learnings from assistant responses via pattern matching + salience scoring. Saves to ShieldCortex memory via mcporter. **Fire-and-forget.** |
+| `llm_output` | Optional memory extraction from assistant responses (disabled by default). Saves to ShieldCortex memory via mcporter when enabled, with novelty/dedupe filtering to reduce noise. **Fire-and-forget.** |
 
 ## Installation
 
@@ -39,6 +39,40 @@ Find the package root with `npm root -g` (global) or `npm root` (local).
 - ShieldCortex installed globally (`npm i -g shieldcortex`) or at `~/ShieldCortex/`
 - `mcporter` available via npx (for memory saves)
 
+## Optional Auto-Memory
+
+Auto-memory extraction is off by default to avoid duplicate/noisy memory when OpenClaw already has native memory.
+
+Enable it:
+
+```bash
+npx shieldcortex config --openclaw-auto-memory true
+```
+
+Disable it:
+
+```bash
+npx shieldcortex config --openclaw-auto-memory false
+```
+
+Enable it in `~/.shieldcortex/config.json`:
+
+```json
+{
+  "openclawAutoMemory": true
+}
+```
+
+Novelty filtering is enabled by default when auto-memory is on. Optional tuning keys:
+
+```json
+{
+  "openclawAutoMemoryDedupe": true,
+  "openclawAutoMemoryNoveltyThreshold": 0.88,
+  "openclawAutoMemoryMaxRecent": 300
+}
+```
+
 ## Cloud Sync (optional)
 
 Add your API key to `~/.shieldcortex/config.json`:

package/plugins/openclaw/dist/index.js

@@ -2,9 +2,10 @@
  * ShieldCortex Real-time Scanning Plugin for OpenClaw v2026.2.15+
  *
  * Hooks into llm_input/llm_output for real-time defence scanning
- * and memory extraction. All operations are fire-and-forget.
+ * and optional memory extraction. All operations are fire-and-forget.
  */
 import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { homedir } from "node:os";
@@ -20,6 +21,12 @@ async function loadConfig() {
     }
     return _config;
 }
+function isAutoMemoryEnabled(config) {
+    return config.openclawAutoMemory === true;
+}
+function isAutoMemoryDedupeEnabled(config) {
+    return config.openclawAutoMemoryDedupe !== false;
+}
 // ==================== SERVER CMD ====================
 let _serverCmd = null;
 async function resolveServerCmd() {
@@ -154,6 +161,10 @@ function extractUserContent(msgs) {
     return out;
 }
 const AUDIT_DIR = path.join(homedir(), ".shieldcortex", "audit");
+const NOVELTY_CACHE_FILE = path.join(homedir(), ".shieldcortex", "openclaw-memory-cache.json");
+const DEFAULT_NOVELTY_THRESHOLD = 0.88;
+const DEFAULT_MAX_RECENT = 300;
+const MIN_NOVELTY_CHARS = 40;
 async function auditLog(entry) {
     try {
         await fs.mkdir(AUDIT_DIR, { recursive: true });
@@ -175,6 +186,122 @@ async function cloudSync(threat) {
     }
     catch { }
 }
+function normalizeMemoryText(text) {
+    return String(text || "")
+        .toLowerCase()
+        .replace(/[`"'\\]/g, " ")
+        .replace(/https?:\/\/\S+/g, " ")
+        .replace(/[^a-z0-9\s]/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function hashToken(token) {
+    return createHash("sha1").update(token).digest("hex").slice(0, 12);
+}
+function buildTokenHashes(normalized) {
+    const words = normalized.split(" ").filter((w) => w.length >= 3);
+    const set = new Set();
+    for (let i = 0; i < words.length; i++) {
+        set.add(hashToken(words[i]));
+        if (i < words.length - 1)
+            set.add(hashToken(`${words[i]}_${words[i + 1]}`));
+    }
+    return Array.from(set).slice(0, 200);
+}
+function jaccardSimilarity(a, b) {
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const item of a) {
+        if (b.has(item))
+            intersection++;
+    }
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+function clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+}
+async function loadNoveltyCache(maxRecent) {
+    try {
+        const raw = JSON.parse(await fs.readFile(NOVELTY_CACHE_FILE, "utf-8"));
+        if (!Array.isArray(raw))
+            return [];
+        return raw
+            .filter((entry) => entry && typeof entry.hash === "string" && Array.isArray(entry.tokenHashes))
+            .slice(0, maxRecent);
+    }
+    catch {
+        return [];
+    }
+}
+async function saveNoveltyCache(entries) {
+    await fs.mkdir(path.dirname(NOVELTY_CACHE_FILE), { recursive: true });
+    await fs.writeFile(NOVELTY_CACHE_FILE, JSON.stringify(entries, null, 2) + "\n", "utf-8");
+}
+function inspectNovelty(content, entries, threshold) {
+    const normalized = normalizeMemoryText(content);
+    if (normalized.length < MIN_NOVELTY_CHARS) {
+        return { allow: true, contentHash: null, tokenHashes: [] };
+    }
+    const contentHash = createHash("sha256").update(normalized).digest("hex").slice(0, 24);
+    if (entries.some((entry) => entry.hash === contentHash)) {
+        return { allow: false, contentHash, tokenHashes: [], reason: "exact duplicate" };
+    }
+    const tokenHashes = buildTokenHashes(normalized);
+    const currentSet = new Set(tokenHashes);
+    for (const entry of entries) {
+        const score = jaccardSimilarity(currentSet, new Set(entry.tokenHashes || []));
+        if (score >= threshold) {
+            return {
+                allow: false,
+                contentHash,
+                tokenHashes,
+                reason: `near duplicate (similarity ${score.toFixed(2)})`,
+            };
+        }
+    }
+    return { allow: true, contentHash, tokenHashes };
+}
+async function createNoveltyGate(config) {
+    const thresholdRaw = Number(config.openclawAutoMemoryNoveltyThreshold);
+    const maxRecentRaw = Number(config.openclawAutoMemoryMaxRecent);
+    const threshold = Number.isFinite(thresholdRaw)
+        ? clamp(thresholdRaw, 0.6, 0.99)
+        : DEFAULT_NOVELTY_THRESHOLD;
+    const maxRecent = Number.isFinite(maxRecentRaw)
+        ? Math.floor(clamp(maxRecentRaw, 50, 1000))
+        : DEFAULT_MAX_RECENT;
+    const enabled = isAutoMemoryDedupeEnabled(config);
+    const entries = enabled ? await loadNoveltyCache(maxRecent) : [];
+    let dirty = false;
+    return {
+        inspect(content) {
+            if (!enabled)
+                return { allow: true, contentHash: null, tokenHashes: [] };
+            return inspectNovelty(content, entries, threshold);
+        },
+        remember(memory, novelty) {
+            if (!enabled || !novelty.contentHash || novelty.tokenHashes.length === 0)
+                return;
+            entries.unshift({
+                hash: novelty.contentHash,
+                tokenHashes: novelty.tokenHashes,
+                title: String(memory.title || "").slice(0, 120),
+                category: String(memory.category || "note"),
+                createdAt: new Date().toISOString(),
+            });
+            if (entries.length > maxRecent)
+                entries.length = maxRecent;
+            dirty = true;
+        },
+        async flush() {
+            if (!enabled || !dirty)
+                return;
+            await saveNoveltyCache(entries);
+        },
+    };
+}
 // ==================== HOOK HANDLERS ====================
 // Skip scanning internal OpenClaw content (boot checks, system prompts, heartbeats)
 const SKIP_PATTERNS = [
@@ -225,25 +352,38 @@ function handleLlmOutput(event, ctx) {
     // Fire and forget
     (async () => {
         try {
+            const config = await loadConfig();
+            if (!isAutoMemoryEnabled(config))
+                return;
             const texts = event.assistantTexts.filter(t => t && t.length >= 30);
             if (!texts.length)
                 return;
             const memories = extractMemories(texts);
             if (!memories.length)
                 return;
+            const noveltyGate = await createNoveltyGate(config);
             let saved = 0;
+            let skipped = 0;
             for (const mem of memories) {
+                const novelty = noveltyGate.inspect(mem.content);
+                if (!novelty.allow) {
+                    skipped++;
+                    continue;
+                }
                 const r = await callCortex("remember", {
                     title: mem.title, content: mem.content, category: mem.category,
                     project: ctx.agentId || "openclaw", scope: "global",
                     importance: "normal",
                 });
-                if (r)
+                if (r) {
                     saved++;
+                    noveltyGate.remember(mem, novelty);
+                }
             }
+            await noveltyGate.flush();
             if (saved) {
-                console.log(`[shieldcortex] Extracted ${saved} memor${saved === 1 ? "y" : "ies"} from LLM output`);
-                auditLog({ type: "memory", hook: "llm_output", sessionId: event.sessionId, count: saved, ts: new Date().toISOString() });
+                console.log(`[shieldcortex] Extracted ${saved} memor${saved === 1 ? "y" : "ies"} from LLM output (${skipped} duplicates skipped)`);
+                auditLog({ type: "memory", hook: "llm_output", sessionId: event.sessionId, count: saved, skipped, ts: new Date().toISOString() });
             }
         }
         catch (e) {
@@ -255,7 +395,7 @@ function handleLlmOutput(event, ctx) {
 export default {
     id: "shieldcortex-realtime",
     name: "ShieldCortex Real-time Scanner",
-    description: "Real-time defence scanning on LLM inputs and memory extraction from outputs",
+    description: "Real-time defence scanning on LLM inputs with optional memory extraction from outputs",
     version: "__SHIELDCORTEX_VERSION__",
     register(api) {
         api.on("llm_input", handleLlmInput);

package/plugins/openclaw/index.ts

@@ -2,10 +2,11 @@
  * ShieldCortex Real-time Scanning Plugin for OpenClaw v2026.2.15+
  *
  * Hooks into llm_input/llm_output for real-time defence scanning
- * and memory extraction. All operations are fire-and-forget.
+ * and optional memory extraction. All operations are fire-and-forget.
  */
 
 import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
 import fs from "node:fs/promises";
 import { readFileSync } from "node:fs";
 import path from "node:path";
@@ -34,7 +35,15 @@ type PluginApi = {
 
 // ==================== CONFIG ====================
 
-interface SCConfig { cloudApiKey?: string; cloudEndpoint?: string; binaryPath?: string }
+interface SCConfig {
+  cloudApiKey?: string;
+  cloudEndpoint?: string;
+  binaryPath?: string;
+  openclawAutoMemory?: boolean;
+  openclawAutoMemoryDedupe?: boolean;
+  openclawAutoMemoryNoveltyThreshold?: number;
+  openclawAutoMemoryMaxRecent?: number;
+}
 let _config: SCConfig | null = null;
 let _version = "0.0.0";
 try {
@@ -50,6 +59,14 @@ async function loadConfig(): Promise<SCConfig> {
   return _config!;
 }
 
+function isAutoMemoryEnabled(config: SCConfig): boolean {
+  return config.openclawAutoMemory === true;
+}
+
+function isAutoMemoryDedupeEnabled(config: SCConfig): boolean {
+  return config.openclawAutoMemoryDedupe !== false;
+}
+
 // ==================== SERVER CMD ====================
 
 let _serverCmd: string | null = null;
@@ -133,6 +150,10 @@ function extractUserContent(msgs: unknown[]): string[] {
 }
 
 const AUDIT_DIR = path.join(homedir(), ".shieldcortex", "audit");
+const NOVELTY_CACHE_FILE = path.join(homedir(), ".shieldcortex", "openclaw-memory-cache.json");
+const DEFAULT_NOVELTY_THRESHOLD = 0.88;
+const DEFAULT_MAX_RECENT = 300;
+const MIN_NOVELTY_CHARS = 40;
 
 async function auditLog(entry: Record<string, unknown>) {
   try {
@@ -157,6 +178,147 @@ async function cloudSync(threat: Record<string, unknown>) {
   } catch {}
 }
 
+type NoveltyEntry = {
+  hash: string;
+  tokenHashes: string[];
+  title: string;
+  category: string;
+  createdAt: string;
+};
+
+function normalizeMemoryText(text: string): string {
+  return String(text || "")
+    .toLowerCase()
+    .replace(/[`"'\\]/g, " ")
+    .replace(/https?:\/\/\S+/g, " ")
+    .replace(/[^a-z0-9\s]/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function hashToken(token: string): string {
+  return createHash("sha1").update(token).digest("hex").slice(0, 12);
+}
+
+function buildTokenHashes(normalized: string): string[] {
+  const words = normalized.split(" ").filter((w) => w.length >= 3);
+  const set = new Set<string>();
+
+  for (let i = 0; i < words.length; i++) {
+    set.add(hashToken(words[i]));
+    if (i < words.length - 1) set.add(hashToken(`${words[i]}_${words[i + 1]}`));
+  }
+
+  return Array.from(set).slice(0, 200);
+}
+
+function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 || b.size === 0) return 0;
+  let intersection = 0;
+  for (const item of a) {
+    if (b.has(item)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
+async function loadNoveltyCache(maxRecent: number): Promise<NoveltyEntry[]> {
+  try {
+    const raw = JSON.parse(await fs.readFile(NOVELTY_CACHE_FILE, "utf-8"));
+    if (!Array.isArray(raw)) return [];
+    return raw
+      .filter((entry) => entry && typeof entry.hash === "string" && Array.isArray(entry.tokenHashes))
+      .slice(0, maxRecent) as NoveltyEntry[];
+  } catch {
+    return [];
+  }
+}
+
+async function saveNoveltyCache(entries: NoveltyEntry[]): Promise<void> {
+  await fs.mkdir(path.dirname(NOVELTY_CACHE_FILE), { recursive: true });
+  await fs.writeFile(NOVELTY_CACHE_FILE, JSON.stringify(entries, null, 2) + "\n", "utf-8");
+}
+
+function inspectNovelty(content: string, entries: NoveltyEntry[], threshold: number): {
+  allow: boolean;
+  contentHash: string | null;
+  tokenHashes: string[];
+  reason?: string;
+} {
+  const normalized = normalizeMemoryText(content);
+  if (normalized.length < MIN_NOVELTY_CHARS) {
+    return { allow: true, contentHash: null, tokenHashes: [] };
+  }
+
+  const contentHash = createHash("sha256").update(normalized).digest("hex").slice(0, 24);
+  if (entries.some((entry) => entry.hash === contentHash)) {
+    return { allow: false, contentHash, tokenHashes: [], reason: "exact duplicate" };
+  }
+
+  const tokenHashes = buildTokenHashes(normalized);
+  const currentSet = new Set(tokenHashes);
+
+  for (const entry of entries) {
+    const score = jaccardSimilarity(currentSet, new Set(entry.tokenHashes || []));
+    if (score >= threshold) {
+      return {
+        allow: false,
+        contentHash,
+        tokenHashes,
+        reason: `near duplicate (similarity ${score.toFixed(2)})`,
+      };
+    }
+  }
+
+  return { allow: true, contentHash, tokenHashes };
+}
+
+async function createNoveltyGate(config: SCConfig): Promise<{
+  inspect: (content: string) => { allow: boolean; contentHash: string | null; tokenHashes: string[]; reason?: string };
+  remember: (memory: { title: string; category: string }, novelty: { contentHash: string | null; tokenHashes: string[] }) => void;
+  flush: () => Promise<void>;
+}> {
+  const thresholdRaw = Number(config.openclawAutoMemoryNoveltyThreshold);
+  const maxRecentRaw = Number(config.openclawAutoMemoryMaxRecent);
+  const threshold = Number.isFinite(thresholdRaw)
+    ? clamp(thresholdRaw, 0.6, 0.99)
+    : DEFAULT_NOVELTY_THRESHOLD;
+  const maxRecent = Number.isFinite(maxRecentRaw)
+    ? Math.floor(clamp(maxRecentRaw, 50, 1000))
+    : DEFAULT_MAX_RECENT;
+
+  const enabled = isAutoMemoryDedupeEnabled(config);
+  const entries = enabled ? await loadNoveltyCache(maxRecent) : [];
+  let dirty = false;
+
+  return {
+    inspect(content: string) {
+      if (!enabled) return { allow: true, contentHash: null, tokenHashes: [] };
+      return inspectNovelty(content, entries, threshold);
+    },
+    remember(memory, novelty) {
+      if (!enabled || !novelty.contentHash || novelty.tokenHashes.length === 0) return;
+      entries.unshift({
+        hash: novelty.contentHash,
+        tokenHashes: novelty.tokenHashes,
+        title: String(memory.title || "").slice(0, 120),
+        category: String(memory.category || "note"),
+        createdAt: new Date().toISOString(),
+      });
+      if (entries.length > maxRecent) entries.length = maxRecent;
+      dirty = true;
+    },
+    async flush() {
+      if (!enabled || !dirty) return;
+      await saveNoveltyCache(entries);
+    },
+  };
+}
+
 // ==================== HOOK HANDLERS ====================
 
 // Skip scanning internal OpenClaw content (boot checks, system prompts, heartbeats)
@@ -208,23 +370,38 @@ function handleLlmOutput(event: LlmOutputEvent, ctx: AgentCtx): void {
   // Fire and forget
   (async () => {
     try {
+      const config = await loadConfig();
+      if (!isAutoMemoryEnabled(config)) return;
+
       const texts = event.assistantTexts.filter(t => t && t.length >= 30);
       if (!texts.length) return;
       const memories = extractMemories(texts);
       if (!memories.length) return;
 
+      const noveltyGate = await createNoveltyGate(config);
       let saved = 0;
+      let skipped = 0;
       for (const mem of memories) {
+        const novelty = noveltyGate.inspect(mem.content);
+        if (!novelty.allow) {
+          skipped++;
+          continue;
+        }
+
         const r = await callCortex("remember", {
           title: mem.title, content: mem.content, category: mem.category,
           project: ctx.agentId || "openclaw", scope: "global",
           importance: "normal", tags: "auto-extracted,realtime-plugin,llm-output",
         });
-        if (r) saved++;
+        if (r) {
+          saved++;
+          noveltyGate.remember(mem, novelty);
+        }
       }
+      await noveltyGate.flush();
       if (saved) {
-        console.log(`[shieldcortex] Extracted ${saved} memor${saved === 1 ? "y" : "ies"} from LLM output`);
-        auditLog({ type: "memory", hook: "llm_output", sessionId: event.sessionId, count: saved, ts: new Date().toISOString() });
+        console.log(`[shieldcortex] Extracted ${saved} memor${saved === 1 ? "y" : "ies"} from LLM output (${skipped} duplicates skipped)`);
+        auditLog({ type: "memory", hook: "llm_output", sessionId: event.sessionId, count: saved, skipped, ts: new Date().toISOString() });
       }
     } catch (e) {
       console.error("[shieldcortex] llm_output error:", e instanceof Error ? e.message : String(e));
@@ -237,7 +414,7 @@ function handleLlmOutput(event: LlmOutputEvent, ctx: AgentCtx): void {
 export default {
   id: "shieldcortex-realtime",
   name: "ShieldCortex Real-time Scanner",
-  description: "Real-time defence scanning on LLM inputs and memory extraction from outputs",
+  description: "Real-time defence scanning on LLM inputs with optional memory extraction from outputs",
   version: _version,
 
   register(api: PluginApi) {

package/scripts/postinstall.mjs

@@ -3,12 +3,46 @@
  * Postinstall script - prints setup instructions after global install.
  * Does NOT auto-run setup (can fail in CI, user might not have Claude Code).
  */
+import { existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { homedir } from 'os';
+import { spawnSync } from 'child_process';
+import { fileURLToPath } from 'url';
 
 // Only show message for global installs (not local dev or CI)
 const isGlobal = process.env.npm_config_global === 'true';
 const isCI = process.env.CI === 'true' || process.env.CONTINUOUS_INTEGRATION === 'true';
+const skipAutoOpenClaw = process.env.SHIELDCORTEX_SKIP_AUTO_OPENCLAW === '1';
+
+function shouldRefreshOpenClaw() {
+  const home = homedir();
+  const openclawDir = join(home, '.openclaw');
+  const knownHook = join(openclawDir, 'hooks', 'cortex-memory');
+  const knownPlugin = join(openclawDir, 'extensions', 'shieldcortex-realtime');
+  return existsSync(openclawDir) || existsSync(knownHook) || existsSync(knownPlugin);
+}
+
+function refreshOpenClawInstall() {
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
+  const cliPath = join(__dirname, '..', 'dist', 'index.js');
+
+  if (!existsSync(cliPath)) return;
+  const result = spawnSync(process.execPath, [cliPath, 'openclaw', 'install'], {
+    stdio: 'inherit',
+    env: process.env,
+  });
+  if (result.status !== 0) {
+    console.warn('[shieldcortex] OpenClaw auto-refresh skipped (non-fatal).');
+  }
+}
 
 if (isGlobal && !isCI) {
+  if (!skipAutoOpenClaw && shouldRefreshOpenClaw()) {
+    console.log('\n[shieldcortex] OpenClaw detected. Refreshing hook/plugin to latest version...');
+    refreshOpenClawInstall();
+  }
+
   console.log('');
   console.log('\x1b[36m╭───────────────────────────────────────────────────────╮\x1b[0m');
   console.log('\x1b[36m│\x1b[0m  \x1b[1mShieldCortex installed!\x1b[0m                              \x1b[36m│\x1b[0m');