shieldcortex 4.31.1 → 4.32.0

package/dist/xray/sarif.js

@@ -0,0 +1,166 @@
+/**
+ * SARIF 2.1.0 output for ShieldCortex findings.
+ *
+ * Phase 15b — turns X-Ray / audit / mcp-scan findings into a SARIF 2.1.0
+ * document so they surface in GitHub Code Scanning (the Security tab).
+ *
+ * Lives in the xray layer (its `index.ts` already re-exports the other
+ * formatters) but is shape-agnostic: it accepts BOTH the xray `XRayFinding`
+ * and the audit `AuditFinding` via a single internal normaliser, so the SARIF
+ * builder never branches on the source shape.
+ *
+ * Spec: https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html
+ */
+import path from 'path';
+import { createRequire } from 'module';
+const INFORMATION_URI = 'https://shieldcortex.ai';
+const SARIF_SCHEMA = 'https://json.schemastore.org/sarif-2.1.0.json';
+// ── Helpers ────────────────────────────────────────────────────
+/**
+ * Map a ShieldCortex severity onto a SARIF result level.
+ *   critical / high → error
+ *   medium          → warning
+ *   low / info      → note
+ */
+function severityToLevel(severity) {
+    switch (severity) {
+        case 'critical':
+        case 'high':
+            return 'error';
+        case 'medium':
+            return 'warning';
+        case 'low':
+        case 'info':
+        default:
+            return 'note';
+    }
+}
+/**
+ * Stable rule id from a category/scanner key + title. SARIF rule ids should be
+ * opaque, stable, and free of whitespace so they read well in the GitHub UI and
+ * dedupe deterministically across runs.
+ */
+function makeRuleId(prefix, title) {
+    const slug = `${prefix}/${title}`
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '');
+    return `shieldcortex/${slug}`;
+}
+function isXRayFinding(f) {
+    // XRayFinding has `category`; AuditFinding has `scanner`. They are disjoint.
+    return f.category !== undefined;
+}
+/** Make a file path relative to baseDir + posix-separated for SARIF URIs. */
+function toUri(file, baseDir) {
+    let rel = file;
+    if (path.isAbsolute(file)) {
+        rel = path.relative(baseDir, file);
+        // If the file is outside baseDir, relative() yields ../ — fall back to the
+        // absolute path rather than emitting a confusing climb-out URI.
+        if (rel.startsWith('..'))
+            rel = file;
+    }
+    return rel.split(path.sep).join('/');
+}
+function normalise(f) {
+    if (isXRayFinding(f)) {
+        const category = f.category;
+        return {
+            ruleId: makeRuleId(category, f.title),
+            ruleName: f.title,
+            level: severityToLevel(f.severity),
+            message: f.description,
+            file: f.file,
+            line: f.line,
+        };
+    }
+    // AuditFinding. May or may not have a filePath.
+    const audit = f;
+    const norm = {
+        ruleId: makeRuleId(audit.scanner, audit.title),
+        ruleName: audit.title,
+        level: severityToLevel(audit.severity),
+        message: audit.description,
+        helpUri: audit.learnMoreUrl,
+    };
+    if (audit.filePath) {
+        norm.file = audit.filePath;
+    }
+    else {
+        // Fileless: synthesise a logical location from scanner + title so the
+        // result still carries a location and stays schema-valid without a
+        // physicalLocation.
+        norm.logicalName = `${audit.scanner}/${audit.title}`;
+    }
+    return norm;
+}
+// ── Builder ────────────────────────────────────────────────────
+function resolveVersion() {
+    try {
+        const require = createRequire(import.meta.url);
+        const pkg = require('../../package.json');
+        return pkg.version ?? '0.0.0';
+    }
+    catch {
+        return '0.0.0';
+    }
+}
+/**
+ * Build a SARIF 2.1.0 log from a list of findings (either shape).
+ * Rules are deduped into `tool.driver.rules` by ruleId.
+ */
+export function toSarif(findings, options = {}) {
+    const version = options.version ?? resolveVersion();
+    const baseDir = options.baseDir ?? process.cwd();
+    const rulesById = new Map();
+    const results = [];
+    for (const finding of findings) {
+        const n = normalise(finding);
+        if (!rulesById.has(n.ruleId)) {
+            const rule = {
+                id: n.ruleId,
+                name: n.ruleName,
+                shortDescription: { text: n.ruleName },
+            };
+            if (n.helpUri)
+                rule.helpUri = n.helpUri;
+            rulesById.set(n.ruleId, rule);
+        }
+        const result = {
+            ruleId: n.ruleId,
+            level: n.level,
+            message: { text: n.message },
+        };
+        if (n.file) {
+            const physicalLocation = {
+                artifactLocation: { uri: toUri(n.file, baseDir) },
+            };
+            if (typeof n.line === 'number') {
+                physicalLocation.region = { startLine: n.line };
+            }
+            result.locations = [{ physicalLocation }];
+        }
+        else if (n.logicalName) {
+            result.locations = [{ logicalLocations: [{ fullyQualifiedName: n.logicalName }] }];
+        }
+        results.push(result);
+    }
+    return {
+        $schema: SARIF_SCHEMA,
+        version: '2.1.0',
+        runs: [
+            {
+                tool: {
+                    driver: {
+                        name: 'ShieldCortex',
+                        version,
+                        informationUri: INFORMATION_URI,
+                        rules: Array.from(rulesById.values()),
+                    },
+                },
+                results,
+            },
+        ],
+    };
+}

package/dist/xray/watch.d.ts

@@ -4,6 +4,7 @@
  * Watches a directory for file changes and incrementally re-scans
  * only the changed files, printing new findings as they appear.
  */
+export declare function shouldIgnore(filePath: string): boolean;
 /**
  * Watch a directory for changes and incrementally scan changed files.
  */

package/dist/xray/watch.js

@@ -10,6 +10,7 @@ import path from 'path';
 import crypto from 'crypto';
 import { scanFile } from './file-scanner.js';
 import { calculateTrustScore } from './trust-score.js';
+import { ALLOW_HIDDEN_DIRS } from './dir-scanner.js';
 import { appendActivity, emitDetectionEvent, endWatchSession, heartbeatWatchSession, recordWatchSessionEvent, startWatchSession, } from './activity.js';
 import { addFindings } from './findings-store.js';
 // ── Constants ───────────────────────────────────────────────
@@ -34,10 +35,18 @@ const IGNORE_PATH_PATTERNS = [
     '/tmp/com.apple.',
 ];
 // ── Helpers ─────────────────────────────────────────────────
-function shouldIgnore(filePath) {
+export function shouldIgnore(filePath) {
     const parts = filePath.split(path.sep);
     if (parts.some(p => IGNORE_DIRS.has(p)))
         return true;
+    // Security-relevant hidden dirs (.github/.claude/.cursor/.codex/.vscode/…)
+    // are explicitly in scope for watch mode — they hold the agent-instruction /
+    // CI surfaces X-Ray exists to protect. Shares ALLOW_HIDDEN_DIRS with the
+    // directory scanner so a scan and a watch agree on what's watched. `.git`,
+    // `node_modules`, `.venv` etc. are NOT in the allow-list and remain excluded
+    // via the IGNORE_DIRS check above (which runs first).
+    if (parts.some(p => ALLOW_HIDDEN_DIRS.has(p)))
+        return false;
     // Resolve symlinks to check the real path
     let abs;
     try {

package/hooks/openclaw/cortex-memory/handler.ts

@@ -215,6 +215,84 @@ async function getSharedNoveltyGate() {
  * Uses ShieldCortex's scanSkill via mcporter
  * @returns {Promise<Array<{hookName: string, threat: string}>>}
  */
+// Content-hash cache for hook scan_skill verdicts. Byte-identical
+// HOOK.md/handler.js files produce the same sha256, so we skip the (cold MCP)
+// scan_skill shell-out on a cache hit and reuse the stored verdict. Keyed by
+// sha256(content) → "unsafe" | "clean". Persisted to ~/.shieldcortex so the
+// cache survives gateway restarts. A file edit changes the hash and forces a
+// fresh scan, so this never masks a newly-introduced threat.
+const HOOK_SCAN_CACHE_FILE = path.join(homedir(), ".shieldcortex", "openclaw-hook-scan-cache.json");
+
+async function loadHookScanCache() {
+  try {
+    const raw = JSON.parse(await fs.readFile(HOOK_SCAN_CACHE_FILE, "utf-8"));
+    if (raw && typeof raw === "object" && !Array.isArray(raw)) return raw;
+  } catch { /* no cache yet / unreadable */ }
+  return {};
+}
+
+async function saveHookScanCache(cache) {
+  try {
+    await fs.mkdir(path.dirname(HOOK_SCAN_CACHE_FILE), { recursive: true });
+    await fs.writeFile(HOOK_SCAN_CACHE_FILE, JSON.stringify(cache) + "\n", "utf-8");
+  } catch { /* best-effort — never block bootstrap on cache write */ }
+}
+
+function hashHookContent(content) {
+  return createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Parse the scan_skill verdict from its structured markdown output.
+ *
+ * The MCP `scan_skill` tool emits a fixed `**Safe:** Yes|No` field (see
+ * server.ts). We parse THAT exact field rather than substring-matching the
+ * whole report. The old `result.includes("unsafe")` was wrong twice over: the
+ * report never even prints the literal word "unsafe" (the verdict field is
+ * `**Safe:** No`), and any finding `description`/`matchedText` that merely
+ * mentions "unsafe" (e.g. "unsafe deserialization") would false-positive a
+ * clean file into a bootstrap security warning.
+ *
+ * Returns:
+ *   true  → definitively unsafe (`**Safe:** No`)
+ *   false → definitively safe   (`**Safe:** Yes`)
+ *   null  → verdict could not be parsed (treat as indeterminate; don't cache)
+ *
+ * @param {string} result
+ * @returns {boolean|null}
+ */
+function parseScanSkillUnsafe(result) {
+  if (typeof result !== "string") return null;
+  // Tolerate optional markdown bold markers and surrounding whitespace.
+  const m = result.match(/\*{0,2}Safe:\*{0,2}\s*(Yes|No)\b/i);
+  if (!m) return null;
+  return m[1].toLowerCase() === "no";
+}
+
+/**
+ * Scan one hook file, reusing a cached verdict when the content hash matches.
+ * Mutates `cache` in place and sets `dirty.changed` when a fresh scan runs.
+ * @returns {Promise<boolean>} true when the file is flagged unsafe
+ */
+async function scanHookFileCached(content, name, format, cache, dirty) {
+  const hash = hashHookContent(content);
+  const cached = cache[hash];
+  if (cached === "unsafe") return true;
+  if (cached === "clean") return false;
+
+  const result = await callCortex("scan_skill", { content, name, format });
+  // Only memoise a definitive verdict. A null/failed call is NOT cached, so a
+  // transient MCP failure doesn't get pinned as "clean" for an unscanned file.
+  if (result == null) return false;
+  const unsafe = parseScanSkillUnsafe(result);
+  // Indeterminate parse (null) → don't flag and don't cache, so the next scan
+  // retries rather than pinning a guessed verdict.
+  if (unsafe === null) return false;
+  cache[hash] = unsafe ? "unsafe" : "clean";
+  dirty.changed = true;
+  return unsafe;
+}
+
 async function scanInstalledHooks() {
   const path = await import("node:path");
   const { homedir } = await import("node:os");
@@ -224,6 +302,8 @@ async function scanInstalledHooks() {
 
   try {
     const entries = await fs.readdir(hooksDir, { withFileTypes: true });
+    const cache = await loadHookScanCache();
+    const dirty = { changed: false };
 
     for (const entry of entries) {
       if (!entry.isDirectory()) continue;
@@ -237,13 +317,7 @@ async function scanInstalledHooks() {
       const hookMdPath = path.join(hookDir, "HOOK.md");
       try {
         const hookContent = await fs.readFile(hookMdPath, "utf-8");
-        const result = await callCortex("scan_skill", {
-          content: hookContent,
-          name: entry.name,
-          format: "hook-md",
-        });
-
-        if (result && result.includes("unsafe")) {
+        if (await scanHookFileCached(hookContent, entry.name, "hook-md", cache, dirty)) {
           threats.push({ hookName: entry.name, threat: `HOOK.md flagged as unsafe` });
         }
       } catch { /* No HOOK.md, skip */ }
@@ -252,17 +326,13 @@ async function scanInstalledHooks() {
       const handlerPath = path.join(hookDir, "handler.js");
       try {
         const handlerContent = await fs.readFile(handlerPath, "utf-8");
-        const result = await callCortex("scan_skill", {
-          content: handlerContent,
-          name: `${entry.name}/handler.js`,
-          format: "hook-js",
-        });
-
-        if (result && result.includes("unsafe")) {
+        if (await scanHookFileCached(handlerContent, `${entry.name}/handler.js`, "hook-js", cache, dirty)) {
           threats.push({ hookName: entry.name, threat: `handler.js flagged as unsafe` });
         }
       } catch { /* No handler.js, skip */ }
     }
+
+    if (dirty.changed) await saveHookScanCache(cache);
   } catch {
     // Hooks directory doesn't exist or is unreadable
   }
@@ -303,10 +373,37 @@ function assistantConversationText(messages: string[]): string {
  * @param {string} sessionFilePath
  * @returns {Promise<string[]>} Array of "role: content" strings
  */
+// Read only the tail of a (potentially large) JSONL transcript. We need the
+// last 30 lines; reading the whole file just to discard the head wastes memory
+// on long sessions. Read up to TAIL_BYTES from the end via a file descriptor.
+// If the window started mid-file we drop its first (possibly partial) line so
+// only complete lines are parsed — JSON.parse failures are skipped anyway, but
+// dropping the fragment keeps the "last 30 complete lines" semantics exact.
+const TAIL_BYTES = 64 * 1024;
+
+async function readSessionTail(sessionFilePath) {
+  const fh = await fs.open(sessionFilePath, "r");
+  try {
+    const { size } = await fh.stat();
+    if (size <= TAIL_BYTES) {
+      // Small file — read it whole (identical to the original behaviour).
+      return { text: await fh.readFile("utf-8"), truncated: false };
+    }
+    const start = size - TAIL_BYTES;
+    const buf = Buffer.allocUnsafe(TAIL_BYTES);
+    const { bytesRead } = await fh.read(buf, 0, TAIL_BYTES, start);
+    return { text: buf.toString("utf-8", 0, bytesRead), truncated: true };
+  } finally {
+    await fh.close();
+  }
+}
+
 async function getRecentMessages(sessionFilePath) {
   try {
-    const content = await fs.readFile(sessionFilePath, "utf-8");
-    const lines = content.trim().split("\n");
+    const { text, truncated } = await readSessionTail(sessionFilePath);
+    let lines = text.trim().split("\n");
+    // Drop the leading partial line when we started reading mid-file.
+    if (truncated && lines.length > 1) lines = lines.slice(1);
     const recentLines = lines.slice(-30);
 
     const messages = [];
@@ -665,8 +762,15 @@ async function proactiveRecall(event) {
       project: "*",
     });
 
-    if (result && typeof result === "string" && result.includes("Found") && !result.includes("Found 0")) {
-      if (event.messages) {
+    // Parse the structured `Found N memor(y|ies):` header (recall.ts) at the
+    // START of the result, rather than substring-matching "Found" anywhere —
+    // a recalled memory's own content can contain the word "Found" and the old
+    // `!result.includes("Found 0")` clause was dead (the empty result is
+    // "No memories found...", never "Found 0"). Surface only on N >= 1.
+    if (result && typeof result === "string") {
+      const m = result.match(/^Found\s+(\d+)\s+memor/m);
+      const count = m ? Number(m[1]) : 0;
+      if (count >= 1 && event.messages) {
         event.messages.push(`🧠 ${result}`);
       }
     }

package/hooks/openclaw/cortex-memory/runtime.mjs

@@ -143,10 +143,16 @@ export function createOpenClawRuntime({
     const serverCmd = await resolveServerCmd();
 
     return new Promise((resolve) => {
-      const cmdArgs = ["mcporter", "call", "--stdio", serverCmd, tool];
-      for (const [key, value] of Object.entries(args)) {
-        cmdArgs.push(`${key}:${String(value).replace(/'/g, "''")}`);
-      }
+      // Pass arguments as a single `--args <json>` payload, NOT as per-key
+      // `key:value` flags. The old form did `String(value).replace(/'/g, "''")`
+      // — SQL-style apostrophe doubling — on each value, which mangled saved
+      // memory content (an apostrophe in "it's" became "it''s" and was stored
+      // literally; this is a key:value CLI flag, not a bound SQL param, so no
+      // such escaping should ever happen). A JSON payload also round-trips
+      // colons, spaces, and newlines in content without the `key:value`
+      // splitter misreading them. execFile passes argv elements directly (no
+      // shell), so the JSON string needs no further quoting/escaping.
+      const cmdArgs = ["mcporter", "call", "--stdio", serverCmd, tool, "--args", JSON.stringify(args)];
 
       let attempts = 0;
       const maxAttempts = (options.retries || 0) + 1;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shieldcortex",
-  "version": "4.31.1",
+  "version": "4.32.0",
   "description": "Trustworthy memory and security for AI agents. Recall debugging, review queue, OpenClaw session capture, and memory poisoning defence for Claude Code, Codex, OpenClaw, LangChain, and MCP agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -30,6 +30,10 @@
       "import": "./dist/defence/index.js",
       "types": "./dist/defence/index.d.ts"
     },
+    "./scan": {
+      "import": "./dist/scan-only.js",
+      "types": "./dist/scan-only.d.ts"
+    },
     "./environment": {
       "import": "./dist/environment/index.js",
       "types": "./dist/environment/index.d.ts"
@@ -57,12 +61,13 @@
     "test": "node scripts/run-jest.mjs",
     "test:watch": "node scripts/run-jest.mjs --watch",
     "test:coverage": "node scripts/run-jest.mjs --coverage",
+    "test:dist": "node scripts/check-no-bare-require.mjs",
     "bench": "tsx benchmark/longmemeval/run.ts",
     "bench:smoke": "SHIELDCORTEX_SKIP_EMBEDDINGS=1 tsx benchmark/longmemeval/run.ts --quiet",
     "audit:security": "npm audit --audit-level=moderate",
     "prepack": "node scripts/ensure-bin-executable.mjs",
     "version": "node scripts/sync-plugin-version.mjs && git add plugins/openclaw/package.json plugins/openclaw/openclaw.plugin.json",
-    "prepublishOnly": "node scripts/sync-plugin-version.mjs --check && npm run build"
+    "prepublishOnly": "node scripts/sync-plugin-version.mjs --check && npm run build && npm run test:dist"
   },
   "keywords": [
     "ai-memory",
@@ -112,7 +117,6 @@
     "node": ">=20.0.0"
   },
   "dependencies": {
-    "@huggingface/transformers": "^3.7.2",
     "@modelcontextprotocol/sdk": "^1.0.0",
     "better-sqlite3": "^12.0.0",
     "cors": "^2.8.5",
@@ -122,6 +126,9 @@
     "ws": "^8.18.0",
     "zod": "^3.23.0"
   },
+  "optionalDependencies": {
+    "@huggingface/transformers": "^3.7.2"
+  },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.11",
     "@types/cors": "^2.8.17",

package/dist/memory/embedding-cache.d.ts

@@ -1,20 +0,0 @@
-/**
- * Embedding Cache
- *
- * Stores pre-computed embeddings in SQLite to avoid re-embedding on every recall.
- * Self-migrating: creates the table on first use.
- */
-/**
- * Get a cached embedding for a memory ID.
- * Returns null if not cached or DB unavailable.
- */
-export declare function getCachedEmbedding(memoryId: number): Float32Array | null;
-/**
- * Store a computed embedding in the cache.
- */
-export declare function setCachedEmbedding(memoryId: number, embedding: Float32Array): void;
-/**
- * Get embedding from cache, or compute + store if missing.
- * Returns null if both cache and computation fail.
- */
-export declare function getOrComputeEmbedding(memoryId: number, text: string): Promise<Float32Array | null>;

package/dist/memory/embedding-cache.js

@@ -1,91 +0,0 @@
-/**
- * Embedding Cache
- *
- * Stores pre-computed embeddings in SQLite to avoid re-embedding on every recall.
- * Self-migrating: creates the table on first use.
- */
-import { getDatabase, isDatabaseInitialized } from '../database/init.js';
-import { embedText } from './embedding.js';
-const MODEL_VERSION = 'all-MiniLM-L6-v2';
-let tableCreated = false;
-/**
- * Ensure the memory_embeddings table exists (self-migrating).
- */
-function ensureTable() {
-    if (tableCreated)
-        return;
-    if (!isDatabaseInitialized())
-        return;
-    try {
-        const db = getDatabase();
-        db.exec(`
-      CREATE TABLE IF NOT EXISTS memory_embeddings (
-        memory_id INTEGER PRIMARY KEY,
-        embedding BLOB NOT NULL,
-        model_version TEXT NOT NULL,
-        created_at TEXT NOT NULL DEFAULT (datetime('now'))
-      )
-    `);
-        tableCreated = true;
-    }
-    catch (e) {
-        console.warn('[shieldcortex] Failed to create memory_embeddings table:', e.message);
-    }
-}
-/**
- * Get a cached embedding for a memory ID.
- * Returns null if not cached or DB unavailable.
- */
-export function getCachedEmbedding(memoryId) {
-    try {
-        ensureTable();
-        if (!isDatabaseInitialized())
-            return null;
-        const db = getDatabase();
-        const row = db.prepare('SELECT embedding FROM memory_embeddings WHERE memory_id = ? AND model_version = ?').get(memoryId, MODEL_VERSION);
-        if (!row)
-            return null;
-        const buf = row.embedding;
-        return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
-    }
-    catch (e) {
-        console.warn('[shieldcortex] getCachedEmbedding error:', e.message);
-        return null;
-    }
-}
-/**
- * Store a computed embedding in the cache.
- */
-export function setCachedEmbedding(memoryId, embedding) {
-    try {
-        ensureTable();
-        if (!isDatabaseInitialized())
-            return;
-        const db = getDatabase();
-        const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
-        db.prepare(`
-      INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding, model_version, created_at)
-      VALUES (?, ?, ?, datetime('now'))
-    `).run(memoryId, buffer, MODEL_VERSION);
-    }
-    catch (e) {
-        console.warn('[shieldcortex] setCachedEmbedding error:', e.message);
-    }
-}
-/**
- * Get embedding from cache, or compute + store if missing.
- * Returns null if both cache and computation fail.
- */
-export async function getOrComputeEmbedding(memoryId, text) {
-    // Try cache first
-    const cached = getCachedEmbedding(memoryId);
-    if (cached)
-        return cached;
-    // Compute embedding
-    const embedding = await embedText(text);
-    if (!embedding)
-        return null;
-    // Store in cache (best-effort)
-    setCachedEmbedding(memoryId, embedding);
-    return embedding;
-}