shieldcortex 3.3.0 → 3.4.0

package/dist/api/routes/memories.js

@@ -2,9 +2,9 @@ import { homedir } from 'os';
 import { join } from 'path';
 import { existsSync, readdirSync, readFileSync } from 'fs';
 import { getDatabase } from '../../database/init.js';
-import { searchMemories, getRecentMemories, getHighPriorityMemories, getMemoryStats, getMemoryById, addMemory, deleteMemory, accessMemory, updateMemory, promoteMemory, createMemoryLink, rowToMemory, enrichMemory, } from '../../memory/store.js';
+import { searchMemories, getRecentMemories, getHighPriorityMemories, getMemoryStats, getMemoryById, addMemory, deleteMemory, accessMemory, updateMemory, mergeMemories, promoteMemory, createMemoryLink, rowToMemory, enrichMemory, } from '../../memory/store.js';
 import { calculateDecayedScore } from '../../memory/decay.js';
-import { consolidate, formatContextSummary, generateContextSummary, } from '../../memory/consolidate.js';
+import { consolidate, findDuplicateMemoryPairs, formatContextSummary, generateContextSummary, } from '../../memory/consolidate.js';
 import { getActivationStats, getActiveMemories } from '../../memory/activation.js';
 import { detectContradictions, getContradictionsFor } from '../../memory/contradiction.js';
 import { emitConsolidation } from '../events.js';
@@ -383,6 +383,7 @@ export function registerMemoryRoutes(app, requireNotLocked) {
                 minScore: 0.4,
                 limit,
             });
+            const duplicates = findDuplicateMemoryPairs({ project, limit });
             res.json({
                 summary: {
                     stale: stale.length,
@@ -391,6 +392,7 @@ export function registerMemoryRoutes(app, requireNotLocked) {
                     noisyAutoExtracted: noisyAutoExtracted.length,
                     projectless: projectless.length,
                     contradictions: contradictions.length,
+                    duplicates: duplicates.length,
                 },
                 openClaw: {
                     total: openClawSummary.total ?? 0,
@@ -405,6 +407,7 @@ export function registerMemoryRoutes(app, requireNotLocked) {
                     lowTrust: lowTrust.map(rowToMemory),
                     noisyAutoExtracted: noisyAutoExtracted.map(rowToMemory),
                     projectless: projectless.map(rowToMemory),
+                    duplicates,
                     contradictions: contradictions.map((item) => ({
                         memoryA: item.memoryA,
                         memoryB: item.memoryB,
@@ -419,6 +422,25 @@ export function registerMemoryRoutes(app, requireNotLocked) {
             res.status(500).json({ error: error.message });
         }
     });
+    app.post('/api/memories/merge', requireNotLocked, (req, res) => {
+        try {
+            const { keptId, removedId, reviewedBy } = req.body;
+            if (!Number.isInteger(keptId) || !Number.isInteger(removedId)) {
+                return res.status(400).json({ error: 'keptId and removedId are required integers' });
+            }
+            if (keptId === removedId) {
+                return res.status(400).json({ error: 'keptId and removedId must be different' });
+            }
+            const merged = mergeMemories(keptId, removedId, { reviewedBy });
+            if (!merged) {
+                return res.status(404).json({ error: 'Unable to merge memories' });
+            }
+            res.json(merged);
+        }
+        catch (error) {
+            res.status(500).json({ error: error.message });
+        }
+    });
     app.get('/api/memories/:id', requireNotLocked, (req, res) => {
         try {
             const id = parseInt(req.params.id, 10);

package/dist/memory/consolidate.d.ts

@@ -13,6 +13,19 @@ import { Memory, MemoryConfig, ConsolidationResult, ContextSummary } from './typ
  * This is like the brain's sleep consolidation - should be run periodically
  */
 export declare function consolidate(config?: MemoryConfig): ConsolidationResult;
+export interface DuplicateMemoryPair {
+    memoryA: Memory;
+    memoryB: Memory;
+    recommendedKeepId: number;
+    similarity: string;
+    titleSimilarity: number;
+    contentOverlap: number;
+    sharedWords: number;
+}
+export declare function findDuplicateMemoryPairs(options?: {
+    project?: string;
+    limit?: number;
+}): DuplicateMemoryPair[];
 /**
  * Find memories with very similar titles/content and merge them.
  * Keeps the newer one, appends unique content from the older one.

package/dist/memory/consolidate.js

@@ -9,7 +9,7 @@
  */
 import { getDatabase, withTransaction } from '../database/init.js';
 import { DEFAULT_CONFIG, } from './types.js';
-import { getMemoriesByType, getRecentMemories, promoteMemory, deleteMemory, searchMemories, getMemoryStats, updateDecayScores, addMemory, } from './store.js';
+import { getMemoriesByType, getRecentMemories, promoteMemory, deleteMemory, searchMemories, getMemoryStats, updateDecayScores, addMemory, rowToMemory, } from './store.js';
 import { processDecay, } from './decay.js';
 import { detectContradictions, linkContradictions, } from './contradiction.js';
 import { jaccardSimilarity } from './similarity.js';
@@ -191,6 +191,67 @@ function sharedWordCount(a, b) {
     }
     return count;
 }
+export function findDuplicateMemoryPairs(options) {
+    const db = getDatabase();
+    const limit = options?.limit ?? 20;
+    const rows = options?.project
+        ? db.prepare("SELECT * FROM memories WHERE type = 'long_term' AND project = ? ORDER BY created_at ASC").all(options.project)
+        : db.prepare("SELECT * FROM memories WHERE type = 'long_term' ORDER BY created_at ASC").all();
+    const groups = new Map();
+    for (const mem of rows) {
+        const cat = mem.category || 'note';
+        if (!groups.has(cat))
+            groups.set(cat, []);
+        groups.get(cat).push(mem);
+    }
+    const pairs = [];
+    const seen = new Set();
+    for (const [, group] of groups) {
+        if (group.length < 2)
+            continue;
+        for (let i = 0; i < group.length && pairs.length < limit; i++) {
+            const memA = group[i];
+            for (let j = i + 1; j < group.length && pairs.length < limit; j++) {
+                const memB = group[j];
+                const idA = memA.id;
+                const idB = memB.id;
+                const pairKey = `${Math.min(idA, idB)}:${Math.max(idA, idB)}`;
+                if (seen.has(pairKey))
+                    continue;
+                const titleA = memA.title || '';
+                const titleB = memB.title || '';
+                const contentA = memA.content || '';
+                const contentB = memB.content || '';
+                const titleSimilarity = levenshteinSimilarity(titleA.toLowerCase(), titleB.toLowerCase());
+                const sharedWords = sharedWordCount(titleA, titleB);
+                const titlesMatch = titleSimilarity > 0.7 || sharedWords >= 3 || titleA.toLowerCase() === titleB.toLowerCase();
+                if (!titlesMatch)
+                    continue;
+                const contentOverlap = wordOverlap(contentA, contentB);
+                if (contentOverlap <= 0.5)
+                    continue;
+                const salienceA = memA.salience || 0;
+                const salienceB = memB.salience || 0;
+                const createdA = new Date(memA.created_at || 0).getTime();
+                const createdB = new Date(memB.created_at || 0).getTime();
+                const recommendedKeepId = salienceA > salienceB || (salienceA === salienceB && createdA >= createdB)
+                    ? idA
+                    : idB;
+                pairs.push({
+                    memoryA: rowToMemory(memA),
+                    memoryB: rowToMemory(memB),
+                    recommendedKeepId,
+                    similarity: `title=${titleSimilarity.toFixed(2)}, content=${contentOverlap.toFixed(2)}`,
+                    titleSimilarity,
+                    contentOverlap,
+                    sharedWords,
+                });
+                seen.add(pairKey);
+            }
+        }
+    }
+    return pairs;
+}
 /**
  * Find memories with very similar titles/content and merge them.
  * Keeps the newer one, appends unique content from the older one.
@@ -200,81 +261,31 @@ export function deduplicateMemories(options) {
     return withTransaction(() => {
         const db = getDatabase();
         const pairs = [];
-        // Query all LTM memories
-        const ltmMemories = db.prepare("SELECT * FROM memories WHERE type = 'long_term' ORDER BY created_at ASC").all();
-        // Group by category
-        const groups = new Map();
-        for (const mem of ltmMemories) {
-            const cat = mem.category || 'note';
-            if (!groups.has(cat))
-                groups.set(cat, []);
-            groups.get(cat).push(mem);
-        }
         const removed = new Set();
-        for (const [, group] of groups) {
-            if (group.length < 2)
+        for (const pair of findDuplicateMemoryPairs()) {
+            if (removed.has(pair.memoryA.id) || removed.has(pair.memoryB.id))
                 continue;
-            for (let i = 0; i < group.length; i++) {
-                const memA = group[i];
-                if (removed.has(memA.id))
-                    continue;
-                for (let j = i + 1; j < group.length; j++) {
-                    const memB = group[j];
-                    if (removed.has(memB.id))
-                        continue;
-                    const titleA = memA.title || '';
-                    const titleB = memB.title || '';
-                    const contentA = memA.content || '';
-                    const contentB = memB.content || '';
-                    // Check title similarity: Levenshtein > 0.7 OR 3+ shared words
-                    const titleSim = levenshteinSimilarity(titleA.toLowerCase(), titleB.toLowerCase());
-                    const titleSharedWords = sharedWordCount(titleA, titleB);
-                    const titlesMatch = titleSim > 0.7 || titleSharedWords >= 3;
-                    if (!titlesMatch)
-                        continue;
-                    // Check content overlap > 50%
-                    const contentOverlap = wordOverlap(contentA, contentB);
-                    if (contentOverlap <= 0.5)
-                        continue;
-                    // They are duplicates — keep the one with higher salience (or newer if equal)
-                    const salienceA = memA.salience || 0;
-                    const salienceB = memB.salience || 0;
-                    const createdA = new Date(memA.created_at || 0).getTime();
-                    const createdB = new Date(memB.created_at || 0).getTime();
-                    let kept;
-                    let discarded;
-                    if (salienceA > salienceB || (salienceA === salienceB && createdA >= createdB)) {
-                        kept = memA;
-                        discarded = memB;
-                    }
-                    else {
-                        kept = memB;
-                        discarded = memA;
-                    }
-                    const similarity = `title=${titleSim.toFixed(2)}, content=${contentOverlap.toFixed(2)}`;
-                    if (!dryRun) {
-                        // Append unique sentences from discarded to kept
-                        const keptContent = kept.content || '';
-                        const discardedContent = discarded.content || '';
-                        const keptSentences = new Set(keptContent.split(/[.!?\n]+/).map(s => s.trim().toLowerCase()).filter(s => s.length > 0));
-                        const uniqueSentences = discardedContent
-                            .split(/[.!?\n]+/)
-                            .map(s => s.trim())
-                            .filter(s => s.length > 0 && !keptSentences.has(s.toLowerCase()));
-                        if (uniqueSentences.length > 0) {
-                            const mergedContent = keptContent + '\n\nMerged from duplicate:\n' + uniqueSentences.join('. ') + '.';
-                            db.prepare('UPDATE memories SET content = ? WHERE id = ?').run(mergedContent, kept.id);
-                        }
-                        deleteMemory(discarded.id);
-                    }
-                    removed.add(discarded.id);
-                    pairs.push({
-                        kept: kept.id,
-                        removed: discarded.id,
-                        similarity,
-                    });
+            const removedId = pair.recommendedKeepId === pair.memoryA.id ? pair.memoryB.id : pair.memoryA.id;
+            if (!dryRun) {
+                const kept = pair.recommendedKeepId === pair.memoryA.id ? pair.memoryA : pair.memoryB;
+                const discarded = pair.recommendedKeepId === pair.memoryA.id ? pair.memoryB : pair.memoryA;
+                const keptSentences = new Set(kept.content.split(/[.!?\n]+/).map(s => s.trim().toLowerCase()).filter(s => s.length > 0));
+                const uniqueSentences = discarded.content
+                    .split(/[.!?\n]+/)
+                    .map(s => s.trim())
+                    .filter(s => s.length > 0 && !keptSentences.has(s.toLowerCase()));
+                if (uniqueSentences.length > 0) {
+                    const mergedContent = kept.content + '\n\nMerged from duplicate:\n' + uniqueSentences.join('. ') + '.';
+                    db.prepare('UPDATE memories SET content = ? WHERE id = ?').run(mergedContent, kept.id);
                 }
+                deleteMemory(removedId);
             }
+            removed.add(removedId);
+            pairs.push({
+                kept: pair.recommendedKeepId,
+                removed: removedId,
+                similarity: pair.similarity,
+            });
         }
         return { merged: pairs.length, pairs };
     });

package/dist/memory/store.d.ts

@@ -42,6 +42,9 @@ export declare function getMemoryById(id: number, source?: DefenceSource): Memor
  * Update a memory
  */
 export declare function updateMemory(id: number, updates: Partial<MemoryInput>): Memory | null;
+export declare function mergeMemories(keptId: number, removedId: number, options?: {
+    reviewedBy?: string | null;
+}): Memory | null;
 /**
  * Delete a memory
  */

package/dist/memory/store.js

@@ -5,7 +5,7 @@
  * Handles storage, retrieval, and management of memories.
  */
 import { randomUUID } from 'crypto';
-import { getDatabase, isDatabaseInitialized } from '../database/init.js';
+import { getDatabase, isDatabaseInitialized, withTransaction } from '../database/init.js';
 import { DEFAULT_CONFIG, } from './types.js';
 import { calculateSalience, suggestCategory, extractTags, } from './salience.js';
 import { calculateDecayedScore, calculateReinforcementBoost, calculatePriority, } from './decay.js';
@@ -655,6 +655,103 @@ export function updateMemory(id, updates) {
     }
     return updatedMemory;
 }
+function uniqueSentencesFrom(source, existing) {
+    const existingSentences = new Set(existing
+        .split(/[.!?\n]+/)
+        .map((sentence) => sentence.trim().toLowerCase())
+        .filter(Boolean));
+    return source
+        .split(/[.!?\n]+/)
+        .map((sentence) => sentence.trim())
+        .filter((sentence) => sentence.length > 0 && !existingSentences.has(sentence.toLowerCase()));
+}
+export function mergeMemories(keptId, removedId, options) {
+    if (keptId === removedId)
+        return getMemoryById(keptId);
+    return withTransaction(() => {
+        const db = getDatabase();
+        const kept = getMemoryById(keptId);
+        const removed = getMemoryById(removedId);
+        if (!kept || !removed)
+            return null;
+        const mergedSnippets = uniqueSentencesFrom(removed.content, kept.content);
+        const mergedContent = mergedSnippets.length > 0
+            ? `${kept.content}\n\nMerged from duplicate (${removed.title}):\n${mergedSnippets.join('. ')}.`
+            : kept.content;
+        const mergedTags = Array.from(new Set([...(kept.tags ?? []), ...(removed.tags ?? [])]));
+        const mergedFrom = Array.isArray(kept.metadata?.mergedFrom)
+            ? [...kept.metadata.mergedFrom]
+            : [];
+        mergedFrom.push({
+            id: removed.id,
+            uuid: removed.uuid,
+            title: removed.title,
+            mergedAt: new Date().toISOString(),
+        });
+        const mergedMetadata = {
+            ...kept.metadata,
+            mergedFrom,
+            mergedFromCount: mergedFrom.length,
+        };
+        const mergedStatus = kept.status === 'canonical' || removed.status === 'canonical'
+            ? 'canonical'
+            : kept.status === 'active' || removed.status === 'active'
+                ? 'active'
+                : kept.status;
+        const mergedPinned = kept.pinned || removed.pinned || mergedStatus === 'canonical';
+        const mergedCloudExcluded = kept.cloudExcluded || removed.cloudExcluded;
+        const mergedScope = kept.scope === 'global' || removed.scope === 'global' ? 'global' : 'project';
+        const mergedProject = kept.project ?? removed.project ?? null;
+        const mergedTransferable = kept.transferable || removed.transferable;
+        const mergedSalience = Math.max(kept.salience, removed.salience);
+        const mergedTrustScore = Math.max(kept.trustScore ?? 0, removed.trustScore ?? 0);
+        const mergedAccessCount = kept.accessCount + removed.accessCount;
+        const mergedLastAccessed = new Date(Math.max(new Date(kept.lastAccessed).getTime(), new Date(removed.lastAccessed).getTime())).toISOString();
+        const mergedReviewedBy = options?.reviewedBy ?? kept.reviewedBy ?? 'review-merge';
+        const mergedSensitivity = kept.sensitivityLevel === 'SECRET' || removed.sensitivityLevel === 'SECRET'
+            ? 'SECRET'
+            : kept.sensitivityLevel === 'CONFIDENTIAL' || removed.sensitivityLevel === 'CONFIDENTIAL'
+                ? 'CONFIDENTIAL'
+                : kept.sensitivityLevel ?? removed.sensitivityLevel ?? 'INTERNAL';
+        db.prepare(`
+      UPDATE memories
+      SET content = ?,
+          tags = ?,
+          salience = ?,
+          project = ?,
+          metadata = ?,
+          scope = ?,
+          transferable = ?,
+          status = ?,
+          pinned = ?,
+          reviewed_by = ?,
+          reviewed_at = ?,
+          trust_score = ?,
+          sensitivity_level = ?,
+          cloud_excluded = ?,
+          access_count = ?,
+          last_accessed = ?,
+          updated_at = CURRENT_TIMESTAMP
+      WHERE id = ?
+    `).run(mergedContent, JSON.stringify(mergedTags), mergedSalience, mergedProject, JSON.stringify(mergedMetadata), mergedScope, mergedTransferable ? 1 : 0, mergedStatus, mergedPinned ? 1 : 0, mergedReviewedBy, new Date().toISOString(), mergedTrustScore, mergedSensitivity, mergedCloudExcluded ? 1 : 0, mergedAccessCount, mergedLastAccessed, kept.id);
+        const updatedMemory = getMemoryById(kept.id);
+        try {
+            const extraction = extractFromMemory(updatedMemory.title, updatedMemory.content, updatedMemory.category);
+            replaceMemoryGraph(updatedMemory.id, extraction);
+        }
+        catch (e) {
+            console.error('[shieldcortex] Entity extraction refresh failed after merge:', e);
+        }
+        emitMemoryUpdated(updatedMemory);
+        persistEvent('memory_updated', { memory: updatedMemory, mergedFromId: removed.id });
+        if (isFeatureEnabled('cloud_sync')) {
+            syncMemoryUpsertToCloud(updatedMemory);
+            syncGraphForMemoryToCloud(updatedMemory.id);
+        }
+        deleteMemory(removed.id);
+        return updatedMemory;
+    });
+}
 /**
  * Delete a memory
  */

package/dist/setup/codex.js

@@ -13,6 +13,9 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const SERVER_NAME = 'shieldcortex-memory';
 const MCP_ENTRY = path.resolve(__dirname, '..', 'index.js');
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
 function getCodexConfigPath() {
     return path.join(os.homedir(), '.codex', 'config.toml');
 }
@@ -38,32 +41,66 @@ function buildServerBlock() {
         '',
     ].join('\n');
 }
+function parseSections(content) {
+    const lines = content.split(/\r?\n/);
+    const sections = [];
+    let current = { header: null, lines: [] };
+    for (const line of lines) {
+        const headerMatch = line.match(/^\[([^\]]+)\]\s*$/);
+        if (headerMatch) {
+            sections.push(current);
+            current = { header: headerMatch[1], lines: [line] };
+            continue;
+        }
+        current.lines.push(line);
+    }
+    sections.push(current);
+    return sections;
+}
+function normalizeContent(content) {
+    const trimmed = content.replace(/\n{3,}/g, '\n\n').trimEnd();
+    return trimmed.length > 0 ? `${trimmed}\n` : '';
+}
 function hasServerBlock(content) {
-    return new RegExp(`^\\[mcp_servers\\.${SERVER_NAME}\\]`, 'm').test(content);
+    return new RegExp(`^\\[mcp_servers\\.${escapeRegExp(SERVER_NAME)}\\]\\r?$`, 'm').test(content);
 }
 function upsertServerBlock(content) {
-    const block = buildServerBlock();
-    const sectionPattern = new RegExp(`^\\[mcp_servers\\.${SERVER_NAME}\\]\\n[\\s\\S]*?(?=^\\[|\\Z)`, 'm');
-    if (sectionPattern.test(content)) {
-        const nextContent = content.replace(sectionPattern, block);
-        return { content: nextContent, changed: nextContent !== content };
+    const block = buildServerBlock().trimEnd();
+    const sections = parseSections(content);
+    const output = [];
+    let insertionIndex = -1;
+    let found = false;
+    for (const section of sections) {
+        if (section.header === `mcp_servers.${SERVER_NAME}`) {
+            if (insertionIndex === -1)
+                insertionIndex = output.length;
+            found = true;
+            continue;
+        }
+        const rendered = section.lines.join('\n').trimEnd();
+        if (rendered.length > 0)
+            output.push(rendered);
+    }
+    if (insertionIndex === -1)
+        insertionIndex = output.length;
+    output.splice(insertionIndex, 0, block);
+    const normalized = normalizeContent(output.join('\n\n'));
+    if (found) {
+        return { content: normalized, changed: normalized !== content };
     }
-    const trimmed = content.trimEnd();
-    return {
-        content: trimmed.length > 0 ? `${trimmed}\n\n${block}` : block,
-        changed: true,
-    };
+    return { content: normalized, changed: normalized !== content };
 }
 function removeServerBlock(content) {
-    const sectionPattern = new RegExp(`^\\[mcp_servers\\.${SERVER_NAME}\\]\\n[\\s\\S]*?(?=^\\[|\\Z)`, 'm');
-    if (!sectionPattern.test(content)) {
+    const sections = parseSections(content);
+    const filtered = sections.filter((section) => section.header !== `mcp_servers.${SERVER_NAME}`);
+    if (filtered.length === sections.length) {
         return { content, changed: false };
     }
-    const nextContent = content
-        .replace(sectionPattern, '')
-        .replace(/\n{3,}/g, '\n\n')
-        .trimEnd();
-    return { content: nextContent.length > 0 ? `${nextContent}\n` : '', changed: true };
+    const normalized = normalizeContent(filtered
+        .map((section) => section.lines.join('\n').trimEnd())
+        .filter(Boolean)
+        .join('\n\n'));
+    return { content: normalized, changed: true };
 }
 export async function installCodex() {
     if (!fs.existsSync(MCP_ENTRY)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shieldcortex",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "Trustworthy memory and security for AI agents. Recall debugging, review queue, OpenClaw session capture, and memory poisoning defence for Claude Code, Codex, OpenClaw, LangChain, and MCP agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",