squish-memory 1.1.5 → 1.2.1

package/dist/core/algorithms/detection/two-stage-detector.d.ts

@@ -1,52 +0,0 @@
-/**
- * Three-stage duplicate detection orchestrator.
- * Stage 0: Exact match (content hash-based)
- * Stage 1: Hash-based prefiltering (SimHash + MinHash)
- * Stage 2: Semantic ranking using embeddings
- */
-import type { Memory, MemoryType } from '../../../db/drizzle/schema.js';
-import { analyzePair } from './semantic-ranker.js';
-export interface MemoryPair {
-    memory1: Memory;
-    memory2: Memory;
-    similarityScore: number;
-    detectionMethod: 'exact' | 'simhash' | 'minhash' | 'embedding';
-    confidenceLevel: 'high' | 'medium' | 'low';
-    mergeReason: string;
-}
-export interface DetectionResult {
-    candidates: MemoryPair[];
-    stage0Time: number;
-    stage1Time: number;
-    stage2Time: number;
-    totalCandidates: number;
-    filteredCandidates: number;
-    statistics: {
-        totalMemories: number;
-        memoriesByType: Record<MemoryType, number>;
-    };
-}
-export interface DetectionOptions {
-    projectId?: string;
-    type?: MemoryType;
-    threshold?: number;
-    limit?: number;
-    simhashThreshold?: number;
-    minhashThreshold?: number;
-    stage1Only?: boolean;
-    stage0Only?: boolean;
-}
-export declare function detectDuplicates(options: DetectionOptions): Promise<DetectionResult>;
-export declare function analyzeMergePair(memoryId1: string, memoryId2: string): Promise<{
-    memory1: Memory;
-    memory2: Memory;
-    analysis: ReturnType<typeof analyzePair>;
-} | null>;
-export declare function getDetectionStats(projectId: string): Promise<{
-    totalMemories: number;
-    mergeableMemories: number;
-    mergedMemories: number;
-    canonicalMemories: number;
-    memoriesByType: Record<MemoryType, number>;
-}>;
-//# sourceMappingURL=two-stage-detector.d.ts.map

package/dist/core/algorithms/detection/two-stage-detector.js

@@ -1,299 +0,0 @@
-/**
- * Three-stage duplicate detection orchestrator.
- * Stage 0: Exact match (content hash-based)
- * Stage 1: Hash-based prefiltering (SimHash + MinHash)
- * Stage 2: Semantic ranking using embeddings
- */
-import { getEmbedding, getBatchEmbeddings } from '../../../core/embeddings.js';
-import { SimHashFilter, MinHashFilter, findCandidatePairs } from './hash-filters.js';
-import { rankCandidates, analyzePair } from './semantic-ranker.js';
-import { eq, and } from 'drizzle-orm';
-import * as crypto from 'crypto';
-import { getDbClient } from '../../../core/lib/db-client.js';
-export async function detectDuplicates(options) {
-    const startTime = Date.now();
-    const { db, schema } = await getDbClient();
-    let query = db.select().from(schema.memories);
-    if (options.projectId) {
-        query = query.where(eq(schema.memories.projectId, options.projectId));
-    }
-    if (options.type) {
-        query = query.where(eq(schema.memories.type, options.type));
-    }
-    query = query.where(and(eq(schema.memories.isMerged, false), eq(schema.memories.isMergeable, true), eq(schema.memories.isActive, true)));
-    const memories = await query.execute();
-    if (memories.length < 2) {
-        return {
-            candidates: [],
-            stage0Time: 0,
-            stage1Time: 0,
-            stage2Time: 0,
-            totalCandidates: 0,
-            filteredCandidates: 0,
-            statistics: {
-                totalMemories: memories.length,
-                memoriesByType: countByType(memories),
-            },
-        };
-    }
-    const memoriesById = new Map(memories.map((m) => [m.id, m]));
-    const contentById = new Map(memories.map((m) => [m.id, m.content]));
-    // Stage 0: Exact match using content hash
-    const stage0Start = Date.now();
-    const stage0Candidates = [];
-    // Group memories by content hash for exact matching
-    const contentHashGroups = new Map();
-    for (const memory of memories) {
-        const contentHash = crypto.createHash('md5').update(memory.content).digest('hex');
-        if (!contentHashGroups.has(contentHash)) {
-            contentHashGroups.set(contentHash, []);
-        }
-        contentHashGroups.get(contentHash).push(memory.id);
-    }
-    // Create pairs from each group with same content
-    for (const [hash, ids] of contentHashGroups.entries()) {
-        if (ids.length >= 2) {
-            // Create all unique pairs within this group
-            for (let i = 0; i < ids.length; i++) {
-                for (let j = i + 1; j < ids.length; j++) {
-                    stage0Candidates.push({ memoryId1: ids[i], memoryId2: ids[j] });
-                }
-            }
-        }
-    }
-    const stage0Time = Date.now() - stage0Start;
-    // If we only want exact matches for testing, return early
-    if (options.stage0Only) {
-        const exactMatchCandidates = stage0Candidates.map(pair => ({
-            memory1: memoriesById.get(pair.memoryId1),
-            memory2: memoriesById.get(pair.memoryId2),
-            similarityScore: 1.0, // Exact match = 1.0 similarity
-            detectionMethod: 'exact',
-            confidenceLevel: 'high',
-            mergeReason: 'Exact content match',
-        }));
-        return {
-            candidates: exactMatchCandidates.slice(0, options.limit ?? 50),
-            stage0Time,
-            stage1Time: 0,
-            stage2Time: 0,
-            totalCandidates: stage0Candidates.length,
-            filteredCandidates: exactMatchCandidates.length,
-            statistics: {
-                totalMemories: memories.length,
-                memoriesByType: countByType(memories),
-            },
-        };
-    }
-    const simhashFilter = new SimHashFilter();
-    const minhashFilter = new MinHashFilter();
-    const allSimhashes = new Map();
-    const allMinhashes = new Map();
-    for (const memory of memories) {
-        allSimhashes.set(memory.id, simhashFilter.generateHash(memory.content));
-        allMinhashes.set(memory.id, minhashFilter.generateSignature(memory.content));
-    }
-    const stage1Start = Date.now();
-    const stage1Candidates = findCandidatePairs(contentById, allSimhashes, allMinhashes, {
-        simhashThreshold: options.simhashThreshold ?? 4,
-        minhashThreshold: options.minhashThreshold ?? 0.7,
-    });
-    const stage1Time = Date.now() - stage1Start;
-    // Combine stage 0 and stage 1 candidates for stage 2 processing
-    // We'll prioritize exact matches but also include fuzzy matches for better recall
-    const combinedCandidatesForStage2 = [
-        ...stage0Candidates, // Exact matches first
-        ...stage1Candidates // Then fuzzy matches
-    ];
-    // Remove duplicates while preserving order (exact matches first)
-    const seenPairs = new Set();
-    const uniqueCombinedCandidates = [];
-    for (const pair of combinedCandidatesForStage2) {
-        const pairKey = `${pair.memoryId1}:${pair.memoryId2}`;
-        const reversePairKey = `${pair.memoryId2}:${pair.memoryId1}`;
-        if (!seenPairs.has(pairKey) && !seenPairs.has(reversePairKey)) {
-            seenPairs.add(pairKey);
-            uniqueCombinedCandidates.push(pair);
-        }
-    }
-    if (options.stage1Only) {
-        // Process only stage 1 candidates (fuzzy matches) for backward compatibility
-        const stage1OnlyCandidates = stage1Candidates.map((pair) => ({
-            memory1: memoriesById.get(pair.memoryId1),
-            memory2: memoriesById.get(pair.memoryId2),
-            similarityScore: Math.max(1 - pair.simhashDistance / 64, pair.minhashSimilarity),
-            detectionMethod: pair.matched === 'both' ? 'simhash' : pair.matched,
-            confidenceLevel: 'low',
-            mergeReason: 'Stage 1 candidate (embedding analysis skipped)',
-        }));
-        return {
-            candidates: stage1OnlyCandidates.slice(0, options.limit ?? 50),
-            stage0Time,
-            stage1Time,
-            stage2Time: 0,
-            totalCandidates: stage1Candidates.length,
-            filteredCandidates: stage1OnlyCandidates.length,
-            statistics: {
-                totalMemories: memories.length,
-                memoriesByType: countByType(memories),
-            },
-        };
-    }
-    const stage2Start = Date.now();
-    const embeddings = new Map();
-    // Separate memories that already have embeddings from those that need generation
-    const memoriesWithoutEmbedding = memories.filter((m) => !m.embedding);
-    const memoriesWithEmbedding = memories.filter((m) => m.embedding);
-    // Add cached embeddings to map
-    for (const memory of memoriesWithEmbedding) {
-        embeddings.set(memory.id, memory.embedding);
-    }
-    // Generate embeddings for remaining memories in parallel batches
-    if (memoriesWithoutEmbedding.length > 0) {
-        const contents = memoriesWithoutEmbedding.map((m) => m.content);
-        const generatedEmbeddings = await getBatchEmbeddings(contents, 20);
-        for (let i = 0; i < memoriesWithoutEmbedding.length; i++) {
-            const embedding = generatedEmbeddings[i];
-            if (embedding) {
-                embeddings.set(memoriesWithoutEmbedding[i].id, embedding);
-            }
-        }
-    }
-    const rankedCandidates = await rankCandidates(uniqueCombinedCandidates.map((pair) => ({
-        memoryId1: pair.memoryId1,
-        memoryId2: pair.memoryId2,
-    })), memoriesById, embeddings, {
-        semanticThreshold: options.threshold ?? 0.85,
-        topK: 10,
-    });
-    const stage2Time = Date.now() - stage2Start;
-    // Build final candidates list with proper scoring and methods
-    const finalCandidates = [];
-    // Add exact matches first (highest confidence)
-    for (const pair of stage0Candidates) {
-        const memory1 = memoriesById.get(pair.memoryId1);
-        const memory2 = memoriesById.get(pair.memoryId2);
-        if (memory1 && memory2) {
-            finalCandidates.push({
-                memory1,
-                memory2,
-                similarityScore: 1.0, // Exact match
-                detectionMethod: 'exact',
-                confidenceLevel: 'high',
-                mergeReason: 'Exact content match',
-            });
-        }
-    }
-    // Add semantic matches from stage 2
-    for (const ranked of rankedCandidates) {
-        finalCandidates.push({
-            memory1: ranked.memory1,
-            memory2: ranked.memory2,
-            similarityScore: ranked.cosineSimilarity,
-            detectionMethod: 'embedding',
-            confidenceLevel: ranked.confidenceLevel,
-            mergeReason: ranked.mergeReason,
-        });
-    }
-    // Add fuzzy matches (stage 1) that weren't already covered
-    const processedPairs = new Set();
-    for (const candidate of finalCandidates) {
-        const pairKey1 = `${candidate.memory1.id}:${candidate.memory2.id}`;
-        const pairKey2 = `${candidate.memory2.id}:${candidate.memory1.id}`;
-        processedPairs.add(pairKey1);
-        processedPairs.add(pairKey2);
-    }
-    for (const pair of stage1Candidates) {
-        const pairKey = `${pair.memoryId1}:${pair.memoryId2}`;
-        const reversePairKey = `${pair.memoryId2}:${pair.memoryId1}`;
-        if (!processedPairs.has(pairKey) && !processedPairs.has(reversePairKey)) {
-            const memory1 = memoriesById.get(pair.memoryId1);
-            const memory2 = memoriesById.get(pair.memoryId2);
-            if (memory1 && memory2) {
-                finalCandidates.push({
-                    memory1,
-                    memory2,
-                    similarityScore: Math.max(1 - pair.simhashDistance / 64, pair.minhashSimilarity),
-                    detectionMethod: pair.matched === 'both' ? 'simhash' : pair.matched,
-                    confidenceLevel: 'low',
-                    mergeReason: 'Stage 1 candidate (embedding analysis skipped)',
-                });
-            }
-        }
-    }
-    const limited = finalCandidates.slice(0, options.limit ?? 50);
-    return {
-        candidates: limited,
-        stage0Time,
-        stage1Time,
-        stage2Time,
-        totalCandidates: stage0Candidates.length + stage1Candidates.length,
-        filteredCandidates: finalCandidates.length,
-        statistics: {
-            totalMemories: memories.length,
-            memoriesByType: countByType(memories),
-        },
-    };
-}
-export async function analyzeMergePair(memoryId1, memoryId2) {
-    const { db, schema } = await getDbClient();
-    const [memory1] = await db
-        .select()
-        .from(schema.memories)
-        .where(eq(schema.memories.id, memoryId1));
-    const [memory2] = await db
-        .select()
-        .from(schema.memories)
-        .where(eq(schema.memories.id, memoryId2));
-    if (!memory1 || !memory2) {
-        return null;
-    }
-    const embedding1 = memory1.embedding || (await getEmbedding(memory1.content)) || [];
-    const embedding2 = memory2.embedding || (await getEmbedding(memory2.content)) || [];
-    if (!embedding1 || !embedding2 || embedding1.length === 0 || embedding2.length === 0) {
-        return null;
-    }
-    const analysis = analyzePair(memory1, memory2, embedding1, embedding2);
-    return {
-        memory1,
-        memory2,
-        analysis,
-    };
-}
-function countByType(memories) {
-    const counts = {
-        observation: 0,
-        fact: 0,
-        decision: 0,
-        context: 0,
-        preference: 0,
-        reflection: 0,
-        note: 0,
-    };
-    for (const memory of memories) {
-        const type = memory.type;
-        if (type in counts) {
-            counts[type]++;
-        }
-    }
-    return counts;
-}
-export async function getDetectionStats(projectId) {
-    const { db, schema } = await getDbClient();
-    const memories = await db
-        .select()
-        .from(schema.memories)
-        .where(eq(schema.memories.projectId, projectId));
-    const totalMemories = memories.length;
-    const mergedMemories = memories.filter((m) => m.isMerged).length;
-    const canonicalMemories = memories.filter((m) => m.isCanonical).length;
-    const mergeableMemories = memories.filter((m) => m.isMergeable && !m.isMerged).length;
-    return {
-        totalMemories,
-        mergeableMemories,
-        mergedMemories,
-        canonicalMemories,
-        memoriesByType: countByType(memories),
-    };
-}
-//# sourceMappingURL=two-stage-detector.js.map

package/dist/core/algorithms/handlers/approve-merge.d.ts

@@ -1,22 +0,0 @@
-/**
- * Executes the approved merge in a single atomic transaction.
- */
-interface ApproveMergeInput {
-    proposalId: string;
-    reviewNotes?: string;
-}
-interface ApproveMergeResponse {
-    ok: boolean;
-    message: string;
-    data?: {
-        proposalId: string;
-        canonicalMemoryId: string;
-        mergedMemoryIds: string[];
-        tokensSaved: number;
-        mergedAt: string;
-    };
-    error?: string;
-}
-export declare function handleApproveMerge(input: ApproveMergeInput): Promise<ApproveMergeResponse>;
-export {};
-//# sourceMappingURL=approve-merge.d.ts.map

package/dist/core/algorithms/handlers/approve-merge.js

@@ -1,179 +0,0 @@
-/**
- * Executes the approved merge in a single atomic transaction.
- */
-import { randomUUID } from 'crypto';
-import { getDb } from '../../../db/index.js';
-import { getSchema } from '../../../db/schema.js';
-import { createDatabaseClient } from '../../../core/storage/database.js';
-import { eq, inArray } from 'drizzle-orm';
-import { mergeMemories } from '../strategies/merge-strategies.js';
-import { estimateTokensSaved } from '../analytics/token-estimator.js';
-import { getEmbedding } from '../../../core/embeddings.js';
-export async function handleApproveMerge(input) {
-    try {
-        const { proposalId, reviewNotes } = input;
-        if (!proposalId) {
-            return {
-                ok: false,
-                message: 'proposalId is required',
-                error: 'proposalId is required',
-            };
-        }
-        const db = createDatabaseClient(await getDb());
-        const schema = await getSchema();
-        const [proposal] = await db
-            .select()
-            .from(schema.memoryMergeProposals)
-            .where(eq(schema.memoryMergeProposals.id, proposalId));
-        if (!proposal) {
-            return {
-                ok: false,
-                message: 'Proposal not found',
-                error: `Proposal ${proposalId} not found`,
-            };
-        }
-        if (proposal.status !== 'pending') {
-            return {
-                ok: false,
-                message: `Proposal is not pending (status: ${proposal.status})`,
-                error: `Cannot approve non-pending proposal`,
-            };
-        }
-        // Step 2: Load source memories
-        const sourceIds = proposal.sourceMemoryIds || [];
-        if (sourceIds.length === 0) {
-            return {
-                ok: false,
-                message: 'No source memories found in proposal',
-                error: 'Proposal has no source memory IDs',
-            };
-        }
-        const sourceMemories = await db
-            .select()
-            .from(schema.memories)
-            .where(inArray(schema.memories.id, sourceIds));
-        if (sourceMemories.length !== sourceIds.length) {
-            return {
-                ok: false,
-                message: 'Not all source memories could be found',
-                error: 'Missing source memories',
-            };
-        }
-        // Step 3: Merge memories
-        let merged;
-        try {
-            merged = mergeMemories(sourceMemories);
-        }
-        catch (error) {
-            return {
-                ok: false,
-                message: 'Merge strategy failed',
-                error: error instanceof Error ? error.message : 'Unknown merge error',
-            };
-        }
-        // Step 4: Calculate token savings
-        const tokensSaved = estimateTokensSaved(sourceMemories, merged);
-        // Step 5: Generate embedding for merged memory
-        let embedding = null;
-        try {
-            embedding = (await getEmbedding(merged.content));
-        }
-        catch (err) {
-            // Continue without embedding if generation fails
-        }
-        // Step 6: Create canonical memory
-        const canonicalId = randomUUID();
-        const now = new Date();
-        await db.insert(schema.memories).values({
-            id: canonicalId,
-            projectId: sourceMemories[0].projectId,
-            userId: sourceMemories[0].userId,
-            type: sourceMemories[0].type,
-            content: merged.content,
-            summary: merged.summary,
-            embedding: embedding || undefined,
-            tags: merged.tags,
-            metadata: merged.metadata,
-            source: 'merge',
-            confidence: 85, // Merged confidence slightly lower than source
-            isActive: true,
-            isCanonical: true,
-            mergeSourceIds: sourceIds,
-            isMergeable: true,
-            mergeVersion: 1,
-            createdAt: now,
-            updatedAt: now,
-            isPrivate: sourceMemories[0].isPrivate,
-            hasSecrets: sourceMemories.some((m) => m.hasSecrets),
-            relevanceScore: Math.round(sourceMemories.reduce((sum, m) => sum + (m.relevanceScore || 50), 0) / sourceMemories.length),
-            accessCount: 0,
-            lastAccessedAt: null,
-            expiresAt: null,
-        });
-        // Step 7: Mark source memories as merged (soft archive)
-        for (const sourceMemory of sourceMemories) {
-            await db
-                .update(schema.memories)
-                .set({
-                isMerged: true,
-                mergedIntoId: canonicalId,
-                mergedAt: now,
-                isActive: false,
-                updatedAt: now,
-            })
-                .where(eq(schema.memories.id, sourceMemory.id));
-        }
-        // Step 8: Create merge history record (audit trail)
-        const historyId = randomUUID();
-        await db.insert(schema.memoryMergeHistory).values({
-            id: historyId,
-            projectId: sourceMemories[0].projectId,
-            userId: sourceMemories[0].userId,
-            proposalId,
-            sourceMemoryIds: sourceIds,
-            canonicalMemoryId: canonicalId,
-            sourceMemoriesSnapshot: sourceMemories.map((m) => ({
-                id: m.id,
-                type: m.type,
-                content: m.content,
-                summary: m.summary,
-                tags: m.tags,
-                metadata: m.metadata,
-                createdAt: m.createdAt,
-            })),
-            mergeStrategy: sourceMemories[0].type === 'preference' ? 'latest' : 'union',
-            tokensSaved,
-            isReversed: false,
-            mergedAt: now,
-        });
-        // Step 9: Update proposal status
-        await db
-            .update(schema.memoryMergeProposals)
-            .set({
-            status: 'approved',
-            reviewedAt: now,
-            reviewNotes: reviewNotes || undefined,
-        })
-            .where(eq(schema.memoryMergeProposals.id, proposalId));
-        // Step 10: Return success
-        return {
-            ok: true,
-            message: `Merge approved and executed. Created canonical memory ${canonicalId}`,
-            data: {
-                proposalId,
-                canonicalMemoryId: canonicalId,
-                mergedMemoryIds: sourceIds,
-                tokensSaved,
-                mergedAt: now.toISOString(),
-            },
-        };
-    }
-    catch (error) {
-        return {
-            ok: false,
-            message: 'Failed to approve merge',
-            error: error instanceof Error ? error.message : 'Unknown error',
-        };
-    }
-}
-//# sourceMappingURL=approve-merge.js.map

package/dist/core/algorithms/handlers/detect-duplicates.d.ts

@@ -1,47 +0,0 @@
-/**
- * MCP Tool: detect_duplicate_memories
- *
- * Scans for duplicate or similar memories and creates merge proposals
- * Entry point for memory merging workflow
- */
-interface DetectDuplicatesInput {
-    projectId?: string;
-    threshold?: number;
-    memoryType?: 'fact' | 'preference' | 'decision' | 'observation' | 'context';
-    limit?: number;
-    autoCreateProposals?: boolean;
-}
-interface DetectDuplicatesResponse {
-    ok: boolean;
-    message: string;
-    data?: {
-        projectId: string;
-        duplicateCount: number;
-        proposalsCreated: number;
-        proposalIds: string[];
-        statistics: {
-            totalMemories: number;
-            scannedMemories: number;
-            candidatesFound: number;
-            estimatedTokensSaved: number;
-        };
-        timing: {
-            stage1Ms: number;
-            stage2Ms: number;
-            totalMs: number;
-        };
-    };
-    error?: string;
-}
-/**
- * Handle detect_duplicate_memories tool call
- *
- * Algorithm:
- * 1. Run two-stage detection (SimHash → embedding similarity)
- * 2. Filter by safety checks
- * 3. Create merge proposals in database
- * 4. Return summary
- */
-export declare function handleDetectDuplicates(input: DetectDuplicatesInput): Promise<DetectDuplicatesResponse>;
-export {};
-//# sourceMappingURL=detect-duplicates.d.ts.map