squish-memory 1.0.2 → 1.2.0

package/core/algorithms/handlers/detect-duplicates.ts

@@ -0,0 +1,192 @@
+/**
+ * MCP Tool: detect_duplicate_memories
+ *
+ * Scans for duplicate or similar memories and creates merge proposals
+ * Entry point for memory merging workflow
+ */
+
+import type { UUID } from 'crypto';
+import { randomUUID } from 'crypto';
+import { detectDuplicates } from '../detection/two-stage-detector.js';
+import { runSafetyChecks, checkBlockers } from '../safety/safety-checks.js';
+import { mergeMemories } from '../strategies/merge-strategies.js';
+import { estimateTokensSaved } from '../analytics/token-estimator.js';
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+import { buildSuccessResponse, buildErrorResponse } from '../utils/response-builder.js';
+
+interface DetectDuplicatesInput {
+  projectId?: string;
+  threshold?: number;
+  memoryType?: 'fact' | 'preference' | 'decision' | 'observation' | 'context';
+  limit?: number;
+  autoCreateProposals?: boolean;
+}
+
+interface DetectDuplicatesResponse {
+  ok: boolean;
+  message: string;
+  data?: {
+    projectId: string;
+    duplicateCount: number;
+    proposalsCreated: number;
+    proposalIds: string[];
+    statistics: {
+      totalMemories: number;
+      scannedMemories: number;
+      candidatesFound: number;
+      estimatedTokensSaved: number;
+    };
+    timing: {
+      stage1Ms: number;
+      stage2Ms: number;
+      totalMs: number;
+    };
+  };
+  error?: string;
+}
+
+/**
+ * Handle detect_duplicate_memories tool call
+ *
+ * Algorithm:
+ * 1. Run two-stage detection (SimHash → embedding similarity)
+ * 2. Filter by safety checks
+ * 3. Create merge proposals in database
+ * 4. Return summary
+ */
+export async function handleDetectDuplicates(input: DetectDuplicatesInput): Promise<DetectDuplicatesResponse> {
+  try {
+    const projectId = input.projectId;
+    const threshold = input.threshold ?? 0.85;
+    const memoryType = input.memoryType;
+    const limit = input.limit ?? 50;
+    const autoCreateProposals = input.autoCreateProposals !== false;
+
+    if (!projectId) {
+      return buildErrorResponse('projectId is required', 'projectId is required');
+    }
+
+    // Run detection
+    const startTime = Date.now();
+    const detectionResult = await detectDuplicates({
+      projectId,
+      threshold,
+      type: memoryType,
+      limit,
+    });
+    const detectionTime = Date.now() - startTime;
+
+    if (detectionResult.candidates.length === 0) {
+      return buildSuccessResponse('No duplicates found', {
+        projectId,
+        duplicateCount: 0,
+        proposalsCreated: 0,
+        proposalIds: [],
+        statistics: {
+          totalMemories: detectionResult.statistics.totalMemories,
+          scannedMemories: detectionResult.statistics.totalMemories,
+          candidatesFound: 0,
+          estimatedTokensSaved: 0,
+        },
+        timing: {
+          stage1Ms: detectionResult.stage1Time,
+          stage2Ms: detectionResult.stage2Time,
+          totalMs: detectionTime,
+        },
+      });
+    }
+
+    if (!autoCreateProposals) {
+      return buildSuccessResponse(`Found ${detectionResult.candidates.length} potential duplicates`, {
+        projectId,
+        duplicateCount: detectionResult.candidates.length,
+        proposalsCreated: 0,
+        proposalIds: [],
+        statistics: {
+          totalMemories: detectionResult.statistics.totalMemories,
+          scannedMemories: detectionResult.statistics.totalMemories,
+          candidatesFound: detectionResult.candidates.length,
+          estimatedTokensSaved: 0,
+        },
+        timing: {
+          stage1Ms: detectionResult.stage1Time,
+          stage2Ms: detectionResult.stage2Time,
+          totalMs: detectionTime,
+        },
+      });
+    }
+
+    // Create proposals for detected duplicates
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    const proposalIds: string[] = [];
+    let estimatedTokensSaved = 0;
+
+    for (const candidate of detectionResult.candidates) {
+      const sources = [candidate.memory1, candidate.memory2];
+
+      // Run safety checks
+      const safetyResult = runSafetyChecks(sources, {
+        similarityScore: candidate.similarityScore,
+      });
+
+      if (!safetyResult.passed) {
+        // Skip this candidate if safety checks fail
+        continue;
+      }
+
+      // Merge memories
+      const merged = mergeMemories(sources);
+
+      // Estimate tokens saved
+      const tokensSaved = estimateTokensSaved(sources, merged);
+      estimatedTokensSaved += tokensSaved;
+
+      // Create proposal
+      const proposalId = randomUUID();
+      await db.insert(schema.memoryMergeProposals).values({
+        id: proposalId as unknown as UUID,
+        projectId: projectId as unknown as UUID,
+        userId: candidate.memory1.userId,
+        sourceMemoryIds: [candidate.memory1.id, candidate.memory2.id],
+        proposedContent: merged.content,
+        proposedSummary: merged.summary,
+        proposedTags: merged.tags,
+        proposedMetadata: merged.metadata,
+        detectionMethod: candidate.detectionMethod,
+        similarityScore: candidate.similarityScore,
+        confidenceLevel: candidate.confidenceLevel,
+        mergeReason: candidate.mergeReason,
+        conflictWarnings: merged.conflictWarnings,
+        status: 'pending',
+        createdAt: new Date(),
+      } as any);
+
+      proposalIds.push(proposalId);
+    }
+
+    return buildSuccessResponse(`Created ${proposalIds.length} merge proposals from ${detectionResult.candidates.length} duplicates`, {
+      projectId,
+      duplicateCount: detectionResult.candidates.length,
+      proposalsCreated: proposalIds.length,
+      proposalIds,
+      statistics: {
+        totalMemories: detectionResult.statistics.totalMemories,
+        scannedMemories: detectionResult.statistics.totalMemories,
+        candidatesFound: detectionResult.candidates.length,
+        estimatedTokensSaved,
+      },
+      timing: {
+        stage1Ms: detectionResult.stage1Time,
+        stage2Ms: detectionResult.stage2Time,
+        totalMs: detectionTime,
+      },
+    });
+  } catch (error) {
+    return buildErrorResponse('Failed to detect duplicates', error);
+  }
+}

package/core/algorithms/handlers/get-stats.ts

@@ -0,0 +1,132 @@
+/**
+ * MCP Tool: get_merge_stats
+ *
+ * Returns statistics about memory merges for a project
+ */
+
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+import { calculateProjectTokenSavings, formatTokenCount } from '../analytics/token-estimator.js';
+import type { Memory, MemoryMergeHistory } from '../../../db/drizzle/schema.js';
+
+interface GetMergeStatsInput {
+  projectId: string;
+}
+
+interface MergeStats {
+  projectId: string;
+  totalMemories: number;
+  mergeableMemories: number;
+  mergedMemories: number;
+  canonicalMemories: number;
+  pendingProposals: number;
+  approvedMerges: number;
+  rejectedProposals: number;
+  tokensSaved: {
+    total: number;
+    formatted: string;
+    percentage: number;
+  };
+  averageMergeSize: number;
+  reversedMerges: number;
+}
+
+interface GetMergeStatsResponse {
+  ok: boolean;
+  message: string;
+  data?: MergeStats;
+  error?: string;
+}
+
+/**
+ * Handle get_merge_stats tool call
+ *
+ * Gathers merge statistics for a project
+ */
+export async function handleGetMergeStats(input: GetMergeStatsInput): Promise<GetMergeStatsResponse> {
+  try {
+    const { projectId } = input;
+
+    if (!projectId) {
+      return {
+        ok: false,
+        message: 'projectId is required',
+        error: 'projectId is required',
+      };
+    }
+
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Get all memories in project
+    const memories: Memory[] = await db
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.projectId, projectId));
+
+    // Get merge history
+    const mergeHistory: MemoryMergeHistory[] = await db
+      .select()
+      .from(schema.memoryMergeHistory)
+      .where(eq(schema.memoryMergeHistory.projectId, projectId));
+
+    // Get proposals
+    const allProposals: any[] = await db
+      .select()
+      .from(schema.memoryMergeProposals)
+      .where(eq(schema.memoryMergeProposals.projectId, projectId));
+
+    // Calculate statistics
+    const totalMemories = memories.length;
+    const mergedMemories = memories.filter((m) => m.isMerged).length;
+    const canonicalMemories = memories.filter((m) => m.isCanonical).length;
+    const mergeableMemories = memories.filter(
+      (m) => m.isMergeable && !m.isMerged && m.isActive
+    ).length;
+
+    const pendingProposals = allProposals.filter((p) => p.status === 'pending').length;
+    const approvedMerges = allProposals.filter((p) => p.status === 'approved').length;
+    const rejectedProposals = allProposals.filter((p) => p.status === 'rejected').length;
+
+    const reversedMerges = mergeHistory.filter((m) => m.isReversed).length;
+
+    // Calculate token savings
+    const tokenStats = await calculateProjectTokenSavings(projectId);
+
+    // Calculate average merge size (memories per canonical)
+    const averageMergeSize =
+      canonicalMemories > 0
+        ? Math.round((mergedMemories + canonicalMemories) / canonicalMemories)
+        : 1;
+
+    return {
+      ok: true,
+      message: 'Merge statistics retrieved',
+      data: {
+        projectId,
+        totalMemories,
+        mergeableMemories,
+        mergedMemories,
+        canonicalMemories,
+        pendingProposals,
+        approvedMerges,
+        rejectedProposals,
+        tokensSaved: {
+          total: tokenStats.totalSaved,
+          formatted: formatTokenCount(tokenStats.totalSaved),
+          percentage: tokenStats.tokenSavingPercentage,
+        },
+        averageMergeSize,
+        reversedMerges,
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      message: 'Failed to get merge statistics',
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}

package/core/algorithms/handlers/list-proposals.ts

@@ -0,0 +1,130 @@
+/**
+ * MCP Tool: list_merge_proposals
+ *
+ * Lists pending merge proposals for user review
+ */
+
+import type { MemoryMergeProposal } from '../../../db/drizzle/schema.js';
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq, and, desc } from 'drizzle-orm';
+
+interface ListProposalsInput {
+  projectId: string;
+  status?: 'pending' | 'approved' | 'rejected' | 'expired';
+  limit?: number;
+}
+
+interface ProposalSummary {
+  id: string;
+  projectId: string;
+  sourceMemoryIds: string[];
+  status: 'pending' | 'approved' | 'rejected' | 'expired';
+  confidenceLevel: 'high' | 'medium' | 'low';
+  similarityScore: number;
+  mergeReason: string;
+  createdAt: string;
+  conflictWarnings: string[];
+}
+
+interface ListProposalsResponse {
+  ok: boolean;
+  message: string;
+  data?: {
+    projectId: string;
+    count: number;
+    proposals: ProposalSummary[];
+    byStatus: {
+      pending: number;
+      approved: number;
+      rejected: number;
+      expired: number;
+    };
+  };
+  error?: string;
+}
+
+/**
+ * Handle list_merge_proposals tool call
+ *
+ * Lists merge proposals that are waiting for user review
+ */
+export async function handleListProposals(input: ListProposalsInput): Promise<ListProposalsResponse> {
+  try {
+    const { projectId, status, limit = 20 } = input;
+
+    if (!projectId) {
+      return {
+        ok: false,
+        message: 'projectId is required',
+        error: 'projectId is required',
+      };
+    }
+
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Build query
+    let query: any = db
+      .select()
+      .from(schema.memoryMergeProposals)
+      .where(eq(schema.memoryMergeProposals.projectId, projectId));
+
+    // Filter by status if specified
+    if (status) {
+      query = query.where(eq(schema.memoryMergeProposals.status, status));
+    }
+
+    // Order by creation date (newest first)
+    query = query.orderBy(desc(schema.memoryMergeProposals.createdAt));
+
+    // Limit results
+    query = query.limit(limit);
+
+    const proposals: MemoryMergeProposal[] = await query;
+
+    // Count by status for the project
+    const allProposals: MemoryMergeProposal[] = await db
+      .select()
+      .from(schema.memoryMergeProposals)
+      .where(eq(schema.memoryMergeProposals.projectId, projectId));
+
+    const byStatus = {
+      pending: allProposals.filter((p) => p.status === 'pending').length,
+      approved: allProposals.filter((p) => p.status === 'approved').length,
+      rejected: allProposals.filter((p) => p.status === 'rejected').length,
+      expired: allProposals.filter((p) => p.status === 'expired').length,
+    };
+
+    // Format proposals
+    const formattedProposals: ProposalSummary[] = proposals.map((p) => ({
+      id: p.id,
+      projectId: p.projectId,
+      sourceMemoryIds: (p.sourceMemoryIds as unknown as string[]) || [],
+      status: (p.status as 'pending' | 'approved' | 'rejected' | 'expired') || 'pending',
+      confidenceLevel: (p.confidenceLevel as 'high' | 'medium' | 'low') || 'medium',
+      similarityScore: typeof p.similarityScore === 'string' ? parseFloat(p.similarityScore) : p.similarityScore || 0,
+      mergeReason: p.mergeReason || '',
+      createdAt: p.createdAt?.toISOString() || '',
+      conflictWarnings: (p.conflictWarnings as unknown as string[]) || [],
+    }));
+
+    return {
+      ok: true,
+      message: `Found ${formattedProposals.length} proposals (${byStatus.pending} pending)`,
+      data: {
+        projectId,
+        count: formattedProposals.length,
+        proposals: formattedProposals,
+        byStatus,
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      message: 'Failed to list proposals',
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}

package/core/algorithms/handlers/preview-merge.ts

@@ -0,0 +1,139 @@
+/**
+ * Shows a preview of the merged result without executing the merge.
+ */
+
+import type { Memory, MemoryMergeProposal } from '../../../db/drizzle/schema.js';
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+import { estimateMergeSavingsPreview } from '../analytics/token-estimator.js';
+import { mergeMemories } from '../strategies/merge-strategies.js';
+
+interface PreviewMergeInput {
+  proposalId: string;
+}
+
+interface PreviewMergeResponse {
+  ok: boolean;
+  message: string;
+  data?: {
+    proposalId: string;
+    sourceMemories: Array<{
+      id: string;
+      type: string;
+      content: string;
+      summary: string | null;
+      tags: string[];
+      createdAt: string;
+    }>;
+    mergedResult: {
+      content: string;
+      summary: string | null;
+      tags: string[];
+      metadata: Record<string, unknown>;
+    };
+    analysis: {
+      mergeReason: string;
+      conflictWarnings: string[];
+      savedTokens: number;
+      savedPercentage: number;
+      similarityScore: number;
+      confidenceLevel: string;
+    };
+  };
+  error?: string;
+}
+
+export async function handlePreviewMerge(input: PreviewMergeInput): Promise<PreviewMergeResponse> {
+  try {
+    const { proposalId } = input;
+
+    if (!proposalId) {
+      return {
+        ok: false,
+        message: 'proposalId is required',
+        error: 'proposalId is required',
+      };
+    }
+
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Load proposal
+    const [proposal] = await db
+      .select()
+      .from(schema.memoryMergeProposals)
+      .where(eq(schema.memoryMergeProposals.id, proposalId));
+
+    if (!proposal) {
+      return {
+        ok: false,
+        message: 'Proposal not found',
+        error: `Proposal ${proposalId} not found`,
+      };
+    }
+
+    // Load source memories
+    const sourceIds = (proposal.sourceMemoryIds as unknown as string[]) || [];
+    const sourceMemories: Memory[] = [];
+
+    for (const id of sourceIds) {
+      const [memory] = await db.select().from(schema.memories).where(eq(schema.memories.id, id));
+      if (memory) {
+        sourceMemories.push(memory);
+      }
+    }
+
+    if (sourceMemories.length === 0) {
+      return {
+        ok: false,
+        message: 'No source memories found',
+        error: 'Could not load source memories',
+      };
+    }
+
+    // Regenerate merged result from source memories (should match proposal)
+    const merged = mergeMemories(sourceMemories);
+
+    // Calculate token savings
+    const savings = estimateMergeSavingsPreview(sourceMemories, merged);
+
+    // Format response
+    return {
+      ok: true,
+      message: 'Merge preview generated',
+      data: {
+        proposalId,
+        sourceMemories: sourceMemories.map((m) => ({
+          id: m.id,
+          type: m.type,
+          content: m.content,
+          summary: m.summary,
+          tags: m.tags || [],
+          createdAt: m.createdAt?.toISOString() || '',
+        })),
+        mergedResult: {
+          content: merged.content,
+          summary: merged.summary,
+          tags: merged.tags,
+          metadata: merged.metadata,
+        },
+        analysis: {
+          mergeReason: merged.mergeReason,
+          conflictWarnings: merged.conflictWarnings,
+          savedTokens: savings.savedTokens,
+          savedPercentage: savings.savedPercentage,
+          similarityScore: proposal.similarityScore || 0,
+          confidenceLevel: proposal.confidenceLevel || 'unknown',
+        },
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      message: 'Failed to preview merge',
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}

package/core/algorithms/handlers/reject-merge.ts

@@ -0,0 +1,93 @@
+/**
+ * MCP Tool: reject_merge
+ *
+ * Rejects a merge proposal without executing it
+ */
+
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+
+interface RejectMergeInput {
+  proposalId: string;
+  reviewNotes?: string;
+}
+
+interface RejectMergeResponse {
+  ok: boolean;
+  message: string;
+  data?: {
+    proposalId: string;
+    previousStatus: string;
+    newStatus: string;
+    rejectedAt: string;
+  };
+  error?: string;
+}
+
+/**
+ * Handle reject_merge tool call
+ *
+ * Marks a merge proposal as rejected without executing it
+ */
+export async function handleRejectMerge(input: RejectMergeInput): Promise<RejectMergeResponse> {
+  try {
+    const { proposalId, reviewNotes } = input;
+
+    if (!proposalId) {
+      return {
+        ok: false,
+        message: 'proposalId is required',
+        error: 'proposalId is required',
+      };
+    }
+
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Load proposal
+    const [proposal] = await db
+      .select()
+      .from(schema.memoryMergeProposals)
+      .where(eq(schema.memoryMergeProposals.id, proposalId));
+
+    if (!proposal) {
+      return {
+        ok: false,
+        message: 'Proposal not found',
+        error: `Proposal ${proposalId} not found`,
+      };
+    }
+
+    const previousStatus = proposal.status;
+
+    // Update status to rejected
+    const now = new Date();
+    await db
+      .update(schema.memoryMergeProposals)
+      .set({
+        status: 'rejected',
+        reviewedAt: now,
+        reviewNotes: reviewNotes || `Rejected without comment`,
+      })
+      .where(eq(schema.memoryMergeProposals.id, proposalId));
+
+    return {
+      ok: true,
+      message: `Merge proposal rejected`,
+      data: {
+        proposalId,
+        previousStatus: previousStatus || 'unknown',
+        newStatus: 'rejected',
+        rejectedAt: now.toISOString(),
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      message: 'Failed to reject merge',
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}