squish-memory 1.1.5 → 1.2.0

package/core/algorithms/handlers/reverse-merge.ts

@@ -0,0 +1,155 @@
+/**
+ * Reverses/undoes a completed merge and restores original memories.
+ */
+
+import type { Memory } from '../../../db/drizzle/schema.js';
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+
+interface ReverseMergeInput {
+  mergeHistoryId: string;
+  reason?: string;
+}
+
+interface ReverseMergeResponse {
+  ok: boolean;
+  message: string;
+  data?: {
+    mergeHistoryId: string;
+    canonicalMemoryId: string;
+    restoredMemoryIds: string[];
+    reversedAt: string;
+  };
+  error?: string;
+}
+
+export async function handleReverseMerge(input: ReverseMergeInput): Promise<ReverseMergeResponse> {
+  try {
+    const { mergeHistoryId, reason } = input;
+
+    if (!mergeHistoryId) {
+      return {
+        ok: false,
+        message: 'mergeHistoryId is required',
+        error: 'mergeHistoryId is required',
+      };
+    }
+
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Step 1: Load merge history record
+    const [history] = await db
+      .select()
+      .from(schema.memoryMergeHistory)
+      .where(eq(schema.memoryMergeHistory.id, mergeHistoryId));
+
+    if (!history) {
+      return {
+        ok: false,
+        message: 'Merge history record not found',
+        error: `Merge history ${mergeHistoryId} not found`,
+      };
+    }
+
+    // Check if already reversed
+    if (history.isReversed) {
+      return {
+        ok: false,
+        message: 'Merge already reversed',
+        error: 'This merge has already been reversed',
+      };
+    }
+
+    // Step 2: Load and mark canonical memory as inactive
+    const [canonicalMemory] = await db
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, history.canonicalMemoryId));
+
+    if (!canonicalMemory) {
+      return {
+        ok: false,
+        message: 'Canonical memory not found',
+        error: `Canonical memory ${history.canonicalMemoryId} not found`,
+      };
+    }
+
+    const now = new Date();
+
+    // Step 3: Restore source memories from snapshot
+    const sourceSnapshot = (history.sourceMemoriesSnapshot as unknown as Array<Record<string, any>>) || [];
+    const sourceMemoryIds = (history.sourceMemoryIds as unknown as string[]) || [];
+
+    if (sourceSnapshot.length === 0) {
+      return {
+        ok: false,
+        message: 'No snapshot data to restore from',
+        error: 'Merge history has no source memories snapshot',
+      };
+    }
+
+    // Restore each memory from snapshot
+    for (const snapshotData of sourceSnapshot) {
+      const memoryId = snapshotData.id;
+
+      // Load current state of the memory (should be marked as merged)
+      const [currentMemory] = await db
+        .select()
+        .from(schema.memories)
+        .where(eq(schema.memories.id, memoryId));
+
+      if (currentMemory) {
+        // Restore to original state
+        await db
+          .update(schema.memories)
+          .set({
+            isMerged: false,
+            mergedIntoId: null,
+            mergedAt: null,
+            isActive: true,
+            updatedAt: now,
+          })
+          .where(eq(schema.memories.id, memoryId));
+      }
+    }
+
+    // Step 4: Deactivate canonical memory
+    await db
+      .update(schema.memories)
+      .set({
+        isActive: false,
+        updatedAt: now,
+      })
+      .where(eq(schema.memories.id, history.canonicalMemoryId));
+
+    // Step 5: Update merge history record
+    await db
+      .update(schema.memoryMergeHistory)
+      .set({
+        isReversed: true,
+        reversedAt: now,
+        reversedBy: canonicalMemory.userId,
+      })
+      .where(eq(schema.memoryMergeHistory.id, mergeHistoryId));
+
+    return {
+      ok: true,
+      message: `Merge reversed successfully. Restored ${sourceMemoryIds.length} memories`,
+      data: {
+        mergeHistoryId,
+        canonicalMemoryId: history.canonicalMemoryId,
+        restoredMemoryIds: sourceMemoryIds,
+        reversedAt: now.toISOString(),
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      message: 'Failed to reverse merge',
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}

package/core/algorithms/index.ts

@@ -0,0 +1,39 @@
+/**
+ * Memory Merging Feature - Barrel Exports
+ *
+ * Provides a clean public API for memory merging functionality
+ */
+
+// Types
+export * from './types.js';
+
+// Detection
+export { detectDuplicates, analyzeMergePair, getDetectionStats } from './detection/two-stage-detector.js';
+export { SimHashFilter, MinHashFilter, findCandidatePairs } from './detection/hash-filters.js';
+export { rankCandidates, analyzePair } from './detection/semantic-ranker.js';
+
+// Merge Strategies
+export { getMergeStrategy, mergeMemories, MERGE_STRATEGIES } from './strategies/merge-strategies.js';
+export type { MergeStrategy, MergedMemory } from './strategies/merge-strategies.js';
+
+// Safety
+export { runSafetyChecks, checkBlockers, formatSafetyResults, describeSafetyChecks } from './safety/safety-checks.js';
+
+// Analytics & Tokens
+export {
+  estimateTokensSaved,
+  calculateProjectTokenSavings,
+  estimateMergeSavingsPreview,
+  getTokenStatistics,
+  formatTokenCount,
+  formatSavingsReport,
+} from './analytics/token-estimator.js';
+
+// Handlers (MCP Tools)
+export { handleDetectDuplicates } from './handlers/detect-duplicates.js';
+export { handleListProposals } from './handlers/list-proposals.js';
+export { handlePreviewMerge } from './handlers/preview-merge.js';
+export { handleApproveMerge } from './handlers/approve-merge.js';
+export { handleRejectMerge } from './handlers/reject-merge.js';
+export { handleReverseMerge } from './handlers/reverse-merge.js';
+export { handleGetMergeStats } from './handlers/get-stats.js';

package/core/algorithms/operations/cache-maintenance.ts

@@ -0,0 +1,182 @@
+/**
+ * Hash cache maintenance - SimHash/MinHash signatures for duplicate detection
+ */
+
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+import { SimHashFilter, MinHashFilter } from '../detection/hash-filters.js';
+import crypto from 'crypto';
+import { logger } from '../../../core/logger.js';
+
+function hashContent(content: string): string {
+  return crypto.createHash('md5').update(content).digest('hex');
+}
+
+export async function updateCache(memoryId: string): Promise<boolean> {
+  try {
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Load the memory
+    const [memory] = await db
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, memoryId));
+
+    if (!memory) {
+      return false;
+    }
+
+    // Generate hashes
+    const simhashFilter = new SimHashFilter();
+    const minhashFilter = new MinHashFilter();
+
+    const simhash = simhashFilter.generateHash(memory.content);
+    const minhash = minhashFilter.generateSignature(memory.content);
+    const contentHash = hashContent(memory.content);
+
+    // Upsert cache entry
+    const now = new Date();
+
+    // Check if entry exists
+    const [existing] = await db
+      .select()
+      .from(schema.memoryHashCache)
+      .where(eq(schema.memoryHashCache.memoryId, memoryId));
+
+    if (existing) {
+      // Update existing
+      await db
+        .update(schema.memoryHashCache)
+        .set({
+          simhash,
+          minhash: minhash,
+          contentHash,
+          lastUpdated: now,
+        })
+        .where(eq(schema.memoryHashCache.memoryId, memoryId));
+    } else {
+      // Create new
+      await db.insert(schema.memoryHashCache).values({
+        memoryId,
+        projectId: memory.projectId,
+        simhash,
+        minhash: minhash,
+        contentHash,
+        lastUpdated: now,
+      } as any);
+    }
+
+    return true;
+  } catch (error) {
+    logger.error(`Failed to update hash cache for ${memoryId}`, error);
+    return false;
+  }
+}
+
+export async function rebuildCache(projectId: string): Promise<{
+  processed: number;
+  succeeded: number;
+  failed: number;
+}> {
+  try {
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Get all memories in project
+    const memories: any[] = await db
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.projectId, projectId));
+
+    let succeeded = 0;
+    let failed = 0;
+
+    for (const memory of memories) {
+      const ok = await updateCache(memory.id);
+      if (ok) {
+        succeeded++;
+      } else {
+        failed++;
+      }
+    }
+
+    return {
+      processed: memories.length,
+      succeeded,
+      failed,
+    };
+  } catch (error) {
+    logger.error(`Failed to rebuild hash cache for project ${projectId}`, error);
+    return { processed: 0, succeeded: 0, failed: 0 };
+  }
+}
+
+export async function isStale(memoryId: string): Promise<boolean> {
+  try {
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    const [memory] = await db
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, memoryId));
+
+    if (!memory) {
+      return true;
+    }
+
+    const [cacheEntry] = await db
+      .select()
+      .from(schema.memoryHashCache)
+      .where(eq(schema.memoryHashCache.memoryId, memoryId));
+
+    if (!cacheEntry) {
+      return true; // No cache entry = stale
+    }
+
+    const currentContentHash = hashContent(memory.content);
+    return currentContentHash !== cacheEntry.contentHash;
+  } catch (error) {
+    logger.error('Failed to check hash cache staleness', error);
+    return true; // Assume stale on error
+  }
+}
+
+export async function cleanupOrphaned(projectId: string): Promise<number> {
+  try {
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+
+    // Get all cache entries for project
+    const cacheEntries: any[] = await db
+      .select()
+      .from(schema.memoryHashCache)
+      .where(eq(schema.memoryHashCache.projectId, projectId));
+
+    let deleted = 0;
+
+    for (const entry of cacheEntries) {
+      // Check if memory exists
+      const [memory] = await db
+        .select()
+        .from(schema.memories)
+        .where(eq(schema.memories.id, entry.memoryId));
+
+      if (!memory) {
+        // Memory doesn't exist, delete cache entry
+        await db
+          .delete(schema.memoryHashCache)
+          .where(eq(schema.memoryHashCache.memoryId, entry.memoryId));
+        deleted++;
+      }
+    }
+
+    return deleted;
+  } catch (error) {
+    logger.error('Failed to cleanup orphaned hash cache', error);
+    return 0;
+  }
+}

package/core/algorithms/safety/safety-checks.ts

@@ -0,0 +1,256 @@
+/**
+ * Safety checks to prevent bad merges.
+ * Checks run before creating merge proposals and are categorized as BLOCKER or WARNING.
+ */
+
+import type { Memory } from '../../../db/drizzle/schema.js';
+import type { SafetyCheckResult } from '../../lib/types.js';
+
+export interface SafetyCheck {
+  name: string;
+  description: string;
+  type: 'blocker' | 'warning'; // How to treat failures
+  check(memories: Memory[], metadata?: Record<string, unknown>): SafetyCheckResult;
+}
+
+const PASSED_RESULT: SafetyCheckResult = { passed: true, warnings: [], blockers: [] };
+
+function createBlockerCheck(
+  name: string,
+  description: string,
+  checkFn: (memories: Memory[], metadata?: Record<string, unknown>) => { passed: boolean; blockers: string[] }
+): SafetyCheck {
+  return {
+    name,
+    description,
+    type: 'blocker',
+    check: (memories, metadata) => {
+      const result = checkFn(memories, metadata);
+      return result.passed ? PASSED_RESULT : { passed: false, warnings: [], blockers: result.blockers };
+    },
+  };
+}
+
+function createWarningCheck(
+  name: string,
+  description: string,
+  checkFn: (memories: Memory[], metadata?: Record<string, unknown>) => { warnings: string[] }
+): SafetyCheck {
+  return {
+    name,
+    description,
+    type: 'warning',
+    check: (memories, metadata) => {
+      const result = checkFn(memories, metadata);
+      return { passed: true, warnings: result.warnings, blockers: [] };
+    },
+  };
+}
+
+export const SAFETY_CHECKS: SafetyCheck[] = [
+  createBlockerCheck(
+    'immutability',
+    'Prevent merging immutable memories',
+    (memories) => {
+      const immutableMemories = memories.filter((m) => !m.isMergeable);
+      if (immutableMemories.length === 0) {
+        return { passed: true, blockers: [] };
+      }
+      return {
+        passed: false,
+        blockers: [
+          `Cannot merge: ${immutableMemories.length} memory(ies) marked as immutable`,
+          `IDs: ${immutableMemories.map((m) => m.id).join(', ')}`,
+        ],
+      };
+    }
+  ),
+
+  createBlockerCheck(
+    'type_consistency',
+    'Ensure all memories are same type',
+    (memories) => {
+      const types = new Set(memories.map((m) => m.type));
+      if (types.size <= 1) {
+        return { passed: true, blockers: [] };
+      }
+      return {
+        passed: false,
+        blockers: [
+          `Cannot merge different types: ${Array.from(types).join(', ')}`,
+          'All memories must be same type (fact, preference, decision, etc.)',
+        ],
+      };
+    }
+  ),
+
+  createBlockerCheck(
+    'already_merged',
+    'Prevent re-merging of previously merged memories',
+    (memories) => {
+      const alreadyMerged = memories.filter((m) => m.isMerged);
+      if (alreadyMerged.length === 0) {
+        return { passed: true, blockers: [] };
+      }
+      return {
+        passed: false,
+        blockers: [
+          `Cannot merge: ${alreadyMerged.length} memory(ies) already merged`,
+          'Already-merged memories should not be re-merged. Undo previous merge first.',
+        ],
+      };
+    }
+  ),
+
+  createBlockerCheck(
+    'min_similarity',
+    'Ensure similarity is above minimum threshold',
+    (memories, metadata) => {
+      const minThreshold = 0.70;
+      if (!metadata || !('similarityScore' in metadata)) {
+        return { passed: true, blockers: [] };
+      }
+      const similarity = metadata.similarityScore as number;
+      if (similarity >= minThreshold) {
+        return { passed: true, blockers: [] };
+      }
+      return {
+        passed: false,
+        blockers: [
+          `Similarity too low: ${(similarity * 100).toFixed(1)}%`,
+          `Minimum required: ${(minThreshold * 100).toFixed(0)}%`,
+          'Increase similarity threshold or select more similar memories',
+        ],
+      };
+    }
+  ),
+
+  createWarningCheck(
+    'multi_user',
+    'Warn about merging memories from different users',
+    (memories) => {
+      const users = new Set(memories.map((m) => m.userId).filter(Boolean));
+      if (users.size <= 1) {
+        return { warnings: [] };
+      }
+      return {
+        warnings: [
+          `Merging memories from ${users.size} different users`,
+          'This is usually not recommended. Ensure you want to consolidate user-specific memories.',
+        ],
+      };
+    }
+  ),
+
+  createWarningCheck(
+    'privacy',
+    'Warn about mixing private and non-private memories',
+    (memories) => {
+      const privacyStates = new Set(memories.map((m) => m.isPrivate));
+      if (privacyStates.size <= 1) {
+        return { warnings: [] };
+      }
+      return {
+        warnings: [
+          'Merging private and non-private memories',
+          'The merged result will inherit the privacy setting of the canonical memory',
+        ],
+      };
+    }
+  ),
+
+  createWarningCheck(
+    'secrets',
+    'Warn about merging memories with detected secrets',
+    (memories) => {
+      const withSecrets = memories.filter((m) => m.hasSecrets);
+      if (withSecrets.length === 0) {
+        return { warnings: [] };
+      }
+      return {
+        warnings: [
+          `${withSecrets.length} memory(ies) contain detected secrets`,
+          'Ensure merged content does not expose sensitive information',
+          'Consider redacting secrets before merging',
+        ],
+      };
+    }
+  ),
+
+  createBlockerCheck(
+    'active_status',
+    'Ensure all memories are active',
+    (memories) => {
+      const inactive = memories.filter((m) => !m.isActive);
+      if (inactive.length === 0) {
+        return { passed: true, blockers: [] };
+      }
+      return {
+        passed: false,
+        blockers: [
+          `Cannot merge: ${inactive.length} memory(ies) are inactive (archived/expired)`,
+          'Only active memories can be merged',
+        ],
+      };
+    }
+  ),
+];
+
+export function runSafetyChecks(
+  memories: Memory[],
+  metadata?: Record<string, unknown>
+): SafetyCheckResult {
+  const results = SAFETY_CHECKS.map((check) => check.check(memories, metadata));
+
+  const allBlockers = results.flatMap((r) => r.blockers);
+  const allWarnings = results.flatMap((r) => r.warnings);
+
+  return {
+    passed: allBlockers.length === 0,
+    warnings: allWarnings,
+    blockers: allBlockers,
+  };
+}
+
+export function checkBlockers(memories: Memory[]): boolean {
+  const blockerChecks = SAFETY_CHECKS.filter((c) => c.type === 'blocker');
+
+  for (const check of blockerChecks) {
+    const result = check.check(memories);
+    if (!result.passed) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+export function formatSafetyResults(result: SafetyCheckResult): string {
+  if (result.passed && result.warnings.length === 0) {
+    return 'All safety checks passed';
+  }
+
+  const lines: string[] = [];
+
+  if (!result.passed && result.blockers.length > 0) {
+    lines.push('BLOCKERS (merge prevented):');
+    for (const blocker of result.blockers) {
+      lines.push(`  ✗ ${blocker}`);
+    }
+  }
+
+  if (result.warnings.length > 0) {
+    lines.push('WARNINGS (merge allowed with caution):');
+    for (const warning of result.warnings) {
+      lines.push(`  ⚠ ${warning}`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+export function describeSafetyChecks(): string {
+  return SAFETY_CHECKS.map(
+    (check) => `${check.name} [${check.type}]: ${check.description}`
+  ).join('\n');
+}