squish-memory 1.1.5 → 1.2.1

package/core/algorithms/strategies/merge-strategies.ts

@@ -0,0 +1,381 @@
+/**
+ * Type-specific merge strategies for different memory types.
+ * Each type has different merge semantics to preserve meaning and prevent data loss.
+ */
+
+import type { Memory, MemoryType } from '../../../db/drizzle/schema.js';
+
+export interface MergeStrategy {
+  type: MemoryType;
+  /**
+   * Merge a set of source memories into a single canonical memory
+   */
+  merge(sources: Memory[]): MergedMemory;
+  /**
+   * Check if memories can be safely merged
+   * Returns { ok, reason } where reason explains why merging is not allowed
+   */
+  canMerge(sources: Memory[]): { ok: boolean; reason?: string };
+}
+
+export interface MergedMemory {
+  content: string;
+  summary: string | null;
+  tags: string[];
+  metadata: Record<string, unknown>;
+  mergeReason: string;
+  conflictWarnings: string[];
+}
+
+/**
+ * Base implementation of MergeStrategy with common functionality.
+ * Reduces code duplication across specific merge strategies.
+ */
+export abstract class BaseMergeStrategy implements MergeStrategy {
+  abstract type: MemoryType;
+
+  canMerge(sources: Memory[]): { ok: boolean; reason?: string } {
+    if (sources.length < 2) {
+      return { ok: false, reason: 'Need at least 2 memories to merge' };
+    }
+    return { ok: true };
+  }
+
+  protected validateSources(sources: Memory[]): boolean {
+    return sources.length >= 2;
+  }
+
+  protected handleEmptySources(): MergedMemory {
+    return {
+      content: '',
+      summary: null,
+      tags: [],
+      metadata: {},
+      mergeReason: 'Empty source set',
+      conflictWarnings: [],
+    };
+  }
+
+  protected mergeTags(sources: Memory[]): string[] {
+    const tagSet = new Set<string>();
+    for (const source of sources) {
+      for (const tag of source.tags || []) {
+        tagSet.add(tag);
+      }
+    }
+    return Array.from(tagSet);
+  }
+
+  protected buildBaseMetadata(
+    sources: Memory[],
+    extra?: Record<string, unknown>
+  ): Record<string, unknown> {
+    return {
+      mergedFrom: sources.map((m) => m.id),
+      mergeCount: sources.length,
+      ...extra,
+    };
+  }
+
+  protected sortByDate(
+    sources: Memory[],
+    order: 'asc' | 'desc' = 'desc'
+  ): Memory[] {
+    const sorted = [...sources].sort(
+      (a, b) => new Date(b.createdAt!).getTime() - new Date(a.createdAt!).getTime()
+    );
+    return order === 'asc' ? [...sorted].reverse() : sorted;
+  }
+
+  protected sortChronologically(sources: Memory[]): Memory[] {
+    return [...sources].sort(
+      (a, b) => new Date(a.createdAt!).getTime() - new Date(b.createdAt!).getTime()
+    );
+  }
+
+  // Abstract method each subclass must implement
+  abstract merge(sources: Memory[]): MergedMemory;
+}
+
+/**
+ * FACT strategy: Union of information, remove exact duplicates.
+ * Combines all unique facts into a unified statement with provenance tracking.
+ */
+class FactMergeStrategy extends BaseMergeStrategy {
+  type: MemoryType = 'fact';
+
+  merge(sources: Memory[]): MergedMemory {
+    if (sources.length === 0) {
+      return this.handleEmptySources();
+    }
+
+    // Split content into sentences and deduplicate
+    const sentenceSet = new Set<string>();
+    const timestamps: string[] = [];
+
+    for (const source of sources) {
+      // Extract sentences (split by period, exclamation, question)
+      const sentences = source.content
+        .split(/[.!?]\s+/)
+        .map((s: string) => s.trim())
+        .filter((s: string) => s.length > 0);
+
+      for (const sentence of sentences) {
+        sentenceSet.add(sentence);
+      }
+
+      if (source.createdAt) {
+        timestamps.push(source.createdAt);
+      }
+    }
+
+    // Sort sentences for consistency
+    const mergedSentences = Array.from(sentenceSet).sort();
+    const content = mergedSentences.join('. ') + (mergedSentences.length > 0 ? '.' : '');
+
+    // Build merged metadata with provenance
+    const metadata = this.buildBaseMetadata(sources, {
+      timestamps: timestamps.sort(),
+    });
+
+    return {
+      content,
+      summary: null,
+      tags: this.mergeTags(sources),
+      metadata,
+      mergeReason: `Merged ${sources.length} facts by combining all unique statements`,
+      conflictWarnings: [],
+    };
+  }
+}
+
+/**
+ * PREFERENCE strategy: Keep latest by timestamp, track evolution history.
+ * Warns if preferences conflict, indicating user changed their mind.
+ */
+class PreferenceMergeStrategy extends BaseMergeStrategy {
+  type: MemoryType = 'preference';
+
+  merge(sources: Memory[]): MergedMemory {
+    if (sources.length === 0) {
+      return this.handleEmptySources();
+    }
+
+    // Sort by creation date (newest first)
+    const sorted = this.sortByDate(sources);
+
+    const latest = sorted[0];
+    const warnings: string[] = [];
+
+    // Check for conflicting preferences
+    const uniqueContents = new Set(sources.map((m) => m.content));
+    if (uniqueContents.size > 1) {
+      warnings.push(`Multiple preferences detected: ${Array.from(uniqueContents).join(', ')}`);
+      warnings.push(`Using latest preference from ${latest.createdAt || 'unknown date'}`);
+    }
+
+    // Build history timeline
+    const history = sorted.map((m) => ({
+      preference: m.content,
+      createdAt: m.createdAt,
+      confidence: m.confidence || 100,
+      source: m.source,
+    }));
+
+    const metadata = this.buildBaseMetadata(sources, {
+      preferenceHistory: history,
+      latestAt: latest.createdAt,
+    });
+
+    const summary =
+      sources.length > 1
+        ? `Prefers: ${latest.content} (evolved from ${sources.length} preferences)`
+        : null;
+
+    return {
+      content: latest.content,
+      summary,
+      tags: latest.tags || [],
+      metadata,
+      mergeReason: `Merged ${sources.length} preference records, keeping latest from ${latest.createdAt}`,
+      conflictWarnings: warnings,
+    };
+  }
+}
+
+/**
+ * DECISION strategy: Keep latest decision, link to previous ones in timeline.
+ * Warns if decisions contradict, preserving rationale history.
+ */
+class DecisionMergeStrategy extends BaseMergeStrategy {
+  type: MemoryType = 'decision';
+
+  merge(sources: Memory[]): MergedMemory {
+    if (sources.length === 0) {
+      return this.handleEmptySources();
+    }
+
+    // Sort by creation date (newest first)
+    const sorted = this.sortByDate(sources);
+
+    const latest = sorted[0];
+    const warnings: string[] = [];
+
+    // Check for conflicting decisions
+    const uniqueDecisions = new Set(sources.map((m) => m.content));
+    if (uniqueDecisions.size > 1) {
+      warnings.push(
+        `Decision changed: was ${sorted[sorted.length - 1].content}, now ${latest.content}`
+      );
+    }
+
+    // Build decision timeline
+    const timeline = sorted.map((m) => ({
+      decision: m.content,
+      createdAt: m.createdAt,
+      confidence: m.confidence || 100,
+      rationale: m.summary || 'No rationale recorded',
+    }));
+
+    const metadata = this.buildBaseMetadata(sources, {
+      decisionTimeline: timeline,
+      currentDecisionAt: latest.createdAt,
+      supersedes: sorted.slice(1).map((m) => m.id),
+    });
+
+    const summary = sources.length > 1 ? `Decided: ${latest.content} (${sources.length} decisions)` : null;
+
+    return {
+      content: latest.content,
+      summary,
+      tags: latest.tags || [],
+      metadata,
+      mergeReason: `Merged ${sources.length} decision records, keeping latest from ${latest.createdAt}`,
+      conflictWarnings: warnings,
+    };
+  }
+}
+
+/**
+ * OBSERVATION strategy: Aggregate observations in chronological order.
+ * Preserves temporal patterns and frequency information.
+ */
+class ObservationMergeStrategy extends BaseMergeStrategy {
+  type: MemoryType = 'observation';
+
+  merge(sources: Memory[]): MergedMemory {
+    if (sources.length === 0) {
+      return this.handleEmptySources();
+    }
+
+    // Sort chronologically
+    const sorted = this.sortChronologically(sources);
+
+    // Create observation summary
+    const observations = sorted.map((m) => `• ${m.content}`).join('\n');
+
+    const metadata = this.buildBaseMetadata(sources, {
+      observationCount: sources.length,
+      timeSpan: {
+        start: sorted[0].createdAt,
+        end: sorted[sorted.length - 1].createdAt,
+      },
+      chronologicalOrder: sorted.map((m) => ({
+        id: m.id,
+        content: m.content,
+        createdAt: m.createdAt,
+      })),
+    });
+
+    return {
+      content: `Observations (${sources.length} total):\n${observations}`,
+      summary: `${sources.length} observations over time period`,
+      tags: this.mergeTags(sources),
+      metadata,
+      mergeReason: `Merged ${sources.length} observations chronologically`,
+      conflictWarnings: [],
+    };
+  }
+}
+
+/**
+ * CONTEXT strategy: Union of unique context, remove exact duplicates.
+ * Preserves all distinct context items.
+ */
+class ContextMergeStrategy extends BaseMergeStrategy {
+  type: MemoryType = 'context';
+
+  merge(sources: Memory[]): MergedMemory {
+    if (sources.length === 0) {
+      return this.handleEmptySources();
+    }
+
+    // Deduplicate by content (exact matches)
+    const uniqueContexts = new Map<string, Memory>();
+    for (const source of sources) {
+      if (!uniqueContexts.has(source.content)) {
+        uniqueContexts.set(source.content, source);
+      }
+    }
+
+    // Format as list
+    const contextList = Array.from(uniqueContexts.keys())
+      .sort()
+      .map((content) => `• ${content}`)
+      .join('\n');
+
+    const metadata = this.buildBaseMetadata(sources, {
+      uniqueContextCount: uniqueContexts.size,
+      totalContextCount: sources.length,
+      deduplicatedEntries: sources.length - uniqueContexts.size,
+    });
+
+    return {
+      content: contextList,
+      summary: `${uniqueContexts.size} context items (${sources.length} total)`,
+      tags: this.mergeTags(sources),
+      metadata,
+      mergeReason: `Merged ${sources.length} context records into ${uniqueContexts.size} unique items`,
+      conflictWarnings: [],
+    };
+  }
+}
+
+export const MERGE_STRATEGIES: Record<MemoryType, MergeStrategy> = {
+  fact: new FactMergeStrategy(),
+  preference: new PreferenceMergeStrategy(),
+  decision: new DecisionMergeStrategy(),
+  observation: new ObservationMergeStrategy(),
+  context: new ContextMergeStrategy(),
+  reflection: new ObservationMergeStrategy(),
+  note: new ObservationMergeStrategy(), // jot uses observation strategy
+};
+
+export function getMergeStrategy(type: MemoryType): MergeStrategy {
+  const strategy = MERGE_STRATEGIES[type];
+  if (!strategy) {
+    throw new Error(`No merge strategy defined for type: ${type}`);
+  }
+  return strategy;
+}
+
+export function mergeMemories(sources: Memory[]): MergedMemory {
+  if (sources.length === 0) {
+    throw new Error('Cannot merge: no source memories');
+  }
+
+  // All sources must be same type
+  const type = sources[0].type;
+  const allSameType = sources.every((m) => m.type === type);
+  if (!allSameType) {
+    throw new Error('Cannot merge: memories must be same type');
+  }
+
+  const strategy = getMergeStrategy(type);
+  const canMerge = strategy.canMerge(sources);
+  if (!canMerge.ok) {
+    throw new Error(`Cannot merge: ${canMerge.reason}`);
+  }
+
+  return strategy.merge(sources);
+}

package/core/algorithms/types.ts

@@ -0,0 +1,140 @@
+/**
+ * TypeScript types and interfaces for memory merging
+ */
+
+import type { Memory, MemoryMergeProposal, MemoryMergeHistory } from '../../db/drizzle/schema.js';
+import type { SafetyCheckResult } from '../../core/lib/types.js';
+
+/**
+ * Merge proposal status
+ */
+export type MergeProposalStatus = 'pending' | 'approved' | 'rejected' | 'expired';
+
+/**
+ * Confidence level for merge suggestions
+ */
+export type ConfidenceLevel = 'high' | 'medium' | 'low';
+
+/**
+ * Detection method used to find duplicates
+ */
+export type DetectionMethod = 'simhash' | 'minhash' | 'embedding';
+
+/**
+ * Merge strategy type per memory type
+ */
+export type MergeStrategyType = 'union' | 'latest' | 'voting' | 'custom';
+
+/**
+ * Memory pair candidate for merging
+ */
+export interface MemoryMergePair {
+  memory1: Memory;
+  memory2: Memory;
+  similarityScore: number;
+  detectionMethod: DetectionMethod;
+  confidenceLevel: ConfidenceLevel;
+  mergeReason: string;
+}
+
+/**
+ * Merge proposal with all details
+ */
+export interface MergeProposalDetails {
+  proposal: MemoryMergeProposal;
+  sourceMemories: Memory[];
+  previewMerged: {
+    content: string;
+    summary: string | null;
+    tags: string[];
+    metadata: Record<string, unknown>;
+  };
+  analysis: {
+    savedTokens: number;
+    savedPercentage: number;
+    conflictWarnings: string[];
+  };
+}
+
+/**
+ * Merge result after approval
+ */
+export interface MergeResult {
+  proposalId: string;
+  canonicalMemoryId: string;
+  mergedMemoryIds: string[];
+  tokensSaved: number;
+  mergedAt: Date;
+  mergeHistoryId?: string;
+}
+
+/**
+ * Merge reversal result
+ */
+export interface MergeReversal {
+  mergeHistoryId: string;
+  canonicalMemoryId: string;
+  restoredMemoryIds: string[];
+  reversedAt: Date;
+}
+
+/**
+ * Detection statistics
+ */
+export interface DetectionStatistics {
+  totalMemories: number;
+  scannedMemories: number;
+  candidatesFound: number;
+  candidatesRanked: number;
+  estimatedTokensSaved: number;
+  timeMs: {
+    stage1: number;
+    stage2: number;
+    total: number;
+  };
+}
+
+/**
+ * Project merge statistics
+ */
+export interface ProjectMergeStatistics {
+  projectId: string;
+  totalMemories: number;
+  mergeableMemories: number;
+  mergedMemories: number;
+  canonicalMemories: number;
+  pendingProposals: number;
+  approvedMerges: number;
+  rejectedProposals: number;
+  reversedMerges: number;
+  tokensSaved: {
+    total: number;
+    formatted: string;
+    percentage: number;
+  };
+  averageMergeSize: number;
+}
+
+
+
+/**
+ * Hash cache entry for fast duplicate detection
+ */
+export interface HashCacheEntry {
+  memoryId: string;
+  simhash: string;
+  minhash: number[];
+  contentHash: string;
+  lastUpdated: Date;
+}
+
+/**
+ * Stage 1 candidate pair with hash distances
+ */
+export interface Stage1CandidatePair {
+  memoryId1: string;
+  memoryId2: string;
+  simhashDistance: number;
+  minhashSimilarity: number;
+  matched: 'simhash' | 'minhash' | 'both';
+}

package/core/algorithms/utils/response-builder.ts

@@ -0,0 +1,61 @@
+/**
+ * Unified response builder for algorithm handlers
+ * Provides consistent response formatting across all algorithm handlers
+ */
+
+export interface ErrorResponse {
+  ok: false;
+  message: string;
+  error: string;
+}
+
+export interface SuccessResponse<T = unknown> {
+  ok: true;
+  message: string;
+  data?: T;
+}
+
+export type ApiResponse<T = unknown> = ErrorResponse | SuccessResponse<T>;
+
+/**
+ * Build a success response with optional data
+ */
+export function buildSuccessResponse<T = unknown>(
+  message: string,
+  data?: T
+): SuccessResponse<T> {
+  const response: SuccessResponse<T> = { ok: true, message };
+  if (data !== undefined) {
+    response.data = data;
+  }
+  return response;
+}
+
+/**
+ * Build an error response
+ */
+export function buildErrorResponse(
+  message: string,
+  error?: unknown
+): ErrorResponse {
+  return {
+    ok: false,
+    message,
+    error: error instanceof Error ? error.message : String(error || 'Unknown error'),
+  };
+}
+
+/**
+ * Wrap async handlers with consistent error handling
+ */
+export async function withErrorHandler<T>(
+  handler: () => Promise<T>,
+  errorMessage: string
+): Promise<ApiResponse<T>> {
+  try {
+    const result = await handler();
+    return buildSuccessResponse(errorMessage, result);
+  } catch (error) {
+    return buildErrorResponse(errorMessage, error);
+  }
+}