squish-memory 1.1.5 → 1.2.0

package/core/algorithms/analytics/token-estimator.ts

@@ -0,0 +1,216 @@
+/**
+ * Token estimation for calculating context window savings from merges.
+ * Uses simple heuristic: 1 token ≈ 4 characters (can be upgraded to tiktoken for accuracy).
+ */
+
+import type { Memory } from '../../../db/drizzle/schema.js';
+import type { MergedMemory } from '../strategies/merge-strategies.js';
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+
+function estimateTokensSimple(text: string): number {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+function estimateMetadataOverhead(): number {
+  return 25;
+}
+
+function estimateMemoryTokens(memory: Memory): number {
+  let tokens = 0;
+
+  tokens += estimateTokensSimple(memory.content);
+
+  if (memory.summary) {
+    tokens += estimateTokensSimple(memory.summary);
+  }
+
+  if (memory.tags && memory.tags.length > 0) {
+    tokens += estimateTokensSimple(memory.tags.join(' '));
+  }
+
+  if (memory.metadata) {
+    tokens += estimateTokensSimple(JSON.stringify(memory.metadata));
+  }
+
+  tokens += estimateMetadataOverhead();
+
+  return tokens;
+}
+
+function estimateMergedMemoryTokens(merged: MergedMemory): number {
+  let tokens = 0;
+
+  tokens += estimateTokensSimple(merged.content);
+
+  if (merged.summary) {
+    tokens += estimateTokensSimple(merged.summary);
+  }
+
+  if (merged.tags && merged.tags.length > 0) {
+    tokens += estimateTokensSimple(merged.tags.join(' '));
+  }
+
+  tokens += estimateTokensSimple(JSON.stringify(merged.metadata));
+  tokens += estimateMetadataOverhead();
+
+  return tokens;
+}
+
+export function estimateTokensSaved(sources: Memory[], merged: MergedMemory): number {
+  const sourceTokens = sources.reduce((sum, memory) => sum + estimateMemoryTokens(memory), 0);
+  const mergedTokens = estimateMergedMemoryTokens(merged);
+  const savings = sourceTokens - mergedTokens;
+
+  return Math.max(0, savings);
+}
+
+export async function calculateProjectTokenSavings(
+  projectId: string
+): Promise<{
+  totalSaved: number;
+  mergeCount: number;
+  avgSavingsPerMerge: number;
+  tokenSavingPercentage: number;
+  totalMemoryTokens: number;
+}> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const memories: Memory[] = await db
+    .select()
+    .from(schema.memories)
+    .where(eq(schema.memories.projectId, projectId));
+
+  const totalMemoryTokens = memories.reduce((sum, m) => sum + estimateMemoryTokens(m), 0);
+  if (!schema.memoryMergeHistory) {
+    return {
+      totalSaved: 0,
+      mergeCount: 0,
+      avgSavingsPerMerge: 0,
+      tokenSavingPercentage: 0,
+      totalMemoryTokens,
+    };
+  }
+
+  const mergeHistory: any[] = await db
+    .select()
+    .from(schema.memoryMergeHistory)
+    .where(eq(schema.memoryMergeHistory.projectId, projectId));
+
+  // Sum up token savings
+  const totalSaved = mergeHistory.reduce((sum, record) => sum + (record.tokensSaved || 0), 0);
+
+  const mergeCount = mergeHistory.length;
+  const avgSavingsPerMerge = mergeCount > 0 ? Math.round(totalSaved / mergeCount) : 0;
+  const tokenSavingPercentage = totalMemoryTokens > 0 ? (totalSaved / totalMemoryTokens) * 100 : 0;
+
+  return {
+    totalSaved,
+    mergeCount,
+    avgSavingsPerMerge,
+    tokenSavingPercentage,
+    totalMemoryTokens,
+  };
+}
+
+/**
+ * Format token counts for display
+ *
+ * Converts token count to human-readable format with context usage
+ */
+export function formatTokenCount(tokens: number): string {
+  // Show percentage of typical context window
+  // Claude 3 / GPT-4: 128k tokens (~32k practical for context)
+  const contextWindow = 128000;
+  const typicalUseful = 32000;
+
+  const percentage = (tokens / contextWindow) * 100;
+  const usefulPercent = (tokens / typicalUseful) * 100;
+
+  if (tokens < 1000) {
+    return `${tokens} tokens (${percentage.toFixed(3)}% of context)`;
+  }
+
+  const kiloTokens = (tokens / 1000).toFixed(1);
+  return `${kiloTokens}k tokens (${usefulPercent.toFixed(1)}% of typical recall window)`;
+}
+
+/**
+ * Format savings report
+ */
+export function formatSavingsReport(savings: {
+  totalSaved: number;
+  mergeCount: number;
+  avgSavingsPerMerge: number;
+  tokenSavingPercentage: number;
+  totalMemoryTokens: number;
+}): string {
+  const lines: string[] = [];
+
+  lines.push('Memory Merge Savings Report');
+  lines.push('='.repeat(40));
+
+  if (savings.mergeCount === 0) {
+    lines.push('No merges completed yet');
+    return lines.join('\n');
+  }
+
+  lines.push(`Total Merges: ${savings.mergeCount}`);
+  lines.push(`Total Tokens Saved: ${formatTokenCount(savings.totalSaved)}`);
+  lines.push(`Avg Savings per Merge: ${formatTokenCount(savings.avgSavingsPerMerge)}`);
+  lines.push(`Reduction: ${savings.tokenSavingPercentage.toFixed(2)}%`);
+  lines.push(`Total Memory Tokens: ${formatTokenCount(savings.totalMemoryTokens)}`);
+
+  return lines.join('\n');
+}
+
+/**
+ * Estimate savings for a proposed merge (preview)
+ *
+ * Used in merge preview to show user estimated savings
+ */
+export function estimateMergeSavingsPreview(
+  sources: Memory[],
+  merged: MergedMemory
+): { savedTokens: number; savedPercentage: number } {
+  const sourceTokens = sources.reduce((sum, m) => sum + estimateMemoryTokens(m), 0);
+  const mergedTokens = estimateMergedMemoryTokens(merged);
+  const savedTokens = Math.max(0, sourceTokens - mergedTokens);
+  const savedPercentage = sourceTokens > 0 ? (savedTokens / sourceTokens) * 100 : 0;
+
+  return { savedTokens, savedPercentage };
+}
+
+/**
+ * Get token statistics for a set of memories
+ */
+export function getTokenStatistics(memories: Memory[]): {
+  total: number;
+  min: number;
+  max: number;
+  average: number;
+  median: number;
+} {
+  const counts = memories.map((m) => estimateMemoryTokens(m));
+  const total = counts.reduce((a, b) => a + b, 0);
+  const min = Math.min(...counts);
+  const max = Math.max(...counts);
+  const average = Math.round(total / counts.length);
+
+  // Calculate median
+  const sorted = [...counts].sort((a, b) => a - b);
+  const middle = Math.floor(sorted.length / 2);
+  const median = sorted.length % 2 !== 0 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
+
+  return {
+    total,
+    min,
+    max,
+    average,
+    median: Math.round(median),
+  };
+}

package/core/algorithms/detection/hash-filters.ts

@@ -0,0 +1,260 @@
+/**
+ * Hash-based duplicate detection filters (Stage 1).
+ * Uses SimHash and MinHash for fast approximate matching before semantic analysis.
+ */
+
+/**
+ * SimHash: Fast fingerprinting for near-duplicate detection.
+ * Tokenizes content, weights by frequency, and produces comparable hash fingerprints.
+ * Similar documents produce similar hashes with Hamming distance indicating edit distance.
+ */
+export class SimHashFilter {
+  private readonly dimensions = 32; // 32-bit hash
+
+  generateHash(content: string): string {
+    if (!content || content.trim().length === 0) {
+      return '0'.repeat(16);
+    }
+
+    const tokens = content
+      .toLowerCase()
+      .split(/\W+/)
+      .filter((token) => token.length > 0);
+
+    if (tokens.length === 0) {
+      return '0'.repeat(16);
+    }
+
+    const tokenFreq = new Map<string, number>();
+    for (const token of tokens) {
+      tokenFreq.set(token, (tokenFreq.get(token) || 0) + 1);
+    }
+
+    const hashBits = new Array(this.dimensions).fill(0);
+
+    for (const [token, freq] of tokenFreq.entries()) {
+      const tokenHash = this.hashToken(token);
+      for (let i = 0; i < this.dimensions; i++) {
+        const bitSet = (tokenHash >>> i) & 1;
+        hashBits[i] += bitSet === 1 ? freq : -freq;
+      }
+    }
+
+    let result = 0;
+    for (let i = 0; i < this.dimensions; i++) {
+      if (hashBits[i] > 0) {
+        result |= 1 << i;
+      }
+    }
+
+    return (result >>> 0).toString(16).padStart(16, '0');
+  }
+
+  hammingDistance(hash1: string, hash2: string): number {
+    const num1 = BigInt('0x' + hash1);
+    const num2 = BigInt('0x' + hash2);
+    const xor = num1 ^ num2;
+    return this.popcount(xor);
+  }
+
+  findCandidates(
+    targetHash: string,
+    allHashes: Map<string, string>,
+    threshold: number
+  ): string[] {
+    const candidates: string[] = [];
+
+    for (const [memoryId, hash] of allHashes.entries()) {
+      const distance = this.hammingDistance(targetHash, hash);
+      if (distance <= threshold) {
+        candidates.push(memoryId);
+      }
+    }
+
+    return candidates;
+  }
+
+  private hashToken(token: string): number {
+    let hash = 2166136261;
+
+    for (let i = 0; i < token.length; i++) {
+      hash ^= token.charCodeAt(i);
+      hash = (hash * 16777619) & 0xffffffff;
+    }
+
+    const high = hash >>> 16;
+    const low = hash & 0xffff;
+    return (high * 65599 + low) >>> 0;
+  }
+
+  private popcount(n: bigint): number {
+    let count = 0;
+    while (n > 0n) {
+      count += Number(n & 1n);
+      n = n >> 1n;
+    }
+    return count;
+  }
+}
+
+/**
+ * MinHash: Estimates Jaccard similarity using character n-grams and multiple hash functions.
+ * Keeps minimum hash for each function, resulting in comparable signatures.
+ * More effective than SimHash for paraphrases. 128 functions give ~1% error margin.
+ */
+export class MinHashFilter {
+  private readonly numPermutations = 128; // Number of independent hash functions
+  private readonly ngramSize = 3; // Character n-gram size
+
+  generateSignature(content: string): number[] {
+    if (!content || content.trim().length === 0) {
+      return new Array(this.numPermutations).fill(0);
+    }
+
+    const ngrams = this.generateNgrams(content.toLowerCase(), this.ngramSize);
+
+    if (ngrams.length === 0) {
+      return new Array(this.numPermutations).fill(0);
+    }
+
+    const signature = new Array(this.numPermutations).fill(Number.MAX_SAFE_INTEGER);
+
+    for (const ngram of ngrams) {
+      for (let i = 0; i < this.numPermutations; i++) {
+        const hashValue = this.hashNgramWithSeed(ngram, i);
+        signature[i] = Math.min(signature[i], hashValue);
+      }
+    }
+
+    return signature;
+  }
+
+  jaccardSimilarity(sig1: number[], sig2: number[]): number {
+    if (sig1.length !== sig2.length) {
+      return 0;
+    }
+
+    if (sig1.length === 0) {
+      return 1;
+    }
+
+    let matches = 0;
+    for (let i = 0; i < sig1.length; i++) {
+      if (sig1[i] === sig2[i]) {
+        matches++;
+      }
+    }
+
+    return matches / sig1.length;
+  }
+
+  findCandidates(
+    targetSig: number[],
+    allSigs: Map<string, number[]>,
+    threshold: number
+  ): string[] {
+    const candidates: string[] = [];
+
+    for (const [memoryId, sig] of allSigs.entries()) {
+      const similarity = this.jaccardSimilarity(targetSig, sig);
+      if (similarity >= threshold) {
+        candidates.push(memoryId);
+      }
+    }
+
+    return candidates;
+  }
+
+  private generateNgrams(content: string, size: number): string[] {
+    const ngrams: string[] = [];
+    const padded = ' '.repeat(size - 1) + content + ' '.repeat(size - 1);
+
+    for (let i = 0; i <= padded.length - size; i++) {
+      ngrams.push(padded.substring(i, i + size));
+    }
+
+    return ngrams;
+  }
+
+  private hashNgramWithSeed(ngram: string, seed: number): number {
+    let hash = seed;
+
+    for (let i = 0; i < ngram.length; i++) {
+      hash = (hash << 5) - hash + ngram.charCodeAt(i);
+      hash = hash & hash;
+    }
+
+    return Math.abs(hash);
+  }
+}
+
+export interface Stage1CandidatePair {
+  memoryId1: string;
+  memoryId2: string;
+  simhashDistance: number;
+  minhashSimilarity: number;
+  matched: 'simhash' | 'minhash' | 'both';
+}
+
+/**
+ * Combine SimHash and MinHash filters using union approach.
+ * Candidates pass if they match EITHER filter, casting wider net for stage 2 ranking.
+ */
+export function findCandidatePairs(
+  memories: Map<string, string>,
+  allSimhashes: Map<string, string>,
+  allMinhashes: Map<string, number[]>,
+  options: {
+    simhashThreshold?: number;
+    minhashThreshold?: number;
+  }
+): Stage1CandidatePair[] {
+  const simhashThreshold = options.simhashThreshold ?? 4;
+  const minhashThreshold = options.minhashThreshold ?? 0.7;
+
+  const simhashFilter = new SimHashFilter();
+  const minhashFilter = new MinHashFilter();
+
+  const candidatePairs: Stage1CandidatePair[] = [];
+  const seen = new Set<string>();
+
+  const memoryIds = Array.from(memories.keys());
+
+  for (let i = 0; i < memoryIds.length; i++) {
+    const id1 = memoryIds[i];
+    const simhash1 = allSimhashes.get(id1);
+    const minhash1 = allMinhashes.get(id1);
+
+    if (!simhash1 || !minhash1) continue;
+
+    for (let j = i + 1; j < memoryIds.length; j++) {
+      const id2 = memoryIds[j];
+      const simhash2 = allSimhashes.get(id2);
+      const minhash2 = allMinhashes.get(id2);
+
+      if (!simhash2 || !minhash2) continue;
+
+      const pairKey = id1 < id2 ? `${id1}:${id2}` : `${id2}:${id1}`;
+      if (seen.has(pairKey)) continue;
+
+      const simhashDist = simhashFilter.hammingDistance(simhash1, simhash2);
+      const simhashMatch = simhashDist <= simhashThreshold;
+
+      const minhashSim = minhashFilter.jaccardSimilarity(minhash1, minhash2);
+      const minhashMatch = minhashSim >= minhashThreshold;
+
+      if (simhashMatch || minhashMatch) {
+        seen.add(pairKey);
+        candidatePairs.push({
+          memoryId1: id1,
+          memoryId2: id2,
+          simhashDistance: simhashDist,
+          minhashSimilarity: minhashSim,
+          matched: simhashMatch && minhashMatch ? 'both' : simhashMatch ? 'simhash' : 'minhash',
+        });
+      }
+    }
+  }
+
+  return candidatePairs;
+}

package/core/algorithms/detection/semantic-ranker.ts

@@ -0,0 +1,194 @@
+/**
+ * Semantic ranking using embeddings (Stage 2 of two-stage detection).
+ * Ranks candidates from Stage 1 by semantic similarity using cosine distance.
+ */
+
+import type { Memory } from '../../../db/drizzle/schema.js';
+import { cosineSimilarity } from '../../../core/utils/vector-operations.js';
+
+export interface RankedCandidate {
+  memoryId1: string;
+  memoryId2: string;
+  memory1: Memory;
+  memory2: Memory;
+  cosineSimilarity: number;
+  confidenceLevel: 'high' | 'medium' | 'low';
+  mergeReason: string;
+}
+
+function calculateConfidence(
+  memory1: Memory,
+  memory2: Memory,
+  cosineSim: number
+): 'high' | 'medium' | 'low' {
+  if (cosineSim >= 0.90) {
+    const factors = calculateConfidenceFactors(memory1, memory2);
+    if (factors.sameType && factors.tagOverlap > 0.5) {
+      return 'high';
+    }
+    if (factors.sameType || factors.tagOverlap > 0.3) {
+      return 'high';
+    }
+    return 'medium';
+  }
+
+  if (cosineSim >= 0.80) {
+    const factors = calculateConfidenceFactors(memory1, memory2);
+    if (factors.sameType && factors.contentLengthSimilarity > 0.7) {
+      return 'medium';
+    }
+    return 'medium';
+  }
+
+  return 'low';
+}
+
+interface ConfidenceFactors {
+  sameType: boolean;
+  tagOverlap: number;
+  contentLengthSimilarity: number;
+}
+
+function calculateConfidenceFactors(memory1: Memory, memory2: Memory): ConfidenceFactors {
+  const sameType = memory1.type === memory2.type;
+
+  const tags1 = new Set(memory1.tags || []);
+  const tags2 = new Set(memory2.tags || []);
+  const overlap = Array.from(tags1).filter((tag) => tags2.has(tag)).length;
+  const union = new Set([...tags1, ...tags2]).size;
+  const tagOverlap = union === 0 ? 0 : overlap / union;
+
+  const len1 = memory1.content.length;
+  const len2 = memory2.content.length;
+  const maxLen = Math.max(len1, len2);
+  const minLen = Math.min(len1, len2);
+  const contentLengthSimilarity = maxLen === 0 ? 1 : minLen / maxLen;
+
+  return {
+    sameType,
+    tagOverlap,
+    contentLengthSimilarity,
+  };
+}
+
+function generateMergeReason(
+  memory1: Memory,
+  memory2: Memory,
+  cosineSim: number,
+  confidenceLevel: 'high' | 'medium' | 'low'
+): string {
+  const factors = calculateConfidenceFactors(memory1, memory2);
+  const parts: string[] = [];
+
+  parts.push(`Semantic similarity: ${(cosineSim * 100).toFixed(1)}%`);
+
+  if (factors.sameType) {
+    parts.push(`Same type (${memory1.type})`);
+  } else {
+    parts.push(`Different types (${memory1.type} vs ${memory2.type})`);
+  }
+
+  if (factors.tagOverlap > 0) {
+    parts.push(`${(factors.tagOverlap * 100).toFixed(0)}% tag overlap`);
+  }
+
+  const len1 = memory1.content.length;
+  const len2 = memory2.content.length;
+  const diffPercent = Math.abs(len1 - len2) / Math.max(len1, len2);
+  parts.push(`Content length difference: ${(diffPercent * 100).toFixed(0)}%`);
+
+  parts.push(`Confidence: ${confidenceLevel}`);
+
+  return parts.join(' • ');
+}
+
+export async function rankCandidates(
+  candidates: Array<{ memoryId1: string; memoryId2: string }>,
+  memories: Map<string, Memory>,
+  embeddings: Map<string, number[]>,
+  options: {
+    semanticThreshold?: number;
+    topK?: number;
+  }
+): Promise<RankedCandidate[]> {
+  const semanticThreshold = options.semanticThreshold ?? 0.85;
+  const topK = options.topK ?? 10;
+
+  const rankedCandidates: RankedCandidate[] = [];
+
+  for (const { memoryId1, memoryId2 } of candidates) {
+    const memory1 = memories.get(memoryId1);
+    const memory2 = memories.get(memoryId2);
+    const embedding1 = embeddings.get(memoryId1);
+    const embedding2 = embeddings.get(memoryId2);
+
+    if (!memory1 || !memory2 || !embedding1 || !embedding2) {
+      continue;
+    }
+
+    const similarity = cosineSimilarity(embedding1, embedding2);
+
+    if (similarity < semanticThreshold) {
+      continue;
+    }
+
+    const confidence = calculateConfidence(memory1, memory2, similarity);
+    const mergeReason = generateMergeReason(memory1, memory2, similarity, confidence);
+
+    rankedCandidates.push({
+      memoryId1,
+      memoryId2,
+      memory1,
+      memory2,
+      cosineSimilarity: similarity,
+      confidenceLevel: confidence,
+      mergeReason,
+    });
+  }
+
+  rankedCandidates.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity);
+
+  const selectedByMemory = new Map<string, number>();
+  const filtered: RankedCandidate[] = [];
+
+  for (const candidate of rankedCandidates) {
+    const count1 = (selectedByMemory.get(candidate.memoryId1) || 0) + 1;
+    const count2 = (selectedByMemory.get(candidate.memoryId2) || 0) + 1;
+
+    if (count1 <= topK && count2 <= topK) {
+      filtered.push(candidate);
+      selectedByMemory.set(candidate.memoryId1, count1);
+      selectedByMemory.set(candidate.memoryId2, count2);
+    }
+  }
+
+  return filtered;
+}
+
+export function analyzePair(
+  memory1: Memory,
+  memory2: Memory,
+  embedding1: number[],
+  embedding2: number[]
+): {
+  cosineSimilarity: number;
+  confidenceLevel: 'high' | 'medium' | 'low';
+  mergeReason: string;
+  factors: {
+    sameType: boolean;
+    tagOverlap: number;
+    contentLengthSimilarity: number;
+  };
+} {
+  const similarity = cosineSimilarity(embedding1, embedding2);
+  const confidence = calculateConfidence(memory1, memory2, similarity);
+  const mergeReason = generateMergeReason(memory1, memory2, similarity, confidence);
+  const factors = calculateConfidenceFactors(memory1, memory2);
+
+  return {
+    cosineSimilarity: similarity,
+    confidenceLevel: confidence,
+    mergeReason,
+    factors,
+  };
+}