squish-memory 1.0.2 → 1.2.0

package/dist/core/algorithms/operations/cache-maintenance.js

@@ -0,0 +1,157 @@
+/**
+ * Hash cache maintenance - SimHash/MinHash signatures for duplicate detection
+ */
+import { getDb } from '../../../db/index.js';
+import { getSchema } from '../../../db/schema.js';
+import { createDatabaseClient } from '../../../core/storage/database.js';
+import { eq } from 'drizzle-orm';
+import { SimHashFilter, MinHashFilter } from '../detection/hash-filters.js';
+import crypto from 'crypto';
+import { logger } from '../../../core/logger.js';
+function hashContent(content) {
+    return crypto.createHash('md5').update(content).digest('hex');
+}
+export async function updateCache(memoryId) {
+    try {
+        const db = createDatabaseClient(await getDb());
+        const schema = await getSchema();
+        // Load the memory
+        const [memory] = await db
+            .select()
+            .from(schema.memories)
+            .where(eq(schema.memories.id, memoryId));
+        if (!memory) {
+            return false;
+        }
+        // Generate hashes
+        const simhashFilter = new SimHashFilter();
+        const minhashFilter = new MinHashFilter();
+        const simhash = simhashFilter.generateHash(memory.content);
+        const minhash = minhashFilter.generateSignature(memory.content);
+        const contentHash = hashContent(memory.content);
+        // Upsert cache entry
+        const now = new Date();
+        // Check if entry exists
+        const [existing] = await db
+            .select()
+            .from(schema.memoryHashCache)
+            .where(eq(schema.memoryHashCache.memoryId, memoryId));
+        if (existing) {
+            // Update existing
+            await db
+                .update(schema.memoryHashCache)
+                .set({
+                simhash,
+                minhash: minhash,
+                contentHash,
+                lastUpdated: now,
+            })
+                .where(eq(schema.memoryHashCache.memoryId, memoryId));
+        }
+        else {
+            // Create new
+            await db.insert(schema.memoryHashCache).values({
+                memoryId,
+                projectId: memory.projectId,
+                simhash,
+                minhash: minhash,
+                contentHash,
+                lastUpdated: now,
+            });
+        }
+        return true;
+    }
+    catch (error) {
+        logger.error(`Failed to update hash cache for ${memoryId}`, error);
+        return false;
+    }
+}
+export async function rebuildCache(projectId) {
+    try {
+        const db = createDatabaseClient(await getDb());
+        const schema = await getSchema();
+        // Get all memories in project
+        const memories = await db
+            .select()
+            .from(schema.memories)
+            .where(eq(schema.memories.projectId, projectId));
+        let succeeded = 0;
+        let failed = 0;
+        for (const memory of memories) {
+            const ok = await updateCache(memory.id);
+            if (ok) {
+                succeeded++;
+            }
+            else {
+                failed++;
+            }
+        }
+        return {
+            processed: memories.length,
+            succeeded,
+            failed,
+        };
+    }
+    catch (error) {
+        logger.error(`Failed to rebuild hash cache for project ${projectId}`, error);
+        return { processed: 0, succeeded: 0, failed: 0 };
+    }
+}
+export async function isStale(memoryId) {
+    try {
+        const db = createDatabaseClient(await getDb());
+        const schema = await getSchema();
+        const [memory] = await db
+            .select()
+            .from(schema.memories)
+            .where(eq(schema.memories.id, memoryId));
+        if (!memory) {
+            return true;
+        }
+        const [cacheEntry] = await db
+            .select()
+            .from(schema.memoryHashCache)
+            .where(eq(schema.memoryHashCache.memoryId, memoryId));
+        if (!cacheEntry) {
+            return true; // No cache entry = stale
+        }
+        const currentContentHash = hashContent(memory.content);
+        return currentContentHash !== cacheEntry.contentHash;
+    }
+    catch (error) {
+        logger.error('Failed to check hash cache staleness', error);
+        return true; // Assume stale on error
+    }
+}
+export async function cleanupOrphaned(projectId) {
+    try {
+        const db = createDatabaseClient(await getDb());
+        const schema = await getSchema();
+        // Get all cache entries for project
+        const cacheEntries = await db
+            .select()
+            .from(schema.memoryHashCache)
+            .where(eq(schema.memoryHashCache.projectId, projectId));
+        let deleted = 0;
+        for (const entry of cacheEntries) {
+            // Check if memory exists
+            const [memory] = await db
+                .select()
+                .from(schema.memories)
+                .where(eq(schema.memories.id, entry.memoryId));
+            if (!memory) {
+                // Memory doesn't exist, delete cache entry
+                await db
+                    .delete(schema.memoryHashCache)
+                    .where(eq(schema.memoryHashCache.memoryId, entry.memoryId));
+                deleted++;
+            }
+        }
+        return deleted;
+    }
+    catch (error) {
+        logger.error('Failed to cleanup orphaned hash cache', error);
+        return 0;
+    }
+}
+//# sourceMappingURL=cache-maintenance.js.map

package/dist/{algorithms → core/algorithms}/safety/safety-checks.d.ts

@@ -2,12 +2,8 @@
  * Safety checks to prevent bad merges.
  * Checks run before creating merge proposals and are categorized as BLOCKER or WARNING.
  */
-import type { Memory } from '../../drizzle/schema.js';
-export interface SafetyCheckResult {
-    passed: boolean;
-    warnings: string[];
-    blockers: string[];
-}
+import type { Memory } from '../../../db/drizzle/schema.js';
+import type { SafetyCheckResult } from '../../lib/types.js';
 export interface SafetyCheck {
     name: string;
     description: string;

package/dist/{algorithms → core/algorithms}/strategies/merge-strategies.d.ts

@@ -2,7 +2,7 @@
  * Type-specific merge strategies for different memory types.
  * Each type has different merge semantics to preserve meaning and prevent data loss.
  */
-import type { Memory, MemoryType } from '../../drizzle/schema.js';
+import type { Memory, MemoryType } from '../../../db/drizzle/schema.js';
 export interface MergeStrategy {
     type: MemoryType;
     /**
@@ -26,6 +26,24 @@ export interface MergedMemory {
     mergeReason: string;
     conflictWarnings: string[];
 }
+/**
+ * Base implementation of MergeStrategy with common functionality.
+ * Reduces code duplication across specific merge strategies.
+ */
+export declare abstract class BaseMergeStrategy implements MergeStrategy {
+    abstract type: MemoryType;
+    canMerge(sources: Memory[]): {
+        ok: boolean;
+        reason?: string;
+    };
+    protected validateSources(sources: Memory[]): boolean;
+    protected handleEmptySources(): MergedMemory;
+    protected mergeTags(sources: Memory[]): string[];
+    protected buildBaseMetadata(sources: Memory[], extra?: Record<string, unknown>): Record<string, unknown>;
+    protected sortByDate(sources: Memory[], order?: 'asc' | 'desc'): Memory[];
+    protected sortChronologically(sources: Memory[]): Memory[];
+    abstract merge(sources: Memory[]): MergedMemory;
+}
 export declare const MERGE_STRATEGIES: Record<MemoryType, MergeStrategy>;
 export declare function getMergeStrategy(type: MemoryType): MergeStrategy;
 export declare function mergeMemories(sources: Memory[]): MergedMemory;

package/dist/{algorithms → core/algorithms}/strategies/merge-strategies.js

@@ -3,28 +3,62 @@
  * Each type has different merge semantics to preserve meaning and prevent data loss.
  */
 /**
- * FACT strategy: Union of information, remove exact duplicates.
- * Combines all unique facts into a unified statement with provenance tracking.
+ * Base implementation of MergeStrategy with common functionality.
+ * Reduces code duplication across specific merge strategies.
  */
-class FactMergeStrategy {
-    type = 'fact';
+export class BaseMergeStrategy {
     canMerge(sources) {
-        // Facts can almost always be merged
         if (sources.length < 2) {
             return { ok: false, reason: 'Need at least 2 memories to merge' };
         }
         return { ok: true };
     }
+    validateSources(sources) {
+        return sources.length >= 2;
+    }
+    handleEmptySources() {
+        return {
+            content: '',
+            summary: null,
+            tags: [],
+            metadata: {},
+            mergeReason: 'Empty source set',
+            conflictWarnings: [],
+        };
+    }
+    mergeTags(sources) {
+        const tagSet = new Set();
+        for (const source of sources) {
+            for (const tag of source.tags || []) {
+                tagSet.add(tag);
+            }
+        }
+        return Array.from(tagSet);
+    }
+    buildBaseMetadata(sources, extra) {
+        return {
+            mergedFrom: sources.map((m) => m.id),
+            mergeCount: sources.length,
+            ...extra,
+        };
+    }
+    sortByDate(sources, order = 'desc') {
+        const sorted = [...sources].sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+        return order === 'asc' ? [...sorted].reverse() : sorted;
+    }
+    sortChronologically(sources) {
+        return [...sources].sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
+    }
+}
+/**
+ * FACT strategy: Union of information, remove exact duplicates.
+ * Combines all unique facts into a unified statement with provenance tracking.
+ */
+class FactMergeStrategy extends BaseMergeStrategy {
+    type = 'fact';
     merge(sources) {
         if (sources.length === 0) {
-            return {
-                content: '',
-                summary: null,
-                tags: [],
-                metadata: {},
-                mergeReason: 'Empty source set',
-                conflictWarnings: [],
-            };
+            return this.handleEmptySources();
         }
         // Split content into sentences and deduplicate
         const sentenceSet = new Set();
@@ -45,27 +79,14 @@ class FactMergeStrategy {
         // Sort sentences for consistency
         const mergedSentences = Array.from(sentenceSet).sort();
         const content = mergedSentences.join('. ') + (mergedSentences.length > 0 ? '.' : '');
-        // Merge tags (union)
-        const tagSet = new Set();
-        for (const source of sources) {
-            for (const tag of source.tags || []) {
-                tagSet.add(tag);
-            }
-        }
-        // Create merged metadata with provenance
-        const metadata = {
-            mergedFrom: sources.map((m) => ({
-                id: m.id,
-                createdAt: m.createdAt,
-                source: m.source,
-            })),
-            mergeCount: sources.length,
+        // Build merged metadata with provenance
+        const metadata = this.buildBaseMetadata(sources, {
             timestamps: timestamps.sort(),
-        };
+        });
         return {
             content,
             summary: null,
-            tags: Array.from(tagSet),
+            tags: this.mergeTags(sources),
             metadata,
             mergeReason: `Merged ${sources.length} facts by combining all unique statements`,
             conflictWarnings: [],
@@ -76,27 +97,14 @@ class FactMergeStrategy {
  * PREFERENCE strategy: Keep latest by timestamp, track evolution history.
  * Warns if preferences conflict, indicating user changed their mind.
  */
-class PreferenceMergeStrategy {
+class PreferenceMergeStrategy extends BaseMergeStrategy {
     type = 'preference';
-    canMerge(sources) {
-        if (sources.length < 2) {
-            return { ok: false, reason: 'Need at least 2 memories to merge' };
-        }
-        return { ok: true };
-    }
     merge(sources) {
         if (sources.length === 0) {
-            return {
-                content: '',
-                summary: null,
-                tags: [],
-                metadata: {},
-                mergeReason: 'Empty source set',
-                conflictWarnings: [],
-            };
+            return this.handleEmptySources();
         }
         // Sort by creation date (newest first)
-        const sorted = [...sources].sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+        const sorted = this.sortByDate(sources);
         const latest = sorted[0];
         const warnings = [];
         // Check for conflicting preferences
@@ -112,12 +120,10 @@ class PreferenceMergeStrategy {
             confidence: m.confidence || 100,
             source: m.source,
         }));
-        const metadata = {
-            mergedFrom: sources.map((m) => m.id),
+        const metadata = this.buildBaseMetadata(sources, {
             preferenceHistory: history,
-            mergeCount: sources.length,
             latestAt: latest.createdAt,
-        };
+        });
         const summary = sources.length > 1
             ? `Prefers: ${latest.content} (evolved from ${sources.length} preferences)`
             : null;
@@ -135,27 +141,14 @@ class PreferenceMergeStrategy {
  * DECISION strategy: Keep latest decision, link to previous ones in timeline.
  * Warns if decisions contradict, preserving rationale history.
  */
-class DecisionMergeStrategy {
+class DecisionMergeStrategy extends BaseMergeStrategy {
     type = 'decision';
-    canMerge(sources) {
-        if (sources.length < 2) {
-            return { ok: false, reason: 'Need at least 2 memories to merge' };
-        }
-        return { ok: true };
-    }
     merge(sources) {
         if (sources.length === 0) {
-            return {
-                content: '',
-                summary: null,
-                tags: [],
-                metadata: {},
-                mergeReason: 'Empty source set',
-                conflictWarnings: [],
-            };
+            return this.handleEmptySources();
         }
         // Sort by creation date (newest first)
-        const sorted = [...sources].sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+        const sorted = this.sortByDate(sources);
         const latest = sorted[0];
         const warnings = [];
         // Check for conflicting decisions
@@ -170,13 +163,11 @@ class DecisionMergeStrategy {
             confidence: m.confidence || 100,
             rationale: m.summary || 'No rationale recorded',
         }));
-        const metadata = {
-            mergedFrom: sources.map((m) => m.id),
+        const metadata = this.buildBaseMetadata(sources, {
             decisionTimeline: timeline,
-            mergeCount: sources.length,
             currentDecisionAt: latest.createdAt,
             supersedes: sorted.slice(1).map((m) => m.id),
-        };
+        });
         const summary = sources.length > 1 ? `Decided: ${latest.content} (${sources.length} decisions)` : null;
         return {
             content: latest.content,
@@ -192,31 +183,17 @@ class DecisionMergeStrategy {
  * OBSERVATION strategy: Aggregate observations in chronological order.
  * Preserves temporal patterns and frequency information.
  */
-class ObservationMergeStrategy {
+class ObservationMergeStrategy extends BaseMergeStrategy {
     type = 'observation';
-    canMerge(sources) {
-        if (sources.length < 2) {
-            return { ok: false, reason: 'Need at least 2 memories to merge' };
-        }
-        return { ok: true };
-    }
     merge(sources) {
         if (sources.length === 0) {
-            return {
-                content: '',
-                summary: null,
-                tags: [],
-                metadata: {},
-                mergeReason: 'Empty source set',
-                conflictWarnings: [],
-            };
+            return this.handleEmptySources();
         }
         // Sort chronologically
-        const sorted = [...sources].sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
+        const sorted = this.sortChronologically(sources);
         // Create observation summary
         const observations = sorted.map((m) => `• ${m.content}`).join('\n');
-        const metadata = {
-            mergedFrom: sources.map((m) => m.id),
+        const metadata = this.buildBaseMetadata(sources, {
             observationCount: sources.length,
             timeSpan: {
                 start: sorted[0].createdAt,
@@ -227,18 +204,11 @@ class ObservationMergeStrategy {
                 content: m.content,
                 createdAt: m.createdAt,
             })),
-        };
-        // Merge tags
-        const tagSet = new Set();
-        for (const source of sources) {
-            for (const tag of source.tags || []) {
-                tagSet.add(tag);
-            }
-        }
+        });
         return {
             content: `Observations (${sources.length} total):\n${observations}`,
             summary: `${sources.length} observations over time period`,
-            tags: Array.from(tagSet),
+            tags: this.mergeTags(sources),
             metadata,
             mergeReason: `Merged ${sources.length} observations chronologically`,
             conflictWarnings: [],
@@ -249,24 +219,11 @@ class ObservationMergeStrategy {
  * CONTEXT strategy: Union of unique context, remove exact duplicates.
  * Preserves all distinct context items.
  */
-class ContextMergeStrategy {
+class ContextMergeStrategy extends BaseMergeStrategy {
     type = 'context';
-    canMerge(sources) {
-        if (sources.length < 2) {
-            return { ok: false, reason: 'Need at least 2 memories to merge' };
-        }
-        return { ok: true };
-    }
     merge(sources) {
         if (sources.length === 0) {
-            return {
-                content: '',
-                summary: null,
-                tags: [],
-                metadata: {},
-                mergeReason: 'Empty source set',
-                conflictWarnings: [],
-            };
+            return this.handleEmptySources();
         }
         // Deduplicate by content (exact matches)
         const uniqueContexts = new Map();
@@ -280,23 +237,15 @@ class ContextMergeStrategy {
             .sort()
             .map((content) => `• ${content}`)
             .join('\n');
-        const metadata = {
-            mergedFrom: sources.map((m) => m.id),
+        const metadata = this.buildBaseMetadata(sources, {
             uniqueContextCount: uniqueContexts.size,
             totalContextCount: sources.length,
             deduplicatedEntries: sources.length - uniqueContexts.size,
-        };
-        // Merge tags
-        const tagSet = new Set();
-        for (const source of sources) {
-            for (const tag of source.tags || []) {
-                tagSet.add(tag);
-            }
-        }
+        });
         return {
             content: contextList,
             summary: `${uniqueContexts.size} context items (${sources.length} total)`,
-            tags: Array.from(tagSet),
+            tags: this.mergeTags(sources),
             metadata,
             mergeReason: `Merged ${sources.length} context records into ${uniqueContexts.size} unique items`,
             conflictWarnings: [],
@@ -309,6 +258,8 @@ export const MERGE_STRATEGIES = {
     decision: new DecisionMergeStrategy(),
     observation: new ObservationMergeStrategy(),
     context: new ContextMergeStrategy(),
+    reflection: new ObservationMergeStrategy(),
+    note: new ObservationMergeStrategy(), // jot uses observation strategy
 };
 export function getMergeStrategy(type) {
     const strategy = MERGE_STRATEGIES[type];

package/dist/core/algorithms/types.d.ts

@@ -0,0 +1,125 @@
+/**
+ * TypeScript types and interfaces for memory merging
+ */
+import type { Memory, MemoryMergeProposal } from '../../db/drizzle/schema.js';
+/**
+ * Merge proposal status
+ */
+export type MergeProposalStatus = 'pending' | 'approved' | 'rejected' | 'expired';
+/**
+ * Confidence level for merge suggestions
+ */
+export type ConfidenceLevel = 'high' | 'medium' | 'low';
+/**
+ * Detection method used to find duplicates
+ */
+export type DetectionMethod = 'simhash' | 'minhash' | 'embedding';
+/**
+ * Merge strategy type per memory type
+ */
+export type MergeStrategyType = 'union' | 'latest' | 'voting' | 'custom';
+/**
+ * Memory pair candidate for merging
+ */
+export interface MemoryMergePair {
+    memory1: Memory;
+    memory2: Memory;
+    similarityScore: number;
+    detectionMethod: DetectionMethod;
+    confidenceLevel: ConfidenceLevel;
+    mergeReason: string;
+}
+/**
+ * Merge proposal with all details
+ */
+export interface MergeProposalDetails {
+    proposal: MemoryMergeProposal;
+    sourceMemories: Memory[];
+    previewMerged: {
+        content: string;
+        summary: string | null;
+        tags: string[];
+        metadata: Record<string, unknown>;
+    };
+    analysis: {
+        savedTokens: number;
+        savedPercentage: number;
+        conflictWarnings: string[];
+    };
+}
+/**
+ * Merge result after approval
+ */
+export interface MergeResult {
+    proposalId: string;
+    canonicalMemoryId: string;
+    mergedMemoryIds: string[];
+    tokensSaved: number;
+    mergedAt: Date;
+    mergeHistoryId?: string;
+}
+/**
+ * Merge reversal result
+ */
+export interface MergeReversal {
+    mergeHistoryId: string;
+    canonicalMemoryId: string;
+    restoredMemoryIds: string[];
+    reversedAt: Date;
+}
+/**
+ * Detection statistics
+ */
+export interface DetectionStatistics {
+    totalMemories: number;
+    scannedMemories: number;
+    candidatesFound: number;
+    candidatesRanked: number;
+    estimatedTokensSaved: number;
+    timeMs: {
+        stage1: number;
+        stage2: number;
+        total: number;
+    };
+}
+/**
+ * Project merge statistics
+ */
+export interface ProjectMergeStatistics {
+    projectId: string;
+    totalMemories: number;
+    mergeableMemories: number;
+    mergedMemories: number;
+    canonicalMemories: number;
+    pendingProposals: number;
+    approvedMerges: number;
+    rejectedProposals: number;
+    reversedMerges: number;
+    tokensSaved: {
+        total: number;
+        formatted: string;
+        percentage: number;
+    };
+    averageMergeSize: number;
+}
+/**
+ * Hash cache entry for fast duplicate detection
+ */
+export interface HashCacheEntry {
+    memoryId: string;
+    simhash: string;
+    minhash: number[];
+    contentHash: string;
+    lastUpdated: Date;
+}
+/**
+ * Stage 1 candidate pair with hash distances
+ */
+export interface Stage1CandidatePair {
+    memoryId1: string;
+    memoryId2: string;
+    simhashDistance: number;
+    minhashSimilarity: number;
+    matched: 'simhash' | 'minhash' | 'both';
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/algorithms/types.js

@@ -0,0 +1,5 @@
+/**
+ * TypeScript types and interfaces for memory merging
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/associations.d.ts

@@ -8,9 +8,10 @@ export type AssociationType = 'co_occurred' | 'supersedes' | 'contradicts' | 'su
  */
 export declare function createAssociation(fromMemoryId: string, toMemoryId: string, type: AssociationType, weight?: number): Promise<void>;
 /**
- * Track co-activation of multiple memories (they were used together)
- * OPTIMIZED: Uses bulk upsert instead of N² individual database operations
+ * Auto-link memories that share entities
+ * Called after storing a memory to link it to related memories
  */
+export declare function autoLinkByEntities(newMemoryId: string, entityNames: string[], projectId: string): Promise<number>;
 export declare function trackCoactivation(memoryIds: string[]): Promise<void>;
 /**
  * Get related memories via the association graph