squish-memory 1.0.2 → 1.2.0

package/core/utils/vector-operations.ts

@@ -0,0 +1,135 @@
+/**
+ * Vector Operations Utilities
+ *
+ * Provides mathematical operations for vector computations, primarily focused on
+ * cosine similarity calculations for embedding comparisons.
+ *
+ * All functions handle edge cases gracefully and are optimized for performance.
+ */
+
+/**
+ * Calculates the cosine similarity between two vectors.
+ *
+ * Cosine similarity measures the cosine of the angle between two vectors,
+ * providing a normalized measure of similarity that ranges from -1 (opposite)
+ * to 1 (identical), with 0 indicating orthogonality (no similarity).
+ *
+ * The implementation uses the standard formula:
+ *   cos(θ) = (A · B) / (||A|| * ||B||)
+ *
+ * Edge case handling:
+ * - Returns 0 if either vector is null/undefined
+ * - Returns 0 if vectors have different lengths
+ * - Returns 0 if either vector has zero magnitude (norm = 0)
+ *
+ * @param a - First vector as array of numbers
+ * @param b - Second vector as array of numbers
+ * @returns Cosine similarity value in range [-1, 1], or 0 for invalid inputs
+ *
+ * @example
+ * ```typescript
+ * const vec1 = [1, 2, 3];
+ * const vec2 = [4, 5, 6];
+ * const similarity = cosineSimilarity(vec1, vec2); // ~0.974
+ * ```
+ *
+ * @performance
+ * - Time complexity: O(n) where n is vector length
+ * - Space complexity: O(1) - uses only accumulator variables
+ * - Optimized with single-pass computation of dot product and norms
+ */
+export function cosineSimilarity(a: number[] | null | undefined, b: number[] | null | undefined): number {
+  // Guard against null/undefined inputs
+  if (!a || !b) return 0;
+
+  // Vectors must have same dimensions
+  if (a.length !== b.length) return 0;
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  // Single pass to compute dot product and norms
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  // Handle zero vectors (avoid division by zero)
+  if (normA === 0 || normB === 0) return 0;
+
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * Calculates the Euclidean distance between two vectors.
+ *
+ * @param a - First vector
+ * @param b - Second vector
+ * @returns Euclidean distance, or Infinity if vectors have different lengths
+ */
+export function euclideanDistance(a: number[] | null | undefined, b: number[] | null | undefined): number {
+  if (!a || !b) return Infinity;
+  if (a.length !== b.length) return Infinity;
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i];
+    sum += diff * diff;
+  }
+  return Math.sqrt(sum);
+}
+
+/**
+ * Normalizes a vector to unit length (magnitude = 1).
+ *
+ * @param vec - Input vector
+ * @returns Normalized vector, or null if input is null or zero vector
+ */
+export function normalizeVector(vec: number[] | null | undefined): number[] | null {
+  if (!vec) return null;
+
+  let norm = 0;
+  for (let i = 0; i < vec.length; i++) {
+    norm += vec[i] * vec[i];
+  }
+
+  if (norm === 0) return null;
+
+  const magnitude = Math.sqrt(norm);
+  return vec.map(val => val / magnitude);
+}
+
+/**
+ * Computes the dot product of two vectors.
+ *
+ * @param a - First vector
+ * @param b - Second vector
+ * @returns Dot product, or 0 if vectors are invalid or different lengths
+ */
+export function dotProduct(a: number[] | null | undefined, b: number[] | null | undefined): number {
+  if (!a || !b || a.length !== b.length) return 0;
+
+  let result = 0;
+  for (let i = 0; i < a.length; i++) {
+    result += a[i] * b[i];
+  }
+  return result;
+}
+
+/**
+ * Calculates the magnitude (L2 norm) of a vector.
+ *
+ * @param vec - Input vector
+ * @returns Magnitude of the vector, or 0 if input is null
+ */
+export function magnitude(vec: number[] | null | undefined): number {
+  if (!vec) return 0;
+
+  let sum = 0;
+  for (let i = 0; i < vec.length; i++) {
+    sum += vec[i] * vec[i];
+  }
+  return Math.sqrt(sum);
+}

package/core/utils/version-management.ts

@@ -0,0 +1,74 @@
+/**
+ * Shared Version Management Utilities
+ * Common patterns for versioning and updating records
+ */
+
+import { eq } from 'drizzle-orm';
+import { randomUUID } from 'crypto';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+
+/**
+ * Create a new version of a fact by expiring the old one
+ */
+export async function createNewFactVersion(
+  oldFactId: string,
+  newContent: string,
+  additionalFields: Record<string, any> = {},
+  reason?: string
+): Promise<string> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Get the old fact
+    const oldFacts = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, oldFactId))
+      .limit(1);
+
+    if (oldFacts.length === 0) {
+      throw new Error('Previous fact not found');
+    }
+
+    const oldFact = oldFacts[0];
+    const now = new Date();
+    const newFactId = randomUUID();
+
+    // Expire the old fact
+    await (db as any)
+      .update(schema.memories)
+      .set({ validTo: now, supersededBy: newFactId })
+      .where(eq(schema.memories.id, oldFactId));
+
+    // Create new version
+    const newFact: any = {
+      id: newFactId,
+      type: 'fact',
+      sector: 'semantic',
+      content: newContent,
+      confidence: oldFact.confidence,
+      validFrom: now,
+      validTo: null,
+      version: (oldFact.version || 1) + 1,
+      tags: oldFact.tags,
+      projectId: oldFact.projectId,
+      createdAt: now,
+      updatedAt: now,
+      ...additionalFields,
+    };
+
+    if (reason) {
+      newFact.metadata = { supersedes: oldFactId, reason };
+    }
+
+    await (db as any).insert(schema.memories).values(newFact);
+
+    return newFactId;
+  } catch (error) {
+    logger.error('Error creating new fact version', error);
+    throw error;
+  }
+}

package/core/worker.ts

@@ -0,0 +1,324 @@
+/**
+ * Background Worker
+ * Handles lifecycle maintenance, summarization, and other async tasks
+ */
+
+import { config } from '../config.js';
+import { runLifecycleMaintenance } from './lifecycle.js';
+import { pruneWeakAssociations, getAssociationStats } from './associations.js';
+import { pruneOldSummaries, getSummarizationStats } from './summarization.js';
+import { runFullConsolidationJob, getDeduplicationStats } from './consolidation.js';
+import { logger } from './logger.js';
+
+interface WorkerConfig {
+  lifecycleInterval: number;
+  pruningInterval: number;
+  summarizationCheckInterval: number;
+  consolidationInterval: number;
+  dedupInterval: number;
+  associationPruningThreshold: number;
+  summaryPruningAge: number;
+}
+
+const DEFAULT_WORKER_CONFIG: WorkerConfig = {
+  lifecycleInterval: config.lifecycleInterval || 3600000,
+  pruningInterval: 7 * 24 * 60 * 60 * 1000,
+  summarizationCheckInterval: 5 * 60 * 1000,
+  consolidationInterval: 24 * 60 * 60 * 1000, // Daily
+  dedupInterval: 12 * 60 * 60 * 1000, // Twice daily
+  associationPruningThreshold: 5,
+  summaryPruningAge: 30,
+};
+
+class SquishWorker {
+  private lifecycleTimer?: NodeJS.Timeout;
+  private pruningTimer?: NodeJS.Timeout;
+  private summarizationTimer?: NodeJS.Timeout;
+  private consolidationTimer?: NodeJS.Timeout;
+  private dedupTimer?: NodeJS.Timeout;
+  private config: WorkerConfig;
+  private isRunning: boolean = false;
+  private stats = {
+    lifecycleRuns: 0,
+    pruningRuns: 0,
+    summarizationRuns: 0,
+    consolidationRuns: 0,
+    dedupRuns: 0,
+    lastLifecycle: null as any,
+    lastAssociation: null as any,
+    lastSummarization: null as any,
+    lastConsolidation: null as any,
+    lastDedup: null as any,
+  };
+
+  constructor(customConfig: Partial<WorkerConfig> = {}) {
+    this.config = { ...DEFAULT_WORKER_CONFIG, ...customConfig };
+  }
+
+  async start(): Promise<void> {
+    if (this.isRunning) {
+      logger.warn('Worker already running');
+      return;
+    }
+
+    this.isRunning = true;
+    logger.info('Starting background worker');
+
+    if (config.lifecycleEnabled) {
+      this.scheduleLifecycleMaintenance();
+    }
+
+    this.schedulePruning();
+
+    if (config.summarizationEnabled) {
+      this.scheduleSummarizationCheck();
+    }
+
+    // Schedule consolidation and dedup jobs
+    this.scheduleConsolidation();
+    this.scheduleDedup();
+
+    logger.info('Background worker started');
+  }
+
+  async stop(): Promise<void> {
+    if (!this.isRunning) {
+      return;
+    }
+
+    this.isRunning = false;
+
+    if (this.lifecycleTimer) clearInterval(this.lifecycleTimer);
+    if (this.pruningTimer) clearInterval(this.pruningTimer);
+    if (this.summarizationTimer) clearInterval(this.summarizationTimer);
+    if (this.consolidationTimer) clearInterval(this.consolidationTimer);
+    if (this.dedupTimer) clearInterval(this.dedupTimer);
+
+    logger.info('Background worker stopped');
+  }
+
+  private scheduleLifecycleMaintenance(): void {
+    this.runLifecycleMaintenance().catch((err) => {
+      logger.error('Initial lifecycle maintenance failed:', err);
+    });
+
+    this.lifecycleTimer = setInterval(() => {
+      this.runLifecycleMaintenance().catch((err) => {
+        logger.error('Scheduled lifecycle maintenance failed:', err);
+      });
+    }, this.config.lifecycleInterval);
+  }
+
+  private async runLifecycleMaintenance(): Promise<void> {
+    try {
+      const stats = await runLifecycleMaintenance();
+      this.stats.lifecycleRuns++;
+      this.stats.lastLifecycle = {
+        timestamp: new Date().toISOString(),
+        ...stats,
+      };
+
+      logger.info('Lifecycle maintenance completed', {
+        decayed: stats.decayed,
+        evicted: stats.evicted,
+        promoted: stats.promoted,
+      });
+    } catch (error) {
+      logger.error('Lifecycle maintenance error:', error);
+    }
+  }
+
+  private schedulePruning(): void {
+    this.runPruning().catch((err) => {
+      logger.error('Initial pruning failed:', err);
+    });
+
+    this.pruningTimer = setInterval(() => {
+      this.runPruning().catch((err) => {
+        logger.error('Scheduled pruning failed:', err);
+      });
+    }, this.config.pruningInterval);
+  }
+
+  private async runPruning(): Promise<void> {
+    try {
+      this.stats.pruningRuns++;
+
+      const prunedAssociations = await pruneWeakAssociations(
+        this.config.associationPruningThreshold
+      );
+
+      const assocStats = await getAssociationStats();
+      this.stats.lastAssociation = {
+        timestamp: new Date().toISOString(),
+        pruned: prunedAssociations,
+        ...assocStats,
+      };
+
+      const prunedSummaries = await pruneOldSummaries(this.config.summaryPruningAge);
+
+      logger.info('Pruning completed', {
+        associationsPruned: prunedAssociations,
+        summariesPruned: prunedSummaries,
+      });
+    } catch (error) {
+      logger.error('Pruning error:', error);
+    }
+  }
+
+  private scheduleSummarizationCheck(): void {
+    this.summarizationTimer = setInterval(() => {
+      this.runSummarizationCheck().catch((err) => {
+        logger.error('Summarization check failed:', err);
+      });
+    }, this.config.summarizationCheckInterval);
+  }
+
+  private async runSummarizationCheck(): Promise<void> {
+    try {
+      this.stats.summarizationRuns++;
+      const stats = await getSummarizationStats();
+      this.stats.lastSummarization = {
+        timestamp: new Date().toISOString(),
+        ...stats,
+      };
+    } catch (error) {
+      logger.error('Summarization check error:', error);
+    }
+  }
+
+  private scheduleConsolidation(): void {
+    // Run immediately on start
+    this.runConsolidation().catch((err) => {
+      logger.error('Initial consolidation failed:', err);
+    });
+
+    this.consolidationTimer = setInterval(() => {
+      this.runConsolidation().catch((err) => {
+        logger.error('Scheduled consolidation failed:', err);
+      });
+    }, this.config.consolidationInterval);
+  }
+
+  private async runConsolidation(): Promise<void> {
+    try {
+      this.stats.consolidationRuns++;
+      const stats = await runFullConsolidationJob();
+      this.stats.lastConsolidation = {
+        timestamp: new Date().toISOString(),
+        ...stats,
+      };
+
+      logger.info('Consolidation completed', {
+        clustered: stats.clustered,
+        merged: stats.merged,
+        deduped: stats.deduped,
+        consolidated: stats.consolidated,
+        tokensRecovered: stats.tokensRecovered,
+      });
+    } catch (error) {
+      logger.error('Consolidation error:', error);
+    }
+  }
+
+  private scheduleDedup(): void {
+    // Run immediately on start
+    this.runDedup().catch((err) => {
+      logger.error('Initial dedup failed:', err);
+    });
+
+    this.dedupTimer = setInterval(() => {
+      this.runDedup().catch((err) => {
+        logger.error('Scheduled dedup failed:', err);
+      });
+    }, this.config.dedupInterval);
+  }
+
+  private async runDedup(): Promise<void> {
+    try {
+      this.stats.dedupRuns++;
+      const dedupStats = await getDeduplicationStats();
+      this.stats.lastDedup = {
+        timestamp: new Date().toISOString(),
+        ...dedupStats,
+      };
+    } catch (error) {
+      logger.error('Dedup check error:', error);
+    }
+  }
+
+  getStats() {
+    return {
+      isRunning: this.isRunning,
+      stats: this.stats,
+      config: this.config,
+    };
+  }
+
+  async forceLifecycleMaintenance(projectId?: string): Promise<any> {
+    return await runLifecycleMaintenance(projectId);
+  }
+
+  async forcePruning(): Promise<any> {
+    return await this.runPruning();
+  }
+
+  async forceConsolidation(projectId?: string): Promise<any> {
+    return await runFullConsolidationJob(projectId);
+  }
+
+  async forceDedup(projectId?: string): Promise<any> {
+    const { runDeduplicationJob } = await import('./consolidation.js');
+    return await runDeduplicationJob(projectId);
+  }
+}
+
+let globalWorker: SquishWorker | null = null;
+
+export function getWorker(customConfig?: Partial<WorkerConfig>): SquishWorker {
+  if (!globalWorker) {
+    globalWorker = new SquishWorker(customConfig);
+  }
+  return globalWorker;
+}
+
+export async function startWorker(): Promise<void> {
+  const worker = getWorker();
+  await worker.start();
+}
+
+export async function stopWorker(): Promise<void> {
+  if (globalWorker) {
+    await globalWorker.stop();
+  }
+}
+
+export function getWorkerStats() {
+  if (!globalWorker) {
+    return null;
+  }
+  return globalWorker.getStats();
+}
+
+export async function forceLifecycleMaintenance(projectId?: string): Promise<any> {
+  const worker = getWorker();
+  return await worker.forceLifecycleMaintenance(projectId);
+}
+
+export async function forcePruning(): Promise<any> {
+  const worker = getWorker();
+  return await worker.forcePruning();
+}
+
+export async function forceConsolidation(projectId?: string): Promise<any> {
+  const worker = getWorker();
+  return await worker.forceConsolidation(projectId);
+}
+
+export async function forceDedup(projectId?: string): Promise<any> {
+  const worker = getWorker();
+  return await worker.forceDedup(projectId);
+}
+
+export type { WorkerConfig };
+export { SquishWorker };