squish-memory 1.0.2 → 1.2.0

package/core/lib/utils.ts

@@ -0,0 +1,151 @@
+/**
+ * Shared utility functions for the squish codebase
+ */
+
+import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
+import { config } from '../../config.js';
+import { toSqliteJson } from '../memory/serialization.js';
+
+export function normalizeTimestamp(value: any): string | null {
+  if (!value) return null;
+  if (value instanceof Date) return value.toISOString();
+  if (typeof value === 'number') {
+    try {
+      // Handle different timestamp formats using magnitude thresholds
+      // Microseconds: > 100000000000000 (e.g., 1700000000000000)
+      // Milliseconds: > 1000000000000 (e.g., 1700000000000)
+      // Seconds: <= 1000000000000 (e.g., 1700000000)
+      if (value > 100000000000000) {
+        return new Date(value / 1000).toISOString();
+      } else if (value > 1000000000000) {
+        return new Date(value).toISOString();
+      } else if (value >= 0) {
+        return new Date(value * 1000).toISOString();
+      }
+      return null;
+    } catch {
+      return null;
+    }
+  }
+  if (typeof value === 'string') {
+    try {
+      const parsed = new Date(value);
+      if (!isNaN(parsed.getTime())) return parsed.toISOString();
+      return null;
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+
+export function now(): string {
+  return new Date().toISOString();
+}
+
+export function isDatabaseUnavailableError(error: any): boolean {
+  const message = error?.message || '';
+  return [
+    'Database unavailable',
+    'not a valid Win32 application',
+    'invalid ELF header',
+    'bun:',
+    'sql.js wasm asset not found',
+    'SQLite database initialization failed',
+    'working local SQLite driver',
+  ].some((pattern) => message.includes(pattern));
+}
+
+export async function withDatabaseErrorHandling<T>(
+  operation: () => Promise<T>,
+  errorMessage: string
+): Promise<T> {
+  try {
+    return await operation();
+  } catch (dbError: any) {
+    if (isDatabaseUnavailableError(dbError)) {
+      throw new McpError(ErrorCode.InternalError, errorMessage);
+    }
+    throw dbError;
+  }
+}
+
+export function clampLimit(value: number | undefined, defaultValue: number, min: number = 1, max: number = 100): number {
+  return Math.min(Math.max(value ?? defaultValue, min), max);
+}
+
+export function prepareEmbedding(embedding: number[] | null): { embedding?: number[] | null; embeddingJson?: string | null } {
+  if (config.isTeamMode) {
+    return { embedding: embedding ?? null };
+  }
+  return { embeddingJson: toSqliteJson(embedding ?? null) };
+}
+
+export function determineOverallStatus(dbStatus: string, redisOk: boolean): string {
+  if ((dbStatus === 'ok' || dbStatus === 'unavailable') && redisOk) {
+    return 'ok';
+  }
+  if (dbStatus === 'unavailable') {
+    return 'degraded';
+  }
+  return 'error';
+}
+
+// Date parsing utilities - shared between CLI and MCP
+// ============================================================================
+
+export function parseDate(input: string): Date | null {
+  if (!input) return null;
+  const now = new Date();
+  const lower = input.toLowerCase().trim();
+  
+  // Direct date parse
+  const parsed = new Date(input);
+  if (!isNaN(parsed.getTime())) return parsed;
+  
+  // Relative parsing
+  const dayMatch = lower.match(/(\d+)\s*day/i);
+  const weekMatch = lower.match(/(\d+)\s*week/i);
+  const monthMatch = lower.match(/(\d+)\s*month/i);
+  
+  if (lower === 'today') {
+    const d = new Date(now);
+    d.setHours(0, 0, 0, 0);
+    return d;
+  }
+  if (lower === 'yesterday') return new Date(now.getTime() - 86400000);
+  if (lower === 'thisweek' || lower === 'this week') {
+    const d = new Date(now);
+    d.setDate(d.getDate() - d.getDay());
+    d.setHours(0, 0, 0, 0);
+    return d;
+  }
+  if (lower === 'lastweek' || lower === 'last week') {
+    const d = new Date(now);
+    d.setDate(d.getDate() - d.getDay() - 7);
+    return d;
+  }
+  
+  if (dayMatch) return new Date(now.getTime() - parseInt(dayMatch[1]) * 86400000);
+  if (weekMatch) return new Date(now.getTime() - parseInt(weekMatch[1]) * 604800000);
+  if (monthMatch) return new Date(now.getTime() - parseInt(monthMatch[1]) * 2592000000);
+  
+  return null;
+}
+
+export function filterByDateRange<T extends { createdAt?: string | null }>(
+  items: T[], 
+  since?: string, 
+  until?: string
+): T[] {
+  const sinceDate = parseDate(since || '');
+  const untilDate = parseDate(until || '');
+  
+  return items.filter(item => {
+    if (!item.createdAt) return true;
+    const created = new Date(item.createdAt);
+    if (sinceDate && created < sinceDate) return false;
+    if (untilDate && created > untilDate) return false;
+    return true;
+  });
+}

package/core/lib/validation.ts

@@ -0,0 +1,180 @@
+/**
+ * Input validation utilities for Squish
+ * Consolidates scattered validation patterns into a unified module
+ */
+
+import { clampLimit as originalClampLimit } from './utils.js';
+import { normalizeTags } from '../memory/serialization.js';
+
+/**
+ * Validate and normalize a limit value with bounds checking
+ */
+export function validateLimit(
+  value: number | string | undefined,
+  defaultValue: number = 20,
+  min: number = 1,
+  max: number = 100
+): number {
+  // Handle undefined, null, empty string
+  if (value === undefined || value === null || value === '') {
+    return defaultValue;
+  }
+
+  // Convert string to number, truncate decimals
+  let num: number;
+  if (typeof value === 'string') {
+    const parsed = parseInt(value, 10);
+    if (isNaN(parsed)) {
+      return NaN;
+    }
+    num = parsed;
+  } else {
+    // Truncate decimals for numbers
+    num = Math.trunc(value);
+  }
+
+  // Handle NaN
+  if (isNaN(num)) {
+    return NaN;
+  }
+
+  // Clamp to bounds
+  return Math.min(Math.max(num, min), max);
+}
+
+/**
+ * Parse an integer with bounds checking
+ */
+export function parseIntBounded(
+  value: number | string | undefined,
+  defaultValue: number,
+  min: number,
+  max: number
+): number {
+  return validateLimit(value, defaultValue, min, max);
+}
+
+/**
+ * Validate a project path
+ */
+export async function validateProjectPath(
+  path: string | undefined,
+  options?: { createIfMissing?: boolean; require?: boolean }
+): Promise<string> {
+  const { createIfMissing = false, require: requireProject = false } = options || {};
+
+  // If path is undefined, throw if required or return current directory
+  if (path === undefined || path === '') {
+    if (requireProject) {
+      throw new Error('Project path is required');
+    }
+    return process.cwd();
+  }
+
+  // Resolve to absolute path
+  const absolutePath = path.startsWith('/') || /^[a-zA-Z]:\\/.test(path)
+    ? path
+    : path.startsWith('~')
+    ? path.replace(/^~/, process.env.HOME || process.env.USERPROFILE || '')
+    : path.startsWith('.')
+    ? path
+    : path;
+
+  // Check if project exists in database
+  const { getProjectByPath } = await import('../projects.js');
+  const existingProject = await getProjectByPath(absolutePath);
+
+  if (existingProject) {
+    return absolutePath;
+  }
+
+  // Project doesn't exist
+  if (requireProject) {
+    throw new Error(`Project not found: ${absolutePath}`);
+  }
+
+  if (createIfMissing) {
+    const { ensureProject } = await import('../projects.js');
+    await ensureProject(absolutePath);
+    return absolutePath;
+  }
+
+  // Return the path even if it doesn't exist in database (for non-db validation)
+  return absolutePath;
+}
+
+/**
+ * Validate a UUID
+ */
+export function validateUuid(id: string): boolean {
+  if (typeof id !== 'string' || !id) {
+    return false;
+  }
+
+  // UUID v4/v5 regex pattern (8-4-4-4-12 hex digits)
+  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+  return uuidRegex.test(id);
+}
+
+/**
+ * Require a valid UUID, throws if invalid
+ */
+export function requireUuid(id: string): string {
+  if (!validateUuid(id)) {
+    throw new Error('Invalid UUID');
+  }
+  return id;
+}
+
+/**
+ * Validate a date value
+ *
+ * @param value - The date to validate (string, Date, number, or undefined)
+ * @returns Date object if valid, null otherwise
+ */
+export function validateDate(value: string | Date | number | undefined): Date | null {
+  if (!value) {
+    return null;
+  }
+
+  try {
+    let date: Date;
+
+    if (value instanceof Date) {
+      date = value;
+    } else if (typeof value === 'number') {
+      // Handle both milliseconds and seconds timestamps
+      if (value > 100000000000000) {
+        // Microseconds, convert to milliseconds
+        date = new Date(value / 1000);
+      } else if (value > 1000000000000) {
+        // Milliseconds
+        date = new Date(value);
+      } else if (value >= 0) {
+        // Seconds
+        date = new Date(value * 1000);
+      } else {
+        return null;
+      }
+    } else if (typeof value === 'string') {
+      date = new Date(value);
+    } else {
+      return null;
+    }
+
+    // Check if date is valid
+    if (isNaN(date.getTime())) {
+      return null;
+    }
+
+    return date;
+  } catch {
+    return null;
+  }
+}
+
+// Re-export normalizeTags for tag validation
+export { normalizeTags };
+
+// Re-export clampLimit for backward compatibility
+export { clampLimit } from './utils.js';

package/core/lifecycle.ts

@@ -0,0 +1,353 @@
+/**
+ * Memory Lifecycle Management
+ * Implements sector-based decay, tier classification, and eviction policies
+ *
+ * Decay Formula: newScore = importanceScore * (1 - decayRate/100)^days
+ * - decayRate: per-memory integer (e.g., 30 = 30% decay per decay cycle)
+ * - days: days since lastDecayAt
+ * - Tier demotion occurs when score drops below decayThreshold
+ * - Cold memories below threshold get status = 'expired'
+ */
+
+import { and, eq, lt, gte, desc, inArray, gt } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { getSchema } from '../db/schema.js';
+import { config } from '../config.js';
+import { logger } from './logger.js';
+import { triggerTierChange, triggerDecayApplied } from './memory/hooks.js';
+
+/**
+ * Default decay intervals by sector (days until decay check)
+ * These determine how often memories in each sector are evaluated for decay
+ * Can be configured via config.sectorDecayIntervals
+ */
+const SECTOR_DECAY_INTERVAL_DAYS: Record<string, number> = {
+  episodic: config.sectorDecayIntervals?.episodic || 30,
+  semantic: config.sectorDecayIntervals?.semantic || 90,
+  procedural: config.sectorDecayIntervals?.procedural || 180,
+  autobiographical: config.sectorDecayIntervals?.autobiographical || 365,
+  working: config.sectorDecayIntervals?.working || 7,
+};
+
+/**
+ * Default decay rates by memory type (percentage per decay cycle)
+ * Used as fallback when memory.decayRate is not set
+ */
+const DEFAULT_DECAY_RATES: Record<string, number> = {
+  observation: 5,
+  fact: 3,
+  decision: 2,
+  context: 4,
+  preference: 3,
+  note: 5,
+  reflection: 2,
+};
+
+const TIER_THRESHOLDS = {
+  hot: { recency: 7, coactivation: 10, salience: 70 },
+  cold: { recency: Infinity, coactivation: 0, salience: 0 },
+};
+
+export interface LifecycleStats {
+  decayed: number;
+  evicted: number;
+  promoted: number;
+  tierChanges: { hot: number; cold: number };
+  expired: number;
+}
+
+/**
+ * Run full lifecycle maintenance on all memories
+ */
+export async function runLifecycleMaintenance(projectId?: string): Promise<LifecycleStats> {
+  if (!config.lifecycleEnabled) {
+    return { decayed: 0, evicted: 0, promoted: 0, tierChanges: { hot: 0, cold: 0 }, expired: 0 };
+  }
+
+  const stats: LifecycleStats = {
+    decayed: 0,
+    evicted: 0,
+    promoted: 0,
+    tierChanges: { hot: 0, cold: 0 },
+    expired: 0,
+  };
+
+  try {
+    await applyDecay(projectId, stats);
+    await updateTiers(projectId, stats);
+    await evictOldMemories(projectId, stats);
+  } catch (error) {
+    logger.error('Lifecycle maintenance error', error);
+  }
+
+  return stats;
+}
+
+/**
+ * Apply decay to memories using per-memory decay rate formula
+ *
+ * Formula: newScore = importanceScore * (1 - decayRate/100)^days
+ * - Uses per-memory decayRate (stored as integer percentage, e.g., 30 = 30%)
+ * - Calculates days since lastDecayAt
+ * - Demotes tier when score drops below decayThreshold
+ * - Sets status = 'expired' for cold memories below threshold
+ */
+async function applyDecay(projectId: string | undefined, stats: LifecycleStats): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    const now = new Date();
+    const decayThreshold = config.decayThreshold || 0.1;
+
+    // Fetch memories that need decay processing (not protected, not expired)
+    let whereClause: any;
+    const conditions = [
+      eq(schema.memories.isProtected, false),
+      eq(schema.memories.status as any, 'active'),
+    ];
+
+    if (projectId) {
+      conditions.push(eq(schema.memories.projectId, projectId));
+    }
+
+    whereClause = and(...conditions);
+
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(whereClause)
+      .limit(10000);
+
+    let decayed = 0;
+    let expired = 0;
+    const expiredIds: string[] = [];
+
+    for (const memory of memories) {
+      // Get decay rate for this memory (fallback to type-based default)
+      const decayRate = memory.decayRate || DEFAULT_DECAY_RATES[memory.type] || 5;
+
+      // Calculate days since last decay
+      const lastDecayAt = memory.lastDecayAt ? new Date(memory.lastDecayAt) : new Date(memory.createdAt);
+      const daysSinceDecay = Math.max(0, (now.getTime() - lastDecayAt.getTime()) / (24 * 60 * 60 * 1000));
+
+      // Skip if not enough time has passed for this sector
+      const sectorInterval = SECTOR_DECAY_INTERVAL_DAYS[memory.sector] || 30;
+      if (daysSinceDecay < sectorInterval) {
+        continue;
+      }
+
+      // Apply decay formula: newScore = oldScore * (1 - decayRate/100)^days
+      const currentScore = memory.importanceScore || memory.relevanceScore || 50;
+      const decayMultiplier = Math.pow(1 - decayRate / 100, daysSinceDecay / sectorInterval);
+      const newScore = Math.max(0, Math.round(currentScore * decayMultiplier));
+
+      // Check if memory should expire (cold tier + below threshold)
+      const shouldExpire = memory.tier === 'cold' && newScore < (decayThreshold * 100);
+
+      if (shouldExpire) {
+        expiredIds.push(memory.id);
+        expired++;
+        logger.debug('Memory expiring', { id: memory.id, score: newScore, tier: memory.tier });
+      } else if (newScore !== currentScore) {
+        // Only update if score changed
+        await (db as any)
+          .update(schema.memories)
+          .set({
+            importanceScore: newScore,
+            relevanceScore: newScore,
+            lastDecayAt: now,
+            updatedAt: now,
+          })
+          .where(eq(schema.memories.id, memory.id));
+        decayed++;
+        
+        // Trigger decay applied hook
+        try {
+          await triggerDecayApplied({
+            memoryId: memory.id,
+            content: memory.content,
+            type: memory.type,
+            tags: typeof memory.tags === 'string' ? memory.tags.split(',') : [],
+            project: memory.projectId || undefined,
+            source: memory.source || undefined,
+            tier: memory.tier,
+            importance: newScore,
+            oldScore: currentScore,
+            newScore: newScore,
+          });
+        } catch (hookError) {
+          logger.error('Error triggering decayApplied hook', hookError);
+        }
+      }
+    }
+
+    // Batch expire memories
+    if (expiredIds.length > 0) {
+      await (db as any)
+        .update(schema.memories)
+        .set({
+          status: 'expired',
+          updatedAt: now,
+        })
+        .where(inArray(schema.memories.id, expiredIds));
+    }
+
+    stats.decayed += decayed;
+    stats.expired += expired;
+
+    logger.info('Decay applied', { decayed, expired, total: memories.length });
+  } catch (error) {
+    logger.error('Error applying decay', error);
+  }
+}
+
+/**
+ * Update memory tiers based on recency, coactivation, and salience
+ */
+async function updateTiers(projectId: string | undefined, stats: LifecycleStats): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const now = new Date();
+
+    // Fetch all memories to classify
+    const where = projectId ? eq(schema.memories.projectId, projectId) : undefined;
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(where)
+      .limit(10000); // Process larger batches now
+
+    // Calculate tiers in memory (simplified: hot or cold only)
+    const tierAssignments = new Map<string, 'hot' | 'cold'>();
+    const tierCounts = { hot: 0, cold: 0 };
+
+    for (const memory of memories) {
+      const recencyDays = (now.getTime() - new Date(memory.createdAt).getTime()) / (24 * 60 * 60 * 1000);
+      const coactivation = memory.coactivationScore || 0;
+      const salience = memory.relevanceScore || 50;
+
+      let newTier: 'hot' | 'cold' = 'cold';
+
+      if (
+        recencyDays <= TIER_THRESHOLDS.hot.recency &&
+        coactivation >= TIER_THRESHOLDS.hot.coactivation &&
+        salience >= TIER_THRESHOLDS.hot.salience
+      ) {
+        newTier = 'hot';
+      }
+
+      if (newTier !== memory.tier) {
+        tierAssignments.set(memory.id, newTier);
+        tierCounts[newTier]++;
+      }
+    }
+
+    if (tierAssignments.size === 0) return;
+
+    // Group by tier for efficient batched updates
+    const hotIds = Array.from(tierAssignments.entries())
+      .filter(([_, tier]) => tier === 'hot')
+      .map(([id]) => id);
+    const coldIds = Array.from(tierAssignments.entries())
+      .filter(([_, tier]) => tier === 'cold')
+      .map(([id]) => id);
+
+    // Execute batched updates instead of individual queries
+    if (hotIds.length > 0) {
+      await (db as any)
+        .update(schema.memories)
+        .set({ tier: 'hot', updatedAt: now })
+        .where(inArray(schema.memories.id, hotIds));
+    }
+
+    if (coldIds.length > 0) {
+      await (db as any)
+        .update(schema.memories)
+        .set({ tier: 'cold', updatedAt: now })
+        .where(inArray(schema.memories.id, coldIds));
+    }
+
+    if (coldIds.length > 0) {
+      await (db as any)
+        .update(schema.memories)
+        .set({ tier: 'cold', updatedAt: now })
+        .where(inArray(schema.memories.id, coldIds));
+    }
+
+    // Trigger tier change hooks for each memory that changed tier
+    if (tierAssignments.size > 0) {
+      for (const memory of memories) {
+        const newTier = tierAssignments.get(memory.id);
+        if (newTier && newTier !== memory.tier) {
+          try {
+            await triggerTierChange({
+              memoryId: memory.id,
+              content: memory.content,
+              type: memory.type,
+              tags: typeof memory.tags === 'string' ? memory.tags.split(',') : [],
+              project: memory.projectId || undefined,
+              source: memory.source || undefined,
+              tier: newTier,
+              importance: memory.importanceScore || memory.relevanceScore || 50,
+              oldTier: memory.tier,
+              newTier: newTier,
+            });
+          } catch (hookError) {
+            logger.error('Error triggering tierChange hook', hookError);
+          }
+        }
+      }
+    }
+
+    // Update stats
+    stats.tierChanges.hot = tierCounts.hot;
+    stats.tierChanges.cold = tierCounts.cold;
+
+    logger.debug('Tier updates complete', {
+      hot: tierCounts.hot,
+      cold: tierCounts.cold,
+    });
+  } catch (error) {
+    logger.error('Error updating tiers', error);
+  }
+}
+
+/**
+ * Evict old, cold memories with low relevance
+ */
+async function evictOldMemories(projectId: string | undefined, stats: LifecycleStats): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const evictionThreshold = new Date(Date.now() - 180 * 24 * 60 * 60 * 1000); // 180 days
+
+    const where = projectId
+      ? and(
+          eq(schema.memories.projectId, projectId),
+          lt(schema.memories.createdAt as any, evictionThreshold),
+          eq(schema.memories.isProtected, false),
+          eq(schema.memories.isPinned, false),
+          eq(schema.memories.tier as any, 'cold'),
+          lt(schema.memories.relevanceScore as any, 20) // Very low relevance
+        )
+      : and(
+          lt(schema.memories.createdAt as any, evictionThreshold),
+          eq(schema.memories.isProtected, false),
+          eq(schema.memories.isPinned, false),
+          eq(schema.memories.tier as any, 'cold'),
+          lt(schema.memories.relevanceScore as any, 20)
+        );
+
+    const result = await (db as any).delete(schema.memories).where(where);
+    stats.evicted = result?.rowCount || 0;
+  } catch (error) {
+    logger.error('Error evicting memories', error);
+  }
+}
+
+
+
+