squish-memory 1.1.5 → 1.2.1

package/core/memory/hooks.ts

@@ -0,0 +1,230 @@
+/**
+ * Memory Lifecycle Hooks
+ * 
+ * Provides event hooks for memory operations:
+  * - memoryCreated: When a memory is stored (DB or markdown files)
+ * - memoryUpdated: When a memory is updated
+ * - memoryDeleted: When a memory is deleted
+ * - tierChange: When memory tier changes (hot/warm/cold)
+ * - decayApplied: When decay score is updated
+ * 
+ * Each hook can have sync and async handlers.
+ * Usage: Register handlers to auto-capture, sync to external systems, etc.
+ */
+
+import { logger } from '../logger.js';
+
+export type HookEvent = 
+  | 'memoryCreated' 
+  | 'memoryUpdated' 
+  | 'memoryDeleted' 
+  | 'tierChange' 
+  | 'decayApplied';
+
+export interface MemoryHookContext {
+  memoryId: string;
+  content: string;
+  type: string;
+  tags: string[];
+  project?: string;
+  source?: string;
+  tier?: string;
+  importance?: number;
+  oldContent?: string;  // For memoryUpdated events
+}
+
+export interface TierChangeContext extends MemoryHookContext {
+  oldTier: string;
+  newTier: string;
+}
+
+export interface DecayContext extends MemoryHookContext {
+  oldScore: number;
+  newScore: number;
+}
+
+export type HookHandler<T = MemoryHookContext> = (context: T) => void | Promise<void>;
+
+interface HookRegistration {
+  event: HookEvent;
+  handler: HookHandler;
+  priority: number;
+}
+
+// Global hook registry
+const hooks: HookRegistration[] = [];
+
+/**
+ * Register a hook handler for a specific event
+ */
+export function registerHook(
+  event: HookEvent, 
+  handler: HookHandler,
+  priority: number = 100
+): void {
+  hooks.push({ event, handler, priority });
+  hooks.sort((a, b) => b.priority - a.priority);
+  logger.info('[Hooks] Registered handler for: ' + event + ' (priority: ' + priority + ')');
+}
+
+/**
+ * Unregister a hook handler
+ */
+export function unregisterHook(event: HookEvent, handler: HookHandler): void {
+  const idx = hooks.findIndex(h => h.event === event && h.handler === handler);
+  if (idx !== -1) {
+    hooks.splice(idx, 1);
+    logger.info('[Hooks] Unregistered handler for: ' + event);
+  }
+}
+
+/**
+ * Clear all hooks for an event
+ */
+export function clearHooks(event?: HookEvent): void {
+  if (event) {
+    const before = hooks.length;
+    const filtered = hooks.filter(h => h.event !== event);
+    hooks.length = 0;
+    hooks.push(...filtered);
+    logger.info('[Hooks] Cleared ' + (before - filtered.length) + ' hooks for: ' + event);
+  } else {
+    const count = hooks.length;
+    hooks.length = 0;
+    logger.info('[Hooks] Cleared all ' + count + ' hooks');
+  }
+}
+
+/**
+ * Get registered hooks for an event
+ */
+export function getHooks(event: HookEvent): HookHandler[] {
+  return hooks.filter(h => h.event === event).map(h => h.handler);
+}
+
+/**
+ * List all registered hooks (for debugging)
+ */
+export function listHooks(): { event: HookEvent; priority: number }[] {
+  return hooks.map(h => ({ event: h.event, priority: h.priority }));
+}
+
+// --- Trigger functions ---
+
+/**
+ * Trigger memoryCreated hooks
+ * Called when a memory is stored in DB or markdown files
+ */
+export async function triggerMemoryCreated(context: MemoryHookContext): Promise<void> {
+  const handlers = getHooks('memoryCreated');
+  
+  for (const handler of handlers) {
+    try {
+      const result = handler(context);
+      if (result instanceof Promise) {
+        await result;
+      }
+    } catch (error) {
+      logger.error('[Hooks] Error in memoryCreated handler: ' + error);
+    }
+  }
+}
+
+/**
+ * Trigger memoryUpdated hooks
+ * Called when a memory content/tags/etc changes
+ */
+export async function triggerMemoryUpdated(
+  context: MemoryHookContext, 
+  oldContent?: string
+): Promise<void> {
+  const handlers = getHooks('memoryUpdated');
+  
+  for (const handler of handlers) {
+    try {
+      const result = handler({ ...context, oldContent: oldContent || '' });
+      if (result instanceof Promise) {
+        await result;
+      }
+    } catch (error) {
+      logger.error('[Hooks] Error in memoryUpdated handler: ' + error);
+    }
+  }
+}
+
+/**
+ * Trigger memoryDeleted hooks
+ * Called when a memory is deleted
+ */
+export async function triggerMemoryDeleted(context: MemoryHookContext): Promise<void> {
+  const handlers = getHooks('memoryDeleted');
+  
+  for (const handler of handlers) {
+    try {
+      const result = handler(context);
+      if (result instanceof Promise) {
+        await result;
+      }
+    } catch (error) {
+      logger.error('[Hooks] Error in memoryDeleted handler: ' + error);
+    }
+  }
+}
+
+/**
+ * Trigger tierChange hooks
+ * Called when memory tier changes
+ */
+export async function triggerTierChange(context: TierChangeContext): Promise<void> {
+  const handlers = getHooks('tierChange');
+  
+  for (const handler of handlers) {
+    try {
+      const result = handler(context);
+      if (result instanceof Promise) {
+        await result;
+      }
+    } catch (error) {
+      logger.error('[Hooks] Error in tierChange handler: ' + error);
+    }
+  }
+}
+
+/**
+ * Trigger decayApplied hooks
+ * Called when memory decay score changes
+ */
+export async function triggerDecayApplied(context: DecayContext): Promise<void> {
+  const handlers = getHooks('decayApplied');
+  
+  for (const handler of handlers) {
+    try {
+      const result = handler(context);
+      if (result instanceof Promise) {
+        await result;
+      }
+    } catch (error) {
+      logger.error('[Hooks] Error in decayApplied handler: ' + error);
+    }
+  }
+}
+
+/**
+ * Built-in hook: Auto-save to memory files when storing to DB
+ * Use this to have dual storage (DB + markdown files)
+ */
+export function createMarkdownAutoSyncHook() {
+  return async function markdownAutoSync(context: MemoryHookContext) {
+    if (context.tier === 'hot') {
+      const { saveToMarkdown } = await import('./markdown/markdown-storage.js');
+      await saveToMarkdown({
+        content: context.content,
+        type: context.type as any,
+        tags: context.tags,
+        project: context.project,
+        source: context.source,
+      });
+      logger.info('[Hooks] Auto-synced memory to markdown: ' + context.memoryId);
+    }
+  };
+}

package/core/memory/hybrid-retrieval.ts

@@ -0,0 +1,65 @@
+import type { MemoryRecord } from '../lib/types.js';
+import { hybridSearch as currentHybridSearch } from './hybrid-search.js';
+import type { SearchInput } from './memories.js';
+
+export interface HybridSearchOptions extends SearchInput {
+  candidateLimit?: number;
+  resultLimit?: number;
+}
+
+export interface HybridSearchResult extends MemoryRecord {
+  hybridScore: number;
+  semanticScore: number;
+  recencyScore: number;
+  importanceScore: number;
+  coactivationScore: number;
+  rank: number;
+  explanation: string;
+}
+
+type ScoredItem = {
+  memoryId: string;
+  memory: Record<string, unknown>;
+  totalScore: number;
+  components: Record<string, number>;
+  rank: number;
+  explanation: string;
+};
+
+export function applyEntityBoostAndRerank<T extends ScoredItem>(scored: T[]): T[] {
+  return scored
+    .map((item) => {
+      const rawBoost = item.memory._entityBoost;
+      const entityBoost = typeof rawBoost === 'number' ? rawBoost : 0;
+      return {
+        ...item,
+        totalScore: item.totalScore + entityBoost * 20,
+      };
+    })
+    .sort((a, b) => b.totalScore - a.totalScore)
+    .map((item, index) => ({
+      ...item,
+      rank: index + 1,
+    }));
+}
+
+export async function hybridSearch(options: HybridSearchOptions): Promise<HybridSearchResult[]> {
+  const limit = options.resultLimit ?? options.limit ?? 5;
+  const results = await currentHybridSearch(options, {
+    limit,
+    project: options.project,
+    type: options.type,
+    tags: options.tags,
+  });
+
+  return results.map((result, index) => ({
+    ...result,
+    hybridScore: result.similarity ?? 0,
+    semanticScore: result.similarity ?? 0,
+    recencyScore: 0,
+    importanceScore: typeof result.importance === 'number' ? result.importance : 0,
+    coactivationScore: 0,
+    rank: index + 1,
+    explanation: 'Compatibility result from current hybrid search',
+  }));
+}

package/core/memory/hybrid-scorer.ts

@@ -0,0 +1,325 @@
+/** Hybrid Scorer - Multi-factor relevance scoring for memory ranking */
+
+import { logger } from '../../core/logger.js';
+import { cosineSimilarity } from '../utils/vector-operations.js';
+
+export interface ScoredMemory {
+  memoryId: string;
+  memory: any;
+  totalScore: number;
+  components: {
+    semantic: number;
+    recency: number;
+    coactivation: number;
+    importance: number;
+    confidence: number;
+    feedback: number;
+  };
+  rank: number;
+  explanation: string;
+}
+
+export interface HybridScorerOptions {
+  weights?: {
+    semantic?: number;
+    recency?: number;
+    coactivation?: number;
+    importance?: number;
+    confidence?: number;
+    feedback?: number;
+  };
+  decayDays?: number;
+  minSemanticScore?: number;
+  includeExplanation?: boolean;
+}
+
+export async function hybridScore(
+  queryEmbedding: number[] | null,
+  memories: any[],
+  options: HybridScorerOptions = {}
+): Promise<ScoredMemory[]> {
+  const weights = {
+    semantic: options.weights?.semantic ?? 0.30,
+    recency: options.weights?.recency ?? 0.20,
+    coactivation: options.weights?.coactivation ?? 0.10,
+    importance: options.weights?.importance ?? 0.15,
+    confidence: options.weights?.confidence ?? 0.15,
+    feedback: options.weights?.feedback ?? 0.10,
+  };
+
+  const decayDays = options.decayDays ?? 30;
+  const minSemanticScore = options.minSemanticScore ?? 0.0;
+
+  const scored: ScoredMemory[] = [];
+  const now = new Date();
+
+  for (const memory of memories) {
+    if (queryEmbedding && weights.semantic > 0) {
+      const semanticScore = calculateSemanticScore(queryEmbedding, memory);
+      if (semanticScore < minSemanticScore) continue;
+    }
+
+    const components = {
+      semantic: queryEmbedding ? calculateSemanticScore(queryEmbedding, memory) : 50,
+      recency: calculateRecencyScore(memory, now, decayDays),
+      coactivation: calculateCoactivationScore(memory),
+      importance: calculateImportanceScore(memory),
+      confidence: calculateConfidenceScore(memory),
+      feedback: calculateFeedbackScore(memory),
+    };
+
+    const totalScore = Math.min(
+      100,
+      components.semantic * weights.semantic +
+        components.recency * weights.recency +
+        components.coactivation * weights.coactivation +
+        components.importance * weights.importance +
+        components.confidence * weights.confidence +
+        components.feedback * weights.feedback
+    );
+
+    scored.push({
+      memoryId: memory.id,
+      memory,
+      totalScore: Math.round(totalScore * 100) / 100,
+      components: {
+        semantic: Math.round(components.semantic * 100) / 100,
+        recency: Math.round(components.recency * 100) / 100,
+        coactivation: Math.round(components.coactivation * 100) / 100,
+        importance: Math.round(components.importance * 100) / 100,
+        confidence: Math.round(components.confidence * 100) / 100,
+        feedback: Math.round(components.feedback * 100) / 100,
+      },
+      rank: 0,
+      explanation: options.includeExplanation
+        ? generateScoreExplanation(components, weights, memory)
+        : '',
+    });
+  }
+
+  scored.sort((a, b) => b.totalScore - a.totalScore);
+  for (let i = 0; i < scored.length; i++) {
+    scored[i].rank = i + 1;
+  }
+
+  return scored;
+}
+
+function calculateSemanticScore(queryEmbedding: number[], memory: any): number {
+  if (!memory.embedding || queryEmbedding.length === 0) return 50;
+
+  try {
+    let memoryEmbedding: number[] | null = null;
+
+    if (Array.isArray(memory.embedding)) {
+      memoryEmbedding = memory.embedding;
+    } else if (typeof memory.embedding === 'string') {
+      memoryEmbedding = JSON.parse(memory.embedding);
+    }
+
+    if (!memoryEmbedding || memoryEmbedding.length === 0) return 50;
+
+    const semanticScore = cosineSimilarity(queryEmbedding, memoryEmbedding);
+    return Math.max(0, Math.min(100, (semanticScore + 1) * 50));
+  } catch (error) {
+    logger.error('Error calculating semantic score', error);
+    return 50;
+  }
+}
+
+function calculateRecencyScore(memory: any, now: Date, decayDays: number): number {
+  // Enhanced bi-temporal recency scoring: considers validity period and learning time
+  const validFromDate = memory.validFrom ? new Date(memory.validFrom) : null;
+  const validToDate = memory.validTo ? new Date(memory.validTo) : null;
+  const recordedAtDate = memory.recordedAt ? new Date(memory.recordedAt) : null;
+  const createdDate = memory.createdAt ? new Date(memory.createdAt) : null;
+
+  // Calculate score based on how relevant the memory is right now
+  let score = 50; // Default neutral score
+
+  // If we have a validity period, score based on current time's position within that period
+  if (validFromDate && validToDate) {
+    const timeInPeriod = now.getTime() - validFromDate.getTime();
+    const periodLength = validToDate.getTime() - validFromDate.getTime();
+    
+    if (periodLength > 0) {
+      // If now is within the validity period, high score
+      if (now >= validFromDate && now <= validToDate) {
+        // Full score if right in middle, decreasing toward edges
+        const progress = timeInPeriod / periodLength;
+        const distanceFromCenter = Math.abs(0.5 - progress) * 2; // 0 at center, 1 at edges
+        score = 100 * (1 - distanceFromCenter * 0.8); // 100 at center, 40 at edges
+      } else {
+        // Outside validity period - score based on how recently it expired or how far in future
+        if (now < validFromDate) {
+          // Future validity - score based on how soon it becomes valid
+          const daysUntilValid = (validFromDate.getTime() - now.getTime()) / (24 * 60 * 60 * 1000);
+          score = Math.max(20, 100 - Math.min(80, daysUntilValid / decayDays * 100));
+        } else {
+          // Past validity - score based on how recently it expired
+          const daysSinceExpired = (now.getTime() - validToDate.getTime()) / (24 * 60 * 60 * 1000);
+          score = Math.max(10, 60 - Math.min(50, daysSinceExpired / decayDays * 100));
+        }
+      }
+    }
+  } 
+  // If we only have validFrom (open-ended validity)
+  else if (validFromDate) {
+    const daysSinceValid = (now.getTime() - validFromDate.getTime()) / (24 * 60 * 60 * 1000);
+    if (now >= validFromDate) {
+      // Still valid - decay from full score over time
+      score = Math.max(30, 100 - Math.min(70, daysSinceValid / decayDays * 100));
+    } else {
+      // Becomes valid in future
+      const daysUntilValid = (validFromDate.getTime() - now.getTime()) / (24 * 60 * 60 * 1000);
+      score = Math.max(20, 100 - Math.min(80, daysUntilValid / decayDays * 100));
+    }
+  }
+  // Fallback to learned/recorded time
+  else {
+    const learnedDate = recordedAtDate || createdDate;
+    if (learnedDate) {
+      const daysSinceLearned = (now.getTime() - learnedDate.getTime()) / (24 * 60 * 60 * 1000);
+      score = 100 * Math.pow(0.5, daysSinceLearned / decayDays);
+      score = Math.max(0, Math.min(100, score));
+    }
+  }
+
+  return Math.round(score * 10) / 10; // Return one decimal place
+}
+
+function calculateCoactivationScore(memory: any): number {
+  if (!memory.coactivationScore || memory.coactivationScore === 0) return 10;
+  return Math.min(100, memory.coactivationScore * 5);
+}
+
+function calculateImportanceScore(memory: any): number {
+  let score = 50;
+
+  // Simplified: hot or cold only (warm removed)
+  if (memory.tier === 'hot') score += 30;
+  // No bonus for cold - it's the fallback
+
+  if (memory.relevanceScore) score += memory.relevanceScore * 0.2;
+  if (memory.isPinned) score += 10;
+  if (memory.isProtected) score += 10;
+  if (memory.isMergeable || memory.isMerged) score -= 10;
+
+  return Math.max(0, Math.min(100, score));
+}
+
+function calculateConfidenceScore(memory: any): number {
+  let score = 50;
+
+  const signals = memory.metadata?.memorySignals;
+
+  if (signals) {
+    if (signals.priority === 'high') score += 25;
+
+    if (signals.explicitTriggers && signals.explicitTriggers.length > 0) {
+      score += 15 * Math.min(signals.explicitTriggers.length, 2);
+    }
+
+    if (signals.implicit) {
+      if (signals.implicit.decision) score += 10;
+      if (signals.implicit.correction) score += 15;
+      if (signals.implicit.preference) score += 8;
+      if (signals.implicit.workflowRule) score += 12;
+      if (signals.implicit.lesson) score += 10;
+    }
+
+    if (signals.requiresConflictCheck) score -= 5;
+  }
+
+  switch (memory.type) {
+    case 'decision': score += 10; break;
+    case 'preference': score += 5; break;
+    case 'fact': score += 8; break;
+    case 'context': score += 3; break;
+  }
+
+  if (memory.source === 'mcp') score += 5;
+
+  if (memory.accessCount && memory.accessCount > 5) {
+    score += Math.min(10, memory.accessCount / 2);
+  }
+
+  return Math.max(0, Math.min(100, score));
+}
+
+function calculateFeedbackScore(memory: any): number {
+  const retrievalPriority = memory.retrievalPriority ?? 50;
+  return Math.max(0, Math.min(100, retrievalPriority));
+}
+
+function generateScoreExplanation(
+  components: ScoredMemory['components'],
+  weights: ScoredMemory['components'],
+  memory: any
+): string {
+  const parts: string[] = [];
+
+  if (components.semantic > 70) parts.push(`highly relevant (${components.semantic.toFixed(0)})`);
+  else if (components.semantic > 50) parts.push(`somewhat relevant (${components.semantic.toFixed(0)})`);
+  else parts.push(`low relevance (${components.semantic.toFixed(0)})`);
+
+  if (components.recency > 70) parts.push('recent');
+  else if (components.recency > 30) parts.push('moderately recent');
+  else parts.push('older');
+
+  // Simplified: hot or cold only (warm removed)
+  if (memory.tier === 'hot') parts.push('active memory');
+  else parts.push('archived memory');
+
+  if (components.coactivation > 60) parts.push('frequently associated');
+
+  if (components.confidence > 80) parts.push('high confidence');
+  else if (components.confidence > 60) parts.push('moderate confidence');
+  else if (components.confidence < 40) parts.push('low confidence');
+
+  if (components.feedback > 70) parts.push('frequently useful');
+  else if (components.feedback < 30) parts.push('rarely used');
+
+  return parts.join(', ');
+}
+
+export async function scoreAndRankMemories(
+  queryEmbedding: number[] | null,
+  memories: any[],
+  topK: number = 10,
+  options: HybridScorerOptions = {}
+): Promise<ScoredMemory[]> {
+  const scored = await hybridScore(queryEmbedding, memories, options);
+
+  logger.debug('Scored and ranked memories', {
+    total: scored.length,
+    topK,
+    topScores: scored.slice(0, topK).map((s) => ({ id: s.memoryId, score: s.totalScore })),
+  });
+
+  return scored.slice(0, topK);
+}
+
+export function getScoreDistribution(scored: ScoredMemory[]): {
+  min: number;
+  max: number;
+  avg: number;
+  median: number;
+  p95: number;
+  p99: number;
+} {
+  if (scored.length === 0) {
+    return { min: 0, max: 0, avg: 0, median: 0, p95: 0, p99: 0 };
+  }
+
+  const scores = scored.map((s) => s.totalScore).sort((a, b) => a - b);
+
+  return {
+    min: scores[0],
+    max: scores[scores.length - 1],
+    avg: scores.reduce((a, b) => a + b, 0) / scores.length,
+    median: scores[Math.floor(scores.length / 2)],
+    p95: scores[Math.floor((95 / 100) * scores.length)],
+    p99: scores[Math.floor((99 / 100) * scores.length)],
+  };
+}