squish-memory 1.0.2 → 1.2.0

package/core/memory/response-analyzer.ts

@@ -0,0 +1,81 @@
+/** Response Analyzer - Analyze LLM responses for memory references (Echo/Fizzle tracking) */
+
+import { logger } from '../logger.js';
+
+export interface AnalysisResult {
+  referencedMemoryIds: string[];
+  referenceCount: number;
+  hasReferences: boolean;
+}
+
+const REFERENCE_PATTERNS = [
+  /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/gi,
+  /as (?:i |we )?(?:mentioned|noted|remembered|recalled|stored|saved)/gi,
+  /(?:based|drawing) on (?:my|our|the) (?:memory|previous|earlier)/gi,
+  /(?:i |we )?(?:recall|remember|noted) (?:that |earlier )?/gi,
+  /from (?:my|our|the) (?:memory|notes|records)/gi,
+];
+
+export function analyzeResponseForMemoryReferences(
+  responseText: string,
+  injectedMemoryIds: string[],
+  injectedMemoryContent: Map<string, string>
+): AnalysisResult {
+  const referencedMemoryIds: string[] = [];
+  const responseLower = responseText.toLowerCase();
+
+  for (const memoryId of injectedMemoryIds) {
+    if (responseText.includes(memoryId)) {
+      referencedMemoryIds.push(memoryId);
+      logger.debug(`[ResponseAnalyzer] Direct reference found: ${memoryId}`);
+    }
+  }
+
+  for (const [memoryId, content] of injectedMemoryContent) {
+    if (referencedMemoryIds.includes(memoryId)) continue;
+
+    const contentWords = content.toLowerCase().split(/\s+/).filter(w => w.length > 3);
+    if (contentWords.length < 5) continue;
+
+    let matchCount = 0;
+    for (const word of contentWords) {
+      if (responseLower.includes(word)) {
+        matchCount++;
+      }
+    }
+
+    const matchRatio = matchCount / contentWords.length;
+    if (matchRatio > 0.5) {
+      referencedMemoryIds.push(memoryId);
+      logger.debug(`[ResponseAnalyzer] Content match (${(matchRatio * 100).toFixed(0)}%): ${memoryId}`);
+    }
+  }
+
+  for (const pattern of REFERENCE_PATTERNS) {
+    const matches = responseText.match(pattern);
+    if (matches && matches.length > 0) {
+      logger.debug(`[ResponseAnalyzer] Reference pattern found: ${matches.length} matches`);
+    }
+  }
+
+  const result: AnalysisResult = {
+    referencedMemoryIds: [...new Set(referencedMemoryIds)],
+    referenceCount: referencedMemoryIds.length,
+    hasReferences: referencedMemoryIds.length > 0,
+  };
+
+  if (result.hasReferences) {
+    logger.info(`[ResponseAnalyzer] Found ${result.referenceCount} memory references`);
+  }
+
+  return result;
+}
+
+export function mightContainMemoryReferences(responseText: string): boolean {
+  const responseLower = responseText.toLowerCase();
+  const quickPatterns = [
+    'remember', 'recall', 'mentioned', 'noted', 'earlier',
+    'previous', 'as i', 'as we', 'from my', 'from our',
+  ];
+  return quickPatterns.some(pattern => responseLower.includes(pattern));
+}

package/core/memory/retrieval-feedback.ts

@@ -0,0 +1,276 @@
+/**
+ * Retrieval Feedback System
+ * 
+ * Tracks whether retrieved memories were useful in responses.
+ * Strengthens useful paths, prunes stale nodes, tunes edge weights.
+ */
+
+import { eq, and, sql } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export interface RetrievalFeedback {
+  memoryId: string;
+  query: string;
+  wasUseful: boolean;
+  cited: boolean; // Was the memory cited in the response?
+  responseId?: string;
+  timestamp: Date;
+}
+
+export interface FeedbackStats {
+  totalRetrievals: number;
+  usefulRetrievals: number;
+  citedRetrievals: number;
+  usefulnessRate: number;
+  citationRate: number;
+}
+
+// ─── In-Memory Feedback Buffer ───────────────────────────────────────────────
+
+const feedbackBuffer = new Map<string, RetrievalFeedback[]>();
+const MAX_BUFFER_SIZE = 1000;
+const FLUSH_INTERVAL_MS = 60000; // 1 minute
+
+let flushTimer: ReturnType<typeof setInterval> | null = null;
+
+/**
+ * Record that a memory was retrieved for a query.
+ * Call this when a memory appears in search results.
+ */
+export function recordRetrieval(
+  memoryId: string,
+  query: string,
+  options?: { sessionId?: string }
+): void {
+  const key = options?.sessionId || 'default';
+  const buffer = feedbackBuffer.get(key) || [];
+
+  buffer.push({
+    memoryId,
+    query,
+    wasUseful: false, // Will be updated later
+    cited: false,
+    timestamp: new Date(),
+  });
+
+  // Trim buffer if too large
+  if (buffer.length > MAX_BUFFER_SIZE) {
+    buffer.splice(0, buffer.length - MAX_BUFFER_SIZE);
+  }
+
+  feedbackBuffer.set(key, buffer);
+}
+
+/**
+ * Record that a retrieved memory was actually useful.
+ * Call this when a memory is cited or referenced in a response.
+ */
+export function recordUsefulRetrieval(
+  memoryId: string,
+  query: string,
+  options?: { 
+    cited?: boolean;
+    responseId?: string;
+    sessionId?: string;
+  }
+): void {
+  const key = options?.sessionId || 'default';
+  const buffer = feedbackBuffer.get(key) || [];
+
+  // Find the existing retrieval record and update it
+  const existing = buffer.find(f => f.memoryId === memoryId && f.query === query && !f.wasUseful);
+  if (existing) {
+    existing.wasUseful = true;
+    existing.cited = options?.cited ?? false;
+    existing.responseId = options?.responseId;
+  } else {
+    // Add a new record
+    buffer.push({
+      memoryId,
+      query,
+      wasUseful: true,
+      cited: options?.cited ?? false,
+      responseId: options?.responseId,
+      timestamp: new Date(),
+    });
+  }
+
+  feedbackBuffer.set(key, buffer);
+}
+
+/**
+ * Record that a memory was cited in a response.
+ * This is stronger feedback than just "useful" - it means the memory
+ * was explicitly referenced.
+ */
+export function recordCitation(
+  memoryId: string,
+  responseId: string,
+  options?: { sessionId?: string }
+): void {
+  const key = options?.sessionId || 'default';
+  const buffer = feedbackBuffer.get(key) || [];
+
+  // Mark all retrievals of this memory as useful
+  for (const feedback of buffer) {
+    if (feedback.memoryId === memoryId) {
+      feedback.wasUseful = true;
+      feedback.cited = true;
+      feedback.responseId = responseId;
+    }
+  }
+
+  feedbackBuffer.set(key, buffer);
+}
+
+/**
+ * Get feedback statistics for a specific memory.
+ */
+export function getRetrievalStats(memoryId: string): FeedbackStats {
+  let totalRetrievals = 0;
+  let usefulRetrievals = 0;
+  let citedRetrievals = 0;
+
+  for (const [, buffer] of feedbackBuffer) {
+    for (const feedback of buffer) {
+      if (feedback.memoryId === memoryId) {
+        totalRetrievals++;
+        if (feedback.wasUseful) usefulRetrievals++;
+        if (feedback.cited) citedRetrievals++;
+      }
+    }
+  }
+
+  return {
+    totalRetrievals,
+    usefulRetrievals,
+    citedRetrievals,
+    usefulnessRate: totalRetrievals > 0 ? usefulRetrievals / totalRetrievals : 0,
+    citationRate: totalRetrievals > 0 ? citedRetrievals / totalRetrievals : 0,
+  };
+}
+
+/**
+ * Get overall feedback statistics.
+ */
+export function getOverallFeedbackStats(): FeedbackStats {
+  let totalRetrievals = 0;
+  let usefulRetrievals = 0;
+  let citedRetrievals = 0;
+
+  for (const [, buffer] of feedbackBuffer) {
+    for (const feedback of buffer) {
+      totalRetrievals++;
+      if (feedback.wasUseful) usefulRetrievals++;
+      if (feedback.cited) citedRetrievals++;
+    }
+  }
+
+  return {
+    totalRetrievals,
+    usefulRetrievals,
+    citedRetrievals,
+    usefulnessRate: totalRetrievals > 0 ? usefulRetrievals / totalRetrievals : 0,
+    citationRate: totalRetrievals > 0 ? citedRetrievals / totalRetrievals : 0,
+  };
+}
+
+/**
+ * Flush feedback to the database, updating association weights.
+ */
+export async function flushFeedback(): Promise<{
+  strengthened: number;
+  weakened: number;
+  total: number;
+}> {
+  const db = await getDb();
+  const schema = await getSchema();
+
+  let strengthened = 0;
+  let weakened = 0;
+  let total = 0;
+
+  for (const [, buffer] of feedbackBuffer) {
+    for (const feedback of buffer) {
+      total++;
+
+      try {
+        // Get associations involving this memory
+        const associations = await (db as any)
+          .select()
+          .from(schema.memoryAssociations)
+          .where(
+            sql`${schema.memoryAssociations.fromMemoryId} = ${feedback.memoryId} OR ${schema.memoryAssociations.toMemoryId} = ${feedback.memoryId}`
+          );
+
+        for (const assoc of associations) {
+          if (feedback.wasUseful) {
+            // Strengthen: increase weight and coactivation count
+            await (db as any)
+              .update(schema.memoryAssociations)
+              .set({
+                weight: sql`${schema.memoryAssociations.weight} + 1`,
+                coactivationCount: sql`${schema.memoryAssociations.coactivationCount} + 1`,
+                lastCoactivatedAt: new Date(),
+              })
+              .where(eq(schema.memoryAssociations.id, assoc.id));
+            strengthened++;
+          } else {
+            // Weaken: decrease weight slightly (but don't go below 1)
+            await (db as any)
+              .update(schema.memoryAssociations)
+              .set({
+                weight: sql`GREATEST(${schema.memoryAssociations.weight} - 0.5, 1)`,
+              })
+              .where(eq(schema.memoryAssociations.id, assoc.id));
+            weakened++;
+          }
+        }
+      } catch (error) {
+        logger.debug('Error updating association weights', {
+          memoryId: feedback.memoryId,
+          error: error as Error,
+        });
+      }
+    }
+  }
+
+  // Clear the buffer after flushing
+  feedbackBuffer.clear();
+
+  logger.info('Feedback flushed', { strengthened, weakened, total });
+
+  return { strengthened, weakened, total };
+}
+
+/**
+ * Start the periodic feedback flush timer.
+ */
+export function startFeedbackFlushTimer(): void {
+  if (flushTimer) return;
+
+  flushTimer = setInterval(async () => {
+    try {
+      await flushFeedback();
+    } catch (error) {
+      logger.error('Error flushing feedback', { error: error as Error });
+    }
+  }, FLUSH_INTERVAL_MS);
+
+  logger.info('Feedback flush timer started', { intervalMs: FLUSH_INTERVAL_MS });
+}
+
+/**
+ * Stop the periodic feedback flush timer.
+ */
+export function stopFeedbackFlushTimer(): void {
+  if (flushTimer) {
+    clearInterval(flushTimer);
+    flushTimer = null;
+    logger.info('Feedback flush timer stopped');
+  }
+}

package/core/memory/serialization.ts

@@ -0,0 +1,83 @@
+import { config } from '../../config.js';
+
+export function normalizeTags(tags?: string[]): string[] {
+  return (tags || []).map((tag) => tag.trim()).filter((tag) => tag.length > 0);
+}
+
+export function toSqliteJson(value: unknown): string | null {
+  if (value === undefined || value === null) return null;
+  return JSON.stringify(value);
+}
+
+export function fromSqliteJson<T>(value: string | null | undefined): T | null {
+  if (!value) return null;
+  try {
+    return JSON.parse(value) as T;
+  } catch {
+    return null;
+  }
+}
+
+export function toSqliteTags(tags?: string[]): string | null {
+  if (!tags || tags.length === 0) return null;
+  return JSON.stringify(tags);
+}
+
+export function fromSqliteTags(value: string | null | undefined): string[] {
+  if (!value) return [];
+  try {
+    const parsed = JSON.parse(value);
+    return Array.isArray(parsed) ? parsed.map(String) : [];
+  } catch {
+    return value.split(',').map((tag) => tag.trim()).filter(Boolean);
+  }
+}
+
+// High-level helpers that abstract away team mode vs local mode
+
+export function serializeTags(tags?: string[]): string[] | string | null | undefined {
+  if (config.isTeamMode) {
+    // PostgreSQL: store as array directly (or null if empty)
+    return tags && tags.length > 0 ? tags : null;
+  } else {
+    // SQLite: store as JSON string (or null)
+    return toSqliteTags(tags);
+  }
+}
+
+export function deserializeTags(value: string[] | string | null | undefined): string[] {
+  if (config.isTeamMode) {
+    // PostgreSQL: value is already an array or null/undefined
+    return Array.isArray(value) ? value : [];
+  } else {
+    // SQLite: value is a JSON string or null/undefined
+    if (typeof value === 'string' || value === null || value === undefined) {
+      return fromSqliteTags(value);
+    }
+    // Unexpected type, return empty array
+    return [];
+  }
+}
+
+export function serializeMetadata(metadata: Record<string, unknown> | null | undefined): Record<string, unknown> | string | null {
+  if (config.isTeamMode) {
+    // PostgreSQL: store object directly (or null if null/undefined, keep empty object as-is)
+    return metadata === undefined || metadata === null ? null : metadata;
+  } else {
+    // SQLite: store as JSON string (or null)
+    return toSqliteJson(metadata);
+  }
+}
+
+export function deserializeMetadata(value: Record<string, unknown> | string | null | undefined): Record<string, unknown> | null {
+  if (config.isTeamMode) {
+    // PostgreSQL: value is already an object or null/undefined
+    return value == null ? null : (value as Record<string, unknown>);
+  } else {
+    // SQLite: value is a JSON string or null/undefined
+    if (typeof value === 'string' || value === null) {
+      return fromSqliteJson<Record<string, unknown>>(value);
+    }
+    return null;
+  }
+}

package/core/memory/stale-cleaner.ts

@@ -0,0 +1,147 @@
+/**
+ * Stale Memory Cleaner
+ * Deletes memories that are old, low-confidence, and low-importance
+ */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { lt, or, and, eq, lte } from 'drizzle-orm';
+
+export interface StaleMemory {
+  id: string;
+  content: string;
+  type: string;
+  createdAt: Date;
+  confidenceLevel: string | null;
+  importanceScore: number | null;
+  isPinned: boolean;
+}
+
+export interface StaleMemoryQuery {
+  olderThanDays: number;
+  confidenceLevels: string[];
+  minImportance: number;
+  projectId?: string;
+}
+
+export async function getStaleMemories(query: StaleMemoryQuery): Promise<StaleMemory[]> {
+  const db = await getDb();
+  if (!db) return [];
+  
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  
+  const cutoffDate = new Date(Date.now() - query.olderThanDays * 24 * 60 * 60 * 1000);
+  
+  // Build conditions
+  const conditions = [
+    // Not pinned
+    or(
+      eq((schema.memories as any).isPinned, false),
+      eq((schema.memories as any).isPinned, null)
+    ),
+    // Older than cutoff
+    lt((schema.memories as any).createdAt, cutoffDate),
+  ];
+  
+  // Confidence level filter
+  if (query.confidenceLevels.length > 0) {
+    conditions.push(
+      or(
+        ...query.confidenceLevels.map(level => 
+          eq((schema.memories as any).confidenceLevel, level)
+        )
+      )
+    );
+  }
+  
+  // Importance filter (below threshold)
+  conditions.push(lte((schema.memories as any).importanceScore, query.minImportance));
+  
+  // Project filter if specified
+  if (query.projectId) {
+    conditions.push(eq((schema.memories as any).projectId, query.projectId));
+  }
+  
+  try {
+    const results = await sqliteDb
+      .select({
+        id: schema.memories.id,
+        content: schema.memories.content,
+        type: schema.memories.type,
+        createdAt: schema.memories.createdAt,
+        confidenceLevel: (schema.memories as any).confidenceLevel,
+        importanceScore: (schema.memories as any).importanceScore,
+        isPinned: (schema.memories as any).isPinned,
+      })
+      .from(schema.memories)
+      .where(and(...conditions.filter(Boolean)));
+    
+    return results.map((r: any) => ({
+      ...r,
+      isPinned: Boolean(r.isPinned),
+    }));
+  } catch (error) {
+    console.error('[StaleCleaner] Error querying stale memories:', error);
+    return [];
+  }
+}
+
+export async function deleteMemoryPermanently(memoryId: string): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+  
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  
+  try {
+    // Delete associated records first
+    await sqliteDb.delete(schema.memoryAssociations).where(
+      or(
+        eq((schema.memoryAssociations as any).fromMemoryId, memoryId),
+        eq((schema.memoryAssociations as any).toMemoryId, memoryId)
+      )
+    ).catch(() => {}); // Ignore if table doesn't exist
+    
+    // Delete memory tags
+    await sqliteDb.delete((schema as any).memoryTags).where(
+      eq((schema as any).memoryTags.memoryId, memoryId)
+    ).catch(() => {}); // Ignore if table doesn't exist
+    
+    // Delete the memory itself
+    await sqliteDb.delete(schema.memories).where(eq(schema.memories.id, memoryId));
+  } catch (error) {
+    console.error(`[StaleCleaner] Error deleting memory ${memoryId}:`, error);
+    throw error;
+  }
+}
+
+export async function runAutoClean(options?: Partial<StaleMemoryQuery>): Promise<{
+  deleted: number;
+  summary: Record<string, unknown>;
+}> {
+  const defaultOptions: StaleMemoryQuery = {
+    olderThanDays: options?.olderThanDays || 30,
+    confidenceLevels: options?.confidenceLevels || ['outdated', 'speculative'],
+    minImportance: options?.minImportance || 40,
+  };
+  
+  const stale = await getStaleMemories(defaultOptions);
+  let deleted = 0;
+  
+  for (const memory of stale) {
+    if (!memory.isPinned) {
+      await deleteMemoryPermanently(memory.id);
+      deleted++;
+    }
+  }
+  
+  return {
+    deleted,
+    summary: {
+      scanned: stale.length,
+      skippedPinned: stale.length - deleted,
+      criteria: defaultOptions,
+    },
+  };
+}