squish-memory 1.0.2 → 1.2.0

package/core/memory/stats.ts

@@ -0,0 +1,181 @@
+/**
+ * Memory Statistics Module
+ * Provides memory usage statistics for CLI and MCP
+ */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { config } from '../../config.js';
+import { ensureProject } from '../../core/projects.js';
+import { createDatabaseClient } from '../storage/database.js';
+import { getProjectSignalStats } from '../session/working-set.js';
+
+export interface MemoryStats {
+  totalMemories: number;
+  byType: Record<string, number>;
+  totalNotes: number;
+  notesByCategory: Record<string, number>;
+  totalLearnings: number;
+  learningsByType: Record<string, number>;
+  totalLinks: number;
+  oldestMemory?: string;
+  newestMemory?: string;
+  projectPath: string;
+  mode: string;
+  signal?: {
+    captured: number;
+    suppressed: number;
+    sessionOnly: number;
+    durable: number;
+    durableWithRaw: number;
+    tokensSaved: number;
+    placeRouted: number;
+    graphEnriched: number;
+  };
+}
+
+/**
+ * Get memory statistics for a project
+ */
+export async function getMemoryStats(projectPath: string = process.cwd()): Promise<MemoryStats> {
+  let db: any;
+  try {
+    db = createDatabaseClient(await getDb());
+  } catch (error) {
+    throw new Error(`Database unavailable: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+
+  const schema = await getSchema();
+  const project = await ensureProject(projectPath);
+  if (!project) {
+    throw new Error(`Project not found: ${projectPath}`);
+  }
+
+  const stats: MemoryStats = {
+    totalMemories: 0,
+    byType: {},
+    totalNotes: 0,
+    notesByCategory: {},
+    totalLearnings: 0,
+    learningsByType: {},
+    totalLinks: 0,
+    projectPath,
+    mode: config.isTeamMode ? 'team' : 'local',
+  };
+
+  try {
+    // Count total memories
+    const countResult = await db
+      .select({ count: schema.memories.id })
+      .from(schema.memories)
+      .where(eq(schema.memories.projectId, project.id));
+
+    stats.totalMemories = countResult.length;
+
+    // Count by type
+    if (config.isTeamMode) {
+      // PostgreSQL - use raw query for GROUP BY
+      const typeCounts = await db.execute(sql`
+        SELECT type, COUNT(*) as count
+        FROM memories
+        ${sql`WHERE project_id = ${project.id}`}
+        GROUP BY type
+      `);
+      for (const row of typeCounts.rows) {
+        stats.byType[row.type] = Number(row.count);
+      }
+    } else {
+      // SQLite - get all and count in memory
+      const allMemories = await db
+        .select({ type: schema.memories.type })
+       .from(schema.memories)
+       .where(eq(schema.memories.projectId, project.id));
+
+      for (const mem of allMemories) {
+        const type = mem.type || 'unknown';
+        stats.byType[type] = (stats.byType[type] || 0) + 1;
+      }
+    }
+
+    // Get oldest and newest
+    const oldest = await db
+      .select({ createdAt: schema.memories.createdAt })
+       .from(schema.memories)
+       .where(eq(schema.memories.projectId, project.id))
+       .orderBy(asc(schema.memories.createdAt))
+      .limit(1);
+
+    const newest = await db
+      .select({ createdAt: schema.memories.createdAt })
+       .from(schema.memories)
+       .where(eq(schema.memories.projectId, project.id))
+       .orderBy(desc(schema.memories.createdAt))
+      .limit(1);
+
+    if (oldest.length > 0 && oldest[0].createdAt) {
+      stats.oldestMemory = oldest[0].createdAt;
+    }
+    if (newest.length > 0 && newest[0].createdAt) {
+      stats.newestMemory = newest[0].createdAt;
+    }
+
+    // Learnings
+    const allLearnings = await db
+      .select({ category: schema.learnings.category, type: schema.learnings.type })
+       .from(schema.learnings)
+       .where(eq(schema.learnings.projectId, project.id));
+
+    stats.totalNotes = allLearnings.length;
+    for (const obs of allLearnings) {
+      const cat = obs.category || 'uncategorized';
+      stats.notesByCategory[cat] = (stats.notesByCategory[cat] || 0) + 1;
+    }
+
+    // Learnings by type
+    const learningTypes = ['success', 'failure', 'fix', 'insight'];
+    const learningRecords = allLearnings.filter((o: any) => {
+      const type = o.type || '';
+      return learningTypes.includes(type.toLowerCase());
+    });
+    stats.totalLearnings = learningRecords.length;
+    // Count by type
+    for (const obs of learningRecords) {
+      const type = (obs as any).type || 'unknown';
+      stats.learningsByType[type] = (stats.learningsByType[type] || 0) + 1;
+    }
+
+    // Links
+    // Links are scoped via their associated memories
+    if (config.isTeamMode) {
+      // PostgreSQL - use raw query to join through memories
+      const linksCount = await db.execute(sql`
+        SELECT COUNT(*) as count FROM memory_associations ma
+        JOIN memories m1 ON ma.from_memory_id = m1.id
+        JOIN memories m2 ON ma.to_memory_id = m2.id
+        ${project ? sql`WHERE m1.project_id = ${project.id} OR m2.project_id = ${project.id}` : sql``}
+      `);
+      stats.totalLinks = Number(linksCount.rows[0]?.count || 0);
+    } else {
+      // SQLite - get all and filter in memory
+      const allLinks = await db
+        .select({
+          fromProjectId: schema.memories.projectId,
+          toProjectId: schema.memories.projectId
+        })
+        .from(schema.memoryAssociations)
+        .innerJoin(schema.memories, eq(schema.memoryAssociations.fromMemoryId, schema.memories.id))
+        .where(project ? eq(schema.memories.projectId, project.id) : undefined);
+      stats.totalLinks = allLinks.length;
+    }
+
+  } catch (error) {
+    // Return empty stats on error
+    console.error('Error getting memory stats:', error);
+  }
+
+  stats.signal = await getProjectSignalStats(projectPath);
+
+  return stats;
+}
+
+import { eq, sql, asc, desc } from 'drizzle-orm';

package/core/memory/telemetry.ts

@@ -0,0 +1,392 @@
+/**
+ * Retrieval Quality Telemetry
+ * Tracks memory usage patterns (echo/fizzle) to improve retrieval quality
+ * Echo: memory was retrieved and used
+ * Fizzle: memory was retrieved but not used
+ */
+
+import { eq, and, desc } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+
+export interface RetrievalEvent {
+  memoryId: string;
+  query: string;
+  position: number; // Position in results
+  score: number;
+  wasUsed: boolean; // true = echo, false = fizzle
+  sessionId?: string;
+  timestamp: Date;
+}
+
+export interface RetrievalStats {
+  totalRetrievals: number;
+  echoes: number;
+  fizzles: number;
+  echoRate: number;
+  avgPosition: number;
+  topMemories: string[];
+  underperformingMemories: string[];
+}
+
+export interface MemoryTelemetry {
+  memoryId: string;
+  retrievalCount: number;
+  echoCount: number;
+  fizzleCount: number;
+  echoRate: number;
+  avgPosition: number;
+  lastRetrieved?: Date;
+  lastEchoed?: Date;
+}
+
+// In-memory cache for recent retrieval events (flushed periodically)
+const retrievalEvents: RetrievalEvent[] = [];
+const MAX_CACHE_SIZE = 1000;
+
+/**
+ * Record a retrieval event
+ */
+export function recordRetrieval(
+  memoryId: string,
+  query: string,
+  position: number,
+  score: number,
+  sessionId?: string
+): void {
+  const event: RetrievalEvent = {
+    memoryId,
+    query,
+    position,
+    score,
+    wasUsed: false, // Initially false, updated when echoed
+    sessionId,
+    timestamp: new Date(),
+  };
+  
+  retrievalEvents.push(event);
+  
+  // Flush if cache is full
+  if (retrievalEvents.length >= MAX_CACHE_SIZE) {
+    flushRetrievalEvents().catch(err => {
+      logger.error('Failed to flush retrieval events', err);
+    });
+  }
+}
+
+/**
+ * Mark a retrieval as "echoed" (memory was actually used)
+ */
+export function recordEcho(
+  memoryId: string,
+  sessionId?: string
+): void {
+  // Find the most recent retrieval for this memory
+  const recentIndex = retrievalEvents.findIndex(
+    (e, i) => 
+      e.memoryId === memoryId && 
+      !e.wasUsed &&
+      (sessionId ? e.sessionId === sessionId : true)
+  );
+  
+  if (recentIndex !== -1) {
+    retrievalEvents[recentIndex].wasUsed = true;
+  } else {
+    // Add a new echo event if not found in cache
+    retrievalEvents.push({
+      memoryId,
+      query: '',
+      position: 0,
+      score: 0,
+      wasUsed: true,
+      sessionId,
+      timestamp: new Date(),
+    });
+  }
+  
+  // Update memory's echo count directly
+  incrementMemoryEcho(memoryId).catch(err => {
+    logger.debug('Failed to increment echo count', err);
+  });
+}
+
+/**
+ * Mark retrievals as "fizzled" (memory was not used)
+ * Call this at the end of a session for non-echoed retrievals
+ */
+export function recordFizzle(
+  memoryId: string,
+  sessionId?: string
+): void {
+  incrementMemoryFizzle(memoryId).catch(err => {
+    logger.debug('Failed to increment fizzle count', err);
+  });
+}
+
+/**
+ * Flush cached retrieval events to database
+ */
+export async function flushRetrievalEvents(): Promise<void> {
+  if (retrievalEvents.length === 0) return;
+  
+  const events = [...retrievalEvents];
+  retrievalEvents.length = 0; // Clear cache
+  
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    
+    // Group by memoryId for batch updates
+    const byMemory = new Map<string, RetrievalEvent[]>();
+    for (const event of events) {
+      const existing = byMemory.get(event.memoryId) || [];
+      existing.push(event);
+      byMemory.set(event.memoryId, existing);
+    }
+    
+    // Update each memory's telemetry
+    for (const [memoryId, memEvents] of byMemory.entries()) {
+      const echoCount = memEvents.filter(e => e.wasUsed).length;
+      const fizzleCount = memEvents.filter(e => !e.wasUsed).length;
+      
+      // Get current stats
+      const memories = await (db as any)
+        .select()
+        .from(schema.memories)
+        .where(eq(schema.memories.id, memoryId))
+        .limit(1);
+      
+      if (memories.length > 0) {
+        const current = memories[0];
+        const newEchoCount = (current.echoCount || 0) + echoCount;
+        const newFizzleCount = (current.fizzleCount || 0) + fizzleCount;
+        const newRetrievalCount = (current.retrievalCount || 0) + memEvents.length;
+        
+        await (db as any)
+          .update(schema.memories)
+          .set({
+            echoCount: newEchoCount,
+            fizzleCount: newFizzleCount,
+            retrievalCount: newRetrievalCount,
+            lastRetrievedAt: new Date(),
+            updatedAt: new Date(),
+          })
+          .where(eq(schema.memories.id, memoryId));
+      }
+    }
+    
+    logger.debug('Flushed retrieval events', { count: events.length });
+    
+  } catch (error) {
+    logger.error('Error flushing retrieval events', error);
+    // Re-add events to cache on failure
+    retrievalEvents.push(...events);
+  }
+}
+
+/**
+ * Increment memory's echo count directly
+ */
+async function incrementMemoryEcho(memoryId: string): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, memoryId))
+      .limit(1);
+    
+    if (memories.length > 0) {
+      const current = memories[0];
+      await (db as any)
+        .update(schema.memories)
+        .set({
+          echoCount: (current.echoCount || 0) + 1,
+          lastEchoedAt: new Date(),
+          coactivationScore: (current.coactivationScore || 0) + 1,
+          updatedAt: new Date(),
+        })
+        .where(eq(schema.memories.id, memoryId));
+    }
+  } catch (error) {
+    logger.debug('Error incrementing echo count', { error });
+  }
+}
+
+/**
+ * Increment memory's fizzle count directly
+ */
+async function incrementMemoryFizzle(memoryId: string): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, memoryId))
+      .limit(1);
+    
+    if (memories.length > 0) {
+      const current = memories[0];
+      await (db as any)
+        .update(schema.memories)
+        .set({
+          fizzleCount: (current.fizzleCount || 0) + 1,
+          updatedAt: new Date(),
+        })
+        .where(eq(schema.memories.id, memoryId));
+    }
+  } catch (error) {
+    logger.debug('Error incrementing fizzle count', { error });
+  }
+}
+
+/**
+ * Get telemetry for a specific memory
+ */
+export async function getMemoryTelemetry(memoryId: string): Promise<MemoryTelemetry | null> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, memoryId))
+      .limit(1);
+    
+    if (memories.length === 0) return null;
+    
+    const m = memories[0];
+    const echoCount = m.echoCount || 0;
+    const fizzleCount = m.fizzleCount || 0;
+    const total = echoCount + fizzleCount;
+    
+    return {
+      memoryId: m.id,
+      retrievalCount: m.retrievalCount || 0,
+      echoCount,
+      fizzleCount,
+      echoRate: total > 0 ? echoCount / total : 0,
+      avgPosition: 0, // Would need to calculate from retrieval events
+      lastRetrieved: m.lastRetrievedAt,
+      lastEchoed: m.lastEchoedAt,
+    };
+    
+  } catch (error) {
+    logger.error('Error getting memory telemetry', error);
+    return null;
+  }
+}
+
+/**
+ * Get overall retrieval statistics
+ */
+export async function getRetrievalStats(projectId?: string): Promise<RetrievalStats> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    
+    const whereClause = projectId
+      ? eq(schema.memories.projectId, projectId)
+      : undefined;
+    
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(whereClause)
+      .orderBy(desc(schema.memories.retrievalCount))
+      .limit(1000);
+    
+    let totalRetrievals = 0;
+    let totalEchoes = 0;
+    let totalFizzles = 0;
+    const topMemories: string[] = [];
+    const underperforming: string[] = [];
+    
+    for (const m of memories) {
+      const echoes = m.echoCount || 0;
+      const fizzles = m.fizzleCount || 0;
+      const retrievals = m.retrievalCount || 0;
+      
+      totalRetrievals += retrievals;
+      totalEchoes += echoes;
+      totalFizzles += fizzles;
+      
+      // Top performers (high echo rate with sufficient data)
+      if (echoes >= 3 && (echoes / (echoes + fizzles)) >= 0.8) {
+        topMemories.push(m.id);
+      }
+      
+      // Underperformers (high fizzle rate with sufficient data)
+      if (fizzles >= 5 && (fizzles / (echoes + fizzles)) >= 0.7) {
+        underperforming.push(m.id);
+      }
+    }
+    
+    const echoRate = (totalEchoes + totalFizzles) > 0
+      ? totalEchoes / (totalEchoes + totalFizzles)
+      : 0;
+    
+    return {
+      totalRetrievals,
+      echoes: totalEchoes,
+      fizzles: totalFizzles,
+      echoRate,
+      avgPosition: 0, // Would need more detailed tracking
+      topMemories: topMemories.slice(0, 10),
+      underperformingMemories: underperforming.slice(0, 10),
+    };
+    
+  } catch (error) {
+    logger.error('Error getting retrieval stats', error);
+    return {
+      totalRetrievals: 0,
+      echoes: 0,
+      fizzles: 0,
+      echoRate: 0,
+      avgPosition: 0,
+      topMemories: [],
+      underperformingMemories: [],
+    };
+  }
+}
+
+/**
+ * Adjust retrieval boosting based on echo/fizzle history
+ * Returns a boost factor (0.5 - 2.0) for a memory
+ */
+export function calculateTelemetryBoost(telemetry: MemoryTelemetry | null): number {
+  if (!telemetry || telemetry.retrievalCount < 3) {
+    return 1.0; // Not enough data
+  }
+  
+  const echoRate = telemetry.echoRate;
+  
+  // High echo rate = boost
+  if (echoRate >= 0.8) {
+    return 1.5;
+  } else if (echoRate >= 0.6) {
+    return 1.2;
+  } else if (echoRate >= 0.4) {
+    return 1.0;
+  } else if (echoRate >= 0.2) {
+    return 0.8;
+  } else {
+    return 0.5; // Low echo rate = penalize
+  }
+}
+
+/**
+ * Periodic cleanup job for telemetry data
+ */
+export async function cleanupOldTelemetry(daysToKeep: number = 90): Promise<number> {
+  // This would typically clean up a dedicated telemetry_events table
+  // For now, we just flush the in-memory cache
+  await flushRetrievalEvents();
+  return 0;
+}