squish-memory 1.0.2 → 1.2.0

package/core/tracing/visualizer.ts

@@ -0,0 +1,195 @@
+/** Trace Visualizer - Creates ASCII visualizations of search traces
+ *
+ * Provides tree-format output showing search pipeline stages and timing
+ */
+
+import { logger } from '../logger.js';
+import { SearchTrace } from './collector.js';
+
+/**
+ * Create visual ASCII visualization of a trace
+ */
+export function visualizeTrace(trace: SearchTrace): string {
+  const lines: string[] = [];
+
+  // Header
+  lines.push('='.repeat(60));
+  lines.push(`Search Trace: ${trace.id}`);
+  lines.push(`Session: ${trace.sessionId || 'N/A'}`);
+  lines.push(`Query: "${trace.query}"`);
+  lines.push('='.repeat(60));
+  lines.push('');
+
+  // Total duration
+  lines.push(`Total Duration: ${trace.totalDurationMs}ms (${(trace.totalDurationMs / 1000).toFixed(2)}s)`);
+  lines.push('');
+
+  // Stage timeline visualization
+  lines.push('─'.repeat(60));
+  lines.push('Pipeline Stages:');
+  lines.push('─'.repeat(60));
+  lines.push('');
+
+  const stageNames: string[] = [];
+  const stageTimings: number[] = [];
+
+  if (trace.queryRewrite) {
+    stageNames.push('Query Rewrite');
+    stageTimings.push(trace.queryRewrite.timeMs || 0);
+  }
+  if (trace.candidateRetrieval) {
+    stageNames.push('Candidate Retrieval');
+    stageTimings.push(trace.candidateRetrieval.timeMs || 0);
+  }
+  if (trace.entityFiltering) {
+    stageNames.push('Entity Filtering');
+    stageTimings.push(trace.entityFiltering.timeMs || 0);
+  }
+  if (trace.hybridScoring) {
+    stageNames.push('Hybrid Scoring');
+    stageTimings.push(trace.hybridScoring.timeMs || 0);
+  }
+  if (trace.reranking) {
+    stageNames.push('Reranking');
+    stageTimings.push(trace.reranking.timeMs || 0);
+  }
+
+  if (stageNames.length > 0) {
+    let cumulativeTime = 0;
+    const maxStageNameLength = Math.max(...stageNames.map(n => n.length));
+
+    stageNames.forEach((name, i) => {
+      const duration = stageTimings[i] || 0;
+      cumulativeTime += duration;
+      const progressPercent = ((cumulativeTime / trace.totalDurationMs) * 100).toFixed(1);
+      const barLength = Math.max(0, Math.min(40, Math.round((duration / trace.totalDurationMs) * 40)));
+
+      // Stage name with timing
+      const paddedName = name.padEnd(maxStageNameLength + 2);
+      lines.push(`${paddedName} │ ${duration.toString().padStart(4)}ms │ ${progressPercent.padStart(5)}%`);
+
+      // Visualization bar
+      const bar = '█'.repeat(barLength) + '░'.repeat(40 - barLength);
+      lines.push(`           └─ ${bar}`);
+
+      lines.push('');
+    });
+  } else {
+    lines.push('  No stage data available');
+    lines.push('');
+  }
+
+  // Top results
+  if (trace.topResults && trace.topResults.length > 0) {
+    lines.push('─'.repeat(60));
+    lines.push(`Top Results (${trace.topResults.length}):`);
+    lines.push('─'.repeat(60));
+    lines.push('');
+
+    trace.topResults.slice(0, 5).forEach((result, i) => {
+      const type = result.type || 'memory';
+      const content = result.content?.substring(0, 60) || '';
+      const score = result.hybridScore?.toFixed(2) || 'N/A';
+      lines.push(`${(i + 1).toString().padStart(2)}. [${type.padEnd(8)}] (score: ${score})`);
+      if (content.length > 0) {
+        lines.push(`    ${content}`);
+      }
+      lines.push('');
+    });
+  } else if (trace.resultCount !== undefined) {
+    lines.push('─'.repeat(60));
+    lines.push(`Results: ${trace.resultCount} returned`);
+    lines.push('─'.repeat(60));
+    lines.push('');
+  }
+
+  // Footer
+  lines.push('='.repeat(60));
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+/**
+ * Create compact single-line trace summary
+ */
+export function summarizeTrace(trace: SearchTrace): string {
+  const parts: string[] = [];
+
+  parts.push(`[${trace.id.substring(0, 8)}]`);
+  parts.push(`"${trace.query.substring(0, 30)}${trace.query.length > 30 ? '...' : ''}"`);
+  parts.push(`${trace.totalDurationMs}ms`);
+  parts.push(trace.resultCount !== undefined ? `${trace.resultCount} results` : 'unknown');
+
+  return parts.join(' | ');
+}
+
+/**
+ * Compare multiple traces
+ */
+export function compareTraces(traces: SearchTrace[]): string {
+  const lines: string[] = [];
+
+  lines.push('─'.repeat(80));
+  lines.push('Trace Comparison');
+  lines.push('─'.repeat(80));
+  lines.push('');
+  lines.push('ID'.padEnd(10) + ' | ' + 'Query'.padEnd(30) + ' | ' + 'Duration'.padEnd(10) + ' | ' + 'Results');
+  lines.push('─'.repeat(80));
+
+  traces.forEach(trace => {
+    const id = trace.id.substring(0, 8);
+    const query = trace.query.substring(0, 30);
+    const duration = `${trace.totalDurationMs}ms`;
+    const results = trace.resultCount !== undefined ? trace.resultCount.toString() : '?';
+    lines.push(`${id.padEnd(10)} | ${query.padEnd(30)} | ${duration.padEnd(10)} | ${results.padEnd(8)}`);
+  });
+
+  return lines.join('\n');
+}
+
+/**
+ * Get stage breakdown table
+ */
+export function getStageBreakdown(trace: SearchTrace): string {
+  const lines: string[] = [];
+
+  lines.push('Stage Timing Breakdown:');
+  lines.push('');
+
+  const stages = [
+    { name: 'Query Rewrite', data: trace.queryRewrite, timeMs: trace.queryRewrite?.timeMs || 0 },
+    { name: 'Candidate Retrieval', data: trace.candidateRetrieval, timeMs: trace.candidateRetrieval?.timeMs || 0 },
+    { name: 'Entity Filtering', data: trace.entityFiltering, timeMs: trace.entityFiltering?.timeMs || 0 },
+    { name: 'Hybrid Scoring', data: trace.hybridScoring, timeMs: trace.hybridScoring?.timeMs || 0 },
+    { name: 'Reranking', data: trace.reranking, timeMs: trace.reranking?.timeMs || 0 },
+  ];
+
+  const validStages = stages.filter(s => s.data && s.timeMs !== undefined);
+
+  if (validStages.length > 0) {
+    const maxNameLength = Math.max(...validStages.map(s => s.name.length));
+
+    validStages.forEach(stage => {
+      const percentage = ((stage.timeMs / trace.totalDurationMs) * 100).toFixed(1);
+      lines.push(`  ${stage.name.padEnd(maxNameLength)} │ ${stage.timeMs.toString().padStart(4)}ms │ ${percentage.padStart(5)}%`);
+    });
+
+    lines.push('');
+    lines.push('  ' + '─'.repeat(maxNameLength + 4 + 5 + 6));
+  } else {
+    lines.push('  No stage timing data available');
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Export all visualizer functions
+ */
+export default {
+  visualizeTrace,
+  summarizeTrace,
+  compareTraces,
+  getStageBreakdown,
+};

package/core/utils/cleanup-operations.ts

@@ -0,0 +1,50 @@
+/**
+ * Shared Cleanup Operations Utilities
+ * Common patterns for age-based cleanup operations
+ */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+
+/**
+ * Cleanup old session summaries
+ */
+export async function cleanupOldSessionSummaries(olderThanDays: number = 30): Promise<number> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const threshold = new Date(Date.now() - olderThanDays * 24 * 60 * 60 * 1000);
+
+    const result = await (db as any)
+      .delete(schema.sessionSummaries)
+      .where((schema.sessionSummaries.createdAt as any) < threshold);
+
+    return result?.rowCount || 0;
+  } catch (error) {
+    logger.error('Error pruning old summaries', error);
+    return 0;
+  }
+}
+
+/**
+ * Cleanup old memory snapshots
+ */
+export async function cleanupOldMemorySnapshots(olderThanDays: number = 90): Promise<number> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const threshold = new Date(Date.now() - olderThanDays * 24 * 60 * 60 * 1000);
+
+    const result = await (db as any)
+      .delete(schema.memorySnapshots)
+      .where((schema.memorySnapshots.createdAt as any) < threshold);
+
+    return result?.rowCount || 0;
+  } catch (error) {
+    logger.error('Error deleting old snapshots', error);
+    return 0;
+  }
+}

package/core/utils/content-extraction.ts

@@ -0,0 +1,95 @@
+/**
+ * Shared Content Extraction Utilities
+ * Common patterns for extracting information from messages
+ */
+
+/**
+ * Extract key information from conversation messages
+ */
+export function extractMessageContent(messages: any[]): {
+  userMessages: any[];
+  assistantMessages: any[];
+  toolCalls: Set<string>;
+  topics: Set<string>;
+  timestamp: string;
+} {
+  // Extract key information from messages
+  const userMessages = messages.filter((m: any) => m.role === 'user');
+  const assistantMessages = messages.filter((m: any) => m.role === 'assistant');
+
+  // Extract tool calls
+  const toolCalls = new Set<string>();
+  for (const msg of messages) {
+    if (msg.toolCalls && Array.isArray(msg.toolCalls)) {
+      for (const tc of msg.toolCalls) {
+        toolCalls.add(tc.name);
+      }
+    }
+  }
+
+  // Extract topics from first and last user messages
+  const topics = new Set<string>();
+  if (userMessages.length > 0) {
+    const firstUser = userMessages[0].content || '';
+    const lastUser = userMessages[userMessages.length - 1].content || '';
+
+    // Simple topic extraction (first 10 words)
+    const extractTopics = (text: string) => {
+      const words = text.split(/\s+/).slice(0, 10);
+      return words.join(' ');
+    };
+
+    if (firstUser) topics.add(extractTopics(firstUser));
+    if (lastUser && lastUser !== firstUser) topics.add(extractTopics(lastUser));
+  }
+
+  const timestamp = messages.length > 0 ? messages[messages.length - 1].createdAt : 'unknown';
+
+  return {
+    userMessages,
+    assistantMessages,
+    toolCalls,
+    topics,
+    timestamp,
+  };
+}
+
+/**
+ * Generate basic extractive summary from extracted content
+ */
+export function generateExtractiveSummary(extracted: ReturnType<typeof extractMessageContent>): string {
+  const hasContent =
+    extracted.userMessages.length > 0 ||
+    extracted.assistantMessages.length > 0 ||
+    extracted.toolCalls.size > 0 ||
+    extracted.topics.size > 0 ||
+    extracted.timestamp !== 'unknown';
+
+  if (!hasContent) {
+    return '';
+  }
+
+  const parts: string[] = [];
+
+  if (extracted.userMessages.length > 0) {
+    parts.push(`User prompts: ${extracted.userMessages.length}`);
+  }
+
+  if (extracted.assistantMessages.length > 0) {
+    parts.push(`Assistant responses: ${extracted.assistantMessages.length}`);
+  }
+
+  if (extracted.toolCalls.size > 0) {
+    parts.push(`Tools used: ${Array.from(extracted.toolCalls).join(', ')}`);
+  }
+
+  if (extracted.topics.size > 0) {
+    parts.push(`Topics: ${Array.from(extracted.topics).join('; ')}`);
+  }
+
+  if (extracted.timestamp !== 'unknown') {
+    parts.push(`Last activity: ${extracted.timestamp}`);
+  }
+
+  return parts.join('. ');
+}

package/core/utils/filter-builder.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared Requirements Filter Builder
+ * Common patterns for building database filters from criteria objects
+ */
+
+import { eq, inArray, gte } from 'drizzle-orm';
+
+/**
+ * Build database filter array from MemoryCriteria
+ */
+export function buildMemoryFilters(criteria: any, schema: any): any[] {
+  const filters: any[] = [];
+
+  if (criteria.type) {
+    filters.push(eq(schema.memories.type as any, criteria.type));
+  }
+
+  if (criteria.types && criteria.types.length > 0) {
+    filters.push(inArray(schema.memories.type as any, criteria.types));
+  }
+
+  if (criteria.sector) {
+    filters.push(eq(schema.memories.sector as any, criteria.sector));
+  }
+
+  if (criteria.minConfidence !== undefined) {
+    filters.push(gte(schema.memories.confidence as any, criteria.minConfidence));
+  }
+
+  if (criteria.minRelevance !== undefined) {
+    filters.push(gte(schema.memories.relevanceScore as any, criteria.minRelevance));
+  }
+
+  if (criteria.projectId) {
+    filters.push(eq(schema.memories.projectId, criteria.projectId));
+  }
+
+  if (criteria.agentId) {
+    filters.push(eq(schema.memories.agentId as any, criteria.agentId));
+  }
+
+  return filters;
+}
+
+/**
+ * Build database filter array from partial criteria (for assertions)
+ */
+export function buildMemoryFiltersPartial(criteria: any, schema: any): any[] {
+  const filters: any[] = [];
+
+  if (criteria.type) {
+    filters.push(eq(schema.memories.type as any, criteria.type));
+  }
+
+  return filters;
+}

package/core/utils/history-traversal.ts

@@ -0,0 +1,63 @@
+/**
+ * Shared History and Version Tracking Utilities
+ * Common patterns for traversing version chains and history
+ */
+
+import { eq } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+
+/**
+ * Traverse superseded fact chain (specific to memories table)
+ * Follows the supersededBy chain to build complete version history
+ */
+export async function traverseSupersededChain(
+  startFactId: string,
+  options: {
+    maxDepth?: number;
+    includeStart?: boolean;
+  } = {}
+): Promise<any[]> {
+  const { maxDepth = 50, includeStart = true } = options;
+
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Get the starting fact
+    const initial = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, startFactId))
+      .limit(1);
+
+    if (initial.length === 0) {
+      return [];
+    }
+
+    const history: any[] = includeStart ? [initial[0]] : [];
+    let currentId = initial[0].supersededBy;
+    let depth = 0;
+
+    // Follow the superseded chain
+    while (currentId && depth < maxDepth) {
+      const next = await (db as any)
+        .select()
+        .from(schema.memories)
+        .where(eq(schema.memories.id, currentId))
+        .limit(1);
+
+      if (next.length === 0) break;
+
+      history.push(next[0]);
+      currentId = next[0].supersededBy;
+      depth++;
+    }
+
+    return history;
+  } catch (error) {
+    logger.error('Error traversing superseded chain', error);
+    return [];
+  }
+}

package/core/utils/memory-operations.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared Memory Operations Utilities
+ * Common patterns for memory governance operations
+ */
+
+import { eq } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { createDatabaseClient } from '../storage/database.js';
+import { config } from '../../config.js';
+import { logger } from '../logger.js';
+
+/**
+ * Generic memory operation with governance checks and error handling
+ */
+export async function performMemoryOperation(
+  memoryId: string,
+  operation: {
+    name: string;
+    updates: Record<string, any>;
+    requiresGovernance?: boolean;
+  }
+): Promise<void> {
+  if (operation.requiresGovernance !== false && !config.governanceEnabled) {
+    return;
+  }
+
+  try {
+    const db = await getDb();
+    const client = createDatabaseClient(db);
+    const schema = await getSchema();
+
+    await client
+      .update(schema.memories)
+      .set(operation.updates)
+      .where(eq(schema.memories.id, memoryId));
+  } catch (error) {
+    logger.error(`Error ${operation.name.toLowerCase()}`, error);
+  }
+}
+
+/**
+ * Redis publish operation with error handling
+ */
+export async function performRedisPublish(
+  getRedisClient: () => Promise<any>,
+  channel: string,
+  message: unknown
+): Promise<void> {
+  try {
+    const redis = await getRedisClient();
+    await redis.publish(channel, JSON.stringify(message));
+  } catch (error) {
+    logger.error('Error publishing to Redis', error);
+  }
+}

package/core/utils/query-operations.ts

@@ -0,0 +1,83 @@
+/**
+ * Shared Requirements Query Operations
+ * Common patterns for database queries with error handling
+ */
+
+import { and } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+
+export class MemoryRequirementError extends Error {
+  constructor(
+    message: string,
+    public readonly missingCriteria: any,
+    public readonly context?: any
+  ) {
+    super(message);
+    this.name = 'MemoryRequirementError';
+    Object.setPrototypeOf(this, MemoryRequirementError.prototype);
+  }
+}
+
+/**
+ * Execute a memory requirement query with standard error handling
+ */
+export async function executeMemoryQuery(
+  criteria: any,
+  filters: any[],
+  limit: number = 1,
+  operation: string = 'memory requirement'
+): Promise<any[]> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const query = filters.length > 0 ? and(...filters) : undefined;
+
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(query)
+      .limit(limit);
+
+    return memories;
+  } catch (error) {
+    if (error instanceof MemoryRequirementError) {
+      throw error;
+    }
+    const msg = error instanceof Error ? error.message : String(error);
+    throw new MemoryRequirementError(`Error checking ${operation}: ${msg}`, criteria);
+  }
+}
+
+/**
+ * Execute a memory assertion query (expecting specific results)
+ */
+export async function executeMemoryAssertion(
+  criteria: any,
+  filters: any[],
+  expectedCount: number,
+  operation: string,
+  errorMessage: string
+): Promise<any[]> {
+  try {
+    const memories = await executeMemoryQuery(criteria, filters, 100, operation);
+
+    if (expectedCount === 0 && memories.length > 0) {
+      throw new MemoryRequirementError(errorMessage, criteria, { found: memories[0].id });
+    }
+
+    if (expectedCount > 0 && memories.length < expectedCount) {
+      const msg = `Required ${expectedCount} memories but found ${memories.length}`;
+      throw new MemoryRequirementError(msg, criteria, { found: memories.length, required: expectedCount });
+    }
+
+    return memories;
+  } catch (error) {
+    if (error instanceof MemoryRequirementError) {
+      throw error;
+    }
+    const msg = error instanceof Error ? error.message : String(error);
+    throw new MemoryRequirementError(`Error ${operation}: ${msg}`, criteria);
+  }
+}

package/core/utils/summarization-helpers.ts

@@ -0,0 +1,45 @@
+/**
+ * Shared Summarization Strategy Utilities
+ * Common patterns for different summarization approaches
+ */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { config } from '../../config.js';
+import { estimateTokens } from '../context/context-window.js';
+
+export { estimateTokens };
+
+/**
+ * Chunk messages for incremental summarization
+ */
+export function chunkMessages(messages: any[], chunkSize: number): any[][] {
+  const chunks: any[][] = [];
+  for (let i = 0; i < messages.length; i += chunkSize) {
+    chunks.push(messages.slice(i, i + chunkSize));
+  }
+  return chunks;
+}
+
+/**
+ * Get rolling window of messages
+ */
+export function getRollingWindow(messages: any[], windowSize: number): any[] {
+  if (windowSize <= 0) {
+    return [];
+  }
+
+  return messages.slice(-windowSize);
+}
+
+/**
+ * Calculate tokens saved by summarization
+ */
+export function calculateTokensSaved(messages: any[], summary: string): number {
+  const originalTokens = messages.reduce(
+    (sum: number, m: any) => sum + estimateTokens(m.content || ''),
+    0
+  );
+  const summaryTokens = estimateTokens(summary);
+  return Math.max(0, originalTokens - summaryTokens);
+}

package/core/utils/temporal-queries.ts

@@ -0,0 +1,39 @@
+/**
+ * Shared Temporal Query Utilities
+ * Common patterns for temporal database queries
+ */
+
+import { and, eq, lte, gte, or, isNull } from 'drizzle-orm';
+
+/**
+ * Build temporal validity filters for queries
+ */
+export function buildTemporalFilters(
+  schema: any,
+  timestamp: Date = new Date(),
+  additionalFilters: any[] = []
+): any[] {
+  const filters = [
+    lte(schema.validFrom as any, timestamp),
+    or(isNull(schema.validTo), gte(schema.validTo as any, timestamp)),
+    ...additionalFilters
+  ];
+
+  return filters;
+}
+
+/**
+ * Build temporal query for facts at a specific time
+ */
+export function buildFactAtTimeQuery(
+  schema: any,
+  timestamp: Date = new Date(),
+  additionalFilters: any[] = []
+): any {
+  const filters = buildTemporalFilters(schema, timestamp, [
+    eq(schema.type as any, 'fact'),
+    ...additionalFilters
+  ]);
+
+  return and(...filters);
+}