squish-memory 1.1.5 → 1.2.0

package/core/temporal-facts.ts

@@ -0,0 +1,244 @@
+/**
+ * Temporal Facts System
+ * Manages versioned facts with validity windows
+ */
+
+import { and, eq, lte, gte, or, isNull, desc } from 'drizzle-orm';
+import { randomUUID } from 'crypto';
+import { getDb } from '../db/index.js';
+import { getSchema } from '../db/schema.js';
+import { traverseSupersededChain } from './utils/history-traversal.js';
+import { createNewFactVersion } from './utils/version-management.js';
+import { buildFactAtTimeQuery } from './utils/temporal-queries.js';
+import { logger } from './logger.js';
+
+/**
+ * Store a new fact with temporal validity
+ */
+export async function storeFact(
+  content: string,
+  options: {
+    validFrom?: Date;
+    validTo?: Date;
+    confidence?: number;
+    tags?: string[];
+    projectId?: string;
+  } = {}
+): Promise<string> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const factId = randomUUID();
+
+    await (db as any).insert(schema.memories).values({
+      id: factId,
+      type: 'fact',
+      sector: 'semantic',
+      content,
+      confidence: options.confidence || 100,
+      validFrom: options.validFrom || new Date(),
+      validTo: options.validTo || null,
+      version: 1,
+      tags: options.tags || [],
+      projectId: options.projectId || null,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    });
+
+    return factId;
+  } catch (error) {
+    logger.error('Error storing fact', error);
+    throw error;
+  }
+}
+
+/**
+ * Update a fact by creating a new version
+ */
+export async function updateFact(
+  previousFactId: string,
+  newContent: string,
+  reason: string
+): Promise<string> {
+  return createNewFactVersion(previousFactId, newContent, {}, reason);
+}
+
+/**
+ * Query facts valid at a specific point in time
+ */
+export async function queryFactsAtTime(
+  timestamp: Date = new Date(),
+  options: {
+    minConfidence?: number;
+    tags?: string[];
+    projectId?: string;
+    limit?: number;
+  } = {}
+): Promise<any[]> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const minConfidence = options.minConfidence || 0;
+    const limit = options.limit || 100;
+
+    const additionalFilters = [gte(schema.memories.confidence as any, minConfidence)];
+    if (options.projectId) {
+      additionalFilters.push(eq(schema.memories.projectId, options.projectId));
+    }
+
+    const where = buildFactAtTimeQuery(schema, timestamp, additionalFilters);
+
+    const results = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(where)
+      .orderBy(desc(schema.memories.confidence))
+      .limit(limit);
+
+    return results;
+  } catch (error) {
+    logger.error('Error querying facts at time', error);
+    return [];
+  }
+}
+
+/**
+ * Get the complete version history of a fact
+ */
+export async function getFactHistory(factId: string): Promise<any[]> {
+  return traverseSupersededChain(factId);
+}
+
+/**
+ * Apply confidence decay based on temporal distance
+ */
+export async function applyConfidenceDecay(factId: string, ageDays: number): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Confidence decays exponentially: newConfidence = originalConfidence * e^(-lambda * ageDays)
+    // Using lambda = 0.01 for moderate decay
+    const decayFactor = Math.exp(-0.01 * ageDays);
+
+    const fact = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, factId))
+      .limit(1);
+
+    if (fact.length === 0) return;
+
+    const newConfidence = Math.max(0, Math.floor(fact[0].confidence * decayFactor));
+
+    await (db as any)
+      .update(schema.memories)
+      .set({ confidence: newConfidence })
+      .where(eq(schema.memories.id, factId));
+  } catch (error) {
+    logger.error('Error applying confidence decay', error);
+  }
+}
+
+/**
+ * Invalidate a fact (mark as no longer valid)
+ */
+export async function invalidateFact(factId: string, reason: string): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    await (db as any)
+      .update(schema.memories)
+      .set({
+        validTo: new Date(),
+        metadata: { invalidationReason: reason, invalidatedAt: new Date().toISOString() },
+      })
+      .where(eq(schema.memories.id, factId));
+  } catch (error) {
+    logger.error('Error invalidating fact', error);
+  }
+}
+
+/**
+ * Check if a fact is currently valid
+ */
+export async function isFactValid(factId: string, atTime: Date = new Date()): Promise<boolean> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const additionalFilters = [eq(schema.memories.id, factId)];
+    const where = buildFactAtTimeQuery(schema, atTime, additionalFilters);
+
+    const fact = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(where)
+      .limit(1);
+
+    return fact.length > 0;
+  } catch (error) {
+    logger.error('Error checking fact validity', error);
+    return false;
+  }
+}
+
+/**
+ * Get statistics about facts
+ */
+export async function getFactStats(projectId?: string): Promise<{
+  totalFacts: number;
+  activeFacts: number;
+  expiredFacts: number;
+  avgConfidence: number;
+  avgAge: number;
+}> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const now = new Date();
+    const where = projectId
+      ? and(eq(schema.memories.type as any, 'fact'), eq(schema.memories.projectId, projectId))
+      : eq(schema.memories.type as any, 'fact');
+
+    const facts = await (db as any).select().from(schema.memories).where(where);
+
+    let active = 0;
+    let expired = 0;
+    let totalConfidence = 0;
+    let totalAge = 0;
+
+    for (const fact of facts) {
+      totalConfidence += fact.confidence || 0;
+
+      if (fact.validTo && new Date(fact.validTo) < now) {
+        expired++;
+      } else if (!fact.validTo || new Date(fact.validTo) >= now) {
+        active++;
+      }
+
+      totalAge += (now.getTime() - new Date(fact.createdAt).getTime()) / (1000 * 60 * 60 * 24);
+    }
+
+    return {
+      totalFacts: facts.length,
+      activeFacts: active,
+      expiredFacts: expired,
+      avgConfidence: facts.length > 0 ? totalConfidence / facts.length : 0,
+      avgAge: facts.length > 0 ? totalAge / facts.length : 0,
+    };
+  } catch (error) {
+    logger.error('Error getting fact stats', error);
+    return {
+      totalFacts: 0,
+      activeFacts: 0,
+      expiredFacts: 0,
+      avgConfidence: 0,
+      avgAge: 0,
+    };
+  }
+}

package/core/tracing/collector.ts

@@ -0,0 +1,470 @@
+/** Trace Collector - Collects search pipeline traces for debugging and performance analysis
+ *
+ * Trace storage format - All search operations are logged with timing information
+ * Trace retrieval - Get traces by session or ID
+ */
+
+import { logger } from '../logger.js';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { eq, desc, sql } from 'drizzle-orm';
+
+export interface SearchTrace {
+  id: string;  // UUID of this trace
+  sessionId: string | null;  // Session that originated this trace
+  query: string;
+  timestamp: Date;  // When trace started
+  totalDurationMs: number;  // Total trace duration in milliseconds
+  metadata: Record<string, unknown>;  // Additional metadata about the trace
+  // Stage data (stored as JSONB in schema)
+  queryRewrite?: QueryRewriteStage;
+  candidateRetrieval?: RetrievalStage;
+  entityFiltering?: RetrievalStage;
+  hybridScoring?: ScoringStage;
+  reranking?: RerankingStage;
+  resultCount?: number;
+  topResults?: TopResult[];
+}
+
+export interface QueryRewriteStage {
+  original?: string;
+  rewritten?: string;
+  method?: string;
+  timeMs?: number;
+}
+
+export interface RetrievalStage {
+  candidates?: number;
+  results?: number;
+  timeMs?: number;
+  entities?: string[];
+}
+
+export interface ScoringStage {
+  results?: number;
+  timeMs?: number;
+}
+
+export interface RerankingStage {
+  results?: number;
+  timeMs?: number;
+}
+
+export interface TopResult {
+  type?: string;
+  content?: string;
+  hybridScore?: number;
+}
+
+export interface TraceOptions {
+  sessionId?: string;
+  limit?: number;  // Limit traces to retrieve
+  session?: string;  // Filter by session ID
+}
+
+export interface TraceStats {
+  totalTraces: number;
+  totalDurationMs: number;
+  avgDurationMs: number;
+  recentSessions: number;
+  totalErrors: number;
+  errorRate: number;
+}
+
+/**
+ * Start a new search trace collection
+ */
+export async function startTrace(sessionId: string, query: string): Promise<string> {
+  const db = await getDb();
+  if (!db) {
+    logger.error('Database unavailable for trace collection');
+    return '';
+  }
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  const traceId = crypto.randomUUID();
+  const timestamp = new Date();
+
+  // Create trace record
+  await sqliteDb.insert(schema.searchTraces).values({
+    id: traceId,
+    sessionId,
+    query,
+    timestamp,
+    resultCount: 0,
+    totalDurationMs: 0,
+  });
+
+  logger.info(`[Tracing] Started trace ${traceId}`);
+
+  return traceId;
+}
+
+/**
+ * Add query rewrite stage data
+ */
+export async function addQueryRewriteStage(traceId: string, stage: QueryRewriteStage): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  const data = JSON.stringify(stage);
+
+  await sqliteDb.update(schema.searchTraces)
+    .set({
+      queryRewrite: sql`CAST(? AS jsonb)`,
+    })
+    .where(eq(schema.searchTraces.id, traceId))
+    .set({ queryRewrite: data });
+
+  logger.debug(`[Tracing] Added queryRewrite stage to trace ${traceId}`);
+}
+
+/**
+ * Add candidate retrieval stage data
+ */
+export async function addCandidateRetrievalStage(traceId: string, stage: RetrievalStage): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  const data = JSON.stringify(stage);
+
+  await sqliteDb.update(schema.searchTraces)
+    .set({ candidateRetrieval: data })
+    .where(eq(schema.searchTraces.id, traceId));
+
+  logger.debug(`[Tracing] Added candidateRetrieval stage to trace ${traceId}`);
+}
+
+/**
+ * Add entity filtering stage data
+ */
+export async function addEntityFilteringStage(traceId: string, stage: RetrievalStage): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  const data = JSON.stringify(stage);
+
+  await sqliteDb.update(schema.searchTraces)
+    .set({ entityFiltering: data })
+    .where(eq(schema.searchTraces.id, traceId));
+
+  logger.debug(`[Tracing] Added entityFiltering stage to trace ${traceId}`);
+}
+
+/**
+ * Add hybrid scoring stage data
+ */
+export async function addHybridScoringStage(traceId: string, stage: ScoringStage): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  const data = JSON.stringify(stage);
+
+  await sqliteDb.update(schema.searchTraces)
+    .set({ hybridScoring: data })
+    .where(eq(schema.searchTraces.id, traceId));
+
+  logger.debug(`[Tracing] Added hybridScoring stage to trace ${traceId}`);
+}
+
+/**
+ * Add reranking stage data
+ */
+export async function addRerankingStage(traceId: string, stage: RerankingStage): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  const data = JSON.stringify(stage);
+
+  await sqliteDb.update(schema.searchTraces)
+    .set({ reranking: data })
+    .where(eq(schema.searchTraces.id, traceId));
+
+  logger.debug(`[Tracing] Added reranking stage to trace ${traceId}`);
+}
+
+/**
+ * Complete current trace (all stages done)
+ */
+export async function completeTrace(traceId: string, results: TopResult[]): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  // Get trace
+  const rows = await sqliteDb.select()
+    .from(schema.searchTraces)
+    .where(eq(schema.searchTraces.id, traceId))
+    .limit(1);
+
+  if (!rows || rows.length === 0) {
+    logger.warn(`[Tracing] Trace ${traceId} not found`);
+    return;
+  }
+
+  const trace = rows[0];
+  const startTime = trace.timestamp.getTime();
+  const endTime = Date.now();
+  const totalDurationMs = endTime - startTime;
+
+  const resultCount = results.length;
+  const topResults = results.slice(0, 10);
+
+  await sqliteDb.update(schema.searchTraces)
+    .set({
+      resultCount,
+      topResults: JSON.stringify(topResults),
+      totalDurationMs,
+    })
+    .where(eq(schema.searchTraces.id, traceId));
+
+  logger.info(`[Tracing] Completed trace ${traceId} with ${resultCount} results`);
+}
+
+/**
+ * Get traces for a session
+ */
+export async function getTraces(options: TraceOptions = {}): Promise<SearchTrace[]> {
+  const db = await getDb();
+  if (!db) return [];
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  let conditions = [];
+
+  if (options.sessionId) {
+    conditions.push(eq(schema.searchTraces.sessionId, options.sessionId));
+  }
+
+  if (options.session && options.session.length > 0) {
+    conditions.push(eq(schema.searchTraces.sessionId, options.session));
+  }
+
+  const query = sqliteDb.select()
+    .from(schema.searchTraces)
+    .where(conditions.length > 0 ? eq(schema.searchTraces.sessionId, options.sessionId || options.session || '') : undefined)
+    .orderBy(desc(schema.searchTraces.timestamp));
+
+  if (options.limit) {
+    query.limit(options.limit);
+  }
+
+  const traces = await query;
+
+  return traces.map((row: any): SearchTrace => ({
+    id: row.id,
+    sessionId: row.session_id,
+    query: row.query,
+    timestamp: row.timestamp,
+    totalDurationMs: row.total_duration_ms,
+    metadata: row.metadata ? JSON.parse(String(row.metadata)) : {},
+    queryRewrite: row.query_rewrite ? JSON.parse(String(row.query_rewrite)) : undefined,
+    candidateRetrieval: row.candidate_retrieval ? JSON.parse(String(row.candidate_retrieval)) : undefined,
+    entityFiltering: row.entity_filtering ? JSON.parse(String(row.entity_filtering)) : undefined,
+    hybridScoring: row.hybrid_scoring ? JSON.parse(String(row.hybrid_scoring)) : undefined,
+    reranking: row.reranking ? JSON.parse(String(row.reranking)) : undefined,
+    resultCount: row.result_count,
+    topResults: row.top_results ? JSON.parse(String(row.top_results)) : undefined,
+  }));
+}
+
+/**
+ * Get a specific trace by ID
+ */
+export async function getTraceById(traceId: string): Promise<SearchTrace | null> {
+  const db = await getDb();
+  if (!db) return null;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  const rows = await sqliteDb.select()
+    .from(schema.searchTraces)
+    .where(eq(schema.searchTraces.id, traceId))
+    .limit(1);
+
+  if (rows.length === 0) return null;
+
+  const row = rows[0];
+
+  return {
+    id: row.id,
+    sessionId: row.session_id,
+    query: row.query,
+    timestamp: row.timestamp,
+    totalDurationMs: row.total_duration_ms,
+    metadata: row.metadata ? JSON.parse(String(row.metadata)) : {},
+    queryRewrite: row.query_rewrite ? JSON.parse(String(row.query_rewrite)) : undefined,
+    candidateRetrieval: row.candidate_retrieval ? JSON.parse(String(row.candidate_retrieval)) : undefined,
+    entityFiltering: row.entity_filtering ? JSON.parse(String(row.entity_filtering)) : undefined,
+    hybridScoring: row.hybrid_scoring ? JSON.parse(String(row.hybrid_scoring)) : undefined,
+    reranking: row.reranking ? JSON.parse(String(row.reranking)) : undefined,
+    resultCount: row.result_count,
+    topResults: row.top_results ? JSON.parse(String(row.top_results)) : undefined,
+  };
+}
+
+/**
+ * Get recent traces (limited)
+ */
+export async function getRecentTraces(limit: number = 10): Promise<SearchTrace[]> {
+  const db = await getDb();
+  if (!db) return [];
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  const traces = await sqliteDb.select()
+    .from(schema.searchTraces)
+    .orderBy(desc(schema.searchTraces.timestamp))
+    .limit(limit);
+
+  return traces.map((row: any): SearchTrace => ({
+    id: row.id,
+    sessionId: row.session_id,
+    query: row.query,
+    timestamp: row.timestamp,
+    totalDurationMs: row.total_duration_ms,
+    metadata: row.metadata ? JSON.parse(String(row.metadata)) : {},
+    queryRewrite: row.query_rewrite ? JSON.parse(String(row.query_rewrite)) : undefined,
+    candidateRetrieval: row.candidate_retrieval ? JSON.parse(String(row.candidate_retrieval)) : undefined,
+    entityFiltering: row.entity_filtering ? JSON.parse(String(row.entity_filtering)) : undefined,
+    hybridScoring: row.hybrid_scoring ? JSON.parse(String(row.hybrid_scoring)) : undefined,
+    reranking: row.reranking ? JSON.parse(String(row.reranking)) : undefined,
+    resultCount: row.result_count,
+    topResults: row.top_results ? JSON.parse(String(row.top_results)) : undefined,
+  }));
+}
+
+/**
+ * Get recent session summary traces (aggregated by session)
+ */
+export async function getSessionTraces(sessionId: string): Promise<SearchTrace[]> {
+  const db = await getDb();
+  if (!db) return [];
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  // Get all traces for this session
+  const traces = await sqliteDb.select()
+    .from(schema.searchTraces)
+    .where(eq(schema.searchTraces.sessionId, sessionId))
+    .orderBy(desc(schema.searchTraces.timestamp));
+
+  if (traces.length === 0) return [];
+
+  return traces.map((row: any): SearchTrace => ({
+    id: row.id,
+    sessionId: row.session_id,
+    query: row.query,
+    timestamp: row.timestamp,
+    totalDurationMs: row.total_duration_ms,
+    metadata: row.metadata ? JSON.parse(String(row.metadata)) : {},
+    queryRewrite: row.query_rewrite ? JSON.parse(String(row.query_rewrite)) : undefined,
+    candidateRetrieval: row.candidate_retrieval ? JSON.parse(String(row.candidate_retrieval)) : undefined,
+    entityFiltering: row.entity_filtering ? JSON.parse(String(row.entity_filtering)) : undefined,
+    hybridScoring: row.hybrid_scoring ? JSON.parse(String(row.hybrid_scoring)) : undefined,
+    reranking: row.reranking ? JSON.parse(String(row.reranking)) : undefined,
+    resultCount: row.result_count,
+    topResults: row.top_results ? JSON.parse(String(row.top_results)) : undefined,
+  }));
+}
+
+/**
+ * Create visual ASCII visualization of a trace
+ */
+export function visualizeTrace(trace: SearchTrace): string {
+  const lines: string[] = [];
+
+  // Header
+  lines.push(`Search Trace: ${trace.id}`);
+  lines.push(`Session: ${trace.sessionId || 'N/A'}`);
+  lines.push(`Query: "${trace.query}"`);
+  lines.push('');
+
+  // Total duration
+  lines.push(`Duration: ${trace.totalDurationMs}ms (${(trace.totalDurationMs / 1000).toFixed(2)}s)`);
+  lines.push('');
+
+  // Stages summary
+  const stageNames: string[] = [];
+  if (trace.queryRewrite) stageNames.push(`Query Rewrite (${trace.queryRewrite.timeMs || 0}ms)`);
+  if (trace.candidateRetrieval) stageNames.push(`Candidate Retrieval (${trace.candidateRetrieval.timeMs || 0}ms)`);
+  if (trace.entityFiltering) stageNames.push(`Entity Filtering (${trace.entityFiltering.timeMs || 0}ms)`);
+  if (trace.hybridScoring) stageNames.push(`Hybrid Scoring (${trace.hybridScoring.timeMs || 0}ms)`);
+  if (trace.reranking) stageNames.push(`Reranking (${trace.reranking.timeMs || 0}ms)`);
+
+  if (stageNames.length > 0) {
+    lines.push('Stages:');
+    stageNames.forEach(name => lines.push(`  - ${name}`));
+  }
+
+  lines.push('');
+
+  // Top results
+  if (trace.topResults && trace.topResults.length > 0) {
+    lines.push(`Top Results (${trace.topResults.length}):`);
+
+    trace.topResults.slice(0, 5).forEach((result, i) =>
+      lines.push(`  ${i + 1}. [${result.type || 'memory'}] ${result.content?.substring(0, 50)}... (score: ${result.hybridScore?.toFixed(2)})`)
+    );
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Get trace statistics (performance metrics)
+ */
+export async function getTraceStats(): Promise<TraceStats> {
+  const db = await getDb();
+  if (!db) {
+    return {
+      totalTraces: 0,
+      totalDurationMs: 0,
+      avgDurationMs: 0,
+      recentSessions: 0,
+      totalErrors: 0,
+      errorRate: 0,
+    };
+  }
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  const traces = await sqliteDb.select()
+    .from(schema.searchTraces)
+    .limit(100);
+
+  const totalTraces = traces.length;
+  const totalDurationMs = traces.reduce((sum: number, t: any) => sum + (t.total_duration_ms || 0), 0);
+  const avgDurationMs = totalTraces > 0 ? Math.round(totalDurationMs / totalTraces) : 0;
+
+  const uniqueSessions = new Set(traces.map((t: any) => t.session_id).filter(Boolean));
+  const recentSessions = uniqueSessions.size;
+
+  return {
+    totalTraces,
+    totalDurationMs,
+    avgDurationMs,
+    recentSessions,
+    totalErrors: 0,
+    errorRate: 0,
+  };
+}