squish-memory 1.0.2 → 1.2.0

package/core/memory/progressive-disclosure.ts

@@ -0,0 +1,392 @@
+import { randomUUID } from 'crypto';
+import { eq, desc, and, sql } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { createDatabaseClient } from '../storage/database.js';
+
+export interface LightweightIndex {
+  id: string;
+  memoryId: string;
+  contentHash: string;
+  contentPreview: string;
+  keyTerms: string[];
+  category: string;
+  importance: number;
+  createdAt: Date;
+}
+
+export interface MemoryPreview {
+  id: string;
+  type: string;
+  contentPreview: string;
+  keyTerms: string[];
+  category: string;
+  importance: number;
+  lastAccessedAt?: Date;
+}
+
+export interface FullMemoryLoad {
+  id: string;
+  content: string;
+  summary?: string;
+  tags: string[];
+  metadata?: Record<string, unknown>;
+}
+
+export interface TokenBudgetStatus {
+  budget: number;
+  used: number;
+  remaining: number;
+  loadedCount: number;
+  preloadCount: number;
+}
+
+const PREVIEW_TOKENS = 50;
+const MAX_PRELOAD_CANDIDATES = 20;
+const TOKEN_ESTIMATE_CHARS = 4;
+
+export async function createLightweightIndex(
+  memoryId: string,
+  content: string,
+  category: string,
+  importance: number
+): Promise<LightweightIndex> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+  
+  const contentHash = hashContent(content);
+  const contentPreview = extractPreview(content);
+  const keyTerms = extractKeyTerms(content);
+
+  const index: LightweightIndex = {
+    id: randomUUID(),
+    memoryId,
+    contentHash,
+    contentPreview,
+    keyTerms,
+    category,
+    importance,
+    createdAt: new Date(),
+  };
+
+  await db.insert(schema.lightweightMemoryIndices).values({
+    id: index.id,
+    memoryId: index.memoryId,
+    contentHash: index.contentHash,
+    contentPreview: index.contentPreview,
+    keyTerms: JSON.stringify(index.keyTerms),
+    category: index.category,
+    importanceScore: index.importance,
+    createdAt: index.createdAt,
+  });
+
+  return index;
+}
+
+export async function searchLightweightIndices(
+  projectId: string,
+  query: string,
+  limit: number = 10
+): Promise<MemoryPreview[]> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const queryLower = query.toLowerCase();
+  const queryTerms = queryLower.split(/\s+/).filter((t) => t.length > 2);
+
+  const results = await db
+    .select({
+      id: schema.lightweightMemoryIndices.id,
+      memoryId: schema.lightweightMemoryIndices.memoryId,
+      contentPreview: schema.lightweightMemoryIndices.contentPreview,
+      keyTerms: schema.lightweightMemoryIndices.keyTerms,
+      category: schema.lightweightMemoryIndices.category,
+      importanceScore: schema.lightweightMemoryIndices.importanceScore,
+      lastAccessedAt: schema.memories.lastAccessedAt,
+    })
+    .from(schema.lightweightMemoryIndices)
+    .leftJoin(
+      schema.memories,
+      eq(schema.lightweightMemoryIndices.memoryId, schema.memories.id)
+    )
+    .where(eq(schema.lightweightMemoryIndices.memoryId, schema.memories.id))
+    .limit(limit * 2);
+
+  const scored = results.map((row: any) => {
+    let score = (row.importanceScore as number) ?? 50;
+    const keyTerms: string[] = typeof row.keyTerms === 'string' 
+      ? JSON.parse(row.keyTerms as string) 
+      : (row.keyTerms as string[]) ?? [];
+    const termsLower = keyTerms.map((t: string) => t.toLowerCase());
+
+    for (const term of queryTerms) {
+      if (row.contentPreview.toLowerCase().includes(term)) {
+        score += 10;
+      }
+      if (termsLower.some((t: string) => t.includes(term))) {
+        score += 15;
+      }
+    }
+
+    return {
+      id: row.id as string,
+      memoryId: row.memoryId as string,
+      contentPreview: row.contentPreview as string,
+      keyTerms,
+      category: row.category as string,
+      importance: (row.importanceScore as number) ?? 50,
+      lastAccessedAt: row.lastAccessedAt as Date | undefined,
+      score,
+    };
+  });
+
+  scored.sort((a: any, b: any) => b.score - a.score);
+
+  return scored.slice(0, limit).map((item: any) => ({
+    id: item.memoryId,
+    type: 'preview',
+    contentPreview: item.contentPreview,
+    keyTerms: item.keyTerms,
+    category: item.category,
+    importance: item.importance,
+    lastAccessedAt: item.lastAccessedAt,
+  }));
+}
+
+export async function loadFullMemory(memoryId: string): Promise<FullMemoryLoad | null> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const rows = await db
+    .select()
+    .from(schema.memories)
+    .where(eq(schema.memories.id, memoryId))
+    .limit(1);
+
+  if (rows.length === 0) return null;
+
+  const row = rows[0];
+  return {
+    id: row.id,
+    content: row.content,
+    summary: row.summary ?? undefined,
+    tags: Array.isArray(row.tags) ? row.tags : [],
+    metadata: row.metadata ?? undefined,
+  };
+}
+
+export async function estimateTokenCount(text: string): Promise<number> {
+  return Math.ceil(text.length / TOKEN_ESTIMATE_CHARS);
+}
+
+export async function allocateTokenBudget(
+  sessionId: string,
+  budget: number
+): Promise<TokenBudgetStatus> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const [session] = await db
+    .select()
+    .from(schema.contextPagingSessions)
+    .where(eq(schema.contextPagingSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (!session) {
+    return {
+      budget,
+      used: 0,
+      remaining: budget,
+      loadedCount: 0,
+      preloadCount: 0,
+    };
+  }
+
+  const tokensUsed = session.tokensUsed ?? 0;
+  const loadedCount = (session.loadedMemoryIds as string[])?.length ?? 0;
+  const preloadCount = (session.preloadCandidateIds as string[])?.length ?? 0;
+
+  return {
+    budget,
+    used: tokensUsed,
+    remaining: Math.max(0, budget - tokensUsed),
+    loadedCount,
+    preloadCount,
+  };
+}
+
+export async function preloadCandidateMemories(
+  sessionId: string,
+  projectId: string,
+  budget: number,
+  excludeLoadedIds: string[]
+): Promise<string[]> {
+  const status = await allocateTokenBudget(sessionId, budget);
+  const availableTokens = status.remaining;
+
+  if (availableTokens < PREVIEW_TOKENS * 2) {
+    return [];
+  }
+
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const rows = await db
+    .select({
+      id: schema.memories.id,
+      content: schema.memories.content,
+      importanceScore: schema.memories.importanceScore,
+      lastAccessedAt: schema.memories.lastAccessedAt,
+    })
+    .from(schema.memories)
+    .where(
+      and(
+        eq(schema.memories.projectId, projectId),
+        eq(schema.memories.isActive, true),
+        excludeLoadedIds.length > 0 
+          ? sql`${schema.memories.id} NOT IN (${excludeLoadedIds.map(id => `'${id.replace(/'/g, "''")}'`).join(',')})` 
+          : sql`TRUE`
+      )
+    )
+    .orderBy(desc(schema.memories.importanceScore), desc(schema.memories.lastAccessedAt))
+    .limit(MAX_PRELOAD_CANDIDATES);
+
+  const candidates: string[] = [];
+  let tokensForPreload = Math.min(availableTokens - PREVIEW_TOKENS, availableTokens * 0.3);
+
+  for (const row of rows) {
+    const tokens = await estimateTokenCount(row.content ?? '');
+    if (tokens <= tokensForPreload) {
+      candidates.push(row.id as string);
+      tokensForPreload -= tokens;
+    }
+    if (candidates.length >= 5) break;
+  }
+
+  await db
+    .update(schema.contextPagingSessions)
+    .set({
+      preloadCandidateIds: candidates,
+      updatedAt: new Date(),
+    })
+    .where(eq(schema.contextPagingSessions.sessionId, sessionId));
+
+  return candidates;
+}
+
+export async function getPreloadCandidates(sessionId: string): Promise<MemoryPreview[]> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const [session] = await db
+    .select()
+    .from(schema.contextPagingSessions)
+    .where(eq(schema.contextPagingSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (!session) return [];
+
+  const candidateIds = (session.preloadCandidateIds as string[]) ?? [];
+  if (candidateIds.length === 0) return [];
+
+  const previews: MemoryPreview[] = [];
+  for (const memoryId of candidateIds.slice(0, 5)) {
+    const [index] = await db
+      .select()
+      .from(schema.lightweightMemoryIndices)
+      .where(eq(schema.lightweightMemoryIndices.memoryId, memoryId))
+      .limit(1);
+
+    if (index) {
+      const keyTerms: string[] = typeof index.keyTerms === 'string'
+        ? JSON.parse(index.keyTerms as string)
+        : (index.keyTerms as string[]) ?? [];
+
+      previews.push({
+        id: index.memoryId as string,
+        type: 'preload',
+        contentPreview: index.contentPreview as string,
+        keyTerms,
+        category: index.category as string,
+        importance: (index.importanceScore as number) ?? 50,
+      });
+    }
+  }
+
+  return previews;
+}
+
+export async function updateIndexImportance(
+  memoryId: string,
+  importance: number
+): Promise<void> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  await db
+    .update(schema.lightweightMemoryIndices)
+    .set({
+      importanceScore: importance,
+    })
+    .where(eq(schema.lightweightMemoryIndices.memoryId, memoryId));
+}
+
+export async function deleteLightweightIndex(memoryId: string): Promise<void> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  await db
+    .delete(schema.lightweightMemoryIndices)
+    .where(eq(schema.lightweightMemoryIndices.memoryId, memoryId));
+}
+
+function hashContent(content: string): string {
+  let hash = 0;
+  for (let i = 0; i < content.length; i++) {
+    const char = content.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash;
+  }
+  return Math.abs(hash).toString(36);
+}
+
+function extractPreview(content: string): string {
+  const firstPeriod = content.indexOf('.');
+  const firstNewline = content.indexOf('\n');
+  
+  let endIndex = content.length;
+  if (firstPeriod > 0 && firstPeriod < 100) {
+    endIndex = firstPeriod + 1;
+  } else if (firstNewline > 0 && firstNewline < 100) {
+    endIndex = firstNewline;
+  }
+  
+  return content.substring(0, Math.min(endIndex, 150)).trim();
+}
+
+function extractKeyTerms(content: string): string[] {
+  const words = content.toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .filter((w) => w.length > 3);
+  
+  const stopWords = new Set([
+    'this', 'that', 'with', 'from', 'have', 'been', 'were', 'they',
+    'their', 'which', 'about', 'would', 'could', 'should', 'there',
+    'what', 'when', 'where', 'who', 'will', 'just', 'like', 'into',
+    'than', 'then', 'both', 'each', 'more', 'most', 'other', 'some',
+    'such', 'only', 'over', 'very', 'also', 'back', 'after', 'being',
+  ]);
+  
+  const termCounts: Record<string, number> = {};
+  for (const word of words) {
+    if (!stopWords.has(word) && !/^\d+$/.test(word)) {
+      termCounts[word] = (termCounts[word] ?? 0) + 1;
+    }
+  }
+  
+  return Object.entries(termCounts)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 5)
+    .map(([term]) => term);
+}

package/core/memory/query-processor.ts

@@ -0,0 +1,130 @@
+/**
+ * Query Processor - Phase 4
+ * Expands queries for better retrieval coverage
+ */
+
+export interface ProcessedQuery {
+  original: string;
+  expanded: string[];
+  entities: string[];
+  synonyms: Map<string, string[]>;
+}
+
+/**
+ * Simple synonym expansion without LLM calls
+ * Fast, deterministic, no API costs
+ */
+export function expandQuery(query: string): ProcessedQuery {
+  const synonyms = new Map<string, string[]>();
+  
+  // Define common synonym mappings
+  const synonymMap: Record<string, string[]> = {
+    'manager': ['manager', 'boss', 'supervisor', 'lead'],
+    'team': ['team', 'group', 'staff', 'crew'],
+    'project': ['project', 'initiative', 'endeavor', 'work'],
+    'start': ['start', 'begin', 'commence', 'initiate'],
+    'budget': ['budget', 'funding', 'cost', 'allocation'],
+    'language': ['language', 'tech', 'stack', 'technology'],
+    'feature': ['feature', 'capability', 'function', 'aspect'],
+    'based': ['based', 'located', 'situated', 'headquartered'],
+  };
+  
+  const lowerQuery = query.toLowerCase();
+  
+  // Find matching synonyms
+  for (const [key, values] of Object.entries(synonymMap)) {
+    if (lowerQuery.includes(key)) {
+      synonyms.set(key, values);
+    }
+  }
+  
+  // Generate expanded queries by substituting synonyms
+  const expanded: string[] = [query]; // Original
+  
+  // Add one variation per synonym found
+  for (const [term, variants] of synonyms) {
+    for (const variant of variants) {
+      if (variant !== term) {
+        expanded.push(query.replace(new RegExp(term, 'gi'), variant));
+      }
+    }
+  }
+  
+  // Extract entities (capitalized words), filtering common stopwords
+  const STOP_WORDS = new Set([
+    'The', 'A', 'An', 'This', 'That', 'These', 'Those',
+    'I', 'You', 'We', 'They', 'He', 'She', 'It',
+    'What', 'When', 'Where', 'Why', 'How',
+    'My', 'Your', 'Our', 'Their', 'Its',
+    'If', 'But', 'Or', 'And', 'So', 'Then',
+    'Just', 'Only', 'Also', 'Very', 'Too',
+    'Has', 'Have', 'Had', 'Does', 'Did', 'Will', 'Would', 'Could', 'Should',
+    'Can', 'May', 'Might', 'Must', 'Shall',
+    'With', 'Without', 'From', 'Into', 'About', 'After', 'Before', 'Between',
+    'During', 'Through', 'Under', 'Over', 'Above', 'Below',
+    'All', 'Some', 'Any', 'No', 'Not', 'Each', 'Every', 'Most', 'Many', 'Much',
+    'Few', 'Both', 'Either', 'Neither', 'Another', 'Other',
+    'Such', 'Same', 'Different', 'New', 'Old', 'First', 'Last', 'Next', 'Last',
+    'See', 'Get', 'Make', 'Do', 'Say', 'Tell', 'Know', 'Think', 'Want', 'Use',
+    'Find', 'Give', 'Take', 'Show', 'Send', 'Put', 'Keep', 'Let', 'Begin', 'Seem',
+  ]);
+  
+  // Common tech stack terms that ARE entities (preserves known entities)
+  const TECH_TERMS = new Set([
+    'TypeScript', 'JavaScript', 'Python', 'Java', 'Go', 'Rust', 'Ruby', 'PHP',
+    'React', 'Vue', 'Angular', 'Svelte', 'Next', 'Nuxt', 'Astro',
+    'Node', 'Express', 'Fastify', 'Deno', 'Bun',
+    'PostgreSQL', 'MySQL', 'MongoDB', 'Redis', 'SQLite', 'DynamoDB',
+    'AWS', 'GCP', 'Azure', 'Docker', 'Kubernetes', 'Terraform',
+    'OpenAI', 'Anthropic', 'Google', 'Meta', 'Microsoft', 'Apple',
+    'HTML', 'CSS', 'Tailwind', 'GraphQL', 'REST', 'gRPC', 'WebSocket',
+    'CI', 'CD', 'Git', 'GitHub', 'GitLab', 'Bitbucket',
+  ]);
+  
+  const entityMatches = query.match(/\b[A-Z][a-z]{2,}\b/g) || [];
+  const entities = [...new Set(entityMatches)].filter(e => {
+    // Always include known tech terms
+    if (TECH_TERMS.has(e)) return true;
+    // Filter out stopwords
+    return !STOP_WORDS.has(e);
+  });
+  
+  return {
+    original: query,
+    expanded: [...new Set(expanded)], // Deduplicate
+    entities,
+    synonyms,
+  };
+}
+
+/**
+ * Multi-query search combiner
+ * Searches with multiple query variants and merges results
+ */
+export async function multiQuerySearch<T>(
+  queries: string[],
+  searchFn: (query: string) => Promise<T[]>,
+  dedupeKey: (item: T) => string
+): Promise<T[]> {
+  const allResults: T[] = [];
+  
+  // Search with each query variant
+  for (const query of queries) {
+    const results = await searchFn(query);
+    allResults.push(...results);
+  }
+  
+  // Deduplicate by key
+  const seen = new Set<string>();
+  const unique: T[] = [];
+  
+  for (const item of allResults) {
+    const key = dedupeKey(item);
+    if (!seen.has(key)) {
+      seen.add(key);
+      unique.push(item);
+    }
+  }
+  
+  return unique;
+}

package/core/memory/query-rewriter.ts

@@ -0,0 +1,153 @@
+/** Query Rewriter - Rewrite user queries for better retrieval using LLM */
+
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+import { formatContextForLLM, type ContextMessage } from './context-collector.js';
+import { expandQuery } from './query-processor.js';
+
+export interface RewriteResult {
+  original: string;
+  rewritten: string;
+  expansions: string[];
+  intent: 'search' | 'recall' | 'question' | 'context';
+  confidence: number;
+  method: 'llm' | 'synonym' | 'none';
+}
+
+const REWRITE_SYSTEM_PROMPT = `You are a search query optimizer. Given a conversation context and a user's latest message, rewrite the message into the single most effective search query to retrieve relevant memories.
+
+Rules:
+1. Output ONLY the optimized search query - no explanations or extra text
+2. Remove filler words (please, can you, etc.)
+3. Focus on the core information need
+4. Preserve important entities (names, dates, technical terms)
+5. Add synonyms for key terms if helpful
+6. Consider what memories would be most relevant
+7. If the message is about recalling something specific, include those details`;
+
+export async function rewriteQuery(
+  query: string,
+  context: ContextMessage[]
+): Promise<RewriteResult> {
+  if (!config.queryRewritingEnabled) {
+    return {
+      original: query,
+      rewritten: query,
+      expansions: [],
+      intent: detectIntent(query),
+      confidence: 1.0,
+      method: 'none',
+    };
+  }
+
+  if (config.queryRewritingFallbackEnabled) {
+    try {
+      const llmResult = await rewriteWithLLM(query, context);
+      if (llmResult) return llmResult;
+    } catch (error) {
+      logger.warn(`[QueryRewriter] LLM rewrite failed, falling back to synonym: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  if (config.queryRewritingFallbackEnabled) {
+    const expanded = expandQuery(query);
+    return {
+      original: query,
+      rewritten: expanded.expanded[0] || query,
+      expansions: expanded.expanded.slice(1),
+      intent: detectIntent(query),
+      confidence: 0.7,
+      method: 'synonym',
+    };
+  }
+
+  return {
+    original: query,
+    rewritten: query,
+    expansions: [],
+    intent: detectIntent(query),
+    confidence: 1.0,
+    method: 'none',
+  };
+}
+
+async function rewriteWithLLM(
+  query: string,
+  context: ContextMessage[]
+): Promise<RewriteResult | null> {
+  if (context.length < 2) return null;
+
+  const lastUserMsg = context.filter(m => m.role === 'user').pop()?.content || '';
+
+  const pronounPattern = /\b(it|that|this|they|them|those|these|he|she|his|her)\b/i;
+  const hasPronouns = pronounPattern.test(query);
+
+  const vaguePattern = /\b(the thing|the stuff|what (we|I) (talked|discussed|mentioned))\b/i;
+  const hasVagueRef = vaguePattern.test(query);
+
+  if (!hasPronouns && !hasVagueRef) return null;
+
+  let rewritten = query;
+
+  const entityPattern = /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b/g;
+  const contextEntities = lastUserMsg.match(entityPattern) || [];
+
+  const techPattern = /\b([a-z]+\.(?:ts|js|py|json|md)|@[a-z]+\/[a-z]+|\b[A-Z]{2,}\b)\b/gi;
+  const techTerms = lastUserMsg.match(techPattern) || [];
+
+  if (contextEntities.length > 0 && hasPronouns) {
+    const mostLikelyEntity = contextEntities[contextEntities.length - 1];
+    if (!query.toLowerCase().includes(mostLikelyEntity.toLowerCase())) {
+      rewritten = `${query} ${mostLikelyEntity}`;
+    }
+  }
+
+  const firstTechTerm = techTerms[0];
+  if (firstTechTerm && !query.includes(firstTechTerm)) {
+    rewritten = `${rewritten} ${firstTechTerm}`;
+  }
+
+  if (rewritten !== query) {
+    logger.info(`[QueryRewriter] Rewrote "${query}" -> "${rewritten}"`);
+    return {
+      original: query,
+      rewritten,
+      expansions: [],
+      intent: detectIntent(query),
+      confidence: 0.8,
+      method: 'llm',
+    };
+  }
+
+  return null;
+}
+
+function detectIntent(query: string): 'search' | 'recall' | 'question' | 'context' {
+  const lower = query.toLowerCase();
+
+  if (/\b(remember|recall|what did (we|I)|what was|retrieve)\b/.test(lower)) {
+    return 'recall';
+  }
+
+  if (/^(what|who|when|where|why|how|which|is|are|can|do|does)/.test(lower)) {
+    return 'question';
+  }
+
+  if (/\b(context|background|about|regarding|concerning)\b/.test(lower)) {
+    return 'context';
+  }
+
+  return 'search';
+}
+
+export function wouldBenefitFromRewrite(query: string): boolean {
+  if (query.split(/\s+/).length < 3) return true;
+
+  const pronounPattern = /\b(it|that|this|they|them|those|these|he|she|his|her)\b/i;
+  if (pronounPattern.test(query)) return true;
+
+  const vaguePattern = /\b(the thing|the stuff|what (we|I) (talked|discussed|mentioned))\b/i;
+  if (vaguePattern.test(query)) return true;
+
+  return false;
+}