squish-memory 1.0.2 → 1.2.0

package/core/memory/entity-extractor.ts

@@ -0,0 +1,426 @@
+/**
+ * Entity Extractor
+ * Extracts named entities from memory content
+ * Supports people, files, functions, dates, locations, concepts, tools, and patterns
+ */
+
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { eq, and } from 'drizzle-orm';
+import { logger } from '../../core/logger.js';
+
+export type EntityType =
+  | 'person'
+  | 'file'
+  | 'function'
+  | 'class'
+  | 'concept'
+  | 'tool'
+  | 'date'
+  | 'location'
+  | 'pattern'
+  | 'technique'
+  | 'other';
+
+export interface ExtractedEntity {
+  name: string;
+  type: EntityType;
+  confidence: number; // 0-1
+  startIndex: number;
+  endIndex: number;
+  context: string;
+  normalized?: string;
+}
+
+// Regex patterns for entity detection
+const PATTERNS = {
+  // File paths: src/components/Button.tsx
+  filePath: /(?:^|[^\w])(?:\.?\/[\w.-]*[\w.-]*(?:\/[\w.-]+)*|[\w.-]+\.(?:ts|tsx|js|jsx|py|rb|go|java|cpp|c|h|json|yaml|yml|env|md|txt|csv|sql|graphql))\b/gm,
+
+  // Functions/methods
+  functionCall: /\b([a-zA-Z_$][a-zA-Z0-9_$]*)\s*(?=\()/g,
+
+  // Classes
+  className: /\b(?:class|new|extends|implements)\s+([A-Z][a-zA-Z0-9_$]*)\b/g,
+
+  // Common names
+  personName: /(?:^|[^.\w])\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\b(?![^.\w])/g,
+
+  // Capitalized concepts
+  concept: /\b(?:[A-Z][a-z]+\s+)+(?:System|Architecture|Pattern|Model|Algorithm|Framework|Library|Process|Concept|Method)\b/g,
+
+  // Tools
+  tool: /\b(?:React|Vue|Angular|Node\.?js?|Express|Django|Flask|FastAPI|PostgreSQL|MongoDB|Redis|Docker|Kubernetes)\b/g,
+
+  // Design patterns
+  designPattern: /\b(?:abstract\s+)?(?:factory|singleton|observer|decorator|strategy|adapter)\s+(?:pattern|method)\b/gi,
+
+  // Techniques
+  technique: /\b(?:memoization|caching|lazy\s+loading|debouncing|throttling)\b/gi,
+
+  // ISO dates
+  date: /\b(\d{4}-\d{2}-\d{2})(?:T\d{2}:\d{2}:\d{2})?\b/g,
+};
+
+/**
+ * Extract unique entity names for auto-linking
+ */
+export function extractEntityNames(content: string): string[] {
+  const names = new Set<string>();
+
+  // Extract person names
+  const namePattern = /(?:^|[^.\w])\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\b(?![^.\w])/g;
+  let match;
+  while ((match = namePattern.exec(content)) !== null) {
+    names.add(match[1].toLowerCase());
+  }
+
+  // Extract capitalized concepts
+  const conceptPattern = /\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+\b/g;
+  while ((match = conceptPattern.exec(content)) !== null) {
+    names.add(match[0].toLowerCase());
+  }
+
+  return Array.from(names).slice(0, 10);
+}
+
+/**
+ * Extract entities from content
+ */
+export async function extractEntities(content: string): Promise<ExtractedEntity[]> {
+  const entities: ExtractedEntity[] = [];
+  const seenKeys = new Set<string>(); // Deduplicate entities
+
+  // Extract file paths
+  const filePaths = Array.from(content.matchAll(PATTERNS.filePath));
+  for (const match of filePaths) {
+    const name = match[0].trim();
+    const key = `file:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'file',
+        confidence: 0.9,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+        normalized: normalizePath(name),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract function calls
+  const functionCalls = Array.from(content.matchAll(PATTERNS.functionCall));
+  for (const match of functionCalls) {
+    const name = match[1];
+    const key = `function:${name}`;
+    if (!seenKeys.has(key) && name.length > 2) {
+      entities.push({
+        name,
+        type: 'function',
+        confidence: 0.85,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract class names
+  const classNames = Array.from(content.matchAll(PATTERNS.className));
+  for (const match of classNames) {
+    const name = match[1];
+    const key = `class:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'class',
+        confidence: 0.95,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract technologies/tools
+  const tools = Array.from(content.matchAll(PATTERNS.tool));
+  for (const match of tools) {
+    const name = match[0];
+    const key = `tool:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'tool',
+        confidence: 0.9,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract design patterns
+  const patterns = Array.from(content.matchAll(PATTERNS.designPattern));
+  for (const match of patterns) {
+    const name = match[0];
+    const key = `pattern:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'pattern',
+        confidence: 0.92,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract techniques/methodologies
+  const techniques = Array.from(content.matchAll(PATTERNS.technique));
+  for (const match of techniques) {
+    const name = match[0];
+    const key = `technique:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'technique',
+        confidence: 0.88,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract dates
+  const dates = Array.from(content.matchAll(PATTERNS.date));
+  for (const match of dates) {
+    const name = match[0];
+    const key = `date:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'date',
+        confidence: 0.95,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract quoted important concepts
+  const quotedConcepts = Array.from(content.matchAll(PATTERNS.quotedConcept));
+  for (const match of quotedConcepts) {
+    const name = match[1];
+    const key = `concept:${name}`;
+    if (!seenKeys.has(key) && name.length > 2) {
+      entities.push({
+        name,
+        type: 'concept',
+        confidence: 0.8,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Extract capitalized concepts
+  const concepts = Array.from(content.matchAll(PATTERNS.concept));
+  for (const match of concepts) {
+    const name = match[0];
+    const key = `concept:${name}`;
+    if (!seenKeys.has(key)) {
+      entities.push({
+        name,
+        type: 'concept',
+        confidence: 0.75,
+        startIndex: match.index || 0,
+        endIndex: (match.index || 0) + match[0].length,
+        context: extractContext(content, match.index || 0),
+      });
+      seenKeys.add(key);
+    }
+  }
+
+  // Sort by start index
+  entities.sort((a, b) => a.startIndex - b.startIndex);
+
+  logger.debug('Entities extracted', {
+    count: entities.length,
+    byType: countByType(entities),
+  });
+
+  return entities;
+}
+
+/**
+ * Link extracted entities to memory records
+ * Creates entity records in the knowledge graph
+ */
+export async function linkEntitiesToMemories(
+  memoryId: string,
+  entities: ExtractedEntity[]
+): Promise<void> {
+  if (entities.length === 0) return;
+
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Get memory to find project
+    const memories = await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.id, memoryId))
+      .limit(1);
+
+    if (memories.length === 0) return;
+    const memory = memories[0];
+
+    // Insert unique entities into the knowledge graph
+    for (const entity of entities) {
+      try {
+        // Check if entity already exists
+        const existing = await (db as any)
+          .select()
+          .from(schema.entities)
+          .where(
+            and(
+              eq(schema.entities.projectId, memory.projectId),
+              eq(schema.entities.name, entity.name),
+              eq(schema.entities.type, entity.type)
+            )
+          )
+          .limit(1);
+
+        if (existing.length === 0) {
+          // Insert new entity with metadata in properties
+          await (db as any).insert(schema.entities).values({
+            name: entity.name,
+            type: entity.type,
+            projectId: memory.projectId,
+            description: entity.context,
+            properties: {
+              confidence: entity.confidence,
+              extractedFrom: memoryId,
+              normalized: entity.normalized,
+            },
+          });
+        }
+      } catch (error) {
+        logger.error('Error inserting entity', { entity, error });
+      }
+    }
+
+    logger.debug('Entities linked to memory', {
+      memoryId,
+      entityCount: entities.length,
+    });
+  } catch (error) {
+    logger.error('Error linking entities to memories', error);
+  }
+}
+
+/**
+ * Get entities extracted from a memory based on properties
+ */
+export async function getMemoryEntities(memoryId: string): Promise<ExtractedEntity[]> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Find all entities with this memory ID in properties
+    const allEntities = await (db as any).select().from(schema.entities);
+
+    const entities: ExtractedEntity[] = [];
+
+    for (const entity of allEntities) {
+      const props = entity.properties as Record<string, unknown>;
+      if (props && props.extractedFrom === memoryId) {
+        entities.push({
+          name: entity.name,
+          type: entity.type as EntityType,
+          confidence: (props.confidence as number) || 0.8,
+          startIndex: 0,
+          endIndex: 0,
+          context: entity.description || '',
+          normalized: props.normalized as string | undefined,
+        });
+      }
+    }
+
+    return entities;
+  } catch (error) {
+    logger.error('Error getting memory entities', error);
+    return [];
+  }
+}
+
+/**
+ * Get all entities for a project
+ */
+export async function getProjectEntities(
+  projectId: string,
+  type?: EntityType
+): Promise<any[]> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    let query: any = (db as any).select().from(schema.entities).where(eq(schema.entities.projectId, projectId));
+
+    if (type) {
+      query = query.where(eq(schema.entities.type, type));
+    }
+
+    return await query;
+  } catch (error) {
+    logger.error('Error getting project entities', error);
+    return [];
+  }
+}
+
+/**
+ * Extract context around entity
+ */
+function extractContext(content: string, index: number, contextLength: number = 50): string {
+  const start = Math.max(0, index - contextLength);
+  const end = Math.min(content.length, index + contextLength);
+  return content.substring(start, end).trim();
+}
+
+/**
+ * Normalize file path
+ */
+function normalizePath(path: string): string {
+  return path
+    .trim()
+    .replace(/^\.\//, '')
+    .toLowerCase()
+    .replace(/\\/g, '/');
+}
+
+/**
+ * Count entities by type
+ */
+function countByType(entities: ExtractedEntity[]): Record<EntityType, number> {
+  const counts = {} as Record<EntityType, number>;
+
+  for (const entity of entities) {
+    counts[entity.type] = (counts[entity.type] || 0) + 1;
+  }
+
+  return counts;
+}

package/core/memory/entity-resolver.ts

@@ -0,0 +1,89 @@
+export interface ExtractedEntities {
+  primary: string[]; // Main entities (people, projects)
+  secondary: string[]; // Supporting entities
+  queryType: 'factual' | 'relational' | 'temporal';
+}
+
+/**
+ * Extract entities from a query using Claude
+ * NO FALLBACKS - real extraction or throw
+ */
+export async function extractQueryEntities(
+  query: string,
+  callClaudeFn: (prompt: string, maxTokens: number) => Promise<string>
+): Promise<ExtractedEntities> {
+  const prompt = `Extract entities from this question. Return ONLY JSON:
+{
+  "primary": ["Alice", "project name"],
+  "secondary": ["team", "budget"],
+  "queryType": "factual"
+}
+
+Question: ${query}`;
+
+  const response = await callClaudeFn(prompt, 500);
+  
+  const jsonMatch = response.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) {
+    throw new Error('No JSON in entity extraction');
+  }
+
+  const parsed = JSON.parse(jsonMatch[0]) as ExtractedEntities;
+  
+  return {
+    primary: parsed.primary || [],
+    secondary: parsed.secondary || [],
+    queryType: parsed.queryType || 'factual',
+  };
+}
+
+/**
+ * Score memory relevance based on entity matching
+ * Returns boost factor (0-1)
+ */
+export function scoreEntityMatch(
+  memoryEntities: string[],
+  queryEntities: string[]
+): number {
+  if (!queryEntities.length || !memoryEntities.length) {
+    return 0.5; // Neutral if no entities
+  }
+
+  const querySet = new Set(queryEntities.map(e => e.toLowerCase()));
+  const memorySet = new Set(memoryEntities.map(e => e.toLowerCase()));
+  
+  // Count matches
+  let matches = 0;
+  for (const entity of querySet) {
+    if (memorySet.has(entity)) {
+      matches++;
+    }
+  }
+  
+  // Calculate boost (0-1)
+  const matchRatio = matches / querySet.size;
+  return 0.5 + (matchRatio * 0.5); // 0.5 to 1.0
+}
+
+/**
+ * Filter and boost memories by entity relevance
+ */
+export function filterByEntities(
+  memories: any[],
+  queryEntities: string[]
+): Array<{ memory: any; entityBoost: number }> {
+  if (!queryEntities.length) {
+    // No entities to filter by, return all with neutral boost
+    return memories.map(m => ({ memory: m, entityBoost: 0.5 }));
+  }
+
+  return memories.map(memory => {
+    const memoryEntities = (memory.metadata?.entities as string[]) || [];
+    const boost = scoreEntityMatch(memoryEntities, queryEntities);
+    
+    return {
+      memory,
+      entityBoost: boost,
+    };
+  });
+}

package/core/memory/explain.ts

@@ -0,0 +1,112 @@
+import { eq } from 'drizzle-orm';
+import { getDbClient } from '../lib/db-client.js';
+import { deserializeMetadata } from './serialization.js';
+import { getMemorySnapshot } from '../snapshots/retrieval.js';
+import { getMemoryPlace } from '../places/memory-places.js';
+import { getPlace } from '../places/places.js';
+import { getBeliefsForMemory } from '../beliefs/store.js';
+import type { StoredBelief } from '../beliefs/types.js';
+
+export interface MemoryInspection {
+  id: string;
+  type: string;
+  classification: string;
+  reasons: string[];
+  rawFallbackSnapshotId?: string | null;
+  nuanceSuppressed: boolean;
+  place?: string | null;
+  placeType?: string | null;
+  graphStatus?: string | null;
+  content: string;
+  legacyMetadata: boolean;
+  beliefs?: StoredBelief[];
+}
+
+export function summarizeInspection(input: MemoryInspection): string {
+  const lines = [
+    `Memory ${input.id}`,
+    `Type: ${input.type}`,
+    `Classification: ${input.classification}`,
+    `Reasons: ${input.reasons.join('; ') || 'n/a'}`,
+    `Raw fallback: ${input.rawFallbackSnapshotId ?? 'none'}`,
+    `nuance suppressed: ${input.nuanceSuppressed ? 'yes' : 'no'}`,
+    `Place: ${input.place ?? 'none'}${input.placeType ? ` (${input.placeType})` : ''}`,
+    `Graph: ${input.graphStatus ?? 'none'}`,
+    `Content: ${input.content}`,
+    `Legacy metadata: ${input.legacyMetadata ? 'yes' : 'no'}`,
+  ];
+  if (input.beliefs && input.beliefs.length > 0) {
+    lines.push('Beliefs:');
+    for (const belief of input.beliefs) {
+      const confidence = typeof belief.confidence === 'number' ? belief.confidence.toFixed(1) : '?';
+      const sourceCount = belief.sourceMemoryIds?.length ?? 1;
+      const origin = sourceCount > 1 ? `derived from ${sourceCount} memories` : 'derived from 1 memory';
+      const evidence = belief.evidenceSummary 
+        ? `\n  Evidence: ${belief.evidenceSummary.slice(0, 100)}${belief.evidenceSummary.length > 100 ? '...' : ''}`
+        : '';
+      lines.push(`- [${belief.type}] ${belief.statement}`);
+      lines.push(`  Status: ${belief.status}, Confidence: ${confidence}/100, ${origin}${evidence}`);
+      if (belief.status === 'superseded') {
+        lines.push(`  Note: This belief supersedes a previous version`);
+      }
+      if (belief.status === 'disputed') {
+        lines.push(`  Warning: This belief has been disputed`);
+      }
+    }
+  } else {
+    lines.push(`Beliefs: ${input.legacyMetadata ? 'No derived beliefs available for this legacy record' : 'none derived'}`);
+  }
+  return lines.join('\n');
+}
+
+export async function explainMemory(id: string): Promise<MemoryInspection | null> {
+  const { db, schema } = await getDbClient();
+  const rows = await db.select().from(schema.memories).where(eq(schema.memories.id, id)).limit(1);
+  const row = rows[0];
+  if (!row) return null;
+
+  const metadata = deserializeMetadata(row.metadata ?? null) as Record<string, unknown> | null;
+  const rawClassification = metadata?.signal?.classification ?? metadata?.classification ?? null;
+  const classification = typeof rawClassification === 'string' && rawClassification.trim().length > 0
+    ? rawClassification
+    : 'legacy-durable';
+  const reasons = Array.isArray(metadata?.signal?.reasons)
+    ? (metadata?.signal?.reasons as string[])
+    : Array.isArray(metadata?.reasons)
+      ? (metadata?.reasons as string[])
+      : ['This record predates signal tracking, so detailed ingestion metadata is unavailable.'];
+  const rawFallbackSnapshotId = typeof metadata?.rawFallbackSnapshotId === 'string'
+    ? metadata.rawFallbackSnapshotId
+    : null;
+  const nuanceSuppressed = Boolean(metadata?.signal?.nuanceSuppressed ?? metadata?.nuanceSuppressed);
+  const graphStatus =
+    typeof metadata?.graphStatus === 'string'
+      ? metadata.graphStatus
+      : typeof metadata?.graph === 'object' && metadata.graph
+        ? JSON.stringify(metadata.graph)
+        : 'Unavailable for this legacy record';
+  const legacyMetadata = rawClassification == null;
+
+  const placeId = await getMemoryPlace(id);
+  const place = placeId ? await getPlace(placeId) : null;
+
+  if (rawFallbackSnapshotId) {
+    await getMemorySnapshot(rawFallbackSnapshotId);
+  }
+  const beliefs = await getBeliefsForMemory(id);
+
+  return {
+    id: row.id,
+    type: row.type,
+    classification,
+    reasons,
+    rawFallbackSnapshotId,
+    nuanceSuppressed,
+    place: place?.name ?? null,
+    placeType: place?.placeType ?? null,
+    graphStatus,
+    content: row.content,
+    legacyMetadata,
+    beliefs,
+  };
+}