squish-memory 1.0.2 → 1.2.0

package/core/memory/fact-deriver.ts

@@ -0,0 +1,300 @@
+/**
+ * Fact Deriver
+ * 
+ * Derives implicit facts from existing relationships in the knowledge graph.
+ * Example: works_on(A, X) + uses(X, Y) -> depends_on(A, Y)
+ */
+
+import { eq, and, or } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+import type { RelationType } from '../graph/llm-entity-extractor.js';
+
+// Extend RelationType to include derived relation types
+type ExtendedRelationType = RelationType | 'oversees' | 'may_affect';
+
+// ─── Derivation Rules ─────────────────────────────────────────────────────────
+
+interface DerivationRule {
+  name: string;
+  fromType: RelationType;
+  toType: RelationType;
+  derivedType: ExtendedRelationType;
+  description: string;
+  confidence: number;
+}
+
+const DERIVATION_RULES: DerivationRule[] = [
+  // Transitivity rules
+  {
+    name: 'works_on_uses_depends_on',
+    fromType: 'works_on',
+    toType: 'uses',
+    derivedType: 'depends_on',
+    description: 'If A works on X and X uses Y, then A depends on Y',
+    confidence: 0.8,
+  },
+  {
+    name: 'manages_works_oversees',
+    fromType: 'manages',
+    toType: 'works_on',
+    derivedType: 'oversees',
+    description: 'If A manages B and B works on X, then A oversees X',
+    confidence: 0.75,
+  },
+  {
+    name: 'caused_affects_may_affect',
+    fromType: 'caused',
+    toType: 'affects',
+    derivedType: 'may_affect',
+    description: 'If X caused Y and Y affects Z, then X may affect Z',
+    confidence: 0.7,
+  },
+  {
+    name: 'part_of_contains',
+    fromType: 'part_of',
+    toType: 'contains',
+    derivedType: 'related_to',
+    description: 'If A is part of X and X contains B, then A is related to B',
+    confidence: 0.6,
+  },
+  {
+    name: 'uses_depends_on',
+    fromType: 'uses',
+    toType: 'depends_on',
+    derivedType: 'depends_on',
+    description: 'If A uses X and X depends on Y, then A depends on Y',
+    confidence: 0.85,
+  },
+  {
+    name: 'created_resolved',
+    fromType: 'created',
+    toType: 'resolved',
+    derivedType: 'related_to',
+    description: 'If A created X and X was resolved by B, then A is related to B',
+    confidence: 0.5,
+  },
+  {
+    name: 'blocks_depends_on',
+    fromType: 'blocks',
+    toType: 'depends_on',
+    derivedType: 'blocks',
+    description: 'If X blocks A and A depends on Y, then X may block Y',
+    confidence: 0.65,
+  },
+];
+
+export interface DerivedFact {
+  fromEntityId: string;
+  fromEntityName: string;
+  toEntityId: string;
+  toEntityName: string;
+  relationType: RelationType;
+  rule: string;
+  confidence: number;
+  isDerived: boolean;
+}
+
+// ─── Main Derivation Function ─────────────────────────────────────────────────
+
+/**
+ * Derive implicit facts from existing relationships in the knowledge graph.
+ * Applies transitivity and causal rules to find new relationships.
+ */
+export async function deriveFacts(
+  projectId: string,
+  options?: {
+    maxDerivations?: number;
+    minConfidence?: number;
+    storeResults?: boolean;
+  }
+): Promise<DerivedFact[]> {
+  const { maxDerivations = 50, minConfidence = 0.6, storeResults = true } = options || {};
+  const db = await getDb();
+  const schema = await getSchema();
+
+  // Get all entities for this project
+  const entities = await (db as any)
+    .select()
+    .from(schema.entities)
+    .where(eq(schema.entities.projectId, projectId));
+
+  if (entities.length < 2) return [];
+
+  const entityMap = new Map<string, any>(entities.map((e: any) => [e.id as string, e] as [string, any]));
+  const derivedFacts: DerivedFact[] = [];
+
+  // Get all relations for this project's entities
+  const entityIds = entities.map((e: any) => e.id);
+  const allRelations: any[] = [];
+
+  // Fetch in batches to avoid query size limits
+  const batchSize = 100;
+  for (let i = 0; i < entityIds.length; i += batchSize) {
+    const batch = entityIds.slice(i, i + batchSize);
+    const relations = await (db as any)
+      .select()
+      .from(schema.entityRelations)
+      .where(
+        or(...batch.map((id: string) => eq(schema.entityRelations.fromEntityId, id)))
+      );
+    allRelations.push(...relations);
+  }
+
+  // Build adjacency list for fast lookup
+  const outgoing = new Map<string, Map<string, { relationType: string; weight: number }>>();
+
+  for (const rel of allRelations) {
+    if (!outgoing.has(rel.fromEntityId)) {
+      outgoing.set(rel.fromEntityId, new Map());
+    }
+    outgoing.get(rel.fromEntityId)!.set(rel.toEntityId, {
+      relationType: rel.type,
+      weight: rel.weight || 1,
+    });
+  }
+
+  // Apply derivation rules
+  for (const rule of DERIVATION_RULES) {
+    if (derivedFacts.length >= maxDerivations) break;
+    if (rule.confidence < minConfidence) continue;
+
+    // Find all pairs where fromType matches
+    for (const [fromId, targets] of outgoing) {
+      if (derivedFacts.length >= maxDerivations) break;
+
+      for (const [midId, fromRel] of targets) {
+        if (fromRel.relationType !== rule.fromType) continue;
+
+        // Check if midId has outgoing toType relations
+        const midTargets = outgoing.get(midId);
+        if (!midTargets) continue;
+
+        for (const [toId, toRel] of midTargets) {
+          if (toRel.relationType !== rule.toType) continue;
+
+          // Found a derivation: fromId --fromType--> midId --toType--> toId
+          // Derive: fromId --derivedType--> toId
+          const fromEntity = entityMap.get(fromId);
+          const toEntity = entityMap.get(toId);
+
+          if (!fromEntity || !toEntity) continue;
+
+          // Check if this derived relation already exists
+          const existing = await (db as any)
+            .select()
+            .from(schema.entityRelations)
+            .where(
+              and(
+                eq(schema.entityRelations.fromEntityId, fromId),
+                eq(schema.entityRelations.toEntityId, toId),
+                eq(schema.entityRelations.type, rule.derivedType)
+              )
+            )
+            .limit(1);
+
+          if (existing.length > 0) continue; // Already exists
+
+          const derivedFact: DerivedFact = {
+            fromEntityId: fromId,
+            fromEntityName: fromEntity.name,
+            toEntityId: toId,
+            toEntityName: toEntity.name,
+            relationType: rule.derivedType as RelationType,
+            rule: rule.name,
+            confidence: rule.confidence,
+            isDerived: true,
+          };
+
+          derivedFacts.push(derivedFact);
+
+          // Store the derived fact if requested
+          if (storeResults) {
+            try {
+              await (db as any).insert(schema.entityRelations).values({
+                fromEntityId: fromId,
+                toEntityId: toId,
+                type: rule.derivedType,
+                weight: Math.round(rule.confidence * 10),
+                properties: {
+                  derived: true,
+                  rule: rule.name,
+                  confidence: rule.confidence,
+                  description: rule.description,
+                } as any,
+              });
+            } catch (error) {
+              logger.debug('Error storing derived fact', {
+                rule: rule.name,
+                from: fromEntity.name,
+                to: toEntity.name,
+                error: error as Error,
+              });
+            }
+          }
+        }
+      }
+    }
+  }
+
+  logger.info('Fact derivation completed', {
+    projectId,
+    derivedCount: derivedFacts.length,
+    rulesApplied: DERIVATION_RULES.length,
+  });
+
+  return derivedFacts;
+}
+
+/**
+ * Get all derived facts for a project.
+ */
+export async function getDerivedFacts(projectId: string): Promise<DerivedFact[]> {
+  const db = await getDb();
+  const schema = await getSchema();
+
+  // Get all entities for this project
+  const entities = await (db as any)
+    .select()
+    .from(schema.entities)
+    .where(eq(schema.entities.projectId, projectId));
+
+  const entityIds = entities.map((e: any) => e.id);
+  if (entityIds.length === 0) return [];
+
+  // Get all derived relations
+  const derivedRelations: any[] = [];
+  const batchSize = 100;
+
+  for (let i = 0; i < entityIds.length; i += batchSize) {
+    const batch = entityIds.slice(i, i + batchSize);
+    const relations = await (db as any)
+      .select()
+      .from(schema.entityRelations)
+      .where(
+        or(...batch.map((id: string) => eq(schema.entityRelations.fromEntityId, id)))
+      );
+
+    for (const rel of relations) {
+      const props = rel.properties as Record<string, unknown> | null;
+      if (props && props.derived === true) {
+        const fromEntity = entities.find((e: any) => e.id === rel.fromEntityId);
+        const toEntity = entities.find((e: any) => e.id === rel.toEntityId);
+
+        derivedRelations.push({
+          fromEntityId: rel.fromEntityId,
+          fromEntityName: fromEntity?.name || 'Unknown',
+          toEntityId: rel.toEntityId,
+          toEntityName: toEntity?.name || 'Unknown',
+          relationType: rel.type as RelationType,
+          rule: (props.rule as string) || 'unknown',
+          confidence: (props.confidence as number) || 0.5,
+          isDerived: true,
+        });
+      }
+    }
+  }
+
+  return derivedRelations;
+}

package/core/memory/fact-extractor.ts

@@ -0,0 +1,120 @@
+import type { RememberInput } from './memories.js';
+
+export interface ExtractedFact {
+  content: string;
+  confidence: number;
+  entities: string[];
+  relation?: string;
+}
+
+export interface FactExtractionResult {
+  facts: ExtractedFact[];
+  summary: string;
+  entities: string[];
+}
+
+/**
+ * Extract facts from conversation text using Claude API
+ * 
+ * This function MUST return valid facts or throw an error.
+ * NO FALLBACKS - we need real extraction to reach 90% accuracy.
+ */
+export async function extractFacts(
+  text: string,
+  callClaude: (prompt: string, maxTokens: number) => Promise<string>
+): Promise<FactExtractionResult> {
+  if (!text || text.length < 50) {
+    // Too short for meaningful extraction - return as single fact
+    return {
+      facts: [{ content: text.trim(), confidence: 1.0, entities: [] }],
+      summary: text.trim(),
+      entities: []
+    };
+  }
+
+  const prompt = `Extract atomic facts from this conversation. 
+
+Requirements:
+- Extract specific, verifiable facts (names, dates, numbers, relationships)
+- Each fact should be a standalone statement
+- Identify entities (people, organizations, projects, locations)
+- Note temporal information and relationships
+
+Return ONLY valid JSON in this exact format:
+{"facts":[{"content":"fact statement","confidence":0.95,"entities":["Name"],"relation":"optional"}],"summary":"brief summary","entities":["all","entities"]}
+
+Text to analyze:
+${text.substring(0, 4000)}`;
+
+  const response = await callClaude(prompt, 2000);
+  
+  // Extract JSON - look for the first { and last }
+  const startIdx = response.indexOf('{');
+  const endIdx = response.lastIndexOf('}');
+  
+  if (startIdx === -1 || endIdx === -1 || startIdx >= endIdx) {
+    throw new Error(`Fact extraction failed: No valid JSON found in response. Response: ${response.substring(0, 200)}`);
+  }
+  
+  const jsonStr = response.substring(startIdx, endIdx + 1);
+  
+  let parsed: FactExtractionResult;
+  try {
+    parsed = JSON.parse(jsonStr) as FactExtractionResult;
+  } catch (parseError) {
+    throw new Error(`Fact extraction failed: JSON parse error. JSON: ${jsonStr.substring(0, 200)}`);
+  }
+  
+  // Validate facts array exists and has content
+  if (!parsed.facts || !Array.isArray(parsed.facts)) {
+    throw new Error(`Fact extraction failed: Missing 'facts' array in response. Keys: ${Object.keys(parsed).join(', ')}`);
+  }
+  
+  // Filter and clean facts
+  const validFacts = parsed.facts
+    .filter((f: any) => f && typeof f.content === 'string' && f.content.trim().length > 10)
+    .map((f: any) => ({
+      content: f.content.trim(),
+      confidence: Math.max(0, Math.min(1, typeof f.confidence === 'number' ? f.confidence : 0.8)),
+      entities: Array.isArray(f.entities) ? f.entities.filter((e: any) => typeof e === 'string') : [],
+      relation: typeof f.relation === 'string' ? f.relation : undefined
+    }));
+  
+  if (validFacts.length === 0) {
+    throw new Error(`Fact extraction failed: No valid facts after filtering. Original count: ${parsed.facts.length}`);
+  }
+  
+  return {
+    facts: validFacts,
+    summary: typeof parsed.summary === 'string' && parsed.summary.trim() 
+      ? parsed.summary.trim() 
+      : text.substring(0, 200),
+    entities: Array.isArray(parsed.entities) 
+      ? parsed.entities.filter((e: any) => typeof e === 'string')
+      : []
+  };
+}
+
+/**
+ * Convert extracted facts to memory inputs
+ */
+export function factsToMemoryInputs(
+  sourceMemory: RememberInput,
+  extraction: FactExtractionResult,
+  sourceId: string
+): RememberInput[] {
+  return extraction.facts.map((fact, index) => ({
+    content: fact.content,
+    type: 'fact' as const,
+    metadata: {
+      sourceMemoryId: sourceId,
+      extractedAt: new Date().toISOString(),
+      entities: fact.entities,
+      relation: fact.relation,
+      confidence: Math.floor(fact.confidence * 100),
+      factIndex: index
+    },
+    tags: [...(sourceMemory.tags || []), 'extracted-fact', ...fact.entities],
+    project: sourceMemory.project
+  }));
+}

package/core/memory/feedback-tracker.ts

@@ -0,0 +1,200 @@
+/** Feedback Tracker - Track memory usage in responses for Echo/Fizzle loop */
+
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+import { getDb } from '../../db/index.js';
+import { memoryFeedback, memories, type MemoryFeedback } from '../../db/drizzle/schema-sqlite.js';
+import { eq, and } from 'drizzle-orm';
+import { analyzeResponseForMemoryReferences, mightContainMemoryReferences } from './response-analyzer.js';
+
+interface InjectionRecord {
+  memoryIds: string[];
+  memoryContent: Map<string, string>;
+  injectedAt: Date;
+}
+
+const injectionTracker = new Map<string, InjectionRecord>();
+
+export async function recordInjection(
+  sessionId: string,
+  memoryIds: string[],
+  memoryContent: Map<string, string>
+): Promise<void> {
+  if (!config.feedbackTrackingEnabled) return;
+
+  injectionTracker.set(sessionId, {
+    memoryIds,
+    memoryContent,
+    injectedAt: new Date(),
+  });
+
+  const db = await getDb();
+  if (!db) return;
+
+  try {
+    const now = new Date();
+    const sqliteDb = db as any;
+    for (const memoryId of memoryIds) {
+      await sqliteDb.insert(memoryFeedback).values({
+        memoryId,
+        sessionId,
+        wasInjected: true,
+        wasReferenced: false,
+        referenceCount: 0,
+        retrievalPriorityDelta: 0,
+        injectedAt: now,
+      }).onConflictDoNothing();
+    }
+    logger.debug(`[FeedbackTracker] Recorded injection of ${memoryIds.length} memories for session ${sessionId}`);
+  } catch (error) {
+    logger.error('[FeedbackTracker] Failed to record injection:', error);
+  }
+}
+
+export async function analyzeAndRecordFeedback(
+  sessionId: string,
+  responseText: string
+): Promise<void> {
+  if (!config.feedbackTrackingEnabled) return;
+
+  const injection = injectionTracker.get(sessionId);
+  if (!injection) {
+    logger.debug(`[FeedbackTracker] No injection record for session ${sessionId}`);
+    return;
+  }
+
+  if (!mightContainMemoryReferences(responseText)) {
+    await applyFizzlePenalty(injection.memoryIds);
+    injectionTracker.delete(sessionId);
+    return;
+  }
+
+  const analysis = analyzeResponseForMemoryReferences(
+    responseText,
+    injection.memoryIds,
+    injection.memoryContent
+  );
+
+  const db = await getDb();
+  if (!db) return;
+
+  try {
+    const now = new Date();
+    const sqliteDb = db as any;
+
+    for (const memoryId of injection.memoryIds) {
+      const wasReferenced = analysis.referencedMemoryIds.includes(memoryId);
+      const delta = wasReferenced ? config.feedbackEchoBonus : -config.feedbackFizzlePenalty;
+
+      const existing = await sqliteDb
+        .select()
+        .from(memoryFeedback)
+        .where(and(
+          eq(memoryFeedback.memoryId, memoryId),
+          eq(memoryFeedback.sessionId, sessionId)
+        ))
+        .limit(1);
+
+      if (existing.length > 0) {
+        await sqliteDb
+          .update(memoryFeedback)
+          .set({
+            wasReferenced,
+            referenceCount: wasReferenced ? 1 : 0,
+            retrievalPriorityDelta: delta,
+            referencedAt: wasReferenced ? now : null,
+          })
+          .where(eq(memoryFeedback.id, existing[0].id));
+      }
+
+      await updateRetrievalPriority(memoryId, delta);
+    }
+
+    logger.info(
+      `[FeedbackTracker] Feedback recorded: ${analysis.referenceCount} echoes, ${injection.memoryIds.length - analysis.referenceCount} fizzles`
+    );
+
+    injectionTracker.delete(sessionId);
+  } catch (error) {
+    logger.error('[FeedbackTracker] Failed to record feedback:', error);
+  }
+}
+
+async function applyFizzlePenalty(memoryIds: string[]): Promise<void> {
+  try {
+    for (const memoryId of memoryIds) {
+      await updateRetrievalPriority(memoryId, -config.feedbackFizzlePenalty);
+    }
+    logger.debug(`[FeedbackTracker] Applied fizzle penalty to ${memoryIds.length} memories`);
+  } catch (error) {
+    logger.error('[FeedbackTracker] Failed to apply fizzle penalty:', error);
+  }
+}
+
+export async function updateRetrievalPriority(memoryId: string, delta: number): Promise<void> {
+  const db = await getDb();
+  if (!db) return;
+
+  try {
+    const sqliteDb = db as any;
+    const [memory] = await sqliteDb
+      .select({ retrievalPriority: memories.retrievalPriority })
+      .from(memories)
+      .where(eq(memories.id, memoryId))
+      .limit(1);
+
+    if (!memory) return;
+
+    const currentPriority = memory.retrievalPriority ?? 50;
+    const newPriority = Math.max(0, Math.min(100, currentPriority + delta));
+
+    await sqliteDb
+      .update(memories)
+      .set({ retrievalPriority: newPriority })
+      .where(eq(memories.id, memoryId));
+
+    logger.debug(`[FeedbackTracker] Updated priority for ${memoryId}: ${currentPriority} -> ${newPriority}`);
+  } catch (error) {
+    logger.error('[FeedbackTracker] Failed to update retrieval priority:', error);
+  }
+}
+
+export async function getMemoryFeedbackStats(memoryId: string): Promise<{
+  totalInjections: number;
+  totalReferences: number;
+  echoRate: number;
+  averagePriorityDelta: number;
+}> {
+  const db = await getDb();
+  if (!db) {
+    return { totalInjections: 0, totalReferences: 0, echoRate: 0, averagePriorityDelta: 0 };
+  }
+
+  try {
+    const sqliteDb = db as any;
+    const records = await sqliteDb
+      .select()
+      .from(memoryFeedback)
+      .where(eq(memoryFeedback.memoryId, memoryId));
+
+    const totalInjections = records.filter((r: MemoryFeedback) => r.wasInjected).length;
+    const totalReferences = records.filter((r: MemoryFeedback) => r.wasReferenced).length;
+    const echoRate = totalInjections > 0 ? totalReferences / totalInjections : 0;
+    const totalDelta = records.reduce((sum: number, r: MemoryFeedback) => sum + (r.retrievalPriorityDelta ?? 0), 0);
+    const averagePriorityDelta = records.length > 0 ? totalDelta / records.length : 0;
+
+    return { totalInjections, totalReferences, echoRate, averagePriorityDelta };
+  } catch (error) {
+    logger.error('[FeedbackTracker] Failed to get feedback stats:', error);
+    return { totalInjections: 0, totalReferences: 0, echoRate: 0, averagePriorityDelta: 0 };
+  }
+}
+
+export function cleanupInjectionTracker(maxAgeMs: number = 3600000): void {
+  const now = Date.now();
+  for (const [sessionId, injection] of injectionTracker.entries()) {
+    if (now - injection.injectedAt.getTime() > maxAgeMs) {
+      injectionTracker.delete(sessionId);
+    }
+  }
+}