squish-memory 1.1.5 → 1.2.0

package/core/associations.ts

@@ -0,0 +1,363 @@
+/**
+ * Memory Association Graph (Waypoint Graph)
+ * Tracks co-occurrence and relationships between memories
+ */
+
+import { eq, and, or, desc, inArray, sql } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { getSchema } from '../db/schema.js';
+import { logger } from './logger.js';
+
+export type AssociationType = 'co_occurred' | 'supersedes' | 'contradicts' | 'supports' | 'relates_to' | 'duplicate' | 'merged';
+
+/**
+ * Create or update an association between two memories
+ */
+export async function createAssociation(
+  fromMemoryId: string,
+  toMemoryId: string,
+  type: AssociationType,
+  weight: number = 1
+): Promise<void> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Check if association already exists
+    const existing = await (db as any)
+      .select()
+      .from(schema.memoryAssociations)
+      .where(
+        and(
+          eq(schema.memoryAssociations.fromMemoryId, fromMemoryId),
+          eq(schema.memoryAssociations.toMemoryId, toMemoryId)
+        )
+      )
+      .limit(1);
+
+    if (existing.length > 0) {
+      // Update weight and coactivation count
+      await (db as any)
+        .update(schema.memoryAssociations)
+        .set({
+          weight: existing[0].weight + weight,
+          coactivationCount: existing[0].coactivationCount + 1,
+          lastCoactivatedAt: new Date(),
+        })
+        .where(eq(schema.memoryAssociations.id, existing[0].id));
+    } else {
+      // Create new association
+      await (db as any).insert(schema.memoryAssociations).values({
+        fromMemoryId,
+        toMemoryId,
+        associationType: type,
+        weight,
+        coactivationCount: 1,
+        lastCoactivatedAt: new Date(),
+      });
+    }
+  } catch (error) {
+    logger.error('Error creating association', error);
+  }
+}
+
+/**
+ * Auto-link memories that share entities
+ * Called after storing a memory to link it to related memories
+ */
+export async function autoLinkByEntities(
+  newMemoryId: string,
+  entityNames: string[],
+  projectId: string
+): Promise<number> {
+  if (entityNames.length === 0) return 0;
+
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Find existing memories that contain any of these entity names
+    // Use simple LIKE query for matching
+    const conditions = entityNames.slice(0, 5).map(name =>
+      sql<boolean>`LOWER(m.content) LIKE ${'%' + name + '%'}`
+    );
+
+    const existing = await (db as any)
+      .select({
+        id: schema.memories.id,
+        content: schema.memories.content
+      })
+      .from(schema.memories)
+      .where(
+        and(
+          eq(schema.memories.projectId, projectId),
+          sql`(${conditions.join(' OR ')})`,
+          sql`id != ${newMemoryId}`
+        )
+      )
+      .limit(10);
+
+    // Create associations for matching memories
+    let linked = 0;
+    for (const mem of existing) {
+      try {
+        await createAssociation(newMemoryId, mem.id, 'relates_to', 0.5);
+        linked++;
+      } catch (e) {
+        // Skip duplicates
+      }
+    }
+
+    return linked;
+  } catch (error) {
+    logger.error('Error auto-linking by entities', error);
+    return 0;
+  }
+}
+export async function trackCoactivation(memoryIds: string[]): Promise<void> {
+  if (memoryIds.length < 2) return;
+
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+    const now = new Date();
+
+    // Generate all pairs
+    const pairs: Array<{ from: string; to: string }> = [];
+    for (let i = 0; i < memoryIds.length; i++) {
+      for (let j = i + 1; j < memoryIds.length; j++) {
+        pairs.push({ from: memoryIds[i], to: memoryIds[j] });
+        pairs.push({ from: memoryIds[j], to: memoryIds[i] }); // Bidirectional
+      }
+    }
+
+    if (pairs.length === 0) return;
+
+    // Batch check existing associations with single SELECT
+    const pairIds = pairs.map(p => ({ from: p.from, to: p.to }));
+
+    // Check which pairs already exist
+    const existingPairs = await (db as any)
+      .select({ fromId: schema.memoryAssociations.fromMemoryId, toId: schema.memoryAssociations.toMemoryId })
+      .from(schema.memoryAssociations)
+      .where(
+        or(
+          ...pairIds.map((p: any) =>
+            and(
+              eq(schema.memoryAssociations.fromMemoryId, p.from),
+              eq(schema.memoryAssociations.toMemoryId, p.to)
+            )
+          )
+        )
+      );
+
+    const existingMap = new Set(
+      existingPairs.map((p: any) => `${p.fromId}:${p.toId}`)
+    );
+
+    // Separate into new pairs and existing pairs
+    const newPairs: any[] = [];
+    const existingPairsToUpdate: string[] = [];
+
+    for (const pair of pairs) {
+      const key = `${pair.from}:${pair.to}`;
+      if (existingMap.has(key)) {
+        existingPairsToUpdate.push(key);
+      } else {
+        newPairs.push({
+          fromMemoryId: pair.from,
+          toMemoryId: pair.to,
+          associationType: 'co_occurred',
+          weight: 1,
+          coactivationCount: 1,
+          lastCoactivatedAt: now,
+        });
+      }
+    }
+
+    // Bulk insert new associations
+    if (newPairs.length > 0) {
+      const BATCH_SIZE = 100;
+      for (let i = 0; i < newPairs.length; i += BATCH_SIZE) {
+        const batch = newPairs.slice(i, i + BATCH_SIZE);
+        try {
+          // For PostgreSQL with ON CONFLICT support
+          if ((db as any).insert && (db as any).onConflict) {
+            await (db as any)
+              .insert(schema.memoryAssociations)
+              .values(batch)
+                .onConflict({
+                 target: [schema.memoryAssociations.fromMemoryId, schema.memoryAssociations.toMemoryId],
+                    set: {
+                   weight: sql.raw('EXCLUDED.weight'),
+                   coactivationCount: sql.raw('EXCLUDED.coactivation_count'),
+                   lastCoactivatedAt: now,
+                 },
+               })
+              .catch(() => {
+                // Fallback for SQLite
+                return (db as any).insert(schema.memoryAssociations).values(batch);
+              });
+          } else {
+            // Direct insert for SQLite
+            await (db as any).insert(schema.memoryAssociations).values(batch);
+          }
+        } catch (error) {
+          logger.error('Error inserting batch of associations', { batchSize: batch.length, error });
+        }
+      }
+    }
+
+    // Bulk update existing associations
+    if (existingPairsToUpdate.length > 0) {
+      const BATCH_SIZE = 100;
+      for (let i = 0; i < existingPairsToUpdate.length; i += BATCH_SIZE) {
+        const batch = existingPairsToUpdate.slice(i, i + BATCH_SIZE);
+
+        // Extract from/to pairs for this batch
+        const batchPairs = batch.map(key => {
+          const [from, to] = key.split(':');
+          return { from, to };
+        });
+
+        try {
+          for (const pair of batchPairs) {
+            await (db as any)
+              .update(schema.memoryAssociations)
+              .set({
+                weight: sql`${schema.memoryAssociations.weight} + 1`,
+                coactivationCount: sql`${schema.memoryAssociations.coactivationCount} + 1`,
+                lastCoactivatedAt: now,
+              })
+              .where(
+                and(
+                  eq(schema.memoryAssociations.fromMemoryId, pair.from),
+                  eq(schema.memoryAssociations.toMemoryId, pair.to)
+                )
+              );
+          }
+        } catch (error) {
+          logger.error('Error updating batch of associations', { batchSize: batch.length, error });
+        }
+      }
+    }
+
+    logger.debug('Coactivation tracked', {
+      totalPairs: pairs.length,
+      newAssociations: newPairs.length,
+      updatedAssociations: existingPairsToUpdate.length,
+    });
+  } catch (error) {
+    logger.error('Error tracking coactivation', error);
+  }
+}
+
+/**
+ * Get related memories via the association graph
+ */
+export async function getRelatedMemories(
+  memoryId: string,
+  limit: number = 10
+): Promise<any[]> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    // Get all associated memories, sorted by weight
+    const associations = await (db as any)
+      .select()
+      .from(schema.memoryAssociations)
+      .where(
+        or(
+          eq(schema.memoryAssociations.fromMemoryId, memoryId),
+          eq(schema.memoryAssociations.toMemoryId, memoryId)
+        )
+      )
+      .orderBy(desc(schema.memoryAssociations.weight))
+      .limit(limit);
+
+    const relatedIds = associations.map((a: any) =>
+      a.fromMemoryId === memoryId ? a.toMemoryId : a.fromMemoryId
+    );
+
+    if (relatedIds.length === 0) return [];
+
+    // Fetch the actual memories
+    return await (db as any)
+      .select()
+      .from(schema.memories)
+      .where(inArray(schema.memories.id, relatedIds));
+  } catch (error) {
+    logger.error('Error getting related memories', error);
+    return [];
+  }
+}
+
+
+
+/**
+ * Prune weak associations (weight < threshold)
+ */
+export async function pruneWeakAssociations(weightThreshold: number = 5): Promise<number> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const result = await (db as any)
+      .delete(schema.memoryAssociations)
+      .where(schema.memoryAssociations.weight as any <= weightThreshold);
+
+    return result?.rowCount || 0;
+  } catch (error) {
+    logger.error('Error pruning weak associations', error);
+    return 0;
+  }
+}
+
+/**
+ * Get association statistics
+ */
+export async function getAssociationStats(): Promise<{
+  totalAssociations: number;
+  byType: Record<string, number>;
+  avgWeight: number;
+  maxWeight: number;
+}> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    const associations = await (db as any)
+      .select()
+      .from(schema.memoryAssociations);
+
+    const stats = {
+      totalAssociations: associations.length,
+      byType: {} as Record<string, number>,
+      avgWeight: 0,
+      maxWeight: 0,
+    };
+
+    let totalWeight = 0;
+
+    for (const assoc of associations) {
+      stats.byType[assoc.associationType] = (stats.byType[assoc.associationType] || 0) + 1;
+      totalWeight += assoc.weight;
+      if (assoc.weight > stats.maxWeight) stats.maxWeight = assoc.weight;
+    }
+
+    stats.avgWeight = associations.length > 0 ? totalWeight / associations.length : 0;
+
+    return stats;
+  } catch (error) {
+    logger.error('Error getting association stats', error);
+    return {
+      totalAssociations: 0,
+      byType: {},
+      avgWeight: 0,
+      maxWeight: 0,
+    };
+  }
+}
+
+

package/core/beliefs/decay.ts

@@ -0,0 +1,289 @@
+import { getDbClient } from '../lib/db-client.js';
+import { logger } from './logger.js';
+
+/**
+ * Belief Decay Engine
+ * 
+ * Manages confidence decay for beliefs over time
+ * Formula: newConfidence = oldConfidence * (1 - decayRate/100)^(daysSinceLastConfirmation / halfLife)
+ * - Default 30 days half-life (belief confidence halves every 30 days without reinforcement)
+ * - Disputed beliefs decay faster
+ * - Beliefs with more sources decay slower
+ */
+
+const DEFAULT_BELIEF_HALF_LIFE = 30; // days
+const DISPUTE_DECAY_MULTIPLIER = 1.5; // disputes decay 1.5x faster
+const SOURCE_BOOST_THRESHOLD = 3; // 3+ sources = slower decay
+
+interface DecayStats {
+  decayed: number;
+  sourceCountUpdated: number;
+  errors: string[];
+}
+
+export async function applyBeliefDecay(projectId?: string): Promise<DecayStats> {
+  const stats: DecayStats = { decayed: 0, sourceCountUpdated: 0, errors: [] };
+  
+  try {
+    const { raw } = await getDbClient();
+    const sqlite = (raw as any).$client;
+    const isPg = typeof (raw as any).query === 'function';
+
+    if (!sqlite && !isPg) {
+      logger.warn('No database client available for belief decay');
+      return stats;
+    }
+
+    const now = Date.now();
+    
+    if (isPg) {
+      // PostgreSQL version
+      const pg = raw as any;
+      
+      // Get all active beliefs
+      const beliefResult = await pg.query(
+        `SELECT id, confidence, belief_decay_rate, last_confirmed_at, status, source_count 
+         FROM beliefs WHERE project_id = $1 AND status = 'active'`,
+        [projectId]
+      );
+      
+      for (const belief of beliefResult.rows) {
+        try {
+          const lastConfirmed = belief.last_confirmed_at ? new Date(belief.last_confirmed_at).getTime() : new Date(belief.created_at).getTime();
+          const daysSince = Math.max(0, (now - lastConfirmed) / (24 * 60 * 60 * 1000));
+          
+          // Get source count for this belief
+          const sourceResult = await pg.query(
+            `SELECT COUNT(*) as cnt FROM belief_memory_sources WHERE belief_id = $1`,
+            [belief.id]
+          );
+          const sourceCount = parseInt(sourceResult.rows[0]?.cnt ?? 1);
+          
+          // Calculate decay rate (disputes decay faster)
+          const baseDecayRate = belief.belief_decay_rate ?? DEFAULT_BELIEF_HALF_LIFE;
+          const decayMultiplier = belief.status === 'disputed' ? DISPUTE_DECAY_MULTIPLIER : 1;
+          const sourceMultiplier = sourceCount >= SOURCE_BOOST_THRESHOLD ? 0.8 : 1;
+          
+          const effectiveHalfLife = baseDecayRate * decayMultiplier * sourceMultiplier;
+          
+          // Apply exponential decay: confidence = initial * (0.5)^(days/halfLife)
+          const newConfidence = Math.round(
+            belief.confidence * Math.pow(0.5, daysSince / effectiveHalfLife)
+          );
+          
+          if (newConfidence < belief.confidence) {
+            await pg.query(
+              `UPDATE beliefs SET confidence = $1, updated_at = NOW() WHERE id = $2`,
+              [newConfidence, belief.id]
+            );
+            stats.decayed++;
+          }
+          
+          if (sourceCount !== belief.source_count) {
+            await pg.query(
+              `UPDATE beliefs SET source_count = $1 WHERE id = $2`,
+              [sourceCount, belief.id]
+            );
+            stats.sourceCountUpdated++;
+          }
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          stats.errors.push(`Belief ${belief.id}: ${msg}`);
+        }
+      }
+    } else if (sqlite) {
+      // SQLite version
+      const beliefs = sqlite.prepare(`
+        SELECT id, confidence, belief_decay_rate, last_confirmed_at, status, source_count, created_at
+        FROM beliefs WHERE project_id = ? AND status = 'active'
+      `).all(projectId) as any[];
+
+      for (const belief of beliefs) {
+        try {
+          const lastConfirmed = belief.last_confirmed_at 
+            ? belief.last_confirmed_at 
+            : belief.created_at;
+          const daysSince = Math.max(0, (now - lastConfirmed) / (24 * 60 * 60 * 1000));
+          
+          // Get source count
+          const sourceResult = sqlite.prepare(
+            `SELECT COUNT(*) as cnt FROM belief_memory_sources WHERE belief_id = ?`
+          ).get(belief.id) as any;
+          const sourceCount = sourceResult?.cnt ?? 1;
+          
+          // Calculate decay rate
+          const baseDecayRate = belief.belief_decay_rate ?? DEFAULT_BELIEF_HALF_LIFE;
+          const decayMultiplier = belief.status === 'disputed' ? DISPUTE_DECAY_MULTIPLIER : 1;
+          const sourceMultiplier = sourceCount >= SOURCE_BOOST_THRESHOLD ? 0.8 : 1;
+          
+          const effectiveHalfLife = baseDecayRate * decayMultiplier * sourceMultiplier;
+          
+          const newConfidence = Math.round(
+            belief.confidence * Math.pow(0.5, daysSince / effectiveHalfLife)
+          );
+          
+          if (newConfidence < belief.confidence) {
+            sqlite.prepare(`
+              UPDATE beliefs SET confidence = ?, updated_at = ?
+              WHERE id = ?
+            `).run(newConfidence, Math.floor(now / 1000), belief.id);
+            stats.decayed++;
+          }
+          
+          if (sourceCount !== belief.source_count) {
+            sqlite.prepare(`
+              UPDATE beliefs SET source_count = ? WHERE id = ?
+            `).run(sourceCount, belief.id);
+            stats.sourceCountUpdated++;
+          }
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          stats.errors.push(`Belief ${belief.id}: ${msg}`);
+        }
+      }
+    }
+    
+    logger.info('Belief decay applied', stats);
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    logger.error('Belief decay failed', { error: msg });
+    stats.errors.push(msg);
+  }
+  
+  return stats;
+}
+
+/**
+ * Get beliefs that need attention (low confidence, disputed, or stale)
+ */
+export async function getProblemBeliefs(projectId: string, options?: {
+  minDaysSinceConfirmation?: number;
+  confidenceThreshold?: number;
+}): Promise<Array<{
+  id: string;
+  type: string;
+  statement: string;
+  confidence: number;
+  status: string;
+  lastConfirmedAt: Date | null;
+  issue: string;
+}>> {
+  const problems: Array<{
+    id: string;
+    type: string;
+    statement: string;
+    confidence: number;
+    status: string;
+    lastConfirmedAt: Date | null;
+    issue: string;
+  }> = [];
+  
+  try {
+    const { raw } = await getDbClient();
+    const sqlite = (raw as any).$client;
+    const isPg = typeof (raw as any).query === 'function';
+    
+    const minDays = options?.minDaysSinceConfirmation ?? 30;
+    const confThreshold = options?.confidenceThreshold ?? 20;
+    const now = Date.now();
+    
+    if (isPg) {
+      const pg = raw as any;
+      const result = await pg.query(
+        `SELECT id, belief_type, statement, confidence, status, last_confirmed_at, created_at
+         FROM beliefs WHERE project_id = $1`,
+        [projectId]
+      );
+      
+      for (const belief of result.rows) {
+        const lastConfirmed = belief.last_confirmed_at ? new Date(belief.last_confirmed_at) : new Date(belief.created_at);
+        const daysSince = (now - lastConfirmed.getTime()) / (24 * 60 * 60 * 1000);
+        
+        let issue = '';
+        if (belief.status === 'disputed') issue = 'disputed';
+        else if (belief.confidence < confThreshold) issue = `low confidence (${belief.confidence})`;
+        else if (daysSince > minDays) issue = `stale (${Math.round(daysSince)} days)`;
+        
+        if (issue) {
+          problems.push({
+            id: belief.id,
+            type: belief.belief_type,
+            statement: belief.statement,
+            confidence: belief.confidence,
+            status: belief.status,
+            lastConfirmedAt: belief.last_confirmed_at,
+            issue,
+          });
+        }
+      }
+    } else if (sqlite) {
+      const beliefs = sqlite.prepare(`
+        SELECT id, belief_type, statement, confidence, status, last_confirmed_at, created_at
+        FROM beliefs WHERE project_id = ?
+      `).all(projectId) as any[];
+      
+      for (const belief of beliefs) {
+        const lastConfirmed = belief.last_confirmed_at 
+          ? new Date(belief.last_confirmed_at * 1000) 
+          : new Date(belief.created_at * 1000);
+        const daysSince = (now - lastConfirmed.getTime()) / (24 * 60 * 60 * 1000);
+        
+        let issue = '';
+        if (belief.status === 'disputed') issue = 'disputed';
+        else if (belief.confidence < confThreshold) issue = `low confidence (${belief.confidence})`;
+        else if (daysSince > minDays) issue = `stale (${Math.round(daysSince)} days)`;
+        
+        if (issue) {
+          problems.push({
+            id: belief.id,
+            type: belief.belief_type,
+            statement: belief.statement,
+            confidence: belief.confidence,
+            status: belief.status,
+            lastConfirmedAt: belief.last_confirmed_at ? new Date(belief.last_confirmed_at * 1000) : null,
+            issue,
+          });
+        }
+      }
+    }
+  } catch (error) {
+    logger.error('Failed to get problem beliefs', error);
+  }
+  
+  return problems;
+}
+
+/**
+ * Reinforce a belief (increase confidence when re-referenced)
+ */
+export async function reinforceBelief(beliefId: string, boostAmount: number = 5): Promise<boolean> {
+  try {
+    const { raw } = await getDbClient();
+    const isPg = typeof (raw as any).query === 'function';
+    
+    if (isPg) {
+      await (raw as any).query(
+        `UPDATE beliefs SET 
+          confidence = LEAST(100, confidence + $1),
+          last_confirmed_at = NOW(),
+          updated_at = NOW()
+         WHERE id = $2`,
+        [boostAmount, beliefId]
+      );
+    } else {
+      const sqlite = (raw as any).$client;
+      sqlite.prepare(`
+        UPDATE beliefs SET 
+          confidence = MIN(100, confidence + ?),
+          last_confirmed_at = ?,
+          updated_at = ?
+        WHERE id = ?
+      `).run(boostAmount, Math.floor(Date.now() / 1000), Math.floor(Date.now() / 1000), beliefId);
+    }
+    
+    return true;
+  } catch (error) {
+    logger.error('Failed to reinforce belief', { beliefId, error });
+    return false;
+  }
+}