squish-memory 1.1.5 → 1.2.0

package/core/ingestion/learnings.ts

@@ -0,0 +1,260 @@
+/**
+ * Learnings Module
+ * Agent learnings: success, failure, fix, insight
+ * With auto-linking to similar memories and learnings
+ */
+
+import { randomUUID } from 'crypto';
+import { desc, eq, sql, and } from 'drizzle-orm';
+import { config } from '../../config.js';
+import { getEmbedding } from '../embeddings.js';
+import { getOrCreateProject, requireProject } from '../projects.js';
+import { serializeMetadata, deserializeMetadata } from '../memory/serialization.js';
+import { normalizeTimestamp, prepareEmbedding } from '../lib/utils.js';
+import { logger } from '../logger.js';
+import { getDbClient } from '../lib/db-client.js';
+import { createAssociation } from '../associations.js';
+import { search, type SearchResult } from '../memory/memories.js';
+import { updateAgentPreference } from '../agent-preferences.js';
+
+// Learning type: success, failure, fix, insight
+export type LearningType = 'success' | 'failure' | 'fix' | 'insight';
+
+export interface LearningInput {
+  type: LearningType;
+  content: string;
+  context?: string;
+  action?: string;
+  target?: string;
+  project?: string;
+  memoryId?: string;  // Optional link to a memory
+  autoLink?: boolean;  // Auto-link to similar memories/learnings (default: true)
+}
+
+export interface LearningRecord {
+  id: string;
+  projectId?: string | null;
+  conversationId?: string | null;
+  type: LearningType;
+  action: string;
+  target?: string | null;
+  summary: string;
+  details?: Record<string, unknown> | null;
+  memoryId?: string | null;
+  isImported?: boolean;
+  createdAt?: string | null;
+}
+
+/**
+ * Create a learning and optionally auto-link to similar memories/learnings
+ */
+export async function createLearning(input: LearningInput): Promise<LearningRecord> {
+  const { db, schema } = await getDbClient();
+  const project = await getOrCreateProject(input.project);
+  const embedding = await getEmbedding(input.content);
+  const id = randomUUID();
+
+  const learningType = input.type;
+  const autoLink = input.autoLink !== false; // default true
+
+  // Build details with learning metadata
+  const details = serializeMetadata({
+    learningType: learningType,
+    learningContent: input.content,
+    learningContext: input.context,
+  });
+
+  const embeddingValues = prepareEmbedding(embedding);
+
+  // Insert the learning
+  await db.insert(schema.learnings).values({
+    id,
+    projectId: project?.id ?? null,
+    type: learningType,
+    action: input.action ?? input.content,
+    summary: input.context || input.content,
+    target: input.target ?? null,
+    details,
+    memoryId: input.memoryId ?? null,
+    ...embeddingValues,
+    createdAt: new Date(),
+  });
+
+  // If memoryId provided, create bidirectional link
+  if (input.memoryId) {
+    await createAssociation(
+      input.memoryId,  // from memory
+      id,             // to learning
+      'relates_to',
+      1
+    );
+  }
+
+  // Auto-link to similar memories and learnings (if enabled)
+  if (autoLink) {
+    await autoLinkLearning(id, input.content, project?.id ?? null);
+  }
+
+  // Extract and store agent preferences from this learning
+  await updateAgentPreference(project?.id ?? 'default', input.content, id);
+
+  return {
+    id,
+    projectId: project?.id ?? null,
+    conversationId: null,
+    type: learningType,
+    action: input.action ?? input.content,
+    target: input.target ?? null,
+    summary: input.context || input.content,
+    details: { learningType: learningType, learningContent: input.content, learningContext: input.context },
+    memoryId: input.memoryId ?? null,
+    createdAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * Auto-link a learning to similar memories above threshold
+ */
+async function autoLinkLearning(learningId: string, content: string, projectId: string | null) {
+  const SIMILARITY_THRESHOLD = 0.85;
+  const MAX_LINKS = 5;
+
+  if (!projectId) return;
+
+  try {
+    // Search for similar memories using the search function
+    const similarMemories = await search({
+      query: content,
+      limit: MAX_LINKS,
+    });
+
+    // Filter by similarity threshold
+    const relevantMemories = (similarMemories as SearchResult[]).filter((m: SearchResult) => (m.similarity ?? 0) >= SIMILARITY_THRESHOLD);
+
+    // Create associations with memories
+    for (const mem of relevantMemories) {
+      await createAssociation(
+        learningId,     // from learning
+        mem.id,         // to memory
+        'relates_to',
+        mem.similarity ?? 0.9
+      );
+    }
+
+    logger.info(`Auto-linked learning ${learningId} to ${relevantMemories.length} memories`);
+  } catch (error) {
+    logger.error('Error in auto-linking:', error);
+    // Don't fail learning creation if auto-link fails
+  }
+}
+
+/**
+ * Get learnings for a project
+ */
+export async function getLearnings(projectPath: string, limit: number): Promise<LearningRecord[]> {
+  try {
+    const { db, schema } = await getDbClient();
+    const project = await requireProject(projectPath);
+
+    const rows = await db.select().from(schema.learnings)
+      .where(eq(schema.learnings.projectId, project.id))
+      .orderBy(desc(schema.learnings.createdAt))
+      .limit(limit);
+
+    return rows.map((row: any) => normalizeLearning(row));
+  } catch (error: any) {
+    throw error;
+  }
+}
+
+/**
+ * Get recent learnings
+ */
+export async function getRecentLearnings(projectPath: string, limit: number = 10): Promise<LearningRecord[]> {
+  try {
+    const { db, schema } = await getDbClient();
+    const project = await requireProject(projectPath);
+
+    const rows = await db.select().from(schema.learnings)
+      .where(eq(schema.learnings.projectId, project.id))
+      .orderBy(desc(schema.learnings.createdAt))
+      .limit(limit);
+
+    return rows.map((row: any) => normalizeLearning(row));
+  } catch (error: any) {
+    logger.error('Error getting recent learnings', error);
+    throw error;
+  }
+}
+
+/**
+ * Get learning by ID
+ */
+export async function getLearningById(learningId: string): Promise<LearningRecord | null> {
+  try {
+    const { db, schema } = await getDbClient();
+    const rows = await db.select().from(schema.learnings)
+      .where(eq(schema.learnings.id, learningId))
+      .limit(1);
+
+    if (rows.length === 0) return null;
+    return normalizeLearning(rows[0]);
+  } catch (error: any) {
+    logger.error('Error getting learning', error);
+    throw error;
+  }
+}
+
+/**
+ * Get learnings linked to a specific memory
+ */
+export async function getLearningsForMemory(memoryId: string): Promise<LearningRecord[]> {
+  try {
+    const { db, schema } = await getDbClient();
+    const rows = await db.select().from(schema.learnings)
+      .where(eq(schema.learnings.memoryId, memoryId))
+      .orderBy(desc(schema.learnings.createdAt));
+
+    return rows.map((row: any) => normalizeLearning(row));
+  } catch (error: any) {
+    logger.error('Error getting learnings for memory', error);
+    throw error;
+  }
+}
+
+/**
+ * Delete a learning
+ */
+export async function deleteLearning(learningId: string): Promise<boolean> {
+  try {
+    const { db, schema } = await getDbClient();
+    await db.delete(schema.learnings)
+      .where(eq(schema.learnings.id, learningId));
+    return true;
+  } catch (error: any) {
+    logger.error('Error deleting learning', error);
+    return false;
+  }
+}
+
+function normalizeLearning(row: any): LearningRecord {
+  const details = deserializeMetadata(row.details ?? null);
+  return {
+    id: row.id,
+    projectId: row.projectId ?? row.project_id ?? null,
+    conversationId: row.conversationId ?? row.conversation_id ?? null,
+    type: row.type as LearningType,
+    action: row.action,
+    target: row.target ?? null,
+    summary: row.summary,
+    details,
+    memoryId: row.memoryId ?? row.memory_id ?? null,
+    isImported: row.isImported ?? row.is_imported ?? false,
+    createdAt: normalizeTimestamp(row.createdAt ?? row.created_at),
+  };
+}
+
+// Backwards compatibility aliases
+export const getObservations = getLearnings;
+export const getRecentObservations = getRecentLearnings;
+export const getObservationById = getLearningById;

package/core/ingestion/signal-engine.ts

@@ -0,0 +1,266 @@
+import { createHash } from 'crypto';
+
+export type SignalClassification =
+  | 'discard'
+  | 'session-only'
+  | 'durable-distilled'
+  | 'durable-raw+distilled';
+
+export interface SignalEventInput {
+  toolName: string;
+  toolInput?: Record<string, unknown>;
+  toolResult?: unknown;
+  sessionId?: string | null;
+}
+
+export interface SignalDecision {
+  classification: SignalClassification;
+  reasons: string[];
+  storeRaw: boolean;
+  importance: 'low' | 'medium' | 'high';
+  wakeUpPriority: 'low' | 'medium' | 'high';
+  placeHint: {
+    placeType: 'inbox' | 'ref' | 'wip' | 'sandbox' | 'board' | 'sparks' | 'archive' | null;
+    confidence: number;
+  };
+  graphHint: {
+    shouldEnrich: boolean;
+    entityTerms: string[];
+    reason?: string;
+  };
+  content: string;
+  contentHash: string;
+  estimatedSavings: number;
+}
+
+export interface DistillInput {
+  toolName: string;
+  command?: string;
+  content: string;
+  classification: SignalClassification;
+}
+
+function normalizeContent(toolResult: unknown): string {
+  if (typeof toolResult === 'string') return toolResult;
+  if (toolResult == null) return '';
+  if (typeof toolResult === 'object') {
+    try {
+      return JSON.stringify(toolResult, null, 2);
+    } catch {
+      return String(toolResult);
+    }
+  }
+  return String(toolResult);
+}
+
+function estimateNoiseSavings(raw: string, distilled: string): number {
+  return Math.max(0, raw.length - distilled.length);
+}
+
+function extractEntityTerms(content: string): string[] {
+  const matches = content.match(/\b([A-Z][A-Za-z0-9]+|[a-z]+(?:\.[a-z0-9_-]+)+|[A-Za-z0-9_-]+\.ts)\b/g) ?? [];
+  return [...new Set(matches.filter((term) => term.length > 2))].slice(0, 6);
+}
+
+function inferPlaceHint(toolName: string, normalized: string): SignalDecision['placeHint'] {
+  if (['write', 'edit', 'multiedit'].includes(toolName.toLowerCase()) || /\b(fix|bug|implement|refactor|edited|wrote)\b/.test(normalized)) {
+    return { placeType: 'wip', confidence: 0.88 };
+  }
+  if (/\b(test|assertionerror|traceback|exception|experiment|verify)\b/.test(normalized)) {
+    return { placeType: 'sandbox', confidence: 0.92 };
+  }
+  if (/\b(hypothesis|plan|decided|roadmap|task|todo|investigate)\b/.test(normalized)) {
+    return { placeType: 'board', confidence: 0.74 };
+  }
+  if (/\b(research|docs|pattern|reference|api|architecture)\b/.test(normalized)) {
+    return { placeType: 'ref', confidence: 0.72 };
+  }
+  if (/\b(idea|explore|future|concept)\b/.test(normalized)) {
+    return { placeType: 'sparks', confidence: 0.68 };
+  }
+  return { placeType: null, confidence: 0 };
+}
+
+function inferGraphHint(normalized: string, content: string, classification: SignalClassification): SignalDecision['graphHint'] {
+  if (classification === 'discard' || classification === 'session-only') {
+    return { shouldEnrich: false, entityTerms: [], reason: 'non-durable signal' };
+  }
+
+  const entityTerms = extractEntityTerms(content);
+  const hasStructuredSignal = entityTerms.length > 0 || /\b(fix|decision|preference|error|graph|ranking|entity|relationship)\b/.test(normalized);
+  if (!hasStructuredSignal) {
+    return { shouldEnrich: false, entityTerms: [], reason: 'no graph-worthy entities' };
+  }
+
+  return {
+    shouldEnrich: true,
+    entityTerms,
+    reason: 'durable signal with identifiable entities',
+  };
+}
+
+export function hashSignalContent(content: string): string {
+  return createHash('sha1').update(content).digest('hex');
+}
+
+export function classifySignalEvent(input: SignalEventInput): SignalDecision {
+  const toolName = input.toolName;
+  const command = String(input.toolInput?.command ?? input.toolInput?.cmd ?? '');
+  const content = normalizeContent(input.toolResult);
+  const normalized = `${command}\n${content}`.toLowerCase();
+  const reasons: string[] = [];
+
+  const noisyCommand =
+    /\b(npm|bun|pnpm|yarn)\s+(install|add)\b/.test(normalized) ||
+    /\bdownload(ed|ing)?\b|\bfetch(ed|ing)?\b|\bprogress\b|\bresolved\b/.test(normalized);
+  if (noisyCommand) {
+    const placeHint = inferPlaceHint(toolName, normalized);
+    reasons.push('noise filtered from install/build output');
+    return {
+      classification: 'discard',
+      reasons,
+      storeRaw: false,
+      importance: 'low',
+      wakeUpPriority: 'low',
+      placeHint,
+      graphHint: inferGraphHint(normalized, content, 'discard'),
+      content,
+      contentHash: hashSignalContent(normalized),
+      estimatedSavings: content.length,
+    };
+  }
+
+  if (/\b(current hypothesis|hypothesis:|investigate|trying next)\b/.test(normalized)) {
+    const placeHint = inferPlaceHint(toolName, normalized);
+    reasons.push('active session hypothesis');
+    return {
+      classification: 'session-only',
+      reasons,
+      storeRaw: false,
+      importance: 'medium',
+      wakeUpPriority: 'high',
+      placeHint,
+      graphHint: inferGraphHint(normalized, content, 'session-only'),
+      content,
+      contentHash: hashSignalContent(normalized),
+      estimatedSavings: 0,
+    };
+  }
+
+  if (/\bcorrection:|actually\b|\binstead of\b|\bprefer(s)?\b/.test(normalized)) {
+    const placeHint = inferPlaceHint(toolName, normalized);
+    reasons.push('correction or preference worth durable retention');
+    return {
+      classification: 'durable-distilled',
+      reasons,
+      storeRaw: false,
+      importance: 'high',
+      wakeUpPriority: 'high',
+      placeHint,
+      graphHint: inferGraphHint(normalized, content, 'durable-distilled'),
+      content,
+      contentHash: hashSignalContent(normalized),
+      estimatedSavings: estimateNoiseSavings(content, distillSignalEvent({
+        toolName,
+        command,
+        content,
+        classification: 'durable-distilled',
+      })),
+    };
+  }
+
+  if (
+    /\b(fail|failed|failing|assertionerror|stack trace|traceback|exception)\b/.test(normalized) ||
+    /\b(test|pytest|vitest|jest|bun test)\b/.test(normalized)
+  ) {
+    const placeHint = inferPlaceHint(toolName, normalized);
+    reasons.push('test or debugging signal with nuance-sensitive output');
+    return {
+      classification: 'durable-raw+distilled',
+      reasons,
+      storeRaw: true,
+      importance: 'high',
+      wakeUpPriority: 'high',
+      placeHint,
+      graphHint: inferGraphHint(normalized, content, 'durable-raw+distilled'),
+      content,
+      contentHash: hashSignalContent(normalized),
+      estimatedSavings: estimateNoiseSavings(content, distillSignalEvent({
+        toolName,
+        command,
+        content,
+        classification: 'durable-raw+distilled',
+      })),
+    };
+  }
+
+  if (['write', 'edit', 'multiedit'].includes(toolName.toLowerCase())) {
+    const placeHint = inferPlaceHint(toolName, normalized);
+    reasons.push('meaningful file modification');
+    return {
+      classification: 'durable-distilled',
+      reasons,
+      storeRaw: false,
+      importance: 'medium',
+      wakeUpPriority: 'medium',
+      placeHint,
+      graphHint: inferGraphHint(normalized, content, 'durable-distilled'),
+      content,
+      contentHash: hashSignalContent(normalized),
+      estimatedSavings: estimateNoiseSavings(content, distillSignalEvent({
+        toolName,
+        command,
+        content,
+        classification: 'durable-distilled',
+      })),
+    };
+  }
+
+  reasons.push('session-scoped operational context');
+  const placeHint = inferPlaceHint(toolName, normalized);
+  return {
+    classification: 'session-only',
+    reasons,
+    storeRaw: false,
+    importance: 'low',
+    wakeUpPriority: 'low',
+    placeHint,
+    graphHint: inferGraphHint(normalized, content, 'session-only'),
+    content,
+    contentHash: hashSignalContent(normalized),
+    estimatedSavings: 0,
+  };
+}
+
+export function distillSignalEvent(input: DistillInput): string {
+  const lines = input.content
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+
+  const important = lines.filter((line) =>
+    /\b(fail|error|exception|traceback|assert|stack|correction|prefer|decision|fix|edited|wrote)\b/i.test(line)
+  );
+
+  const selected = important.length > 0 ? important : lines.slice(0, 4);
+  const cleaned = selected.filter(
+    (line) => !/\b(pass(ing)?|resolved|download(ed|ing)?|progress|packages? added)\b/i.test(line)
+  );
+
+  if (/assertionerror|traceback|exception|stack/i.test(input.content) && !cleaned.some((line) => /assertionerror|traceback|exception|stack/i.test(line))) {
+    const extra = lines.find((line) => /assertionerror|traceback|exception|stack/i.test(line));
+    if (extra) cleaned.push(extra);
+  }
+
+  return cleaned.slice(0, 6).join('\n');
+}
+
+export function shouldReturnRawFallback(input: {
+  query: string;
+  hasRawFallback: boolean;
+  nuanceSuppressed: boolean;
+}): boolean {
+  if (!input.hasRawFallback) return false;
+  if (input.nuanceSuppressed) return true;
+  return /\b(raw|original|full|stack trace|traceback|stderr|stdout|rewind)\b/i.test(input.query);
+}