vellum 0.2.14 → 0.3.2

package/src/memory/fingerprint.ts

@@ -1,20 +0,0 @@
-import { createHash } from 'node:crypto';
-
-/**
- * Compute a scope-salted fingerprint for a memory item.
- *
- * Format: sha256(`${scopeId}|${kind}|${subject}|${statement}`)
- *
- * All writers (memory_save, memory_update, items-extractor, playbook-create,
- * playbook-update, gmail-analyze-style) MUST use this function so the
- * fingerprint scheme stays consistent and deduplication works correctly.
- */
-export function computeMemoryFingerprint(
-  scopeId: string,
-  kind: string,
-  subject: string,
-  statement: string,
-): string {
-  const normalized = `${scopeId}|${kind}|${subject.toLowerCase()}|${statement.toLowerCase()}`;
-  return createHash('sha256').update(normalized).digest('hex');
-}

package/src/memory/indexer.ts

@@ -1,156 +0,0 @@
-import { createHash } from 'crypto';
-import { desc, eq } from 'drizzle-orm';
-import type { MemoryConfig } from '../config/types.js';
-import { getLogger } from '../util/logger.js';
-import { getMemoryCheckpoint, setMemoryCheckpoint } from './checkpoints.js';
-import { getDb } from './db.js';
-import { enqueueMemoryJob, enqueueResolvePendingConflictsForMessageJob } from './jobs-store.js';
-import { extractTextFromStoredMessageContent } from './message-content.js';
-import { segmentText } from './segmenter.js';
-import { memorySegments } from './schema.js';
-
-const log = getLogger('memory-indexer');
-const SUMMARY_JOB_CHECKPOINT_KEY = 'memory:summary_jobs:last_scheduled_at';
-const SUMMARY_SCHEDULE_INTERVAL_MS = 6 * 60 * 60 * 1000;
-
-export interface IndexMessageInput {
-  messageId: string;
-  conversationId: string;
-  role: string;
-  content: string;
-  createdAt: number;
-  scopeId?: string;
-}
-
-export interface IndexMessageResult {
-  indexedSegments: number;
-  enqueuedJobs: number;
-}
-
-export function indexMessageNow(
-  input: IndexMessageInput,
-  config: MemoryConfig,
-): IndexMessageResult {
-  if (!config.enabled) return { indexedSegments: 0, enqueuedJobs: 0 };
-
-  const text = extractTextFromStoredMessageContent(input.content);
-  if (text.length === 0) {
-    enqueueMemoryJob('build_conversation_summary', { conversationId: input.conversationId });
-    return { indexedSegments: 0, enqueuedJobs: 1 };
-  }
-
-  const db = getDb();
-  const now = Date.now();
-  const segments = segmentText(
-    text,
-    config.segmentation.targetTokens,
-    config.segmentation.overlapTokens,
-  );
-  const shouldExtract =
-    input.role === 'user' ||
-    (input.role === 'assistant' && config.extraction.extractFromAssistant);
-  const shouldResolveConflicts = input.role === 'user' && config.conflicts.enabled;
-
-  // Wrap all segment inserts and job enqueues in a single transaction so they
-  // either all succeed or all roll back, preventing partial/orphaned state.
-  let skippedEmbedJobs = 0;
-  db.transaction((tx) => {
-    for (const segment of segments) {
-      const segmentId = buildSegmentId(input.messageId, segment.segmentIndex);
-      const hash = createHash('sha256').update(segment.text).digest('hex');
-
-      // Check if this segment already exists with the same content hash
-      const existing = tx.select({ contentHash: memorySegments.contentHash })
-        .from(memorySegments)
-        .where(eq(memorySegments.id, segmentId))
-        .get();
-
-      tx.insert(memorySegments).values({
-        id: segmentId,
-        messageId: input.messageId,
-        conversationId: input.conversationId,
-        role: input.role,
-        segmentIndex: segment.segmentIndex,
-        text: segment.text,
-        tokenEstimate: segment.tokenEstimate,
-        scopeId: input.scopeId ?? 'default',
-        contentHash: hash,
-        createdAt: input.createdAt,
-        updatedAt: now,
-      }).onConflictDoUpdate({
-        target: memorySegments.id,
-        set: {
-          text: segment.text,
-          tokenEstimate: segment.tokenEstimate,
-          scopeId: input.scopeId ?? 'default',
-          contentHash: hash,
-          updatedAt: now,
-        },
-      }).run();
-
-      if (existing?.contentHash === hash) {
-        skippedEmbedJobs++;
-      } else {
-        enqueueMemoryJob('embed_segment', { segmentId }, Date.now(), tx);
-      }
-    }
-
-    if (shouldExtract) {
-      enqueueMemoryJob('extract_items', { messageId: input.messageId, scopeId: input.scopeId ?? 'default' }, Date.now(), tx);
-    }
-    if (shouldResolveConflicts) {
-      enqueueResolvePendingConflictsForMessageJob(input.messageId, input.scopeId ?? 'default', tx);
-    }
-    enqueueMemoryJob('build_conversation_summary', { conversationId: input.conversationId }, Date.now(), tx);
-  });
-
-  if (skippedEmbedJobs > 0) {
-    log.debug(`Skipped ${skippedEmbedJobs}/${segments.length} embed_segment jobs (content unchanged)`);
-  }
-
-  enqueueSummaryRollupJobsIfDue();
-
-  const enqueuedJobs = (segments.length - skippedEmbedJobs) + (shouldExtract ? 2 : 1) + (shouldResolveConflicts ? 1 : 0);
-  return {
-    indexedSegments: segments.length,
-    enqueuedJobs,
-  };
-}
-
-export function enqueueBackfillJob(force = false): string {
-  return enqueueMemoryJob('backfill', { force });
-}
-
-export function enqueueRebuildIndexJob(): string {
-  return enqueueMemoryJob('rebuild_index', {});
-}
-
-export function getRecentSegmentsForConversation(
-  conversationId: string,
-  limit: number,
-): Array<typeof memorySegments.$inferSelect> {
-  const db = getDb();
-  return db
-    .select()
-    .from(memorySegments)
-    .where(eq(memorySegments.conversationId, conversationId))
-    .orderBy(desc(memorySegments.createdAt))
-    .limit(limit)
-    .all();
-}
-
-function enqueueSummaryRollupJobsIfDue(): void {
-  const now = Date.now();
-  const raw = getMemoryCheckpoint(SUMMARY_JOB_CHECKPOINT_KEY);
-  const last = raw ? Number.parseInt(raw, 10) : 0;
-  if (Number.isFinite(last) && now - last < SUMMARY_SCHEDULE_INTERVAL_MS) return;
-
-  enqueueMemoryJob('refresh_weekly_summary', {});
-  enqueueMemoryJob('refresh_monthly_summary', {});
-  setMemoryCheckpoint(SUMMARY_JOB_CHECKPOINT_KEY, String(now));
-  log.debug('Scheduled periodic global summary jobs');
-}
-
-function buildSegmentId(messageId: string, segmentIndex: number): string {
-  return `${messageId}:${segmentIndex}`;
-}

package/src/memory/items-extractor.ts

@@ -1,461 +0,0 @@
-import Anthropic from '@anthropic-ai/sdk';
-import { and, eq, sql } from 'drizzle-orm';
-import { v4 as uuid } from 'uuid';
-import { getConfig } from '../config/loader.js';
-import type { MemoryExtractionConfig } from '../config/types.js';
-import { getLogger } from '../util/logger.js';
-import { truncate } from '../util/truncate.js';
-import { computeMemoryFingerprint } from './fingerprint.js';
-import { enqueueMemoryJob } from './jobs-store.js';
-import { extractTextFromStoredMessageContent } from './message-content.js';
-import { getDb } from './db.js';
-import { memoryItemConflicts, memoryItems, memoryItemSources, messages } from './schema.js';
-
-const log = getLogger('memory-items-extractor');
-
-export type MemoryItemKind =
-  | 'preference'
-  | 'profile'
-  | 'project'
-  | 'decision'
-  | 'todo'
-  | 'fact'
-  | 'constraint'
-  | 'relationship'
-  | 'event'
-  | 'opinion'
-  | 'instruction'
-  | 'style';
-
-interface ExtractedItem {
-  kind: MemoryItemKind;
-  subject: string;
-  statement: string;
-  confidence: number;
-  importance: number;
-  fingerprint: string;
-}
-
-const VALID_KINDS = new Set<string>([
-  'preference', 'profile', 'project', 'decision', 'todo',
-  'fact', 'constraint', 'relationship', 'event', 'opinion', 'instruction', 'style',
-]);
-
-const SUPERSEDE_KINDS = new Set<MemoryItemKind>(['decision', 'preference', 'constraint']);
-
-// ── Semantic density gating ────────────────────────────────────────────
-// Skip messages that are too short or consist of low-value filler.
-
-const LOW_VALUE_PATTERNS = new Set([
-  'ok', 'okay', 'k', 'sure', 'yes', 'no', 'yep', 'nope', 'yeah', 'nah',
-  'thanks', 'thank you', 'ty', 'thx', 'thanks!', 'thank you!',
-  'got it', 'understood', 'makes sense', 'sounds good', 'sounds great',
-  'cool', 'nice', 'great', 'awesome', 'perfect', 'done', 'lgtm',
-  'agreed', 'right', 'correct', 'exactly', 'yup', 'ack',
-  'hm', 'hmm', 'hmmm', 'ah', 'oh', 'i see',
-]);
-
-function hasSemanticDensity(text: string): boolean {
-  const trimmed = text.trim();
-  if (trimmed.length < 15) return false;
-  const lower = trimmed.toLowerCase().replace(/[.!?,;:\s]+$/, '');
-  if (LOW_VALUE_PATTERNS.has(lower)) return false;
-  // Very short messages with only 1-2 words are typically not memorable
-  const wordCount = trimmed.split(/\s+/).length;
-  if (wordCount <= 2) return false;
-  return true;
-}
-
-// ── LLM-powered extraction ────────────────────────────────────────────
-
-const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction system. Given a message from a conversation, extract structured memory items that would be valuable to remember for future interactions.
-
-Extract items in these categories:
-- preference: User likes, dislikes, preferred approaches/tools/styles
-- profile: Personal info (name, role, location, timezone, background)
-- project: Project names, repos, tech stacks, architecture details
-- decision: Choices made, approaches selected, trade-offs resolved
-- todo: Action items, follow-ups, things to do later
-- fact: Notable facts, definitions, technical details worth remembering
-- constraint: Rules, requirements, things that must/must not be done
-- relationship: Connections between people, teams, projects, systems
-- event: Deadlines, milestones, meetings, releases, dates
-- opinion: Viewpoints, assessments, evaluations of tools/approaches
-- instruction: Explicit directives on how the assistant should behave
-- style: Communication style patterns — writing tone, formatting habits, vocabulary choices, greeting/sign-off conventions
-
-For each item, provide:
-- kind: One of the categories above
-- subject: A short label (2-8 words) identifying what this is about
-- statement: The full factual statement to remember (1-2 sentences)
-- confidence: How confident you are this is accurate (0.0-1.0)
-- importance: How valuable this is to remember (0.0-1.0)
-  - 1.0: Explicit user instructions about assistant behavior
-  - 0.8-0.9: Personal facts, strong preferences, key decisions
-  - 0.6-0.7: Project details, constraints, opinions
-  - 0.3-0.5: Contextual details, minor preferences
-
-Rules:
-- Only extract genuinely memorable information. Skip pleasantries, filler, and transient discussion.
-- Do NOT extract information about what tools the assistant used or what files it read — only extract substantive facts about the user, their projects, and their preferences.
-- Prefer fewer high-quality items over many low-quality ones.
-- If the message contains no memorable information, return an empty array.`;
-
-interface LLMExtractedItem {
-  kind: string;
-  subject: string;
-  statement: string;
-  confidence: number;
-  importance: number;
-}
-
-async function extractItemsWithLLM(
-  text: string,
-  extractionConfig: MemoryExtractionConfig,
-  scopeId: string,
-): Promise<ExtractedItem[]> {
-  const config = getConfig();
-  const apiKey = config.apiKeys.anthropic ?? process.env.ANTHROPIC_API_KEY;
-  if (!apiKey) {
-    log.debug('No Anthropic API key available for LLM extraction, falling back to pattern-based');
-    return extractItemsPatternBased(text, scopeId);
-  }
-
-  try {
-    const client = new Anthropic({ apiKey });
-    const abortController = new AbortController();
-    let timer: ReturnType<typeof setTimeout>;
-    const apiCall = client.messages.create({
-      model: extractionConfig.model,
-      max_tokens: 1024,
-      system: EXTRACTION_SYSTEM_PROMPT,
-      tools: [{
-        name: 'store_memory_items',
-        description: 'Store extracted memory items from the message',
-        input_schema: {
-          type: 'object' as const,
-          properties: {
-            items: {
-              type: 'array',
-              items: {
-                type: 'object',
-                properties: {
-                  kind: {
-                    type: 'string',
-                    enum: [...VALID_KINDS],
-                    description: 'Category of memory item',
-                  },
-                  subject: {
-                    type: 'string',
-                    description: 'Short label (2-8 words) for what this is about',
-                  },
-                  statement: {
-                    type: 'string',
-                    description: 'Full factual statement to remember (1-2 sentences)',
-                  },
-                  confidence: {
-                    type: 'number',
-                    description: 'Confidence that this is accurate (0.0-1.0)',
-                  },
-                  importance: {
-                    type: 'number',
-                    description: 'How valuable this is to remember (0.0-1.0)',
-                  },
-                },
-                required: ['kind', 'subject', 'statement', 'confidence', 'importance'],
-              },
-            },
-          },
-          required: ['items'],
-        },
-      }],
-      tool_choice: { type: 'tool' as const, name: 'store_memory_items' },
-      messages: [{ role: 'user' as const, content: text }],
-    }, { signal: abortController.signal });
-    // Swallow the abort rejection that fires when the timeout wins the race
-    apiCall.catch(() => {});
-    const response = await Promise.race([
-      apiCall.finally(() => clearTimeout(timer)),
-      new Promise<never>((_, reject) => {
-        timer = setTimeout(() => {
-          abortController.abort();
-          reject(new Error('LLM extraction timeout'));
-        }, 15000);
-      }),
-    ]);
-
-    const toolBlock = response.content.find((b) => b.type === 'tool_use');
-    if (!toolBlock || toolBlock.type !== 'tool_use') {
-      log.warn('No tool_use block in LLM extraction response, falling back to pattern-based');
-      return extractItemsPatternBased(text, scopeId);
-    }
-
-    const input = toolBlock.input as { items?: LLMExtractedItem[] };
-    if (!Array.isArray(input.items)) {
-      log.warn('Invalid items in LLM extraction response, falling back to pattern-based');
-      return extractItemsPatternBased(text, scopeId);
-    }
-
-    const items: ExtractedItem[] = [];
-    for (const raw of input.items) {
-      if (!VALID_KINDS.has(raw.kind)) continue;
-      if (!raw.subject || !raw.statement) continue;
-      const subject = truncate(String(raw.subject), 80, '');
-      const statement = truncate(String(raw.statement), 500, '');
-      const confidence = clamp(parseScore(raw.confidence, 0.5), 0, 1);
-      const importance = clamp(parseScore(raw.importance, 0.5), 0, 1);
-      const fingerprint = computeMemoryFingerprint(scopeId, raw.kind, subject, statement);
-      items.push({
-        kind: raw.kind as MemoryItemKind,
-        subject,
-        statement,
-        confidence,
-        importance,
-        fingerprint,
-      });
-    }
-
-    return deduplicateItems(items);
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    log.warn({ err: message }, 'LLM extraction failed, falling back to pattern-based');
-    return extractItemsPatternBased(text, scopeId);
-  }
-}
-
-// ── Public API ─────────────────────────────────────────────────────────
-
-export async function extractAndUpsertMemoryItemsForMessage(messageId: string, scopeId?: string): Promise<number> {
-  const db = getDb();
-  const message = db
-    .select({
-      id: messages.id,
-      role: messages.role,
-      content: messages.content,
-      createdAt: messages.createdAt,
-    })
-    .from(messages)
-    .where(eq(messages.id, messageId))
-    .get();
-
-  if (!message) return 0;
-
-  const text = extractTextFromStoredMessageContent(message.content);
-  if (!hasSemanticDensity(text)) {
-    log.debug({ messageId }, 'Skipping extraction — message lacks semantic density');
-    return 0;
-  }
-
-  const config = getConfig();
-  const extractionConfig = config.memory.extraction;
-  const effectiveScopeId = scopeId ?? 'default';
-  const extracted = extractionConfig.useLLM
-    ? await extractItemsWithLLM(text, extractionConfig, effectiveScopeId)
-    : extractItemsPatternBased(text, effectiveScopeId);
-
-  if (extracted.length === 0) return 0;
-
-  // Determine verification state from message role
-  const verificationState = message.role === 'user' ? 'user_reported' : 'assistant_inferred';
-
-  let upserted = 0;
-  for (const item of extracted) {
-    const now = Date.now();
-    const seenAt = message.createdAt;
-    const existing = db
-      .select()
-      .from(memoryItems)
-      .where(and(
-        eq(memoryItems.fingerprint, item.fingerprint),
-        eq(memoryItems.scopeId, effectiveScopeId),
-      ))
-      .get();
-
-    let memoryItemId: string;
-    let effectiveStatus: string = 'active';
-    if (existing) {
-      memoryItemId = existing.id;
-      // Promote verification state if re-seen from a more trusted source
-      const promotedState =
-        existing.verificationState === 'assistant_inferred' && verificationState === 'user_reported'
-          ? 'user_reported'
-          : existing.verificationState;
-      // Preserve pending_clarification if this item has an unresolved conflict
-      effectiveStatus = existing.status === 'pending_clarification' && hasPendingConflict(existing.id)
-        ? 'pending_clarification'
-        : 'active';
-      db.update(memoryItems)
-        .set({
-          status: effectiveStatus,
-          confidence: Math.max(existing.confidence, item.confidence),
-          importance: Math.max(existing.importance ?? 0, item.importance),
-          lastSeenAt: Math.max(existing.lastSeenAt, seenAt),
-          verificationState: promotedState,
-        })
-        .where(eq(memoryItems.id, existing.id))
-        .run();
-    } else {
-      memoryItemId = uuid();
-      db.insert(memoryItems).values({
-        id: memoryItemId,
-        kind: item.kind,
-        subject: item.subject,
-        statement: item.statement,
-        status: 'active',
-        confidence: item.confidence,
-        importance: item.importance,
-        fingerprint: item.fingerprint,
-        verificationState,
-        scopeId: effectiveScopeId,
-        firstSeenAt: message.createdAt,
-        lastSeenAt: seenAt,
-        lastUsedAt: null,
-      }).run();
-      upserted += 1;
-    }
-
-    // Only supersede other items when this item is active — a
-    // pending_clarification item should not demote the existing active
-    // item, since that would leave no retrievable memory until manual
-    // conflict resolution occurs.
-    if (SUPERSEDE_KINDS.has(item.kind) && effectiveStatus === 'active') {
-      db.update(memoryItems)
-        .set({ status: 'superseded' })
-        .where(and(
-          eq(memoryItems.kind, item.kind),
-          eq(memoryItems.subject, item.subject),
-          eq(memoryItems.status, 'active'),
-          eq(memoryItems.scopeId, effectiveScopeId),
-          sql`${memoryItems.id} <> ${memoryItemId}`,
-        ))
-        .run();
-    }
-
-    db.insert(memoryItemSources).values({
-      memoryItemId,
-      messageId,
-      evidence: truncate(item.statement, 500, ''),
-      createdAt: now,
-    }).onConflictDoNothing().run();
-
-    enqueueMemoryJob('embed_item', { itemId: memoryItemId });
-
-    // Queue contradiction check for newly inserted items
-    if (!existing) {
-      enqueueMemoryJob('check_contradictions', { itemId: memoryItemId });
-    }
-  }
-
-  log.debug({ messageId, extracted: extracted.length, upserted }, 'Extracted memory items from message');
-  return upserted;
-}
-
-// ── Pattern-based extraction (fallback) ────────────────────────────────
-
-function extractItemsPatternBased(text: string, scopeId: string = 'default'): ExtractedItem[] {
-  const sentences = text
-    .split(/[\n\r]+|(?<=[.!?])\s+/)
-    .map((s) => s.trim())
-    .filter((s) => s.length >= 20 && s.length <= 500);
-
-  const items: ExtractedItem[] = [];
-  for (const sentence of sentences) {
-    const lower = sentence.toLowerCase();
-    const classification = classifySentence(lower);
-    if (!classification) continue;
-    const subject = inferSubject(sentence, classification.kind);
-    const statement = sentence.replace(/\s+/g, ' ').trim();
-    const fingerprint = computeMemoryFingerprint(scopeId, classification.kind, subject, statement);
-    items.push({
-      kind: classification.kind,
-      subject,
-      statement,
-      confidence: classification.confidence,
-      importance: classification.importance,
-      fingerprint,
-    });
-  }
-
-  return deduplicateItems(items);
-}
-
-function classifySentence(lower: string): { kind: MemoryItemKind; confidence: number; importance: number } | null {
-  if (includesAny(lower, ['i prefer', 'prefer to', 'favorite', 'i like', 'i dislike'])) {
-    return { kind: 'preference', confidence: 0.78, importance: 0.7 };
-  }
-  if (includesAny(lower, ['my name is', 'i am ', 'i work as', 'i live in', 'timezone'])) {
-    return { kind: 'profile', confidence: 0.72, importance: 0.8 };
-  }
-  if (includesAny(lower, ['project', 'repository', 'repo', 'codebase'])) {
-    return { kind: 'project', confidence: 0.68, importance: 0.6 };
-  }
-  if (includesAny(lower, ['we decided', 'decision', 'chosen approach', 'we will'])) {
-    return { kind: 'decision', confidence: 0.75, importance: 0.7 };
-  }
-  if (includesAny(lower, ['todo', 'to do', 'next step', 'follow up', 'need to'])) {
-    return { kind: 'todo', confidence: 0.74, importance: 0.6 };
-  }
-  if (includesAny(lower, ['must', 'cannot', 'should not', 'constraint', 'requirement'])) {
-    return { kind: 'constraint', confidence: 0.7, importance: 0.7 };
-  }
-  if (includesAny(lower, ['remember', 'important', 'fact', 'noted'])) {
-    return { kind: 'fact', confidence: 0.62, importance: 0.5 };
-  }
-  return null;
-}
-
-function inferSubject(sentence: string, kind: MemoryItemKind): string {
-  const trimmed = sentence.trim();
-  if (kind === 'project') {
-    const match = trimmed.match(/(?:project|repo(?:sitory)?)\s+([A-Za-z0-9._/-]{2,80})/i);
-    if (match) return match[1];
-  }
-  const words = trimmed.split(/\s+/).slice(0, 6).join(' ');
-  return truncate(words, 80, '');
-}
-
-function includesAny(text: string, needles: string[]): boolean {
-  for (const needle of needles) {
-    if (text.includes(needle)) return true;
-  }
-  return false;
-}
-
-// ── Helpers ────────────────────────────────────────────────────────────
-
-function deduplicateItems(items: ExtractedItem[]): ExtractedItem[] {
-  const seen = new Set<string>();
-  const unique: ExtractedItem[] = [];
-  for (const item of items) {
-    if (seen.has(item.fingerprint)) continue;
-    seen.add(item.fingerprint);
-    unique.push(item);
-  }
-  return unique;
-}
-
-/** Parse a score value, returning `fallback` for null, undefined, empty strings, and non-finite numbers. */
-function parseScore(value: unknown, fallback: number): number {
-  if (value == null || value === '') return fallback;
-  const n = Number(value);
-  return Number.isFinite(n) ? n : fallback;
-}
-
-function clamp(value: number, min: number, max: number): number {
-  return Math.min(max, Math.max(min, value));
-}
-
-/** Returns true if the given memory item is the candidate in an unresolved conflict. */
-function hasPendingConflict(itemId: string): boolean {
-  const db = getDb();
-  const row = db
-    .select({ id: memoryItemConflicts.id })
-    .from(memoryItemConflicts)
-    .where(and(
-      eq(memoryItemConflicts.candidateItemId, itemId),
-      eq(memoryItemConflicts.status, 'pending_clarification'),
-    ))
-    .limit(1)
-    .get();
-  return row != null;
-}