vellum 0.0.16 → 0.2.0

package/src/memory/jobs-worker.ts

@@ -0,0 +1,274 @@
+import type { AssistantConfig } from '../config/types.js';
+import { getConfig } from '../config/loader.js';
+import { getLogger } from '../util/logger.js';
+import { getDb } from './db.js';
+import {
+  claimMemoryJobs,
+  completeMemoryJob,
+  deferMemoryJob,
+  enqueueCleanupResolvedConflictsJob,
+  enqueueCleanupStaleSupersededItemsJob,
+  failMemoryJob,
+  type MemoryJob,
+  resetRunningJobsToPending,
+} from './jobs-store.js';
+import {
+  BackendUnavailableError,
+  classifyError,
+  retryDelayForAttempt,
+  RETRY_MAX_ATTEMPTS,
+} from './job-utils.js';
+
+// ── Per-job-type handlers ──────────────────────────────────────────
+
+import { embedSegmentJob, embedItemJob, embedSummaryJob } from './job-handlers/embedding.js';
+import { extractItemsJob, extractEntitiesJob } from './job-handlers/extraction.js';
+import { resolvePendingConflictsForMessageJob, cleanupResolvedConflictsJob } from './job-handlers/conflict.js';
+import { checkContradictionsJob, cleanupStaleSupersededItemsJob } from './job-handlers/cleanup.js';
+import { buildConversationSummaryJob, buildGlobalSummaryJob } from './job-handlers/summarization.js';
+import { backfillJob, backfillEntityRelationsJob } from './job-handlers/backfill.js';
+import { rebuildIndexJob, deleteQdrantVectorsJob } from './job-handlers/index-maintenance.js';
+
+// Re-export public utilities consumed by tests and other modules
+export { currentWeekWindow } from './job-utils.js';
+
+const log = getLogger('memory-jobs-worker');
+
+export interface MemoryJobsWorker {
+  runOnce(): Promise<number>;
+  stop(): void;
+}
+
+export function startMemoryJobsWorker(): MemoryJobsWorker {
+  const recovered = resetRunningJobsToPending();
+  if (recovered > 0) {
+    log.info({ recovered }, 'Recovered stale running memory jobs');
+  }
+
+  let stopped = false;
+  let tickRunning = false;
+
+  const tick = async () => {
+    if (stopped || tickRunning) return;
+    tickRunning = true;
+    try {
+      await runMemoryJobsOnce({ enableScheduledCleanup: true });
+    } catch (err) {
+      log.error({ err }, 'Memory worker tick failed');
+    } finally {
+      tickRunning = false;
+    }
+  };
+
+  const timer = setInterval(() => { void tick(); }, 1500);
+  timer.unref();
+  void tick();
+
+  return {
+    async runOnce(): Promise<number> {
+      return runMemoryJobsOnce({ enableScheduledCleanup: true });
+    },
+    stop(): void {
+      stopped = true;
+      clearInterval(timer);
+    },
+  };
+}
+
+export async function runMemoryJobsOnce(
+  options: { enableScheduledCleanup?: boolean } = {},
+): Promise<number> {
+  const config = getConfig();
+  if (!config.memory.enabled) return 0;
+  const enableScheduledCleanup = options.enableScheduledCleanup === true;
+
+  // Periodic stale item sweep (throttled to at most once per hour)
+  sweepStaleItems(config);
+
+  const concurrency = Math.max(1, config.memory.jobs.workerConcurrency);
+  const jobs = claimMemoryJobs(concurrency);
+  if (jobs.length === 0) {
+    if (enableScheduledCleanup) {
+      maybeEnqueueScheduledCleanupJobs(config);
+    }
+    return 0;
+  }
+
+  let processed = 0;
+  for (const job of jobs) {
+    try {
+      await processJob(job, config);
+      completeMemoryJob(job.id);
+      processed += 1;
+    } catch (err) {
+      if (err instanceof BackendUnavailableError) {
+        // Backend not configured yet -- put the job back with exponential backoff.
+        const result = deferMemoryJob(job.id);
+        if (result === 'failed') {
+          log.warn({ jobId: job.id, type: job.type }, 'Embedding backend unavailable, job exceeded max deferrals');
+        } else {
+          log.debug({ jobId: job.id, type: job.type }, 'Embedding backend unavailable, deferring job');
+        }
+      } else {
+        const message = err instanceof Error ? err.message : String(err);
+        const category = classifyError(err);
+        if (category === 'retryable') {
+          const delay = retryDelayForAttempt(job.attempts + 1);
+          failMemoryJob(job.id, message, {
+            retryDelayMs: delay,
+            maxAttempts: RETRY_MAX_ATTEMPTS,
+          });
+          log.warn({ err, jobId: job.id, type: job.type, delay, category }, 'Memory job failed (retryable)');
+        } else {
+          failMemoryJob(job.id, message, { maxAttempts: 1 });
+          log.warn({ err, jobId: job.id, type: job.type, category }, 'Memory job failed (fatal)');
+        }
+      }
+    }
+  }
+  if (enableScheduledCleanup) {
+    maybeEnqueueScheduledCleanupJobs(config);
+  }
+  return processed;
+}
+
+// ── Job dispatch ───────────────────────────────────────────────────
+
+async function processJob(job: MemoryJob, config: AssistantConfig): Promise<void> {
+  switch (job.type) {
+    case 'embed_segment':
+      await embedSegmentJob(job, config);
+      return;
+    case 'embed_item':
+      await embedItemJob(job, config);
+      return;
+    case 'embed_summary':
+      await embedSummaryJob(job, config);
+      return;
+    case 'extract_items':
+      await extractItemsJob(job);
+      return;
+    case 'extract_entities':
+      await extractEntitiesJob(job, config);
+      return;
+    case 'resolve_pending_conflicts_for_message':
+      await resolvePendingConflictsForMessageJob(job, config);
+      return;
+    case 'cleanup_resolved_conflicts':
+      cleanupResolvedConflictsJob(job, config);
+      return;
+    case 'cleanup_stale_superseded_items':
+      cleanupStaleSupersededItemsJob(job, config);
+      return;
+    case 'check_contradictions':
+      await checkContradictionsJob(job);
+      return;
+    case 'build_conversation_summary':
+      await buildConversationSummaryJob(job, config);
+      return;
+    case 'refresh_weekly_summary':
+      await buildGlobalSummaryJob('weekly_global', config);
+      return;
+    case 'refresh_monthly_summary':
+      await buildGlobalSummaryJob('monthly_global', config);
+      return;
+    case 'backfill':
+      backfillJob(job, config);
+      return;
+    case 'backfill_entity_relations':
+      backfillEntityRelationsJob(job, config);
+      return;
+    case 'rebuild_index':
+      rebuildIndexJob();
+      return;
+    case 'delete_qdrant_vectors':
+      await deleteQdrantVectorsJob(job);
+      return;
+    default:
+      throw new Error(`Unknown memory job type: ${(job as { type: string }).type}`);
+  }
+}
+
+// ── Cleanup scheduling ─────────────────────────────────────────────
+
+let lastScheduledCleanupEnqueueMs = 0;
+
+/** Reset the cleanup enqueue throttle so tests can run deterministic checks. */
+export function resetCleanupScheduleThrottle(): void {
+  lastScheduledCleanupEnqueueMs = 0;
+}
+
+/**
+ * Enqueue periodic cleanup jobs using config-driven retention windows.
+ * Enqueue is deduped in jobs-store, so repeated calls remain safe.
+ */
+export function maybeEnqueueScheduledCleanupJobs(config: AssistantConfig, nowMs = Date.now()): boolean {
+  const cleanup = config.memory.cleanup;
+  if (!cleanup.enabled) return false;
+  if (nowMs - lastScheduledCleanupEnqueueMs < cleanup.enqueueIntervalMs) return false;
+
+  const resolvedConflictsJobId = enqueueCleanupResolvedConflictsJob(cleanup.resolvedConflictRetentionMs);
+  const staleSupersededItemsJobId = enqueueCleanupStaleSupersededItemsJob(cleanup.supersededItemRetentionMs);
+  lastScheduledCleanupEnqueueMs = nowMs;
+  log.debug({
+    resolvedConflictsJobId,
+    staleSupersededItemsJobId,
+    enqueueIntervalMs: cleanup.enqueueIntervalMs,
+    resolvedConflictRetentionMs: cleanup.resolvedConflictRetentionMs,
+    supersededItemRetentionMs: cleanup.supersededItemRetentionMs,
+  }, 'Enqueued scheduled memory cleanup jobs');
+  return true;
+}
+
+// ── Stale item sweep ───────────────────────────────────────────────
+
+const STALE_SWEEP_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
+let lastStaleSweepMs = 0;
+
+/** Reset the sweep throttle so tests can call sweepStaleItems back-to-back. */
+export function resetStaleSweepThrottle(): void {
+  lastStaleSweepMs = 0;
+}
+
+/**
+ * Mark deeply stale memory items as invalid. An item is considered deeply
+ * stale when it has exceeded 2x its freshness window for its kind and has
+ * not been recently accessed.
+ *
+ * This is non-destructive: items keep their data but get an `invalid_at`
+ * timestamp that excludes them from retrieval queries.
+ */
+export function sweepStaleItems(config: AssistantConfig): number {
+  const freshness = config.memory.retrieval.freshness;
+  if (!freshness.enabled) return 0;
+
+  const now = Date.now();
+  // Throttle: at most once per hour
+  if (now - lastStaleSweepMs < STALE_SWEEP_INTERVAL_MS) return 0;
+  lastStaleSweepMs = now;
+
+  const db = getDb();
+  const raw = (db as unknown as { $client: { query: (q: string) => { run: (...params: unknown[]) => { changes: number } } } }).$client;
+
+  let totalMarked = 0;
+  for (const [kind, maxAgeDays] of Object.entries(freshness.maxAgeDays)) {
+    if (maxAgeDays <= 0) continue;
+    // Mark invalid if: past 2x window, no access in the shield period, and not already invalid
+    const cutoffMs = now - maxAgeDays * 2 * 86_400_000;
+    const shieldCutoffMs = now - freshness.reinforcementShieldDays * 86_400_000;
+    const result = raw.query(`
+      UPDATE memory_items
+      SET invalid_at = ?
+      WHERE kind = ?
+        AND status = 'active'
+        AND invalid_at IS NULL
+        AND last_seen_at < ?
+        AND (access_count = 0 OR COALESCE(last_used_at, 0) < ?)
+    `).run(now, kind, cutoffMs, shieldCutoffMs);
+    if (result.changes > 0) {
+      log.info({ kind, marked: result.changes, cutoffMs }, 'Marked stale memory items as invalid');
+      totalMarked += result.changes;
+    }
+  }
+  return totalMarked;
+}

package/src/memory/llm-request-log-store.ts

@@ -0,0 +1,45 @@
+import { and, gte, lte, eq } from 'drizzle-orm';
+import { v4 as uuid } from 'uuid';
+import { getDb } from './db.js';
+import { llmRequestLogs } from './schema.js';
+
+export function recordRequestLog(
+  conversationId: string,
+  requestPayload: string,
+  responsePayload: string,
+): void {
+  const db = getDb();
+  db.insert(llmRequestLogs).values({
+    id: uuid(),
+    conversationId,
+    requestPayload,
+    responsePayload,
+    createdAt: Date.now(),
+  }).run();
+}
+
+export function queryRequestLogs(
+  conversationId: string,
+  startTime: number,
+  endTime: number,
+): Array<{
+  id: string;
+  conversationId: string;
+  requestPayload: string;
+  responsePayload: string;
+  createdAt: number;
+}> {
+  const db = getDb();
+  return db
+    .select()
+    .from(llmRequestLogs)
+    .where(
+      and(
+        eq(llmRequestLogs.conversationId, conversationId),
+        gte(llmRequestLogs.createdAt, startTime),
+        lte(llmRequestLogs.createdAt, endTime),
+      ),
+    )
+    .orderBy(llmRequestLogs.createdAt)
+    .all();
+}

package/src/memory/llm-usage-store.ts

@@ -0,0 +1,62 @@
+import { desc } from 'drizzle-orm';
+import { v4 as uuid } from 'uuid';
+import { getDb } from './db.js';
+import { llmUsageEvents } from './schema.js';
+import type { UsageEventInput, UsageEvent, PricingResult } from '../usage/types.js';
+
+export function recordUsageEvent(input: UsageEventInput, pricing: PricingResult): UsageEvent {
+  const db = getDb();
+  const event: UsageEvent = {
+    id: uuid(),
+    createdAt: Date.now(),
+    ...input,
+    estimatedCostUsd: pricing.estimatedCostUsd,
+    pricingStatus: pricing.pricingStatus,
+  };
+  db.insert(llmUsageEvents).values({
+    id: event.id,
+    createdAt: event.createdAt,
+    assistantId: event.assistantId,
+    conversationId: event.conversationId,
+    runId: event.runId,
+    requestId: event.requestId,
+    actor: event.actor,
+    provider: event.provider,
+    model: event.model,
+    inputTokens: event.inputTokens,
+    outputTokens: event.outputTokens,
+    cacheCreationInputTokens: event.cacheCreationInputTokens,
+    cacheReadInputTokens: event.cacheReadInputTokens,
+    estimatedCostUsd: event.estimatedCostUsd,
+    pricingStatus: event.pricingStatus,
+    metadataJson: null,
+  }).run();
+  return event;
+}
+
+export function listUsageEvents(options?: { limit?: number }): UsageEvent[] {
+  const db = getDb();
+  const rows = db
+    .select()
+    .from(llmUsageEvents)
+    .orderBy(desc(llmUsageEvents.createdAt))
+    .limit(options?.limit ?? 100)
+    .all();
+  return rows.map(row => ({
+    id: row.id,
+    createdAt: row.createdAt,
+    assistantId: row.assistantId,
+    conversationId: row.conversationId,
+    runId: row.runId,
+    requestId: row.requestId,
+    actor: row.actor as UsageEvent['actor'],
+    provider: row.provider,
+    model: row.model,
+    inputTokens: row.inputTokens,
+    outputTokens: row.outputTokens,
+    cacheCreationInputTokens: row.cacheCreationInputTokens,
+    cacheReadInputTokens: row.cacheReadInputTokens,
+    estimatedCostUsd: row.estimatedCostUsd,
+    pricingStatus: row.pricingStatus as 'priced' | 'unpriced',
+  }));
+}

package/src/memory/message-content.ts

@@ -0,0 +1,54 @@
+import type { ContentBlock } from '../providers/types.js';
+import { escapeXmlAttr } from '../util/xml.js';
+
+export function extractTextFromStoredMessageContent(raw: string): string {
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    if (typeof parsed === 'string') return parsed;
+    if (!Array.isArray(parsed)) return raw;
+    const blocks = parsed as ContentBlock[];
+    const lines: string[] = [];
+    for (const block of blocks) {
+      switch (block.type) {
+        case 'text':
+          lines.push(block.text);
+          break;
+        case 'tool_use':
+          lines.push(`Tool use (${block.name}): ${stableJson(block.input)}`);
+          break;
+        case 'tool_result':
+          lines.push(`Tool result${block.is_error ? ' <error />' : ''}: ${block.content}`);
+          break;
+        case 'thinking':
+          lines.push(block.thinking);
+          break;
+        case 'redacted_thinking':
+          lines.push('<redacted_thinking />');
+          break;
+        case 'image':
+          lines.push(`<image type="${escapeXmlAttr(block.source.media_type)}" />`);
+          break;
+        case 'file':
+          if (block.extracted_text) {
+            lines.push(`File (${block.source.filename}): ${block.extracted_text}`);
+          } else {
+            lines.push(`<file name="${escapeXmlAttr(block.source.filename)}" type="${escapeXmlAttr(block.source.media_type)}" />`);
+          }
+          break;
+        default:
+          lines.push('<unknown_content_block />');
+      }
+    }
+    return lines.join('\n').trim();
+  } catch {
+    return raw;
+  }
+}
+
+function stableJson(value: unknown): string {
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return '<unserializable />';
+  }
+}

package/src/memory/profile-compiler.ts

@@ -0,0 +1,160 @@
+import { and, desc, eq, inArray, isNull } from 'drizzle-orm';
+import { getConfig } from '../config/loader.js';
+import { estimateTextTokens } from '../context/token-estimator.js';
+import { getDb } from './db.js';
+import { memoryItems } from './schema.js';
+
+const PROFILE_KIND_ALLOWLIST = ['profile', 'preference', 'constraint', 'instruction', 'style'] as const;
+
+const TRUST_RANK: Record<string, number> = {
+  user_confirmed: 3,
+  user_reported: 2,
+  assistant_inferred: 1,
+};
+
+export interface CompileProfileOptions {
+  scopeId?: string;
+  maxInjectTokensOverride?: number;
+  /** When true and scopeId is not 'default', query both the given scope and 'default'. */
+  includeDefaultFallback?: boolean;
+}
+
+export interface CompiledProfile {
+  text: string;
+  sourceCount: number;
+  selectedCount: number;
+  budgetTokens: number;
+  tokenEstimate: number;
+}
+
+interface ProfileCandidate {
+  kind: string;
+  subject: string;
+  statement: string;
+  verificationState: string;
+  confidence: number;
+  importance: number | null;
+  lastSeenAt: number;
+  firstSeenAt: number;
+  scopeId: string;
+}
+
+export function compileDynamicProfile(options?: CompileProfileOptions): CompiledProfile {
+  const config = getConfig();
+  const profileConfig = config.memory.profile;
+  const scopeId = options?.scopeId ?? 'default';
+  const budgetTokens = Math.max(0, Math.floor(options?.maxInjectTokensOverride ?? profileConfig.maxInjectTokens));
+  if (!profileConfig.enabled || budgetTokens <= 0) {
+    return { text: '', sourceCount: 0, selectedCount: 0, budgetTokens, tokenEstimate: 0 };
+  }
+
+  const db = getDb();
+  const shouldFallback = options?.includeDefaultFallback === true && scopeId !== 'default';
+  const scopeFilter = shouldFallback
+    ? inArray(memoryItems.scopeId, [scopeId, 'default'])
+    : eq(memoryItems.scopeId, scopeId);
+  const rows = db
+    .select({
+      kind: memoryItems.kind,
+      subject: memoryItems.subject,
+      statement: memoryItems.statement,
+      verificationState: memoryItems.verificationState,
+      confidence: memoryItems.confidence,
+      importance: memoryItems.importance,
+      lastSeenAt: memoryItems.lastSeenAt,
+      firstSeenAt: memoryItems.firstSeenAt,
+      scopeId: memoryItems.scopeId,
+    })
+    .from(memoryItems)
+    .where(and(
+      scopeFilter,
+      eq(memoryItems.status, 'active'),
+      isNull(memoryItems.invalidAt),
+      inArray(memoryItems.kind, [...PROFILE_KIND_ALLOWLIST]),
+    ))
+    .orderBy(desc(memoryItems.lastSeenAt))
+    .all();
+
+  const nowMs = Date.now();
+  const trusted = rows
+    .filter((row) => TRUST_RANK[row.verificationState] !== undefined)
+    .sort((a, b) => compareProfileCandidates(a, b, nowMs, shouldFallback ? scopeId : undefined));
+
+  const selectedLines: string[] = [];
+  const seenKeys = new Set<string>();
+  for (const candidate of trusted) {
+    const subject = normalizeWhitespace(candidate.subject, 80);
+    const statement = normalizeWhitespace(candidate.statement, 220);
+    if (!subject || !statement) continue;
+    const dedupeKey = `${candidate.kind}|${subject.toLowerCase()}`;
+    if (seenKeys.has(dedupeKey)) continue;
+
+    const line = `- ${subject}: ${statement}`;
+    const tentative = renderProfileText([...selectedLines, line]);
+    if (estimateTextTokens(tentative) > budgetTokens) continue;
+    seenKeys.add(dedupeKey);
+    selectedLines.push(line);
+  }
+
+  const text = renderProfileText(selectedLines);
+  const tokenEstimate = text ? estimateTextTokens(text) : 0;
+  return {
+    text,
+    sourceCount: trusted.length,
+    selectedCount: selectedLines.length,
+    budgetTokens,
+    tokenEstimate,
+  };
+}
+
+/** Half-life for recency decay — items seen this many ms ago score ~0.5. (30 days) */
+const RECENCY_HALF_LIFE_MS = 30 * 24 * 60 * 60 * 1000;
+
+function recencyScore(lastSeenAt: number, nowMs: number): number {
+  const ageMs = Math.max(0, nowMs - lastSeenAt);
+  return Math.pow(0.5, ageMs / RECENCY_HALF_LIFE_MS);
+}
+
+function candidateRankScore(c: ProfileCandidate, nowMs: number): number {
+  const importance = c.importance ?? 0.5;
+  const recency = recencyScore(c.lastSeenAt, nowMs);
+  // Use weighted additive formula: importance dominates, recency is a minor boost
+  // This preserves high-importance items even when they haven't been seen recently
+  return importance * 0.7 + recency * 0.3;
+}
+
+function compareProfileCandidates(
+  left: ProfileCandidate,
+  right: ProfileCandidate,
+  nowMs: number,
+  /** When set, entries matching this scope sort before 'default' entries. */
+  preferredScopeId?: string,
+): number {
+  // Scoped entries beat default fallback entries so local overrides win deduplication
+  if (preferredScopeId !== undefined) {
+    const leftIsPreferred = left.scopeId === preferredScopeId;
+    const rightIsPreferred = right.scopeId === preferredScopeId;
+    if (leftIsPreferred && !rightIsPreferred) return -1;
+    if (!leftIsPreferred && rightIsPreferred) return 1;
+  }
+
+  const trustDelta = (TRUST_RANK[right.verificationState] ?? 0) - (TRUST_RANK[left.verificationState] ?? 0);
+  if (trustDelta !== 0) return trustDelta;
+
+  const scoreDelta = candidateRankScore(right, nowMs) - candidateRankScore(left, nowMs);
+  if (scoreDelta !== 0) return scoreDelta;
+
+  const confidenceDelta = right.confidence - left.confidence;
+  if (confidenceDelta !== 0) return confidenceDelta;
+
+  return right.firstSeenAt - left.firstSeenAt;
+}
+
+function normalizeWhitespace(input: string, maxLength: number): string {
+  return input.replace(/\s+/g, ' ').trim().slice(0, maxLength);
+}
+
+function renderProfileText(lines: string[]): string {
+  if (lines.length === 0) return '';
+  return ['<dynamic-user-profile>', ...lines, '</dynamic-user-profile>'].join('\n');
+}