vellum 0.2.1 → 0.2.2

package/src/__tests__/call-store.test.ts

@@ -23,7 +23,7 @@ mock.module('../util/logger.js', () => ({
   }),
 }));
 
-import { initializeDb, getDb } from '../memory/db.js';
+import { initializeDb, getDb, resetDb } from '../memory/db.js';
 import { conversations } from '../memory/schema.js';
 import {
   createCallSession,
@@ -37,11 +37,15 @@ import {
   getPendingQuestion,
   answerPendingQuestion,
   expirePendingQuestions,
+  claimCallback,
+  releaseCallbackClaim,
+  finalizeCallbackClaim,
 } from '../calls/call-store.js';
 
 initializeDb();
 
 afterAll(() => {
+  resetDb();
   try { rmSync(testDir, { recursive: true }); } catch { /* best effort */ }
 });
 
@@ -65,6 +69,7 @@ function resetTables() {
   db.run('DELETE FROM call_pending_questions');
   db.run('DELETE FROM call_events');
   db.run('DELETE FROM call_sessions');
+  db.run('DELETE FROM processed_callbacks');
   db.run('DELETE FROM conversations');
   ensuredConvIds = new Set();
 }
@@ -473,4 +478,214 @@ describe('call-store', () => {
     const q1Row = raw.query('SELECT status FROM call_pending_questions WHERE id = ?').get(q1.id) as { status: string };
     expect(q1Row.status).toBe('answered');
   });
+
+  // ── Callback Claim ──────────────────────────────────────────────
+
+  test('claimCallback returns a claim ID on first call', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-22',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    const result = claimCallback('test-dedupe-key-1', session.id);
+    expect(result).toBeTypeOf('string');
+    expect(result!.length).toBeGreaterThan(0);
+  });
+
+  test('claimCallback returns null on duplicate key', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-23',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    const first = claimCallback('test-dedupe-key-2', session.id);
+    const second = claimCallback('test-dedupe-key-2', session.id);
+
+    expect(first).toBeTypeOf('string');
+    expect(second).toBeNull();
+  });
+
+  test('releaseCallbackClaim allows re-claim', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-24',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    const first = claimCallback('test-dedupe-key-3', session.id);
+    expect(first).toBeTypeOf('string');
+
+    releaseCallbackClaim('test-dedupe-key-3', first!);
+
+    const second = claimCallback('test-dedupe-key-3', session.id);
+    expect(second).toBeTypeOf('string');
+  });
+
+  test('releaseCallbackClaim with wrong claimId does not release', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-24b',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    const claimId = claimCallback('test-dedupe-key-3b', session.id);
+    expect(claimId).toBeTypeOf('string');
+
+    // Attempt to release with a wrong claim ID — should be a no-op
+    releaseCallbackClaim('test-dedupe-key-3b', 'wrong-claim-id');
+
+    // The claim should still be held, so re-claiming should fail
+    const second = claimCallback('test-dedupe-key-3b', session.id);
+    expect(second).toBeNull();
+  });
+
+  test('claimCallback INSERT OR IGNORE pattern is safe for same key', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-25',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    // Claim the key
+    const first = claimCallback('test-dedupe-key-4', session.id);
+    expect(first).toBeTypeOf('string');
+
+    // Subsequent claims with the same key should all return null without throwing
+    expect(claimCallback('test-dedupe-key-4', session.id)).toBeNull();
+    expect(claimCallback('test-dedupe-key-4', session.id)).toBeNull();
+
+    // Only one row should exist in the table for this key
+    const raw = (getDb() as unknown as { $client: import('bun:sqlite').Database }).$client;
+    const rows = raw.query('SELECT COUNT(*) as cnt FROM processed_callbacks WHERE dedupe_key = ?').get('test-dedupe-key-4') as { cnt: number };
+    expect(rows.cnt).toBe(1);
+  });
+
+  test('claimCallback reclaims expired orphaned claims', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-26',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    // Claim the key
+    const first = claimCallback('test-dedupe-key-expired', session.id);
+    expect(first).toBeTypeOf('string');
+
+    // Simulate an orphaned claim by backdating the created_at to well past expiry
+    const raw = (getDb() as unknown as { $client: import('bun:sqlite').Database }).$client;
+    const oldTimestamp = Date.now() - 120_000; // 2 minutes ago, well past 60s expiry
+    raw.query('UPDATE processed_callbacks SET created_at = ? WHERE dedupe_key = ?').run(oldTimestamp, 'test-dedupe-key-expired');
+
+    // Reclaim should succeed because the old claim has expired
+    const second = claimCallback('test-dedupe-key-expired', session.id);
+    expect(second).toBeTypeOf('string');
+
+    // The new claim should have a different claim ID
+    expect(second).not.toBe(first);
+  });
+
+  test('claimCallback does not reclaim finalized claims', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-27',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    // Claim and finalize
+    const first = claimCallback('test-dedupe-key-finalized', session.id);
+    expect(first).toBeTypeOf('string');
+    finalizeCallbackClaim('test-dedupe-key-finalized', first!);
+
+    // Attempting to reclaim a finalized key should fail because the far-future
+    // timestamp means it will never be considered expired
+    const second = claimCallback('test-dedupe-key-finalized', session.id);
+    expect(second).toBeNull();
+  });
+
+  test('finalizeCallbackClaim makes claim permanent', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-28',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    // Claim and finalize
+    const claimId = claimCallback('test-dedupe-key-permanent', session.id)!;
+    finalizeCallbackClaim('test-dedupe-key-permanent', claimId);
+
+    // Verify the created_at is set far in the future
+    const raw = (getDb() as unknown as { $client: import('bun:sqlite').Database }).$client;
+    const row = raw.query('SELECT created_at FROM processed_callbacks WHERE dedupe_key = ?').get('test-dedupe-key-permanent') as { created_at: number };
+    // Should be at least 50 years in the future from now
+    const fiftyYearsMs = 50 * 365 * 24 * 60 * 60 * 1000;
+    expect(row.created_at).toBeGreaterThan(Date.now() + fiftyYearsMs);
+  });
+
+  test('finalizeCallbackClaim with wrong claimId does not finalize', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-28b',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    // Claim the key
+    const claimId = claimCallback('test-dedupe-key-permanent-b', session.id)!;
+    expect(claimId).toBeTypeOf('string');
+
+    // Try to finalize with wrong claimId — should be a no-op
+    finalizeCallbackClaim('test-dedupe-key-permanent-b', 'wrong-claim-id');
+
+    // Verify the created_at was NOT set to far-future (it should still be close to now)
+    const raw = (getDb() as unknown as { $client: import('bun:sqlite').Database }).$client;
+    const row = raw.query('SELECT created_at FROM processed_callbacks WHERE dedupe_key = ?').get('test-dedupe-key-permanent-b') as { created_at: number };
+    const oneMinuteMs = 60 * 1000;
+    expect(row.created_at).toBeLessThan(Date.now() + oneMinuteMs);
+  });
+
+  test('handler A cannot release handler B claim after reclaim', () => {
+    const session = createTestCallSession({
+      conversationId: 'conv-29',
+      provider: 'twilio',
+      fromNumber: '+15551111111',
+      toNumber: '+15552222222',
+    });
+
+    // Handler A claims
+    const claimA = claimCallback('test-dedupe-key-ownership', session.id)!;
+    expect(claimA).toBeTypeOf('string');
+
+    // Simulate handler A taking too long: backdate the claim so it expires
+    const raw = (getDb() as unknown as { $client: import('bun:sqlite').Database }).$client;
+    const oldTimestamp = Date.now() - 120_000;
+    raw.query('UPDATE processed_callbacks SET created_at = ? WHERE dedupe_key = ?').run(oldTimestamp, 'test-dedupe-key-ownership');
+
+    // Handler B reclaims (succeeds because the old claim expired)
+    const claimB = claimCallback('test-dedupe-key-ownership', session.id)!;
+    expect(claimB).toBeTypeOf('string');
+    expect(claimB).not.toBe(claimA);
+
+    // Handler B finalizes
+    finalizeCallbackClaim('test-dedupe-key-ownership', claimB);
+
+    // Handler A tries to release using its old claimId — should be a no-op
+    releaseCallbackClaim('test-dedupe-key-ownership', claimA);
+
+    // Verify B's finalized claim is still intact
+    const row = raw.query('SELECT created_at, claim_id FROM processed_callbacks WHERE dedupe_key = ?').get('test-dedupe-key-ownership') as { created_at: number; claim_id: string };
+    expect(row).not.toBeNull();
+    expect(row.claim_id).toBe(claimB);
+    const fiftyYearsMs = 50 * 365 * 24 * 60 * 60 * 1000;
+    expect(row.created_at).toBeGreaterThan(Date.now() + fiftyYearsMs);
+  });
 });

package/src/__tests__/cli-discover.test.ts

@@ -71,7 +71,7 @@ describe('cliDiscoverTool', () => {
     expect(result.isError).toBe(false);
     // Should at least find git which is nearly universally available
     expect(result.content).toContain('**git**');
-  }, 30_000);
+  }, 60_000);
 
   test('includes version info for found CLIs', async () => {
     const result = await cliDiscoverTool.execute(

package/src/__tests__/commit-message-enrichment-service.test.ts

@@ -3,12 +3,13 @@ import { mkdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import { execFileSync } from 'node:child_process';
+import type { CommitContext } from '../workspace/commit-message-provider.js';
+
 import {
   CommitEnrichmentService,
   _resetEnrichmentService,
 } from '../workspace/commit-message-enrichment-service.js';
 import { WorkspaceGitService, _resetGitServiceRegistry } from '../workspace/git-service.js';
-import type { CommitContext } from '../workspace/commit-message-provider.js';
 
 describe('CommitEnrichmentService', () => {
   let testDir: string;
@@ -48,6 +49,16 @@ describe('CommitEnrichmentService', () => {
     return await gitService.getHeadHash();
   }
 
+  async function waitForDrain(service: CommitEnrichmentService, timeoutMs = 5000): Promise<void> {
+    const started = Date.now();
+    while (service._getQueueSize() > 0 || service._getActiveWorkers() > 0) {
+      if (Date.now() - started > timeoutMs) {
+        throw new Error(`Timed out waiting for enrichment queue to drain after ${timeoutMs}ms`);
+      }
+      await new Promise(resolve => setTimeout(resolve, 50));
+    }
+  }
+
   test('enqueue and execute writes git note on success', async () => {
     const commitHash = await createCommit();
     const service = new CommitEnrichmentService({
@@ -280,9 +291,7 @@ describe('CommitEnrichmentService', () => {
     });
 
     // Wait for queue to drain before shutdown (avoids discarding pending jobs)
-    while (service._getQueueSize() > 0 || service._getActiveWorkers() > 0) {
-      await new Promise(resolve => setTimeout(resolve, 50));
-    }
+    await waitForDrain(service, 5000);
     await service.shutdown();
 
     // Both notes should exist
@@ -318,9 +327,7 @@ describe('CommitEnrichmentService', () => {
     // Wait for all retries to complete (initial + 2 retries, with backoff)
     // Backoff: 1s after attempt 1, 2s after attempt 2 = ~3s total
     // But since the job itself is very fast to time out, total time is dominated by backoff
-    while (service._getActiveWorkers() > 0 || service._getQueueSize() > 0) {
-      await new Promise(resolve => setTimeout(resolve, 100));
-    }
+    await waitForDrain(service, 10000);
     await service.shutdown();
 
     // After 1 initial attempt + 2 retries (3 total), the job should be counted as failed
@@ -373,6 +380,140 @@ describe('CommitEnrichmentService', () => {
     expect(service._getDroppedCount()).toBe(4);
   });
 
+  test('timed-out enrichment work is cancelled via AbortSignal', async () => {
+    // Track whether the slow enrichment work actually ran to completion
+    let enrichmentCompleted = false;
+    const commitHash = await createCommit();
+
+    const service = new CommitEnrichmentService({
+      maxQueueSize: 10,
+      maxConcurrency: 1,
+      jobTimeoutMs: 50, // Very short timeout
+      maxRetries: 0,
+    });
+
+    // Monkey-patch writeNote to simulate slow work that respects the abort signal.
+    // The real writeNote now passes the signal to execFileAsync which kills the
+    // child process on abort. This mock replicates that behavior by rejecting
+    // when the signal fires.
+    const originalWriteNote = gitService.writeNote.bind(gitService);
+    gitService.writeNote = async (_hash: string, _note: string, signal?: AbortSignal) => {
+      // Simulate slow work that is cancellable via AbortSignal
+      await new Promise<void>((resolve, reject) => {
+        const timer = setTimeout(() => {
+          enrichmentCompleted = true;
+          resolve();
+        }, 2000);
+        signal?.addEventListener('abort', () => {
+          clearTimeout(timer);
+          reject(new Error('aborted'));
+        }, { once: true });
+      });
+    };
+
+    service.enqueue({
+      workspaceDir: testDir,
+      commitHash,
+      context: makeContext(),
+      gitService,
+    });
+
+    await waitForDrain(service, 5000);
+    await service.shutdown();
+
+    // Allow any zombie work to settle — if abort didn't work, the 2s timer
+    // would still be running and would set enrichmentCompleted=true. Wait
+    // longer than the 2000ms mock delay to reliably catch the regression.
+    await new Promise(resolve => setTimeout(resolve, 2500));
+
+    // The job should have timed out and been counted as failed
+    expect(service._getFailedCount()).toBe(1);
+    expect(service._getSucceededCount()).toBe(0);
+    // The slow enrichment work should NOT have completed since the signal was aborted
+    expect(enrichmentCompleted).toBe(false);
+
+    // Restore original
+    gitService.writeNote = originalWriteNote;
+  });
+
+  test('shutdown does not hang on timed-out jobs', async () => {
+    const commitHash = await createCommit();
+
+    const service = new CommitEnrichmentService({
+      maxQueueSize: 10,
+      maxConcurrency: 1,
+      jobTimeoutMs: 50, // Short timeout
+      maxRetries: 0,
+    });
+
+    // Make writeNote artificially slow so the job will always time out.
+    // The mock respects the abort signal so the subprocess is killed on timeout.
+    const originalWriteNote = gitService.writeNote.bind(gitService);
+    gitService.writeNote = async (_hash: string, _note: string, signal?: AbortSignal) => {
+      await new Promise<void>((resolve, reject) => {
+        const timer = setTimeout(resolve, 5000);
+        signal?.addEventListener('abort', () => {
+          clearTimeout(timer);
+          reject(new Error('aborted'));
+        }, { once: true });
+      });
+    };
+
+    service.enqueue({
+      workspaceDir: testDir,
+      commitHash,
+      context: makeContext(),
+      gitService,
+    });
+
+    // Shutdown should complete promptly, not hang for 5s waiting on the slow writeNote
+    const shutdownStart = Date.now();
+    await service.shutdown();
+    const shutdownElapsed = Date.now() - shutdownStart;
+
+    // Shutdown should complete well under the 5s slow-work duration
+    expect(shutdownElapsed).toBeLessThan(3000);
+    expect(service._getFailedCount()).toBe(1);
+
+    gitService.writeNote = originalWriteNote;
+  }, 10000);
+
+  test('abort signal is triggered on non-timeout errors before retry', async () => {
+    const commitHash = await createCommit();
+
+    const service = new CommitEnrichmentService({
+      maxQueueSize: 10,
+      maxConcurrency: 1,
+      jobTimeoutMs: 5000,
+      maxRetries: 0,
+    });
+
+    // Make writeNote throw an error and observe whether the signal gets aborted
+    const originalWriteNote = gitService.writeNote.bind(gitService);
+    gitService.writeNote = async (_hash: string, _note: string) => {
+      // Set up a listener on the abort controller's signal to track abortion.
+      // We access the signal indirectly by throwing, which triggers the catch
+      // block in executeJob where controller.abort() is called.
+      throw new Error('Simulated writeNote failure');
+    };
+
+    service.enqueue({
+      workspaceDir: testDir,
+      commitHash,
+      context: makeContext(),
+      gitService,
+    });
+
+    await waitForDrain(service, 5000);
+    await service.shutdown();
+
+    // The job should have failed (no retries configured)
+    expect(service._getFailedCount()).toBe(1);
+    expect(service._getSucceededCount()).toBe(0);
+
+    gitService.writeNote = originalWriteNote;
+  });
+
   test('enqueue is fire-and-forget and never throws even when called rapidly', async () => {
     const service = new CommitEnrichmentService({
       maxQueueSize: 3,

package/src/__tests__/compaction.benchmark.test.ts

@@ -0,0 +1,176 @@
+/**
+ * Context Window Compaction Benchmark
+ *
+ * Measures compaction cost with a mock provider:
+ * - compaction latency under threshold pressure
+ * - no-op fast path for below-threshold histories
+ * - token reduction ratio after compaction
+ * - summary call count within expected range
+ * - severe pressure overriding cooldown
+ */
+import { describe, expect, mock, test } from 'bun:test';
+
+import { DEFAULT_CONFIG } from '../config/defaults.js';
+import { ContextWindowManager } from '../context/window-manager.js';
+import { estimatePromptTokens } from '../context/token-estimator.js';
+import type { Message, Provider } from '../providers/types.js';
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+function makeSummaryProvider(counter: { calls: number }): Provider {
+  return {
+    name: 'mock',
+    async sendMessage() {
+      counter.calls += 1;
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `## Goals\n- Preserve state\n## Constraints\n- Keep PRs small\n## Decisions\n- Call ${counter.calls}`,
+          },
+        ],
+        model: 'mock-model',
+        usage: { inputTokens: 420, outputTokens: 85 },
+        stopReason: 'end_turn',
+      };
+    },
+  };
+}
+
+function makeLongMessages(turns: number): Message[] {
+  const rows: Message[] = [];
+  for (let i = 0; i < turns; i++) {
+    rows.push({
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          text: `[U${i}] User message with enough content to estimate tokens. Topic ${i % 9}.`,
+        },
+      ],
+    });
+    rows.push({
+      role: 'assistant',
+      content: [
+        {
+          type: 'text',
+          text: `[A${i}] Assistant response with relevant content. Result ${i % 7}.`,
+        },
+      ],
+    });
+  }
+  return rows;
+}
+
+function makeConfig() {
+  return {
+    ...DEFAULT_CONFIG.contextWindow,
+    maxInputTokens: 6000,
+    targetInputTokens: 3200,
+    compactThreshold: 0.6,
+    preserveRecentUserTurns: 8,
+    chunkTokens: 1200,
+  };
+}
+
+describe('Compaction benchmark', () => {
+  test('compaction with mock provider completes under 500ms', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    // 90 turns = 180 messages, well above 60% of 6000 = 3600 threshold
+    const messages = makeLongMessages(90);
+    const before = estimatePromptTokens(messages, 'system prompt', {
+      providerName: 'mock',
+    });
+    expect(before).toBeGreaterThan(config.maxInputTokens * config.compactThreshold);
+
+    const start = performance.now();
+    const result = await manager.maybeCompact(messages);
+    const elapsed = performance.now() - start;
+
+    expect(result.compacted).toBe(true);
+    expect(elapsed).toBeLessThan(500);
+  });
+
+  test('below-threshold check returns in under 50ms (no-op)', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    // 3 turns = 6 messages, well below threshold
+    const messages = makeLongMessages(3);
+
+    const start = performance.now();
+    const result = await manager.maybeCompact(messages);
+    const elapsed = performance.now() - start;
+
+    expect(result.compacted).toBe(false);
+    expect(result.reason).toBe('below compaction threshold');
+    expect(elapsed).toBeLessThan(50);
+    expect(counter.calls).toBe(0);
+  });
+
+  test('token reduction ratio exceeds 30% after compaction', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const result = await manager.maybeCompact(messages);
+
+    expect(result.compacted).toBe(true);
+    const reductionRatio =
+      (result.previousEstimatedInputTokens - result.estimatedInputTokens) /
+      result.previousEstimatedInputTokens;
+    expect(reductionRatio).toBeGreaterThan(0.3);
+  });
+
+  test('summary calls fall within 2-6 range', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const result = await manager.maybeCompact(messages);
+
+    expect(result.compacted).toBe(true);
+    expect(result.summaryCalls).toBeGreaterThanOrEqual(2);
+    expect(result.summaryCalls).toBeLessThanOrEqual(6);
+    expect(result.summaryCalls).toBe(counter.calls);
+  });
+
+  test('severe pressure triggers compaction even during cooldown', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    // Use a tighter maxInputTokens so 90 turns exceeds the 95% severe threshold
+    const config = {
+      ...makeConfig(),
+      maxInputTokens: 4000,
+      targetInputTokens: 2000,
+    };
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const estimated = estimatePromptTokens(messages, 'system prompt', {
+      providerName: 'mock',
+    });
+    expect(estimated).toBeGreaterThan(config.maxInputTokens * 0.95);
+
+    // Simulate being within cooldown by setting lastCompactedAt to now
+    const result = await manager.maybeCompact(messages, undefined, {
+      lastCompactedAt: Date.now(),
+    });
+
+    expect(result.compacted).toBe(true);
+    expect(result.summaryCalls).toBeGreaterThan(0);
+  });
+});