vellum 0.2.1 → 0.2.2

package/src/__tests__/parallel-tool.benchmark.test.ts

@@ -0,0 +1,294 @@
+import { describe, test, expect } from 'bun:test';
+import { AgentLoop } from '../agent/loop.js';
+import type { AgentEvent } from '../agent/loop.js';
+import type {
+  Provider,
+  Message,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+  ContentBlock,
+} from '../providers/types.js';
+
+// ---------------------------------------------------------------------------
+// Helpers (mirrors agent-loop.test.ts patterns)
+// ---------------------------------------------------------------------------
+
+function createMockProvider(
+  responses: ProviderResponse[],
+): { provider: Provider; calls: { messages: Message[]; tools?: ToolDefinition[]; systemPrompt?: string }[] } {
+  const calls: { messages: Message[]; tools?: ToolDefinition[]; systemPrompt?: string }[] = [];
+  let callIndex = 0;
+
+  const provider: Provider = {
+    name: 'mock',
+    async sendMessage(
+      messages: Message[],
+      tools?: ToolDefinition[],
+      systemPrompt?: string,
+      options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      calls.push({ messages: [...messages], tools, systemPrompt });
+      const response = responses[callIndex] ?? responses[responses.length - 1];
+      callIndex++;
+
+      if (options?.onEvent) {
+        for (const block of response.content) {
+          if (block.type === 'text') {
+            options.onEvent({ type: 'text_delta', text: block.text });
+          }
+        }
+      }
+
+      return response;
+    },
+  };
+
+  return { provider, calls };
+}
+
+function textResponse(text: string): ProviderResponse {
+  return {
+    content: [{ type: 'text', text }],
+    model: 'mock-model',
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: 'end_turn',
+  };
+}
+
+/** Build a provider response containing N parallel tool_use blocks. */
+function parallelToolUseResponse(
+  tools: Array<{ id: string; name: string; input: Record<string, unknown> }>,
+): ProviderResponse {
+  return {
+    content: tools.map((t) => ({
+      type: 'tool_use' as const,
+      id: t.id,
+      name: t.name,
+      input: t.input,
+    })),
+    model: 'mock-model',
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: 'tool_use' as const,
+  };
+}
+
+const dummyTools: ToolDefinition[] = [
+  { name: 'delay_tool', description: 'Delays', input_schema: { type: 'object', properties: {} } },
+];
+
+const userMessage: Message = {
+  role: 'user',
+  content: [{ type: 'text', text: 'Run benchmarks' }],
+};
+
+function collectEvents(events: AgentEvent[]): (event: AgentEvent) => void {
+  return (event) => events.push(event);
+}
+
+// ---------------------------------------------------------------------------
+// Benchmark Tests
+// ---------------------------------------------------------------------------
+
+describe('Parallel tool execution benchmarks', () => {
+  // 1. 5 tools at 50ms each should complete in ~50ms (parallel), not ~250ms (sequential)
+  test('5 tools at 50ms each complete in parallel (~50ms, not ~250ms)', async () => {
+    const toolCount = 5;
+    const delayMs = 50;
+
+    const toolUseBlocks = Array.from({ length: toolCount }, (_, i) => ({
+      id: `t${i}`,
+      name: 'delay_tool',
+      input: { index: i },
+    }));
+
+    const { provider } = createMockProvider([
+      parallelToolUseResponse(toolUseBlocks),
+      textResponse('All done.'),
+    ]);
+
+    const toolExecutor = async () => {
+      await new Promise((r) => setTimeout(r, delayMs));
+      return { content: 'ok', isError: false };
+    };
+
+    const loop = new AgentLoop(provider, 'system', {}, dummyTools, toolExecutor);
+    const start = Date.now();
+    await loop.run([userMessage], () => {});
+    const elapsed = Date.now() - start;
+
+    // Parallel: ~50ms + overhead. Sequential would be ~250ms.
+    // Allow up to 150ms for CI/scheduling overhead.
+    expect(elapsed).toBeLessThan(150);
+  });
+
+  // 2. 10 tools at 50ms each should still complete quickly in parallel
+  test('10 tools at 50ms each still complete in parallel (< 200ms)', async () => {
+    const toolCount = 10;
+    const delayMs = 50;
+
+    const toolUseBlocks = Array.from({ length: toolCount }, (_, i) => ({
+      id: `t${i}`,
+      name: 'delay_tool',
+      input: { index: i },
+    }));
+
+    const { provider } = createMockProvider([
+      parallelToolUseResponse(toolUseBlocks),
+      textResponse('All done.'),
+    ]);
+
+    const executionLog: { index: number; start: number; end: number }[] = [];
+    const toolExecutor = async (_name: string, input: Record<string, unknown>) => {
+      const start = Date.now();
+      await new Promise((r) => setTimeout(r, delayMs));
+      const end = Date.now();
+      executionLog.push({ index: input.index as number, start, end });
+      return { content: 'ok', isError: false };
+    };
+
+    const loop = new AgentLoop(provider, 'system', {}, dummyTools, toolExecutor);
+    const start = Date.now();
+    await loop.run([userMessage], () => {});
+    const elapsed = Date.now() - start;
+
+    // All 10 tools should have executed
+    expect(executionLog).toHaveLength(toolCount);
+
+    // Parallel: ~50ms + overhead. Sequential would be ~500ms.
+    // Allow up to 200ms for CI/scheduling overhead with 10 concurrent timers.
+    expect(elapsed).toBeLessThan(200);
+
+    // Verify overlap: all tools should start before any finishes
+    const allStarts = executionLog.map((e) => e.start);
+    const allEnds = executionLog.map((e) => e.end);
+    const lastStart = Math.max(...allStarts);
+    const firstEnd = Math.min(...allEnds);
+    expect(lastStart).toBeLessThanOrEqual(firstEnd);
+  });
+
+  // 3. Mixed latencies: 1 slow (2s) + 4 fast (100ms) = ~2s parallel, ~2.4s sequential
+  test('mixed latencies: 1 slow + 4 fast tools complete in slow-tool time', async () => {
+    const toolUseBlocks = [
+      { id: 'slow', name: 'delay_tool', input: { delayMs: 2000 } },
+      { id: 'fast1', name: 'delay_tool', input: { delayMs: 100 } },
+      { id: 'fast2', name: 'delay_tool', input: { delayMs: 100 } },
+      { id: 'fast3', name: 'delay_tool', input: { delayMs: 100 } },
+      { id: 'fast4', name: 'delay_tool', input: { delayMs: 100 } },
+    ];
+
+    const { provider } = createMockProvider([
+      parallelToolUseResponse(toolUseBlocks),
+      textResponse('Mixed done.'),
+    ]);
+
+    const completionOrder: string[] = [];
+    const toolExecutor = async (_name: string, input: Record<string, unknown>) => {
+      const delay = input.delayMs as number;
+      await new Promise((r) => setTimeout(r, delay));
+      completionOrder.push(
+        toolUseBlocks.find((t) => t.input.delayMs === delay && !completionOrder.includes(t.id))?.id
+          ?? `unknown-${delay}`,
+      );
+      return { content: 'ok', isError: false };
+    };
+
+    const loop = new AgentLoop(provider, 'system', {}, dummyTools, toolExecutor);
+    const events: AgentEvent[] = [];
+    const start = Date.now();
+    await loop.run([userMessage], collectEvents(events));
+    const elapsed = Date.now() - start;
+
+    // Parallel: ~2000ms (dominated by slow tool). Sequential: ~2400ms (2000 + 4*100).
+    // Upper bound of 2200ms ensures a sequential implementation would fail.
+    expect(elapsed).toBeGreaterThanOrEqual(1900);
+    expect(elapsed).toBeLessThan(2200);
+
+    // tool_result events should be emitted in tool_use order (slow first),
+    // even though fast tools finish earlier
+    const toolResultEvents = events.filter(
+      (e): e is Extract<AgentEvent, { type: 'tool_result' }> => e.type === 'tool_result',
+    );
+    expect(toolResultEvents).toHaveLength(5);
+    expect(toolResultEvents[0].toolUseId).toBe('slow');
+    expect(toolResultEvents[1].toolUseId).toBe('fast1');
+  }, 10000);
+
+  // 4. Abort during parallel execution cancels within 200ms
+  test('abort during parallel execution cancels within 200ms', async () => {
+    const unhandledRejections: Error[] = [];
+    const handler = (event: PromiseRejectionEvent) => {
+      unhandledRejections.push(event.reason);
+      event.preventDefault();
+    };
+    globalThis.addEventListener('unhandledrejection', handler);
+
+    try {
+      const toolCount = 5;
+
+      const toolUseBlocks = Array.from({ length: toolCount }, (_, i) => ({
+        id: `t${i}`,
+        name: 'delay_tool',
+        input: { index: i },
+      }));
+
+      const { provider } = createMockProvider([
+        parallelToolUseResponse(toolUseBlocks),
+        textResponse('Should not reach.'),
+      ]);
+
+      const controller = new AbortController();
+
+      // Track each tool executor's promise so we can wait for them all to
+      // settle after abort, ensuring late rejections are caught by our listener.
+      const toolPromises: Promise<unknown>[] = [];
+
+      const toolExecutor = async () => {
+        const p = new Promise<void>((resolve) => {
+          // Each tool takes 500ms — abort fires at 50ms, well before completion.
+          // Shorter than the original 10s so we can actually wait for settlement.
+          setTimeout(resolve, 500);
+        });
+        toolPromises.push(p);
+
+        setTimeout(() => controller.abort(), 50);
+        await p;
+        return { content: 'should not return', isError: false };
+      };
+
+      const loop = new AgentLoop(provider, 'system', {}, dummyTools, toolExecutor);
+      const start = Date.now();
+      const history = await loop.run([userMessage], () => {}, controller.signal);
+      const elapsed = Date.now() - start;
+
+      // Should exit quickly after the 50ms abort, not wait 500ms
+      expect(elapsed).toBeLessThan(200);
+
+      // History should have: user msg, assistant (tool_use), user (cancelled tool_results)
+      expect(history).toHaveLength(3);
+
+      const lastMsg = history[history.length - 1];
+      expect(lastMsg.role).toBe('user');
+
+      const toolResultBlocks = lastMsg.content.filter(
+        (b): b is Extract<ContentBlock, { type: 'tool_result' }> => b.type === 'tool_result',
+      );
+      expect(toolResultBlocks).toHaveLength(toolCount);
+
+      // All results should be cancelled
+      for (const block of toolResultBlocks) {
+        expect(block.content).toBe('Cancelled by user');
+        expect(block.is_error).toBe(true);
+      }
+
+      // Wait for all abandoned tool promises to settle so any late rejections
+      // fire while our listener is still active
+      await Promise.allSettled(toolPromises);
+
+      // Verify no unhandled rejections occurred
+      expect(unhandledRejections).toHaveLength(0);
+    } finally {
+      globalThis.removeEventListener('unhandledrejection', handler);
+    }
+  });
+});

package/src/__tests__/playbook-tools.test.ts

@@ -0,0 +1,342 @@
+import { describe, test, expect, beforeEach, afterAll, mock } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+const testDir = mkdtempSync(join(tmpdir(), 'playbook-tools-test-'));
+
+mock.module('../util/platform.js', () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === 'darwin',
+  isLinux: () => process.platform === 'linux',
+  isWindows: () => process.platform === 'win32',
+  getSocketPath: () => join(testDir, 'test.sock'),
+  getPidPath: () => join(testDir, 'test.pid'),
+  getDbPath: () => join(testDir, 'test.db'),
+  getLogPath: () => join(testDir, 'test.log'),
+  ensureDataDir: () => {},
+  migrateToDataLayout: () => {},
+  migrateToWorkspaceLayout: () => {},
+}));
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () => new Proxy({} as Record<string, unknown>, {
+    get: () => () => {},
+  }),
+}));
+
+mock.module('../config/loader.js', () => ({
+  getConfig: () => ({ memory: {} }),
+}));
+
+// Stub memory job queue to avoid side effects
+mock.module('../memory/jobs-store.js', () => ({
+  enqueueMemoryJob: () => {},
+}));
+
+import type { Database } from 'bun:sqlite';
+import { initializeDb, getDb, resetDb } from '../memory/db.js';
+import type { ToolContext } from '../tools/types.js';
+import {
+  executePlaybookCreate,
+  executePlaybookList,
+  executePlaybookUpdate,
+  executePlaybookDelete,
+} from '../tools/playbooks/index.js';
+
+initializeDb();
+
+afterAll(() => {
+  resetDb();
+  try { rmSync(testDir, { recursive: true }); } catch { /* best effort */ }
+});
+
+function getRawDb(): Database {
+  return (getDb() as unknown as { $client: Database }).$client;
+}
+
+const ctx: ToolContext = {
+  workingDir: '/tmp',
+  sessionId: 'test-session',
+  conversationId: 'test-conversation',
+};
+
+function clearPlaybooks(): void {
+  getRawDb().run("DELETE FROM memory_items WHERE kind = 'playbook'");
+}
+
+function extractPlaybookId(content: string): string {
+  const match = content.match(/ID: (\S+)/);
+  expect(match).not.toBeNull();
+  return match![1];
+}
+
+// ── playbook_create ─────────────────────────────────────────────────
+
+describe('playbook_create tool', () => {
+  beforeEach(clearPlaybooks);
+
+  test('creates a playbook with required fields', async () => {
+    const result = await executePlaybookCreate({
+      trigger: 'meeting request',
+      action: 'check calendar, propose 3 times',
+    }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('Playbook created successfully');
+    expect(result.content).toContain('meeting request');
+    expect(result.content).toContain('check calendar, propose 3 times');
+    expect(result.content).toContain('Autonomy: draft for review'); // default
+    expect(result.content).toContain('Channel: *'); // default
+    expect(result.content).toContain('Category: general'); // default
+    expect(result.content).toContain('Priority: 0'); // default
+  });
+
+  test('creates a playbook with all optional fields', async () => {
+    const result = await executePlaybookCreate({
+      trigger: 'from:ceo@*',
+      action: 'prioritize and draft response',
+      channel: 'email',
+      category: 'triage',
+      autonomy_level: 'auto',
+      priority: 10,
+    }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('from:ceo@*');
+    expect(result.content).toContain('Channel: email');
+    expect(result.content).toContain('Category: triage');
+    expect(result.content).toContain('Autonomy: execute automatically');
+    expect(result.content).toContain('Priority: 10');
+  });
+
+  test('creates with notify autonomy level', async () => {
+    const result = await executePlaybookCreate({
+      trigger: 'newsletter',
+      action: 'archive',
+      autonomy_level: 'notify',
+    }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('Autonomy: notify only');
+  });
+
+  test('rejects duplicate playbook', async () => {
+    await executePlaybookCreate({
+      trigger: 'unique trigger',
+      action: 'unique action',
+    }, ctx);
+
+    const result = await executePlaybookCreate({
+      trigger: 'unique trigger',
+      action: 'unique action',
+    }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('already exists');
+  });
+
+  test('rejects missing trigger', async () => {
+    const result = await executePlaybookCreate({
+      action: 'do something',
+    }, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('trigger is required');
+  });
+
+  test('rejects missing action', async () => {
+    const result = await executePlaybookCreate({
+      trigger: 'test trigger',
+    }, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('action is required');
+  });
+});
+
+// ── playbook_list ───────────────────────────────────────────────────
+
+describe('playbook_list tool', () => {
+  beforeEach(clearPlaybooks);
+
+  test('returns empty message when no playbooks exist', async () => {
+    const result = await executePlaybookList({}, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('No playbooks found');
+  });
+
+  test('lists all playbooks', async () => {
+    await executePlaybookCreate({
+      trigger: 'meeting request',
+      action: 'check calendar',
+    }, ctx);
+    await executePlaybookCreate({
+      trigger: 'newsletter',
+      action: 'archive it',
+    }, ctx);
+
+    const result = await executePlaybookList({}, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('Found 2 playbook(s)');
+    expect(result.content).toContain('meeting request');
+    expect(result.content).toContain('newsletter');
+  });
+
+  test('filters by channel', async () => {
+    await executePlaybookCreate({
+      trigger: 'email trigger',
+      action: 'handle email',
+      channel: 'email',
+    }, ctx);
+    await executePlaybookCreate({
+      trigger: 'slack trigger',
+      action: 'handle slack',
+      channel: 'slack',
+    }, ctx);
+
+    const result = await executePlaybookList({ channel: 'email' }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('email trigger');
+    expect(result.content).not.toContain('slack trigger');
+  });
+
+  test('filters by category', async () => {
+    await executePlaybookCreate({
+      trigger: 'scheduling trigger',
+      action: 'schedule it',
+      category: 'scheduling',
+    }, ctx);
+    await executePlaybookCreate({
+      trigger: 'triage trigger',
+      action: 'triage it',
+      category: 'triage',
+    }, ctx);
+
+    const result = await executePlaybookList({ category: 'scheduling' }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('scheduling trigger');
+    expect(result.content).not.toContain('triage trigger');
+  });
+
+  test('includes wildcard channel playbooks in channel filter', async () => {
+    await executePlaybookCreate({
+      trigger: 'wildcard trigger',
+      action: 'handle anything',
+      channel: '*',
+    }, ctx);
+
+    const result = await executePlaybookList({ channel: 'email' }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('wildcard trigger');
+  });
+});
+
+// ── playbook_update ─────────────────────────────────────────────────
+
+describe('playbook_update tool', () => {
+  beforeEach(clearPlaybooks);
+
+  test('updates the trigger', async () => {
+    const createResult = await executePlaybookCreate({
+      trigger: 'old trigger',
+      action: 'do something',
+    }, ctx);
+    const id = extractPlaybookId(createResult.content);
+
+    const result = await executePlaybookUpdate({
+      playbook_id: id,
+      trigger: 'new trigger',
+    }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('Playbook updated successfully');
+    expect(result.content).toContain('new trigger');
+  });
+
+  test('updates multiple fields at once', async () => {
+    const createResult = await executePlaybookCreate({
+      trigger: 'test',
+      action: 'old action',
+    }, ctx);
+    const id = extractPlaybookId(createResult.content);
+
+    const result = await executePlaybookUpdate({
+      playbook_id: id,
+      action: 'new action',
+      channel: 'slack',
+      category: 'notifications',
+      autonomy_level: 'auto',
+      priority: 5,
+    }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('new action');
+    expect(result.content).toContain('Channel: slack');
+    expect(result.content).toContain('Category: notifications');
+    expect(result.content).toContain('Autonomy: execute automatically');
+    expect(result.content).toContain('Priority: 5');
+  });
+
+  test('rejects missing playbook_id', async () => {
+    const result = await executePlaybookUpdate({
+      trigger: 'new trigger',
+    }, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('playbook_id is required');
+  });
+
+  test('returns error for nonexistent playbook_id', async () => {
+    const result = await executePlaybookUpdate({
+      playbook_id: 'nonexistent',
+      trigger: 'test',
+    }, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('not found');
+  });
+});
+
+// ── playbook_delete ─────────────────────────────────────────────────
+
+describe('playbook_delete tool', () => {
+  beforeEach(clearPlaybooks);
+
+  test('deletes a playbook', async () => {
+    const createResult = await executePlaybookCreate({
+      trigger: 'delete me',
+      action: 'to be deleted',
+    }, ctx);
+    const id = extractPlaybookId(createResult.content);
+
+    const result = await executePlaybookDelete({ playbook_id: id }, ctx);
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('Playbook deleted');
+    expect(result.content).toContain('delete me');
+
+    // Verify it no longer appears in list
+    const listResult = await executePlaybookList({}, ctx);
+    expect(listResult.content).toContain('No playbooks found');
+  });
+
+  test('rejects missing playbook_id', async () => {
+    const result = await executePlaybookDelete({}, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('playbook_id is required');
+  });
+
+  test('returns error for nonexistent playbook_id', async () => {
+    const result = await executePlaybookDelete({ playbook_id: 'nonexistent' }, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('not found');
+  });
+});

package/src/__tests__/profile-compiler.test.ts

@@ -42,13 +42,14 @@ mock.module('../config/loader.js', () => ({
 }));
 
 import { estimateTextTokens } from '../context/token-estimator.js';
-import { getDb, initializeDb } from '../memory/db.js';
+import { getDb, initializeDb, resetDb } from '../memory/db.js';
 import { compileDynamicProfile } from '../memory/profile-compiler.js';
 import { memoryItems } from '../memory/schema.js';
 
 initializeDb();
 
 afterAll(() => {
+  resetDb();
   try { rmSync(testDir, { recursive: true }); } catch { /* best effort */ }
 });