vellum 0.2.1 → 0.2.7

package/src/__tests__/intent-routing.test.ts

@@ -1,5 +1,5 @@
 import { describe, test, expect, beforeEach, afterEach, mock } from 'bun:test';
-import { mkdirSync, rmSync, existsSync } from 'node:fs';
+import { mkdirSync, rmSync, existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 
@@ -51,13 +51,24 @@ mock.module('../config/loader.js', () => ({
 
 // ── Import after mocks ───────────────────────────────────────────────
 const { buildSystemPrompt } = await import('../config/system-prompt.js');
-const { taskListAddTool } = await import('../tools/tasks/work-item-enqueue.js');
-const { reminderTool } = await import('../tools/reminder/reminder.js');
 
-// schedule_create is registered via side-effect import; import the module
-// to access the tool description through the registry.
-await import('../tools/schedule/create.js');
-const { getTool } = await import('../tools/registry.js');
+// Load task_list_add description from the bundled skill TOOLS.json
+const tasksToolsJson = JSON.parse(
+  readFileSync(join(import.meta.dirname, '../config/bundled-skills/tasks/TOOLS.json'), 'utf-8'),
+);
+const taskListAddDef = tasksToolsJson.tools.find((t: { name: string }) => t.name === 'task_list_add');
+
+// Load reminder_create description from the bundled skill TOOLS.json
+const reminderToolsJson = JSON.parse(
+  readFileSync(join(import.meta.dirname, '../config/bundled-skills/reminder/TOOLS.json'), 'utf-8'),
+);
+const reminderCreateDef = reminderToolsJson.tools.find((t: { name: string }) => t.name === 'reminder_create');
+
+// Load schedule_create description from the bundled skill TOOLS.json
+const scheduleToolsJson = JSON.parse(
+  readFileSync(join(import.meta.dirname, '../config/bundled-skills/schedule/TOOLS.json'), 'utf-8'),
+);
+const scheduleCreateDef = scheduleToolsJson.tools.find((t: { name: string }) => t.name === 'schedule_create');
 
 // =====================================================================
 // 1. System prompt: buildTaskScheduleReminderRoutingSection
@@ -81,9 +92,9 @@ describe('Task/Schedule/Reminder routing section in system prompt', () => {
 
   test('routing section explains all three subsystems', () => {
     const prompt = buildSystemPrompt();
-    expect(prompt).toContain('### Task Queue (task_list_add / task_list_show)');
+    expect(prompt).toContain('### Task Queue (task_list_add / task_list_show / task_list_update / task_list_remove)');
     expect(prompt).toContain('### Schedules (schedule_create / schedule_list / schedule_update / schedule_delete)');
-    expect(prompt).toContain('### Reminders (reminder)');
+    expect(prompt).toContain('### Reminders (reminder_create / reminder_list / reminder_cancel)');
   });
 
   test('routing section contains key routing phrases for task queue', () => {
@@ -103,10 +114,30 @@ describe('Task/Schedule/Reminder routing section in system prompt', () => {
   test('routing section clarifies schedules are for recurring automation only', () => {
     const prompt = buildSystemPrompt();
     expect(prompt).toContain('recurring automated jobs');
-    expect(prompt).toContain('cron schedule');
+    expect(prompt).toContain('recurrence schedule (cron or RRULE)');
     expect(prompt).toContain('ONLY when the user explicitly wants');
   });
 
+  test('routing section documents supported RRULE set constructs', () => {
+    const prompt = buildSystemPrompt();
+    expect(prompt).toContain('#### RRULE Set Constructs');
+    expect(prompt).toContain('**RDATE**');
+    expect(prompt).toContain('**EXDATE**');
+    expect(prompt).toContain('**EXRULE**');
+    expect(prompt).toContain('multiple RRULE lines form a union');
+  });
+
+  test('routing section documents bounded recurrence patterns', () => {
+    const prompt = buildSystemPrompt();
+    expect(prompt).toContain('Bounded recurrence');
+    expect(prompt).toContain('COUNT or UNTIL');
+  });
+
+  test('routing section states exclusion precedence', () => {
+    const prompt = buildSystemPrompt();
+    expect(prompt).toContain('Exclusions (EXDATE, EXRULE) take precedence over inclusions (RRULE, RDATE)');
+  });
+
   test('routing section clarifies reminders are for time-triggered notifications', () => {
     const prompt = buildSystemPrompt();
     expect(prompt).toContain('one-time time-triggered notifications');
@@ -117,7 +148,7 @@ describe('Task/Schedule/Reminder routing section in system prompt', () => {
     const prompt = buildSystemPrompt();
     expect(prompt).toContain('### Common mistakes to avoid');
     // The key mis-routing guard: "add this to my tasks" should go to task_list_add
-    expect(prompt).toContain('"Add this to my tasks" → task_list_add (NOT schedule_create or reminder)');
+    expect(prompt).toContain('"Add this to my tasks" → task_list_add (NOT schedule_create or reminder_create)');
   });
 
   test('routing section distinguishes timed vs untimed "remind me"', () => {
@@ -125,7 +156,7 @@ describe('Task/Schedule/Reminder routing section in system prompt', () => {
     // Without a time → task queue
     expect(prompt).toContain('"Remind me to buy groceries" without a time → task_list_add');
     // With a time → reminder
-    expect(prompt).toContain('"Remind me at 5pm to buy groceries" → reminder');
+    expect(prompt).toContain('"Remind me at 5pm to buy groceries" → reminder_create');
   });
 
   test('routing section appears after tool routing by content type', () => {
@@ -144,85 +175,70 @@ describe('Task/Schedule/Reminder routing section in system prompt', () => {
 
 describe('task_list_add tool description', () => {
   test('mentions "add to my tasks" routing phrase', () => {
-    const def = taskListAddTool.getDefinition();
+    const def = taskListAddDef;
     expect(def.description).toContain('add to my tasks');
   });
 
   test('mentions "add to my queue" routing phrase', () => {
-    const def = taskListAddTool.getDefinition();
+    const def = taskListAddDef;
     expect(def.description).toContain('add to my queue');
   });
 
   test('mentions "put this on my task list" routing phrase', () => {
-    const def = taskListAddTool.getDefinition();
+    const def = taskListAddDef;
     expect(def.description).toContain('put this on my task list');
   });
 
   test('mentions ad-hoc title-only mode', () => {
-    const def = taskListAddTool.getDefinition();
+    const def = taskListAddDef;
     expect(def.description).toContain('just a title');
   });
 
   test('explicitly warns NOT to use schedule_create or reminder for task requests', () => {
-    const def = taskListAddTool.getDefinition();
+    const def = taskListAddDef;
     expect(def.description).toContain('Do NOT use schedule_create or reminder');
   });
 });
 
 describe('schedule_create tool description', () => {
   test('mentions recurring scheduled automation', () => {
-    const tool = getTool('schedule_create');
-    expect(tool).toBeDefined();
-    const def = tool!.getDefinition();
-    expect(def.description).toContain('recurring');
+    expect(scheduleCreateDef).toBeDefined();
+    expect(scheduleCreateDef.description).toContain('recurring');
   });
 
   test('mentions cron interval', () => {
-    const tool = getTool('schedule_create');
-    const def = tool!.getDefinition();
-    expect(def.description).toContain('cron');
+    expect(scheduleCreateDef.description).toContain('cron');
   });
 
   test('warns against using for "add to my tasks" requests', () => {
-    const tool = getTool('schedule_create');
-    const def = tool!.getDefinition();
-    expect(def.description).toContain('Do NOT use this for "add to my tasks"');
+    expect(scheduleCreateDef.description).toContain('Do NOT use this for "add to my tasks"');
   });
 
   test('redirects to task_list_add for task queue items', () => {
-    const tool = getTool('schedule_create');
-    const def = tool!.getDefinition();
-    expect(def.description).toContain('task_list_add');
+    expect(scheduleCreateDef.description).toContain('task_list_add');
   });
 
   test('does NOT suggest it handles task queue items', () => {
-    const tool = getTool('schedule_create');
-    const def = tool!.getDefinition();
-    // Should not claim to handle one-off task items
-    expect(def.description).not.toContain('task queue');
-    expect(def.description).not.toContain('one-off');
+    expect(scheduleCreateDef.description).not.toContain('task queue');
+    expect(scheduleCreateDef.description).not.toContain('one-off');
   });
 });
 
 describe('reminder tool description', () => {
   test('mentions time-based reminders', () => {
-    const def = reminderTool.getDefinition();
-    expect(def.description).toContain('time-based reminders');
+    expect(reminderCreateDef.description).toContain('time-based reminder');
   });
 
   test('scopes to time-triggered notifications only', () => {
-    const def = reminderTool.getDefinition();
-    expect(def.description).toContain('ONLY when the user wants a time-triggered notification');
+    expect(reminderCreateDef.description).toContain('ONLY when the user wants a time-triggered notification');
   });
 
   test('warns against using for "add to my tasks" requests', () => {
-    const def = reminderTool.getDefinition();
-    expect(def.description).toContain('Do NOT use this for "add to my tasks"');
+    expect(reminderCreateDef.description).toContain('Do NOT use this for "add to my tasks"');
   });
 
   test('redirects to task_list_add for task queue items', () => {
-    const def = reminderTool.getDefinition();
-    expect(def.description).toContain('task_list_add');
+    expect(reminderCreateDef.description).toContain('task_list_add');
   });
 });
 
@@ -232,24 +248,15 @@ describe('reminder tool description', () => {
 
 describe('cross-tool routing consistency', () => {
   test('all three tools reference task_list_add as the task-queue tool', () => {
-    const enqueueDef = taskListAddTool.getDefinition();
-    const scheduleTool = getTool('schedule_create')!;
-    const scheduleDef = scheduleTool.getDefinition();
-    const reminderDef = reminderTool.getDefinition();
-
     // task_list_add is the canonical name in all three descriptions
-    expect(enqueueDef.name).toBe('task_list_add');
-    expect(scheduleDef.description).toContain('task_list_add');
-    expect(reminderDef.description).toContain('task_list_add');
+    expect(taskListAddDef.name).toBe('task_list_add');
+    expect(scheduleCreateDef.description).toContain('task_list_add');
+    expect(reminderCreateDef.description).toContain('task_list_add');
   });
 
   test('schedule_create and reminder both reject "add to my queue" usage', () => {
-    const scheduleTool = getTool('schedule_create')!;
-    const scheduleDef = scheduleTool.getDefinition();
-    const reminderDef = reminderTool.getDefinition();
-
     // Both should redirect away from task-queue requests
-    expect(scheduleDef.description).toContain('add to my queue');
-    expect(reminderDef.description).toContain('add to my queue');
+    expect(scheduleCreateDef.description).toContain('add to my queue');
+    expect(reminderCreateDef.description).toContain('add to my queue');
   });
 });

package/src/__tests__/ipc-roundtrip.benchmark.test.ts

@@ -0,0 +1,237 @@
+/**
+ * IPC Serialization/Parsing Benchmark
+ *
+ * Measures serialize + parse round-trip performance in isolation
+ * (no daemon required). Target ranges:
+ * - Small message p95: < 1ms (averaged over 1000 runs)
+ * - Large message (1MB): < 50ms
+ * - Rapid-fire: no message loss across 100 messages
+ * - Round-trip: content preserved exactly
+ */
+import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test';
+import * as net from 'node:net';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+import { serialize, createMessageParser } from '../daemon/ipc-protocol.js';
+import type { ClientMessage, ServerMessage } from '../daemon/ipc-contract.js';
+
+function percentile(values: number[], p: number): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.ceil((p / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, idx)];
+}
+
+describe('IPC round-trip benchmark', () => {
+  test('small message serialize + parse p95 < 1ms over 1000 runs', () => {
+    const msg: ClientMessage = { type: 'ping' };
+    const parser = createMessageParser();
+    const timings: number[] = [];
+
+    for (let i = 0; i < 1000; i++) {
+      const start = performance.now();
+      const serialized = serialize(msg);
+      const parsed = parser.feed(serialized);
+      const elapsed = performance.now() - start;
+      timings.push(elapsed);
+
+      // Sanity: each iteration should yield exactly one message
+      expect(parsed).toHaveLength(1);
+    }
+
+    const p95 = percentile(timings, 95);
+    expect(p95).toBeLessThan(1);
+  });
+
+  test('large message (1MB) serialize + parse < 50ms', () => {
+    // Build a ~1MB payload using assistant_text_delta with a large text field
+    const largeText = 'x'.repeat(1024 * 1024);
+    const msg: ServerMessage = { type: 'assistant_text_delta', text: largeText };
+    const parser = createMessageParser();
+    const timings: number[] = [];
+
+    for (let i = 0; i < 10; i++) {
+      const start = performance.now();
+      const serialized = serialize(msg);
+      const parsed = parser.feed(serialized);
+      const elapsed = performance.now() - start;
+      timings.push(elapsed);
+
+      expect(parsed).toHaveLength(1);
+    }
+
+    const p95 = percentile(timings, 95);
+    expect(p95).toBeLessThan(50);
+  });
+
+  test('no message loss under rapid-fire (100 messages)', () => {
+    const parser = createMessageParser();
+    const messages: ClientMessage[] = [];
+
+    for (let i = 0; i < 100; i++) {
+      messages.push({ type: 'session_list' } as ClientMessage);
+    }
+
+    // Serialize all messages and concatenate into a single buffer
+    const combined = messages.map((m) => serialize(m)).join('');
+
+    // Feed the entire buffer at once
+    const parsed = parser.feed(combined);
+
+    expect(parsed).toHaveLength(100);
+    for (const p of parsed) {
+      expect(p).toHaveProperty('type', 'session_list');
+    }
+  });
+
+  test('serialize + parse round-trip preserves message content', () => {
+    const parser = createMessageParser();
+
+    const clientMsg: ClientMessage = {
+      type: 'user_message',
+      sessionId: 'sess-abc-123',
+      content: 'Hello, world! Special chars: \u00e9\u00e0\u00fc \ud83d\ude00 "quotes" & <angle>',
+      attachments: [
+        {
+          filename: 'test.txt',
+          mimeType: 'text/plain',
+          data: 'SGVsbG8gV29ybGQ=',
+        },
+      ],
+    };
+
+    const serialized = serialize(clientMsg);
+    const parsed = parser.feed(serialized);
+
+    expect(parsed).toHaveLength(1);
+    const roundTripped = parsed[0] as ClientMessage;
+    expect(roundTripped).toEqual(clientMsg);
+
+    // Verify specific fields survived the round-trip
+    expect(roundTripped.type).toBe('user_message');
+    if (roundTripped.type === 'user_message') {
+      expect(roundTripped.sessionId).toBe('sess-abc-123');
+      expect(roundTripped.content).toContain('\ud83d\ude00');
+      expect(roundTripped.attachments).toHaveLength(1);
+      expect(roundTripped.attachments![0].filename).toBe('test.txt');
+    }
+  });
+});
+
+describe('IPC Unix socket round-trip benchmark', () => {
+  let server: net.Server;
+  let client: net.Socket;
+  let tmpDir: string;
+  let socketPath: string;
+
+  beforeAll(async () => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ipc-bench-'));
+    socketPath = path.join(tmpDir, 'bench.sock');
+
+    // Server: parse incoming messages, echo back a session_list_response for each
+    server = net.createServer((socket) => {
+      const parser = createMessageParser();
+      socket.on('data', (data) => {
+        const msgs = parser.feed(data.toString());
+        for (const _msg of msgs) {
+          const response: ServerMessage = {
+            type: 'session_list_response',
+            sessions: [],
+          };
+          socket.write(serialize(response));
+        }
+      });
+    });
+
+    await new Promise<void>((resolve) => server.listen(socketPath, resolve));
+
+    client = net.createConnection(socketPath);
+    await new Promise<void>((resolve) => client.on('connect', resolve));
+  });
+
+  afterAll(async () => {
+    client?.destroy();
+    await new Promise<void>((resolve) => server?.close(() => resolve()));
+    try { fs.unlinkSync(socketPath); } catch { /* server.close() may already remove it */ }
+    try { fs.rmdirSync(tmpDir); } catch { /* best-effort */ }
+  });
+
+  test('session_list round-trip p50 < 5ms, p99 < 50ms', async () => {
+    const clientParser = createMessageParser();
+    const timings: number[] = [];
+    const iterations = 50;
+
+    for (let i = 0; i < iterations; i++) {
+      const start = performance.now();
+
+      const responsePromise = new Promise<ServerMessage>((resolve) => {
+        const handler = (data: Buffer) => {
+          const msgs = clientParser.feed(data.toString());
+          if (msgs.length > 0) {
+            client.removeListener('data', handler);
+            resolve(msgs[0] as ServerMessage);
+          }
+        };
+        client.on('data', handler);
+      });
+
+      const request: ClientMessage = { type: 'session_list' };
+      client.write(serialize(request));
+
+      const response = await responsePromise;
+      const elapsed = performance.now() - start;
+      timings.push(elapsed);
+
+      expect(response.type).toBe('session_list_response');
+    }
+
+    const p50 = percentile(timings, 50);
+    const p99 = percentile(timings, 99);
+    expect(p50).toBeLessThan(5);
+    expect(p99).toBeLessThan(50);
+  });
+
+  test('rapid-fire: 100 messages over socket without loss', async () => {
+    const clientParser = createMessageParser();
+    const messageCount = 100;
+    const received: ServerMessage[] = [];
+
+    const allReceived = new Promise<void>((resolve) => {
+      const handler = (data: Buffer) => {
+        const msgs = clientParser.feed(data.toString());
+        for (const msg of msgs) {
+          received.push(msg as ServerMessage);
+        }
+        if (received.length >= messageCount) {
+          client.removeListener('data', handler);
+          resolve();
+        }
+      };
+      client.on('data', handler);
+    });
+
+    const start = performance.now();
+    for (let i = 0; i < messageCount; i++) {
+      const request: ClientMessage = { type: 'session_list' };
+      client.write(serialize(request));
+    }
+
+    await allReceived;
+    const elapsed = performance.now() - start;
+
+    expect(received).toHaveLength(messageCount);
+    for (const msg of received) {
+      expect(msg.type).toBe('session_list_response');
+    }
+    // All 100 messages should complete well within 100ms on a local socket
+    expect(elapsed).toBeLessThan(100);
+  });
+});

package/src/__tests__/ipc-snapshot.test.ts

@@ -339,10 +339,24 @@ const clientMessages: Record<ClientMessageType, ClientMessage> = {
     type: 'slack_webhook_config',
     action: 'get',
   },
+  twilio_webhook_config: {
+    type: 'twilio_webhook_config',
+    action: 'get',
+  },
   vercel_api_config: {
     type: 'vercel_api_config',
     action: 'get',
   },
+  twitter_integration_config: {
+    type: 'twitter_integration_config',
+    action: 'get',
+  },
+  twitter_auth_start: {
+    type: 'twitter_auth_start',
+  },
+  twitter_auth_status: {
+    type: 'twitter_auth_status',
+  },
   link_open_request: {
     type: 'link_open_request',
     url: 'https://example.com',
@@ -423,14 +437,6 @@ const clientMessages: Record<ClientMessageType, ClientMessage> = {
     type: 'work_item_get',
     id: 'wi-001',
   },
-  work_item_create: {
-    type: 'work_item_create',
-    taskId: 'task-001',
-    title: 'Process report',
-    notes: 'High priority',
-    priorityTier: 1,
-    sortIndex: 0,
-  },
   work_item_update: {
     type: 'work_item_update',
     id: 'wi-001',
@@ -453,6 +459,19 @@ const clientMessages: Record<ClientMessageType, ClientMessage> = {
     type: 'work_item_output',
     id: 'wi-001',
   },
+  work_item_preflight: {
+    type: 'work_item_preflight',
+    id: 'wi-001',
+  },
+  work_item_approve_permissions: {
+    type: 'work_item_approve_permissions',
+    id: 'wi-001',
+    approvedTools: ['bash', 'file_write'],
+  },
+  work_item_cancel: {
+    type: 'work_item_cancel',
+    id: 'wi-001',
+  },
   document_save: {
     type: 'document_save',
     surfaceId: 'doc-001',
@@ -476,7 +495,6 @@ const clientMessages: Record<ClientMessageType, ClientMessage> = {
   subagent_status: {
     type: 'subagent_status',
     subagentId: 'sub-001',
-    sessionId: 'sess-001',
   },
   subagent_message: {
     type: 'subagent_message',
@@ -907,6 +925,8 @@ const serverMessages: Record<ServerMessageType, ServerMessage> = {
         id: 'sched-001',
         name: 'Daily standup reminder',
         enabled: true,
+        syntax: 'cron',
+        expression: '0 9 * * 1-5',
         cronExpression: '0 9 * * 1-5',
         timezone: 'America/Los_Angeles',
         message: 'Remind me about the standup',
@@ -1109,11 +1129,35 @@ const serverMessages: Record<ServerMessageType, ServerMessage> = {
     webhookUrl: 'https://hooks.slack.com/services/T00/B00/xxx',
     success: true,
   },
+  twilio_webhook_config_response: {
+    type: 'twilio_webhook_config_response',
+    webhookBaseUrl: 'https://example.com/twilio',
+    success: true,
+  },
   vercel_api_config_response: {
     type: 'vercel_api_config_response',
     hasToken: true,
     success: true,
   },
+  twitter_integration_config_response: {
+    type: 'twitter_integration_config_response',
+    success: true,
+    mode: 'local_byo',
+    managedAvailable: false,
+    localClientConfigured: true,
+    connected: false,
+  },
+  twitter_auth_result: {
+    type: 'twitter_auth_result',
+    success: true,
+    accountInfo: '@vellum_test',
+  },
+  twitter_auth_status_response: {
+    type: 'twitter_auth_status_response',
+    connected: true,
+    accountInfo: '@vellum_test',
+    mode: 'local_byo',
+  },
   open_url: {
     type: 'open_url',
     url: 'https://example.com',
@@ -1283,25 +1327,6 @@ const serverMessages: Record<ServerMessageType, ServerMessage> = {
       updatedAt: 1700000000,
     },
   },
-  work_item_create_response: {
-    type: 'work_item_create_response',
-    item: {
-      id: 'wi-001',
-      taskId: 'task-001',
-      title: 'Process report',
-      notes: null,
-      status: 'queued',
-      priorityTier: 1,
-      sortIndex: null,
-      lastRunId: null,
-      lastRunConversationId: null,
-      lastRunStatus: null,
-      sourceType: null,
-      sourceId: null,
-      createdAt: 1700000000,
-      updatedAt: 1700000000,
-    },
-  },
   work_item_update_response: {
     type: 'work_item_update_response',
     item: {
@@ -1346,6 +1371,24 @@ const serverMessages: Record<ServerMessageType, ServerMessage> = {
       highlights: ['- Key finding 1', '- Key finding 2'],
     },
   },
+  work_item_preflight_response: {
+    type: 'work_item_preflight_response',
+    id: 'wi-001',
+    success: true,
+    permissions: [
+      { tool: 'bash', description: 'Run shell commands', riskLevel: 'medium', currentDecision: 'prompt' },
+    ],
+  },
+  work_item_approve_permissions_response: {
+    type: 'work_item_approve_permissions_response',
+    id: 'wi-001',
+    success: true,
+  },
+  work_item_cancel_response: {
+    type: 'work_item_cancel_response',
+    id: 'wi-001',
+    success: true,
+  },
   work_item_status_changed: {
     type: 'work_item_status_changed',
     item: {