vellum 0.2.0 → 0.2.2

package/src/__tests__/compaction.benchmark.test.ts

@@ -0,0 +1,176 @@
+/**
+ * Context Window Compaction Benchmark
+ *
+ * Measures compaction cost with a mock provider:
+ * - compaction latency under threshold pressure
+ * - no-op fast path for below-threshold histories
+ * - token reduction ratio after compaction
+ * - summary call count within expected range
+ * - severe pressure overriding cooldown
+ */
+import { describe, expect, mock, test } from 'bun:test';
+
+import { DEFAULT_CONFIG } from '../config/defaults.js';
+import { ContextWindowManager } from '../context/window-manager.js';
+import { estimatePromptTokens } from '../context/token-estimator.js';
+import type { Message, Provider } from '../providers/types.js';
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+function makeSummaryProvider(counter: { calls: number }): Provider {
+  return {
+    name: 'mock',
+    async sendMessage() {
+      counter.calls += 1;
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `## Goals\n- Preserve state\n## Constraints\n- Keep PRs small\n## Decisions\n- Call ${counter.calls}`,
+          },
+        ],
+        model: 'mock-model',
+        usage: { inputTokens: 420, outputTokens: 85 },
+        stopReason: 'end_turn',
+      };
+    },
+  };
+}
+
+function makeLongMessages(turns: number): Message[] {
+  const rows: Message[] = [];
+  for (let i = 0; i < turns; i++) {
+    rows.push({
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          text: `[U${i}] User message with enough content to estimate tokens. Topic ${i % 9}.`,
+        },
+      ],
+    });
+    rows.push({
+      role: 'assistant',
+      content: [
+        {
+          type: 'text',
+          text: `[A${i}] Assistant response with relevant content. Result ${i % 7}.`,
+        },
+      ],
+    });
+  }
+  return rows;
+}
+
+function makeConfig() {
+  return {
+    ...DEFAULT_CONFIG.contextWindow,
+    maxInputTokens: 6000,
+    targetInputTokens: 3200,
+    compactThreshold: 0.6,
+    preserveRecentUserTurns: 8,
+    chunkTokens: 1200,
+  };
+}
+
+describe('Compaction benchmark', () => {
+  test('compaction with mock provider completes under 500ms', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    // 90 turns = 180 messages, well above 60% of 6000 = 3600 threshold
+    const messages = makeLongMessages(90);
+    const before = estimatePromptTokens(messages, 'system prompt', {
+      providerName: 'mock',
+    });
+    expect(before).toBeGreaterThan(config.maxInputTokens * config.compactThreshold);
+
+    const start = performance.now();
+    const result = await manager.maybeCompact(messages);
+    const elapsed = performance.now() - start;
+
+    expect(result.compacted).toBe(true);
+    expect(elapsed).toBeLessThan(500);
+  });
+
+  test('below-threshold check returns in under 50ms (no-op)', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    // 3 turns = 6 messages, well below threshold
+    const messages = makeLongMessages(3);
+
+    const start = performance.now();
+    const result = await manager.maybeCompact(messages);
+    const elapsed = performance.now() - start;
+
+    expect(result.compacted).toBe(false);
+    expect(result.reason).toBe('below compaction threshold');
+    expect(elapsed).toBeLessThan(50);
+    expect(counter.calls).toBe(0);
+  });
+
+  test('token reduction ratio exceeds 30% after compaction', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const result = await manager.maybeCompact(messages);
+
+    expect(result.compacted).toBe(true);
+    const reductionRatio =
+      (result.previousEstimatedInputTokens - result.estimatedInputTokens) /
+      result.previousEstimatedInputTokens;
+    expect(reductionRatio).toBeGreaterThan(0.3);
+  });
+
+  test('summary calls fall within 2-6 range', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const result = await manager.maybeCompact(messages);
+
+    expect(result.compacted).toBe(true);
+    expect(result.summaryCalls).toBeGreaterThanOrEqual(2);
+    expect(result.summaryCalls).toBeLessThanOrEqual(6);
+    expect(result.summaryCalls).toBe(counter.calls);
+  });
+
+  test('severe pressure triggers compaction even during cooldown', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    // Use a tighter maxInputTokens so 90 turns exceeds the 95% severe threshold
+    const config = {
+      ...makeConfig(),
+      maxInputTokens: 4000,
+      targetInputTokens: 2000,
+    };
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const estimated = estimatePromptTokens(messages, 'system prompt', {
+      providerName: 'mock',
+    });
+    expect(estimated).toBeGreaterThan(config.maxInputTokens * 0.95);
+
+    // Simulate being within cooldown by setting lastCompactedAt to now
+    const result = await manager.maybeCompact(messages, undefined, {
+      lastCompactedAt: Date.now(),
+    });
+
+    expect(result.compacted).toBe(true);
+    expect(result.summaryCalls).toBeGreaterThan(0);
+  });
+});

package/src/__tests__/computer-use-tools.test.ts

@@ -0,0 +1,250 @@
+import { describe, test, expect } from 'bun:test';
+import {
+  allComputerUseTools,
+  computerUseClickTool,
+  computerUseDoubleClickTool,
+  computerUseRightClickTool,
+  computerUseTypeTextTool,
+  computerUseKeyTool,
+  computerUseScrollTool,
+  computerUseDragTool,
+  computerUseWaitTool,
+  computerUseOpenAppTool,
+  computerUseRunAppleScriptTool,
+  computerUseDoneTool,
+  computerUseRespondTool,
+} from '../tools/computer-use/definitions.js';
+import { requestComputerControlTool } from '../tools/computer-use/request-computer-control.js';
+import { forwardComputerUseProxyTool } from '../tools/computer-use/skill-proxy-bridge.js';
+import type { ToolContext } from '../tools/types.js';
+
+interface JsonSchema {
+  type?: string;
+  required?: string[];
+  properties?: Record<string, unknown>;
+}
+
+/** Cast a tool definition's input_schema to a usable JSON Schema shape. */
+function schema(tool: { getDefinition(): { input_schema: object } }): JsonSchema {
+  return tool.getDefinition().input_schema as JsonSchema;
+}
+
+const ctx: ToolContext = {
+  workingDir: '/tmp',
+  sessionId: 'test-session',
+  conversationId: 'test-conversation',
+};
+
+// ── Tool definitions ────────────────────────────────────────────────
+
+describe('computer-use tool definitions', () => {
+  test('allComputerUseTools contains 12 tools', () => {
+    expect(allComputerUseTools.length).toBe(12);
+  });
+
+  test('all tools have proxy execution mode', () => {
+    for (const tool of allComputerUseTools) {
+      expect(tool.executionMode).toBe('proxy');
+    }
+    expect(requestComputerControlTool.executionMode).toBe('proxy');
+  });
+
+  test('all tools belong to computer-use category', () => {
+    for (const tool of allComputerUseTools) {
+      expect(tool.category).toBe('computer-use');
+    }
+    expect(requestComputerControlTool.category).toBe('computer-use');
+  });
+
+  test('all tools have unique names', () => {
+    const names = allComputerUseTools.map((t) => t.name);
+    expect(new Set(names).size).toBe(names.length);
+  });
+
+  test('all tools have descriptions', () => {
+    for (const tool of allComputerUseTools) {
+      expect(tool.description.length).toBeGreaterThan(0);
+    }
+  });
+});
+
+// ── Click tool variants ─────────────────────────────────────────────
+
+describe('click tool variants', () => {
+  for (const [tool, label] of [
+    [computerUseClickTool, 'click'],
+    [computerUseDoubleClickTool, 'double_click'],
+    [computerUseRightClickTool, 'right_click'],
+  ] as const) {
+    test(`${tool.name} has correct name`, () => {
+      expect(tool.name).toBe(`computer_use_${label}`);
+    });
+
+    test(`${tool.name} schema requires reasoning`, () => {
+      expect(schema(tool).required).toContain('reasoning');
+    });
+
+    test(`${tool.name} schema supports element_id and coordinates`, () => {
+      const props = schema(tool).properties as Record<string, { type: string }>;
+      expect(props.element_id.type).toBe('integer');
+      expect(props.x.type).toBe('integer');
+      expect(props.y.type).toBe('integer');
+    });
+
+    test(`${tool.name} execute throws proxy error`, () => {
+      expect(() => tool.execute({}, ctx)).toThrow('Proxy tool');
+    });
+  }
+});
+
+// ── type_text ───────────────────────────────────────────────────────
+
+describe('computer_use_type_text', () => {
+  test('requires text and reasoning', () => {
+    expect(schema(computerUseTypeTextTool).required).toContain('text');
+    expect(schema(computerUseTypeTextTool).required).toContain('reasoning');
+  });
+
+  test('execute throws proxy error', () => {
+    expect(() => computerUseTypeTextTool.execute({}, ctx)).toThrow('Proxy tool');
+  });
+});
+
+// ── key ─────────────────────────────────────────────────────────────
+
+describe('computer_use_key', () => {
+  test('requires key and reasoning', () => {
+    expect(schema(computerUseKeyTool).required).toContain('key');
+    expect(schema(computerUseKeyTool).required).toContain('reasoning');
+  });
+
+  test('execute throws proxy error', () => {
+    expect(() => computerUseKeyTool.execute({}, ctx)).toThrow('Proxy tool');
+  });
+});
+
+// ── scroll ──────────────────────────────────────────────────────────
+
+describe('computer_use_scroll', () => {
+  test('requires direction, amount, and reasoning', () => {
+    expect(schema(computerUseScrollTool).required).toContain('direction');
+    expect(schema(computerUseScrollTool).required).toContain('amount');
+    expect(schema(computerUseScrollTool).required).toContain('reasoning');
+  });
+
+  test('direction enum includes up, down, left, right', () => {
+    const props = schema(computerUseScrollTool).properties as Record<string, { enum?: string[] }>;
+    expect(props.direction.enum).toEqual(['up', 'down', 'left', 'right']);
+  });
+});
+
+// ── drag ────────────────────────────────────────────────────────────
+
+describe('computer_use_drag', () => {
+  test('supports source and destination coordinates', () => {
+    const props = schema(computerUseDragTool).properties as Record<string, { type: string }>;
+    expect(props.element_id.type).toBe('integer');
+    expect(props.to_element_id.type).toBe('integer');
+    expect(props.x.type).toBe('integer');
+    expect(props.y.type).toBe('integer');
+    expect(props.to_x.type).toBe('integer');
+    expect(props.to_y.type).toBe('integer');
+  });
+
+  test('requires reasoning only', () => {
+    expect(schema(computerUseDragTool).required).toEqual(['reasoning']);
+  });
+});
+
+// ── wait ────────────────────────────────────────────────────────────
+
+describe('computer_use_wait', () => {
+  test('requires duration_ms and reasoning', () => {
+    expect(schema(computerUseWaitTool).required).toContain('duration_ms');
+    expect(schema(computerUseWaitTool).required).toContain('reasoning');
+  });
+});
+
+// ── open_app ────────────────────────────────────────────────────────
+
+describe('computer_use_open_app', () => {
+  test('requires app_name and reasoning', () => {
+    expect(schema(computerUseOpenAppTool).required).toContain('app_name');
+    expect(schema(computerUseOpenAppTool).required).toContain('reasoning');
+  });
+});
+
+// ── run_applescript ─────────────────────────────────────────────────
+
+describe('computer_use_run_applescript', () => {
+  test('requires script and reasoning', () => {
+    expect(schema(computerUseRunAppleScriptTool).required).toContain('script');
+    expect(schema(computerUseRunAppleScriptTool).required).toContain('reasoning');
+  });
+
+  test('description warns against do shell script', () => {
+    expect(computerUseRunAppleScriptTool.description).toContain('do shell script');
+    expect(computerUseRunAppleScriptTool.description).toContain('blocked');
+  });
+});
+
+// ── done ────────────────────────────────────────────────────────────
+
+describe('computer_use_done', () => {
+  test('requires summary', () => {
+    expect(schema(computerUseDoneTool).required).toContain('summary');
+  });
+});
+
+// ── respond ─────────────────────────────────────────────────────────
+
+describe('computer_use_respond', () => {
+  test('requires answer and reasoning', () => {
+    expect(schema(computerUseRespondTool).required).toContain('answer');
+    expect(schema(computerUseRespondTool).required).toContain('reasoning');
+  });
+});
+
+// ── request_computer_control ────────────────────────────────────────
+
+describe('computer_use_request_control', () => {
+  test('requires task parameter', () => {
+    expect(schema(requestComputerControlTool).required).toContain('task');
+  });
+
+  test('execute throws proxy error', () => {
+    expect(() => requestComputerControlTool.execute({}, ctx)).toThrow('surfaceProxyResolver');
+  });
+});
+
+// ── skill-proxy-bridge ──────────────────────────────────────────────
+
+describe('forwardComputerUseProxyTool', () => {
+  test('returns error when no proxy resolver available', async () => {
+    const result = await forwardComputerUseProxyTool('computer_use_click', {}, ctx);
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('no proxy resolver available');
+    expect(result.content).toContain('computer_use_click');
+  });
+
+  test('delegates to proxy resolver when available', async () => {
+    const ctxWithProxy: ToolContext = {
+      ...ctx,
+      proxyToolResolver: async (name: string, input: Record<string, unknown>) => ({
+        content: `Forwarded ${name} with ${JSON.stringify(input)}`,
+        isError: false,
+      }),
+    };
+
+    const result = await forwardComputerUseProxyTool(
+      'computer_use_screenshot',
+      { reasoning: 'test' },
+      ctxWithProxy,
+    );
+
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain('Forwarded computer_use_screenshot');
+    expect(result.content).toContain('test');
+  });
+});