vellum 0.2.14 → 0.3.2

package/src/__tests__/ephemeral-permissions.test.ts

@@ -1,362 +0,0 @@
-import { describe, test, expect, beforeAll, beforeEach, afterEach, mock } from 'bun:test';
-import { mkdtempSync, mkdirSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-
-// Use a temp directory so trust-store doesn't touch ~/.vellum
-const testDir = mkdtempSync(join(tmpdir(), 'ephemeral-perm-test-'));
-
-mock.module('../util/platform.js', () => ({
-  getRootDir: () => testDir,
-  getDataDir: () => join(testDir, 'data'),
-  getWorkspaceSkillsDir: () => join(testDir, 'skills'),
-  isMacOS: () => process.platform === 'darwin',
-  isLinux: () => process.platform === 'linux',
-  isWindows: () => process.platform === 'win32',
-  getSocketPath: () => join(testDir, 'test.sock'),
-  getPidPath: () => join(testDir, 'test.pid'),
-  getDbPath: () => join(testDir, 'test.db'),
-  getLogPath: () => join(testDir, 'test.log'),
-  ensureDataDir: () => {},
-}));
-
-mock.module('../util/logger.js', () => ({
-  getLogger: () => new Proxy({} as Record<string, unknown>, {
-    get: () => () => {},
-  }),
-}));
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const testConfig: Record<string, any> = {
-  permissions: { mode: 'legacy' as 'legacy' | 'strict' | 'workspace' },
-  skills: { load: { extraDirs: [] as string[] } },
-  sandbox: { enabled: false },
-};
-
-mock.module('../config/loader.js', () => ({
-  getConfig: () => testConfig,
-  loadConfig: () => testConfig,
-  invalidateConfigCache: () => {},
-  saveConfig: () => {},
-  loadRawConfig: () => ({}),
-  saveRawConfig: () => {},
-  getNestedValue: () => undefined,
-  setNestedValue: () => {},
-}));
-
-import { setTaskRunRules, getTaskRunRules, clearTaskRunRules, buildTaskRules } from '../tasks/ephemeral-permissions.js';
-import { findHighestPriorityRule, addRule, clearCache } from '../permissions/trust-store.js';
-import { check, classifyRisk } from '../permissions/checker.js';
-import type { TrustRule, PolicyContext } from '../permissions/types.js';
-
-// Ensure the protected directory exists for trust-store disk operations
-mkdirSync(join(testDir, 'protected'), { recursive: true });
-
-describe('ephemeral-permissions', () => {
-  // Warm up the shell parser (loads WASM) — required before any check() call
-  // that involves bash commands, otherwise the parser init may hang.
-  beforeAll(async () => {
-    await classifyRisk('bash', { command: 'echo warmup' });
-  });
-  describe('buildTaskRules', () => {
-    test('generates correct rule structure for each required tool', () => {
-      const taskRunId = 'run-123';
-      const requiredTools = ['file_read', 'bash', 'file_write'];
-      const workingDir = '/home/user/project';
-
-      const rules = buildTaskRules(taskRunId, requiredTools, workingDir);
-
-      expect(rules).toHaveLength(3);
-
-      // Check the first rule in detail
-      const fileReadRule = rules[0];
-      expect(fileReadRule.id).toBe('ephemeral:run-123:file_read');
-      expect(fileReadRule.tool).toBe('file_read');
-      expect(fileReadRule.pattern).toBe('**');
-      expect(fileReadRule.scope).toBe('everywhere');
-      expect(fileReadRule.decision).toBe('allow');
-      expect(fileReadRule.priority).toBe(75);
-      expect(fileReadRule.createdAt).toBeGreaterThan(0);
-
-      // allowHighRisk is set because task runs execute asynchronously
-      // without interactive confirmation — pre-approved via preflight
-      expect(fileReadRule.allowHighRisk).toBe(true);
-
-      // Check other rules have correct tool names
-      expect(rules[1].tool).toBe('bash');
-      expect(rules[1].id).toBe('ephemeral:run-123:bash');
-      expect(rules[2].tool).toBe('file_write');
-      expect(rules[2].id).toBe('ephemeral:run-123:file_write');
-    });
-
-    test('returns empty array for empty required tools', () => {
-      const rules = buildTaskRules('run-456', [], '/tmp');
-      expect(rules).toHaveLength(0);
-    });
-  });
-
-  describe('setTaskRunRules / getTaskRunRules / clearTaskRunRules', () => {
-    beforeEach(() => {
-      // Clean up any leftover state
-      clearTaskRunRules('test-run-1');
-      clearTaskRunRules('test-run-2');
-    });
-
-    test('returns empty array for unknown task run', () => {
-      expect(getTaskRunRules('nonexistent')).toEqual([]);
-    });
-
-    test('stores and retrieves rules for a task run', () => {
-      const rules = buildTaskRules('test-run-1', ['file_read'], '/tmp');
-      setTaskRunRules('test-run-1', rules);
-
-      const retrieved = getTaskRunRules('test-run-1');
-      expect(retrieved).toHaveLength(1);
-      expect(retrieved[0].tool).toBe('file_read');
-    });
-
-    test('clears rules for a task run', () => {
-      const rules = buildTaskRules('test-run-1', ['file_read'], '/tmp');
-      setTaskRunRules('test-run-1', rules);
-      expect(getTaskRunRules('test-run-1')).toHaveLength(1);
-
-      clearTaskRunRules('test-run-1');
-      expect(getTaskRunRules('test-run-1')).toEqual([]);
-    });
-
-    test('isolates rules between task runs', () => {
-      const rules1 = buildTaskRules('test-run-1', ['file_read'], '/tmp');
-      const rules2 = buildTaskRules('test-run-2', ['bash', 'file_write'], '/home');
-
-      setTaskRunRules('test-run-1', rules1);
-      setTaskRunRules('test-run-2', rules2);
-
-      expect(getTaskRunRules('test-run-1')).toHaveLength(1);
-      expect(getTaskRunRules('test-run-2')).toHaveLength(2);
-
-      clearTaskRunRules('test-run-1');
-      expect(getTaskRunRules('test-run-1')).toEqual([]);
-      expect(getTaskRunRules('test-run-2')).toHaveLength(2);
-    });
-  });
-
-  describe('findHighestPriorityRule with ephemeral rules', () => {
-    beforeEach(() => {
-      clearCache();
-    });
-
-    test('ephemeral rules are found by findHighestPriorityRule', () => {
-      const ephemeralRules: TrustRule[] = [{
-        id: 'ephemeral:run-1:file_read',
-        tool: 'file_read',
-        pattern: '**',
-        scope: '/home/user/project',
-        decision: 'allow',
-        priority: 50,
-        createdAt: Date.now(),
-      }];
-
-      const ctx: PolicyContext = {
-        ephemeralRules,
-      };
-
-      const result = findHighestPriorityRule(
-        'file_read',
-        ['file_read:/home/user/project/foo.txt'],
-        '/home/user/project',
-        ctx,
-      );
-
-      expect(result).not.toBeNull();
-      expect(result!.id).toBe('ephemeral:run-1:file_read');
-      expect(result!.decision).toBe('allow');
-    });
-
-    test('user deny rules at higher priority override ephemeral allow rules', () => {
-      // Add a persistent user deny rule with priority 100
-      addRule('file_read', '**', '/home/user/project', 'deny', 100);
-
-      const ephemeralRules: TrustRule[] = [{
-        id: 'ephemeral:run-1:file_read',
-        tool: 'file_read',
-        pattern: '**',
-        scope: '/home/user/project',
-        decision: 'allow',
-        priority: 50,
-        createdAt: Date.now(),
-      }];
-
-      const ctx: PolicyContext = {
-        ephemeralRules,
-      };
-
-      const result = findHighestPriorityRule(
-        'file_read',
-        ['file_read:/home/user/project/foo.txt'],
-        '/home/user/project',
-        ctx,
-      );
-
-      expect(result).not.toBeNull();
-      // The user deny rule (priority 100) should win over the ephemeral allow (priority 50)
-      expect(result!.decision).toBe('deny');
-    });
-
-    test('ephemeral rules do not match when scope is outside working dir', () => {
-      const ephemeralRules: TrustRule[] = [{
-        id: 'ephemeral:run-1:file_read',
-        tool: 'file_read',
-        pattern: '**',
-        scope: '/home/user/project',
-        decision: 'allow',
-        priority: 50,
-        createdAt: Date.now(),
-      }];
-
-      const ctx: PolicyContext = {
-        ephemeralRules,
-      };
-
-      // Query with a different scope that doesn't match the ephemeral rule's scope
-      const result = findHighestPriorityRule(
-        'file_read',
-        ['file_read:/other/path/foo.txt'],
-        '/other/path',
-        ctx,
-      );
-
-      // Should not match the ephemeral rule (scope mismatch)
-      // May or may not match a default rule
-      if (result) {
-        expect(result.id).not.toBe('ephemeral:run-1:file_read');
-      }
-    });
-  });
-
-  describe('check() with ephemeral rules', () => {
-    beforeEach(() => {
-      clearCache();
-      testConfig.permissions.mode = 'legacy';
-    });
-
-    test('ephemeral allow rule auto-allows non-high-risk tool', async () => {
-      const ephemeralRules: TrustRule[] = [{
-        id: 'ephemeral:run-1:file_read',
-        tool: 'file_read',
-        pattern: '**',
-        scope: 'everywhere',
-        decision: 'allow',
-        priority: 50,
-        createdAt: Date.now(),
-      }];
-
-      const ctx: PolicyContext = {
-        ephemeralRules,
-      };
-
-      // Use testDir (a real temp path) to avoid EPERM on macOS /home
-      const fakePath = join(testDir, 'foo.txt');
-      const result = await check(
-        'file_read',
-        { path: fakePath },
-        testDir,
-        ctx,
-      );
-
-      expect(result.decision).toBe('allow');
-    });
-
-    test('high-risk tool still prompts even with ephemeral allow rule (no allowHighRisk)', async () => {
-      const ephemeralRules: TrustRule[] = [{
-        id: 'ephemeral:run-1:bash',
-        tool: 'bash',
-        pattern: '**',
-        scope: 'everywhere',
-        decision: 'allow',
-        priority: 50,
-        createdAt: Date.now(),
-        // Note: allowHighRisk is NOT set
-      }];
-
-      const ctx: PolicyContext = {
-        ephemeralRules,
-      };
-
-      // sudo is high-risk
-      const result = await check(
-        'bash',
-        { command: 'sudo rm -rf /' },
-        '/home/user/project',
-        ctx,
-      );
-
-      expect(result.decision).toBe('prompt');
-    });
-  });
-
-  describe('workspace mode interactions', () => {
-    beforeEach(() => {
-      clearCache();
-      testConfig.permissions.mode = 'workspace';
-    });
-
-    afterEach(() => {
-      testConfig.permissions.mode = 'legacy';
-    });
-
-    test('workspace mode auto-allows workspace-scoped file_write (medium risk)', async () => {
-      const filePath = join(testDir, 'workspace-test-file.txt');
-      const result = await check('file_write', { path: filePath }, testDir);
-      expect(result.decision).toBe('allow');
-      expect(result.reason).toContain('Workspace mode');
-    });
-
-    test('workspace mode still prompts for file_write outside workspace', async () => {
-      const result = await check('file_write', { path: '/etc/config' }, testDir);
-      expect(result.decision).toBe('prompt');
-    });
-
-    test('explicit deny rule overrides workspace mode auto-allow', async () => {
-      addRule('file_write', '**', testDir, 'deny', 100);
-      const filePath = join(testDir, 'should-be-denied.txt');
-      const result = await check('file_write', { path: filePath }, testDir);
-      expect(result.decision).toBe('deny');
-    });
-
-    test('proxied bash still prompts in workspace mode', async () => {
-      const result = await check(
-        'bash',
-        { command: 'echo hello', network_mode: 'proxied' },
-        testDir,
-      );
-      expect(result.decision).toBe('prompt');
-      expect(result.reason).toContain('Proxied');
-    });
-
-    test('ephemeral task rules + workspace mode: deny rule wins', async () => {
-      // Add a persistent deny rule for file_write in the workspace
-      addRule('file_write', '**', testDir, 'deny', 100);
-
-      // Create ephemeral allow rules (lower priority than deny)
-      const ephemeralRules: TrustRule[] = [{
-        id: 'ephemeral:run-ws:file_write',
-        tool: 'file_write',
-        pattern: '**',
-        scope: testDir,
-        decision: 'allow',
-        priority: 50,
-        createdAt: Date.now(),
-      }];
-
-      const ctx: PolicyContext = {
-        ephemeralRules,
-      };
-
-      const filePath = join(testDir, 'task-file.txt');
-      const result = await check('file_write', { path: filePath }, testDir, ctx);
-      // The persistent deny rule (priority 100) should override
-      // both the ephemeral allow (priority 50) and workspace mode auto-allow
-      expect(result.decision).toBe('deny');
-    });
-  });
-});

package/src/__tests__/evaluate-typescript-tool.test.ts

@@ -1,286 +0,0 @@
-import { afterEach, describe, expect, mock, test } from 'bun:test';
-import { existsSync } from 'node:fs';
-import { join } from 'node:path';
-import { mkdtempSync, rmSync, readdirSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-
-const mockConfig = {
-  provider: 'anthropic',
-  model: 'test',
-  apiKeys: {},
-  maxTokens: 4096,
-  dataDir: '/tmp',
-  timeouts: {
-    shellDefaultTimeoutSec: 120,
-    shellMaxTimeoutSec: 600,
-    permissionTimeoutSec: 300,
-  },
-  sandbox: { enabled: false },
-  rateLimit: { maxRequestsPerMinute: 0, maxTokensPerSession: 0 },
-  secretDetection: { enabled: true, action: 'warn' as const, entropyThreshold: 4.0 },
-  auditLog: { retentionDays: 0 },
-};
-
-mock.module('../config/loader.js', () => ({
-  getConfig: () => mockConfig,
-  loadConfig: () => mockConfig,
-  invalidateConfigCache: () => {},
-  saveConfig: () => {},
-  loadRawConfig: () => ({}),
-  saveRawConfig: () => {},
-  getNestedValue: () => undefined,
-  setNestedValue: () => {},
-}));
-
-mock.module('../util/logger.js', () => ({
-  getLogger: () => new Proxy({} as Record<string, unknown>, {
-    get: () => () => {},
-  }),
-}));
-
-// Mock wrapCommand to pass through without actual sandboxing (tests run bun directly)
-mock.module('../tools/terminal/sandbox.js', () => ({
-  wrapCommand: (command: string, _workingDir: string, _config: unknown) => ({
-    command: 'bash',
-    args: ['-c', '--', command],
-    sandboxed: false,
-  }),
-}));
-
-import { EvaluateTypescriptTool } from '../tools/terminal/evaluate-typescript.js';
-import type { ToolContext } from '../tools/types.js';
-
-const tool = new EvaluateTypescriptTool();
-const testDirs: string[] = [];
-
-function makeContext(workingDir?: string): ToolContext {
-  const dir = workingDir ?? mkdtempSync(join(tmpdir(), 'eval-ts-test-'));
-  if (!workingDir) testDirs.push(dir);
-  return {
-    workingDir: dir,
-    sessionId: 'test-session',
-    conversationId: 'test-conversation',
-  };
-}
-
-afterEach(() => {
-  for (const dir of testDirs.splice(0)) {
-    rmSync(dir, { recursive: true, force: true });
-  }
-});
-
-describe('evaluate_typescript_code tool', () => {
-  test('successful execution returns exitCode=0 and expected output', async () => {
-    const result = await tool.execute({
-      code: 'export default function(input: unknown) { return { greeting: "hello" }; }',
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(true);
-    expect(parsed.exitCode).toBe(0);
-    expect(parsed.timeout).toBe(false);
-    expect(parsed.result).toEqual({ greeting: 'hello' });
-    expect(typeof parsed.durationMs).toBe('number');
-  });
-
-  test('passes mock_input_json to snippet', async () => {
-    const result = await tool.execute({
-      code: 'export default function(input: any) { return { received: input }; }',
-      mock_input_json: '{"name":"test"}',
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(true);
-    expect(parsed.result).toEqual({ received: { name: 'test' } });
-  });
-
-  test('syntax error returns non-zero and stderr', async () => {
-    const result = await tool.execute({
-      code: 'export default function( { return }',
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(false);
-    expect(parsed.exitCode).not.toBe(0);
-    expect(parsed.stderr.length).toBeGreaterThan(0);
-  });
-
-  test('runtime throw returns non-zero and stack fragment in stderr', async () => {
-    const result = await tool.execute({
-      code: 'export default function() { throw new Error("boom"); }',
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(false);
-    expect(parsed.exitCode).not.toBe(0);
-    expect(parsed.stderr).toContain('boom');
-  });
-
-  test('timeout kills process and returns timeout marker', async () => {
-    const result = await tool.execute({
-      code: 'export default async function() { await new Promise(r => setTimeout(r, 60000)); }',
-      timeout_seconds: 1,
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(false);
-    expect(parsed.timeout).toBe(true);
-  }, 10_000);
-
-  test('invalid JSON in mock_input_json returns deterministic validation error', async () => {
-    const result = await tool.execute({
-      code: 'export default function() { return 1; }',
-      mock_input_json: 'not-json{',
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    expect(result.content).toBe('Error: mock_input_json must be valid JSON');
-  });
-
-  test('missing export returns deterministic contract error', async () => {
-    const result = await tool.execute({
-      code: 'const x = 42;',
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(false);
-    expect(parsed.stderr).toContain('must export a function');
-  });
-
-  test('entrypoint=run calls the run export', async () => {
-    const result = await tool.execute({
-      code: 'export function run(input: any) { return "ran"; }',
-      entrypoint: 'run',
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(true);
-    expect(parsed.result).toBe('ran');
-  });
-
-  test('oversized code is rejected before execution', async () => {
-    const result = await tool.execute({
-      code: 'x'.repeat(200_000),
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    expect(result.content).toContain('exceeds maximum size');
-  });
-
-  test('oversized mock_input_json is rejected before execution', async () => {
-    const result = await tool.execute({
-      code: 'export default function() { return 1; }',
-      mock_input_json: JSON.stringify({ data: 'x'.repeat(200_000) }),
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    expect(result.content).toContain('exceeds maximum size');
-  });
-
-  test('temp directory is removed after completion', async () => {
-    const ctx = makeContext();
-
-    await tool.execute({
-      code: 'export default function() { return 1; }',
-    }, ctx);
-
-    const evalDir = join(ctx.workingDir, '.vellum-eval');
-    // The parent .vellum-eval dir may exist but should have no children
-    if (existsSync(evalDir)) {
-      expect(readdirSync(evalDir).length).toBe(0);
-    }
-  });
-
-  test('empty code is rejected', async () => {
-    const result = await tool.execute({
-      code: '',
-    }, makeContext());
-
-    expect(result.isError).toBe(true);
-    expect(result.content).toContain('code is required');
-  });
-
-  test('missing code is rejected', async () => {
-    const result = await tool.execute({}, makeContext());
-
-    expect(result.isError).toBe(true);
-    expect(result.content).toContain('code is required');
-  });
-
-  test('oversized stdout is truncated', async () => {
-    const result = await tool.execute({
-      code: `export default function() {
-        // Generate output larger than max_output_chars
-        const big = 'x'.repeat(30000);
-        console.log(big);
-        return 'done';
-      }`,
-      max_output_chars: 1000,
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.truncated).toBe(true);
-    expect(parsed.stdout).toContain('[stdout truncated]');
-    expect(parsed.stdout.length).toBeLessThan(2000);
-  }, 10_000);
-
-  test('filename "__runner.ts" is rejected to avoid collision with harness', async () => {
-    const result = await tool.execute({
-      code: 'export default function() { return "safe"; }',
-      filename: '__runner.ts',
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(true);
-    expect(parsed.result).toBe('safe');
-  });
-
-  test('filename "__runner" (no extension) is rejected to avoid collision', async () => {
-    const result = await tool.execute({
-      code: 'export default function() { return "safe"; }',
-      filename: '__runner',
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(true);
-    expect(parsed.result).toBe('safe');
-  });
-
-  test('result is preserved even when stdout is truncated', async () => {
-    const result = await tool.execute({
-      code: `export default function() {
-        // Fill stdout so it exceeds max_output_chars
-        for (let i = 0; i < 200; i++) console.log('x'.repeat(100));
-        return { preserved: true };
-      }`,
-      max_output_chars: 500,
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.truncated).toBe(true);
-    expect(parsed.result).toEqual({ preserved: true });
-  }, 10_000);
-
-  test('async snippet works', async () => {
-    const result = await tool.execute({
-      code: 'export default async function(input: any) { return { async: true, got: input }; }',
-      mock_input_json: '42',
-    }, makeContext());
-
-    expect(result.isError).toBe(false);
-    const parsed = JSON.parse(result.content);
-    expect(parsed.ok).toBe(true);
-    expect(parsed.result).toEqual({ async: true, got: 42 });
-  });
-});