vellum 0.0.16 → 0.2.0

package/src/tools/terminal/backends/native.ts

@@ -0,0 +1,188 @@
+import { writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { execSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { isMacOS, isLinux } from '../../../util/platform.js';
+import { ToolError } from '../../../util/errors.js';
+import { getLogger } from '../../../util/logger.js';
+import type { SandboxBackend, SandboxResult } from './types.js';
+
+const log = getLogger('sandbox');
+
+/**
+ * macOS sandbox-exec profile that restricts shell commands:
+ * - Denies all by default
+ * - Allows read access to most of the filesystem (needed for toolchains)
+ * - Allows write access only to the working directory and temp dirs
+ * - Blocks outbound network access
+ * - Blocks process debugging (ptrace)
+ *
+ * The WORKING_DIR placeholder is replaced at runtime with the actual
+ * working directory path.
+ */
+const SANDBOX_PROFILE = `
+(version 1)
+(deny default)
+
+;; Allow read access to the filesystem (tools, libraries, etc.)
+(allow file-read*)
+
+;; Allow write access to the working directory and its children
+(allow file-write*
+  (subpath "__WORKING_DIR__")
+  (subpath "/private/tmp")
+  (subpath "/tmp")
+  (subpath "/var/folders"))
+
+;; Allow process execution (needed to run commands)
+(allow process-exec*)
+(allow process-fork)
+
+;; Allow signal delivery between parent and child
+(allow signal (target others))
+
+;; Allow sysctl reads (needed by many tools)
+(allow sysctl-read)
+
+;; Allow mach lookups (IPC, needed for basic process operation)
+(allow mach-lookup)
+(allow mach-register)
+
+;; Allow IOKit (needed for some system calls)
+(allow iokit-open)
+
+;; Block network access
+(deny network*)
+
+;; Block process debugging
+(deny process-info-pidinfo (target others))
+`.trim();
+
+/** Characters that are meaningful in SBPL syntax and must not appear in paths. */
+const SBPL_UNSAFE = /["()\\;\n\r]/;
+
+/**
+ * Validate that a path is safe to embed in an SBPL profile string.
+ * Returns true if the path contains no SBPL metacharacters.
+ */
+function isSafeForSBPL(path: string): boolean {
+  return !SBPL_UNSAFE.test(path);
+}
+
+/**
+ * Get the path to the sandbox profile file, creating it if needed.
+ *
+ * Each distinct working directory gets its own profile file (keyed by
+ * a hash of the path) to avoid race conditions when concurrent commands
+ * use different working directories.
+ */
+function getProfilePath(workingDir: string): string {
+  const dir = join(process.env.HOME ?? '/tmp', '.vellum');
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  const hash = createHash('sha256').update(workingDir).digest('hex').slice(0, 12);
+  const path = join(dir, `sandbox-profile-${hash}.sb`);
+
+  const profile = SANDBOX_PROFILE.replace(/__WORKING_DIR__/g, workingDir);
+  writeFileSync(path, profile + '\n');
+  return path;
+}
+
+/**
+ * Cache positive bwrap results only. Negative results are not cached so that
+ * installing bwrap after the daemon starts takes effect without a restart.
+ */
+let bwrapAvailable = false;
+
+/**
+ * Check whether bwrap is installed AND functional (can create namespaces).
+ *
+ * Just testing `bwrap --version` is not enough — the binary may exist but
+ * namespace creation can be blocked by the kernel (e.g. inside containers
+ * or when user namespaces are disabled). We run a minimal sandbox that
+ * exercises all namespace types used by buildBwrapArgs() (mount, network,
+ * PID) to verify end-to-end functionality.
+ *
+ * Only positive results are cached — if bwrap is unavailable, we re-check
+ * on every call so that a mid-session install is picked up immediately.
+ */
+function isBwrapAvailable(): boolean {
+  if (bwrapAvailable) return true;
+  try {
+    execSync('bwrap --ro-bind / / --unshare-net --unshare-pid true', { stdio: 'ignore', timeout: 5000 });
+    bwrapAvailable = true;
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Build bwrap arguments for Linux sandboxing.
+ *
+ * Strategy mirrors the macOS sandbox-exec profile:
+ * - Read-only bind-mount of the root filesystem (toolchains, libs, etc.)
+ * - Read-write bind-mount of the working directory
+ * - Read-write tmpfs for /tmp
+ * - /proc mounted for basic process operation
+ * - /dev bind-mounted for device access (needed by many tools)
+ * - Network access blocked (--unshare-net)
+ * - PID namespace isolated (--unshare-pid)
+ */
+function buildBwrapArgs(workingDir: string, command: string): string[] {
+  return [
+    // Filesystem: read-only root, writable working dir and temp
+    '--ro-bind', '/', '/',
+    '--bind', workingDir, workingDir,
+    '--bind', '/tmp', '/tmp',
+    '--dev', '/dev',
+    '--proc', '/proc',
+    // Isolation
+    '--unshare-net',
+    '--unshare-pid',
+    // Run bash inside the sandbox
+    'bash', '-c', '--', command,
+  ];
+}
+
+/**
+ * Native sandbox backend using OS-level sandboxing:
+ * macOS sandbox-exec (SBPL profiles) and Linux bwrap (bubblewrap).
+ */
+export class NativeBackend implements SandboxBackend {
+  wrap(command: string, workingDir: string, _options?: import('./types.js').WrapOptions): SandboxResult {
+    if (isMacOS()) {
+      if (!isSafeForSBPL(workingDir)) {
+        throw new ToolError(
+          `Sandbox is enabled but the working directory contains characters unsafe for the sandbox profile (SBPL metacharacters). ` +
+          `Refusing to execute unsandboxed. Change to a directory without special characters in its path, or disable sandboxing.`,
+          'bash',
+        );
+      }
+      const profile = getProfilePath(workingDir);
+      return {
+        command: 'sandbox-exec',
+        args: ['-f', profile, 'bash', '-c', '--', command],
+        sandboxed: true,
+      };
+    }
+
+    if (isLinux()) {
+      if (!isBwrapAvailable()) {
+        const msg = 'Sandbox is enabled but bwrap is not available or cannot create namespaces. Refusing to execute unsandboxed. Install bubblewrap (for example: apt install bubblewrap), or disable sandboxing.';
+        log.error(msg);
+        throw new ToolError(msg, 'bash');
+      }
+      return {
+        command: 'bwrap',
+        args: buildBwrapArgs(workingDir, command),
+        sandboxed: true,
+      };
+    }
+
+    const msg = `Sandbox is enabled but not supported on this platform (${process.platform}). Refusing to execute unsandboxed. Disable sandboxing to run shell commands.`;
+    log.error(msg);
+    throw new ToolError(msg, 'bash');
+  }
+}

package/src/tools/terminal/backends/types.ts

@@ -0,0 +1,26 @@
+export interface SandboxResult {
+  /** The command/args to use for spawning. */
+  command: string;
+  args: string[];
+  /** Whether sandboxing was applied. */
+  sandboxed: boolean;
+}
+
+/** Per-invocation options that override backend defaults. */
+export interface WrapOptions {
+  /**
+   * Network mode for this invocation.
+   * - 'off': no container network (--network=none). This is the default.
+   * - 'proxied': bridge network so the container can reach a host proxy (--network=bridge).
+   */
+  networkMode?: 'off' | 'proxied';
+}
+
+/**
+ * A sandbox backend knows how to wrap a shell command so it runs
+ * inside an OS-level sandbox (macOS sandbox-exec, Linux bwrap, Docker, etc.).
+ */
+export interface SandboxBackend {
+  /** Wrap a command for sandboxed execution in the given working directory. */
+  wrap(command: string, workingDir: string, options?: WrapOptions): SandboxResult;
+}

package/src/tools/terminal/evaluate-typescript.ts

@@ -0,0 +1,275 @@
+import { spawn } from 'node:child_process';
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { join, basename } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { RiskLevel } from '../../permissions/types.js';
+import type { Tool, ToolContext, ToolExecutionResult } from '../types.js';
+import type { ToolDefinition } from '../../providers/types.js';
+import { registerTool } from '../registry.js';
+import { getConfig } from '../../config/loader.js';
+import { getLogger } from '../../util/logger.js';
+import { wrapCommand } from './sandbox.js';
+import { buildSanitizedEnv } from './safe-env.js';
+
+const log = getLogger('evaluate-typescript');
+
+const MAX_CODE_BYTES = 100_000;
+const MAX_MOCK_INPUT_BYTES = 100_000;
+const DEFAULT_MAX_OUTPUT_CHARS = 25_000;
+const DEFAULT_TIMEOUT_SEC = 10;
+const MIN_TIMEOUT_SEC = 1;
+const MAX_TIMEOUT_SEC = 20;
+
+interface EvalResult {
+  ok: boolean;
+  exitCode: number | null;
+  durationMs: number;
+  result?: unknown;
+  stdout: string;
+  stderr: string;
+  truncated: boolean;
+  timeout: boolean;
+}
+
+function sanitizeFilename(raw: string): string {
+  const base = basename(raw).replace(/[^a-zA-Z0-9._-]/g, '_');
+  if (!base || base.startsWith('.')) return 'snippet.ts';
+  if (base === '__runner.ts' || base === '__runner') return 'snippet.ts';
+  if (!base.endsWith('.ts')) return base + '.ts';
+  return base;
+}
+
+function buildWrapperSource(snippetFilename: string, entrypoint: string): string {
+  const exportName = entrypoint === 'run' ? 'run' : 'default';
+  return `
+import * as mod from './${snippetFilename}';
+
+const fn = (mod as any)['${exportName}'];
+if (typeof fn !== 'function') {
+  console.error(JSON.stringify({ __eval_error: 'Snippet must export a function named "${exportName}"' }));
+  process.exit(1);
+}
+
+const inputJson = process.env.__EVAL_INPUT_JSON ?? '{}';
+let input: unknown;
+try {
+  input = JSON.parse(inputJson);
+} catch {
+  console.error(JSON.stringify({ __eval_error: 'Invalid JSON in __EVAL_INPUT_JSON' }));
+  process.exit(1);
+}
+
+try {
+  const result = await fn(input);
+  console.log(JSON.stringify({ __eval_result: result }));
+} catch (err: unknown) {
+  console.error(err instanceof Error ? err.stack ?? String(err) : String(err));
+  process.exit(1);
+}
+`;
+}
+
+export class EvaluateTypescriptTool implements Tool {
+  name = 'evaluate_typescript_code';
+  description = 'Evaluate a TypeScript snippet in an isolated sandbox. Use this to test code before persisting it as a managed skill.';
+  category = 'terminal';
+  defaultRiskLevel = RiskLevel.High;
+
+  getDefinition(): ToolDefinition {
+    return {
+      name: this.name,
+      description: this.description,
+      input_schema: {
+        type: 'object',
+        properties: {
+          code: {
+            type: 'string',
+            description: 'The TypeScript source code to evaluate. Must export a `default` or `run` function with signature `(input: unknown) => unknown | Promise<unknown>`.',
+          },
+          mock_input_json: {
+            type: 'string',
+            description: 'Optional JSON string to pass as the input argument to the exported function. Defaults to "{}".',
+          },
+          timeout_seconds: {
+            type: 'number',
+            description: `Optional timeout in seconds (${MIN_TIMEOUT_SEC}-${MAX_TIMEOUT_SEC}). Defaults to ${DEFAULT_TIMEOUT_SEC}.`,
+          },
+          filename: {
+            type: 'string',
+            description: 'Optional filename for the snippet (default: "snippet.ts"). Sanitized to basename only.',
+          },
+          entrypoint: {
+            type: 'string',
+            enum: ['default', 'run'],
+            description: 'Which export to call: "default" or "run". Defaults to "default".',
+          },
+          max_output_chars: {
+            type: 'number',
+            description: `Optional max output characters (1-${DEFAULT_MAX_OUTPUT_CHARS}). Defaults to ${DEFAULT_MAX_OUTPUT_CHARS}.`,
+          },
+        },
+        required: ['code'],
+      },
+    };
+  }
+
+  async execute(input: Record<string, unknown>, context: ToolContext): Promise<ToolExecutionResult> {
+    const code = input.code;
+    if (typeof code !== 'string' || !code.trim()) {
+      return { content: 'Error: code is required and must be a non-empty string', isError: true };
+    }
+    if (Buffer.byteLength(code) > MAX_CODE_BYTES) {
+      return { content: `Error: code exceeds maximum size of ${MAX_CODE_BYTES} bytes`, isError: true };
+    }
+
+    const mockInputJson = typeof input.mock_input_json === 'string' ? input.mock_input_json : '{}';
+    if (Buffer.byteLength(mockInputJson) > MAX_MOCK_INPUT_BYTES) {
+      return { content: `Error: mock_input_json exceeds maximum size of ${MAX_MOCK_INPUT_BYTES} bytes`, isError: true };
+    }
+    try {
+      JSON.parse(mockInputJson);
+    } catch {
+      return { content: 'Error: mock_input_json must be valid JSON', isError: true };
+    }
+
+    const rawTimeout = typeof input.timeout_seconds === 'number' ? input.timeout_seconds : DEFAULT_TIMEOUT_SEC;
+    const timeoutSec = Math.max(MIN_TIMEOUT_SEC, Math.min(MAX_TIMEOUT_SEC, Math.round(rawTimeout)));
+    const timeoutMs = timeoutSec * 1000;
+
+    const filename = sanitizeFilename(typeof input.filename === 'string' ? input.filename : 'snippet.ts');
+    const entrypoint = input.entrypoint === 'run' ? 'run' : 'default';
+    const maxOutputChars = typeof input.max_output_chars === 'number'
+      ? Math.max(1, Math.min(DEFAULT_MAX_OUTPUT_CHARS, Math.round(input.max_output_chars)))
+      : DEFAULT_MAX_OUTPUT_CHARS;
+
+    const evalDir = join(context.workingDir, '.vellum-eval', randomUUID());
+
+    try {
+      mkdirSync(evalDir, { recursive: true });
+      writeFileSync(join(evalDir, filename), code, 'utf-8');
+      writeFileSync(join(evalDir, '__runner.ts'), buildWrapperSource(filename, entrypoint), 'utf-8');
+
+      const result = await this.runSnippet(evalDir, mockInputJson, timeoutSec, timeoutMs, maxOutputChars, context);
+      return { content: JSON.stringify(result), isError: !result.ok };
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      log.error({ err }, 'evaluate_typescript_code failed');
+      return { content: `Error: ${msg}`, isError: true };
+    } finally {
+      try {
+        rmSync(evalDir, { recursive: true, force: true });
+      } catch {
+        // best effort
+      }
+    }
+  }
+
+  private runSnippet(
+    evalDir: string,
+    mockInputJson: string,
+    timeoutSec: number,
+    timeoutMs: number,
+    maxOutputChars: number,
+    _context: ToolContext,
+  ): Promise<EvalResult> {
+    return new Promise<EvalResult>((resolve) => {
+      const startTime = Date.now();
+      const stdoutChunks: Buffer[] = [];
+      const stderrChunks: Buffer[] = [];
+      let timedOut = false;
+
+      const config = getConfig();
+      // Force sandbox on regardless of global config
+      const sandboxConfig = { ...config.sandbox, enabled: true };
+      const bunRunCmd = `bun run __runner.ts`;
+      const wrapped = wrapCommand(bunRunCmd, evalDir, sandboxConfig);
+
+      const env = buildSanitizedEnv();
+      env.__EVAL_INPUT_JSON = mockInputJson;
+
+      const child = spawn(wrapped.command, wrapped.args, {
+        cwd: evalDir,
+        env,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+
+      const timer = setTimeout(() => {
+        timedOut = true;
+        child.kill('SIGKILL');
+      }, timeoutMs);
+
+      child.stdout.on('data', (data: Buffer) => stdoutChunks.push(data));
+      child.stderr.on('data', (data: Buffer) => stderrChunks.push(data));
+
+      child.on('close', (code) => {
+        clearTimeout(timer);
+        const durationMs = Date.now() - startTime;
+
+        let stdout = Buffer.concat(stdoutChunks).toString();
+        let stderr = Buffer.concat(stderrChunks).toString();
+        let truncated = false;
+
+        // Extract structured result from raw stdout before truncation.
+        // Scan backward from the end for the marker; if the last occurrence
+        // is on a malformed line, keep searching earlier occurrences so a
+        // valid result is not missed due to trailing marker-like log output.
+        let result: unknown = undefined;
+        if (code === 0) {
+          let searchFrom = stdout.length;
+          while (searchFrom > 0) {
+            const markerIdx = stdout.lastIndexOf('__eval_result', searchFrom - 1);
+            if (markerIdx === -1) break;
+            const lineStart = stdout.lastIndexOf('\n', markerIdx) + 1;
+            const lineEnd = stdout.indexOf('\n', markerIdx);
+            const line = stdout.slice(lineStart, lineEnd === -1 ? undefined : lineEnd);
+            try {
+              const parsed = JSON.parse(line);
+              if (parsed && typeof parsed === 'object' && '__eval_result' in parsed) {
+                result = parsed.__eval_result;
+                break;
+              }
+            } catch {
+              // malformed line — continue scanning earlier occurrences
+            }
+            searchFrom = lineStart;
+          }
+        }
+
+        if (stdout.length + stderr.length > maxOutputChars) {
+          truncated = true;
+          const halfMax = Math.floor(maxOutputChars / 2);
+          if (stdout.length > halfMax) stdout = stdout.slice(0, halfMax) + '\n[stdout truncated]';
+          if (stderr.length > halfMax) stderr = stderr.slice(0, halfMax) + '\n[stderr truncated]';
+        }
+
+        resolve({
+          ok: code === 0 && !timedOut,
+          exitCode: code,
+          durationMs,
+          result,
+          stdout,
+          stderr,
+          truncated,
+          timeout: timedOut,
+        });
+      });
+
+      child.on('error', (err) => {
+        clearTimeout(timer);
+        const durationMs = Date.now() - startTime;
+        resolve({
+          ok: false,
+          exitCode: null,
+          durationMs,
+          stdout: '',
+          stderr: `Error spawning process: ${err.message}`,
+          truncated: false,
+          timeout: false,
+        });
+      });
+    });
+  }
+}
+
+export const evaluateTypescriptTool: Tool = new EvaluateTypescriptTool();
+registerTool(evaluateTypescriptTool);