self-bench 0.3.0 → 0.3.3

package/src/modal-executor.ts

@@ -0,0 +1,176 @@
+import { type Image, ModalClient, type Secret } from "modal";
+import type { SelfBenchConfig } from "./config.js";
+import { InactivityTimeoutError, RollingOutput } from "./process.js";
+import type {
+  SandboxExecutor,
+  SandboxRequest,
+  SandboxResult,
+  SandboxRunOptions,
+} from "./sandbox.js";
+
+export class ModalSandboxExecutor implements SandboxExecutor {
+  readonly #client: ModalClient;
+  readonly #config: Extract<SelfBenchConfig["execution"], { kind: "modal" }>;
+  #image: Image | undefined;
+
+  constructor(
+    config: Extract<SelfBenchConfig["execution"], { kind: "modal" }>,
+    client = new ModalClient(),
+  ) {
+    this.#config = config;
+    this.#client = client;
+  }
+
+  async run(request: SandboxRequest, options: SandboxRunOptions = {}): Promise<SandboxResult> {
+    const app = await this.#client.apps.fromName(this.#config.app, {
+      createIfMissing: true,
+      ...(this.#config.environment ? { environment: this.#config.environment } : {}),
+    });
+    if (!this.#image) {
+      this.#image = this.#buildImage();
+    }
+    const sandbox = await this.#client.sandboxes.create(app, this.#image, {
+      timeoutMs: request.timeoutMs,
+      idleTimeoutMs: Math.min(request.timeoutMs, 10 * 60 * 1000),
+      cpu: request.cpu ?? 4,
+      memoryMiB: request.memoryMiB ?? 8192,
+      workdir: "/work",
+      name: sandboxName(request.runId, request.stage),
+      tags: { run_id: request.runId, stage: request.stage },
+    });
+
+    const abort = () => {
+      void sandbox.terminate();
+    };
+    options.signal?.addEventListener("abort", abort, { once: true });
+    try {
+      for (const file of request.files ?? []) {
+        if (typeof file.contents === "string") {
+          await sandbox.filesystem.writeText(file.contents, file.path);
+        } else {
+          await sandbox.filesystem.writeBytes(file.contents, file.path);
+        }
+      }
+
+      const secrets = await this.#secrets(request.secrets);
+      const process = await sandbox.exec([...request.command], {
+        env: { ...request.environment },
+        ...(secrets.length > 0 ? { secrets } : {}),
+      });
+      await process.closeStdin();
+      const stdoutOutput = new RollingOutput();
+      const stderrOutput = new RollingOutput();
+      let inactivityTimer: ReturnType<typeof setTimeout> | undefined;
+      let inactivityError: InactivityTimeoutError | undefined;
+      const clearInactivityTimer = (): void => {
+        if (inactivityTimer) {
+          clearTimeout(inactivityTimer);
+          inactivityTimer = undefined;
+        }
+      };
+      const armInactivityTimer = (): void => {
+        clearInactivityTimer();
+        if (!request.inactivityTimeoutMs) {
+          return;
+        }
+        inactivityTimer = setTimeout(() => {
+          inactivityError = new InactivityTimeoutError(
+            `Modal sandbox ${sandbox.sandboxId} stage ${request.stage}`,
+            request.inactivityTimeoutMs ?? 0,
+          );
+          void sandbox.terminate();
+        }, request.inactivityTimeoutMs);
+        inactivityTimer.unref();
+      };
+      const consume = async (
+        streamName: "stdout" | "stderr",
+        stream: ReadableStream<string | Uint8Array>,
+        output: RollingOutput,
+      ): Promise<void> => {
+        const reader = stream.getReader();
+        try {
+          while (true) {
+            const { value, done } = await reader.read();
+            if (done) {
+              return;
+            }
+            if (value !== undefined) {
+              const chunk = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
+              output.push(chunk);
+              options.onProgress?.({ stream: streamName, bytes: chunk.byteLength });
+              armInactivityTimer();
+            }
+          }
+        } finally {
+          reader.releaseLock();
+        }
+      };
+      armInactivityTimer();
+      const [exitCode, stdout, stderr] = await Promise.all([
+        process.wait(),
+        consume("stdout", process.stdout, stdoutOutput),
+        consume("stderr", process.stderr, stderrOutput),
+      ])
+        .then(([exitCode]) => {
+          if (inactivityError) {
+            throw inactivityError;
+          }
+          return [exitCode, stdoutOutput.text(), stderrOutput.text()] as const;
+        })
+        .catch((error: unknown) => {
+          throw inactivityError ?? error;
+        })
+        .finally(clearInactivityTimer);
+      const outputs: Record<string, Uint8Array> = {};
+      for (const path of request.outputPaths ?? []) {
+        try {
+          outputs[path] = await sandbox.filesystem.readBytes(path);
+        } catch {
+          // Callers validate required outputs after persisting stdout/stderr. Returning an
+          // absent output keeps the model or command failure diagnosable.
+        }
+      }
+      return { sandboxId: sandbox.sandboxId, exitCode, stdout, stderr, outputs };
+    } finally {
+      options.signal?.removeEventListener("abort", abort);
+      await sandbox.terminate().catch(() => undefined);
+    }
+  }
+
+  close(): void {
+    this.#client.close();
+  }
+
+  #buildImage(): Image {
+    return this.#client.images
+      .fromRegistry(this.#config.image)
+      .dockerfileCommands([
+        "USER root",
+        "RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl git jq ripgrep unzip xz-utils && rm -rf /var/lib/apt/lists/*",
+        "RUN curl -fsSL https://github.com/cli/cli/releases/download/v2.89.0/gh_2.89.0_linux_amd64.tar.gz -o /tmp/gh.tar.gz && echo 'd0422caade520530e76c1c558da47daebaa8e1203d6b7ff10ad7d6faba3490d8  /tmp/gh.tar.gz' | sha256sum -c - && tar -xzf /tmp/gh.tar.gz -C /tmp && mv /tmp/gh_2.89.0_linux_amd64/bin/gh /usr/local/bin/gh && rm -rf /tmp/gh.tar.gz /tmp/gh_2.89.0_linux_amd64",
+        "RUN npm install --global --ignore-scripts @earendil-works/pi-coding-agent@0.84.0",
+        "RUN npm install --global @openai/codex@0.146.1",
+        "ENV PATH=/root/.local/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin",
+        "WORKDIR /work",
+      ]);
+  }
+
+  async #secrets(entries: Readonly<Record<string, string>> | undefined): Promise<Secret[]> {
+    if (!entries || Object.keys(entries).length === 0) {
+      return [];
+    }
+    return [
+      await this.#client.secrets.fromObject(
+        { ...entries },
+        {
+          ...(this.#config.environment ? { environment: this.#config.environment } : {}),
+        },
+      ),
+    ];
+  }
+}
+
+function sandboxName(runId: string, stage: string): string {
+  const suffix = crypto.randomUUID().slice(0, 8);
+  return `${runId.slice(0, 24)}-${stage.slice(0, 16)}-${suffix}`.replace(/[^a-zA-Z0-9._-]/g, "-");
+}

package/src/parallel.ts

@@ -0,0 +1,24 @@
+export async function parallelMap<T, R>(
+  values: readonly T[],
+  concurrency: number,
+  action: (value: T) => Promise<R>,
+): Promise<R[]> {
+  if (!Number.isInteger(concurrency) || concurrency < 1) {
+    throw new Error("concurrency must be a positive integer");
+  }
+  const output = new Array<R>(values.length);
+  let cursor = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, values.length) }, async () => {
+      while (cursor < values.length) {
+        const index = cursor;
+        cursor += 1;
+        const value = values[index];
+        if (value !== undefined) {
+          output[index] = await action(value);
+        }
+      }
+    }),
+  );
+  return output;
+}

package/src/process.ts

@@ -0,0 +1,165 @@
+import { spawn } from "node:child_process";
+
+export interface CommandResult {
+  readonly stdout: string;
+  readonly stderr: string;
+  readonly exitCode: number;
+}
+
+export interface CommandOptions {
+  readonly cwd?: string;
+  readonly env?: NodeJS.ProcessEnv;
+  readonly input?: Uint8Array | string;
+  readonly timeoutMs?: number;
+  readonly inactivityTimeoutMs?: number;
+  readonly allowFailure?: boolean;
+  readonly signal?: AbortSignal;
+  readonly onOutput?: (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
+}
+
+export class InactivityTimeoutError extends Error {
+  constructor(
+    readonly scope: string,
+    readonly timeoutMs: number,
+  ) {
+    super(`${scope} produced no output for ${timeoutMs}ms`);
+    this.name = "InactivityTimeoutError";
+  }
+}
+
+export async function runCommand(
+  command: string,
+  args: readonly string[],
+  options: CommandOptions = {},
+): Promise<CommandResult> {
+  return await new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd,
+      env: options.env ?? process.env,
+      stdio: [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"],
+    });
+    const stdout = new RollingOutput();
+    const stderr = new RollingOutput();
+    if (!child.stdout || !child.stderr) {
+      reject(new Error(`failed to capture ${command} output`));
+      return;
+    }
+    let inactivityTimer: NodeJS.Timeout | undefined;
+    let forceKillTimer: NodeJS.Timeout | undefined;
+    let inactivityError: InactivityTimeoutError | undefined;
+    const clearInactivityTimer = (): void => {
+      if (inactivityTimer) {
+        clearTimeout(inactivityTimer);
+        inactivityTimer = undefined;
+      }
+    };
+    const terminate = (): void => {
+      child.kill("SIGTERM");
+      forceKillTimer ??= setTimeout(() => child.kill("SIGKILL"), 5_000);
+      forceKillTimer.unref();
+    };
+    const armInactivityTimer = (): void => {
+      clearInactivityTimer();
+      if (!options.inactivityTimeoutMs) {
+        return;
+      }
+      inactivityTimer = setTimeout(() => {
+        inactivityError = new InactivityTimeoutError(
+          `${command} ${args.join(" ")}`,
+          options.inactivityTimeoutMs ?? 0,
+        );
+        terminate();
+      }, options.inactivityTimeoutMs);
+      inactivityTimer.unref();
+    };
+    const capture = (stream: "stdout" | "stderr", output: RollingOutput, chunk: Buffer): void => {
+      output.push(chunk);
+      options.onOutput?.(stream, chunk);
+      armInactivityTimer();
+    };
+    child.stdout.on("data", (chunk: Buffer) => capture("stdout", stdout, chunk));
+    child.stderr.on("data", (chunk: Buffer) => capture("stderr", stderr, chunk));
+    child.on("error", reject);
+
+    const abort = () => terminate();
+    if (options.signal?.aborted) {
+      abort();
+    } else {
+      options.signal?.addEventListener("abort", abort, { once: true });
+    }
+
+    const timeout = options.timeoutMs
+      ? setTimeout(() => terminate(), options.timeoutMs)
+      : undefined;
+    timeout?.unref();
+    armInactivityTimer();
+
+    child.on("close", (exitCode, signal) => {
+      options.signal?.removeEventListener("abort", abort);
+      clearInactivityTimer();
+      if (forceKillTimer) {
+        clearTimeout(forceKillTimer);
+      }
+      if (timeout) {
+        clearTimeout(timeout);
+      }
+      const result = {
+        stdout: stdout.text(),
+        stderr: stderr.text(),
+        exitCode: exitCode ?? (signal ? 128 : 1),
+      };
+      if (inactivityError) {
+        reject(inactivityError);
+        return;
+      }
+      if (result.exitCode !== 0 && !options.allowFailure) {
+        const detail = result.stderr.trim() || result.stdout.trim();
+        reject(new Error(`${command} exited ${result.exitCode}${detail ? `: ${detail}` : ""}`));
+        return;
+      }
+      resolve(result);
+    });
+
+    if (child.stdin && options.input !== undefined) {
+      child.stdin.end(options.input);
+    }
+  });
+}
+
+export class RollingOutput {
+  readonly #limit = 8 * 1024 * 1024;
+  readonly #chunks: Buffer[] = [];
+  #size = 0;
+  #truncated = false;
+
+  push(chunk: Buffer): void {
+    if (chunk.byteLength >= this.#limit) {
+      this.#chunks.length = 0;
+      this.#chunks.push(chunk.subarray(chunk.byteLength - this.#limit));
+      this.#size = this.#limit;
+      this.#truncated = true;
+      return;
+    }
+    this.#chunks.push(chunk);
+    this.#size += chunk.byteLength;
+    while (this.#size > this.#limit) {
+      const first = this.#chunks.shift();
+      if (!first) {
+        break;
+      }
+      const overflow = this.#size - this.#limit;
+      if (first.byteLength > overflow) {
+        this.#chunks.unshift(first.subarray(overflow));
+        this.#size -= overflow;
+      } else {
+        this.#size -= first.byteLength;
+      }
+      this.#truncated = true;
+    }
+  }
+
+  text(): string {
+    const value = Buffer.concat(this.#chunks, this.#size).toString("utf8");
+    return this.#truncated ? `[selfbench: earlier output truncated]\n${value}` : value;
+  }
+}