self-bench 0.3.5 → 0.3.8

package/src/modal-executor.ts

@@ -1,176 +0,0 @@
-import { type Image, ModalClient, type Secret } from "modal";
-import type { SelfBenchConfig } from "./config.js";
-import { InactivityTimeoutError, RollingOutput } from "./process.js";
-import type {
-  SandboxExecutor,
-  SandboxRequest,
-  SandboxResult,
-  SandboxRunOptions,
-} from "./sandbox.js";
-
-export class ModalSandboxExecutor implements SandboxExecutor {
-  readonly #client: ModalClient;
-  readonly #config: Extract<SelfBenchConfig["execution"], { kind: "modal" }>;
-  #image: Image | undefined;
-
-  constructor(
-    config: Extract<SelfBenchConfig["execution"], { kind: "modal" }>,
-    client = new ModalClient(),
-  ) {
-    this.#config = config;
-    this.#client = client;
-  }
-
-  async run(request: SandboxRequest, options: SandboxRunOptions = {}): Promise<SandboxResult> {
-    const app = await this.#client.apps.fromName(this.#config.app, {
-      createIfMissing: true,
-      ...(this.#config.environment ? { environment: this.#config.environment } : {}),
-    });
-    if (!this.#image) {
-      this.#image = this.#buildImage();
-    }
-    const sandbox = await this.#client.sandboxes.create(app, this.#image, {
-      timeoutMs: request.timeoutMs,
-      idleTimeoutMs: Math.min(request.timeoutMs, 10 * 60 * 1000),
-      cpu: request.cpu ?? 4,
-      memoryMiB: request.memoryMiB ?? 8192,
-      workdir: "/work",
-      name: sandboxName(request.runId, request.stage),
-      tags: { run_id: request.runId, stage: request.stage },
-    });
-
-    const abort = () => {
-      void sandbox.terminate();
-    };
-    options.signal?.addEventListener("abort", abort, { once: true });
-    try {
-      for (const file of request.files ?? []) {
-        if (typeof file.contents === "string") {
-          await sandbox.filesystem.writeText(file.contents, file.path);
-        } else {
-          await sandbox.filesystem.writeBytes(file.contents, file.path);
-        }
-      }
-
-      const secrets = await this.#secrets(request.secrets);
-      const process = await sandbox.exec([...request.command], {
-        env: { ...request.environment },
-        ...(secrets.length > 0 ? { secrets } : {}),
-      });
-      await process.closeStdin();
-      const stdoutOutput = new RollingOutput();
-      const stderrOutput = new RollingOutput();
-      let inactivityTimer: ReturnType<typeof setTimeout> | undefined;
-      let inactivityError: InactivityTimeoutError | undefined;
-      const clearInactivityTimer = (): void => {
-        if (inactivityTimer) {
-          clearTimeout(inactivityTimer);
-          inactivityTimer = undefined;
-        }
-      };
-      const armInactivityTimer = (): void => {
-        clearInactivityTimer();
-        if (!request.inactivityTimeoutMs) {
-          return;
-        }
-        inactivityTimer = setTimeout(() => {
-          inactivityError = new InactivityTimeoutError(
-            `Modal sandbox ${sandbox.sandboxId} stage ${request.stage}`,
-            request.inactivityTimeoutMs ?? 0,
-          );
-          void sandbox.terminate();
-        }, request.inactivityTimeoutMs);
-        inactivityTimer.unref();
-      };
-      const consume = async (
-        streamName: "stdout" | "stderr",
-        stream: ReadableStream<string | Uint8Array>,
-        output: RollingOutput,
-      ): Promise<void> => {
-        const reader = stream.getReader();
-        try {
-          while (true) {
-            const { value, done } = await reader.read();
-            if (done) {
-              return;
-            }
-            if (value !== undefined) {
-              const chunk = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
-              output.push(chunk);
-              options.onProgress?.({ stream: streamName, bytes: chunk.byteLength });
-              armInactivityTimer();
-            }
-          }
-        } finally {
-          reader.releaseLock();
-        }
-      };
-      armInactivityTimer();
-      const [exitCode, stdout, stderr] = await Promise.all([
-        process.wait(),
-        consume("stdout", process.stdout, stdoutOutput),
-        consume("stderr", process.stderr, stderrOutput),
-      ])
-        .then(([exitCode]) => {
-          if (inactivityError) {
-            throw inactivityError;
-          }
-          return [exitCode, stdoutOutput.text(), stderrOutput.text()] as const;
-        })
-        .catch((error: unknown) => {
-          throw inactivityError ?? error;
-        })
-        .finally(clearInactivityTimer);
-      const outputs: Record<string, Uint8Array> = {};
-      for (const path of request.outputPaths ?? []) {
-        try {
-          outputs[path] = await sandbox.filesystem.readBytes(path);
-        } catch {
-          // Callers validate required outputs after persisting stdout/stderr. Returning an
-          // absent output keeps the model or command failure diagnosable.
-        }
-      }
-      return { sandboxId: sandbox.sandboxId, exitCode, stdout, stderr, outputs };
-    } finally {
-      options.signal?.removeEventListener("abort", abort);
-      await sandbox.terminate().catch(() => undefined);
-    }
-  }
-
-  close(): void {
-    this.#client.close();
-  }
-
-  #buildImage(): Image {
-    return this.#client.images
-      .fromRegistry(this.#config.image)
-      .dockerfileCommands([
-        "USER root",
-        "RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl git jq ripgrep unzip xz-utils && rm -rf /var/lib/apt/lists/*",
-        "RUN curl -fsSL https://github.com/cli/cli/releases/download/v2.89.0/gh_2.89.0_linux_amd64.tar.gz -o /tmp/gh.tar.gz && echo 'd0422caade520530e76c1c558da47daebaa8e1203d6b7ff10ad7d6faba3490d8  /tmp/gh.tar.gz' | sha256sum -c - && tar -xzf /tmp/gh.tar.gz -C /tmp && mv /tmp/gh_2.89.0_linux_amd64/bin/gh /usr/local/bin/gh && rm -rf /tmp/gh.tar.gz /tmp/gh_2.89.0_linux_amd64",
-        "RUN npm install --global --ignore-scripts @earendil-works/pi-coding-agent@0.84.0",
-        "RUN npm install --global @openai/codex@0.146.1",
-        "ENV PATH=/root/.local/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin",
-        "WORKDIR /work",
-      ]);
-  }
-
-  async #secrets(entries: Readonly<Record<string, string>> | undefined): Promise<Secret[]> {
-    if (!entries || Object.keys(entries).length === 0) {
-      return [];
-    }
-    return [
-      await this.#client.secrets.fromObject(
-        { ...entries },
-        {
-          ...(this.#config.environment ? { environment: this.#config.environment } : {}),
-        },
-      ),
-    ];
-  }
-}
-
-function sandboxName(runId: string, stage: string): string {
-  const suffix = crypto.randomUUID().slice(0, 8);
-  return `${runId.slice(0, 24)}-${stage.slice(0, 16)}-${suffix}`.replace(/[^a-zA-Z0-9._-]/g, "-");
-}

package/src/parallel.ts

@@ -1,24 +0,0 @@
-export async function parallelMap<T, R>(
-  values: readonly T[],
-  concurrency: number,
-  action: (value: T) => Promise<R>,
-): Promise<R[]> {
-  if (!Number.isInteger(concurrency) || concurrency < 1) {
-    throw new Error("concurrency must be a positive integer");
-  }
-  const output = new Array<R>(values.length);
-  let cursor = 0;
-  await Promise.all(
-    Array.from({ length: Math.min(concurrency, values.length) }, async () => {
-      while (cursor < values.length) {
-        const index = cursor;
-        cursor += 1;
-        const value = values[index];
-        if (value !== undefined) {
-          output[index] = await action(value);
-        }
-      }
-    }),
-  );
-  return output;
-}

package/src/process.ts

@@ -1,167 +0,0 @@
-import { spawn } from "node:child_process";
-
-export interface CommandResult {
-  readonly stdout: string;
-  readonly stderr: string;
-  readonly exitCode: number;
-}
-
-export type CommandOutputHandler = (stream: "stdout" | "stderr", chunk: Uint8Array) => void;
-
-export interface CommandOptions {
-  readonly cwd?: string;
-  readonly env?: NodeJS.ProcessEnv;
-  readonly input?: Uint8Array | string;
-  readonly timeoutMs?: number;
-  readonly inactivityTimeoutMs?: number;
-  readonly allowFailure?: boolean;
-  readonly signal?: AbortSignal;
-  readonly onOutput?: CommandOutputHandler;
-}
-
-export class InactivityTimeoutError extends Error {
-  constructor(
-    readonly scope: string,
-    readonly timeoutMs: number,
-  ) {
-    super(`${scope} produced no output for ${timeoutMs}ms`);
-    this.name = "InactivityTimeoutError";
-  }
-}
-
-export async function runCommand(
-  command: string,
-  args: readonly string[],
-  options: CommandOptions = {},
-): Promise<CommandResult> {
-  return await new Promise((resolve, reject) => {
-    const child = spawn(command, args, {
-      cwd: options.cwd,
-      env: options.env ?? process.env,
-      stdio: [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"],
-    });
-    const stdout = new RollingOutput();
-    const stderr = new RollingOutput();
-    if (!child.stdout || !child.stderr) {
-      reject(new Error(`failed to capture ${command} output`));
-      return;
-    }
-    let inactivityTimer: NodeJS.Timeout | undefined;
-    let forceKillTimer: NodeJS.Timeout | undefined;
-    let inactivityError: InactivityTimeoutError | undefined;
-    const clearInactivityTimer = (): void => {
-      if (inactivityTimer) {
-        clearTimeout(inactivityTimer);
-        inactivityTimer = undefined;
-      }
-    };
-    const terminate = (): void => {
-      child.kill("SIGTERM");
-      forceKillTimer ??= setTimeout(() => child.kill("SIGKILL"), 5_000);
-      forceKillTimer.unref();
-    };
-    const armInactivityTimer = (): void => {
-      clearInactivityTimer();
-      if (!options.inactivityTimeoutMs) {
-        return;
-      }
-      inactivityTimer = setTimeout(() => {
-        inactivityError = new InactivityTimeoutError(
-          `${command} ${args.join(" ")}`,
-          options.inactivityTimeoutMs ?? 0,
-        );
-        terminate();
-      }, options.inactivityTimeoutMs);
-      inactivityTimer.unref();
-    };
-    const capture = (stream: "stdout" | "stderr", output: RollingOutput, chunk: Buffer): void => {
-      output.push(chunk);
-      options.onOutput?.(stream, chunk);
-      armInactivityTimer();
-    };
-    child.stdout.on("data", (chunk: Buffer) => capture("stdout", stdout, chunk));
-    child.stderr.on("data", (chunk: Buffer) => capture("stderr", stderr, chunk));
-    child.on("error", reject);
-
-    const abort = () => terminate();
-    if (options.signal?.aborted) {
-      abort();
-    } else {
-      options.signal?.addEventListener("abort", abort, { once: true });
-    }
-
-    const timeout = options.timeoutMs
-      ? setTimeout(() => terminate(), options.timeoutMs)
-      : undefined;
-    timeout?.unref();
-    armInactivityTimer();
-
-    child.on("close", (exitCode, signal) => {
-      options.signal?.removeEventListener("abort", abort);
-      clearInactivityTimer();
-      if (forceKillTimer) {
-        clearTimeout(forceKillTimer);
-      }
-      if (timeout) {
-        clearTimeout(timeout);
-      }
-      const result = {
-        stdout: stdout.text(),
-        stderr: stderr.text(),
-        exitCode: exitCode ?? (signal ? 128 : 1),
-      };
-      if (inactivityError) {
-        reject(inactivityError);
-        return;
-      }
-      if (result.exitCode !== 0 && !options.allowFailure) {
-        const detail = result.stderr.trim() || result.stdout.trim();
-        reject(new Error(`${command} exited ${result.exitCode}${detail ? `: ${detail}` : ""}`));
-        return;
-      }
-      resolve(result);
-    });
-
-    if (child.stdin && options.input !== undefined) {
-      child.stdin.end(options.input);
-    }
-  });
-}
-
-export class RollingOutput {
-  readonly #limit = 8 * 1024 * 1024;
-  readonly #chunks: Buffer[] = [];
-  #size = 0;
-  #truncated = false;
-
-  push(chunk: Buffer): void {
-    if (chunk.byteLength >= this.#limit) {
-      this.#chunks.length = 0;
-      this.#chunks.push(chunk.subarray(chunk.byteLength - this.#limit));
-      this.#size = this.#limit;
-      this.#truncated = true;
-      return;
-    }
-    this.#chunks.push(chunk);
-    this.#size += chunk.byteLength;
-    while (this.#size > this.#limit) {
-      const first = this.#chunks.shift();
-      if (!first) {
-        break;
-      }
-      const overflow = this.#size - this.#limit;
-      if (first.byteLength > overflow) {
-        this.#chunks.unshift(first.subarray(overflow));
-        this.#size -= overflow;
-      } else {
-        this.#size -= first.byteLength;
-      }
-      this.#truncated = true;
-    }
-  }
-
-  text(): string {
-    const value = Buffer.concat(this.#chunks, this.#size).toString("utf8");
-    return this.#truncated ? `[selfbench: earlier output truncated]\n${value}` : value;
-  }
-}