self-bench 0.3.6 → 0.3.8

package/src/sandbox/providers/vercel/command.ts

@@ -1,127 +0,0 @@
-import type { Command, Session } from "@vercel/sandbox";
-import { InactivityTimeoutError, type RollingOutput } from "../../../process.js";
-import type { SandboxRequest, SandboxResult, SandboxRunOptions } from "../../contracts.js";
-import { VercelCommandStartError } from "./fetch.js";
-
-export const VERCEL_WORK_DIRECTORY = "/work";
-
-const COMPLETION_REQUEST_TIMEOUT_MS = 15_000;
-
-export async function executeVercelCommand(input: {
-  readonly session: Session;
-  readonly request: SandboxRequest;
-  readonly options: SandboxRunOptions;
-  readonly signal: AbortSignal;
-  readonly terminate: (error: unknown) => void;
-  readonly stdout: RollingOutput;
-  readonly stderr: RollingOutput;
-}): Promise<Omit<SandboxResult, "sandboxId">> {
-  const { session, request, options, signal, terminate, stdout, stderr } = input;
-  let inactivityTimer: ReturnType<typeof setTimeout> | undefined;
-  const clearInactivityTimer = (): void => {
-    if (inactivityTimer) {
-      clearTimeout(inactivityTimer);
-      inactivityTimer = undefined;
-    }
-  };
-  const armInactivityTimer = (): void => {
-    clearInactivityTimer();
-    if (!request.inactivityTimeoutMs) {
-      return;
-    }
-    inactivityTimer = setTimeout(() => {
-      terminate(
-        new InactivityTimeoutError(
-          `Vercel sandbox ${session.sessionId} stage ${request.stage}`,
-          request.inactivityTimeoutMs ?? 0,
-        ),
-      );
-    }, request.inactivityTimeoutMs);
-    inactivityTimer.unref();
-  };
-
-  try {
-    const [commandName, ...commandArguments] = request.command;
-    if (!commandName) {
-      throw new Error("sandbox command must not be empty");
-    }
-    const command = await startCommand(session, commandName, commandArguments, request, signal);
-
-    armInactivityTimer();
-    await consumeLogs(command, stdout, stderr, options, armInactivityTimer, signal);
-    // The log stream closes when remote execution ends. From here on, the
-    // bounded completion request is provider settlement rather than command
-    // output inactivity.
-    clearInactivityTimer();
-    const completed = await command.wait({
-      signal: AbortSignal.any([signal, AbortSignal.timeout(COMPLETION_REQUEST_TIMEOUT_MS)]),
-    });
-
-    const outputs: Record<string, Uint8Array> = {};
-    for (const path of request.outputPaths ?? []) {
-      const output = await session.readFileToBuffer({ path }, { signal });
-      if (output === null) {
-        if (completed.exitCode === 0) {
-          throw new Error(
-            `sandbox ${session.sessionId} exited successfully without output ${path}`,
-          );
-        }
-      } else {
-        outputs[path] = output;
-      }
-    }
-    return {
-      exitCode: completed.exitCode,
-      stdout: stdout.text(),
-      stderr: stderr.text(),
-      outputs,
-    };
-  } finally {
-    clearInactivityTimer();
-  }
-}
-
-async function startCommand(
-  session: Session,
-  commandName: string,
-  commandArguments: readonly string[],
-  request: SandboxRequest,
-  signal: AbortSignal,
-): Promise<Command> {
-  try {
-    return await session.runCommand({
-      cmd: commandName,
-      args: [...commandArguments],
-      cwd: VERCEL_WORK_DIRECTORY,
-      detached: true,
-      env: { ...request.environment, ...request.secrets },
-      signal,
-      timeoutMs: request.timeoutMs,
-    });
-  } catch (error) {
-    if (error instanceof VercelCommandStartError) {
-      error.restorePublicName();
-    }
-    throw error;
-  }
-}
-
-async function consumeLogs(
-  command: Command,
-  stdout: RollingOutput,
-  stderr: RollingOutput,
-  options: SandboxRunOptions,
-  onOutput: () => void,
-  signal: AbortSignal,
-): Promise<void> {
-  for await (const event of command.logs({ signal })) {
-    const chunk = Buffer.from(event.data);
-    if (chunk.byteLength === 0) {
-      continue;
-    }
-    const output = event.stream === "stdout" ? stdout : stderr;
-    output.push(chunk);
-    options.onProgress?.({ stream: event.stream, bytes: chunk.byteLength });
-    onOutput();
-  }
-}

package/src/sandbox/providers/vercel/executor.ts

@@ -1,448 +0,0 @@
-import { posix } from "node:path";
-import { setTimeout as delay } from "node:timers/promises";
-import { APIError, Sandbox } from "@vercel/sandbox";
-import {
-  isDigestPinnedOciImage,
-  type SelfBenchWorkerConfig,
-  type VercelCredentials,
-} from "../../../config.js";
-import { RollingOutput } from "../../../process.js";
-import type {
-  SandboxExecutor,
-  SandboxRequest,
-  SandboxResult,
-  SandboxRunOptions,
-} from "../../contracts.js";
-import { executeVercelCommand, VERCEL_WORK_DIRECTORY } from "./command.js";
-import { preventAmbiguousVercelCommandStartRetries } from "./fetch.js";
-
-const CLEANUP_TIMEOUT_MS = 60_000;
-const CLEANUP_RETRY_DELAY_MS = 1_000;
-const CREATE_RATE_LIMIT_RETRIES = 2;
-const CREATE_RATE_LIMIT_FALLBACK_DELAY_MS = 30_000;
-const CREATE_RATE_LIMIT_MAX_DELAY_MS = 60_000;
-const CREATE_REQUEST_TIMEOUT_MS = 60_000;
-const HARD_TIMEOUT_EXIT_CODE = 124;
-const LATE_CREATE_RECOVERY_DELAYS_MS = [0, 250, 750, 1_500] as const;
-
-type VercelExecutionConfig = Extract<
-  SelfBenchWorkerConfig["execution"],
-  { readonly kind: "vercel" }
->;
-
-type RunOutcome =
-  | { readonly ok: true; readonly result: SandboxResult }
-  | { readonly ok: false; readonly error: unknown };
-
-type Sleep = (delayMs: number, signal: AbortSignal) => Promise<void>;
-
-class VercelHardTimeoutError extends Error {
-  constructor(name: string, stage: string, timeoutMs: number) {
-    super(`Vercel sandbox ${name} stage ${stage} exceeded ${timeoutMs}ms`);
-    // Passed as the abort reason to in-flight SDK calls, so it carries the
-    // standard DOMException timeout name. run() converts it into an exit-124
-    // result rather than throwing.
-    this.name = "TimeoutError";
-  }
-}
-
-export class VercelSandboxExecutor implements SandboxExecutor {
-  readonly #config: VercelExecutionConfig;
-  readonly #fetch: typeof globalThis.fetch;
-  readonly #sleep: Sleep;
-
-  constructor(
-    config: VercelExecutionConfig,
-    fetch = globalThis.fetch,
-    sleep: Sleep = abortableDelay,
-  ) {
-    this.#config = validateConfig(config);
-    this.#fetch = preventAmbiguousVercelCommandStartRetries(fetch);
-    this.#sleep = sleep;
-  }
-
-  async run(request: SandboxRequest, options: SandboxRunOptions = {}): Promise<SandboxResult> {
-    options.signal?.throwIfAborted();
-    const resources = validateRequest(request);
-
-    const name = sandboxName(request.runId, request.stage);
-    const tags = {
-      selfbench_run: request.runId.slice(0, 256),
-      selfbench_stage: request.stage.slice(0, 256),
-    };
-    const controller = new AbortController();
-    const stdout = new RollingOutput();
-    const stderr = new RollingOutput();
-    let terminationError: unknown;
-    let sandbox: Sandbox | undefined;
-    let deletePromise: Promise<void> | undefined;
-    let allocationMayExist = false;
-
-    const deleteHandle = (): Promise<void> => {
-      if (!sandbox) {
-        return Promise.resolve();
-      }
-      deletePromise ??= sandbox.delete({ signal: AbortSignal.timeout(CLEANUP_TIMEOUT_MS) });
-      return deletePromise;
-    };
-    const terminate = (error: unknown): void => {
-      if (terminationError !== undefined) {
-        return;
-      }
-      terminationError = error;
-      controller.abort(error);
-      if (sandbox) {
-        void deleteHandle().catch(() => undefined);
-      }
-    };
-    const abort = (): void => terminate(abortReason(options.signal));
-    options.signal?.addEventListener("abort", abort, { once: true });
-    if (options.signal?.aborted) {
-      abort();
-    }
-    const hardTimeout = setTimeout(() => {
-      terminate(new VercelHardTimeoutError(name, request.stage, request.timeoutMs));
-    }, request.timeoutMs);
-    hardTimeout.unref();
-
-    let outcome: RunOutcome;
-    try {
-      sandbox = await this.#createSandbox(
-        name,
-        tags,
-        resources.vcpus,
-        request.timeoutMs,
-        controller.signal,
-        (value) => {
-          allocationMayExist = value;
-        },
-      );
-      allocationMayExist = false;
-      throwIfTerminated(terminationError);
-
-      const session = sandbox.currentSession();
-      if (session.status !== "running") {
-        throw new Error(`Vercel sandbox ${name} was created in unexpected state ${session.status}`);
-      }
-      if (session.cwd !== VERCEL_WORK_DIRECTORY) {
-        throw new Error(
-          `Vercel image ${this.#config.image} has workdir ${session.cwd}; expected ${VERCEL_WORK_DIRECTORY}`,
-        );
-      }
-      if (session.vcpus !== resources.vcpus || session.memory !== resources.memoryMiB) {
-        throw new Error(
-          `Vercel sandbox ${name} returned ${session.vcpus} vCPU/${session.memory} MiB; expected ${resources.vcpus} vCPU/${resources.memoryMiB} MiB`,
-        );
-      }
-
-      if (request.files && request.files.length > 0) {
-        await session.writeFiles(
-          request.files.map((file) => ({ path: file.path, content: file.contents })),
-          { signal: controller.signal },
-        );
-      }
-      throwIfTerminated(terminationError);
-
-      const outputs = await executeVercelCommand({
-        session,
-        request,
-        options,
-        signal: controller.signal,
-        terminate,
-        stdout,
-        stderr,
-      });
-      throwIfTerminated(terminationError);
-      outcome = { ok: true, result: { sandboxId: sandbox.name, ...outputs } };
-    } catch (error) {
-      const failure = terminationError ?? error;
-      outcome =
-        failure instanceof VercelHardTimeoutError
-          ? {
-              ok: true,
-              result: {
-                sandboxId: sandbox?.name ?? name,
-                exitCode: HARD_TIMEOUT_EXIT_CODE,
-                stdout: stdout.text(),
-                stderr: stderr.text(),
-                outputs: {},
-              },
-            }
-          : { ok: false, error: failure };
-    }
-
-    clearTimeout(hardTimeout);
-    options.signal?.removeEventListener("abort", abort);
-    try {
-      await this.#cleanup(name, sandbox, deleteHandle, allocationMayExist);
-    } catch (cleanupError) {
-      const publicCleanupError = sanitizeCleanupError(cleanupError, this.#config.credentials.token);
-      outcome = outcome.ok
-        ? { ok: false, error: publicCleanupError }
-        : {
-            ok: false,
-            error: attachCleanupError(outcome.error, publicCleanupError),
-          };
-    }
-    if (!outcome.ok) {
-      throw outcome.error;
-    }
-    return outcome.result;
-  }
-
-  close(): void {}
-
-  async #createSandbox(
-    name: string,
-    tags: Readonly<Record<string, string>>,
-    vcpus: number,
-    timeoutMs: number,
-    signal: AbortSignal,
-    setAllocationMayExist: (value: boolean) => void,
-  ): Promise<Sandbox> {
-    for (let retries = 0; ; retries += 1) {
-      signal.throwIfAborted();
-      setAllocationMayExist(true);
-      try {
-        return await Sandbox.create({
-          ...this.#config.credentials,
-          fetch: this.#fetch,
-          image: this.#config.image,
-          name,
-          persistent: false,
-          resources: { vcpus },
-          signal: withTimeout(signal, CREATE_REQUEST_TIMEOUT_MS),
-          tags: { ...tags },
-          timeout: timeoutMs,
-        });
-      } catch (error) {
-        if (createErrorConfirmsNoAllocation(error)) {
-          setAllocationMayExist(false);
-        }
-        if (
-          !(error instanceof APIError) ||
-          error.response.status !== 429 ||
-          retries >= CREATE_RATE_LIMIT_RETRIES
-        ) {
-          throw error;
-        }
-        await this.#sleep(createRetryDelayMs(error.response), signal);
-      }
-    }
-  }
-
-  async #cleanup(
-    name: string,
-    sandbox: Sandbox | undefined,
-    deleteHandle: () => Promise<void>,
-    allocationMayExist: boolean,
-  ): Promise<void> {
-    let directDeleteError: unknown;
-    if (sandbox) {
-      try {
-        await deleteHandle();
-        return;
-      } catch (error) {
-        directDeleteError = error;
-      }
-    }
-    if (!sandbox && !allocationMayExist) {
-      return;
-    }
-
-    try {
-      const recoveryDelays = sandbox ? [CLEANUP_RETRY_DELAY_MS] : LATE_CREATE_RECOVERY_DELAYS_MS;
-      for (const delayMs of recoveryDelays) {
-        if (delayMs > 0) {
-          await this.#sleep(delayMs, AbortSignal.timeout(CLEANUP_TIMEOUT_MS));
-        }
-        let handle: Sandbox;
-        try {
-          handle = await Sandbox.get({
-            ...this.#config.credentials,
-            fetch: this.#fetch,
-            name,
-            resume: false,
-            signal: AbortSignal.timeout(CLEANUP_TIMEOUT_MS),
-          });
-        } catch (error) {
-          if (error instanceof APIError && error.response.status === 404) {
-            if (sandbox) {
-              return;
-            }
-            continue;
-          }
-          throw error;
-        }
-        try {
-          await handle.delete({ signal: AbortSignal.timeout(CLEANUP_TIMEOUT_MS) });
-        } catch (error) {
-          if (error instanceof APIError && error.response.status === 404) {
-            return;
-          }
-          throw error;
-        }
-        return;
-      }
-      throw new Error("sandbox absence remained unconfirmed after an ambiguous create failure");
-    } catch (recoveryError) {
-      const errors =
-        directDeleteError === undefined ? [recoveryError] : [directDeleteError, recoveryError];
-      throw new AggregateError(
-        errors,
-        `failed to delete Vercel sandbox ${name}: ${errors.map(errorMessage).join("; ")}`,
-      );
-    }
-  }
-}
-
-function validateConfig(config: VercelExecutionConfig): VercelExecutionConfig {
-  const credentials: VercelCredentials = {
-    token: config.credentials.token.trim(),
-    teamId: config.credentials.teamId.trim(),
-    projectId: config.credentials.projectId.trim(),
-  };
-  if (!credentials.token || !credentials.teamId || !credentials.projectId) {
-    throw new Error("Vercel execution requires a complete nonblank credential triple");
-  }
-  const image = config.image.trim();
-  if (!isDigestPinnedOciImage(image)) {
-    throw new Error("Vercel execution requires a digest-pinned image");
-  }
-  return { kind: "vercel", credentials, image, timeoutCapMs: config.timeoutCapMs };
-}
-
-function validateRequest(request: SandboxRequest): {
-  readonly vcpus: number;
-  readonly memoryMiB: number;
-} {
-  if (!Number.isInteger(request.timeoutMs) || request.timeoutMs < 100) {
-    throw new Error("sandbox timeout must be an integer of at least 100ms");
-  }
-  if (request.timeoutMs > 24 * 60 * 60 * 1_000) {
-    throw new Error("Vercel sandbox timeout cannot exceed 24 hours");
-  }
-  if (
-    request.inactivityTimeoutMs !== undefined &&
-    (!Number.isInteger(request.inactivityTimeoutMs) || request.inactivityTimeoutMs < 1)
-  ) {
-    throw new Error("sandbox inactivity timeout must be a positive integer");
-  }
-  if (request.command.length === 0 || !request.command[0]) {
-    throw new Error("sandbox command must not be empty");
-  }
-  for (const path of [
-    ...(request.files ?? []).map((file) => file.path),
-    ...(request.outputPaths ?? []),
-  ]) {
-    assertWorkPath(path);
-  }
-
-  const vcpus = request.cpu ?? 4;
-  if (!Number.isInteger(vcpus) || (vcpus !== 1 && (vcpus < 2 || vcpus > 32 || vcpus % 2 !== 0))) {
-    throw new Error("Vercel sandbox CPU must be 1 or an even integer from 2 through 32");
-  }
-  const memoryMiB = request.memoryMiB ?? vcpus * 2048;
-  if (memoryMiB !== vcpus * 2048) {
-    throw new Error(
-      `Vercel fixes memory at 2048 MiB per vCPU; ${vcpus} vCPU requires ${vcpus * 2048} MiB`,
-    );
-  }
-  return { vcpus, memoryMiB };
-}
-
-function assertWorkPath(path: string): void {
-  const normalized = posix.normalize(path);
-  if (
-    !path.startsWith(`${VERCEL_WORK_DIRECTORY}/`) ||
-    !normalized.startsWith(`${VERCEL_WORK_DIRECTORY}/`)
-  ) {
-    throw new Error(`sandbox path must be beneath ${VERCEL_WORK_DIRECTORY}: ${path}`);
-  }
-}
-
-function sandboxName(runId: string, stage: string): string {
-  const suffix = crypto.randomUUID().replaceAll("-", "").slice(0, 12);
-  const context = `${runId}-${stage}`.replace(/[^a-zA-Z0-9_-]/g, "-");
-  const prefix = `selfbench-${context}`.slice(0, 128 - suffix.length - 1).replace(/-+$/g, "");
-  return `${prefix}-${suffix}`;
-}
-
-function withTimeout(signal: AbortSignal, timeoutMs: number): AbortSignal {
-  return AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
-}
-
-async function abortableDelay(delayMs: number, signal: AbortSignal): Promise<void> {
-  await delay(delayMs, undefined, { signal });
-}
-
-function createRetryDelayMs(response: Response): number {
-  const header = response.headers.get("retry-after")?.trim();
-  if (!header) {
-    return CREATE_RATE_LIMIT_FALLBACK_DELAY_MS;
-  }
-  const seconds = Number(header);
-  const requestedDelay = Number.isFinite(seconds)
-    ? seconds * 1_000
-    : Date.parse(header) - Date.now();
-  if (!Number.isFinite(requestedDelay)) {
-    return CREATE_RATE_LIMIT_FALLBACK_DELAY_MS;
-  }
-  return Math.min(CREATE_RATE_LIMIT_MAX_DELAY_MS, Math.max(0, Math.ceil(requestedDelay)));
-}
-
-function createErrorConfirmsNoAllocation(error: unknown): boolean {
-  if (!(error instanceof APIError)) {
-    return false;
-  }
-  const status = error.response.status;
-  return status >= 400 && status < 500 && status !== 408 && status !== 409 && status !== 499;
-}
-
-function throwIfTerminated(error: unknown): void {
-  if (error !== undefined) {
-    throw error;
-  }
-}
-
-function abortReason(signal: AbortSignal | undefined): unknown {
-  return signal?.reason ?? new DOMException("The operation was aborted", "AbortError");
-}
-
-function attachCleanupError(primaryError: unknown, cleanupError: unknown): unknown {
-  if (primaryError instanceof Error) {
-    const cleanupMessage = errorMessage(cleanupError).slice(0, 500);
-    const message = `${primaryError.message}; Vercel sandbox cleanup also failed: ${cleanupMessage}`;
-    try {
-      Object.defineProperties(primaryError, {
-        cleanupError: { configurable: true, value: cleanupError },
-        message: { configurable: true, value: message, writable: true },
-      });
-      return primaryError;
-    } catch {
-      const wrapped = new Error(message, { cause: primaryError });
-      wrapped.name = primaryError.name;
-      Object.defineProperty(wrapped, "cleanupError", {
-        configurable: true,
-        value: cleanupError,
-      });
-      return wrapped;
-    }
-  }
-  return new AggregateError(
-    [primaryError, cleanupError],
-    "Vercel sandbox execution and cleanup both failed",
-  );
-}
-
-function sanitizeCleanupError(error: unknown, redactedValue: string): Error {
-  const sanitized = new Error(
-    errorMessage(error).replaceAll(redactedValue, "[redacted]").slice(0, 1_000),
-  );
-  sanitized.name = "VercelSandboxCleanupError";
-  return sanitized;
-}
-
-function errorMessage(error: unknown): string {
-  return error instanceof Error ? error.message : String(error);
-}

package/src/sandbox/providers/vercel/fetch.ts

@@ -1,45 +0,0 @@
-export class VercelCommandStartError extends Error {
-  constructor(message: string, cause?: unknown) {
-    super(`${message}; automatic retry was suppressed to prevent duplicate execution`, { cause });
-    // @vercel/sandbox's retry layer only honors AbortError as a non-retryable
-    // transport failure. The command layer restores the truthful public name.
-    this.name = "AbortError";
-  }
-
-  restorePublicName(): void {
-    this.name = "VercelCommandStartError";
-  }
-}
-
-export function preventAmbiguousVercelCommandStartRetries(
-  fetch: typeof globalThis.fetch,
-): typeof globalThis.fetch {
-  const wrapped = async (
-    input: Parameters<typeof fetch>[0],
-    init?: Parameters<typeof fetch>[1],
-  ) => {
-    const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url);
-    const isCommandStart =
-      init?.method === "POST" && /\/v2\/sandboxes\/sessions\/[^/]+\/cmd$/.test(url.pathname);
-    try {
-      const response = await fetch(input, init);
-      if (isCommandStart && response.status >= 500) {
-        await response.body?.cancel().catch(() => undefined);
-        throw new VercelCommandStartError(`Vercel command start returned HTTP ${response.status}`);
-      }
-      return response;
-    } catch (error) {
-      if (error instanceof VercelCommandStartError || !isCommandStart || init?.signal?.aborted) {
-        throw error;
-      }
-      throw new VercelCommandStartError("Vercel command start response was not received", error);
-    }
-  };
-  return Object.assign(wrapped, {
-    preconnect: (...args: Parameters<typeof fetch.preconnect>): void => {
-      if (typeof fetch.preconnect === "function") {
-        fetch.preconnect(...args);
-      }
-    },
-  });
-}

package/src/sandbox/timeout.ts

@@ -1,56 +0,0 @@
-import type {
-  SandboxExecutor,
-  SandboxRequest,
-  SandboxResult,
-  SandboxRunOptions,
-} from "./contracts.js";
-
-export const STANDARD_VERCEL_TIMEOUT_CAP_MS = 2 * 60 * 60 * 1_000;
-export const HOBBY_VERCEL_TIMEOUT_CAP_MS = 45 * 60 * 1_000;
-
-export function parseSandboxTimeoutCapText(value: string): number | undefined {
-  const normalized = value.trim().toLowerCase();
-  if (normalized === "") {
-    return undefined;
-  }
-  const match = /^(\d+)(ms|s|m|h)?$/.exec(normalized);
-  if (!match?.[1]) {
-    return Number.NaN;
-  }
-  const amount = Number(match[1]);
-  const multiplier =
-    match[2] === "h"
-      ? 60 * 60 * 1_000
-      : match[2] === "m"
-        ? 60 * 1_000
-        : match[2] === "s"
-          ? 1_000
-          : 1;
-  return amount * multiplier;
-}
-
-export class TimeoutCappedSandboxExecutor implements SandboxExecutor {
-  readonly #delegate: SandboxExecutor;
-  readonly #timeoutCapMs: number;
-
-  constructor(delegate: SandboxExecutor, timeoutCapMs: number) {
-    if (!Number.isInteger(timeoutCapMs) || timeoutCapMs < 100) {
-      throw new Error("sandbox timeout cap must be an integer of at least 100ms");
-    }
-    this.#delegate = delegate;
-    this.#timeoutCapMs = timeoutCapMs;
-  }
-
-  async run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult> {
-    return await this.#delegate.run(
-      request.timeoutMs > this.#timeoutCapMs
-        ? { ...request, timeoutMs: this.#timeoutCapMs }
-        : request,
-      options,
-    );
-  }
-
-  close(): void {
-    this.#delegate.close();
-  }
-}