self-bench 0.3.3 → 0.3.4

package/src/repair-main.ts

@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
-import { loadConfig } from "./config.js";
+import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
 import { parallelMap } from "./parallel.js";
 import { runCommand } from "./process.js";
 import { createSandboxExecutor } from "./sandbox.js";
@@ -17,7 +17,7 @@ const parsed = parseArgs({
     review: { type: "string" },
     output: { type: "string" },
     task: { type: "string" },
-    concurrency: { type: "string", default: "9" },
+    concurrency: { type: "string" },
     model: { type: "string", default: "gpt-5.6-sol" },
     help: { type: "boolean", short: "h" },
   },
@@ -30,7 +30,7 @@ Usage:
   self-bench-repair --tasks DIRECTORY --review REPORT.json --output DIRECTORY [options]
 
 Options:
-  --concurrency N  Concurrent repair sandboxes (default: 9)
+  --concurrency N  Concurrent repair sandboxes (default: 9; Vercel: worker setting, default 4)
   --model MODEL    Codex subscription model (default: gpt-5.6-sol)
   --task ID        Repair only one coupled task
   -h, --help       Show this help`);
@@ -43,7 +43,11 @@ const outputDirectory = resolve(parsed.values.output ?? fail("--output is requir
 if (outputDirectory === tasksDirectory || outputDirectory.startsWith(`${tasksDirectory}/`)) {
   throw new Error("--output must be outside --tasks");
 }
-const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
+const config = loadWorkerConfig();
+const concurrency = positiveInteger(
+  parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 9)),
+  "--concurrency",
+);
 const rawReview = JSON.parse(await readFile(reviewPath, "utf8")) as unknown;
 const review = reviewReport(rawReview);
 const coupled = new Map(
@@ -77,7 +81,6 @@ for (const source of sourceTasks) {
   }
 }
 
-const config = loadConfig();
 const sandbox = createSandboxExecutor(config.execution);
 const [modelAuth, repairer] = await Promise.all([
   loadCodexModelAuth(),

package/src/sandbox-timeout.ts

@@ -0,0 +1,56 @@
+import type {
+  SandboxExecutor,
+  SandboxRequest,
+  SandboxResult,
+  SandboxRunOptions,
+} from "./sandbox.js";
+
+export const STANDARD_VERCEL_TIMEOUT_CAP_MS = 2 * 60 * 60 * 1_000;
+export const HOBBY_VERCEL_TIMEOUT_CAP_MS = 45 * 60 * 1_000;
+
+export function parseSandboxTimeoutCapText(value: string): number | undefined {
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "") {
+    return undefined;
+  }
+  const match = /^(\d+)(ms|s|m|h)?$/.exec(normalized);
+  if (!match?.[1]) {
+    return Number.NaN;
+  }
+  const amount = Number(match[1]);
+  const multiplier =
+    match[2] === "h"
+      ? 60 * 60 * 1_000
+      : match[2] === "m"
+        ? 60 * 1_000
+        : match[2] === "s"
+          ? 1_000
+          : 1;
+  return amount * multiplier;
+}
+
+export class TimeoutCappedSandboxExecutor implements SandboxExecutor {
+  readonly #delegate: SandboxExecutor;
+  readonly #timeoutCapMs: number;
+
+  constructor(delegate: SandboxExecutor, timeoutCapMs: number) {
+    if (!Number.isInteger(timeoutCapMs) || timeoutCapMs < 100) {
+      throw new Error("sandbox timeout cap must be an integer of at least 100ms");
+    }
+    this.#delegate = delegate;
+    this.#timeoutCapMs = timeoutCapMs;
+  }
+
+  async run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult> {
+    return await this.#delegate.run(
+      request.timeoutMs > this.#timeoutCapMs
+        ? { ...request, timeoutMs: this.#timeoutCapMs }
+        : request,
+      options,
+    );
+  }
+
+  close(): void {
+    this.#delegate.close();
+  }
+}

package/src/sandbox.ts

@@ -1,6 +1,8 @@
-import type { SelfBenchConfig } from "./config.js";
+import type { SelfBenchWorkerConfig } from "./config.js";
 import { DockerSandboxExecutor } from "./docker-executor.js";
 import { ModalSandboxExecutor } from "./modal-executor.js";
+import { TimeoutCappedSandboxExecutor } from "./sandbox-timeout.js";
+import { VercelSandboxExecutor } from "./vercel-executor.js";
 
 export interface SandboxFile {
   readonly path: string;
@@ -44,8 +46,16 @@ export interface SandboxExecutor {
   close(): void;
 }
 
-export function createSandboxExecutor(config: SelfBenchConfig["execution"]): SandboxExecutor {
-  return config.kind === "modal"
-    ? new ModalSandboxExecutor(config)
-    : new DockerSandboxExecutor(config);
+export function createSandboxExecutor(config: SelfBenchWorkerConfig["execution"]): SandboxExecutor {
+  switch (config.kind) {
+    case "docker":
+      return new DockerSandboxExecutor(config);
+    case "modal":
+      return new ModalSandboxExecutor(config);
+    case "vercel":
+      return new TimeoutCappedSandboxExecutor(
+        new VercelSandboxExecutor(config),
+        config.timeoutCapMs,
+      );
+  }
 }

package/src/temporal/activities.ts

@@ -12,7 +12,7 @@ import {
   couplingReviewInput,
   couplingReviewSchema,
 } from "../codex-review.js";
-import type { SelfBenchConfig } from "../config.js";
+import type { SelfBenchConfig, SelfBenchWorkerConfig } from "../config.js";
 import {
   type ArtifactRef,
   type AuditResult,
@@ -34,6 +34,7 @@ import {
   scanBaseContractArtifacts,
 } from "../coupling.js";
 import { assertPullRequestBelongsToRepository } from "../github.js";
+import { harborChildEnvironment } from "../harbor-environment.js";
 import {
   type HarborJobResult,
   harborInfrastructureError,
@@ -114,7 +115,7 @@ export interface SelfBenchActivities {
   buildExport(input: ExportInput): Promise<ArtifactRef>;
 }
 
-export function createActivities(config: SelfBenchConfig): SelfBenchActivities {
+export function createActivities(config: SelfBenchWorkerConfig): SelfBenchActivities {
   const store = createArtifactStore(config.artifact);
   const sandbox = createSandboxExecutor(config.execution);
   return {
@@ -1054,7 +1055,12 @@ async function runHarborGate(
       "--yes",
       "--quiet",
     ],
-    { allowFailure: true, timeoutMs: 3 * 60 * 60 * 1000, signal },
+    {
+      allowFailure: true,
+      env: harborChildEnvironment(),
+      timeoutMs: 3 * 60 * 60 * 1000,
+      signal,
+    },
   );
   if (result.exitCode !== 0) {
     throw ApplicationFailure.create({

package/src/temporal/worker-main.ts

@@ -1,13 +1,13 @@
 import { fileURLToPath } from "node:url";
 import { Worker } from "@temporalio/worker";
-import { loadConfig } from "../config.js";
+import { loadWorkerConfig } from "../config.js";
 import { removeEmptyModalCredentialOverrides } from "../modal-auth.js";
 import { runCommand } from "../process.js";
 import { createActivities } from "./activities.js";
 import { connectTemporalWorker } from "./connection.js";
 
 removeEmptyModalCredentialOverrides();
-const config = loadConfig();
+const config = loadWorkerConfig();
 if (config.execution.kind === "docker" || config.harborEnvironment === "docker") {
   await runCommand("docker", ["info"], { timeoutMs: 30_000 });
   await runCommand("docker", ["compose", "version"], { timeoutMs: 30_000 });

package/src/terminal-prompts.ts

@@ -0,0 +1,197 @@
+import {
+  autocomplete,
+  cancel as cancelPrompt,
+  confirm as confirmPrompt,
+  isCancel,
+  password,
+  select as selectPrompt,
+  text as textPrompt,
+} from "@clack/prompts";
+
+export class SetupCanceledError extends Error {
+  constructor(message = "Setup canceled") {
+    super(message);
+    this.name = "SetupCanceledError";
+  }
+}
+
+export interface PromptChoice {
+  readonly value: string;
+  readonly label: string;
+  readonly hint?: string;
+}
+
+export interface SetupPrompter {
+  select(message: string, choices: readonly PromptChoice[], defaultValue?: string): Promise<string>;
+  search(message: string, choices: readonly PromptChoice[], defaultValue?: string): Promise<string>;
+  text(message: string, defaultValue?: string): Promise<string>;
+  secret(message: string): Promise<string>;
+  confirm(message: string, defaultValue: boolean): Promise<boolean>;
+}
+
+export class TerminalPrompter implements SetupPrompter {
+  readonly #input: NodeJS.ReadStream;
+  readonly #output: NodeJS.WriteStream;
+
+  constructor(
+    input: NodeJS.ReadStream = process.stdin,
+    output: NodeJS.WriteStream = process.stdout,
+  ) {
+    this.#input = input;
+    this.#output = output;
+  }
+
+  async select(
+    message: string,
+    choices: readonly PromptChoice[],
+    defaultValue?: string,
+  ): Promise<string> {
+    this.#assertChoices(message, choices);
+    return await this.#run((signal) =>
+      selectPrompt<string>({
+        message,
+        options: choices.map(toPromptOption),
+        maxItems: this.#visibleItemCount(),
+        showInstructions: true,
+        ...(defaultValue !== undefined && choices.some(({ value }) => value === defaultValue)
+          ? { initialValue: defaultValue }
+          : {}),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async search(
+    message: string,
+    choices: readonly PromptChoice[],
+    defaultValue?: string,
+  ): Promise<string> {
+    this.#assertChoices(message, choices);
+    return await this.#run((signal) =>
+      autocomplete<string>({
+        message,
+        options: choices.map(toPromptOption),
+        placeholder: "Type to filter…",
+        maxItems: this.#visibleItemCount(),
+        filter: (query, option) =>
+          `${option.label ?? ""}\n${option.hint ?? ""}\n${option.value}`
+            .toLowerCase()
+            .includes(query.toLowerCase()),
+        ...(defaultValue !== undefined && choices.some(({ value }) => value === defaultValue)
+          ? { initialValue: defaultValue }
+          : {}),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async text(message: string, defaultValue?: string): Promise<string> {
+    return await this.#run((signal) =>
+      textPrompt({
+        message,
+        ...(defaultValue === undefined ? {} : { placeholder: defaultValue, defaultValue }),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async secret(message: string): Promise<string> {
+    if (!this.#input.isTTY || !this.#input.setRawMode) {
+      throw new Error("secure token input requires an interactive terminal");
+    }
+    return await this.#run((signal) =>
+      password({
+        message,
+        // An empty mask keeps both the token and its length out of terminal output.
+        mask: "",
+        validate: (value) => (value?.trim() ? undefined : "Enter a project-scoped token."),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async confirm(message: string, defaultValue: boolean): Promise<boolean> {
+    return await this.#run((signal) =>
+      confirmPrompt({
+        message,
+        initialValue: defaultValue,
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async #run<T>(prompt: (signal: AbortSignal) => Promise<T | symbol>): Promise<T> {
+    const controller = new AbortController();
+    const wasRaw = this.#input.isRaw;
+    const wasFlowing = this.#input.readableFlowing;
+    let inputFailure: Error | undefined;
+    const abort = (error: Error): void => {
+      inputFailure = error;
+      controller.abort(error);
+    };
+    const onEnd = (): void => abort(new SetupCanceledError("Setup canceled at end of input"));
+    const onError = (error: Error): void => abort(error);
+    this.#input.once("end", onEnd);
+    this.#input.once("error", onError);
+    if (this.#input.readableEnded) {
+      onEnd();
+    }
+
+    try {
+      const result = await prompt(controller.signal);
+      if (isCancel(result)) {
+        const error = inputFailure ?? new SetupCanceledError();
+        if (error instanceof SetupCanceledError) {
+          cancelPrompt("Setup canceled.", { output: this.#output, withGuide: true });
+        }
+        throw error;
+      }
+      return result;
+    } finally {
+      this.#input.off("end", onEnd);
+      this.#input.off("error", onError);
+      if (this.#input.isTTY && this.#input.setRawMode && this.#input.isRaw !== wasRaw) {
+        this.#input.setRawMode(Boolean(wasRaw));
+      }
+      if (wasFlowing !== true) {
+        this.#input.pause();
+      }
+    }
+  }
+
+  #assertChoices(message: string, choices: readonly PromptChoice[]): void {
+    if (choices.length === 0) {
+      throw new Error(`${message} has no available choices`);
+    }
+  }
+
+  #visibleItemCount(): number {
+    return Math.max(3, Math.min(8, (this.#output.rows ?? 20) - 6));
+  }
+}
+
+export function isInteractiveTerminal(): boolean {
+  return Boolean(process.stdin.isTTY && process.stdout.isTTY);
+}
+
+function toPromptOption(choice: PromptChoice): {
+  readonly value: string;
+  readonly label: string;
+  readonly hint?: string;
+} {
+  return {
+    value: choice.value,
+    label: choice.label,
+    ...(choice.hint ? { hint: choice.hint } : {}),
+  };
+}

package/src/terminal-reporter.ts

@@ -0,0 +1,126 @@
+import type { Writable } from "node:stream";
+import { stripVTControlCharacters } from "node:util";
+import { cancel as cancelPrompt, intro, log, outro, spinner } from "@clack/prompts";
+import { type CommandOutputHandler, RollingOutput } from "./process.js";
+import { SetupCanceledError } from "./terminal-prompts.js";
+
+export interface SetupTaskLabels {
+  readonly pending: string;
+  readonly success: string;
+  readonly failure: string;
+}
+
+export interface SetupReporter {
+  intro(title: string): void;
+  message(message: string): void;
+  warn(message: string): void;
+  cancel(message: string): void;
+  task<T>(
+    labels: SetupTaskLabels,
+    operation: (onOutput: CommandOutputHandler) => Promise<T>,
+  ): Promise<T>;
+  finish(title: string, details: readonly string[]): void;
+}
+
+export class TerminalSetupReporter implements SetupReporter {
+  readonly #output: Writable;
+  readonly #errorOutput: Writable;
+  readonly #verbose: boolean;
+
+  constructor(
+    options: {
+      readonly output?: Writable;
+      readonly errorOutput?: Writable;
+      readonly verbose?: boolean;
+    } = {},
+  ) {
+    this.#output = options.output ?? process.stdout;
+    this.#errorOutput = options.errorOutput ?? process.stderr;
+    this.#verbose = options.verbose ?? false;
+  }
+
+  intro(title: string): void {
+    intro(title, { output: this.#output, withGuide: true });
+  }
+
+  message(message: string): void {
+    log.message(message, { output: this.#output, withGuide: true });
+  }
+
+  warn(message: string): void {
+    log.warn(message, { output: this.#output, withGuide: true });
+  }
+
+  cancel(message: string): void {
+    cancelPrompt(message, { output: this.#output, withGuide: true });
+  }
+
+  async task<T>(
+    labels: SetupTaskLabels,
+    operation: (onOutput: CommandOutputHandler) => Promise<T>,
+  ): Promise<T> {
+    const captured = new RollingOutput();
+    const onOutput: CommandOutputHandler = (stream, chunk) => {
+      captured.push(Buffer.from(chunk));
+      if (this.#verbose) {
+        (stream === "stderr" ? this.#errorOutput : this.#output).write(chunk);
+      }
+    };
+
+    if (this.#verbose) {
+      log.step(labels.pending, { output: this.#output, withGuide: true });
+      try {
+        const result = await operation(onOutput);
+        log.success(labels.success, { output: this.#output, withGuide: true });
+        return result;
+      } catch (error) {
+        if (error instanceof SetupCanceledError) {
+          this.cancel("Setup canceled.");
+          throw error;
+        }
+        log.error(labels.failure, { output: this.#output, withGuide: true });
+        throw error;
+      }
+    }
+
+    const progress = spinner({ indicator: "timer", output: this.#output, withGuide: true });
+    progress.start(labels.pending);
+    try {
+      const result = await operation(onOutput);
+      progress.stop(labels.success);
+      return result;
+    } catch (error) {
+      if (error instanceof SetupCanceledError) {
+        progress.cancel("Setup canceled.");
+        throw error;
+      }
+      progress.error(labels.failure);
+      this.#showFailureOutput(captured.text());
+      throw error;
+    }
+  }
+
+  finish(title: string, details: readonly string[]): void {
+    log.success(title, { output: this.#output, withGuide: true });
+    log.message([...details], {
+      output: this.#output,
+      spacing: 0,
+      withGuide: true,
+    });
+    outro("Vercel Sandbox is ready.", { output: this.#output, withGuide: true });
+  }
+
+  #showFailureOutput(value: string): void {
+    const cleaned = stripVTControlCharacters(value)
+      .replace(/\r(?!\n)/g, "\n")
+      .trim();
+    if (!cleaned) {
+      return;
+    }
+    log.message(["Command output:", ...cleaned.split("\n")], {
+      output: this.#output,
+      spacing: 0,
+      withGuide: true,
+    });
+  }
+}

package/src/validate-main.ts

@@ -3,9 +3,11 @@
 import { access, mkdir, readdir, writeFile } from "node:fs/promises";
 import { basename, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
+import { harborChildEnvironment } from "./harbor-environment.js";
 import { harborInfrastructureError, readHarborJobResult } from "./harbor-results.js";
 import { parallelMap } from "./parallel.js";
 import { runCommand } from "./process.js";
+import { type HarborEnvironment, isHarborEnvironment } from "./providers.js";
 
 const parsed = parseArgs({
   options: {
@@ -33,7 +35,7 @@ Options:
 const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
 const jobsDirectory = resolve(parsed.values.jobs ?? fail("--jobs is required"));
 const environment = parsed.values.environment ?? "modal";
-if (environment !== "modal" && environment !== "docker") {
+if (!isHarborEnvironment(environment)) {
   throw new Error("--environment must be modal or docker");
 }
 const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
@@ -90,7 +92,7 @@ async function runGate(
   taskDirectory: string,
   taskId: string,
   agent: "nop" | "oracle",
-  environment: "docker" | "modal",
+  environment: HarborEnvironment,
   jobsDirectory: string,
 ): Promise<{ readonly passed: boolean; readonly rewards: Readonly<Record<string, number>> }> {
   const jobName = `${taskId}-${agent}-${crypto.randomUUID().slice(0, 8)}`;
@@ -116,7 +118,11 @@ async function runGate(
       "--yes",
       "--quiet",
     ],
-    { allowFailure: true, timeoutMs: 4 * 60 * 60 * 1000 },
+    {
+      allowFailure: true,
+      env: harborChildEnvironment(),
+      timeoutMs: 4 * 60 * 60 * 1000,
+    },
   );
   if (process.exitCode !== 0) {
     throw new Error(`Harbor ${agent} exited ${process.exitCode}: ${process.stderr.slice(-1_000)}`);