self-bench 0.3.5 → 0.3.8

package/src/repair-main.ts

@@ -1,209 +0,0 @@
-#!/usr/bin/env node
-
-import { access, cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { basename, dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { parseArgs } from "node:util";
-import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import { createSandboxExecutor } from "./sandbox.js";
-import { loadCodexModelAuth } from "./subscription-auth.js";
-
-const parsed = parseArgs({
-  options: {
-    tasks: { type: "string" },
-    review: { type: "string" },
-    output: { type: "string" },
-    task: { type: "string" },
-    concurrency: { type: "string" },
-    model: { type: "string", default: "gpt-5.6-sol" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Repair coupled tests in expanded Harbor tasks.
-
-Usage:
-  self-bench-repair --tasks DIRECTORY --review REPORT.json --output DIRECTORY [options]
-
-Options:
-  --concurrency N  Concurrent repair sandboxes (default: 9; Vercel: worker setting, default 4)
-  --model MODEL    Codex subscription model (default: gpt-5.6-sol)
-  --task ID        Repair only one coupled task
-  -h, --help       Show this help`);
-  process.exit(0);
-}
-
-const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
-const reviewPath = resolve(parsed.values.review ?? fail("--review is required"));
-const outputDirectory = resolve(parsed.values.output ?? fail("--output is required"));
-if (outputDirectory === tasksDirectory || outputDirectory.startsWith(`${tasksDirectory}/`)) {
-  throw new Error("--output must be outside --tasks");
-}
-const config = loadWorkerConfig();
-const concurrency = positiveInteger(
-  parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 9)),
-  "--concurrency",
-);
-const rawReview = JSON.parse(await readFile(reviewPath, "utf8")) as unknown;
-const review = reviewReport(rawReview);
-const coupled = new Map(
-  review.tasks
-    .filter((task) => task.resolution.verdict === "coupled")
-    .filter((task) => parsed.values.task === undefined || task.taskId === parsed.values.task)
-    .map((task) => [task.taskId, task]),
-);
-if (parsed.values.task !== undefined && coupled.size === 0) {
-  throw new Error(`coupled task not found in review: ${parsed.values.task}`);
-}
-const sourceTasks = (
-  await Promise.all(
-    (
-      await readdir(tasksDirectory)
-    ).map(async (taskId) => {
-      const directory = join(tasksDirectory, taskId);
-      return await access(join(directory, "instruction.md")).then(
-        () => directory,
-        () => undefined,
-      );
-    }),
-  )
-).filter((directory): directory is string => directory !== undefined);
-
-await rm(outputDirectory, { recursive: true, force: true });
-await mkdir(outputDirectory, { recursive: true });
-for (const source of sourceTasks) {
-  if (parsed.values.task === undefined && !coupled.has(basename(source))) {
-    await cp(source, join(outputDirectory, basename(source)), { recursive: true });
-  }
-}
-
-const sandbox = createSandboxExecutor(config.execution);
-const [modelAuth, repairer] = await Promise.all([
-  loadCodexModelAuth(),
-  readFile(join(assetRoot(), "dist/sandbox-repair.bundle.js")),
-]);
-const results = await parallelMap([...coupled.values()], concurrency, async (taskReview) => {
-  const taskId = taskReview.taskId;
-  const source = join(tasksDirectory, taskId);
-  console.error(`repairing ${taskId}`);
-  const scratch = await mkdtemp(join(tmpdir(), `selfbench-repair-${taskId}-`));
-  try {
-    const archive = join(scratch, "task.tar.gz");
-    await runCommand("tar", ["-czf", archive, "-C", source, "."]);
-    const result = await sandbox.run({
-      runId: "selfbench-repair-v1",
-      stage: taskId,
-      timeoutMs: 2 * 60 * 60 * 1000,
-      cpu: 4,
-      memoryMiB: 8192,
-      files: [
-        { path: "/work/task.tar.gz", contents: await readFile(archive) },
-        { path: "/work/review.json", contents: `${JSON.stringify(taskReview, null, 2)}\n` },
-        { path: "/work/sandbox-repair.js", contents: repairer },
-      ],
-      outputPaths: ["/work/repaired-task.tar.gz", "/work/repair-report.json"],
-      secrets: {
-        ...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
-        ...(modelAuth.authJson ? { SELFBENCH_CODEX_AUTH_JSON: modelAuth.authJson } : {}),
-      },
-      environment: { SELFBENCH_REPAIR_MODEL: parsed.values.model ?? "gpt-5.6-sol" },
-      command: [
-        "node",
-        "/work/sandbox-repair.js",
-        "/work/task.tar.gz",
-        "/work/review.json",
-        "/work/repaired-task.tar.gz",
-        "/work/repair-report.json",
-      ],
-    });
-    const repaired = result.outputs["/work/repaired-task.tar.gz"];
-    const report = result.outputs["/work/repair-report.json"];
-    if (result.exitCode !== 0 || !repaired || !report) {
-      throw new Error(
-        `repair sandbox ${result.sandboxId} failed: ${result.stderr.trim() || result.stdout.trim()}`,
-      );
-    }
-    const destination = join(outputDirectory, taskId);
-    await mkdir(destination);
-    const repairedArchive = join(scratch, "repaired-task.tar.gz");
-    await writeFile(repairedArchive, repaired);
-    await runCommand("tar", ["-xzf", repairedArchive, "-C", destination]);
-    const parsedReport = JSON.parse(Buffer.from(report).toString("utf8")) as Record<
-      string,
-      unknown
-    >;
-    console.error(`${taskId}: repaired in ${result.sandboxId}`);
-    return { taskId, status: "repaired" as const, sandboxId: result.sandboxId, ...parsedReport };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    console.error(`${taskId}: error: ${message}`);
-    return { taskId, status: "error" as const, error: message };
-  } finally {
-    await rm(scratch, { recursive: true, force: true });
-  }
-});
-sandbox.close();
-const reportOutput = {
-  schemaVersion: 1,
-  generatedAt: new Date().toISOString(),
-  sourceTasks: tasksDirectory,
-  sourceReview: reviewPath,
-  outputTasks: outputDirectory,
-  model: parsed.values.model,
-  copiedClean: parsed.values.task === undefined ? sourceTasks.length - coupled.size : 0,
-  repaired: results.filter((result) => result.status === "repaired").length,
-  errors: results.filter((result) => result.status === "error").length,
-  tasks: results,
-};
-await writeFile(
-  join(outputDirectory, "repair-report.json"),
-  `${JSON.stringify(reportOutput, null, 2)}\n`,
-);
-console.log(JSON.stringify(reportOutput, null, 2));
-
-interface ReviewTask {
-  readonly taskId: string;
-  readonly resolution: { readonly verdict: "clean" | "coupled" };
-  readonly [key: string]: unknown;
-}
-
-function reviewReport(value: unknown): { readonly tasks: readonly ReviewTask[] } {
-  if (!isRecord(value) || !Array.isArray(value.tasks)) {
-    throw new Error("invalid coupling review report");
-  }
-  const tasks = value.tasks.filter(
-    (task): task is ReviewTask =>
-      isRecord(task) &&
-      typeof task.taskId === "string" &&
-      isRecord(task.resolution) &&
-      (task.resolution.verdict === "clean" || task.resolution.verdict === "coupled"),
-  );
-  if (tasks.length !== value.tasks.length) {
-    throw new Error("coupling review report contains invalid task entries");
-  }
-  return { tasks };
-}
-
-function assetRoot(): string {
-  return resolve(dirname(fileURLToPath(import.meta.url)), "..");
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const number = Number(value);
-  if (!Number.isInteger(number) || number < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return number;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function fail(message: string): never {
-  throw new Error(message);
-}

package/src/repair.ts

@@ -1,55 +0,0 @@
-export function patchPaths(patch: string): readonly string[] {
-  const paths = new Set<string>();
-  for (const line of patch.split("\n")) {
-    const match = /^diff --git a\/(.+) b\/(.+)$/.exec(line);
-    if (match?.[2]) {
-      paths.add(match[2]);
-    }
-  }
-  return [...paths].sort();
-}
-
-export function assertRepairPaths(
-  originalTestPatch: string,
-  changedPaths: readonly string[],
-): void {
-  const allowed = new Set(patchPaths(originalTestPatch));
-  if (allowed.size === 0) {
-    throw new Error("original held-out test patch changes no files");
-  }
-  const outside = changedPaths.filter((path) => !allowed.has(path));
-  if (outside.length > 0) {
-    throw new Error(`repair changed files outside the held-out tests: ${outside.join(", ")}`);
-  }
-}
-
-export function repairPrompt(input: {
-  readonly taskId: string;
-  readonly authenticRequest: string;
-  readonly couplingReport: string;
-  readonly allowedPaths: readonly string[];
-}): string {
-  return `Repair the held-out tests for SelfBench task ${input.taskId}.
-
-The repository is the exact base snapshot with the current held-out test patch already applied. You may edit only these existing test files:
-
-${input.allowedPaths.map((path) => `- ${path}`).join("\n")}
-
-The authentic engineer request is:
-
-<authentic_request>
-${input.authenticRequest.trim()}
-</authentic_request>
-
-The independent coupling report is:
-
-<coupling_report>
-${input.couplingReport.trim()}
-</coupling_report>
-
-Rewrite the tests so they verify every material requested behavior through stable public boundaries without requiring the gold patch's exact internal names, helper structure, error prose, mock input shape, response presentation, or incidental implementation choices. Preserve meaningful negative, authorization, compatibility, and regression coverage. Do not edit application code, the request, or the gold patch. Do not delete assertions merely to silence the report. A coherent alternative implementation must be able to pass, while the unchanged base must still fail the fail-to-pass tests and the gold implementation must still pass.
-
-The final working tree must retain a non-empty test patch relative to HEAD and every originally added fail-to-pass test must still have an equivalent behavioral test. Do not reset, revert, delete, or commit the test patch. If an exact public response field is the only stable seam, prefer asserting the underlying externally visible behavior through existing endpoints, persistence, or follow-up actions. If no rigorous uncoupled test is possible within the allowed files, leave the current tests in place and explain the blocker in your final message instead of removing coverage.
-
-Inspect /work/task/solution/gold.patch only to understand the intended behavior and available seams, never to copy its private structure into assertions. Inspect /work/task/tests/test.sh for the verifier command. Run focused tests when feasible. Finish only after the working tree contains the repaired test files and no other changes.`;
-}

package/src/run-wait.ts

@@ -1,40 +0,0 @@
-export interface PolledRunStatus {
-  readonly phase: string;
-  readonly error?: unknown;
-  readonly [key: string]: unknown;
-}
-
-export interface WaitForRunOptions {
-  readonly poll: () => Promise<PolledRunStatus>;
-  readonly onPhase?: (status: PolledRunStatus) => void;
-  readonly intervalMs?: number;
-  readonly delay?: (milliseconds: number) => Promise<void>;
-}
-
-const FAILED_PHASES = new Set(["blocked", "failed", "cancelled"]);
-
-export async function waitForRun(options: WaitForRunOptions): Promise<PolledRunStatus> {
-  const intervalMs = options.intervalMs ?? 2_000;
-  const delay = options.delay ?? defaultDelay;
-  let previousPhase: string | undefined;
-
-  while (true) {
-    const status = await options.poll();
-    if (status.phase !== previousPhase) {
-      options.onPhase?.(status);
-      previousPhase = status.phase;
-    }
-    if (status.phase === "complete") {
-      return status;
-    }
-    if (FAILED_PHASES.has(status.phase)) {
-      const detail = typeof status.error === "string" ? `: ${status.error}` : "";
-      throw new Error(`SelfBench run ${status.phase}${detail}`);
-    }
-    await delay(intervalMs);
-  }
-}
-
-function defaultDelay(milliseconds: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, milliseconds));
-}

package/src/sandbox-author.ts

@@ -1,19 +0,0 @@
-#!/usr/bin/env node
-
-import { readdir } from "node:fs/promises";
-import { join } from "node:path";
-import { compileHarborTask } from "./harbor-task.js";
-
-const [tasksRoot, repositoryDirectory, outputDirectory] = process.argv.slice(2);
-if (!tasksRoot || !repositoryDirectory || !outputDirectory) {
-  throw new Error("usage: sandbox-author TASKS_ROOT REPOSITORY OUTPUT");
-}
-
-const entries = await readdir(tasksRoot, { withFileTypes: true });
-const taskDirectories = entries
-  .filter((entry) => entry.isDirectory())
-  .map((entry) => join(tasksRoot, entry.name));
-if (taskDirectories.length !== 1) {
-  throw new Error(`authoring must produce exactly one task; found ${taskDirectories.length}`);
-}
-await compileHarborTask(taskDirectories[0] as string, repositoryDirectory, outputDirectory);

package/src/sandbox-repair.ts

@@ -1,156 +0,0 @@
-#!/usr/bin/env node
-
-import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
-import { homedir, tmpdir } from "node:os";
-import { join } from "node:path";
-import { sha256 } from "./hash.js";
-import { runCommand } from "./process.js";
-import { assertRepairPaths, patchPaths, repairPrompt } from "./repair.js";
-
-const [archivePath, reviewPath, outputArchive, outputReport] = process.argv.slice(2);
-if (!archivePath || !reviewPath || !outputArchive || !outputReport) {
-  throw new Error("usage: sandbox-repair TASK.tar.gz REVIEW.json OUTPUT.tar.gz OUTPUT-REPORT.json");
-}
-
-const extractedDirectory = "/work/task";
-const repositoryDirectory = "/work/repo";
-await Promise.all([mkdir(extractedDirectory, { recursive: true }), mkdir(repositoryDirectory)]);
-await runCommand("tar", ["-xzf", archivePath, "-C", extractedDirectory]);
-const taskDirectory = await access(join(extractedDirectory, "instruction.md")).then(
-  () => extractedDirectory,
-  () => join(extractedDirectory, "harbor-task"),
-);
-
-const [instruction, originalPatch, review] = await Promise.all([
-  readFile(join(taskDirectory, "instruction.md"), "utf8"),
-  readFile(join(taskDirectory, "tests/test.patch"), "utf8"),
-  readFile(reviewPath, "utf8"),
-]);
-const manifestPath = join(taskDirectory, ".selfbench-manifest.json");
-const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as Record<string, unknown>;
-const taskId = typeof manifest.taskId === "string" ? manifest.taskId : "unknown-task";
-const allowedPaths = patchPaths(originalPatch);
-
-await runCommand("tar", [
-  "-xzf",
-  join(taskDirectory, "tests/repo.tar.gz"),
-  "-C",
-  repositoryDirectory,
-]);
-await runCommand("git", ["-C", repositoryDirectory, "init", "-q"]);
-await runCommand("git", ["-C", repositoryDirectory, "config", "user.name", "SelfBench"]);
-await runCommand("git", ["-C", repositoryDirectory, "config", "user.email", "selfbench@local"]);
-await runCommand("git", ["-C", repositoryDirectory, "add", "-A"]);
-await runCommand("git", ["-C", repositoryDirectory, "commit", "-qm", "base"]);
-await runCommand("git", [
-  "-C",
-  repositoryDirectory,
-  "apply",
-  join(taskDirectory, "tests/test.patch"),
-]);
-await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
-
-const codexAuth = process.env.SELFBENCH_CODEX_AUTH_JSON;
-const apiKey = process.env.OPENAI_API_KEY;
-if (!apiKey && !codexAuth) {
-  fail("OPENAI_API_KEY or SELFBENCH_CODEX_AUTH_JSON is required");
-}
-const codexHome = join(homedir(), ".codex");
-const authPath = join(codexHome, "auth.json");
-await mkdir(codexHome, { recursive: true });
-await writeFile(authPath, codexAuth ?? `${JSON.stringify({ OPENAI_API_KEY: apiKey })}\n`);
-await chmod(authPath, 0o600);
-const promptPath = join(tmpdir(), `selfbench-repair-${taskId}.md`);
-await writeFile(
-  promptPath,
-  repairPrompt({
-    taskId,
-    authenticRequest: instruction,
-    couplingReport: review,
-    allowedPaths,
-  }),
-);
-
-const codex = await runCommand(
-  "codex",
-  [
-    "exec",
-    "--model",
-    process.env.SELFBENCH_REPAIR_MODEL ?? "gpt-5.6-sol",
-    "--dangerously-bypass-approvals-and-sandbox",
-    "--ephemeral",
-    "--ignore-user-config",
-    "--json",
-    "-C",
-    repositoryDirectory,
-    "-",
-  ],
-  {
-    allowFailure: true,
-    timeoutMs: 90 * 60 * 1000,
-    env: process.env,
-    input: await readFile(promptPath, "utf8"),
-    onOutput: (stream, chunk) => {
-      (stream === "stdout" ? process.stdout : process.stderr).write(chunk);
-    },
-  },
-);
-if (codex.exitCode !== 0) {
-  throw new Error(`Codex repair exited ${codex.exitCode}: ${codex.stderr.slice(-2_000)}`);
-}
-
-const [tracked, untracked] = await Promise.all([
-  runCommand("git", ["-C", repositoryDirectory, "diff", "--name-only", "HEAD"]),
-  runCommand("git", ["-C", repositoryDirectory, "ls-files", "--others", "--exclude-standard"]),
-]);
-const changedPaths = [...tracked.stdout.split("\n"), ...untracked.stdout.split("\n")]
-  .filter(Boolean)
-  .sort();
-assertRepairPaths(originalPatch, changedPaths);
-const repaired = await runCommand("git", ["-C", repositoryDirectory, "diff", "--binary", "HEAD"]);
-if (!repaired.stdout.startsWith("diff --git ")) {
-  throw new Error(
-    `repair produced no held-out test patch; status=${JSON.stringify(changedPaths)}; Codex tail=${codex.stdout.slice(-4_000)}`,
-  );
-}
-if (repaired.stdout === originalPatch) {
-  throw new Error("repair left the held-out test patch unchanged");
-}
-
-await writeFile(join(taskDirectory, "tests/test.patch"), repaired.stdout);
-await writeFile(
-  manifestPath,
-  `${JSON.stringify(
-    {
-      ...manifest,
-      testPatchSha256: sha256(repaired.stdout),
-      repair: {
-        model: process.env.SELFBENCH_REPAIR_MODEL ?? "gpt-5.6-sol",
-        originalTestPatchSha256: sha256(originalPatch),
-      },
-    },
-    null,
-    2,
-  )}\n`,
-);
-await runCommand("tar", ["-czf", outputArchive, "-C", extractedDirectory, "."]);
-await writeFile(
-  outputReport,
-  `${JSON.stringify(
-    {
-      schemaVersion: 1,
-      taskId,
-      model: process.env.SELFBENCH_REPAIR_MODEL ?? "gpt-5.6-sol",
-      changedPaths,
-      originalTestPatchSha256: sha256(originalPatch),
-      repairedTestPatchSha256: sha256(repaired.stdout),
-      codexOutputTail: codex.stdout.slice(-4_000),
-    },
-    null,
-    2,
-  )}\n`,
-);
-
-function fail(message: string): never {
-  throw new Error(message);
-}

package/src/sandbox-review.ts

@@ -1,19 +0,0 @@
-import { readFile, writeFile } from "node:fs/promises";
-import { reviewCouplingWithCodex } from "./codex-review.js";
-
-const apiKey = process.env.OPENAI_API_KEY;
-const authJson = process.env.SELFBENCH_PI_AUTH_JSON;
-const outputPath = process.env.SELFBENCH_REVIEW_OUTPUT;
-if (!apiKey && !authJson) {
-  throw new Error("OPENAI_API_KEY or SELFBENCH_PI_AUTH_JSON is required");
-}
-if (!outputPath) {
-  throw new Error("SELFBENCH_REVIEW_OUTPUT is required");
-}
-
-const review = await reviewCouplingWithCodex({
-  ...(apiKey ? { apiKey } : {}),
-  ...(authJson ? { authJson } : {}),
-  prompt: await readFile("/work/review-input.md", "utf8"),
-});
-await writeFile(outputPath, `${JSON.stringify(review, null, 2)}\n`, { flag: "wx" });

package/src/sandbox-timeout.ts

@@ -1,56 +0,0 @@
-import type {
-  SandboxExecutor,
-  SandboxRequest,
-  SandboxResult,
-  SandboxRunOptions,
-} from "./sandbox.js";
-
-export const STANDARD_VERCEL_TIMEOUT_CAP_MS = 2 * 60 * 60 * 1_000;
-export const HOBBY_VERCEL_TIMEOUT_CAP_MS = 45 * 60 * 1_000;
-
-export function parseSandboxTimeoutCapText(value: string): number | undefined {
-  const normalized = value.trim().toLowerCase();
-  if (normalized === "") {
-    return undefined;
-  }
-  const match = /^(\d+)(ms|s|m|h)?$/.exec(normalized);
-  if (!match?.[1]) {
-    return Number.NaN;
-  }
-  const amount = Number(match[1]);
-  const multiplier =
-    match[2] === "h"
-      ? 60 * 60 * 1_000
-      : match[2] === "m"
-        ? 60 * 1_000
-        : match[2] === "s"
-          ? 1_000
-          : 1;
-  return amount * multiplier;
-}
-
-export class TimeoutCappedSandboxExecutor implements SandboxExecutor {
-  readonly #delegate: SandboxExecutor;
-  readonly #timeoutCapMs: number;
-
-  constructor(delegate: SandboxExecutor, timeoutCapMs: number) {
-    if (!Number.isInteger(timeoutCapMs) || timeoutCapMs < 100) {
-      throw new Error("sandbox timeout cap must be an integer of at least 100ms");
-    }
-    this.#delegate = delegate;
-    this.#timeoutCapMs = timeoutCapMs;
-  }
-
-  async run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult> {
-    return await this.#delegate.run(
-      request.timeoutMs > this.#timeoutCapMs
-        ? { ...request, timeoutMs: this.#timeoutCapMs }
-        : request,
-      options,
-    );
-  }
-
-  close(): void {
-    this.#delegate.close();
-  }
-}