self-bench 0.3.0 → 0.3.2

package/src/evaluate.ts

@@ -0,0 +1,293 @@
+import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { basename, join, resolve } from "node:path";
+import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
+import { parallelMap } from "./parallel.js";
+import { runCommand } from "./process.js";
+import { assertCodexSubscriptionAuth } from "./subscription-auth.js";
+
+export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"] as const;
+export type MatrixModel = (typeof MATRIX_MODELS)[number];
+const CODEX_VERSION = "0.146.1";
+
+export interface MatrixOptions {
+  readonly exportPath?: string;
+  readonly tasksPath?: string;
+  readonly jobsDirectory: string;
+  readonly harborPath?: string;
+  readonly environment?: "docker" | "modal";
+  readonly concurrency?: number;
+  readonly authPath?: string;
+  readonly models?: readonly MatrixModel[];
+  readonly onTrialComplete?: (
+    summary: MatrixTrialSummary,
+    completed: number,
+    total: number,
+  ) => void;
+}
+
+export interface MatrixTrialSummary {
+  readonly taskId: string;
+  readonly model: MatrixModel;
+  readonly jobName: string;
+  readonly passed: boolean;
+  readonly rewards: Readonly<Record<string, number>>;
+  readonly exception?: string;
+}
+
+export async function runMatrix(options: MatrixOptions): Promise<readonly MatrixTrialSummary[]> {
+  const root = resolve(options.jobsDirectory);
+  const tasksDirectory = join(root, "tasks");
+  await mkdir(root, { recursive: true });
+  const taskDirectories = await resolveMatrixTasks(options, tasksDirectory);
+  const authPath = resolve(options.authPath ?? join(homedir(), ".codex/auth.json"));
+  await assertSubscriptionAuth(authPath);
+  const models = options.models ?? MATRIX_MODELS;
+  if (models.length < 1) {
+    throw new Error("provide at least one evaluation model");
+  }
+  const work = models.flatMap((model) =>
+    taskDirectories.map((taskDirectory) => ({ model, taskDirectory })),
+  );
+  let completed = 0;
+  const summaries = await parallelMap(work, options.concurrency ?? 3, async (item) => {
+    const summary = await runTrial({
+      jobsDirectory: root,
+      harborPath: options.harborPath ?? "harbor",
+      environment: options.environment ?? "modal",
+      authPath,
+      ...item,
+    });
+    completed += 1;
+    options.onTrialComplete?.(summary, completed, work.length);
+    return summary;
+  });
+  await writeFile(
+    join(root, "summary.json"),
+    `${JSON.stringify(
+      {
+        schemaVersion: 1,
+        agent: "codex",
+        agentVersion: CODEX_VERSION,
+        auth: "codex-subscription",
+        reasoningEffort: "high",
+        models,
+        taskCount: taskDirectories.length,
+        trials: summaries,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+  return summaries;
+}
+
+async function resolveMatrixTasks(
+  options: MatrixOptions,
+  materializedTasksDirectory: string,
+): Promise<string[]> {
+  if (options.exportPath && options.tasksPath) {
+    throw new Error("provide exactly one of exportPath or tasksPath");
+  }
+  if (options.tasksPath) {
+    const tasks = await taskDirectories(resolve(options.tasksPath));
+    if (tasks.length < 1) {
+      throw new Error(`found no Harbor tasks in ${resolve(options.tasksPath)}`);
+    }
+    return tasks;
+  }
+  if (!options.exportPath) {
+    throw new Error("provide exactly one of exportPath or tasksPath");
+  }
+  const tasks = await materializeExport(resolve(options.exportPath), materializedTasksDirectory);
+  if (tasks.length < 1) {
+    throw new Error(`found no Harbor tasks in ${resolve(options.exportPath)}`);
+  }
+  return tasks;
+}
+
+async function runTrial(input: {
+  readonly taskDirectory: string;
+  readonly model: MatrixModel;
+  readonly jobsDirectory: string;
+  readonly harborPath: string;
+  readonly environment: "docker" | "modal";
+  readonly authPath: string;
+}): Promise<MatrixTrialSummary> {
+  const taskId = basename(input.taskDirectory);
+  const jobName = `${taskId}-${input.model}`.toLowerCase().replace(/[^a-z0-9_.-]/g, "-");
+  const existing = await tryReadHarborJobResult(input.jobsDirectory, jobName);
+  if (existing) {
+    return summarizeResult(taskId, input.model, jobName, existing.trial);
+  }
+  await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
+
+  const environment = { ...process.env };
+  delete environment.OPENAI_API_KEY;
+  environment.CODEX_FORCE_AUTH_JSON = "1";
+  environment.CODEX_AUTH_JSON_PATH = input.authPath;
+  let result: Awaited<ReturnType<typeof runCommand>>;
+  try {
+    result = await runCommand(
+      input.harborPath,
+      [
+        "run",
+        "--path",
+        input.taskDirectory,
+        "--agent",
+        "codex",
+        "--model",
+        input.model,
+        "--ak",
+        `version=${CODEX_VERSION}`,
+        "--ak",
+        "reasoning_effort=high",
+        "--env",
+        input.environment,
+        "--job-name",
+        jobName,
+        "--jobs-dir",
+        input.jobsDirectory,
+        "--n-concurrent",
+        "1",
+        "--max-retries",
+        "1",
+        "--delete",
+        "--yes",
+        "--quiet",
+      ],
+      { allowFailure: true, env: environment, timeoutMs: 4 * 60 * 60 * 1000 },
+    );
+  } catch (error) {
+    return failedTrial(taskId, input.model, jobName, errorMessage(error));
+  }
+  const completed = await tryReadHarborJobResult(input.jobsDirectory, jobName);
+  if (!completed) {
+    const detail = result.stderr.trim() || result.stdout.trim();
+    return failedTrial(
+      taskId,
+      input.model,
+      jobName,
+      `Harbor produced no result (exit ${result.exitCode})${detail ? `: ${detail.slice(-1000)}` : ""}`,
+    );
+  }
+  return summarizeResult(taskId, input.model, jobName, completed.trial);
+}
+
+export function summarizeResult(
+  taskId: string,
+  model: MatrixModel,
+  jobName: string,
+  result: unknown,
+): MatrixTrialSummary {
+  const trial =
+    isRecord(result) && Array.isArray(result.trial_results) ? result.trial_results[0] : result;
+  if (!isRecord(trial)) {
+    throw new Error(`invalid Harbor result for ${taskId}/${model}`);
+  }
+  const rawRewards =
+    isRecord(trial.verifier_result) && isRecord(trial.verifier_result.rewards)
+      ? trial.verifier_result.rewards
+      : {};
+  const rewards = Object.fromEntries(
+    Object.entries(rawRewards).flatMap(([key, value]) =>
+      typeof value === "number" ? [[key, value] as const] : [],
+    ),
+  );
+  const exception = exceptionText(trial.exception_info ?? trial.exception);
+  return {
+    taskId,
+    model,
+    jobName,
+    passed:
+      !exception &&
+      Object.keys(rewards).length > 0 &&
+      (rewards.reward === undefined
+        ? Object.values(rewards).every((reward) => reward >= 1)
+        : rewards.reward >= 1),
+    rewards,
+    ...(exception ? { exception } : {}),
+  };
+}
+
+function failedTrial(
+  taskId: string,
+  model: MatrixModel,
+  jobName: string,
+  exception: string,
+): MatrixTrialSummary {
+  return { taskId, model, jobName, passed: false, rewards: {}, exception };
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+async function materializeExport(exportPath: string, tasksDirectory: string): Promise<string[]> {
+  const existing = await taskDirectories(tasksDirectory);
+  if (existing.length > 0) {
+    return existing;
+  }
+  return await withTemporaryDirectory("selfbench-matrix-", async (temporary) => {
+    await runCommand("tar", ["-xzf", exportPath, "-C", temporary]);
+    const archivesRoot = join(temporary, "tasks");
+    const archives = (await readdir(archivesRoot))
+      .filter((name) => name.endsWith(".tar.gz"))
+      .sort();
+    await mkdir(tasksDirectory, { recursive: true });
+    for (const archive of archives) {
+      const taskId = archive.slice(0, -".tar.gz".length);
+      const expanded = join(temporary, `expanded-${taskId}`);
+      await mkdir(expanded);
+      await runCommand("tar", ["-xzf", join(archivesRoot, archive), "-C", expanded]);
+      await cp(join(expanded, "harbor-task"), join(tasksDirectory, taskId), { recursive: true });
+    }
+    return await taskDirectories(tasksDirectory);
+  });
+}
+
+async function taskDirectories(root: string): Promise<string[]> {
+  const entries = await readdir(root, { withFileTypes: true }).catch(() => []);
+  const paths: string[] = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) {
+      continue;
+    }
+    const path = join(root, entry.name);
+    if (await readFile(join(path, "task.toml")).catch(() => undefined)) {
+      paths.push(path);
+    }
+  }
+  return paths.sort();
+}
+
+async function assertSubscriptionAuth(path: string): Promise<void> {
+  const parsed = JSON.parse(await readFile(path, "utf8")) as unknown;
+  assertCodexSubscriptionAuth(parsed, path);
+}
+
+async function withTemporaryDirectory<T>(
+  prefix: string,
+  action: (root: string) => Promise<T>,
+): Promise<T> {
+  const root = await mkdtemp(join(tmpdir(), prefix));
+  try {
+    return await action(root);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+function exceptionText(value: unknown): string | undefined {
+  if (typeof value === "string" && value) {
+    return value;
+  }
+  if (isRecord(value)) {
+    return typeof value.message === "string" ? value.message : JSON.stringify(value);
+  }
+  return undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/github.ts

@@ -0,0 +1,26 @@
+export function assertPullRequestBelongsToRepository(
+  repositoryUrl: string,
+  pullRequestUrl: string,
+  pullRequestNumber: number,
+): void {
+  const repository = githubRepository(repositoryUrl);
+  const match = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)\/?$/i.exec(pullRequestUrl);
+  if (!match?.[1] || !match[2] || !match[3]) {
+    throw new Error(`invalid GitHub pull request URL: ${pullRequestUrl}`);
+  }
+  const candidateRepository = `${match[1]}/${match[2]}`.toLowerCase();
+  if (candidateRepository !== repository || Number(match[3]) !== pullRequestNumber) {
+    throw new Error(
+      `pull request ${pullRequestUrl} does not match ${repository}#${pullRequestNumber}`,
+    );
+  }
+}
+
+export function githubRepository(url: string): string {
+  const match =
+    /^(?:https:\/\/github\.com\/|git@github\.com:)([^/]+)\/([^/]+?)(?:\.git)?\/?$/i.exec(url);
+  if (!match?.[1] || !match[2]) {
+    throw new Error(`unsupported GitHub repository URL: ${url}`);
+  }
+  return `${match[1]}/${match[2]}`.toLowerCase();
+}

package/src/harbor-results.ts

@@ -0,0 +1,142 @@
+import { readdir, readFile, rename } from "node:fs/promises";
+import { join } from "node:path";
+
+export interface HarborVerifierOutput {
+  readonly combined?: string;
+  readonly stderr?: string;
+}
+
+export interface HarborJobResult {
+  readonly job: unknown;
+  readonly trial: unknown;
+  readonly verifier?: HarborVerifierOutput;
+}
+
+const infrastructurePatterns = [
+  /unknown flag: --project-name/i,
+  /cannot connect to the docker daemon/i,
+  /error during connect/i,
+  /connection refused/i,
+  /modal.*(?:unavailable|timed out|timeout)/i,
+  /image build for im-[a-z0-9]+ failed/i,
+  /all predefined address pools have been fully subnetted/i,
+];
+
+class IncompleteHarborJobError extends Error {}
+
+export function harborInfrastructureError(trial: unknown): string | undefined {
+  if (!isRecord(trial) || !isRecord(trial.exception_info)) {
+    return undefined;
+  }
+  const message = trial.exception_info.exception_message;
+  const type = trial.exception_info.exception_type;
+  if (
+    typeof message !== "string" ||
+    (type !== "AuthError" && !infrastructurePatterns.some((pattern) => pattern.test(message)))
+  ) {
+    return undefined;
+  }
+  return `${typeof type === "string" ? type : "HarborError"}: ${message}`;
+}
+
+export async function readHarborJobResult(
+  jobsDirectory: string,
+  jobName: string,
+): Promise<HarborJobResult> {
+  const jobDirectory = join(jobsDirectory, jobName);
+  const job = JSON.parse(await readFile(join(jobDirectory, "result.json"), "utf8")) as unknown;
+  const entries = await readdir(jobDirectory, { withFileTypes: true });
+  const trials: Array<{ directory: string; result: unknown }> = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) {
+      continue;
+    }
+    const directory = join(jobDirectory, entry.name);
+    const raw = await readFile(join(directory, "result.json"), "utf8").catch((error: unknown) =>
+      isNotFound(error) ? undefined : Promise.reject(error),
+    );
+    if (raw) {
+      trials.push({ directory, result: JSON.parse(raw) as unknown });
+    }
+  }
+  const aggregateTrials =
+    isRecord(job) && Array.isArray(job.trial_results) ? job.trial_results : [];
+  const [onlyTrial] = trials;
+  if (onlyTrial && trials.length === 1) {
+    return {
+      job,
+      trial: onlyTrial.result,
+      ...(await readVerifierOutput(onlyTrial.directory)),
+    };
+  }
+  if (aggregateTrials.length === 1) {
+    return { job, trial: aggregateTrials[0] };
+  }
+  if (trials.length === 0 && isRecord(job) && job.finished_at === null) {
+    throw new IncompleteHarborJobError(`Harbor job ${jobName} has not finished`);
+  }
+  throw new Error(`expected one Harbor trial result in ${jobDirectory}, found ${trials.length}`);
+}
+
+export async function tryReadHarborJobResult(
+  jobsDirectory: string,
+  jobName: string,
+): Promise<HarborJobResult | undefined> {
+  try {
+    return await readHarborJobResult(jobsDirectory, jobName);
+  } catch (error) {
+    if (isNotFound(error) || error instanceof IncompleteHarborJobError) {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+export async function archiveIncompleteHarborJob(
+  jobsDirectory: string,
+  jobName: string,
+): Promise<string | undefined> {
+  const source = join(jobsDirectory, jobName);
+  const destination = `${source}.incomplete-${Date.now()}`;
+  try {
+    await rename(source, destination);
+    return destination;
+  } catch (error) {
+    if (isNotFound(error)) {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+async function readVerifierOutput(
+  trialDirectory: string,
+): Promise<{ readonly verifier?: HarborVerifierOutput }> {
+  const verifierDirectory = join(trialDirectory, "verifier");
+  const [combined, stderr] = await Promise.all([
+    readOptionalText(join(verifierDirectory, "test-stdout.txt")),
+    readOptionalText(join(verifierDirectory, "test-stderr.txt")),
+  ]);
+  return combined || stderr
+    ? { verifier: { ...(combined ? { combined } : {}), ...(stderr ? { stderr } : {}) } }
+    : {};
+}
+
+async function readOptionalText(path: string): Promise<string | undefined> {
+  return await readFile(path, "utf8").catch((error: unknown) =>
+    isNotFound(error) ? undefined : Promise.reject(error),
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isNotFound(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error as { code?: unknown }).code === "ENOENT"
+  );
+}