self-bench 0.3.0 → 0.3.2

package/src/sandbox.ts

@@ -0,0 +1,51 @@
+import type { SelfBenchConfig } from "./config.js";
+import { DockerSandboxExecutor } from "./docker-executor.js";
+import { ModalSandboxExecutor } from "./modal-executor.js";
+
+export interface SandboxFile {
+  readonly path: string;
+  readonly contents: Uint8Array | string;
+}
+
+export interface SandboxRequest {
+  readonly runId: string;
+  readonly stage: string;
+  readonly command: readonly string[];
+  readonly files?: readonly SandboxFile[];
+  readonly outputPaths?: readonly string[];
+  readonly environment?: Readonly<Record<string, string>>;
+  readonly secrets?: Readonly<Record<string, string>>;
+  readonly timeoutMs: number;
+  readonly inactivityTimeoutMs?: number;
+  readonly cpu?: number;
+  readonly memoryMiB?: number;
+}
+
+export interface SandboxProgress {
+  readonly stream: "stdout" | "stderr";
+  readonly bytes: number;
+}
+
+export interface SandboxRunOptions {
+  readonly signal?: AbortSignal;
+  readonly onProgress?: (progress: SandboxProgress) => void;
+}
+
+export interface SandboxResult {
+  readonly sandboxId: string;
+  readonly exitCode: number;
+  readonly stdout: string;
+  readonly stderr: string;
+  readonly outputs: Readonly<Record<string, Uint8Array>>;
+}
+
+export interface SandboxExecutor {
+  run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult>;
+  close(): void;
+}
+
+export function createSandboxExecutor(config: SelfBenchConfig["execution"]): SandboxExecutor {
+  return config.kind === "modal"
+    ? new ModalSandboxExecutor(config)
+    : new DockerSandboxExecutor(config);
+}

package/src/subscription-auth.ts

@@ -0,0 +1,67 @@
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { runCommand } from "./process.js";
+
+export async function loadPiSubscriptionAuth(): Promise<string> {
+  const raw =
+    process.env.SELFBENCH_PI_AUTH_JSON ??
+    (await readFile(join(homedir(), ".pi/agent/auth.json"), "utf8"));
+  const parsed = JSON.parse(raw) as unknown;
+  const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
+  if (
+    !isRecord(credential) ||
+    credential.type !== "oauth" ||
+    typeof credential.access !== "string" ||
+    typeof credential.refresh !== "string"
+  ) {
+    throw new Error("Pi auth does not contain an openai-codex subscription credential");
+  }
+  return JSON.stringify({ "openai-codex": credential });
+}
+
+export function assertCodexSubscriptionAuth(value: unknown, source = "Codex auth"): void {
+  if (!isRecord(value) || value.auth_mode !== "chatgpt" || !isRecord(value.tokens)) {
+    throw new Error(`${source} does not contain a ChatGPT subscription token set`);
+  }
+}
+
+export async function loadCodexSubscriptionAuth(): Promise<string> {
+  const path = process.env.CODEX_AUTH_JSON_PATH ?? join(homedir(), ".codex/auth.json");
+  const parsed = JSON.parse(await readFile(path, "utf8")) as unknown;
+  assertCodexSubscriptionAuth(parsed, path);
+  const auth = parsed as Record<string, unknown>;
+  return JSON.stringify({
+    auth_mode: "chatgpt",
+    tokens: auth.tokens,
+    ...(typeof auth.last_refresh === "string" ? { last_refresh: auth.last_refresh } : {}),
+  });
+}
+
+export async function subscriptionBearerToken(model: string): Promise<string> {
+  const result = await runCommand("pi", [
+    "auth",
+    "print-bearer-token",
+    "--provider",
+    "openai-codex",
+    "--model",
+    model,
+  ]);
+  const token = result.stdout.trim();
+  if (!token) {
+    throw new Error(`Pi returned no openai-codex subscription bearer token for ${model}`);
+  }
+  return token;
+}
+
+export async function githubToken(): Promise<string | undefined> {
+  if (process.env.GH_TOKEN) {
+    return process.env.GH_TOKEN;
+  }
+  const result = await runCommand("gh", ["auth", "token"], { allowFailure: true });
+  return result.exitCode === 0 && result.stdout.trim() ? result.stdout.trim() : undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/temporal.ts

@@ -0,0 +1,23 @@
+import { Connection } from "@temporalio/client";
+import { NativeConnection } from "@temporalio/worker";
+import type { SelfBenchConfig } from "./config.js";
+
+export async function connectTemporalClient(
+  config: SelfBenchConfig["temporal"],
+): Promise<Connection> {
+  return await Connection.connect({
+    address: config.address,
+    tls: config.tls,
+    ...(config.apiKey ? { apiKey: config.apiKey } : {}),
+  });
+}
+
+export async function connectTemporalWorker(
+  config: SelfBenchConfig["temporal"],
+): Promise<NativeConnection> {
+  return await NativeConnection.connect({
+    address: config.address,
+    tls: config.tls,
+    ...(config.apiKey ? { apiKey: config.apiKey } : {}),
+  });
+}

package/src/validate-main.ts

@@ -0,0 +1,171 @@
+#!/usr/bin/env node
+
+import { access, mkdir, readdir, writeFile } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { harborInfrastructureError, readHarborJobResult } from "./harbor-results.js";
+import { parallelMap } from "./parallel.js";
+import { runCommand } from "./process.js";
+
+const parsed = parseArgs({
+  options: {
+    tasks: { type: "string" },
+    jobs: { type: "string" },
+    environment: { type: "string", default: "modal" },
+    concurrency: { type: "string", default: "10" },
+    help: { type: "boolean", short: "h" },
+  },
+  strict: true,
+});
+if (parsed.values.help) {
+  console.log(`Run Harbor nop and oracle gates for expanded SelfBench tasks.
+
+Usage:
+  self-bench-validate --tasks DIRECTORY --jobs DIRECTORY [options]
+
+Options:
+  --environment NAME  Harbor environment (default: modal)
+  --concurrency N     Concurrent tasks (default: 10)
+  -h, --help          Show this help`);
+  process.exit(0);
+}
+
+const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
+const jobsDirectory = resolve(parsed.values.jobs ?? fail("--jobs is required"));
+const environment = parsed.values.environment ?? "modal";
+if (environment !== "modal" && environment !== "docker") {
+  throw new Error("--environment must be modal or docker");
+}
+const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
+const directories = (
+  await Promise.all(
+    (
+      await readdir(tasksDirectory)
+    ).map(async (name) => {
+      const directory = join(tasksDirectory, name);
+      return await access(join(directory, "task.toml")).then(
+        () => directory,
+        () => undefined,
+      );
+    }),
+  )
+).filter((directory): directory is string => directory !== undefined);
+await mkdir(jobsDirectory, { recursive: true });
+
+const tasks = await parallelMap(directories.sort(), concurrency, async (taskDirectory) => {
+  const taskId = basename(taskDirectory);
+  console.error(`validating ${taskId}`);
+  try {
+    const nop = await runGate(taskDirectory, taskId, "nop", environment, jobsDirectory);
+    const oracle = await runGate(taskDirectory, taskId, "oracle", environment, jobsDirectory);
+    const accepted = nop.passed && oracle.passed;
+    console.error(`${taskId}: ${accepted ? "accepted" : "rejected"}`);
+    return { taskId, status: "validated" as const, accepted, nop, oracle };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(`${taskId}: error: ${message}`);
+    return { taskId, status: "error" as const, accepted: false, error: message };
+  }
+});
+const report = {
+  schemaVersion: 1,
+  generatedAt: new Date().toISOString(),
+  sourceTasks: tasksDirectory,
+  environment,
+  summary: {
+    total: tasks.length,
+    accepted: tasks.filter((task) => task.accepted).length,
+    rejected: tasks.filter((task) => task.status === "validated" && !task.accepted).length,
+    errors: tasks.filter((task) => task.status === "error").length,
+  },
+  tasks,
+};
+await writeFile(
+  join(jobsDirectory, "validation-summary.json"),
+  `${JSON.stringify(report, null, 2)}\n`,
+);
+console.log(JSON.stringify(report.summary, null, 2));
+
+async function runGate(
+  taskDirectory: string,
+  taskId: string,
+  agent: "nop" | "oracle",
+  environment: "docker" | "modal",
+  jobsDirectory: string,
+): Promise<{ readonly passed: boolean; readonly rewards: Readonly<Record<string, number>> }> {
+  const jobName = `${taskId}-${agent}-${crypto.randomUUID().slice(0, 8)}`;
+  const process = await runCommand(
+    "harbor",
+    [
+      "run",
+      "--path",
+      taskDirectory,
+      "--agent",
+      agent,
+      "--env",
+      environment,
+      "--job-name",
+      jobName,
+      "--jobs-dir",
+      jobsDirectory,
+      "--n-concurrent",
+      "1",
+      "--max-retries",
+      "1",
+      "--delete",
+      "--yes",
+      "--quiet",
+    ],
+    { allowFailure: true, timeoutMs: 4 * 60 * 60 * 1000 },
+  );
+  if (process.exitCode !== 0) {
+    throw new Error(`Harbor ${agent} exited ${process.exitCode}: ${process.stderr.slice(-1_000)}`);
+  }
+  const result = await readHarborJobResult(jobsDirectory, jobName);
+  const infrastructure = harborInfrastructureError(result.trial);
+  if (infrastructure) {
+    throw new Error(`Harbor ${agent} infrastructure failure: ${infrastructure}`);
+  }
+  const rewards = rewardValues(result.trial);
+  const passed =
+    agent === "nop"
+      ? rewards.fail_to_pass === 0 &&
+        (rewards.pass_to_pass ?? 0) >= 1 &&
+        (rewards.setup_completed ?? 0) >= 1
+      : (rewards.patch_applied ?? 0) >= 1 &&
+        (rewards.fail_to_pass ?? 0) >= 1 &&
+        (rewards.pass_to_pass ?? 0) >= 1 &&
+        (rewards.deterministic ?? 0) >= 1 &&
+        (rewards.setup_completed ?? 0) >= 1;
+  return { passed, rewards };
+}
+
+function rewardValues(value: unknown): Readonly<Record<string, number>> {
+  if (!isRecord(value) || !isRecord(value.verifier_result)) {
+    return {};
+  }
+  const raw = value.verifier_result.rewards;
+  return isRecord(raw)
+    ? Object.fromEntries(
+        Object.entries(raw).filter(
+          (entry): entry is [string, number] => typeof entry[1] === "number",
+        ),
+      )
+    : {};
+}
+
+function positiveInteger(value: string | undefined, label: string): number {
+  const number = Number(value);
+  if (!Number.isInteger(number) || number < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return number;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function fail(message: string): never {
+  throw new Error(message);
+}

package/src/validation-repair.ts

@@ -0,0 +1,94 @@
+import type { TaskDefinition } from "./contracts.js";
+import { patchPaths } from "./repair.js";
+
+export function validationRepairPaths(originalTestPatch: string): readonly string[] {
+  const paths = patchPaths(originalTestPatch);
+  if (paths.length === 0) {
+    throw new Error("original held-out test patch changes no files");
+  }
+  const nonTestPaths = paths.filter((path) => !isTestOnlyPath(path));
+  if (nonTestPaths.length > 0) {
+    throw new Error(`validation repair requires test-only patch paths: ${nonTestPaths.join(", ")}`);
+  }
+  return paths;
+}
+
+function isTestOnlyPath(path: string): boolean {
+  const segments = path.toLowerCase().split("/");
+  const filename = segments.at(-1) ?? "";
+  return (
+    segments.some((segment) =>
+      /^(?:tests?|__tests__|e2e|integration|fixtures?|selfbench(?:-tests|_tests)?|\.selfbench-tests)$/.test(
+        segment,
+      ),
+    ) || /(?:^|\.)(?:test|spec)\.[^.]+$/.test(filename)
+  );
+}
+
+export function assertValidationRepair(
+  original: TaskDefinition,
+  repaired: TaskDefinition,
+  originalTestPatch: string,
+  changedPaths: readonly string[],
+): void {
+  const immutableKeys = [
+    "schemaVersion",
+    "difficulty",
+    "taskId",
+    "repo",
+    "baseCommit",
+    "workdir",
+    "sourcePr",
+    "sourceUrl",
+    "prompt",
+  ] as const;
+  for (const key of immutableKeys) {
+    if (JSON.stringify(repaired[key]) !== JSON.stringify(original[key])) {
+      throw new Error(`validation repair changed immutable definition field ${key}`);
+    }
+  }
+  const allowed = new Set(validationRepairPaths(originalTestPatch));
+  const outside = changedPaths.filter((path) => !allowed.has(path));
+  if (outside.length > 0) {
+    throw new Error(
+      `validation repair changed files outside held-out tests: ${outside.join(", ")}`,
+    );
+  }
+}
+
+export function validationRepairPrompt(input: {
+  readonly definition: TaskDefinition;
+  readonly authenticRequest: string;
+  readonly diagnostics: string;
+  readonly allowedPaths: readonly string[];
+}): string {
+  return `Repair the verifier harness and held-out tests for SelfBench task ${input.definition.taskId}.
+
+The repository is the exact base snapshot with the current held-out test patch already applied. You may edit only these existing held-out test files:
+
+${input.allowedPaths.map((path) => `- ${path}`).join("\n")}
+
+The authentic engineer request is:
+
+<authentic_request>
+${input.authenticRequest.trim()}
+</authentic_request>
+
+The failed nop/oracle validation diagnostics are:
+
+<validation_diagnostics>
+${input.diagnostics.trim()}
+</validation_diagnostics>
+
+Fix the task so its repository-native verifier is rigorous and runnable. You may edit the allowed held-out tests and /work/definition.json. In definition.json you may change only setupCommand, testCommand, failToPass, passToPass, testPaths, toolchains, timeouts, and resources. Never change task identity, difficulty, repository, base commit, workdir, source pull request, prompt, or reference solution.
+
+The required validation split is:
+- nop: the base repository plus held-out tests must make failToPass fail while passToPass succeeds;
+- oracle: after /work/task/solution/gold.patch is applied, failToPass and passToPass must both succeed deterministically.
+
+Use the verifier diagnostics rather than guessing. Inspect repository package scripts and CI for the native test command. Keep {tests} as a list-safe placeholder; do not quote the whole placeholder, assign it to one scalar, or hard-code selected paths elsewhere. Use one test mode/bundler per command instead of chaining equivalent suites. Put dependency installation, native builds, fixture generation, and browser/runtime installation in setupCommand, not testCommand. Pin the repository's declared package manager and use frozen installs. Browser-backed tests must install required browser binaries and OS dependencies during setup. Run focused nop/oracle-style commands in the sandbox when feasible.
+
+Do not weaken, delete, or skip requested behavioral coverage merely to make commands exit zero. A passing base with no solution is not acceptable. If no rigorous runnable split exists, leave the files and definition unchanged and explain why in the final response.
+
+Inspect /work/gold.patch and /work/test.sh only to verify the intended behavior and oracle seam, never to couple assertions to private implementation details. Do not modify application code, the instruction, gold patch, or commit files. Finish with the repaired definition at /work/definition.json and any held-out test edits present in the working tree; do not commit.`;
+}

package/src/worker-main.ts

@@ -0,0 +1,32 @@
+import { existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { Worker } from "@temporalio/worker";
+import { createActivities } from "./activities.js";
+import { loadConfig } from "./config.js";
+import { removeEmptyModalCredentialOverrides } from "./modal-auth.js";
+import { runCommand } from "./process.js";
+import { connectTemporalWorker } from "./temporal.js";
+
+removeEmptyModalCredentialOverrides();
+const config = loadConfig();
+if (config.execution.kind === "docker" || config.harborEnvironment === "docker") {
+  await runCommand("docker", ["info"], { timeoutMs: 30_000 });
+  await runCommand("docker", ["compose", "version"], { timeoutMs: 30_000 });
+}
+const connection = await connectTemporalWorker(config.temporal);
+const javascriptWorkflow = fileURLToPath(new URL("./workflow.js", import.meta.url));
+const workflowsPath = existsSync(javascriptWorkflow)
+  ? javascriptWorkflow
+  : fileURLToPath(new URL("./workflow.ts", import.meta.url));
+const worker = await Worker.create({
+  connection,
+  namespace: config.temporal.namespace,
+  taskQueue: config.temporal.taskQueue,
+  workflowsPath,
+  activities: createActivities(config),
+  maxConcurrentActivityTaskExecutions: config.activityConcurrency,
+});
+console.log(
+  `SelfBench worker polling ${config.temporal.namespace}/${config.temporal.taskQueue} with activity concurrency ${config.activityConcurrency}`,
+);
+await worker.run();