self-bench 0.3.0 → 0.3.2

package/src/contracts.ts

@@ -0,0 +1,209 @@
+import { z } from "zod";
+
+export const commitSchema = z.string().regex(/^[0-9a-f]{40}$/i, "expected a full commit SHA");
+
+export const repositoryRefSchema = z.object({
+  url: z.string().url(),
+  commit: commitSchema,
+});
+
+export type RepositoryRef = z.infer<typeof repositoryRefSchema>;
+
+export const artifactRefSchema = z.object({
+  uri: z.string().min(1),
+  sha256: z.string().regex(/^[0-9a-f]{64}$/),
+  sizeBytes: z.number().int().nonnegative(),
+  contentType: z.string().min(1),
+});
+
+export type ArtifactRef = z.infer<typeof artifactRefSchema>;
+
+export const difficultySchema = z.enum(["easy", "medium", "hard"]);
+export type Difficulty = z.infer<typeof difficultySchema>;
+
+const candidateCountsSchema = z
+  .object({
+    easy: z.number().int().min(0).max(100),
+    medium: z.number().int().min(0).max(100),
+    hard: z.number().int().min(0).max(100),
+  })
+  .refine(({ easy, medium, hard }) => easy + medium + hard >= 1, {
+    message: "at least one candidate must be requested",
+  })
+  .refine(({ easy, medium, hard }) => easy + medium + hard <= 100, {
+    message: "at most 100 candidates may be requested",
+  });
+
+export const runRequestSchema = z.object({
+  runId: z.string().regex(/^[a-z0-9][a-z0-9-]{2,62}$/),
+  repository: repositoryRefSchema,
+  provenance: artifactRefSchema,
+  candidateCounts: candidateCountsSchema,
+  authoring: z.object({
+    provider: z.literal("openai-codex"),
+    model: z.string().min(1),
+    reasoningEffort: z.literal("high"),
+  }),
+  version: z.object({
+    selfbenchCommit: commitSchema,
+    executionBackend: z.enum(["docker", "modal"]),
+    sandboxImage: z.string().min(1),
+    schema: z.literal(1),
+  }),
+});
+
+export type RunRequest = z.infer<typeof runRequestSchema>;
+
+export const candidateSchema = z.object({
+  candidateId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/),
+  difficulty: difficultySchema,
+  sourcePr: z.number().int().positive(),
+  sourceUrl: z.string().url(),
+  baseCommit: commitSchema,
+  completedCommit: commitSchema,
+  request: z.string().min(1),
+  provenance: artifactRefSchema,
+});
+
+export type Candidate = z.infer<typeof candidateSchema>;
+
+export const taskDefinitionSchema = z.object({
+  schemaVersion: z.literal(1),
+  difficulty: difficultySchema,
+  taskId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/),
+  repo: z.string().min(1),
+  baseCommit: commitSchema,
+  workdir: z.string().min(1),
+  setupCommand: z.string().min(1),
+  testCommand: z.string().refine((value) => value.includes("{tests}"), {
+    message: 'testCommand must contain "{tests}"',
+  }),
+  failToPass: z.array(z.string().min(1)).min(1),
+  passToPass: z.array(z.string().min(1)),
+  testPaths: z.array(z.string().min(1)).min(1),
+  toolchains: z.array(z.enum(["uv", "bun", "go", "node", "python", "rust"])).min(1),
+  sourcePr: z.number().int().positive(),
+  sourceUrl: z.string().url(),
+  prompt: z.string().min(1),
+  timeouts: z.object({
+    setupSeconds: z.number().int().positive(),
+    agentSeconds: z.number().int().positive(),
+    testsSeconds: z.number().int().positive(),
+  }),
+  resources: z.object({
+    cpus: z.number().positive(),
+    memoryMb: z.number().int().positive(),
+    storageMb: z.number().int().positive(),
+  }),
+});
+
+export type TaskDefinition = z.infer<typeof taskDefinitionSchema>;
+
+export const authoredTaskSchema = z.object({
+  candidateId: z.string().min(1),
+  taskId: z.string().min(1),
+  definition: artifactRefSchema,
+  bundle: artifactRefSchema,
+});
+
+export type AuthoredTask = z.infer<typeof authoredTaskSchema>;
+
+export const validationResultSchema = z.object({
+  taskId: z.string().min(1),
+  accepted: z.boolean(),
+  nop: z.object({
+    passed: z.boolean(),
+    result: artifactRefSchema,
+    output: artifactRefSchema.optional(),
+  }),
+  oracle: z.object({
+    passed: z.boolean(),
+    result: artifactRefSchema,
+    output: artifactRefSchema.optional(),
+  }),
+  reason: z.string().optional(),
+});
+
+export type ValidationResult = z.infer<typeof validationResultSchema>;
+
+export type RunPhase =
+  | "queued"
+  | "discovering"
+  | "authoring"
+  | "validating"
+  | "reviewing"
+  | "repairing"
+  | "auditing"
+  | "exporting"
+  | "complete"
+  | "blocked"
+  | "failed"
+  | "cancelling"
+  | "cancelled";
+
+export interface TaskProgress {
+  taskId: string;
+  candidateId: string;
+  difficulty: Difficulty;
+  status:
+    | "authoring"
+    | "auditing"
+    | "validating"
+    | "reviewing"
+    | "repairing"
+    | "infrastructure_failed"
+    | "rejected"
+    | "accepted";
+  reason?: string;
+}
+
+export interface DiscoveryProgress {
+  readonly wave: number;
+  readonly totalShards: number;
+  readonly completedShards: number;
+  readonly failedShards: number;
+  readonly candidates: number;
+}
+
+export interface RunStatus {
+  readonly runId: string;
+  readonly phase: RunPhase;
+  readonly requested: number;
+  readonly requestedByDifficulty: Readonly<Record<Difficulty, number>>;
+  readonly discovered: number;
+  readonly accepted: number;
+  readonly rejected: number;
+  readonly tasks: readonly TaskProgress[];
+  readonly discovery?: DiscoveryProgress;
+  readonly export?: ArtifactRef;
+  readonly error?: string;
+}
+
+export interface RunResult {
+  readonly runId: string;
+  readonly export: ArtifactRef;
+  readonly acceptedTaskIds: readonly string[];
+}
+
+export interface DiscoveryResult {
+  readonly candidates: readonly Candidate[];
+  readonly report: ArtifactRef;
+}
+
+export type AuthorOutcome =
+  | { readonly kind: "authored"; readonly task: AuthoredTask }
+  | { readonly kind: "rejected"; readonly candidateId: string; readonly reason: string };
+
+export interface ReviewResult {
+  readonly taskId: string;
+  readonly accepted: boolean;
+  readonly report: ArtifactRef;
+  readonly reason?: string;
+}
+
+export interface AuditResult {
+  readonly taskId: string;
+  readonly accepted: boolean;
+  readonly report: ArtifactRef;
+  readonly reason?: string;
+}

package/src/coupling.ts

@@ -0,0 +1,259 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { runCommand } from "./process.js";
+
+export type ContractArtifactCategory =
+  | "endpoint_path"
+  | "field_name"
+  | "header_name"
+  | "media_type";
+
+export interface ContractArtifactEvidence {
+  readonly artifact: string;
+  readonly category: ContractArtifactCategory;
+  readonly testLocations: readonly string[];
+  readonly introducedByGold: boolean;
+  readonly presentInPrompt: boolean;
+  readonly presentInBase: boolean;
+}
+
+export interface CouplingEvidence {
+  readonly schemaVersion: 1;
+  readonly artifacts: readonly ContractArtifactEvidence[];
+  readonly blockers: readonly string[];
+}
+
+export interface CouplingReviewInput {
+  readonly verdict: "clean" | "coupled";
+  readonly reason: string;
+  readonly findings: readonly {
+    readonly artifact: string;
+    readonly disposition:
+      | "base_contract"
+      | "prompt_contract"
+      | "external_contract"
+      | "gold_only"
+      | "not_contract";
+  }[];
+}
+
+export interface CouplingReviewResolution {
+  readonly verdict: "clean" | "coupled";
+  readonly reason: string;
+  readonly missingArtifacts: readonly string[];
+  readonly goldOnlyArtifacts: readonly string[];
+}
+
+interface AddedLine {
+  readonly location: string;
+  readonly text: string;
+}
+
+export interface ContractArtifactCandidate {
+  readonly artifact: string;
+  readonly category: ContractArtifactCategory;
+  readonly testLocations: readonly string[];
+}
+
+export function discoverContractArtifacts(testPatch: string): readonly ContractArtifactCandidate[] {
+  const found = new Map<string, { category: ContractArtifactCategory; locations: Set<string> }>();
+  for (const line of addedLines(testPatch)) {
+    for (const candidate of contractArtifacts(line.text)) {
+      const key = `${candidate.category}\u0000${candidate.artifact}`;
+      const existing = found.get(key);
+      if (existing) {
+        existing.locations.add(line.location);
+      } else {
+        found.set(key, {
+          category: candidate.category,
+          locations: new Set([line.location]),
+        });
+      }
+    }
+  }
+  return [...found.entries()]
+    .map(([key, value]) => ({
+      artifact: key.slice(key.indexOf("\u0000") + 1),
+      category: value.category,
+      testLocations: [...value.locations].sort(),
+    }))
+    .sort((left, right) =>
+      left.category === right.category
+        ? left.artifact.localeCompare(right.artifact)
+        : left.category.localeCompare(right.category),
+    );
+}
+
+export function buildCouplingEvidence(input: {
+  readonly prompt: string;
+  readonly testPatch: string;
+  readonly goldPatch: string;
+  readonly baseArtifacts: ReadonlySet<string>;
+}): CouplingEvidence {
+  const gold = addedLines(input.goldPatch)
+    .map((line) => line.text)
+    .join("\n");
+  const artifacts = discoverContractArtifacts(input.testPatch)
+    .filter((candidate) => containsArtifact(gold, candidate.artifact, candidate.category))
+    .map(
+      (candidate): ContractArtifactEvidence => ({
+        ...candidate,
+        introducedByGold: true,
+        presentInPrompt: containsArtifact(input.prompt, candidate.artifact, candidate.category),
+        presentInBase: input.baseArtifacts.has(candidate.artifact),
+      }),
+    );
+  const blockers = artifacts
+    .filter((artifact) => !artifact.presentInPrompt && !artifact.presentInBase)
+    .map(
+      (artifact) =>
+        `held-out tests assert gold-only ${artifact.category} ${JSON.stringify(artifact.artifact)} at ${artifact.testLocations.join(", ")}`,
+    );
+  return { schemaVersion: 1, artifacts, blockers };
+}
+
+export function resolveCouplingReview(
+  evidence: CouplingEvidence,
+  review: CouplingReviewInput,
+): CouplingReviewResolution {
+  const unresolvedArtifacts = evidence.artifacts
+    .filter((artifact) => !artifact.presentInBase && !artifact.presentInPrompt)
+    .map((artifact) => artifact.artifact);
+  const reviewedArtifacts = new Set(review.findings.map((finding) => finding.artifact));
+  const missingArtifacts = unresolvedArtifacts.filter(
+    (artifact) => !reviewedArtifacts.has(artifact),
+  );
+  const goldOnlyArtifacts = review.findings
+    .filter((finding) => finding.disposition === "gold_only")
+    .map((finding) => finding.artifact);
+  const verdict =
+    review.verdict === "coupled" || missingArtifacts.length > 0 || goldOnlyArtifacts.length > 0
+      ? "coupled"
+      : "clean";
+  const reason =
+    missingArtifacts.length > 0
+      ? `review did not resolve deterministic coupling evidence for: ${missingArtifacts.join(", ")}`
+      : goldOnlyArtifacts.length > 0 && review.verdict === "clean"
+        ? `review identified gold-only artifacts: ${goldOnlyArtifacts.join(", ")}`
+        : review.reason;
+  return { verdict, reason, missingArtifacts, goldOnlyArtifacts };
+}
+
+export async function scanBaseContractArtifacts(
+  baseDirectory: string,
+  scratchDirectory: string,
+  artifacts: readonly { readonly artifact: string }[],
+): Promise<ReadonlySet<string>> {
+  const values = [...new Set(artifacts.map((artifact) => artifact.artifact))].filter(
+    (artifact) => !artifact.includes("\n") && artifact.length > 0,
+  );
+  if (values.length === 0) {
+    return new Set();
+  }
+  const patterns = join(scratchDirectory, "coupling-patterns.txt");
+  await writeFile(patterns, `${values.join("\n")}\n`);
+  const result = await runCommand(
+    "rg",
+    [
+      "--fixed-strings",
+      "--only-matching",
+      "--no-filename",
+      "--hidden",
+      "--glob",
+      "!.git/**",
+      "--file",
+      patterns,
+      baseDirectory,
+    ],
+    { allowFailure: true },
+  );
+  if (result.exitCode !== 0 && result.exitCode !== 1) {
+    throw new Error(`failed to scan base repository for contract artifacts: ${result.stderr}`);
+  }
+  return new Set(result.stdout.split("\n").filter((value) => value.length > 0));
+}
+
+function addedLines(patch: string): readonly AddedLine[] {
+  const lines: AddedLine[] = [];
+  let path = "unknown";
+  for (const [index, line] of patch.split("\n").entries()) {
+    if (line.startsWith("diff --git a/")) {
+      const match = /^diff --git a\/(.+) b\/(.+)$/.exec(line);
+      path = match?.[2] ?? "unknown";
+      continue;
+    }
+    if (line.startsWith("+") && !line.startsWith("+++")) {
+      lines.push({ location: `${path}:patch-line-${index + 1}`, text: line.slice(1) });
+    }
+  }
+  return lines;
+}
+
+function contractArtifacts(
+  line: string,
+): readonly { artifact: string; category: ContractArtifactCategory }[] {
+  const artifacts: { artifact: string; category: ContractArtifactCategory }[] = [];
+  const quoted = /(["'`])((?:\\.|.)*?)\1/g;
+  for (const match of line.matchAll(quoted)) {
+    const artifact = unescapeLiteral(match[2] ?? "");
+    const start = match.index ?? 0;
+    const prefix = line.slice(0, start);
+    const suffix = line.slice(start + match[0].length);
+    const category = classifyQuotedArtifact(artifact, prefix, suffix, line);
+    if (category) {
+      artifacts.push({ artifact, category });
+    }
+  }
+  const bareKey = /(?:^\s*|[{,]\s*)([A-Za-z][A-Za-z0-9_]{2,63})\s*:/g;
+  for (const match of line.matchAll(bareKey)) {
+    if (match[1]) {
+      artifacts.push({ artifact: match[1], category: "field_name" });
+    }
+  }
+  const member = /\.([A-Za-z][A-Za-z0-9_]{2,63})\b/g;
+  for (const match of line.matchAll(member)) {
+    if (match[1]) {
+      artifacts.push({ artifact: match[1], category: "field_name" });
+    }
+  }
+  return artifacts;
+}
+
+function classifyQuotedArtifact(
+  artifact: string,
+  prefix: string,
+  suffix: string,
+  line: string,
+): ContractArtifactCategory | undefined {
+  if (/^\/[A-Za-z0-9]/.test(artifact) && !/\s/.test(artifact)) {
+    return "endpoint_path";
+  }
+  if (/^[a-z][a-z0-9.+-]*\/[a-z0-9.+-]+(?:;.*)?$/i.test(artifact)) {
+    return "media_type";
+  }
+  if (/^[A-Za-z][A-Za-z0-9_-]{2,63}$/.test(artifact)) {
+    if (/^\s*:/.test(suffix) || (/\[\s*$/.test(prefix) && /^\s*\]/.test(suffix))) {
+      return "field_name";
+    }
+    if (/headers?|content-type/i.test(line) && artifact.includes("-")) {
+      return "header_name";
+    }
+  }
+  return undefined;
+}
+
+function unescapeLiteral(value: string): string {
+  return value.replace(/\\([\\"'`])/g, "$1");
+}
+
+function containsArtifact(
+  text: string,
+  artifact: string,
+  category: ContractArtifactCategory,
+): boolean {
+  if (category !== "field_name") {
+    return text.includes(artifact);
+  }
+  const escaped = artifact.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return new RegExp(`(^|[^A-Za-z0-9_])${escaped}(?=$|[^A-Za-z0-9_])`).test(text);
+}

package/src/docker-executor.ts

@@ -0,0 +1,115 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve } from "node:path";
+import type { SelfBenchConfig } from "./config.js";
+import { runCommand } from "./process.js";
+import type {
+  SandboxExecutor,
+  SandboxRequest,
+  SandboxResult,
+  SandboxRunOptions,
+} from "./sandbox.js";
+
+export class DockerSandboxExecutor implements SandboxExecutor {
+  readonly #config: Extract<SelfBenchConfig["execution"], { kind: "docker" }>;
+
+  constructor(config: Extract<SelfBenchConfig["execution"], { kind: "docker" }>) {
+    this.#config = config;
+  }
+
+  async run(request: SandboxRequest, options: SandboxRunOptions = {}): Promise<SandboxResult> {
+    const root = await mkdtemp(join(tmpdir(), "selfbench-docker-"));
+    const sandboxId = sandboxName(request.runId, request.stage);
+    try {
+      for (const file of request.files ?? []) {
+        const destination = hostPath(root, file.path);
+        await mkdir(dirname(destination), { recursive: true });
+        await writeFile(destination, file.contents);
+      }
+
+      const environment = { ...request.environment, ...request.secrets };
+      const args = [
+        "create",
+        "--name",
+        sandboxId,
+        "--cpus",
+        String(request.cpu ?? 4),
+        "--memory",
+        `${request.memoryMiB ?? 8192}m`,
+        "--volume",
+        `${sandboxId}:/work`,
+        "--workdir",
+        "/work",
+      ];
+      for (const key of Object.keys(environment)) {
+        args.push("--env", key);
+      }
+      args.push(this.#config.image, ...request.command);
+
+      await runCommand("docker", ["volume", "create", sandboxId]);
+      await runCommand("docker", args, { env: { ...process.env, ...environment } });
+      await runCommand("docker", ["cp", `${root}/.`, `${sandboxId}:/work/`]);
+
+      const abort = () => {
+        void runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
+      };
+      options.signal?.addEventListener("abort", abort, { once: true });
+      try {
+        const result = await runCommand("docker", ["start", "--attach", sandboxId], {
+          allowFailure: true,
+          timeoutMs: request.timeoutMs,
+          ...(request.inactivityTimeoutMs
+            ? { inactivityTimeoutMs: request.inactivityTimeoutMs }
+            : {}),
+          onOutput: (stream, chunk) => options.onProgress?.({ stream, bytes: chunk.byteLength }),
+        });
+        const outputs: Record<string, Uint8Array> = {};
+        for (const path of request.outputPaths ?? []) {
+          const destination = hostPath(root, path);
+          await mkdir(dirname(destination), { recursive: true });
+          const copied = await runCommand("docker", ["cp", `${sandboxId}:${path}`, destination], {
+            allowFailure: true,
+          });
+          if (copied.exitCode === 0) {
+            outputs[path] = await readFile(destination);
+          } else if (result.exitCode === 0) {
+            throw new Error(`sandbox ${sandboxId} exited successfully without output ${path}`);
+          }
+        }
+        return { sandboxId, ...result, outputs };
+      } finally {
+        options.signal?.removeEventListener("abort", abort);
+        await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
+        await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
+          allowFailure: true,
+        });
+      }
+    } finally {
+      await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
+      await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
+        allowFailure: true,
+      });
+      await rm(root, { recursive: true, force: true });
+    }
+  }
+
+  close(): void {}
+}
+
+function hostPath(root: string, containerPath: string): string {
+  if (!containerPath.startsWith("/work/")) {
+    throw new Error(`sandbox path must be beneath /work: ${containerPath}`);
+  }
+  const path = resolve(root, relative("/work", containerPath));
+  if (path !== root && !path.startsWith(`${root}/`)) {
+    throw new Error(`sandbox path escapes /work: ${containerPath}`);
+  }
+  return path;
+}
+
+function sandboxName(runId: string, stage: string): string {
+  const suffix = crypto.randomUUID().slice(0, 8);
+  return `selfbench-${runId.slice(0, 20)}-${stage.slice(0, 16)}-${suffix}`
+    .toLowerCase()
+    .replace(/[^a-z0-9_.-]/g, "-");
+}

package/src/eval-main.ts

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+
+import { parseArgs } from "node:util";
+import { MATRIX_MODELS, type MatrixModel, runMatrix } from "./evaluate.js";
+
+const parsed = parseArgs({
+  options: {
+    export: { type: "string" },
+    tasks: { type: "string" },
+    jobs: { type: "string" },
+    harbor: { type: "string" },
+    environment: { type: "string", default: "modal" },
+    concurrency: { type: "string", default: "3" },
+    auth: { type: "string" },
+    model: { type: "string", multiple: true },
+    help: { type: "boolean", short: "h" },
+  },
+  strict: true,
+});
+if (parsed.values.help) {
+  console.log(`Run Harbor tasks through the fixed Codex subscription model matrix.
+
+Usage:
+  self-bench-eval (--export FILE.tar.gz | --tasks DIRECTORY) --jobs DIRECTORY [options]
+
+Options:
+  --tasks DIRECTORY          Expanded Harbor tasks (one or more)
+  --harbor PATH              Harbor executable (default: harbor)
+  --environment docker|modal Execution environment (default: modal)
+  --concurrency N            Concurrent trials (default: 3)
+  --model MODEL              Run only this model; may be repeated
+  --auth FILE                Codex ChatGPT auth.json path
+  -h, --help                 Show this help`);
+  process.exit(0);
+}
+const environment = parsed.values.environment;
+if (environment !== "docker" && environment !== "modal") {
+  throw new Error("--environment must be docker or modal");
+}
+const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
+const models = evaluationModels(parsed.values.model);
+const summaries = await runMatrix({
+  ...(parsed.values.export ? { exportPath: parsed.values.export } : {}),
+  ...(parsed.values.tasks ? { tasksPath: parsed.values.tasks } : {}),
+  jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
+  environment,
+  concurrency,
+  ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
+  ...(parsed.values.auth ? { authPath: parsed.values.auth } : {}),
+  ...(models ? { models } : {}),
+  onTrialComplete: (summary, completed, total) => {
+    const status = summary.passed ? "passed" : summary.exception ? "error" : "failed";
+    console.error(`[${completed}/${total}] ${summary.model} ${summary.taskId}: ${status}`);
+  },
+});
+console.log(
+  JSON.stringify(
+    {
+      trials: summaries.length,
+      passed: summaries.filter((summary) => summary.passed).length,
+      failed: summaries.filter((summary) => !summary.passed).length,
+    },
+    null,
+    2,
+  ),
+);
+
+function fail(message: string): never {
+  throw new Error(message);
+}
+
+function evaluationModels(values: string[] | undefined): readonly MatrixModel[] | undefined {
+  if (!values) {
+    return undefined;
+  }
+  const allowed = new Set<string>(MATRIX_MODELS);
+  const invalid = values.filter((value) => !allowed.has(value));
+  if (invalid.length > 0) {
+    throw new Error(
+      `--model must be one of ${MATRIX_MODELS.join(", ")}; got ${invalid.join(", ")}`,
+    );
+  }
+  return [...new Set(values)] as MatrixModel[];
+}
+
+function positiveInteger(value: string | undefined, label: string): number {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return parsed;
+}