self-bench 0.3.5 → 0.3.8

package/src/coupling.ts

@@ -1,259 +0,0 @@
-import { writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { runCommand } from "./process.js";
-
-export type ContractArtifactCategory =
-  | "endpoint_path"
-  | "field_name"
-  | "header_name"
-  | "media_type";
-
-export interface ContractArtifactEvidence {
-  readonly artifact: string;
-  readonly category: ContractArtifactCategory;
-  readonly testLocations: readonly string[];
-  readonly introducedByGold: boolean;
-  readonly presentInPrompt: boolean;
-  readonly presentInBase: boolean;
-}
-
-export interface CouplingEvidence {
-  readonly schemaVersion: 1;
-  readonly artifacts: readonly ContractArtifactEvidence[];
-  readonly blockers: readonly string[];
-}
-
-export interface CouplingReviewInput {
-  readonly verdict: "clean" | "coupled";
-  readonly reason: string;
-  readonly findings: readonly {
-    readonly artifact: string;
-    readonly disposition:
-      | "base_contract"
-      | "prompt_contract"
-      | "external_contract"
-      | "gold_only"
-      | "not_contract";
-  }[];
-}
-
-export interface CouplingReviewResolution {
-  readonly verdict: "clean" | "coupled";
-  readonly reason: string;
-  readonly missingArtifacts: readonly string[];
-  readonly goldOnlyArtifacts: readonly string[];
-}
-
-interface AddedLine {
-  readonly location: string;
-  readonly text: string;
-}
-
-export interface ContractArtifactCandidate {
-  readonly artifact: string;
-  readonly category: ContractArtifactCategory;
-  readonly testLocations: readonly string[];
-}
-
-export function discoverContractArtifacts(testPatch: string): readonly ContractArtifactCandidate[] {
-  const found = new Map<string, { category: ContractArtifactCategory; locations: Set<string> }>();
-  for (const line of addedLines(testPatch)) {
-    for (const candidate of contractArtifacts(line.text)) {
-      const key = `${candidate.category}\u0000${candidate.artifact}`;
-      const existing = found.get(key);
-      if (existing) {
-        existing.locations.add(line.location);
-      } else {
-        found.set(key, {
-          category: candidate.category,
-          locations: new Set([line.location]),
-        });
-      }
-    }
-  }
-  return [...found.entries()]
-    .map(([key, value]) => ({
-      artifact: key.slice(key.indexOf("\u0000") + 1),
-      category: value.category,
-      testLocations: [...value.locations].sort(),
-    }))
-    .sort((left, right) =>
-      left.category === right.category
-        ? left.artifact.localeCompare(right.artifact)
-        : left.category.localeCompare(right.category),
-    );
-}
-
-export function buildCouplingEvidence(input: {
-  readonly prompt: string;
-  readonly testPatch: string;
-  readonly goldPatch: string;
-  readonly baseArtifacts: ReadonlySet<string>;
-}): CouplingEvidence {
-  const gold = addedLines(input.goldPatch)
-    .map((line) => line.text)
-    .join("\n");
-  const artifacts = discoverContractArtifacts(input.testPatch)
-    .filter((candidate) => containsArtifact(gold, candidate.artifact, candidate.category))
-    .map(
-      (candidate): ContractArtifactEvidence => ({
-        ...candidate,
-        introducedByGold: true,
-        presentInPrompt: containsArtifact(input.prompt, candidate.artifact, candidate.category),
-        presentInBase: input.baseArtifacts.has(candidate.artifact),
-      }),
-    );
-  const blockers = artifacts
-    .filter((artifact) => !artifact.presentInPrompt && !artifact.presentInBase)
-    .map(
-      (artifact) =>
-        `held-out tests assert gold-only ${artifact.category} ${JSON.stringify(artifact.artifact)} at ${artifact.testLocations.join(", ")}`,
-    );
-  return { schemaVersion: 1, artifacts, blockers };
-}
-
-export function resolveCouplingReview(
-  evidence: CouplingEvidence,
-  review: CouplingReviewInput,
-): CouplingReviewResolution {
-  const unresolvedArtifacts = evidence.artifacts
-    .filter((artifact) => !artifact.presentInBase && !artifact.presentInPrompt)
-    .map((artifact) => artifact.artifact);
-  const reviewedArtifacts = new Set(review.findings.map((finding) => finding.artifact));
-  const missingArtifacts = unresolvedArtifacts.filter(
-    (artifact) => !reviewedArtifacts.has(artifact),
-  );
-  const goldOnlyArtifacts = review.findings
-    .filter((finding) => finding.disposition === "gold_only")
-    .map((finding) => finding.artifact);
-  const verdict =
-    review.verdict === "coupled" || missingArtifacts.length > 0 || goldOnlyArtifacts.length > 0
-      ? "coupled"
-      : "clean";
-  const reason =
-    missingArtifacts.length > 0
-      ? `review did not resolve deterministic coupling evidence for: ${missingArtifacts.join(", ")}`
-      : goldOnlyArtifacts.length > 0 && review.verdict === "clean"
-        ? `review identified gold-only artifacts: ${goldOnlyArtifacts.join(", ")}`
-        : review.reason;
-  return { verdict, reason, missingArtifacts, goldOnlyArtifacts };
-}
-
-export async function scanBaseContractArtifacts(
-  baseDirectory: string,
-  scratchDirectory: string,
-  artifacts: readonly { readonly artifact: string }[],
-): Promise<ReadonlySet<string>> {
-  const values = [...new Set(artifacts.map((artifact) => artifact.artifact))].filter(
-    (artifact) => !artifact.includes("\n") && artifact.length > 0,
-  );
-  if (values.length === 0) {
-    return new Set();
-  }
-  const patterns = join(scratchDirectory, "coupling-patterns.txt");
-  await writeFile(patterns, `${values.join("\n")}\n`);
-  const result = await runCommand(
-    "rg",
-    [
-      "--fixed-strings",
-      "--only-matching",
-      "--no-filename",
-      "--hidden",
-      "--glob",
-      "!.git/**",
-      "--file",
-      patterns,
-      baseDirectory,
-    ],
-    { allowFailure: true },
-  );
-  if (result.exitCode !== 0 && result.exitCode !== 1) {
-    throw new Error(`failed to scan base repository for contract artifacts: ${result.stderr}`);
-  }
-  return new Set(result.stdout.split("\n").filter((value) => value.length > 0));
-}
-
-function addedLines(patch: string): readonly AddedLine[] {
-  const lines: AddedLine[] = [];
-  let path = "unknown";
-  for (const [index, line] of patch.split("\n").entries()) {
-    if (line.startsWith("diff --git a/")) {
-      const match = /^diff --git a\/(.+) b\/(.+)$/.exec(line);
-      path = match?.[2] ?? "unknown";
-      continue;
-    }
-    if (line.startsWith("+") && !line.startsWith("+++")) {
-      lines.push({ location: `${path}:patch-line-${index + 1}`, text: line.slice(1) });
-    }
-  }
-  return lines;
-}
-
-function contractArtifacts(
-  line: string,
-): readonly { artifact: string; category: ContractArtifactCategory }[] {
-  const artifacts: { artifact: string; category: ContractArtifactCategory }[] = [];
-  const quoted = /(["'`])((?:\\.|.)*?)\1/g;
-  for (const match of line.matchAll(quoted)) {
-    const artifact = unescapeLiteral(match[2] ?? "");
-    const start = match.index ?? 0;
-    const prefix = line.slice(0, start);
-    const suffix = line.slice(start + match[0].length);
-    const category = classifyQuotedArtifact(artifact, prefix, suffix, line);
-    if (category) {
-      artifacts.push({ artifact, category });
-    }
-  }
-  const bareKey = /(?:^\s*|[{,]\s*)([A-Za-z][A-Za-z0-9_]{2,63})\s*:/g;
-  for (const match of line.matchAll(bareKey)) {
-    if (match[1]) {
-      artifacts.push({ artifact: match[1], category: "field_name" });
-    }
-  }
-  const member = /\.([A-Za-z][A-Za-z0-9_]{2,63})\b/g;
-  for (const match of line.matchAll(member)) {
-    if (match[1]) {
-      artifacts.push({ artifact: match[1], category: "field_name" });
-    }
-  }
-  return artifacts;
-}
-
-function classifyQuotedArtifact(
-  artifact: string,
-  prefix: string,
-  suffix: string,
-  line: string,
-): ContractArtifactCategory | undefined {
-  if (/^\/[A-Za-z0-9]/.test(artifact) && !/\s/.test(artifact)) {
-    return "endpoint_path";
-  }
-  if (/^[a-z][a-z0-9.+-]*\/[a-z0-9.+-]+(?:;.*)?$/i.test(artifact)) {
-    return "media_type";
-  }
-  if (/^[A-Za-z][A-Za-z0-9_-]{2,63}$/.test(artifact)) {
-    if (/^\s*:/.test(suffix) || (/\[\s*$/.test(prefix) && /^\s*\]/.test(suffix))) {
-      return "field_name";
-    }
-    if (/headers?|content-type/i.test(line) && artifact.includes("-")) {
-      return "header_name";
-    }
-  }
-  return undefined;
-}
-
-function unescapeLiteral(value: string): string {
-  return value.replace(/\\([\\"'`])/g, "$1");
-}
-
-function containsArtifact(
-  text: string,
-  artifact: string,
-  category: ContractArtifactCategory,
-): boolean {
-  if (category !== "field_name") {
-    return text.includes(artifact);
-  }
-  const escaped = artifact.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  return new RegExp(`(^|[^A-Za-z0-9_])${escaped}(?=$|[^A-Za-z0-9_])`).test(text);
-}

package/src/docker-executor.ts

@@ -1,115 +0,0 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { dirname, join, relative, resolve } from "node:path";
-import type { SelfBenchConfig } from "./config.js";
-import { runCommand } from "./process.js";
-import type {
-  SandboxExecutor,
-  SandboxRequest,
-  SandboxResult,
-  SandboxRunOptions,
-} from "./sandbox.js";
-
-export class DockerSandboxExecutor implements SandboxExecutor {
-  readonly #config: Extract<SelfBenchConfig["execution"], { kind: "docker" }>;
-
-  constructor(config: Extract<SelfBenchConfig["execution"], { kind: "docker" }>) {
-    this.#config = config;
-  }
-
-  async run(request: SandboxRequest, options: SandboxRunOptions = {}): Promise<SandboxResult> {
-    const root = await mkdtemp(join(tmpdir(), "selfbench-docker-"));
-    const sandboxId = sandboxName(request.runId, request.stage);
-    try {
-      for (const file of request.files ?? []) {
-        const destination = hostPath(root, file.path);
-        await mkdir(dirname(destination), { recursive: true });
-        await writeFile(destination, file.contents);
-      }
-
-      const environment = { ...request.environment, ...request.secrets };
-      const args = [
-        "create",
-        "--name",
-        sandboxId,
-        "--cpus",
-        String(request.cpu ?? 4),
-        "--memory",
-        `${request.memoryMiB ?? 8192}m`,
-        "--volume",
-        `${sandboxId}:/work`,
-        "--workdir",
-        "/work",
-      ];
-      for (const key of Object.keys(environment)) {
-        args.push("--env", key);
-      }
-      args.push(this.#config.image, ...request.command);
-
-      await runCommand("docker", ["volume", "create", sandboxId]);
-      await runCommand("docker", args, { env: { ...process.env, ...environment } });
-      await runCommand("docker", ["cp", `${root}/.`, `${sandboxId}:/work/`]);
-
-      const abort = () => {
-        void runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
-      };
-      options.signal?.addEventListener("abort", abort, { once: true });
-      try {
-        const result = await runCommand("docker", ["start", "--attach", sandboxId], {
-          allowFailure: true,
-          timeoutMs: request.timeoutMs,
-          ...(request.inactivityTimeoutMs
-            ? { inactivityTimeoutMs: request.inactivityTimeoutMs }
-            : {}),
-          onOutput: (stream, chunk) => options.onProgress?.({ stream, bytes: chunk.byteLength }),
-        });
-        const outputs: Record<string, Uint8Array> = {};
-        for (const path of request.outputPaths ?? []) {
-          const destination = hostPath(root, path);
-          await mkdir(dirname(destination), { recursive: true });
-          const copied = await runCommand("docker", ["cp", `${sandboxId}:${path}`, destination], {
-            allowFailure: true,
-          });
-          if (copied.exitCode === 0) {
-            outputs[path] = await readFile(destination);
-          } else if (result.exitCode === 0) {
-            throw new Error(`sandbox ${sandboxId} exited successfully without output ${path}`);
-          }
-        }
-        return { sandboxId, ...result, outputs };
-      } finally {
-        options.signal?.removeEventListener("abort", abort);
-        await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
-        await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
-          allowFailure: true,
-        });
-      }
-    } finally {
-      await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
-      await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
-        allowFailure: true,
-      });
-      await rm(root, { recursive: true, force: true });
-    }
-  }
-
-  close(): void {}
-}
-
-function hostPath(root: string, containerPath: string): string {
-  if (!containerPath.startsWith("/work/")) {
-    throw new Error(`sandbox path must be beneath /work: ${containerPath}`);
-  }
-  const path = resolve(root, relative("/work", containerPath));
-  if (path !== root && !path.startsWith(`${root}/`)) {
-    throw new Error(`sandbox path escapes /work: ${containerPath}`);
-  }
-  return path;
-}
-
-function sandboxName(runId: string, stage: string): string {
-  const suffix = crypto.randomUUID().slice(0, 8);
-  return `selfbench-${runId.slice(0, 20)}-${stage.slice(0, 16)}-${suffix}`
-    .toLowerCase()
-    .replace(/[^a-z0-9_.-]/g, "-");
-}

package/src/eval-main.ts

@@ -1,93 +0,0 @@
-#!/usr/bin/env node
-
-import { parseArgs } from "node:util";
-import { MATRIX_MODELS, type MatrixModel, runMatrix } from "./evaluate.js";
-import { isHarborEnvironment } from "./providers.js";
-
-const parsed = parseArgs({
-  options: {
-    export: { type: "string" },
-    tasks: { type: "string" },
-    jobs: { type: "string" },
-    harbor: { type: "string" },
-    environment: { type: "string", default: "modal" },
-    concurrency: { type: "string", default: "3" },
-    auth: { type: "string" },
-    model: { type: "string", multiple: true },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Run Harbor tasks through the fixed Codex model matrix.
-
-Usage:
-  self-bench-eval (--export FILE.tar.gz | --tasks DIRECTORY) --jobs DIRECTORY [options]
-
-Options:
-  --tasks DIRECTORY          Expanded Harbor tasks (one or more)
-  --harbor PATH              Harbor executable (default: harbor)
-  --environment docker|modal Execution environment (default: modal)
-  --concurrency N            Concurrent trials (default: 3)
-  --model MODEL              Run only this model; may be repeated
-  --auth FILE                Optional Codex ChatGPT auth.json fallback
-  -h, --help                 Show this help`);
-  process.exit(0);
-}
-const environment = parsed.values.environment ?? "modal";
-if (!isHarborEnvironment(environment)) {
-  throw new Error("--environment must be docker or modal");
-}
-const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
-const models = evaluationModels(parsed.values.model);
-const summaries = await runMatrix({
-  ...(parsed.values.export ? { exportPath: parsed.values.export } : {}),
-  ...(parsed.values.tasks ? { tasksPath: parsed.values.tasks } : {}),
-  jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
-  environment,
-  concurrency,
-  ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
-  ...(parsed.values.auth ? { authPath: parsed.values.auth } : {}),
-  ...(models ? { models } : {}),
-  onTrialComplete: (summary, completed, total) => {
-    const status = summary.passed ? "passed" : summary.exception ? "error" : "failed";
-    console.error(`[${completed}/${total}] ${summary.model} ${summary.taskId}: ${status}`);
-  },
-});
-console.log(
-  JSON.stringify(
-    {
-      trials: summaries.length,
-      passed: summaries.filter((summary) => summary.passed).length,
-      failed: summaries.filter((summary) => !summary.passed).length,
-    },
-    null,
-    2,
-  ),
-);
-
-function fail(message: string): never {
-  throw new Error(message);
-}
-
-function evaluationModels(values: string[] | undefined): readonly MatrixModel[] | undefined {
-  if (!values) {
-    return undefined;
-  }
-  const allowed = new Set<string>(MATRIX_MODELS);
-  const invalid = values.filter((value) => !allowed.has(value));
-  if (invalid.length > 0) {
-    throw new Error(
-      `--model must be one of ${MATRIX_MODELS.join(", ")}; got ${invalid.join(", ")}`,
-    );
-  }
-  return [...new Set(values)] as MatrixModel[];
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return parsed;
-}