self-bench 0.3.6 → 0.3.8

package/src/terminal-prompts.ts

@@ -1,197 +0,0 @@
-import {
-  autocomplete,
-  cancel as cancelPrompt,
-  confirm as confirmPrompt,
-  isCancel,
-  password,
-  select as selectPrompt,
-  text as textPrompt,
-} from "@clack/prompts";
-
-export class SetupCanceledError extends Error {
-  constructor(message = "Setup canceled") {
-    super(message);
-    this.name = "SetupCanceledError";
-  }
-}
-
-export interface PromptChoice {
-  readonly value: string;
-  readonly label: string;
-  readonly hint?: string;
-}
-
-export interface SetupPrompter {
-  select(message: string, choices: readonly PromptChoice[], defaultValue?: string): Promise<string>;
-  search(message: string, choices: readonly PromptChoice[], defaultValue?: string): Promise<string>;
-  text(message: string, defaultValue?: string): Promise<string>;
-  secret(message: string): Promise<string>;
-  confirm(message: string, defaultValue: boolean): Promise<boolean>;
-}
-
-export class TerminalPrompter implements SetupPrompter {
-  readonly #input: NodeJS.ReadStream;
-  readonly #output: NodeJS.WriteStream;
-
-  constructor(
-    input: NodeJS.ReadStream = process.stdin,
-    output: NodeJS.WriteStream = process.stdout,
-  ) {
-    this.#input = input;
-    this.#output = output;
-  }
-
-  async select(
-    message: string,
-    choices: readonly PromptChoice[],
-    defaultValue?: string,
-  ): Promise<string> {
-    this.#assertChoices(message, choices);
-    return await this.#run((signal) =>
-      selectPrompt<string>({
-        message,
-        options: choices.map(toPromptOption),
-        maxItems: this.#visibleItemCount(),
-        showInstructions: true,
-        ...(defaultValue !== undefined && choices.some(({ value }) => value === defaultValue)
-          ? { initialValue: defaultValue }
-          : {}),
-        input: this.#input,
-        output: this.#output,
-        signal,
-      }),
-    );
-  }
-
-  async search(
-    message: string,
-    choices: readonly PromptChoice[],
-    defaultValue?: string,
-  ): Promise<string> {
-    this.#assertChoices(message, choices);
-    return await this.#run((signal) =>
-      autocomplete<string>({
-        message,
-        options: choices.map(toPromptOption),
-        placeholder: "Type to filter…",
-        maxItems: this.#visibleItemCount(),
-        filter: (query, option) =>
-          `${option.label ?? ""}\n${option.hint ?? ""}\n${option.value}`
-            .toLowerCase()
-            .includes(query.toLowerCase()),
-        ...(defaultValue !== undefined && choices.some(({ value }) => value === defaultValue)
-          ? { initialValue: defaultValue }
-          : {}),
-        input: this.#input,
-        output: this.#output,
-        signal,
-      }),
-    );
-  }
-
-  async text(message: string, defaultValue?: string): Promise<string> {
-    return await this.#run((signal) =>
-      textPrompt({
-        message,
-        ...(defaultValue === undefined ? {} : { placeholder: defaultValue, defaultValue }),
-        input: this.#input,
-        output: this.#output,
-        signal,
-      }),
-    );
-  }
-
-  async secret(message: string): Promise<string> {
-    if (!this.#input.isTTY || !this.#input.setRawMode) {
-      throw new Error("secure token input requires an interactive terminal");
-    }
-    return await this.#run((signal) =>
-      password({
-        message,
-        // An empty mask keeps both the token and its length out of terminal output.
-        mask: "",
-        validate: (value) => (value?.trim() ? undefined : "Enter a project-scoped token."),
-        input: this.#input,
-        output: this.#output,
-        signal,
-      }),
-    );
-  }
-
-  async confirm(message: string, defaultValue: boolean): Promise<boolean> {
-    return await this.#run((signal) =>
-      confirmPrompt({
-        message,
-        initialValue: defaultValue,
-        input: this.#input,
-        output: this.#output,
-        signal,
-      }),
-    );
-  }
-
-  async #run<T>(prompt: (signal: AbortSignal) => Promise<T | symbol>): Promise<T> {
-    const controller = new AbortController();
-    const wasRaw = this.#input.isRaw;
-    const wasFlowing = this.#input.readableFlowing;
-    let inputFailure: Error | undefined;
-    const abort = (error: Error): void => {
-      inputFailure = error;
-      controller.abort(error);
-    };
-    const onEnd = (): void => abort(new SetupCanceledError("Setup canceled at end of input"));
-    const onError = (error: Error): void => abort(error);
-    this.#input.once("end", onEnd);
-    this.#input.once("error", onError);
-    if (this.#input.readableEnded) {
-      onEnd();
-    }
-
-    try {
-      const result = await prompt(controller.signal);
-      if (isCancel(result)) {
-        const error = inputFailure ?? new SetupCanceledError();
-        if (error instanceof SetupCanceledError) {
-          cancelPrompt("Setup canceled.", { output: this.#output, withGuide: true });
-        }
-        throw error;
-      }
-      return result;
-    } finally {
-      this.#input.off("end", onEnd);
-      this.#input.off("error", onError);
-      if (this.#input.isTTY && this.#input.setRawMode && this.#input.isRaw !== wasRaw) {
-        this.#input.setRawMode(Boolean(wasRaw));
-      }
-      if (wasFlowing !== true) {
-        this.#input.pause();
-      }
-    }
-  }
-
-  #assertChoices(message: string, choices: readonly PromptChoice[]): void {
-    if (choices.length === 0) {
-      throw new Error(`${message} has no available choices`);
-    }
-  }
-
-  #visibleItemCount(): number {
-    return Math.max(3, Math.min(8, (this.#output.rows ?? 20) - 6));
-  }
-}
-
-export function isInteractiveTerminal(): boolean {
-  return Boolean(process.stdin.isTTY && process.stdout.isTTY);
-}
-
-function toPromptOption(choice: PromptChoice): {
-  readonly value: string;
-  readonly label: string;
-  readonly hint?: string;
-} {
-  return {
-    value: choice.value,
-    label: choice.label,
-    ...(choice.hint ? { hint: choice.hint } : {}),
-  };
-}

package/src/terminal-reporter.ts

@@ -1,126 +0,0 @@
-import type { Writable } from "node:stream";
-import { stripVTControlCharacters } from "node:util";
-import { cancel as cancelPrompt, intro, log, outro, spinner } from "@clack/prompts";
-import { type CommandOutputHandler, RollingOutput } from "./process.js";
-import { SetupCanceledError } from "./terminal-prompts.js";
-
-export interface SetupTaskLabels {
-  readonly pending: string;
-  readonly success: string;
-  readonly failure: string;
-}
-
-export interface SetupReporter {
-  intro(title: string): void;
-  message(message: string): void;
-  warn(message: string): void;
-  cancel(message: string): void;
-  task<T>(
-    labels: SetupTaskLabels,
-    operation: (onOutput: CommandOutputHandler) => Promise<T>,
-  ): Promise<T>;
-  finish(title: string, details: readonly string[]): void;
-}
-
-export class TerminalSetupReporter implements SetupReporter {
-  readonly #output: Writable;
-  readonly #errorOutput: Writable;
-  readonly #verbose: boolean;
-
-  constructor(
-    options: {
-      readonly output?: Writable;
-      readonly errorOutput?: Writable;
-      readonly verbose?: boolean;
-    } = {},
-  ) {
-    this.#output = options.output ?? process.stdout;
-    this.#errorOutput = options.errorOutput ?? process.stderr;
-    this.#verbose = options.verbose ?? false;
-  }
-
-  intro(title: string): void {
-    intro(title, { output: this.#output, withGuide: true });
-  }
-
-  message(message: string): void {
-    log.message(message, { output: this.#output, withGuide: true });
-  }
-
-  warn(message: string): void {
-    log.warn(message, { output: this.#output, withGuide: true });
-  }
-
-  cancel(message: string): void {
-    cancelPrompt(message, { output: this.#output, withGuide: true });
-  }
-
-  async task<T>(
-    labels: SetupTaskLabels,
-    operation: (onOutput: CommandOutputHandler) => Promise<T>,
-  ): Promise<T> {
-    const captured = new RollingOutput();
-    const onOutput: CommandOutputHandler = (stream, chunk) => {
-      captured.push(Buffer.from(chunk));
-      if (this.#verbose) {
-        (stream === "stderr" ? this.#errorOutput : this.#output).write(chunk);
-      }
-    };
-
-    if (this.#verbose) {
-      log.step(labels.pending, { output: this.#output, withGuide: true });
-      try {
-        const result = await operation(onOutput);
-        log.success(labels.success, { output: this.#output, withGuide: true });
-        return result;
-      } catch (error) {
-        if (error instanceof SetupCanceledError) {
-          this.cancel("Setup canceled.");
-          throw error;
-        }
-        log.error(labels.failure, { output: this.#output, withGuide: true });
-        throw error;
-      }
-    }
-
-    const progress = spinner({ indicator: "timer", output: this.#output, withGuide: true });
-    progress.start(labels.pending);
-    try {
-      const result = await operation(onOutput);
-      progress.stop(labels.success);
-      return result;
-    } catch (error) {
-      if (error instanceof SetupCanceledError) {
-        progress.cancel("Setup canceled.");
-        throw error;
-      }
-      progress.error(labels.failure);
-      this.#showFailureOutput(captured.text());
-      throw error;
-    }
-  }
-
-  finish(title: string, details: readonly string[]): void {
-    log.success(title, { output: this.#output, withGuide: true });
-    log.message([...details], {
-      output: this.#output,
-      spacing: 0,
-      withGuide: true,
-    });
-    outro("Vercel Sandbox is ready.", { output: this.#output, withGuide: true });
-  }
-
-  #showFailureOutput(value: string): void {
-    const cleaned = stripVTControlCharacters(value)
-      .replace(/\r(?!\n)/g, "\n")
-      .trim();
-    if (!cleaned) {
-      return;
-    }
-    log.message(["Command output:", ...cleaned.split("\n")], {
-      output: this.#output,
-      spacing: 0,
-      withGuide: true,
-    });
-  }
-}

package/src/validate-main.ts

@@ -1,177 +0,0 @@
-#!/usr/bin/env node
-
-import { access, mkdir, readdir, writeFile } from "node:fs/promises";
-import { basename, join, resolve } from "node:path";
-import { parseArgs } from "node:util";
-import { harborChildEnvironment } from "./harbor-environment.js";
-import { harborInfrastructureError, readHarborJobResult } from "./harbor-results.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import { type HarborEnvironment, isHarborEnvironment } from "./providers.js";
-
-const parsed = parseArgs({
-  options: {
-    tasks: { type: "string" },
-    jobs: { type: "string" },
-    environment: { type: "string", default: "modal" },
-    concurrency: { type: "string", default: "10" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Run Harbor nop and oracle gates for expanded SelfBench tasks.
-
-Usage:
-  self-bench-validate --tasks DIRECTORY --jobs DIRECTORY [options]
-
-Options:
-  --environment NAME  Harbor environment (default: modal)
-  --concurrency N     Concurrent tasks (default: 10)
-  -h, --help          Show this help`);
-  process.exit(0);
-}
-
-const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
-const jobsDirectory = resolve(parsed.values.jobs ?? fail("--jobs is required"));
-const environment = parsed.values.environment ?? "modal";
-if (!isHarborEnvironment(environment)) {
-  throw new Error("--environment must be modal or docker");
-}
-const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
-const directories = (
-  await Promise.all(
-    (
-      await readdir(tasksDirectory)
-    ).map(async (name) => {
-      const directory = join(tasksDirectory, name);
-      return await access(join(directory, "task.toml")).then(
-        () => directory,
-        () => undefined,
-      );
-    }),
-  )
-).filter((directory): directory is string => directory !== undefined);
-await mkdir(jobsDirectory, { recursive: true });
-
-const tasks = await parallelMap(directories.sort(), concurrency, async (taskDirectory) => {
-  const taskId = basename(taskDirectory);
-  console.error(`validating ${taskId}`);
-  try {
-    const nop = await runGate(taskDirectory, taskId, "nop", environment, jobsDirectory);
-    const oracle = await runGate(taskDirectory, taskId, "oracle", environment, jobsDirectory);
-    const accepted = nop.passed && oracle.passed;
-    console.error(`${taskId}: ${accepted ? "accepted" : "rejected"}`);
-    return { taskId, status: "validated" as const, accepted, nop, oracle };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    console.error(`${taskId}: error: ${message}`);
-    return { taskId, status: "error" as const, accepted: false, error: message };
-  }
-});
-const report = {
-  schemaVersion: 1,
-  generatedAt: new Date().toISOString(),
-  sourceTasks: tasksDirectory,
-  environment,
-  summary: {
-    total: tasks.length,
-    accepted: tasks.filter((task) => task.accepted).length,
-    rejected: tasks.filter((task) => task.status === "validated" && !task.accepted).length,
-    errors: tasks.filter((task) => task.status === "error").length,
-  },
-  tasks,
-};
-await writeFile(
-  join(jobsDirectory, "validation-summary.json"),
-  `${JSON.stringify(report, null, 2)}\n`,
-);
-console.log(JSON.stringify(report.summary, null, 2));
-
-async function runGate(
-  taskDirectory: string,
-  taskId: string,
-  agent: "nop" | "oracle",
-  environment: HarborEnvironment,
-  jobsDirectory: string,
-): Promise<{ readonly passed: boolean; readonly rewards: Readonly<Record<string, number>> }> {
-  const jobName = `${taskId}-${agent}-${crypto.randomUUID().slice(0, 8)}`;
-  const process = await runCommand(
-    "harbor",
-    [
-      "run",
-      "--path",
-      taskDirectory,
-      "--agent",
-      agent,
-      "--env",
-      environment,
-      "--job-name",
-      jobName,
-      "--jobs-dir",
-      jobsDirectory,
-      "--n-concurrent",
-      "1",
-      "--max-retries",
-      "1",
-      "--delete",
-      "--yes",
-      "--quiet",
-    ],
-    {
-      allowFailure: true,
-      env: harborChildEnvironment(),
-      timeoutMs: 4 * 60 * 60 * 1000,
-    },
-  );
-  if (process.exitCode !== 0) {
-    throw new Error(`Harbor ${agent} exited ${process.exitCode}: ${process.stderr.slice(-1_000)}`);
-  }
-  const result = await readHarborJobResult(jobsDirectory, jobName);
-  const infrastructure = harborInfrastructureError(result.trial);
-  if (infrastructure) {
-    throw new Error(`Harbor ${agent} infrastructure failure: ${infrastructure}`);
-  }
-  const rewards = rewardValues(result.trial);
-  const passed =
-    agent === "nop"
-      ? rewards.fail_to_pass === 0 &&
-        (rewards.pass_to_pass ?? 0) >= 1 &&
-        (rewards.setup_completed ?? 0) >= 1
-      : (rewards.patch_applied ?? 0) >= 1 &&
-        (rewards.fail_to_pass ?? 0) >= 1 &&
-        (rewards.pass_to_pass ?? 0) >= 1 &&
-        (rewards.deterministic ?? 0) >= 1 &&
-        (rewards.setup_completed ?? 0) >= 1;
-  return { passed, rewards };
-}
-
-function rewardValues(value: unknown): Readonly<Record<string, number>> {
-  if (!isRecord(value) || !isRecord(value.verifier_result)) {
-    return {};
-  }
-  const raw = value.verifier_result.rewards;
-  return isRecord(raw)
-    ? Object.fromEntries(
-        Object.entries(raw).filter(
-          (entry): entry is [string, number] => typeof entry[1] === "number",
-        ),
-      )
-    : {};
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const number = Number(value);
-  if (!Number.isInteger(number) || number < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return number;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function fail(message: string): never {
-  throw new Error(message);
-}

package/src/validation-repair.ts

@@ -1,94 +0,0 @@
-import type { TaskDefinition } from "./contracts.js";
-import { patchPaths } from "./repair.js";
-
-export function validationRepairPaths(originalTestPatch: string): readonly string[] {
-  const paths = patchPaths(originalTestPatch);
-  if (paths.length === 0) {
-    throw new Error("original held-out test patch changes no files");
-  }
-  const nonTestPaths = paths.filter((path) => !isTestOnlyPath(path));
-  if (nonTestPaths.length > 0) {
-    throw new Error(`validation repair requires test-only patch paths: ${nonTestPaths.join(", ")}`);
-  }
-  return paths;
-}
-
-function isTestOnlyPath(path: string): boolean {
-  const segments = path.toLowerCase().split("/");
-  const filename = segments.at(-1) ?? "";
-  return (
-    segments.some((segment) =>
-      /^(?:tests?|__tests__|e2e|integration|fixtures?|selfbench(?:-tests|_tests)?|\.selfbench-tests)$/.test(
-        segment,
-      ),
-    ) || /(?:^|\.)(?:test|spec)\.[^.]+$/.test(filename)
-  );
-}
-
-export function assertValidationRepair(
-  original: TaskDefinition,
-  repaired: TaskDefinition,
-  originalTestPatch: string,
-  changedPaths: readonly string[],
-): void {
-  const immutableKeys = [
-    "schemaVersion",
-    "difficulty",
-    "taskId",
-    "repo",
-    "baseCommit",
-    "workdir",
-    "sourcePr",
-    "sourceUrl",
-    "prompt",
-  ] as const;
-  for (const key of immutableKeys) {
-    if (JSON.stringify(repaired[key]) !== JSON.stringify(original[key])) {
-      throw new Error(`validation repair changed immutable definition field ${key}`);
-    }
-  }
-  const allowed = new Set(validationRepairPaths(originalTestPatch));
-  const outside = changedPaths.filter((path) => !allowed.has(path));
-  if (outside.length > 0) {
-    throw new Error(
-      `validation repair changed files outside held-out tests: ${outside.join(", ")}`,
-    );
-  }
-}
-
-export function validationRepairPrompt(input: {
-  readonly definition: TaskDefinition;
-  readonly authenticRequest: string;
-  readonly diagnostics: string;
-  readonly allowedPaths: readonly string[];
-}): string {
-  return `Repair the verifier harness and held-out tests for SelfBench task ${input.definition.taskId}.
-
-The repository is the exact base snapshot with the current held-out test patch already applied. You may edit only these existing held-out test files:
-
-${input.allowedPaths.map((path) => `- ${path}`).join("\n")}
-
-The authentic engineer request is:
-
-<authentic_request>
-${input.authenticRequest.trim()}
-</authentic_request>
-
-The failed nop/oracle validation diagnostics are:
-
-<validation_diagnostics>
-${input.diagnostics.trim()}
-</validation_diagnostics>
-
-Fix the task so its repository-native verifier is rigorous and runnable. You may edit the allowed held-out tests and /work/definition.json. In definition.json you may change only setupCommand, testCommand, failToPass, passToPass, testPaths, toolchains, timeouts, and resources. Never change task identity, difficulty, repository, base commit, workdir, source pull request, prompt, or reference solution.
-
-The required validation split is:
-- nop: the base repository plus held-out tests must make failToPass fail while passToPass succeeds;
-- oracle: after /work/task/solution/gold.patch is applied, failToPass and passToPass must both succeed deterministically.
-
-Use the verifier diagnostics rather than guessing. Inspect repository package scripts and CI for the native test command. Keep {tests} as a list-safe placeholder; do not quote the whole placeholder, assign it to one scalar, or hard-code selected paths elsewhere. Use one test mode/bundler per command instead of chaining equivalent suites. Put dependency installation, native builds, fixture generation, and browser/runtime installation in setupCommand, not testCommand. Pin the repository's declared package manager and use frozen installs. Browser-backed tests must install required browser binaries and OS dependencies during setup. Run focused nop/oracle-style commands in the sandbox when feasible.
-
-Do not weaken, delete, or skip requested behavioral coverage merely to make commands exit zero. A passing base with no solution is not acceptable. If no rigorous runnable split exists, leave the files and definition unchanged and explain why in the final response.
-
-Inspect /work/gold.patch and /work/test.sh only to verify the intended behavior and oracle seam, never to couple assertions to private implementation details. Do not modify application code, the instruction, gold patch, or commit files. Finish with the repaired definition at /work/definition.json and any held-out test edits present in the working tree; do not commit.`;
-}

package/tsconfig.build.json

@@ -1,13 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "compilerOptions": {
-    "noEmit": false,
-    "outDir": "dist",
-    "rootDir": "src",
-    "sourceMap": true,
-    "declaration": true,
-    "declarationMap": true
-  },
-  "include": ["src/**/*.ts"],
-  "exclude": ["src/**/*.test.ts", "src/review_dist"]
-}

package/tsconfig.json

@@ -1,21 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
-    "types": ["node", "bun"],
-    "strict": true,
-    "noUncheckedIndexedAccess": true,
-    "exactOptionalPropertyTypes": true,
-    "noImplicitOverride": true,
-    "noFallthroughCasesInSwitch": true,
-    "noEmit": true,
-    "resolveJsonModule": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "skipLibCheck": true
-  },
-  "include": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.ts"],
-  "exclude": ["review", "dist", "src/review_dist"]
-}