self-bench 0.3.6 → 0.3.8

package/src/reaudit-main.ts

@@ -1,198 +0,0 @@
-#!/usr/bin/env node
-
-import { access, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { basename, dirname, join, resolve } from "node:path";
-import { parseArgs } from "node:util";
-import {
-  COUPLING_REVIEW_MODEL,
-  couplingReviewInput,
-  couplingReviewSchema,
-} from "./codex-review.js";
-import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
-import {
-  buildCouplingEvidence,
-  discoverContractArtifacts,
-  resolveCouplingReview,
-  scanBaseContractArtifacts,
-} from "./coupling.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import { projectRoot } from "./project-paths.js";
-import { createSandboxExecutor, type SandboxExecutor } from "./sandbox/index.js";
-import { loadPiModelAuth } from "./subscription-auth.js";
-
-const parsed = parseArgs({
-  options: {
-    tasks: { type: "string" },
-    task: { type: "string" },
-    output: { type: "string" },
-    concurrency: { type: "string" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Re-audit expanded Harbor tasks for test-to-gold coupling.
-
-Usage:
-  self-bench-reaudit --tasks DIRECTORY --output REPORT.json [options]
-
-Options:
-  --concurrency N  Concurrent Sol reviews (default: 10; Vercel: worker setting, default 4)
-  --task ID        Review only one task ID
-  -h, --help       Show this help`);
-  process.exit(0);
-}
-
-const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
-const outputPath = resolve(parsed.values.output ?? fail("--output is required"));
-const config = loadWorkerConfig();
-const concurrency = positiveInteger(
-  parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 10)),
-  "--concurrency",
-);
-const sandbox = createSandboxExecutor(config.execution);
-const [taskIds, modelAuth, reviewer] = await Promise.all([
-  readdir(tasksDirectory),
-  loadPiModelAuth(),
-  readFile(join(assetRoot(), "dist/sandbox-review.bundle.js")),
-]);
-const directories = (
-  await Promise.all(
-    taskIds.map(async (taskId) => {
-      const taskDirectory = join(tasksDirectory, taskId);
-      return await access(join(taskDirectory, "instruction.md")).then(
-        () => taskDirectory,
-        () => undefined,
-      );
-    }),
-  )
-)
-  .filter((directory): directory is string => directory !== undefined)
-  .filter(
-    (directory) => parsed.values.task === undefined || basename(directory) === parsed.values.task,
-  )
-  .sort();
-if (directories.length === 0) {
-  throw new Error(parsed.values.task ? `task not found: ${parsed.values.task}` : "no tasks found");
-}
-
-const tasks = await parallelMap(directories, concurrency, async (taskDirectory) => {
-  const taskId = basename(taskDirectory);
-  console.error(`reviewing ${taskId}`);
-  try {
-    const report = await auditTask(taskDirectory, taskId, modelAuth, reviewer, sandbox);
-    console.error(`${taskId}: ${report.resolution.verdict}`);
-    return report;
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    console.error(`${taskId}: error: ${message}`);
-    return { taskId, status: "error" as const, error: message };
-  }
-});
-sandbox.close();
-const report = {
-  schemaVersion: 1,
-  reviewer: COUPLING_REVIEW_MODEL,
-  generatedAt: new Date().toISOString(),
-  sourceTasks: tasksDirectory,
-  provisional: true,
-  tasks,
-  summary: {
-    total: tasks.length,
-    clean: tasks.filter((task) => task.status === "reviewed" && task.resolution.verdict === "clean")
-      .length,
-    coupled: tasks.filter(
-      (task) => task.status === "reviewed" && task.resolution.verdict === "coupled",
-    ).length,
-    errors: tasks.filter((task) => task.status === "error").length,
-  },
-};
-await mkdir(dirname(outputPath), { recursive: true });
-await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`);
-console.log(JSON.stringify({ output: outputPath, ...report.summary }, null, 2));
-
-async function auditTask(
-  taskDirectory: string,
-  taskId: string,
-  modelAuth: { apiKey?: string; authJson?: string },
-  reviewer: Uint8Array,
-  sandbox: SandboxExecutor,
-) {
-  const scratch = await mkdtemp(join(tmpdir(), "selfbench-reaudit-"));
-  try {
-    const baseDirectory = join(scratch, "base");
-    const baseArchivePath = join(taskDirectory, "environment/repo.tar.gz");
-    await mkdir(baseDirectory);
-    await runCommand("tar", ["-xzf", baseArchivePath, "-C", baseDirectory]);
-    const [prompt, testPatch, goldPatch] = await Promise.all([
-      readFile(join(taskDirectory, "instruction.md"), "utf8"),
-      readFile(join(taskDirectory, "tests/test.patch"), "utf8"),
-      readFile(join(taskDirectory, "solution/gold.patch"), "utf8"),
-    ]);
-    const baseArtifacts = await scanBaseContractArtifacts(
-      baseDirectory,
-      scratch,
-      discoverContractArtifacts(testPatch),
-    );
-    const couplingEvidence = buildCouplingEvidence({
-      prompt,
-      testPatch,
-      goldPatch,
-      baseArtifacts,
-    });
-    const result = await sandbox.run({
-      runId: "selfbench-reaudit-v2",
-      stage: taskId,
-      timeoutMs: 15 * 60 * 1000,
-      files: [
-        { path: "/work/sandbox-review.js", contents: reviewer },
-        {
-          path: "/work/review-input.md",
-          contents: couplingReviewInput(prompt, testPatch, goldPatch, couplingEvidence),
-        },
-      ],
-      outputPaths: ["/work/review.json"],
-      secrets: {
-        ...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
-        ...(modelAuth.authJson ? { SELFBENCH_PI_AUTH_JSON: modelAuth.authJson } : {}),
-      },
-      environment: { SELFBENCH_REVIEW_OUTPUT: "/work/review.json" },
-      command: ["node", "/work/sandbox-review.js"],
-    });
-    const output = result.outputs["/work/review.json"];
-    if (result.exitCode !== 0 || !output) {
-      throw new Error(
-        `sandboxed coupling review failed in ${result.sandboxId}: ${result.stderr.trim() || result.stdout.trim()}`,
-      );
-    }
-    const review = couplingReviewSchema.parse(JSON.parse(Buffer.from(output).toString("utf8")));
-    return {
-      taskId,
-      status: "reviewed" as const,
-      sandboxId: result.sandboxId,
-      couplingEvidence,
-      review,
-      resolution: resolveCouplingReview(couplingEvidence, review),
-    };
-  } finally {
-    await rm(scratch, { recursive: true, force: true });
-  }
-}
-
-function assetRoot(): string {
-  return projectRoot(import.meta.url);
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const parsed = Number(value);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return parsed;
-}
-
-function fail(message: string): never {
-  throw new Error(message);
-}

package/src/repair-main.ts

@@ -1,209 +0,0 @@
-#!/usr/bin/env node
-
-import { access, cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { basename, join, resolve } from "node:path";
-import { parseArgs } from "node:util";
-import { defaultStandaloneConcurrency, loadWorkerConfig } from "./config.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import { projectRoot } from "./project-paths.js";
-import { createSandboxExecutor } from "./sandbox/index.js";
-import { loadCodexModelAuth } from "./subscription-auth.js";
-
-const parsed = parseArgs({
-  options: {
-    tasks: { type: "string" },
-    review: { type: "string" },
-    output: { type: "string" },
-    task: { type: "string" },
-    concurrency: { type: "string" },
-    model: { type: "string", default: "gpt-5.6-sol" },
-    help: { type: "boolean", short: "h" },
-  },
-  strict: true,
-});
-if (parsed.values.help) {
-  console.log(`Repair coupled tests in expanded Harbor tasks.
-
-Usage:
-  self-bench-repair --tasks DIRECTORY --review REPORT.json --output DIRECTORY [options]
-
-Options:
-  --concurrency N  Concurrent repair sandboxes (default: 9; Vercel: worker setting, default 4)
-  --model MODEL    Codex subscription model (default: gpt-5.6-sol)
-  --task ID        Repair only one coupled task
-  -h, --help       Show this help`);
-  process.exit(0);
-}
-
-const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
-const reviewPath = resolve(parsed.values.review ?? fail("--review is required"));
-const outputDirectory = resolve(parsed.values.output ?? fail("--output is required"));
-if (outputDirectory === tasksDirectory || outputDirectory.startsWith(`${tasksDirectory}/`)) {
-  throw new Error("--output must be outside --tasks");
-}
-const config = loadWorkerConfig();
-const concurrency = positiveInteger(
-  parsed.values.concurrency ?? String(defaultStandaloneConcurrency(config, 9)),
-  "--concurrency",
-);
-const rawReview = JSON.parse(await readFile(reviewPath, "utf8")) as unknown;
-const review = reviewReport(rawReview);
-const coupled = new Map(
-  review.tasks
-    .filter((task) => task.resolution.verdict === "coupled")
-    .filter((task) => parsed.values.task === undefined || task.taskId === parsed.values.task)
-    .map((task) => [task.taskId, task]),
-);
-if (parsed.values.task !== undefined && coupled.size === 0) {
-  throw new Error(`coupled task not found in review: ${parsed.values.task}`);
-}
-const sourceTasks = (
-  await Promise.all(
-    (
-      await readdir(tasksDirectory)
-    ).map(async (taskId) => {
-      const directory = join(tasksDirectory, taskId);
-      return await access(join(directory, "instruction.md")).then(
-        () => directory,
-        () => undefined,
-      );
-    }),
-  )
-).filter((directory): directory is string => directory !== undefined);
-
-await rm(outputDirectory, { recursive: true, force: true });
-await mkdir(outputDirectory, { recursive: true });
-for (const source of sourceTasks) {
-  if (parsed.values.task === undefined && !coupled.has(basename(source))) {
-    await cp(source, join(outputDirectory, basename(source)), { recursive: true });
-  }
-}
-
-const sandbox = createSandboxExecutor(config.execution);
-const [modelAuth, repairer] = await Promise.all([
-  loadCodexModelAuth(),
-  readFile(join(assetRoot(), "dist/sandbox-repair.bundle.js")),
-]);
-const results = await parallelMap([...coupled.values()], concurrency, async (taskReview) => {
-  const taskId = taskReview.taskId;
-  const source = join(tasksDirectory, taskId);
-  console.error(`repairing ${taskId}`);
-  const scratch = await mkdtemp(join(tmpdir(), `selfbench-repair-${taskId}-`));
-  try {
-    const archive = join(scratch, "task.tar.gz");
-    await runCommand("tar", ["-czf", archive, "-C", source, "."]);
-    const result = await sandbox.run({
-      runId: "selfbench-repair-v1",
-      stage: taskId,
-      timeoutMs: 2 * 60 * 60 * 1000,
-      cpu: 4,
-      memoryMiB: 8192,
-      files: [
-        { path: "/work/task.tar.gz", contents: await readFile(archive) },
-        { path: "/work/review.json", contents: `${JSON.stringify(taskReview, null, 2)}\n` },
-        { path: "/work/sandbox-repair.js", contents: repairer },
-      ],
-      outputPaths: ["/work/repaired-task.tar.gz", "/work/repair-report.json"],
-      secrets: {
-        ...(modelAuth.apiKey ? { OPENAI_API_KEY: modelAuth.apiKey } : {}),
-        ...(modelAuth.authJson ? { SELFBENCH_CODEX_AUTH_JSON: modelAuth.authJson } : {}),
-      },
-      environment: { SELFBENCH_REPAIR_MODEL: parsed.values.model ?? "gpt-5.6-sol" },
-      command: [
-        "node",
-        "/work/sandbox-repair.js",
-        "/work/task.tar.gz",
-        "/work/review.json",
-        "/work/repaired-task.tar.gz",
-        "/work/repair-report.json",
-      ],
-    });
-    const repaired = result.outputs["/work/repaired-task.tar.gz"];
-    const report = result.outputs["/work/repair-report.json"];
-    if (result.exitCode !== 0 || !repaired || !report) {
-      throw new Error(
-        `repair sandbox ${result.sandboxId} failed: ${result.stderr.trim() || result.stdout.trim()}`,
-      );
-    }
-    const destination = join(outputDirectory, taskId);
-    await mkdir(destination);
-    const repairedArchive = join(scratch, "repaired-task.tar.gz");
-    await writeFile(repairedArchive, repaired);
-    await runCommand("tar", ["-xzf", repairedArchive, "-C", destination]);
-    const parsedReport = JSON.parse(Buffer.from(report).toString("utf8")) as Record<
-      string,
-      unknown
-    >;
-    console.error(`${taskId}: repaired in ${result.sandboxId}`);
-    return { taskId, status: "repaired" as const, sandboxId: result.sandboxId, ...parsedReport };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    console.error(`${taskId}: error: ${message}`);
-    return { taskId, status: "error" as const, error: message };
-  } finally {
-    await rm(scratch, { recursive: true, force: true });
-  }
-});
-sandbox.close();
-const reportOutput = {
-  schemaVersion: 1,
-  generatedAt: new Date().toISOString(),
-  sourceTasks: tasksDirectory,
-  sourceReview: reviewPath,
-  outputTasks: outputDirectory,
-  model: parsed.values.model,
-  copiedClean: parsed.values.task === undefined ? sourceTasks.length - coupled.size : 0,
-  repaired: results.filter((result) => result.status === "repaired").length,
-  errors: results.filter((result) => result.status === "error").length,
-  tasks: results,
-};
-await writeFile(
-  join(outputDirectory, "repair-report.json"),
-  `${JSON.stringify(reportOutput, null, 2)}\n`,
-);
-console.log(JSON.stringify(reportOutput, null, 2));
-
-interface ReviewTask {
-  readonly taskId: string;
-  readonly resolution: { readonly verdict: "clean" | "coupled" };
-  readonly [key: string]: unknown;
-}
-
-function reviewReport(value: unknown): { readonly tasks: readonly ReviewTask[] } {
-  if (!isRecord(value) || !Array.isArray(value.tasks)) {
-    throw new Error("invalid coupling review report");
-  }
-  const tasks = value.tasks.filter(
-    (task): task is ReviewTask =>
-      isRecord(task) &&
-      typeof task.taskId === "string" &&
-      isRecord(task.resolution) &&
-      (task.resolution.verdict === "clean" || task.resolution.verdict === "coupled"),
-  );
-  if (tasks.length !== value.tasks.length) {
-    throw new Error("coupling review report contains invalid task entries");
-  }
-  return { tasks };
-}
-
-function assetRoot(): string {
-  return projectRoot(import.meta.url);
-}
-
-function positiveInteger(value: string | undefined, label: string): number {
-  const number = Number(value);
-  if (!Number.isInteger(number) || number < 1) {
-    throw new Error(`${label} must be a positive integer`);
-  }
-  return number;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function fail(message: string): never {
-  throw new Error(message);
-}

package/src/repair.ts

@@ -1,55 +0,0 @@
-export function patchPaths(patch: string): readonly string[] {
-  const paths = new Set<string>();
-  for (const line of patch.split("\n")) {
-    const match = /^diff --git a\/(.+) b\/(.+)$/.exec(line);
-    if (match?.[2]) {
-      paths.add(match[2]);
-    }
-  }
-  return [...paths].sort();
-}
-
-export function assertRepairPaths(
-  originalTestPatch: string,
-  changedPaths: readonly string[],
-): void {
-  const allowed = new Set(patchPaths(originalTestPatch));
-  if (allowed.size === 0) {
-    throw new Error("original held-out test patch changes no files");
-  }
-  const outside = changedPaths.filter((path) => !allowed.has(path));
-  if (outside.length > 0) {
-    throw new Error(`repair changed files outside the held-out tests: ${outside.join(", ")}`);
-  }
-}
-
-export function repairPrompt(input: {
-  readonly taskId: string;
-  readonly authenticRequest: string;
-  readonly couplingReport: string;
-  readonly allowedPaths: readonly string[];
-}): string {
-  return `Repair the held-out tests for SelfBench task ${input.taskId}.
-
-The repository is the exact base snapshot with the current held-out test patch already applied. You may edit only these existing test files:
-
-${input.allowedPaths.map((path) => `- ${path}`).join("\n")}
-
-The authentic engineer request is:
-
-<authentic_request>
-${input.authenticRequest.trim()}
-</authentic_request>
-
-The independent coupling report is:
-
-<coupling_report>
-${input.couplingReport.trim()}
-</coupling_report>
-
-Rewrite the tests so they verify every material requested behavior through stable public boundaries without requiring the gold patch's exact internal names, helper structure, error prose, mock input shape, response presentation, or incidental implementation choices. Preserve meaningful negative, authorization, compatibility, and regression coverage. Do not edit application code, the request, or the gold patch. Do not delete assertions merely to silence the report. A coherent alternative implementation must be able to pass, while the unchanged base must still fail the fail-to-pass tests and the gold implementation must still pass.
-
-The final working tree must retain a non-empty test patch relative to HEAD and every originally added fail-to-pass test must still have an equivalent behavioral test. Do not reset, revert, delete, or commit the test patch. If an exact public response field is the only stable seam, prefer asserting the underlying externally visible behavior through existing endpoints, persistence, or follow-up actions. If no rigorous uncoupled test is possible within the allowed files, leave the current tests in place and explain the blocker in your final message instead of removing coverage.
-
-Inspect /work/task/solution/gold.patch only to understand the intended behavior and available seams, never to copy its private structure into assertions. Inspect /work/task/tests/test.sh for the verifier command. Run focused tests when feasible. Finish only after the working tree contains the repaired test files and no other changes.`;
-}

package/src/run-wait.ts

@@ -1,40 +0,0 @@
-export interface PolledRunStatus {
-  readonly phase: string;
-  readonly error?: unknown;
-  readonly [key: string]: unknown;
-}
-
-export interface WaitForRunOptions {
-  readonly poll: () => Promise<PolledRunStatus>;
-  readonly onPhase?: (status: PolledRunStatus) => void;
-  readonly intervalMs?: number;
-  readonly delay?: (milliseconds: number) => Promise<void>;
-}
-
-const FAILED_PHASES = new Set(["blocked", "failed", "cancelled"]);
-
-export async function waitForRun(options: WaitForRunOptions): Promise<PolledRunStatus> {
-  const intervalMs = options.intervalMs ?? 2_000;
-  const delay = options.delay ?? defaultDelay;
-  let previousPhase: string | undefined;
-
-  while (true) {
-    const status = await options.poll();
-    if (status.phase !== previousPhase) {
-      options.onPhase?.(status);
-      previousPhase = status.phase;
-    }
-    if (status.phase === "complete") {
-      return status;
-    }
-    if (FAILED_PHASES.has(status.phase)) {
-      const detail = typeof status.error === "string" ? `: ${status.error}` : "";
-      throw new Error(`SelfBench run ${status.phase}${detail}`);
-    }
-    await delay(intervalMs);
-  }
-}
-
-function defaultDelay(milliseconds: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, milliseconds));
-}

package/src/sandbox/contracts.ts

@@ -1,52 +0,0 @@
-export interface SandboxFile {
-  readonly path: string;
-  readonly contents: Uint8Array | string;
-}
-
-export interface SandboxRequest {
-  readonly runId: string;
-  readonly stage: string;
-  readonly command: readonly string[];
-  readonly files?: readonly SandboxFile[];
-  readonly outputPaths?: readonly string[];
-  readonly environment?: Readonly<Record<string, string>>;
-  readonly secrets?: Readonly<Record<string, string>>;
-  readonly timeoutMs: number;
-  readonly inactivityTimeoutMs?: number;
-  readonly cpu?: number;
-  readonly memoryMiB?: number;
-}
-
-export interface SandboxProgress {
-  readonly stream: "stdout" | "stderr";
-  readonly bytes: number;
-}
-
-export interface SandboxRunOptions {
-  readonly signal?: AbortSignal;
-  readonly onProgress?: (progress: SandboxProgress) => void;
-}
-
-export interface SandboxResult {
-  readonly sandboxId: string;
-  readonly exitCode: number;
-  readonly stdout: string;
-  readonly stderr: string;
-  readonly outputs: Readonly<Record<string, Uint8Array>>;
-}
-
-export class SandboxExecutionError extends Error {
-  constructor(
-    message: string,
-    readonly result: SandboxResult,
-    options?: ErrorOptions,
-  ) {
-    super(message, options);
-    this.name = "SandboxExecutionError";
-  }
-}
-
-export interface SandboxExecutor {
-  run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult>;
-  close(): void;
-}

package/src/sandbox/index.ts

@@ -1,23 +0,0 @@
-import type { SelfBenchWorkerConfig } from "../config.js";
-import { DockerSandboxExecutor } from "./providers/docker/executor.js";
-import { ModalSandboxExecutor } from "./providers/modal/executor.js";
-import { VercelSandboxExecutor } from "./providers/vercel/executor.js";
-import { TimeoutCappedSandboxExecutor } from "./timeout.js";
-
-export * from "./contracts.js";
-
-import type { SandboxExecutor } from "./contracts.js";
-
-export function createSandboxExecutor(config: SelfBenchWorkerConfig["execution"]): SandboxExecutor {
-  switch (config.kind) {
-    case "docker":
-      return new DockerSandboxExecutor(config);
-    case "modal":
-      return new ModalSandboxExecutor(config);
-    case "vercel":
-      return new TimeoutCappedSandboxExecutor(
-        new VercelSandboxExecutor(config),
-        config.timeoutCapMs,
-      );
-  }
-}

package/src/sandbox/programs/author.ts

@@ -1,19 +0,0 @@
-#!/usr/bin/env node
-
-import { readdir } from "node:fs/promises";
-import { join } from "node:path";
-import { compileHarborTask } from "../../harbor-task.js";
-
-const [tasksRoot, repositoryDirectory, outputDirectory] = process.argv.slice(2);
-if (!tasksRoot || !repositoryDirectory || !outputDirectory) {
-  throw new Error("usage: sandbox-author TASKS_ROOT REPOSITORY OUTPUT");
-}
-
-const entries = await readdir(tasksRoot, { withFileTypes: true });
-const taskDirectories = entries
-  .filter((entry) => entry.isDirectory())
-  .map((entry) => join(tasksRoot, entry.name));
-if (taskDirectories.length !== 1) {
-  throw new Error(`authoring must produce exactly one task; found ${taskDirectories.length}`);
-}
-await compileHarborTask(taskDirectories[0] as string, repositoryDirectory, outputDirectory);

package/src/sandbox/programs/prepare-repair.ts

@@ -1,44 +0,0 @@
-import { access, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { runCommand } from "../../process.js";
-
-export interface PreparedRepairTask {
-  readonly extractedDirectory: string;
-  readonly taskDirectory: string;
-  readonly repositoryDirectory: string;
-}
-
-export async function prepareRepairTask(
-  archivePath: string,
-  workDirectory = "/work",
-): Promise<PreparedRepairTask> {
-  const extractedDirectory = join(workDirectory, "task");
-  const repositoryDirectory = join(workDirectory, "repo");
-  await Promise.all([mkdir(extractedDirectory, { recursive: true }), mkdir(repositoryDirectory)]);
-  await runCommand("tar", ["-xzf", archivePath, "-C", extractedDirectory]);
-  const taskDirectory = await access(join(extractedDirectory, "instruction.md")).then(
-    () => extractedDirectory,
-    () => join(extractedDirectory, "harbor-task"),
-  );
-
-  await runCommand("tar", [
-    "-xzf",
-    join(taskDirectory, "tests/repo.tar.gz"),
-    "-C",
-    repositoryDirectory,
-  ]);
-  await runCommand("git", ["-C", repositoryDirectory, "init", "-q"]);
-  await runCommand("git", ["-C", repositoryDirectory, "config", "user.name", "SelfBench"]);
-  await runCommand("git", ["-C", repositoryDirectory, "config", "user.email", "selfbench@local"]);
-  await runCommand("git", ["-C", repositoryDirectory, "add", "-A"]);
-  await runCommand("git", ["-C", repositoryDirectory, "commit", "-qm", "base"]);
-  await runCommand("git", [
-    "-C",
-    repositoryDirectory,
-    "apply",
-    join(taskDirectory, "tests/test.patch"),
-  ]);
-  await runCommand("git", ["-C", repositoryDirectory, "add", "-N", "--all"]);
-
-  return { extractedDirectory, taskDirectory, repositoryDirectory };
-}