self-bench 0.3.6 → 0.3.8

package/dist/validate-main.d.ts

@@ -1,3 +0,0 @@
-#!/usr/bin/env node
-export {};
-//# sourceMappingURL=validate-main.d.ts.map

package/dist/validate-main.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"validate-main.d.ts","sourceRoot":"","sources":["../src/validate-main.ts"],"names":[],"mappings":""}

package/dist/validate-main.js

@@ -1,143 +0,0 @@
-#!/usr/bin/env node
-import { access, mkdir, readdir, writeFile } from "node:fs/promises";
-import { basename, join, resolve } from "node:path";
-import { parseArgs } from "node:util";
-import { harborChildEnvironment } from "./harbor-environment.js";
-import { harborInfrastructureError, readHarborJobResult } from "./harbor-results.js";
-import { parallelMap } from "./parallel.js";
-import { runCommand } from "./process.js";
-import { isHarborEnvironment } from "./providers.js";
-const parsed = parseArgs({
-    options: {
-        tasks: { type: "string" },
-        jobs: { type: "string" },
-        environment: { type: "string", default: "modal" },
-        concurrency: { type: "string", default: "10" },
-        help: { type: "boolean", short: "h" },
-    },
-    strict: true,
-});
-if (parsed.values.help) {
-    console.log(`Run Harbor nop and oracle gates for expanded SelfBench tasks.
-
-Usage:
-  self-bench-validate --tasks DIRECTORY --jobs DIRECTORY [options]
-
-Options:
-  --environment NAME  Harbor environment (default: modal)
-  --concurrency N     Concurrent tasks (default: 10)
-  -h, --help          Show this help`);
-    process.exit(0);
-}
-const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
-const jobsDirectory = resolve(parsed.values.jobs ?? fail("--jobs is required"));
-const environment = parsed.values.environment ?? "modal";
-if (!isHarborEnvironment(environment)) {
-    throw new Error("--environment must be modal or docker");
-}
-const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
-const directories = (await Promise.all((await readdir(tasksDirectory)).map(async (name) => {
-    const directory = join(tasksDirectory, name);
-    return await access(join(directory, "task.toml")).then(() => directory, () => undefined);
-}))).filter((directory) => directory !== undefined);
-await mkdir(jobsDirectory, { recursive: true });
-const tasks = await parallelMap(directories.sort(), concurrency, async (taskDirectory) => {
-    const taskId = basename(taskDirectory);
-    console.error(`validating ${taskId}`);
-    try {
-        const nop = await runGate(taskDirectory, taskId, "nop", environment, jobsDirectory);
-        const oracle = await runGate(taskDirectory, taskId, "oracle", environment, jobsDirectory);
-        const accepted = nop.passed && oracle.passed;
-        console.error(`${taskId}: ${accepted ? "accepted" : "rejected"}`);
-        return { taskId, status: "validated", accepted, nop, oracle };
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        console.error(`${taskId}: error: ${message}`);
-        return { taskId, status: "error", accepted: false, error: message };
-    }
-});
-const report = {
-    schemaVersion: 1,
-    generatedAt: new Date().toISOString(),
-    sourceTasks: tasksDirectory,
-    environment,
-    summary: {
-        total: tasks.length,
-        accepted: tasks.filter((task) => task.accepted).length,
-        rejected: tasks.filter((task) => task.status === "validated" && !task.accepted).length,
-        errors: tasks.filter((task) => task.status === "error").length,
-    },
-    tasks,
-};
-await writeFile(join(jobsDirectory, "validation-summary.json"), `${JSON.stringify(report, null, 2)}\n`);
-console.log(JSON.stringify(report.summary, null, 2));
-async function runGate(taskDirectory, taskId, agent, environment, jobsDirectory) {
-    const jobName = `${taskId}-${agent}-${crypto.randomUUID().slice(0, 8)}`;
-    const process = await runCommand("harbor", [
-        "run",
-        "--path",
-        taskDirectory,
-        "--agent",
-        agent,
-        "--env",
-        environment,
-        "--job-name",
-        jobName,
-        "--jobs-dir",
-        jobsDirectory,
-        "--n-concurrent",
-        "1",
-        "--max-retries",
-        "1",
-        "--delete",
-        "--yes",
-        "--quiet",
-    ], {
-        allowFailure: true,
-        env: harborChildEnvironment(),
-        timeoutMs: 4 * 60 * 60 * 1000,
-    });
-    if (process.exitCode !== 0) {
-        throw new Error(`Harbor ${agent} exited ${process.exitCode}: ${process.stderr.slice(-1_000)}`);
-    }
-    const result = await readHarborJobResult(jobsDirectory, jobName);
-    const infrastructure = harborInfrastructureError(result.trial);
-    if (infrastructure) {
-        throw new Error(`Harbor ${agent} infrastructure failure: ${infrastructure}`);
-    }
-    const rewards = rewardValues(result.trial);
-    const passed = agent === "nop"
-        ? rewards.fail_to_pass === 0 &&
-            (rewards.pass_to_pass ?? 0) >= 1 &&
-            (rewards.setup_completed ?? 0) >= 1
-        : (rewards.patch_applied ?? 0) >= 1 &&
-            (rewards.fail_to_pass ?? 0) >= 1 &&
-            (rewards.pass_to_pass ?? 0) >= 1 &&
-            (rewards.deterministic ?? 0) >= 1 &&
-            (rewards.setup_completed ?? 0) >= 1;
-    return { passed, rewards };
-}
-function rewardValues(value) {
-    if (!isRecord(value) || !isRecord(value.verifier_result)) {
-        return {};
-    }
-    const raw = value.verifier_result.rewards;
-    return isRecord(raw)
-        ? Object.fromEntries(Object.entries(raw).filter((entry) => typeof entry[1] === "number"))
-        : {};
-}
-function positiveInteger(value, label) {
-    const number = Number(value);
-    if (!Number.isInteger(number) || number < 1) {
-        throw new Error(`${label} must be a positive integer`);
-    }
-    return number;
-}
-function isRecord(value) {
-    return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function fail(message) {
-    throw new Error(message);
-}
-//# sourceMappingURL=validate-main.js.map

package/dist/validate-main.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"validate-main.js","sourceRoot":"","sources":["../src/validate-main.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAA0B,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAE7E,MAAM,MAAM,GAAG,SAAS,CAAC;IACvB,OAAO,EAAE;QACP,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE;QACjD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;QAC9C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACtC;IACD,MAAM,EAAE,IAAI;CACb,CAAC,CAAC;AACH,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,OAAO,CAAC,GAAG,CAAC;;;;;;;;qCAQuB,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;AACnF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;AAChF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC;AACzD,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC;IACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;AAC3D,CAAC;AACD,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;AAChF,MAAM,WAAW,GAAG,CAClB,MAAM,OAAO,CAAC,GAAG,CACf,CACE,MAAM,OAAO,CAAC,cAAc,CAAC,CAC9B,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACnB,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IAC7C,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,IAAI,CACpD,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;AACJ,CAAC,CAAC,CACH,CACF,CAAC,MAAM,CAAC,CAAC,SAAS,EAAuB,EAAE,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;AACtE,MAAM,KAAK,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAEhD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE;IACvF,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC;IACvC,OAAO,CAAC,KAAK,CAAC,cAAc,MAAM,EAAE,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;QACpF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;QAClE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,WAAoB,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IACzE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,YAAY,OAAO,EAAE,CAAC,CAAC;QAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAgB,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC/E,CAAC;AACH,CAAC,CAAC,CAAC;AACH,MAAM,MAAM,GAAG;IACb,aAAa,EAAE,CAAC;IAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IACrC,WAAW,EAAE,cAAc;IAC3B,WAAW;IACX,OAAO,EAAE;QACP,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM;QACtD,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM;QACtF,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;KAC/D;IACD,KAAK;CACN,CAAC;AACF,MAAM,SAAS,CACb,IAAI,CAAC,aAAa,EAAE,yBAAyB,CAAC,EAC9C,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACvC,CAAC;AACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAErD,KAAK,UAAU,OAAO,CACpB,aAAqB,EACrB,MAAc,EACd,KAAuB,EACvB,WAA8B,EAC9B,aAAqB;IAErB,MAAM,OAAO,GAAG,GAAG,MAAM,IAAI,KAAK,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IACxE,MAAM,OAAO,GAAG,MAAM,UAAU,CAC9B,QAAQ,EACR;QACE,KAAK;QACL,QAAQ;QACR,aAAa;QACb,SAAS;QACT,KAAK;QACL,OAAO;QACP,WAAW;QACX,YAAY;QACZ,OAAO;QACP,YAAY;QACZ,aAAa;QACb,gBAAgB;QAChB,GAAG;QACH,eAAe;QACf,GAAG;QACH,UAAU;QACV,OAAO;QACP,SAAS;KACV,EACD;QACE,YAAY,EAAE,IAAI;QAClB,GAAG,EAAE,sBAAsB,EAAE;QAC7B,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;KAC9B,CACF,CAAC;IACF,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,WAAW,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IACjE,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,4BAA4B,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GACV,KAAK,KAAK,KAAK;QACb,CAAC,CAAC,OAAO,CAAC,YAAY,KAAK,CAAC;YAC1B,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC;YAChC,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,IAAI,CAAC;QACrC,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,CAAC;YACjC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC;YAChC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC;YAChC,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,CAAC;YACjC,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;QACzD,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,eAAe,CAAC,OAAO,CAAC;IAC1C,OAAO,QAAQ,CAAC,GAAG,CAAC;QAClB,CAAC,CAAC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CACxB,CAAC,KAAK,EAA6B,EAAE,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,CACnE,CACF;QACH,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB,EAAE,KAAa;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,IAAI,CAAC,OAAe;IAC3B,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;AAC3B,CAAC"}

package/dist/validation-repair.d.ts

@@ -1,10 +0,0 @@
-import type { TaskDefinition } from "./contracts.js";
-export declare function validationRepairPaths(originalTestPatch: string): readonly string[];
-export declare function assertValidationRepair(original: TaskDefinition, repaired: TaskDefinition, originalTestPatch: string, changedPaths: readonly string[]): void;
-export declare function validationRepairPrompt(input: {
-    readonly definition: TaskDefinition;
-    readonly authenticRequest: string;
-    readonly diagnostics: string;
-    readonly allowedPaths: readonly string[];
-}): string;
-//# sourceMappingURL=validation-repair.d.ts.map

package/dist/validation-repair.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"validation-repair.d.ts","sourceRoot":"","sources":["../src/validation-repair.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGrD,wBAAgB,qBAAqB,CAAC,iBAAiB,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAUlF;AAcD,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,cAAc,EACxB,QAAQ,EAAE,cAAc,EACxB,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,SAAS,MAAM,EAAE,GAC9B,IAAI,CAwBN;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;IACpC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C,GAAG,MAAM,CA8BT"}

package/dist/validation-repair.js

@@ -1,72 +0,0 @@
-import { patchPaths } from "./repair.js";
-export function validationRepairPaths(originalTestPatch) {
-    const paths = patchPaths(originalTestPatch);
-    if (paths.length === 0) {
-        throw new Error("original held-out test patch changes no files");
-    }
-    const nonTestPaths = paths.filter((path) => !isTestOnlyPath(path));
-    if (nonTestPaths.length > 0) {
-        throw new Error(`validation repair requires test-only patch paths: ${nonTestPaths.join(", ")}`);
-    }
-    return paths;
-}
-function isTestOnlyPath(path) {
-    const segments = path.toLowerCase().split("/");
-    const filename = segments.at(-1) ?? "";
-    return (segments.some((segment) => /^(?:tests?|__tests__|e2e|integration|fixtures?|selfbench(?:-tests|_tests)?|\.selfbench-tests)$/.test(segment)) || /(?:^|\.)(?:test|spec)\.[^.]+$/.test(filename));
-}
-export function assertValidationRepair(original, repaired, originalTestPatch, changedPaths) {
-    const immutableKeys = [
-        "schemaVersion",
-        "difficulty",
-        "taskId",
-        "repo",
-        "baseCommit",
-        "workdir",
-        "sourcePr",
-        "sourceUrl",
-        "prompt",
-    ];
-    for (const key of immutableKeys) {
-        if (JSON.stringify(repaired[key]) !== JSON.stringify(original[key])) {
-            throw new Error(`validation repair changed immutable definition field ${key}`);
-        }
-    }
-    const allowed = new Set(validationRepairPaths(originalTestPatch));
-    const outside = changedPaths.filter((path) => !allowed.has(path));
-    if (outside.length > 0) {
-        throw new Error(`validation repair changed files outside held-out tests: ${outside.join(", ")}`);
-    }
-}
-export function validationRepairPrompt(input) {
-    return `Repair the verifier harness and held-out tests for SelfBench task ${input.definition.taskId}.
-
-The repository is the exact base snapshot with the current held-out test patch already applied. You may edit only these existing held-out test files:
-
-${input.allowedPaths.map((path) => `- ${path}`).join("\n")}
-
-The authentic engineer request is:
-
-<authentic_request>
-${input.authenticRequest.trim()}
-</authentic_request>
-
-The failed nop/oracle validation diagnostics are:
-
-<validation_diagnostics>
-${input.diagnostics.trim()}
-</validation_diagnostics>
-
-Fix the task so its repository-native verifier is rigorous and runnable. You may edit the allowed held-out tests and /work/definition.json. In definition.json you may change only setupCommand, testCommand, failToPass, passToPass, testPaths, toolchains, timeouts, and resources. Never change task identity, difficulty, repository, base commit, workdir, source pull request, prompt, or reference solution.
-
-The required validation split is:
-- nop: the base repository plus held-out tests must make failToPass fail while passToPass succeeds;
-- oracle: after /work/task/solution/gold.patch is applied, failToPass and passToPass must both succeed deterministically.
-
-Use the verifier diagnostics rather than guessing. Inspect repository package scripts and CI for the native test command. Keep {tests} as a list-safe placeholder; do not quote the whole placeholder, assign it to one scalar, or hard-code selected paths elsewhere. Use one test mode/bundler per command instead of chaining equivalent suites. Put dependency installation, native builds, fixture generation, and browser/runtime installation in setupCommand, not testCommand. Pin the repository's declared package manager and use frozen installs. Browser-backed tests must install required browser binaries and OS dependencies during setup. Run focused nop/oracle-style commands in the sandbox when feasible.
-
-Do not weaken, delete, or skip requested behavioral coverage merely to make commands exit zero. A passing base with no solution is not acceptable. If no rigorous runnable split exists, leave the files and definition unchanged and explain why in the final response.
-
-Inspect /work/gold.patch and /work/test.sh only to verify the intended behavior and oracle seam, never to couple assertions to private implementation details. Do not modify application code, the instruction, gold patch, or commit files. Finish with the repaired definition at /work/definition.json and any held-out test edits present in the working tree; do not commit.`;
-}
-//# sourceMappingURL=validation-repair.js.map

package/dist/validation-repair.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"validation-repair.js","sourceRoot":"","sources":["../src/validation-repair.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,UAAU,qBAAqB,CAAC,iBAAyB;IAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,qDAAqD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,OAAO,CACL,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CACxB,gGAAgG,CAAC,IAAI,CACnG,OAAO,CACR,CACF,IAAI,+BAA+B,CAAC,IAAI,CAAC,QAAQ,CAAC,CACpD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,QAAwB,EACxB,QAAwB,EACxB,iBAAyB,EACzB,YAA+B;IAE/B,MAAM,aAAa,GAAG;QACpB,eAAe;QACf,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,YAAY;QACZ,SAAS;QACT,UAAU;QACV,WAAW;QACX,QAAQ;KACA,CAAC;IACX,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,wDAAwD,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAClE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2DAA2D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAKtC;IACC,OAAO,qEAAqE,KAAK,CAAC,UAAU,CAAC,MAAM;;;;EAInG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;EAKxD,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE;;;;;;EAM7B,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE;;;;;;;;;;;;;kXAawV,CAAC;AACnX,CAAC"}

package/review/index.html

@@ -1,14 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="color-scheme" content="light dark" />
-    <meta name="theme-color" content="#f4f5f7" />
-    <title>SelfBench Review</title>
-  </head>
-  <body>
-    <div id="root"></div>
-    <script type="module" src="/src/main.tsx"></script>
-  </body>
-</html>

package/review/src/App.tsx

@@ -1,338 +0,0 @@
-import { parsePatchFiles } from "@pierre/diffs";
-import { FileDiff } from "@pierre/diffs/react";
-import React from "react";
-import { buildCleanExport, loadExport } from "./export-loader";
-import type { ExportTask, LoadedExport, RunStatus, RunSummary } from "./types";
-
-export function App() {
-  const [runs, setRuns] = React.useState<RunSummary[]>([]);
-  const [runId, setRunId] = React.useState<string | null>(null);
-  const [status, setStatus] = React.useState<RunStatus | null>(null);
-  const [loaded, setLoaded] = React.useState<LoadedExport | null>(null);
-  const [removed, setRemoved] = React.useState<Set<string>>(new Set());
-  const [selectedTaskId, setSelectedTaskId] = React.useState<string | null>(null);
-  const [error, setError] = React.useState<string | null>(null);
-  const [busy, setBusy] = React.useState(false);
-  const [token, setToken] = React.useState("");
-  const fileInput = React.useRef<HTMLInputElement>(null);
-  const apiFetch = React.useCallback(
-    (path: string) =>
-      fetch(path, token ? { headers: { Authorization: `Bearer ${token}` } } : undefined),
-    [token],
-  );
-
-  const refreshRuns = React.useCallback(async () => {
-    try {
-      setError(null);
-      const response = await apiFetch("/v1/runs");
-      if (!response.ok) throw new Error(`Unable to list runs (${response.status})`);
-      setRuns((await response.json()) as RunSummary[]);
-    } catch (cause) {
-      setError(cause instanceof Error ? cause.message : String(cause));
-    }
-  }, [apiFetch]);
-
-  React.useEffect(() => {
-    void refreshRuns();
-  }, [refreshRuns]);
-
-  const openRun = async (nextRunId: string) => {
-    try {
-      setBusy(true);
-      setError(null);
-      setRunId(nextRunId);
-      setLoaded(null);
-      setRemoved(new Set());
-      const [statusResponse, exportResponse] = await Promise.all([
-        apiFetch(`/v1/runs/${encodeURIComponent(nextRunId)}`),
-        apiFetch(`/v1/runs/${encodeURIComponent(nextRunId)}/export`),
-      ]);
-      if (!statusResponse.ok) throw new Error(`Unable to load run (${statusResponse.status})`);
-      if (!exportResponse.ok) throw new Error(`Export is not ready (${exportResponse.status})`);
-      setStatus((await statusResponse.json()) as RunStatus);
-      setLoaded(await loadExport(await exportResponse.blob()));
-    } catch (cause) {
-      setError(cause instanceof Error ? cause.message : String(cause));
-    } finally {
-      setBusy(false);
-    }
-  };
-
-  const openFile = async (file: File) => {
-    try {
-      setBusy(true);
-      setError(null);
-      const next = await loadExport(file);
-      setLoaded(next);
-      setRunId(next.manifest.runId);
-      setStatus({
-        runId: next.manifest.runId,
-        phase: "local export",
-        requested: next.manifest.acceptedCount,
-        accepted: next.manifest.acceptedCount,
-        rejected: 0,
-        tasks: [],
-      });
-      setRemoved(new Set());
-      setSelectedTaskId(null);
-    } catch (cause) {
-      setError(cause instanceof Error ? cause.message : String(cause));
-    } finally {
-      setBusy(false);
-    }
-  };
-
-  const selectedTask =
-    loaded?.tasks.find((task) => task.taskId === selectedTaskId) ?? loaded?.tasks[0];
-  const toggleRemoved = (taskId: string) =>
-    setRemoved((current) => {
-      const next = new Set(current);
-      if (next.has(taskId)) next.delete(taskId);
-      else next.add(taskId);
-      return next;
-    });
-
-  const downloadClean = async () => {
-    if (!loaded || !runId) return;
-    const blob = await buildCleanExport(loaded, removed);
-    const url = URL.createObjectURL(blob);
-    const link = document.createElement("a");
-    link.href = url;
-    link.download = `selfbench-${runId}-cleaned.tar.gz`;
-    link.click();
-    URL.revokeObjectURL(url);
-  };
-
-  return (
-    <main className="app-shell">
-      <header className="topbar">
-        <div>
-          <p className="eyebrow">SELFBENCH / EXPORT REVIEW</p>
-          <h1>Review accepted tasks before publishing.</h1>
-        </div>
-        <div className="topbar-actions">
-          <input
-            className="token-input"
-            type="password"
-            value={token}
-            onChange={(event) => setToken(event.target.value)}
-            placeholder="API token (session only)"
-            aria-label="API token"
-          />
-          <input
-            ref={fileInput}
-            hidden
-            type="file"
-            accept=".gz,.tgz,application/gzip"
-            onChange={(event) => {
-              const file = event.target.files?.[0];
-              if (file) void openFile(file);
-            }}
-          />
-          <button type="button" className="button" onClick={() => fileInput.current?.click()}>
-            Open gzip
-          </button>
-          <button type="button" className="button" onClick={() => void refreshRuns()}>
-            Refresh runs
-          </button>
-        </div>
-      </header>
-      {error && <div className="error-banner">{error}</div>}
-      <section className="workspace">
-        <aside className="run-list surface">
-          <div className="section-heading">
-            <h2>Runs</h2>
-            <span>{runs.length}</span>
-          </div>
-          {runs.length === 0 && <p className="muted">No runs found.</p>}
-          {runs.map((run) => (
-            <button
-              type="button"
-              className={`run-row ${run.runId === runId ? "selected" : ""}`}
-              key={run.runId}
-              onClick={() => void openRun(run.runId)}
-            >
-              <strong>{run.runId}</strong>
-              <span>{run.status}</span>
-              <small>{new Date(run.startedAt).toLocaleString()}</small>
-            </button>
-          ))}
-        </aside>
-        <section className="content">
-          {!loaded && (
-            <div className="empty surface">
-              <h2>{busy ? "Loading export…" : "Select a completed run"}</h2>
-              <p>Choose a run or open a local export gzip to inspect its accepted tasks.</p>
-            </div>
-          )}
-          {loaded && status && (
-            <>
-              <div className="summary surface">
-                <div>
-                  <p className="eyebrow">{loaded.manifest.repository.url}</p>
-                  <h2>{runId}</h2>
-                  <p className="muted">
-                    {status.phase} · {loaded.manifest.acceptedCount} accepted tasks · commit{" "}
-                    {loaded.manifest.repository.commit.slice(0, 12)}
-                  </p>
-                </div>
-                <button
-                  type="button"
-                  className="button primary"
-                  onClick={() => void downloadClean()}
-                >
-                  {removed.size
-                    ? `Download cleaned export (${loaded.tasks.length - removed.size})`
-                    : "Download export copy"}
-                </button>
-              </div>
-              <div className="review-grid">
-                <div className="task-list surface">
-                  <div className="section-heading">
-                    <h2>Tasks</h2>
-                    <span>{removed.size} marked for removal</span>
-                  </div>
-                  {loaded.tasks.map((task) => (
-                    <button
-                      type="button"
-                      className={`task-row ${task.taskId === selectedTask?.taskId ? "selected" : ""} ${removed.has(task.taskId) ? "removed" : ""}`}
-                      key={task.taskId}
-                      onClick={() => setSelectedTaskId(task.taskId)}
-                    >
-                      <strong>{task.taskId}</strong>
-                      <span>{removed.has(task.taskId) ? "marked for removal" : "keep"}</span>
-                    </button>
-                  ))}
-                </div>
-                {selectedTask && (
-                  <TaskPanel
-                    task={selectedTask}
-                    removed={removed.has(selectedTask.taskId)}
-                    onToggle={() => toggleRemoved(selectedTask.taskId)}
-                  />
-                )}
-              </div>
-            </>
-          )}
-        </section>
-      </section>
-    </main>
-  );
-}
-
-function TaskPanel({
-  task,
-  removed,
-  onToggle,
-}: {
-  task: ExportTask;
-  removed: boolean;
-  onToggle: () => void;
-}) {
-  const definition = parseJson(task.textFiles.get("definition.json"));
-  return (
-    <article className="task-detail surface">
-      <div className="detail-header">
-        <div>
-          <p className="eyebrow">TASK</p>
-          <h2>{task.taskId}</h2>
-          <p className="muted">
-            {String(definition?.difficulty ?? "unknown")} · {String(definition?.repo ?? "")}
-          </p>
-        </div>
-        <button
-          type="button"
-          className={`button ${removed ? "undo" : "danger"}`}
-          onClick={onToggle}
-        >
-          {removed ? "Keep task" : "Mark for removal"}
-        </button>
-      </div>
-      <DetailBlock title="Prompt">
-        <pre>
-          {String(definition?.prompt ?? task.textFiles.get("instruction.md") ?? "No prompt")}
-        </pre>
-      </DetailBlock>
-      <DetailBlock title="Task definition">
-        <pre>{task.textFiles.get("definition.json") ?? "Missing definition.json"}</pre>
-      </DetailBlock>
-      <DetailBlock title="Gold patch">
-        <PatchBlock patch={task.textFiles.get("solution/gold.patch")} />
-      </DetailBlock>
-      <DetailBlock title="Held-out test patch">
-        <PatchBlock patch={task.textFiles.get("tests/test.patch")} />
-      </DetailBlock>
-    </article>
-  );
-}
-
-function PatchBlock({ patch }: { patch: string | undefined }) {
-  const files = React.useMemo(
-    () => (patch ? parsePatchFiles(patch).flatMap((parsed) => parsed.files) : []),
-    [patch],
-  );
-  if (!patch) return <pre>Missing patch</pre>;
-  if (!files.length) return <pre>{patch}</pre>;
-  const options = {
-    themeType: window.matchMedia("(prefers-color-scheme: dark)").matches
-      ? ("dark" as const)
-      : ("light" as const),
-    diffStyle: window.matchMedia("(max-width: 900px)").matches
-      ? ("unified" as const)
-      : ("split" as const),
-    diffIndicators: "bars" as const,
-    overflow: "scroll" as const,
-    stickyHeader: true,
-  };
-  return (
-    <DiffErrorBoundary fallback={patch}>
-      <div className="patch-files">
-        {files.map((file) => (
-          <FileDiff
-            key={`${file.prevName ?? ""}:${file.name}`}
-            fileDiff={file}
-            disableWorkerPool
-            options={options}
-          />
-        ))}
-      </div>
-    </DiffErrorBoundary>
-  );
-}
-
-class DiffErrorBoundary extends React.Component<
-  React.PropsWithChildren<{ fallback: string }>,
-  { failed: boolean }
-> {
-  state = { failed: false };
-
-  static getDerivedStateFromError() {
-    return { failed: true };
-  }
-
-  componentDidCatch(error: unknown) {
-    console.error("Pierre diff renderer failed", error);
-  }
-
-  render() {
-    return this.state.failed ? <pre>{this.props.fallback}</pre> : this.props.children;
-  }
-}
-
-function DetailBlock({ title, children }: { title: string; children: React.ReactNode }) {
-  return (
-    <section className="detail-block">
-      <h3>{title}</h3>
-      {children}
-    </section>
-  );
-}
-
-function parseJson(value: string | undefined): Record<string, unknown> | null {
-  if (!value) return null;
-  try {
-    return JSON.parse(value) as Record<string, unknown>;
-  } catch {
-    return null;
-  }
-}

package/review/src/export-loader.test.ts

@@ -1,57 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { buildCleanExport, loadExport } from "./export-loader";
-import type { LoadedExport } from "./types";
-
-describe("export review bundles", () => {
-  test("round-trips nested task archives and removes marked tasks", async () => {
-    const loaded: LoadedExport = {
-      manifest: {
-        schemaVersion: 1,
-        runId: "run-123",
-        candidateCounts: { easy: 2, medium: 0, hard: 0 },
-        repository: { url: "https://github.com/example/repo", commit: "a".repeat(40) },
-        version: { schema: 1 },
-        acceptedCount: 2,
-        tasks: [
-          { taskId: "task-a", sha256: "a".repeat(64) },
-          { taskId: "task-b", sha256: "b".repeat(64) },
-        ],
-      },
-      tasks: [],
-      taskArchives: new Map([
-        ["task-a", await taskArchive("task-a")],
-        ["task-b", await taskArchive("task-b")],
-      ]),
-    };
-    loaded.tasks = [
-      { taskId: "task-a", files: new Map(), textFiles: new Map() },
-      { taskId: "task-b", files: new Map(), textFiles: new Map() },
-    ];
-
-    const cleaned = await buildCleanExport(loaded, new Set(["task-b"]));
-    const result = await loadExport(cleaned);
-
-    expect(result.manifest.acceptedCount).toBe(1);
-    expect(result.manifest.tasks.map((task) => task.taskId)).toEqual(["task-a"]);
-    expect(result.tasks[0]?.textFiles.get("definition.json")).toBe('{"taskId":"task-a"}');
-  });
-});
-
-async function taskArchive(taskId: string): Promise<Uint8Array> {
-  const data = new TextEncoder().encode(JSON.stringify({ taskId }));
-  const header = new Uint8Array(512);
-  write(header, 0, "harbor-task/definition.json");
-  write(header, 124, data.length.toString(8).padStart(11, "0"));
-  header[156] = 48;
-  const tar = new Uint8Array(512 + Math.ceil(data.length / 512) * 512 + 1024);
-  tar.set(header);
-  tar.set(data, 512);
-  const compressed = await new Response(
-    new Blob([tar.buffer]).stream().pipeThrough(new CompressionStream("gzip")),
-  ).arrayBuffer();
-  return new Uint8Array(compressed);
-}
-
-function write(target: Uint8Array, offset: number, value: string): void {
-  target.set(new TextEncoder().encode(value), offset);
-}