self-bench 0.3.0

package/dist/docker-executor.d.ts

@@ -0,0 +1,11 @@
+import type { SelfBenchConfig } from "./config.js";
+import type { SandboxExecutor, SandboxRequest, SandboxResult, SandboxRunOptions } from "./sandbox.js";
+export declare class DockerSandboxExecutor implements SandboxExecutor {
+    #private;
+    constructor(config: Extract<SelfBenchConfig["execution"], {
+        kind: "docker";
+    }>);
+    run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult>;
+    close(): void;
+}
+//# sourceMappingURL=docker-executor.d.ts.map

package/dist/docker-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-executor.d.ts","sourceRoot":"","sources":["../src/docker-executor.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,aAAa,EACb,iBAAiB,EAClB,MAAM,cAAc,CAAC;AAEtB,qBAAa,qBAAsB,YAAW,eAAe;;gBAG/C,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,EAAE;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;IAIvE,GAAG,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;IA4E3F,KAAK,IAAI,IAAI;CACd"}

package/dist/docker-executor.js

@@ -0,0 +1,103 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, relative, resolve } from "node:path";
+import { runCommand } from "./process.js";
+export class DockerSandboxExecutor {
+    #config;
+    constructor(config) {
+        this.#config = config;
+    }
+    async run(request, options = {}) {
+        const root = await mkdtemp(join(tmpdir(), "selfbench-docker-"));
+        const sandboxId = sandboxName(request.runId, request.stage);
+        try {
+            for (const file of request.files ?? []) {
+                const destination = hostPath(root, file.path);
+                await mkdir(dirname(destination), { recursive: true });
+                await writeFile(destination, file.contents);
+            }
+            const environment = { ...request.environment, ...request.secrets };
+            const args = [
+                "create",
+                "--name",
+                sandboxId,
+                "--cpus",
+                String(request.cpu ?? 4),
+                "--memory",
+                `${request.memoryMiB ?? 8192}m`,
+                "--volume",
+                `${sandboxId}:/work`,
+                "--workdir",
+                "/work",
+            ];
+            for (const key of Object.keys(environment)) {
+                args.push("--env", key);
+            }
+            args.push(this.#config.image, ...request.command);
+            await runCommand("docker", ["volume", "create", sandboxId]);
+            await runCommand("docker", args, { env: { ...process.env, ...environment } });
+            await runCommand("docker", ["cp", `${root}/.`, `${sandboxId}:/work/`]);
+            const abort = () => {
+                void runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
+            };
+            options.signal?.addEventListener("abort", abort, { once: true });
+            try {
+                const result = await runCommand("docker", ["start", "--attach", sandboxId], {
+                    allowFailure: true,
+                    timeoutMs: request.timeoutMs,
+                    ...(request.inactivityTimeoutMs
+                        ? { inactivityTimeoutMs: request.inactivityTimeoutMs }
+                        : {}),
+                    onOutput: (stream, chunk) => options.onProgress?.({ stream, bytes: chunk.byteLength }),
+                });
+                const outputs = {};
+                for (const path of request.outputPaths ?? []) {
+                    const destination = hostPath(root, path);
+                    await mkdir(dirname(destination), { recursive: true });
+                    const copied = await runCommand("docker", ["cp", `${sandboxId}:${path}`, destination], {
+                        allowFailure: true,
+                    });
+                    if (copied.exitCode === 0) {
+                        outputs[path] = await readFile(destination);
+                    }
+                    else if (result.exitCode === 0) {
+                        throw new Error(`sandbox ${sandboxId} exited successfully without output ${path}`);
+                    }
+                }
+                return { sandboxId, ...result, outputs };
+            }
+            finally {
+                options.signal?.removeEventListener("abort", abort);
+                await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
+                await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
+                    allowFailure: true,
+                });
+            }
+        }
+        finally {
+            await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
+            await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
+                allowFailure: true,
+            });
+            await rm(root, { recursive: true, force: true });
+        }
+    }
+    close() { }
+}
+function hostPath(root, containerPath) {
+    if (!containerPath.startsWith("/work/")) {
+        throw new Error(`sandbox path must be beneath /work: ${containerPath}`);
+    }
+    const path = resolve(root, relative("/work", containerPath));
+    if (path !== root && !path.startsWith(`${root}/`)) {
+        throw new Error(`sandbox path escapes /work: ${containerPath}`);
+    }
+    return path;
+}
+function sandboxName(runId, stage) {
+    const suffix = crypto.randomUUID().slice(0, 8);
+    return `selfbench-${runId.slice(0, 20)}-${stage.slice(0, 16)}-${suffix}`
+        .toLowerCase()
+        .replace(/[^a-z0-9_.-]/g, "-");
+}
+//# sourceMappingURL=docker-executor.js.map

package/dist/docker-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-executor.js","sourceRoot":"","sources":["../src/docker-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAQ1C,MAAM,OAAO,qBAAqB;IACvB,OAAO,CAA4D;IAE5E,YAAY,MAAiE;QAC3E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAuB,EAAE,UAA6B,EAAE;QAChE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5D,IAAI,CAAC;YACH,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBACvC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG;gBACX,QAAQ;gBACR,QAAQ;gBACR,SAAS;gBACT,QAAQ;gBACR,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;gBACxB,UAAU;gBACV,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,GAAG;gBAC/B,UAAU;gBACV,GAAG,SAAS,QAAQ;gBACpB,WAAW;gBACX,OAAO;aACR,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAElD,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAC5D,MAAM,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC;YAC9E,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,SAAS,SAAS,CAAC,CAAC,CAAC;YAEvE,MAAM,KAAK,GAAG,GAAG,EAAE;gBACjB,KAAK,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAClF,CAAC,CAAC;YACF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE;oBAC1E,YAAY,EAAE,IAAI;oBAClB,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,GAAG,CAAC,OAAO,CAAC,mBAAmB;wBAC7B,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE;wBACtD,CAAC,CAAC,EAAE,CAAC;oBACP,QAAQ,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;iBACvF,CAAC,CAAC;gBACH,MAAM,OAAO,GAA+B,EAAE,CAAC;gBAC/C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;oBAC7C,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBACzC,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACvD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,SAAS,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,EAAE;wBACrF,YAAY,EAAE,IAAI;qBACnB,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAC;oBAC9C,CAAC;yBAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;wBACjC,MAAM,IAAI,KAAK,CAAC,WAAW,SAAS,uCAAuC,IAAI,EAAE,CAAC,CAAC;oBACrF,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC;YAC3C,CAAC;oBAAS,CAAC;gBACT,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACpD,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;gBACjF,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE;oBACjE,YAAY,EAAE,IAAI;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE;gBACjE,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,KAAK,KAAU,CAAC;CACjB;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,aAAqB;IACnD,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,aAAa,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;IAC7D,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,+BAA+B,aAAa,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,KAAa;IAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,OAAO,aAAa,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE;SACrE,WAAW,EAAE;SACb,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;AACnC,CAAC"}

package/dist/eval-main.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=eval-main.d.ts.map

package/dist/eval-main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-main.d.ts","sourceRoot":"","sources":["../src/eval-main.ts"],"names":[],"mappings":""}

package/dist/eval-main.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+import { parseArgs } from "node:util";
+import { MATRIX_MODELS, runMatrix } from "./evaluate.js";
+const parsed = parseArgs({
+    options: {
+        export: { type: "string" },
+        tasks: { type: "string" },
+        jobs: { type: "string" },
+        harbor: { type: "string" },
+        environment: { type: "string", default: "modal" },
+        concurrency: { type: "string", default: "3" },
+        auth: { type: "string" },
+        model: { type: "string", multiple: true },
+        help: { type: "boolean", short: "h" },
+    },
+    strict: true,
+});
+if (parsed.values.help) {
+    console.log(`Run Harbor tasks through the fixed Codex subscription model matrix.
+
+Usage:
+  selfbench-eval (--export FILE.tar.gz | --tasks DIRECTORY) --jobs DIRECTORY [options]
+
+Options:
+  --tasks DIRECTORY          Expanded Harbor tasks (one or more)
+  --harbor PATH              Harbor executable (default: harbor)
+  --environment docker|modal Execution environment (default: modal)
+  --concurrency N            Concurrent trials (default: 3)
+  --model MODEL              Run only this model; may be repeated
+  --auth FILE                Codex ChatGPT auth.json path
+  -h, --help                 Show this help`);
+    process.exit(0);
+}
+const environment = parsed.values.environment;
+if (environment !== "docker" && environment !== "modal") {
+    throw new Error("--environment must be docker or modal");
+}
+const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
+const models = evaluationModels(parsed.values.model);
+const summaries = await runMatrix({
+    ...(parsed.values.export ? { exportPath: parsed.values.export } : {}),
+    ...(parsed.values.tasks ? { tasksPath: parsed.values.tasks } : {}),
+    jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
+    environment,
+    concurrency,
+    ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
+    ...(parsed.values.auth ? { authPath: parsed.values.auth } : {}),
+    ...(models ? { models } : {}),
+    onTrialComplete: (summary, completed, total) => {
+        const status = summary.passed ? "passed" : summary.exception ? "error" : "failed";
+        console.error(`[${completed}/${total}] ${summary.model} ${summary.taskId}: ${status}`);
+    },
+});
+console.log(JSON.stringify({
+    trials: summaries.length,
+    passed: summaries.filter((summary) => summary.passed).length,
+    failed: summaries.filter((summary) => !summary.passed).length,
+}, null, 2));
+function fail(message) {
+    throw new Error(message);
+}
+function evaluationModels(values) {
+    if (!values) {
+        return undefined;
+    }
+    const allowed = new Set(MATRIX_MODELS);
+    const invalid = values.filter((value) => !allowed.has(value));
+    if (invalid.length > 0) {
+        throw new Error(`--model must be one of ${MATRIX_MODELS.join(", ")}; got ${invalid.join(", ")}`);
+    }
+    return [...new Set(values)];
+}
+function positiveInteger(value, label) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < 1) {
+        throw new Error(`${label} must be a positive integer`);
+    }
+    return parsed;
+}
+//# sourceMappingURL=eval-main.js.map

package/dist/eval-main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-main.js","sourceRoot":"","sources":["../src/eval-main.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,aAAa,EAAoB,SAAS,EAAE,MAAM,eAAe,CAAC;AAE3E,MAAM,MAAM,GAAG,SAAS,CAAC;IACvB,OAAO,EAAE;QACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE;QACjD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;QAC7C,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACtC;IACD,MAAM,EAAE,IAAI;CACb,CAAC,CAAC;AACH,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;4CAY8B,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;AAC9C,IAAI,WAAW,KAAK,QAAQ,IAAI,WAAW,KAAK,OAAO,EAAE,CAAC;IACxD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;AAC3D,CAAC;AACD,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;AAChF,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACrD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;IAChC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC;IAC/D,WAAW;IACX,WAAW;IACX,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7B,eAAe,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;QAClF,OAAO,CAAC,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,KAAK,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IACzF,CAAC;CACF,CAAC,CAAC;AACH,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CACZ;IACE,MAAM,EAAE,SAAS,CAAC,MAAM;IACxB,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM;IAC5D,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM;CAC9D,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;AAEF,SAAS,IAAI,CAAC,OAAe;IAC3B,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,gBAAgB,CAAC,MAA4B;IACpD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,aAAa,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,0BAA0B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAkB,CAAC;AAC/C,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB,EAAE,KAAa;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/evaluate.d.ts

@@ -0,0 +1,24 @@
+export declare const MATRIX_MODELS: readonly ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"];
+export type MatrixModel = (typeof MATRIX_MODELS)[number];
+export interface MatrixOptions {
+    readonly exportPath?: string;
+    readonly tasksPath?: string;
+    readonly jobsDirectory: string;
+    readonly harborPath?: string;
+    readonly environment?: "docker" | "modal";
+    readonly concurrency?: number;
+    readonly authPath?: string;
+    readonly models?: readonly MatrixModel[];
+    readonly onTrialComplete?: (summary: MatrixTrialSummary, completed: number, total: number) => void;
+}
+export interface MatrixTrialSummary {
+    readonly taskId: string;
+    readonly model: MatrixModel;
+    readonly jobName: string;
+    readonly passed: boolean;
+    readonly rewards: Readonly<Record<string, number>>;
+    readonly exception?: string;
+}
+export declare function runMatrix(options: MatrixOptions): Promise<readonly MatrixTrialSummary[]>;
+export declare function summarizeResult(taskId: string, model: MatrixModel, jobName: string, result: unknown): MatrixTrialSummary;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAQA,eAAO,MAAM,aAAa,2DAA4D,CAAC;AACvF,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAGzD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,eAAe,CAAC,EAAE,CACzB,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,KACV,IAAI,CAAC;CACX;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CA6C9F;AA8FD,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,OAAO,GACd,kBAAkB,CA6BpB"}

package/dist/evaluate.js

@@ -0,0 +1,208 @@
+import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { basename, join, resolve } from "node:path";
+import { archiveIncompleteHarborJob, tryReadHarborJobResult } from "./harbor-results.js";
+import { parallelMap } from "./parallel.js";
+import { runCommand } from "./process.js";
+import { assertCodexSubscriptionAuth } from "./subscription-auth.js";
+export const MATRIX_MODELS = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"];
+const CODEX_VERSION = "0.146.1";
+export async function runMatrix(options) {
+    const root = resolve(options.jobsDirectory);
+    const tasksDirectory = join(root, "tasks");
+    await mkdir(root, { recursive: true });
+    const taskDirectories = await resolveMatrixTasks(options, tasksDirectory);
+    const authPath = resolve(options.authPath ?? join(homedir(), ".codex/auth.json"));
+    await assertSubscriptionAuth(authPath);
+    const models = options.models ?? MATRIX_MODELS;
+    if (models.length < 1) {
+        throw new Error("provide at least one evaluation model");
+    }
+    const work = models.flatMap((model) => taskDirectories.map((taskDirectory) => ({ model, taskDirectory })));
+    let completed = 0;
+    const summaries = await parallelMap(work, options.concurrency ?? 3, async (item) => {
+        const summary = await runTrial({
+            jobsDirectory: root,
+            harborPath: options.harborPath ?? "harbor",
+            environment: options.environment ?? "modal",
+            authPath,
+            ...item,
+        });
+        completed += 1;
+        options.onTrialComplete?.(summary, completed, work.length);
+        return summary;
+    });
+    await writeFile(join(root, "summary.json"), `${JSON.stringify({
+        schemaVersion: 1,
+        agent: "codex",
+        agentVersion: CODEX_VERSION,
+        auth: "codex-subscription",
+        reasoningEffort: "high",
+        models,
+        taskCount: taskDirectories.length,
+        trials: summaries,
+    }, null, 2)}\n`);
+    return summaries;
+}
+async function resolveMatrixTasks(options, materializedTasksDirectory) {
+    if (options.exportPath && options.tasksPath) {
+        throw new Error("provide exactly one of exportPath or tasksPath");
+    }
+    if (options.tasksPath) {
+        const tasks = await taskDirectories(resolve(options.tasksPath));
+        if (tasks.length < 1) {
+            throw new Error(`found no Harbor tasks in ${resolve(options.tasksPath)}`);
+        }
+        return tasks;
+    }
+    if (!options.exportPath) {
+        throw new Error("provide exactly one of exportPath or tasksPath");
+    }
+    const tasks = await materializeExport(resolve(options.exportPath), materializedTasksDirectory);
+    if (tasks.length < 1) {
+        throw new Error(`found no Harbor tasks in ${resolve(options.exportPath)}`);
+    }
+    return tasks;
+}
+async function runTrial(input) {
+    const taskId = basename(input.taskDirectory);
+    const jobName = `${taskId}-${input.model}`.toLowerCase().replace(/[^a-z0-9_.-]/g, "-");
+    const existing = await tryReadHarborJobResult(input.jobsDirectory, jobName);
+    if (existing) {
+        return summarizeResult(taskId, input.model, jobName, existing.trial);
+    }
+    await archiveIncompleteHarborJob(input.jobsDirectory, jobName);
+    const environment = { ...process.env };
+    delete environment.OPENAI_API_KEY;
+    environment.CODEX_FORCE_AUTH_JSON = "1";
+    environment.CODEX_AUTH_JSON_PATH = input.authPath;
+    let result;
+    try {
+        result = await runCommand(input.harborPath, [
+            "run",
+            "--path",
+            input.taskDirectory,
+            "--agent",
+            "codex",
+            "--model",
+            input.model,
+            "--ak",
+            `version=${CODEX_VERSION}`,
+            "--ak",
+            "reasoning_effort=high",
+            "--env",
+            input.environment,
+            "--job-name",
+            jobName,
+            "--jobs-dir",
+            input.jobsDirectory,
+            "--n-concurrent",
+            "1",
+            "--max-retries",
+            "1",
+            "--delete",
+            "--yes",
+            "--quiet",
+        ], { allowFailure: true, env: environment, timeoutMs: 4 * 60 * 60 * 1000 });
+    }
+    catch (error) {
+        return failedTrial(taskId, input.model, jobName, errorMessage(error));
+    }
+    const completed = await tryReadHarborJobResult(input.jobsDirectory, jobName);
+    if (!completed) {
+        const detail = result.stderr.trim() || result.stdout.trim();
+        return failedTrial(taskId, input.model, jobName, `Harbor produced no result (exit ${result.exitCode})${detail ? `: ${detail.slice(-1000)}` : ""}`);
+    }
+    return summarizeResult(taskId, input.model, jobName, completed.trial);
+}
+export function summarizeResult(taskId, model, jobName, result) {
+    const trial = isRecord(result) && Array.isArray(result.trial_results) ? result.trial_results[0] : result;
+    if (!isRecord(trial)) {
+        throw new Error(`invalid Harbor result for ${taskId}/${model}`);
+    }
+    const rawRewards = isRecord(trial.verifier_result) && isRecord(trial.verifier_result.rewards)
+        ? trial.verifier_result.rewards
+        : {};
+    const rewards = Object.fromEntries(Object.entries(rawRewards).flatMap(([key, value]) => typeof value === "number" ? [[key, value]] : []));
+    const exception = exceptionText(trial.exception_info ?? trial.exception);
+    return {
+        taskId,
+        model,
+        jobName,
+        passed: !exception &&
+            Object.keys(rewards).length > 0 &&
+            (rewards.reward === undefined
+                ? Object.values(rewards).every((reward) => reward >= 1)
+                : rewards.reward >= 1),
+        rewards,
+        ...(exception ? { exception } : {}),
+    };
+}
+function failedTrial(taskId, model, jobName, exception) {
+    return { taskId, model, jobName, passed: false, rewards: {}, exception };
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+async function materializeExport(exportPath, tasksDirectory) {
+    const existing = await taskDirectories(tasksDirectory);
+    if (existing.length > 0) {
+        return existing;
+    }
+    return await withTemporaryDirectory("selfbench-matrix-", async (temporary) => {
+        await runCommand("tar", ["-xzf", exportPath, "-C", temporary]);
+        const archivesRoot = join(temporary, "tasks");
+        const archives = (await readdir(archivesRoot))
+            .filter((name) => name.endsWith(".tar.gz"))
+            .sort();
+        await mkdir(tasksDirectory, { recursive: true });
+        for (const archive of archives) {
+            const taskId = archive.slice(0, -".tar.gz".length);
+            const expanded = join(temporary, `expanded-${taskId}`);
+            await mkdir(expanded);
+            await runCommand("tar", ["-xzf", join(archivesRoot, archive), "-C", expanded]);
+            await cp(join(expanded, "harbor-task"), join(tasksDirectory, taskId), { recursive: true });
+        }
+        return await taskDirectories(tasksDirectory);
+    });
+}
+async function taskDirectories(root) {
+    const entries = await readdir(root, { withFileTypes: true }).catch(() => []);
+    const paths = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory()) {
+            continue;
+        }
+        const path = join(root, entry.name);
+        if (await readFile(join(path, "task.toml")).catch(() => undefined)) {
+            paths.push(path);
+        }
+    }
+    return paths.sort();
+}
+async function assertSubscriptionAuth(path) {
+    const parsed = JSON.parse(await readFile(path, "utf8"));
+    assertCodexSubscriptionAuth(parsed, path);
+}
+async function withTemporaryDirectory(prefix, action) {
+    const root = await mkdtemp(join(tmpdir(), prefix));
+    try {
+        return await action(root);
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+}
+function exceptionText(value) {
+    if (typeof value === "string" && value) {
+        return value;
+    }
+    if (isRecord(value)) {
+        return typeof value.message === "string" ? value.message : JSON.stringify(value);
+    }
+    return undefined;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACxF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAErE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,eAAe,EAAE,cAAc,CAAU,CAAC;AAEvF,MAAM,aAAa,GAAG,SAAS,CAAC;AA2BhC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAsB;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,eAAe,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;IAClF,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACpC,eAAe,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,CACnE,CAAC;IACF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QACjF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC;YAC7B,aAAa,EAAE,IAAI;YACnB,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,QAAQ;YAC1C,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,OAAO;YAC3C,QAAQ;YACR,GAAG,IAAI;SACR,CAAC,CAAC;QACH,SAAS,IAAI,CAAC,CAAC;QACf,OAAO,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3D,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAC1B,GAAG,IAAI,CAAC,SAAS,CACf;QACE,aAAa,EAAE,CAAC;QAChB,KAAK,EAAE,OAAO;QACd,YAAY,EAAE,aAAa;QAC3B,IAAI,EAAE,oBAAoB;QAC1B,eAAe,EAAE,MAAM;QACvB,MAAM;QACN,SAAS,EAAE,eAAe,CAAC,MAAM;QACjC,MAAM,EAAE,SAAS;KAClB,EACD,IAAI,EACJ,CAAC,CACF,IAAI,CACN,CAAC;IACF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,OAAsB,EACtB,0BAAkC;IAElC,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAChE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,0BAA0B,CAAC,CAAC;IAC/F,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAOvB;IACC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,GAAG,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,KAAK,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC5E,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,0BAA0B,CAAC,KAAK,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE/D,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACvC,OAAO,WAAW,CAAC,cAAc,CAAC;IAClC,WAAW,CAAC,qBAAqB,GAAG,GAAG,CAAC;IACxC,WAAW,CAAC,oBAAoB,GAAG,KAAK,CAAC,QAAQ,CAAC;IAClD,IAAI,MAA8C,CAAC;IACnD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,UAAU,CACvB,KAAK,CAAC,UAAU,EAChB;YACE,KAAK;YACL,QAAQ;YACR,KAAK,CAAC,aAAa;YACnB,SAAS;YACT,OAAO;YACP,SAAS;YACT,KAAK,CAAC,KAAK;YACX,MAAM;YACN,WAAW,aAAa,EAAE;YAC1B,MAAM;YACN,uBAAuB;YACvB,OAAO;YACP,KAAK,CAAC,WAAW;YACjB,YAAY;YACZ,OAAO;YACP,YAAY;YACZ,KAAK,CAAC,aAAa;YACnB,gBAAgB;YAChB,GAAG;YACH,eAAe;YACf,GAAG;YACH,UAAU;YACV,OAAO;YACP,SAAS;SACV,EACD,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CACxE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,KAAK,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC7E,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5D,OAAO,WAAW,CAChB,MAAM,EACN,KAAK,CAAC,KAAK,EACX,OAAO,EACP,mCAAmC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACjG,CAAC;IACJ,CAAC;IACD,OAAO,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,MAAc,EACd,KAAkB,EAClB,OAAe,EACf,MAAe;IAEf,MAAM,KAAK,GACT,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7F,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,UAAU,GACd,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,OAAO,CAAC;QACxE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,OAAO;QAC/B,CAAC,CAAC,EAAE,CAAC;IACT,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAChC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAClD,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CACzD,CACF,CAAC;IACF,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IACzE,OAAO;QACL,MAAM;QACN,KAAK;QACL,OAAO;QACP,MAAM,EACJ,CAAC,SAAS;YACV,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAC/B,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;gBAC3B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC;gBACvD,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC1B,OAAO;QACP,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,MAAc,EACd,KAAkB,EAClB,OAAe,EACf,SAAiB;IAEjB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,UAAkB,EAAE,cAAsB;IACzE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,sBAAsB,CAAC,mBAAmB,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE;QAC3E,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;aAC3C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;aAC1C,IAAI,EAAE,CAAC;QACV,MAAM,KAAK,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,MAAM,EAAE,CAAC,CAAC;YACvD,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACtB,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC/E,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,OAAO,MAAM,eAAe,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,IAAY;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;IACnE,2BAA2B,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,MAAoC;IAEpC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACnD,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/extensions/authoring.d.ts

@@ -0,0 +1,3 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+export default function authoringExtension(pi: ExtensionAPI): void;
+//# sourceMappingURL=authoring.d.ts.map

package/dist/extensions/authoring.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoring.d.ts","sourceRoot":"","sources":["../../src/extensions/authoring.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AA4CpE,MAAM,CAAC,OAAO,UAAU,kBAAkB,CAAC,EAAE,EAAE,YAAY,GAAG,IAAI,CAuCjE"}

package/dist/extensions/authoring.js

@@ -0,0 +1,71 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { Type } from "@sinclair/typebox";
+const taskDefinition = Type.Object({
+    schemaVersion: Type.Literal(1),
+    difficulty: Type.Union([Type.Literal("easy"), Type.Literal("medium"), Type.Literal("hard")]),
+    taskId: Type.String({ pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*$" }),
+    repo: Type.String({ minLength: 1 }),
+    baseCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
+    workdir: Type.String({ minLength: 1 }),
+    setupCommand: Type.String({ minLength: 1 }),
+    testCommand: Type.String({ minLength: 1 }),
+    failToPass: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+    passToPass: Type.Array(Type.String({ minLength: 1 })),
+    testPaths: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+    toolchains: Type.Array(Type.Union([
+        Type.Literal("uv"),
+        Type.Literal("bun"),
+        Type.Literal("go"),
+        Type.Literal("node"),
+        Type.Literal("python"),
+        Type.Literal("rust"),
+    ]), { minItems: 1 }),
+    sourcePr: Type.Integer({ minimum: 1 }),
+    sourceUrl: Type.String({ minLength: 1 }),
+    prompt: Type.String({ minLength: 1 }),
+    timeouts: Type.Object({
+        setupSeconds: Type.Integer({ minimum: 1 }),
+        agentSeconds: Type.Integer({ minimum: 1 }),
+        testsSeconds: Type.Integer({ minimum: 1 }),
+    }),
+    resources: Type.Object({
+        cpus: Type.Number({ exclusiveMinimum: 0 }),
+        memoryMb: Type.Integer({ minimum: 1 }),
+        storageMb: Type.Integer({ minimum: 1 }),
+    }),
+}, { additionalProperties: false });
+export default function authoringExtension(pi) {
+    pi.registerTool({
+        name: "submit_task",
+        label: "Submit SelfBench task",
+        description: "Submit exactly one complete task at the assigned difficulty.",
+        parameters: Type.Object({
+            definition: taskDefinition,
+            testPatch: Type.String({ minLength: 1 }),
+            goldPatch: Type.String({ minLength: 1 }),
+        }, { additionalProperties: false }),
+        async execute(_toolCallId, input) {
+            const root = process.env.SELFBENCH_TASK_OUTPUT;
+            if (!root) {
+                throw new Error("SELFBENCH_TASK_OUTPUT is not configured");
+            }
+            const { definition, testPatch, goldPatch } = input;
+            if (!definition?.taskId || typeof testPatch !== "string" || typeof goldPatch !== "string") {
+                throw new Error("submit_task received an incomplete task");
+            }
+            const directory = join(root, definition.taskId);
+            mkdirSync(directory, { recursive: false });
+            writeFileSync(join(directory, "definition.json"), `${JSON.stringify(definition, null, 2)}\n`, {
+                flag: "wx",
+            });
+            writeFileSync(join(directory, "test.patch"), testPatch, { flag: "wx" });
+            writeFileSync(join(directory, "gold.patch"), goldPatch, { flag: "wx" });
+            return {
+                content: [{ type: "text", text: `Submitted ${definition.taskId}.` }],
+                details: { taskId: definition.taskId },
+            };
+        },
+    });
+}
+//# sourceMappingURL=authoring.js.map

package/dist/extensions/authoring.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoring.js","sourceRoot":"","sources":["../../src/extensions/authoring.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAChC;IACE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5F,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,8BAA8B,EAAE,CAAC;IAChE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACnC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;IACzD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACtC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAC3C,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAC1C,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACtE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IACrD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACrE,UAAU,EAAE,IAAI,CAAC,KAAK,CACpB,IAAI,CAAC,KAAK,CAAC;QACT,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAClB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAClB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACtB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KACrB,CAAC,EACF,EAAE,QAAQ,EAAE,CAAC,EAAE,CAChB;IACD,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACtC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACxC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACrC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;QACpB,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1C,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1C,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;KAC3C,CAAC;IACF,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;QAC1C,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACtC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;KACxC,CAAC;CACH,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAChC,CAAC;AAEF,MAAM,CAAC,OAAO,UAAU,kBAAkB,CAAC,EAAgB;IACzD,EAAE,CAAC,YAAY,CAAC;QACd,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,uBAAuB;QAC9B,WAAW,EAAE,8DAA8D;QAC3E,UAAU,EAAE,IAAI,CAAC,MAAM,CACrB;YACE,UAAU,EAAE,cAAc;YAC1B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YACxC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;SACzC,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAChC;QACD,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK;YAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;YAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;YACnD,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC1F,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YAChD,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3C,aAAa,CACX,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAClC,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAC1C;gBACE,IAAI,EAAE,IAAI;aACX,CACF,CAAC;YACF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;gBACpE,OAAO,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE;aACvC,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/extensions/discovery.d.ts

@@ -0,0 +1,3 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+export default function discoveryExtension(pi: ExtensionAPI): void;
+//# sourceMappingURL=discovery.d.ts.map

package/dist/extensions/discovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/extensions/discovery.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAGpE,MAAM,CAAC,OAAO,UAAU,kBAAkB,CAAC,EAAE,EAAE,YAAY,GAAG,IAAI,CAwDjE"}

package/dist/extensions/discovery.js

@@ -0,0 +1,51 @@
+import { writeFileSync } from "node:fs";
+import { Type } from "@sinclair/typebox";
+export default function discoveryExtension(pi) {
+    pi.registerTool({
+        name: "submit_discovery",
+        label: "Submit SelfBench discovery",
+        description: "Submit the final ranked tiered candidate list exactly once.",
+        parameters: Type.Object({
+            candidates: Type.Array(Type.Object({
+                candidateId: Type.String({ minLength: 1 }),
+                difficulty: Type.Union([
+                    Type.Literal("easy"),
+                    Type.Literal("medium"),
+                    Type.Literal("hard"),
+                ]),
+                sourcePr: Type.Integer({ minimum: 1 }),
+                sourceUrl: Type.String({ minLength: 1 }),
+                baseCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
+                completedCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
+                request: Type.String({ minLength: 1 }),
+                provenance: Type.Object({
+                    sourceType: Type.Union([
+                        Type.Literal("pi"),
+                        Type.Literal("claude-code"),
+                        Type.Literal("codex"),
+                        Type.Literal("generic"),
+                        Type.Literal("github-pull-request"),
+                    ]),
+                    sessionId: Type.String({ minLength: 1 }),
+                    messageIndex: Type.Integer({ minimum: 0 }),
+                }),
+            }, { additionalProperties: false }), { maxItems: 100 }),
+        }, { additionalProperties: false }),
+        async execute(_toolCallId, input) {
+            const output = process.env.SELFBENCH_DISCOVERY_OUTPUT;
+            if (!output) {
+                throw new Error("SELFBENCH_DISCOVERY_OUTPUT is not configured");
+            }
+            const candidates = input.candidates;
+            if (!candidates) {
+                throw new Error("submit_discovery received no candidates");
+            }
+            writeFileSync(output, `${JSON.stringify({ candidates }, null, 2)}\n`, { flag: "wx" });
+            return {
+                content: [{ type: "text", text: `Submitted ${candidates.length} candidates.` }],
+                details: { candidates },
+            };
+        },
+    });
+}
+//# sourceMappingURL=discovery.js.map

package/dist/extensions/discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.js","sourceRoot":"","sources":["../../src/extensions/discovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAEzC,MAAM,CAAC,OAAO,UAAU,kBAAkB,CAAC,EAAgB;IACzD,EAAE,CAAC,YAAY,CAAC;QACd,IAAI,EAAE,kBAAkB;QACxB,KAAK,EAAE,4BAA4B;QACnC,WAAW,EAAE,6DAA6D;QAC1E,UAAU,EAAE,IAAI,CAAC,MAAM,CACrB;YACE,UAAU,EAAE,IAAI,CAAC,KAAK,CACpB,IAAI,CAAC,MAAM,CACT;gBACE,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;gBAC1C,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC;oBACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;oBACpB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;oBACtB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;iBACrB,CAAC;gBACF,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;gBACtC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;gBACxC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;gBACzD,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;gBAC9D,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;gBACtC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC;oBACtB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC;wBACrB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;wBAClB,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;wBAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;wBACrB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;wBACvB,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC;qBACpC,CAAC;oBACF,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;oBACxC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;iBAC3C,CAAC;aACH,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAChC,EACD,EAAE,QAAQ,EAAE,GAAG,EAAE,CAClB;SACF,EACD,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAChC;QACD,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK;YAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;YACtD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YACD,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;YACpC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YAC7D,CAAC;YACD,aAAa,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACtF,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,UAAU,CAAC,MAAM,cAAc,EAAE,CAAC;gBAC/E,OAAO,EAAE,EAAE,UAAU,EAAE;aACxB,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}