self-bench 0.3.6 → 0.3.8

package/dist/evaluation/store.js

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+export function evaluationPrefix(repoId, id = "") {
+    if (!Number.isSafeInteger(repoId) || repoId < 1 || (id && !/^[a-f0-9-]{36}$/.test(id)))
+        throw new Error("Invalid evaluation identity");
+    return `evaluations/repos/${repoId}/${id ? `${id}/` : ""}`;
+}
+export function initialEvaluation(input, modelLabel) {
+    const { tasks, ...metadata } = input;
+    return {
+        ...metadata,
+        revision: 0,
+        datasetKey: createHash("sha256")
+            .update(JSON.stringify(tasks
+            .map(({ runId, taskId, bundleKey }) => [runId, taskId, bundleKey])
+            .sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)))))
+            .digest("hex"),
+        modelLabel,
+        status: "queued",
+        trials: tasks.flatMap((task) => input.harnesses.map((harness) => ({
+            taskId: task.taskId,
+            runId: task.runId,
+            harness,
+            status: "queued",
+            rewards: {},
+            log: "",
+            steps: [],
+            artifacts: [],
+        }))),
+    };
+}
+export async function saveEvaluation(store, run) {
+    run.revision += 1;
+    await store.put(`${evaluationPrefix(run.repoId, run.id)}snapshots/${String(run.revision).padStart(10, "0")}.json`, Buffer.from(JSON.stringify(run)), "application/json");
+}
+export async function getEvaluation(store, repoId, id) {
+    const entries = await store.list(`${evaluationPrefix(repoId, id)}snapshots`);
+    const latest = entries.sort((left, right) => right.key.localeCompare(left.key))[0];
+    const bytes = latest ? await store.getByKey(latest.key) : undefined;
+    return bytes ? JSON.parse(Buffer.from(bytes).toString("utf8")) : undefined;
+}
+export async function listEvaluations(store, repoId) {
+    const prefix = evaluationPrefix(repoId);
+    const entries = (await store.list(prefix.slice(0, -1))).filter((entry) => entry.key.startsWith(prefix));
+    const latest = new Map();
+    for (const entry of entries) {
+        if (!/\/snapshots\/\d{10}\.json$/.test(entry.key))
+            continue;
+        const id = entry.key.slice(prefix.length).split("/")[0];
+        if (id && entry.key > (latest.get(id) ?? ""))
+            latest.set(id, entry.key);
+    }
+    const runs = await Promise.all([...latest.values()].map(async (key) => {
+        const bytes = await store.getByKey(key);
+        return bytes ? JSON.parse(Buffer.from(bytes).toString("utf8")) : undefined;
+    }));
+    return runs
+        .filter((run) => !!run)
+        .sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+}
+//# sourceMappingURL=store.js.map

package/dist/evaluation/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/evaluation/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,EAAE,GAAG,EAAE;IACtD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,OAAO,qBAAqB,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AAC7D,CAAC;AACD,MAAM,UAAU,iBAAiB,CAAC,KAAsB,EAAE,UAAkB;IAC1E,MAAM,EAAE,KAAK,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAC;IACrC,OAAO;QACL,GAAG,QAAQ;QACX,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC;aAC7B,MAAM,CACL,IAAI,CAAC,SAAS,CACZ,KAAK;aACF,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;aACjE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CACpF,CACF;aACA,MAAM,CAAC,KAAK,CAAC;QAChB,UAAU;QACV,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7B,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAChC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,EAAE;YACX,GAAG,EAAE,EAAE;YACP,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CACJ;KACF,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAoB,EAAE,GAAkB;IAC3E,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClB,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,aAAa,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,EACjG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAChC,kBAAkB,CACnB,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAoB,EACpB,MAAc,EACd,EAAU;IAEV,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpE,OAAO,KAAK,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAmB,CAAC,CAAC,CAAC,SAAS,CAAC;AAChG,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAoB,EACpB,MAAc;IAEd,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CACvE,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAC7B,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5D,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,IAAI,EAAE,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,KAAK,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAmB,CAAC,CAAC,CAAC,SAAS,CAAC;IAChG,CAAC,CAAC,CACH,CAAC;IACF,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,GAAG,EAAwB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SAC5C,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;AAC1E,CAAC"}

package/dist/evaluation/types.d.ts

@@ -0,0 +1,98 @@
+import type { Harness } from "./harnesses.js";
+import type { ThinkingLevel } from "./model-options.js";
+export type { Harness } from "./harnesses.js";
+export type EvaluationSandbox = "docker" | "modal" | "e2b" | "daytona";
+export interface EvaluationPricing {
+    maxInputTokens?: number;
+    input: number;
+    output: number;
+    cacheRead: number;
+    cacheWrite: number;
+    source: string;
+    asOf: string;
+}
+export interface EvaluationTask {
+    runId: string;
+    taskId: string;
+    bundleKey: string;
+}
+export interface EvaluationInput {
+    id: string;
+    repoId: number;
+    tenant: string;
+    startedBy: string;
+    createdAt: string;
+    model: string;
+    modelName: string;
+    thinking?: ThinkingLevel;
+    harnesses: Harness[];
+    sandbox: EvaluationSandbox;
+    tasks: EvaluationTask[];
+    pricing?: EvaluationPricing;
+    credentialOwnerId?: number;
+    credentialOrgId?: number;
+    comparisonId?: string;
+    credentials?: {
+        modelCredentialId: string;
+        sandboxCredentialId: string;
+        provider: "openai" | "anthropic" | "openrouter" | "custom";
+    };
+}
+export interface SolverStep {
+    id: string;
+    role: string;
+    text: string;
+    tools: {
+        id: string;
+        name: string;
+        input: string;
+        output: string;
+    }[];
+}
+export interface TokenUsage {
+    input: number;
+    output: number;
+    cacheRead: number;
+    cacheWrite: number;
+}
+export interface EvaluationTrial {
+    taskId: string;
+    runId: string;
+    harness: Harness;
+    status: "queued" | "running" | "completed" | "failed";
+    rewards: Record<string, number>;
+    error?: string;
+    log: string;
+    steps: SolverStep[];
+    artifacts: string[];
+    startedAt?: string;
+    finishedAt?: string;
+    modelVerified?: boolean;
+    apiCostUsd?: number;
+    tokenUsage?: TokenUsage;
+    costSource?: "harbor" | "reference-rates";
+}
+export interface EvaluationRun extends Omit<EvaluationInput, "tasks"> {
+    revision: number;
+    datasetKey?: string;
+    modelLabel: string;
+    status: "queued" | "running" | "completed" | "failed";
+    trials: EvaluationTrial[];
+    finishedAt?: string;
+    error?: string;
+}
+export interface EvaluationModel {
+    id: string;
+    label: string;
+    model: string;
+    harnesses: Harness[];
+    pricing?: EvaluationPricing;
+    sandbox?: EvaluationSandbox;
+}
+export interface EvaluationChoices {
+    models: EvaluationModel[];
+    sandboxes: EvaluationSandbox[];
+    harborVersion: string;
+    configurable?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/evaluation/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evaluation/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,YAAY,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,SAAS,CAAC;AACvE,MAAM,WAAW,iBAAiB;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AACD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AACD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACZ,iBAAiB,EAAE,MAAM,CAAC;QAC1B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,QAAQ,EAAE,QAAQ,GAAG,WAAW,GAAG,YAAY,GAAG,QAAQ,CAAC;KAC5D,CAAC;CACH;AACD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtE;AACD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,UAAU,CAAC,EAAE,QAAQ,GAAG,iBAAiB,CAAC;CAC3C;AACD,MAAM,WAAW,aAAc,SAAQ,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtD,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AACD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC7B;AACD,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB"}

package/dist/evaluation/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/evaluation/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/evaluation/types.ts"],"names":[],"mappings":""}

package/dist/evaluation/worker-main.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-main.d.ts.map

package/dist/evaluation/worker-main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-main.d.ts","sourceRoot":"","sources":["../../src/evaluation/worker-main.ts"],"names":[],"mappings":""}

package/dist/evaluation/worker-main.js

@@ -0,0 +1,26 @@
+import { fileURLToPath } from "node:url";
+import { Worker } from "@temporalio/worker";
+import { createArtifactStore } from "../artifacts.js";
+import { loadConfig } from "../config.js";
+import { connectTemporalWorker } from "../temporal/connection.js";
+import { createEvaluationActivities } from "./activities.js";
+import { openWorkerRecords } from "./worker-records.js";
+const config = loadConfig();
+const connection = await connectTemporalWorker(config.temporal);
+const credentials = await openWorkerRecords();
+const worker = await Worker.create({
+    connection,
+    namespace: config.temporal.namespace,
+    taskQueue: process.env.SELFBENCH_EVAL_TASK_QUEUE ?? "selfbench-evaluations",
+    workflowsPath: fileURLToPath(new URL("./workflow.js", import.meta.url)),
+    activities: createEvaluationActivities(createArtifactStore(config.artifact), credentials?.records),
+    maxConcurrentActivityTaskExecutions: 1,
+});
+console.log("Evaluation worker ready; waiting for explicit Run submissions");
+try {
+    await worker.run();
+}
+finally {
+    await credentials?.close();
+}
+//# sourceMappingURL=worker-main.js.map

package/dist/evaluation/worker-main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-main.js","sourceRoot":"","sources":["../../src/evaluation/worker-main.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;AAC5B,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AAChE,MAAM,WAAW,GAAG,MAAM,iBAAiB,EAAE,CAAC;AAC9C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC;IACjC,UAAU;IACV,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;IACpC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,uBAAuB;IAC3E,aAAa,EAAE,aAAa,CAAC,IAAI,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvE,UAAU,EAAE,0BAA0B,CACpC,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,EACpC,WAAW,EAAE,OAAO,CACrB;IACD,mCAAmC,EAAE,CAAC;CACvC,CAAC,CAAC;AACH,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;AAC7E,IAAI,CAAC;IACH,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;AACrB,CAAC;QAAS,CAAC;IACT,MAAM,WAAW,EAAE,KAAK,EAAE,CAAC;AAC7B,CAAC"}

package/dist/evaluation/worker-records.d.ts

@@ -0,0 +1,5 @@
+export declare function openWorkerRecords(env?: NodeJS.ProcessEnv): Promise<{
+    records: import("./encrypted-records.js").EncryptedRecordStore;
+    close: () => Promise<void>;
+} | undefined>;
+//# sourceMappingURL=worker-records.d.ts.map

package/dist/evaluation/worker-records.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-records.d.ts","sourceRoot":"","sources":["../../src/evaluation/worker-records.ts"],"names":[],"mappings":"AAGA,wBAAsB,iBAAiB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB;;;eAM3E"}

package/dist/evaluation/worker-records.js

@@ -0,0 +1,11 @@
+import { openDatabase } from "../db/client.js";
+import { createEncryptedRecords } from "./encrypted-records.js";
+export async function openWorkerRecords(env = process.env) {
+    const key = env.SELFBENCH_EVAL_CREDENTIAL_KEY;
+    const url = env.SELFBENCH_DATABASE_URL;
+    if (!key || !url)
+        return undefined;
+    const connection = await openDatabase(url);
+    return { records: createEncryptedRecords(connection.db, key), close: () => connection.close() };
+}
+//# sourceMappingURL=worker-records.js.map

package/dist/evaluation/worker-records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-records.js","sourceRoot":"","sources":["../../src/evaluation/worker-records.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC1E,MAAM,GAAG,GAAG,GAAG,CAAC,6BAA6B,CAAC;IAC9C,MAAM,GAAG,GAAG,GAAG,CAAC,sBAAsB,CAAC;IACvC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IACnC,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,EAAE,OAAO,EAAE,sBAAsB,CAAC,UAAU,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;AAClG,CAAC"}

package/dist/evaluation/workflow.d.ts

@@ -0,0 +1,3 @@
+import type { EvaluationInput } from "./types.js";
+export declare function selfBenchEvaluationWorkflow(input: EvaluationInput): Promise<void>;
+//# sourceMappingURL=workflow.d.ts.map

package/dist/evaluation/workflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../../src/evaluation/workflow.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAYlD,wBAAsB,2BAA2B,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAMvF"}

package/dist/evaluation/workflow.js

@@ -0,0 +1,20 @@
+import { ActivityCancellationType, CancellationScope, proxyActivities } from "@temporalio/workflow";
+const solver = proxyActivities({
+    startToCloseTimeout: "72 hours",
+    heartbeatTimeout: "2 minutes",
+    cancellationType: ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
+    retry: { maximumAttempts: 1 },
+});
+const finalizer = proxyActivities({
+    startToCloseTimeout: "1 minute",
+    retry: { maximumAttempts: 5 },
+});
+export async function selfBenchEvaluationWorkflow(input) {
+    try {
+        await solver.executeSolverEvaluation(input);
+    }
+    catch {
+        await CancellationScope.nonCancellable(() => finalizer.failSolverEvaluation(input));
+    }
+}
+//# sourceMappingURL=workflow.js.map

package/dist/evaluation/workflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow.js","sourceRoot":"","sources":["../../src/evaluation/workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAIpG,MAAM,MAAM,GAAG,eAAe,CAAuB;IACnD,mBAAmB,EAAE,UAAU;IAC/B,gBAAgB,EAAE,WAAW;IAC7B,gBAAgB,EAAE,wBAAwB,CAAC,2BAA2B;IACtE,KAAK,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE;CAC9B,CAAC,CAAC;AACH,MAAM,SAAS,GAAG,eAAe,CAAuB;IACtD,mBAAmB,EAAE,UAAU;IAC/B,KAAK,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,KAAsB;IACtE,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,CAAC;AACH,CAAC"}

package/dist/execution-environment.d.ts

@@ -0,0 +1,3 @@
+export declare function executionEnvironment(): NodeJS.ProcessEnv;
+export declare function withExecutionEnvironment<T>(environment: NodeJS.ProcessEnv, action: () => Promise<T>): Promise<T>;
+//# sourceMappingURL=execution-environment.d.ts.map

package/dist/execution-environment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-environment.d.ts","sourceRoot":"","sources":["../src/execution-environment.ts"],"names":[],"mappings":"AAGA,wBAAgB,oBAAoB,IAAI,MAAM,CAAC,UAAU,CAExD;AACD,wBAAgB,wBAAwB,CAAC,CAAC,EACxC,WAAW,EAAE,MAAM,CAAC,UAAU,EAC9B,MAAM,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,cAGzB"}

package/dist/execution-environment.js

@@ -0,0 +1,9 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+const environments = new AsyncLocalStorage();
+export function executionEnvironment() {
+    return environments.getStore() ?? process.env;
+}
+export function withExecutionEnvironment(environment, action) {
+    return environments.run(environment, action);
+}
+//# sourceMappingURL=execution-environment.js.map

package/dist/execution-environment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-environment.js","sourceRoot":"","sources":["../src/execution-environment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,MAAM,YAAY,GAAG,IAAI,iBAAiB,EAAqB,CAAC;AAChE,MAAM,UAAU,oBAAoB;IAClC,OAAO,YAAY,CAAC,QAAQ,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC;AAChD,CAAC;AACD,MAAM,UAAU,wBAAwB,CACtC,WAA8B,EAC9B,MAAwB;IAExB,OAAO,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;AAC/C,CAAC"}

package/dist/extension-authoring.bundle.js

@@ -0,0 +1,347 @@
+// src/extensions/authoring.ts
+import { cpSync, mkdirSync as mkdirSync2 } from "node:fs";
+import { join as join4 } from "node:path";
+import { Type as Type2 } from "@sinclair/typebox";
+
+// src/extensions/shared/deliverable.ts
+import { existsSync, readFileSync } from "node:fs";
+import { join as join2 } from "node:path";
+
+// src/extensions/shared/schemas.ts
+import { Type } from "@sinclair/typebox";
+var environmentVariableName = Type.String({ pattern: "^[A-Za-z_][A-Za-z0-9_]*$" });
+var environmentVariables = Type.Record(environmentVariableName, Type.String());
+var service = Type.Object({
+  name: Type.String({ pattern: "^[a-z][a-z0-9_-]*$" }),
+  image: Type.String({ minLength: 1 }),
+  environmentVariables,
+  command: Type.Optional(Type.Array(Type.String())),
+  healthcheck: Type.Object({
+    test: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+    intervalSeconds: Type.Integer({ minimum: 1 }),
+    timeoutSeconds: Type.Integer({ minimum: 1 }),
+    retries: Type.Integer({ minimum: 1 }),
+    startPeriodSeconds: Type.Integer({ minimum: 0 })
+  }, { additionalProperties: false })
+}, { additionalProperties: false });
+var environmentContract = Type.Object({
+  schemaVersion: Type.Literal(1),
+  baseImage: Type.String({ minLength: 1 }),
+  rootSetupCommand: Type.String({ minLength: 1 }),
+  setupCommand: Type.String({ minLength: 1 }),
+  smokeCommand: Type.String({ minLength: 1 }),
+  environmentVariables,
+  services: Type.Array(service),
+  source: Type.Union([
+    Type.Literal("repository-dockerfile"),
+    Type.Literal("devcontainer"),
+    Type.Literal("ci-adapted"),
+    Type.Literal("generated")
+  ]),
+  evidence: Type.Array(Type.Object({ path: Type.String({ minLength: 1 }), reason: Type.String({ minLength: 1 }) }, { additionalProperties: false }), { minItems: 1 })
+}, { additionalProperties: false });
+var taskTimeouts = Type.Object({
+  setupSeconds: Type.Integer({ minimum: 1 }),
+  agentSeconds: Type.Integer({ minimum: 1 }),
+  testsSeconds: Type.Integer({ minimum: 1 })
+}, { additionalProperties: false });
+var taskResources = Type.Object({
+  cpus: Type.Number({ exclusiveMinimum: 0 }),
+  memoryMb: Type.Integer({ minimum: 1 }),
+  storageMb: Type.Integer({ minimum: 1 })
+}, { additionalProperties: false });
+var taskDefinition = Type.Object({
+  schemaVersion: Type.Literal(2),
+  difficulty: Type.Union([Type.Literal("easy"), Type.Literal("medium"), Type.Literal("hard")]),
+  taskId: Type.String({ pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*$" }),
+  repo: Type.String({ minLength: 1 }),
+  baseCommit: Type.String({ pattern: "^[0-9a-fA-F]{40}$" }),
+  workdir: Type.String({ minLength: 1 }),
+  testCommand: Type.String({ minLength: 1 }),
+  failToPass: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+  passToPass: Type.Array(Type.String({ minLength: 1 })),
+  testPaths: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+  sourcePr: Type.Integer({ minimum: 1 }),
+  sourceUrl: Type.String({ minLength: 1 }),
+  prompt: Type.String({ minLength: 1 }),
+  timeouts: taskTimeouts,
+  resources: taskResources,
+  environment: environmentContract
+}, { additionalProperties: false });
+
+// src/extensions/shared/static-check.ts
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+function requiredEnvironment(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(`${name} is not configured`);
+  }
+  return value;
+}
+function stageSubmission(definition, testPatch, goldPatch, prefix = "selfbench-submit-") {
+  const directory = mkdtempSync(join(tmpdir(), prefix));
+  const definitionJson = `${JSON.stringify(definition, null, 2)}
+`;
+  writeFileSync(join(directory, "definition.json"), definitionJson);
+  writeFileSync(join(directory, "test.patch"), testPatch);
+  writeFileSync(join(directory, "gold.patch"), goldPatch);
+  return {
+    directory,
+    definitionJson,
+    dispose: () => rmSync(directory, { recursive: true, force: true })
+  };
+}
+function runStaticCheck(staging, extra = [], applyAgainst) {
+  const program = requiredEnvironment("SELFBENCH_CHECK_PROGRAM");
+  const repository = applyAgainst?.repository ?? process.env.SELFBENCH_REPO_DIRECTORY ?? "/work/repo";
+  const result = spawnSync(process.execPath, [
+    program,
+    join(staging, "definition.json"),
+    join(staging, "test.patch"),
+    join(staging, "gold.patch"),
+    process.env.SELFBENCH_RENDER_OUTPUT ?? "/work",
+    ...extra,
+    ...applyAgainst ? ["--repository", repository, "--base", applyAgainst.base] : []
+  ], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+  if (result.status !== 0) {
+    throw new Error(`static check program failed: ${(result.stderr || result.stdout).slice(-4000)}`);
+  }
+  return JSON.parse(result.stdout);
+}
+function staticCheckFailure(verdict, subject) {
+  const lines = verdict.errors.map((error) => `- [${error.gate}] ${error.message}`).join(`
+`);
+  const rendered = verdict.renderedDirectory ? `
+The dry-rendered Harbor tree (as far as rendering succeeded) is under ${verdict.renderedDirectory}.` : "";
+  return failure(`The ${subject} failed the static check; nothing was recorded. Fix every item and try again:
+${lines}${rendered}`);
+}
+function failure(text) {
+  return { content: [{ type: "text", text }], details: {}, isError: true };
+}
+
+// src/extensions/shared/deliverable.ts
+var TASK_DELIVERABLE_FILES = [
+  "definition.json",
+  "instruction.md",
+  "test.patch",
+  "gold.patch"
+];
+function loadTaskDeliverable(root) {
+  const missing = TASK_DELIVERABLE_FILES.filter((name) => !existsSync(join2(root, name)));
+  if (missing.length > 0) {
+    return deliverableFailure(root, missing.map((name) => `${name} is missing`));
+  }
+  let definition;
+  try {
+    const parsed = JSON.parse(readFileSync(join2(root, "definition.json"), "utf8"));
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      throw new Error("must be a JSON object");
+    }
+    definition = parsed;
+  } catch (error) {
+    return deliverableFailure(root, [
+      `definition.json is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
+    ]);
+  }
+  const instruction = readFileSync(join2(root, "instruction.md"), "utf8").trim();
+  const testPatch = readFileSync(join2(root, "test.patch"), "utf8");
+  const goldPatch = readFileSync(join2(root, "gold.patch"), "utf8");
+  const problems = [];
+  if (!instruction) {
+    problems.push("instruction.md is empty");
+  }
+  if (typeof definition.prompt === "string" && definition.prompt.trim() !== instruction) {
+    problems.push("definition.json prompt differs from instruction.md; remove prompt or make them identical");
+  }
+  if (!testPatch.trim()) {
+    problems.push("test.patch is empty");
+  }
+  if (!goldPatch.trim()) {
+    problems.push("gold.patch is empty");
+  }
+  if (problems.length > 0) {
+    return deliverableFailure(root, problems);
+  }
+  return { definition: { ...definition, prompt: instruction }, testPatch, goldPatch };
+}
+function deliverableFailure(root, problems) {
+  return failure(`The deliverable in ${root} is incomplete; nothing was recorded. Fix every item and call the tool again:
+${problems.map((problem) => `- [files] ${problem}`).join(`
+`)}`);
+}
+
+// src/extensions/shared/mailbox.ts
+import { createHash } from "node:crypto";
+import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, renameSync, writeFileSync as writeFileSync2 } from "node:fs";
+import { join as join3 } from "node:path";
+var DEFAULT_MAILBOX = "/work/mailbox";
+var DEFAULT_POLL_MS = 2000;
+var DEFAULT_TIMEOUT_MS = 60 * 60 * 1000;
+function payloadHash(payload) {
+  return createHash("sha256").update(`${JSON.stringify(payload.definition)}\x00${payload.testPatch}\x00${payload.goldPatch}`).digest("hex");
+}
+
+class VerifyClient {
+  #mailbox = process.env.SELFBENCH_MAILBOX ?? DEFAULT_MAILBOX;
+  #budget = Number(process.env.SELFBENCH_VERIFY_BUDGET ?? "0");
+  #pollMs = Number(process.env.SELFBENCH_VERIFY_POLL_MS ?? String(DEFAULT_POLL_MS));
+  #timeoutMs = Number(process.env.SELFBENCH_VERIFY_TIMEOUT_MS ?? String(DEFAULT_TIMEOUT_MS));
+  #used = 0;
+  #sequence = 0;
+  #lastGreenHash;
+  get remaining() {
+    return Math.max(0, this.#budget - this.#used);
+  }
+  verifiedGreen(payload) {
+    return this.#lastGreenHash !== undefined && this.#lastGreenHash === payloadHash(payload);
+  }
+  async verify(kind, payload) {
+    if (this.remaining === 0) {
+      return { kind: "exhausted" };
+    }
+    const requests = join3(this.#mailbox, "requests");
+    const responses = join3(this.#mailbox, "responses");
+    mkdirSync(requests, { recursive: true });
+    mkdirSync(responses, { recursive: true });
+    this.#sequence += 1;
+    const id = `${Date.now()}-${this.#sequence}`;
+    const request = join3(requests, `${id}.json`);
+    writeFileSync2(`${request}.tmp`, `${JSON.stringify({ id, kind, ...payload })}
+`);
+    renameSync(`${request}.tmp`, request);
+    const response = join3(responses, `${id}.json`);
+    const deadline = Date.now() + this.#timeoutMs;
+    while (Date.now() < deadline) {
+      if (existsSync2(response)) {
+        let parsed;
+        try {
+          parsed = JSON.parse(readFileSync2(response, "utf8"));
+        } catch {
+          parsed = undefined;
+        }
+        if (parsed && typeof parsed === "object") {
+          return this.#record(parsed, payload);
+        }
+      }
+      await new Promise((resolve) => setTimeout(resolve, this.#pollMs));
+    }
+    this.#used += 1;
+    return {
+      kind: "error",
+      message: `verify timed out after ${Math.round(this.#timeoutMs / 60000)} minutes; the worker may still be building. ${this.remaining} verify call(s) remain.`
+    };
+  }
+  #record(response, payload) {
+    if (response.kind !== "report") {
+      return {
+        kind: "error",
+        message: typeof response.message === "string" ? response.message : "verify failed on the worker"
+      };
+    }
+    this.#used += 1;
+    const green = response.green === true;
+    if (green) {
+      this.#lastGreenHash = payloadHash(payload);
+    }
+    return {
+      kind: "report",
+      green,
+      summary: typeof response.summary === "string" ? response.summary : "",
+      rendered: typeof response.rendered === "string" ? response.rendered : "",
+      remaining: this.remaining
+    };
+  }
+}
+function verifyOutcomeText(outcome, submitTool) {
+  if (outcome.kind === "exhausted") {
+    return `No verify calls remain in this session. Call ${submitTool} with your best task or explain in your final message why it cannot be made fair.`;
+  }
+  if (outcome.kind === "error") {
+    return `verify could not complete: ${outcome.message}`;
+  }
+  const next = outcome.green ? `Call ${submitTool} with exactly this payload to record it; the worker will reuse this report.` : `Fix the red gates and call verify again, or ${submitTool} if you must (the worker verifies again and a round is spent if it fails).`;
+  return `${outcome.rendered.trim()}
+
+verify result: green=${outcome.green}. ${outcome.remaining} verify call(s) remain in this session. ${next}`;
+}
+
+// src/extensions/authoring.ts
+var noArguments = Type2.Object({}, { additionalProperties: false });
+function applyTarget(definition) {
+  return { base: typeof definition.baseCommit === "string" ? definition.baseCommit : "HEAD" };
+}
+function authoringExtension(pi) {
+  const client = new VerifyClient;
+  const deliverable = () => process.env.SELFBENCH_DELIVERABLE ?? "/work/task";
+  pi.registerTool({
+    name: "verify",
+    label: "Verify SelfBench task",
+    description: "Takes no arguments. Reads the deliverable in /work/task (definition.json, instruction.md, test.patch, gold.patch), runs the static check, then the harness's real verification: trusted compile, audit, image build, smoke, nop, and oracle. Blocks until the report is back (can take up to an hour). Budget-limited per session; the result says how many calls remain.",
+    parameters: noArguments,
+    async execute() {
+      const loaded = loadTaskDeliverable(deliverable());
+      if ("isError" in loaded) {
+        return loaded;
+      }
+      const staging = stageSubmission(loaded.definition, loaded.testPatch, loaded.goldPatch, "selfbench-verify-");
+      try {
+        const verdict = runStaticCheck(staging.directory, [], applyTarget(loaded.definition));
+        if (!verdict.ok) {
+          return staticCheckFailure(verdict, "task");
+        }
+      } finally {
+        staging.dispose();
+      }
+      const outcome = await client.verify("task", loaded);
+      return {
+        content: [{ type: "text", text: verifyOutcomeText(outcome, "submit_task") }],
+        details: { kind: outcome.kind, remaining: client.remaining },
+        ...outcome.kind === "exhausted" ? { isError: true } : {}
+      };
+    }
+  });
+  pi.registerTool({
+    name: "submit_task",
+    label: "Submit SelfBench task",
+    description: "Takes no arguments. Reads the deliverable in /work/task, runs the static check, and records the task; a passing submission ends the session. Run verify first and submit only what verify reported green.",
+    parameters: noArguments,
+    async execute() {
+      const root = requiredEnvironment("SELFBENCH_TASK_OUTPUT");
+      const loaded = loadTaskDeliverable(deliverable());
+      if ("isError" in loaded) {
+        return loaded;
+      }
+      const taskId = String(loaded.definition.taskId ?? "");
+      const staging = stageSubmission(loaded.definition, loaded.testPatch, loaded.goldPatch);
+      try {
+        const verdict = runStaticCheck(staging.directory, [], applyTarget(loaded.definition));
+        if (!verdict.ok) {
+          return staticCheckFailure(verdict, "submission");
+        }
+        const directory = join4(root, taskId);
+        mkdirSync2(root, { recursive: true });
+        mkdirSync2(directory, { recursive: false });
+        cpSync(staging.directory, directory, { recursive: true });
+        const verified = client.verifiedGreen(loaded);
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Submitted ${taskId}; the static check passed and the Harbor tree is dry-rendered under ${verdict.renderedDirectory ?? "/work/rendered"}. ${verified ? "This deliverable matches your last green verify, so the worker reuses that report." : "This deliverable was not verified green in this session, so the worker verifies it now and a round is spent if it fails."} Stop here and wait.`
+            }
+          ],
+          details: { taskId, verified }
+        };
+      } finally {
+        staging.dispose();
+      }
+    }
+  });
+}
+export {
+  authoringExtension as default
+};