self-bench 0.3.6 → 0.3.8

package/dist/evaluation/providers.d.ts

@@ -0,0 +1,11 @@
+import type { Harness } from "./types.js";
+export declare const providerIds: readonly ["openai", "anthropic", "google", "groq", "huggingface", "mistral", "openrouter", "xai"];
+export type EvaluationProvider = (typeof providerIds)[number];
+export declare const providers: Record<EvaluationProvider, {
+    label: string;
+    credential: string;
+    harnesses: Harness[];
+}>;
+export declare const modelIdPattern: RegExp;
+export declare function modelProvider(modelName: string): EvaluationProvider | undefined;
+//# sourceMappingURL=providers.d.ts.map

package/dist/evaluation/providers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../src/evaluation/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,eAAO,MAAM,WAAW,mGASd,CAAC;AACX,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAC9D,eAAO,MAAM,SAAS,EAAE,MAAM,CAC5B,kBAAkB,EAClB;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,EAAE,CAAA;CAAE,CAc5D,CAAC;AACF,eAAO,MAAM,cAAc,QAAyC,CAAC;AACrE,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS,CAG/E"}

package/dist/evaluation/providers.js

@@ -0,0 +1,30 @@
+export const providerIds = [
+    "openai",
+    "anthropic",
+    "google",
+    "groq",
+    "huggingface",
+    "mistral",
+    "openrouter",
+    "xai",
+];
+export const providers = {
+    openai: { label: "OpenAI", credential: "OPENAI_API_KEY", harnesses: ["codex", "pi"] },
+    anthropic: {
+        label: "Anthropic",
+        credential: "ANTHROPIC_API_KEY",
+        harnesses: ["claude-code", "pi"],
+    },
+    google: { label: "Google Gemini", credential: "GEMINI_API_KEY", harnesses: ["pi"] },
+    groq: { label: "Groq", credential: "GROQ_API_KEY", harnesses: ["pi"] },
+    huggingface: { label: "Hugging Face", credential: "HF_TOKEN", harnesses: ["pi"] },
+    mistral: { label: "Mistral", credential: "MISTRAL_API_KEY", harnesses: ["pi"] },
+    openrouter: { label: "OpenRouter", credential: "OPENROUTER_API_KEY", harnesses: ["pi"] },
+    xai: { label: "xAI", credential: "XAI_API_KEY", harnesses: ["pi"] },
+};
+export const modelIdPattern = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/;
+export function modelProvider(modelName) {
+    const prefix = modelName.split("/", 1)[0];
+    return providerIds.find((id) => id === prefix);
+}
+//# sourceMappingURL=providers.js.map

package/dist/evaluation/providers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/evaluation/providers.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,QAAQ;IACR,WAAW;IACX,QAAQ;IACR,MAAM;IACN,aAAa;IACb,SAAS;IACT,YAAY;IACZ,KAAK;CACG,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAGlB;IACF,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;IACrF,SAAS,EAAE;QACT,KAAK,EAAE,WAAW;QAClB,UAAU,EAAE,mBAAmB;QAC/B,SAAS,EAAE,CAAC,aAAa,EAAE,IAAI,CAAC;KACjC;IACD,MAAM,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE;IACnF,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE;IACtE,WAAW,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE;IACjF,OAAO,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE;IAC/E,UAAU,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,oBAAoB,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE;IACxF,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE;CACpE,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG,sCAAsC,CAAC;AACrE,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;AACjD,CAAC"}

package/dist/evaluation/routes.d.ts

@@ -0,0 +1,23 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { ArtifactStore } from "../artifacts.js";
+import type { User, UserStore } from "../auth/users.js";
+import type { RepoStore } from "../site/repo-store.js";
+import type { TaskStore } from "../site/task-store.js";
+import type { CodexLogins } from "./codex-login.js";
+import type { EncryptedRecordStore } from "./encrypted-records.js";
+import type { EvaluationInput } from "./types.js";
+export interface EvaluationRoutesOptions {
+    users: UserStore;
+    repos: RepoStore;
+    tasks: TaskStore;
+    artifacts: ArtifactStore;
+    publicUrl: string;
+    start(input: EvaluationInput): Promise<void>;
+    env?: NodeJS.ProcessEnv;
+    records?: EncryptedRecordStore;
+    codexLogins?: CodexLogins;
+}
+export declare function createEvaluationRoutes(options: EvaluationRoutesOptions): {
+    handle(request: IncomingMessage, url: URL, response: ServerResponse, user: User): Promise<boolean>;
+};
+//# sourceMappingURL=routes.d.ts.map

package/dist/evaluation/routes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"routes.d.ts","sourceRoot":"","sources":["../../src/evaluation/routes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAYnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAmBlD,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,SAAS,EAAE,aAAa,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AACD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,uBAAuB;oBAIxD,eAAe,OACnB,GAAG,YACE,cAAc,QAClB,IAAI,GACT,OAAO,CAAC,OAAO,CAAC;EA6MtB"}

package/dist/evaluation/routes.js

@@ -0,0 +1,212 @@
+import { z } from "zod";
+import { readBody, sendJson, trustedMutation } from "../api/http.js";
+import { tenantFor } from "../site/tenant.js";
+import { codexLoginRoutes } from "./codex-login-routes.js";
+import { evaluationSandboxes } from "./config.js";
+import { harnessIds } from "./harnesses.js";
+import { orgCredentialRoutes } from "./org-credential-routes.js";
+import { platformRoutes } from "./platform-routes.js";
+import { availableChoices, handleSetup } from "./profile-routes.js";
+import { evaluationPrefix, getEvaluation, initialEvaluation, listEvaluations, saveEvaluation, } from "./store.js";
+const route = /^\/api\/orgs\/([A-Za-z0-9_.-]+)\/repos\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)\/evaluations(?:\/(options|profiles|[a-f0-9-]{36}))?(?:\/artifacts)?$/;
+const requestSchema = z
+    .object({
+    id: z.uuid().transform((value) => value.toLowerCase()),
+    model: z.string().min(1).max(60),
+    harnesses: z.array(z.enum(harnessIds)).min(1).max(harnessIds.length),
+    sandbox: z.enum(evaluationSandboxes),
+    tasks: z
+        .array(z
+        .object({ runId: z.string().min(1).max(100), taskId: z.string().min(1).max(200) })
+        .strict())
+        .min(1),
+})
+    .strict();
+export function createEvaluationRoutes(options) {
+    const { users, repos, tasks, artifacts } = options;
+    return {
+        async handle(request, url, response, user) {
+            if (await codexLoginRoutes(options, request, url, response, user))
+                return true;
+            if (await orgCredentialRoutes(options, request, url, response, user))
+                return true;
+            if (await platformRoutes(options, request, url, response, user))
+                return true;
+            const match = route.exec(url.pathname);
+            if (!match?.[1] || !match[2] || !match[3])
+                return false;
+            response.setHeader("cache-control", "no-store");
+            const tenant = await tenantFor(users, user, match[1]);
+            const repo = tenant ? await repos.find(tenant.id, `${match[2]}/${match[3]}`) : undefined;
+            if (!tenant || !repo) {
+                sendJson(response, 404, { error: "Repository is not connected here" });
+                return true;
+            }
+            const env = options.env ?? process.env;
+            if (match[4] === "profiles") {
+                await handleSetup(request, response, artifacts, {
+                    repoId: repo.id,
+                    ownerId: user.githubId,
+                    tenant: tenant.login,
+                    publicUrl: options.publicUrl,
+                    trusted: trustedMutation(request, options.publicUrl, user),
+                }, env);
+                return true;
+            }
+            if (request.method === "GET") {
+                if (match[4] === "options") {
+                    const available = (await tasks.listForRepo(repo.id)).filter((task) => task.bundleKey &&
+                        task.pipelineStatus === "accepted" &&
+                        task.review?.decision === "approve");
+                    sendJson(response, 200, {
+                        ...(await availableChoices(artifacts, repo.id, user.githubId, tenant.login, env)),
+                        tasks: available.map((task) => ({
+                            runId: task.runId,
+                            taskId: task.taskId,
+                            difficulty: task.difficulty,
+                        })),
+                    });
+                }
+                else if (match[4]) {
+                    const run = await getEvaluation(artifacts, repo.id, match[4]);
+                    if (!run) {
+                        sendJson(response, 404, { error: "Evaluation not found" });
+                        return true;
+                    }
+                    if (url.pathname.endsWith("/artifacts")) {
+                        const name = url.searchParams.get("name") ?? "";
+                        if (!run.trials.some((trial) => trial.artifacts.includes(name))) {
+                            sendJson(response, 404, { error: "Artifact not found" });
+                            return true;
+                        }
+                        const bytes = await artifacts.getByKey(`${evaluationPrefix(repo.id, run.id)}artifacts/${name}`);
+                        if (!bytes) {
+                            sendJson(response, 404, { error: "Artifact not found" });
+                            return true;
+                        }
+                        response.writeHead(200, {
+                            "content-type": "text/plain; charset=utf-8",
+                            "x-content-type-options": "nosniff",
+                            "content-disposition": "attachment; filename=solver-artifact.txt",
+                        });
+                        response.end(bytes);
+                    }
+                    else
+                        sendJson(response, 200, run);
+                }
+                else {
+                    sendJson(response, 200, {
+                        runs: (await listEvaluations(artifacts, repo.id)).map((run) => ({
+                            ...run,
+                            trials: run.trials.map((trial) => ({ ...trial, log: "", steps: [], artifacts: [] })),
+                        })),
+                    });
+                }
+                return true;
+            }
+            if (request.method !== "POST" || match[4]) {
+                sendJson(response, 405, { error: "Method not allowed" });
+                return true;
+            }
+            if (!trustedMutation(request, options.publicUrl, user)) {
+                sendJson(response, 403, { error: "Same-origin JSON request required" });
+                return true;
+            }
+            let body;
+            try {
+                body = requestSchema.parse(JSON.parse((await readBody(request, 16_384)).toString("utf8")));
+            }
+            catch {
+                sendJson(response, 400, { error: "Invalid evaluation selection" });
+                return true;
+            }
+            const choices = await availableChoices(artifacts, repo.id, user.githubId, tenant.login, env);
+            const model = choices.models.find((candidate) => candidate.id === body.model);
+            if (!model ||
+                (model.sandbox
+                    ? model.sandbox !== body.sandbox
+                    : !choices.sandboxes.includes(body.sandbox)) ||
+                body.harnesses.some((harness) => !model.harnesses.includes(harness)) ||
+                new Set(body.harnesses).size !== body.harnesses.length ||
+                new Set(body.tasks.map((task) => `${task.runId}/${task.taskId}`)).size !== body.tasks.length) {
+                sendJson(response, 400, {
+                    error: "Select configured models, harnesses, sandbox and unique tasks",
+                });
+                return true;
+            }
+            const selected = await Promise.all(body.tasks.map((task) => tasks.find(repo.id, task.runId, task.taskId)));
+            if (selected.some((task) => !task?.bundleKey ||
+                task.pipelineStatus !== "accepted" ||
+                task.review?.decision !== "approve")) {
+                sendJson(response, 400, {
+                    error: "Every task must belong to this repository and have a human-approved Harbor bundle",
+                });
+                return true;
+            }
+            if (new Set(selected.map((task) => `${task?.runId}/${task?.taskId}`)).size !== selected.length) {
+                sendJson(response, 400, { error: "Select each task only once" });
+                return true;
+            }
+            const existing = await getEvaluation(artifacts, repo.id, body.id);
+            if (existing) {
+                if (existing.model !== body.model ||
+                    existing.sandbox !== body.sandbox ||
+                    JSON.stringify(existing.harnesses) !== JSON.stringify(body.harnesses) ||
+                    JSON.stringify(existing.trials.map((trial) => `${trial.runId}/${trial.taskId}/${trial.harness}`)) !==
+                        JSON.stringify(body.tasks.flatMap((task) => body.harnesses.map((harness) => `${task.runId}/${task.taskId}/${harness}`)))) {
+                    sendJson(response, 409, { error: "This request ID belongs to a different selection" });
+                    return true;
+                }
+                if (existing.status === "queued") {
+                    const saved = await artifacts.getByKey(`${evaluationPrefix(repo.id, body.id)}request.json`);
+                    if (!saved)
+                        throw new Error("Evaluation request snapshot is missing");
+                    await options.start(JSON.parse(Buffer.from(saved).toString("utf8")));
+                }
+                sendJson(response, 202, existing);
+                return true;
+            }
+            let input = {
+                ...body,
+                modelName: model.model,
+                ...(model.id.startsWith("saved-") ? { credentialOwnerId: user.githubId } : {}),
+                ...(model.pricing ? { pricing: model.pricing } : {}),
+                repoId: repo.id,
+                tenant: tenant.login,
+                startedBy: user.login,
+                createdAt: new Date().toISOString(),
+                tasks: selected.map((task) => {
+                    if (!task?.bundleKey)
+                        throw new Error("Task bundle is missing");
+                    return { runId: task.runId, taskId: task.taskId, bundleKey: task.bundleKey };
+                }),
+            };
+            const requestKey = `${evaluationPrefix(repo.id, body.id)}request.json`;
+            const savedRequest = await artifacts.getByKey(requestKey);
+            if (savedRequest) {
+                const previous = JSON.parse(Buffer.from(savedRequest).toString("utf8"));
+                const signature = (value) => JSON.stringify({
+                    model: value.model,
+                    modelName: value.modelName,
+                    sandbox: value.sandbox,
+                    harnesses: value.harnesses,
+                    tasks: value.tasks.map(({ runId, taskId }) => ({ runId, taskId })),
+                });
+                if (signature(previous) !== signature(input)) {
+                    sendJson(response, 409, { error: "This request ID belongs to a different selection" });
+                    return true;
+                }
+                input = previous;
+            }
+            else {
+                await artifacts.put(requestKey, Buffer.from(JSON.stringify(input)), "application/json");
+            }
+            const run = initialEvaluation(input, model.label);
+            await saveEvaluation(artifacts, run);
+            await options.start(input);
+            sendJson(response, 202, run);
+            return true;
+        },
+    };
+}
+//# sourceMappingURL=routes.js.map

package/dist/evaluation/routes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"routes.js","sourceRoot":"","sources":["../../src/evaluation/routes.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAKrE,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,cAAc,GACf,MAAM,YAAY,CAAC;AAGpB,MAAM,KAAK,GACT,mJAAmJ,CAAC;AACtJ,MAAM,aAAa,GAAG,CAAC;KACpB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IACtD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IAChC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;IACpE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC;IACpC,KAAK,EAAE,CAAC;SACL,KAAK,CACJ,CAAC;SACE,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;SACjF,MAAM,EAAE,CACZ;SACA,GAAG,CAAC,CAAC,CAAC;CACV,CAAC;KACD,MAAM,EAAE,CAAC;AAYZ,MAAM,UAAU,sBAAsB,CAAC,OAAgC;IACrE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IACnD,OAAO;QACL,KAAK,CAAC,MAAM,CACV,OAAwB,EACxB,GAAQ,EACR,QAAwB,EACxB,IAAU;YAEV,IAAI,MAAM,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC/E,IAAI,MAAM,mBAAmB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAClF,IAAI,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;YACxD,QAAQ,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YAChD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACzF,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrB,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;YACvC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;gBAC5B,MAAM,WAAW,CACf,OAAO,EACP,QAAQ,EACR,SAAS,EACT;oBACE,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,OAAO,EAAE,IAAI,CAAC,QAAQ;oBACtB,MAAM,EAAE,MAAM,CAAC,KAAK;oBACpB,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,OAAO,EAAE,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;iBAC3D,EACD,GAAG,CACJ,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC7B,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CACzD,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,SAAS;wBACd,IAAI,CAAC,cAAc,KAAK,UAAU;wBAClC,IAAI,CAAC,MAAM,EAAE,QAAQ,KAAK,SAAS,CACtC,CAAC;oBACF,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;wBACtB,GAAG,CAAC,MAAM,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;wBACjF,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;4BAC9B,KAAK,EAAE,IAAI,CAAC,KAAK;4BACjB,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,UAAU,EAAE,IAAI,CAAC,UAAU;yBAC5B,CAAC,CAAC;qBACJ,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpB,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC9D,IAAI,CAAC,GAAG,EAAE,CAAC;wBACT,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE,CAAC,CAAC;wBAC3D,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;wBACxC,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;wBAChD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;4BAChE,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;4BACzD,OAAO,IAAI,CAAC;wBACd,CAAC;wBACD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,QAAQ,CACpC,GAAG,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,aAAa,IAAI,EAAE,CACxD,CAAC;wBACF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACX,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;4BACzD,OAAO,IAAI,CAAC;wBACd,CAAC;wBACD,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE;4BACtB,cAAc,EAAE,2BAA2B;4BAC3C,wBAAwB,EAAE,SAAS;4BACnC,qBAAqB,EAAE,0CAA0C;yBAClE,CAAC,CAAC;wBACH,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACtB,CAAC;;wBAAM,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBACtC,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;wBACtB,IAAI,EAAE,CAAC,MAAM,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;4BAC9D,GAAG,GAAG;4BACN,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;yBACrF,CAAC,CAAC;qBACJ,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1C,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBACvD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBACxE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,IAAmC,CAAC;YACxC,IAAI,CAAC;gBACH,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7F,CAAC;YAAC,MAAM,CAAC;gBACP,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC,CAAC;gBACnE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC7F,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9E,IACE,CAAC,KAAK;gBACN,CAAC,KAAK,CAAC,OAAO;oBACZ,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;oBAChC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC9C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpE,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM;gBACtD,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,EAC5F,CAAC;gBACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;oBACtB,KAAK,EAAE,+DAA+D;iBACvE,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CACvE,CAAC;YACF,IACE,QAAQ,CAAC,IAAI,CACX,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,IAAI,EAAE,SAAS;gBAChB,IAAI,CAAC,cAAc,KAAK,UAAU;gBAClC,IAAI,CAAC,MAAM,EAAE,QAAQ,KAAK,SAAS,CACtC,EACD,CAAC;gBACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;oBACtB,KAAK,EACH,mFAAmF;iBACtF,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IACE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,EAC1F,CAAC;gBACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC,CAAC;gBACjE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,IAAI,QAAQ,EAAE,CAAC;gBACb,IACE,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;oBAC7B,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;oBACjC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC;oBACrE,IAAI,CAAC,SAAS,CACZ,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAClF;wBACC,IAAI,CAAC,SAAS,CACZ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAC1B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAC3E,CACF,EACH,CAAC;oBACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,kDAAkD,EAAE,CAAC,CAAC;oBACvF,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,QAAQ,CACpC,GAAG,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,cAAc,CACpD,CAAC;oBACF,IAAI,CAAC,KAAK;wBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;oBACtE,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAoB,CAAC,CAAC;gBAC1F,CAAC;gBACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,KAAK,GAAoB;gBAC3B,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK,CAAC,KAAK;gBACtB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,MAAM,CAAC,KAAK;gBACpB,SAAS,EAAE,IAAI,CAAC,KAAK;gBACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC3B,IAAI,CAAC,IAAI,EAAE,SAAS;wBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;oBAChE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC/E,CAAC,CAAC;aACH,CAAC;YACF,MAAM,UAAU,GAAG,GAAG,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC;YACvE,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC1D,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAoB,CAAC;gBAC3F,MAAM,SAAS,GAAG,CAAC,KAAsB,EAAE,EAAE,CAC3C,IAAI,CAAC,SAAS,CAAC;oBACb,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;iBACnE,CAAC,CAAC;gBACL,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7C,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,kDAAkD,EAAE,CAAC,CAAC;oBACvF,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,KAAK,GAAG,QAAQ,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAC1F,CAAC;YACD,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,cAAc,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACrC,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3B,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/evaluation/runner.d.ts

@@ -0,0 +1,16 @@
+import type { ArtifactStore } from "../artifacts.js";
+import { runCommand } from "../process.js";
+import type { EncryptedRecordStore } from "./encrypted-records.js";
+import { type ThinkingLevel } from "./model-options.js";
+import type { EvaluationInput, Harness } from "./types.js";
+export declare function solverArguments(taskPath: string, jobs: string, harness: Harness, model: string, sandbox: string, thinking?: ThinkingLevel): string[];
+export interface RunnerOptions {
+    command?: typeof runCommand;
+    env?: NodeJS.ProcessEnv;
+    signal?: AbortSignal;
+    heartbeat?: () => void;
+    pollMs?: number;
+    records?: EncryptedRecordStore;
+}
+export declare function executeEvaluation(store: ArtifactStore, input: EvaluationInput, options?: RunnerOptions): Promise<void>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/evaluation/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evaluation/runner.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAI3C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,OAAO,EAAE,KAAK,aAAa,EAAqB,MAAM,oBAAoB,CAAC;AAY3E,OAAO,KAAK,EAAE,eAAe,EAAkC,OAAO,EAAE,MAAM,YAAY,CAAC;AAE3F,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,aAAa,GACvB,MAAM,EAAE,CAyBV;AACD,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,UAAU,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AACD,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,aAAa,EACpB,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAyGf"}

package/dist/evaluation/runner.js

@@ -0,0 +1,241 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { extractRegularArchive } from "../archive.js";
+import { runCommand } from "../process.js";
+import { HARBOR_VERSION, solverEnvironment } from "./config.js";
+import { trialCost } from "./cost.js";
+import { credentialExecution } from "./credential-execution.js";
+import { gatewayTrial, solverAgent } from "./gateway-execution.js";
+import { thinkingArguments } from "./model-options.js";
+import { boundedSteps, collectOutput, completeLines, piSteps, record, redactOutput, trajectorySteps, } from "./output.js";
+import { profileEnvironment } from "./profiles.js";
+import { evaluationPrefix, getEvaluation, saveEvaluation } from "./store.js";
+export function solverArguments(taskPath, jobs, harness, model, sandbox, thinking) {
+    return [
+        "run",
+        "--path",
+        taskPath,
+        "--agent",
+        solverAgent(harness, model),
+        "--model",
+        model,
+        "--env",
+        sandbox === "e2b" ? "harbor_e2b:SelfBenchE2BEnvironment" : sandbox,
+        "--jobs-dir",
+        jobs,
+        "--job-name",
+        "solver",
+        "--n-attempts",
+        "1",
+        "--n-concurrent",
+        "1",
+        "--max-retries",
+        "0",
+        "--delete",
+        "--yes",
+        ...thinkingArguments(harness, thinking),
+    ];
+}
+export async function executeEvaluation(store, input, options = {}) {
+    const run = await getEvaluation(store, input.repoId, input.id);
+    if (!run)
+        throw new Error("Evaluation record is missing");
+    if (run.status !== "queued")
+        throw new Error("Evaluation already attempted; refusing to repeat model spend");
+    const root = await mkdtemp(join(tmpdir(), "selfbench-evaluation-"));
+    const command = options.command ?? runCommand;
+    const environment = options.env ?? process.env;
+    const secrets = Object.entries(environment)
+        .filter(([name]) => /SECRET|TOKEN|PASSWORD|API_KEY/.test(name))
+        .map(([, value]) => value ?? "")
+        .filter(Boolean);
+    const redact = (text) => redactOutput(text, secrets);
+    run.status = "running";
+    try {
+        await saveEvaluation(store, run);
+        const home = join(root, "home");
+        await mkdir(home, { mode: 0o700 });
+        const resolved = input.credentials
+            ? environment
+            : await profileEnvironment(store, input, environment);
+        secrets.push(...Object.entries(resolved)
+            .filter(([name]) => name.startsWith("SELFBENCH_EVAL_SECRET_"))
+            .map(([, value]) => value ?? "")
+            .filter(Boolean));
+        if (input.credentials && !options.records)
+            throw new Error("Credential storage unavailable on the worker");
+        const execution = input.credentials && options.records
+            ? await credentialExecution(input, home, environment, options.records)
+            : { ...solverEnvironment(input, home, resolved), secrets: [] };
+        secrets.push(...execution.secrets);
+        const { profile, child } = execution;
+        child.PYTHONPATH = dirname(fileURLToPath(import.meta.url));
+        const version = await command("harbor", ["--version"], { env: child, timeoutMs: 15_000 });
+        if (version.stdout.trim() !== HARBOR_VERSION)
+            throw new Error("Worker Harbor version does not match the supported version");
+        for (const [index, trial] of run.trials.entries()) {
+            options.signal?.throwIfAborted();
+            const task = input.tasks.find((candidate) => candidate.runId === trial.runId && candidate.taskId === trial.taskId);
+            if (!task)
+                throw new Error("Evaluation task snapshot is missing");
+            const trialRoot = join(root, String(index));
+            await mkdir(trialRoot);
+            trial.status = "running";
+            trial.startedAt = new Date().toISOString();
+            await saveEvaluation(store, run);
+            try {
+                const bundle = await store.getByKey(task.bundleKey);
+                if (!bundle || bundle.byteLength > 100 * 1024 * 1024)
+                    throw new Error("Task bundle is missing or exceeds 100 MiB");
+                const archive = join(trialRoot, "task.tar.gz");
+                await writeFile(archive, bundle, { mode: 0o600 });
+                const extracted = join(trialRoot, "task");
+                await mkdir(extracted);
+                await extractRegularArchive(archive, extracted, options.signal ? { signal: options.signal } : {});
+                const taskPath = await readFile(join(extracted, "harbor-task", "task.toml")).then(() => join(extracted, "harbor-task"), () => extracted);
+                await readFile(join(taskPath, "task.toml"));
+                await runTrial({
+                    store,
+                    run,
+                    trial,
+                    index,
+                    taskPath,
+                    jobs: join(trialRoot, "jobs"),
+                    ...gatewayTrial(input, trial.harness, profile.model, child),
+                    command,
+                    redact,
+                    options,
+                });
+            }
+            catch (error) {
+                trial.status = "failed";
+                trial.error = redact(error instanceof Error ? error.message : "Solver failed");
+            }
+            trial.finishedAt = new Date().toISOString();
+            await saveEvaluation(store, run);
+        }
+        run.status = run.trials.some((trial) => trial.status === "failed") ? "failed" : "completed";
+    }
+    catch (error) {
+        run.status = "failed";
+        run.error = redact(error instanceof Error ? error.message : "Evaluation failed");
+    }
+    finally {
+        for (const trial of run.trials) {
+            if (trial.status === "queued" || trial.status === "running") {
+                trial.status = "failed";
+                trial.error = "Not completed because the evaluation stopped";
+            }
+        }
+        run.finishedAt = new Date().toISOString();
+        try {
+            await saveEvaluation(store, run);
+        }
+        finally {
+            await rm(root, { recursive: true, force: true });
+        }
+    }
+}
+async function runTrial(context) {
+    const { store, run, trial, index, taskPath, jobs, model, child, command, redact, options } = context;
+    let stdout = "";
+    let polling = Promise.resolve();
+    let refreshing = false;
+    let lastSnapshot = "";
+    const refresh = async (archive) => {
+        const files = await collectOutput(jobs);
+        const clean = new Map([...files].map(([name, text]) => [
+            name,
+            redact(!archive && !name.endsWith(".json") ? completeLines(text) : text),
+        ]));
+        const trajectory = [...clean].find(([name]) => name.endsWith("/trajectory.json"));
+        const pi = [...clean].find(([name]) => name.endsWith("/pi.txt"));
+        if (trajectory) {
+            try {
+                trial.steps = trajectorySteps(JSON.parse(trajectory[1]));
+            }
+            catch {
+                trial.steps = [];
+            }
+        }
+        else if (pi)
+            trial.steps = piSteps(pi[1]);
+        trial.steps = boundedSteps(trial.steps);
+        trial.log = [
+            redact(archive ? stdout : completeLines(stdout)),
+            ...[...clean]
+                .filter(([name]) => /\.(log|txt)$/.test(name))
+                .map(([name, text]) => `\n--- ${name} ---\n${text}`),
+        ]
+            .join("\n")
+            .slice(-100_000);
+        if (archive) {
+            for (const [name, text] of clean) {
+                const artifactName = `${index}/${name}`;
+                await store.put(`${evaluationPrefix(run.repoId, run.id)}artifacts/${artifactName}`, Buffer.from(text), "text/plain");
+                trial.artifacts.push(artifactName);
+            }
+            const result = [...files].find(([name]) => /^solver\/[^/]+\/result\.json$/.test(name));
+            if (!result)
+                throw new Error("Harbor did not produce a trial result");
+            const parsed = record(JSON.parse(result[1]));
+            Object.assign(trial, trialCost(run, trial.harness, files, parsed));
+            const rewards = record(record(parsed.verifier_result).rewards);
+            trial.rewards = Object.fromEntries(Object.entries(rewards).filter((pair) => typeof pair[1] === "number" && Number.isFinite(pair[1])));
+            if (parsed.exception_info)
+                throw new Error(String(record(parsed.exception_info).exception_message ?? "Harbor trial failed"));
+            if (Object.keys(trial.rewards).length === 0)
+                throw new Error("Harbor returned no verifier scores");
+        }
+        const snapshot = JSON.stringify(trial);
+        if (snapshot !== lastSnapshot) {
+            await saveEvaluation(store, run);
+            lastSnapshot = snapshot;
+        }
+    };
+    const timer = setInterval(() => {
+        options.heartbeat?.();
+        if (refreshing)
+            return;
+        refreshing = true;
+        polling = polling
+            .then(() => refresh(false))
+            .catch(() => undefined)
+            .finally(() => {
+            refreshing = false;
+        });
+    }, options.pollMs ?? 3000);
+    try {
+        const result = await command("harbor", solverArguments(taskPath, jobs, trial.harness, model, run.sandbox, run.thinking), {
+            env: child,
+            cwd: taskPath,
+            timeoutMs: 2 * 60 * 60 * 1000,
+            allowFailure: true,
+            ...(options.signal ? { signal: options.signal } : {}),
+            onOutput: (_stream, chunk) => {
+                stdout = `${stdout}${Buffer.from(chunk).toString("utf8")}`.slice(-200_000);
+            },
+        });
+        clearInterval(timer);
+        await polling;
+        await refresh(true);
+        if (result.exitCode !== 0)
+            throw new Error(`Harbor exited with status ${result.exitCode}`);
+        trial.status = "completed";
+    }
+    catch (error) {
+        clearInterval(timer);
+        await polling;
+        if (trial.artifacts.length === 0)
+            await refresh(true).catch(() => undefined);
+        throw error;
+    }
+    finally {
+        clearInterval(timer);
+        await polling;
+        trial.log = `${trial.log}\n${redact(stdout)}`.slice(-100_000);
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/evaluation/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evaluation/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAsB,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EACL,YAAY,EACZ,aAAa,EACb,aAAa,EACb,OAAO,EACP,MAAM,EACN,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG7E,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,IAAY,EACZ,OAAgB,EAChB,KAAa,EACb,OAAe,EACf,QAAwB;IAExB,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ;QACR,SAAS;QACT,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC;QAC3B,SAAS;QACT,KAAK;QACL,OAAO;QACP,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,OAAO;QAClE,YAAY;QACZ,IAAI;QACJ,YAAY;QACZ,QAAQ;QACR,cAAc;QACd,GAAG;QACH,gBAAgB;QAChB,GAAG;QACH,eAAe;QACf,GAAG;QACH,UAAU;QACV,OAAO;QACP,GAAG,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC;KACxC,CAAC;AACJ,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAoB,EACpB,KAAsB,EACtB,UAAyB,EAAE;IAE3B,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC1D,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;QACzB,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,CAAC,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,UAAU,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;SACxC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC9D,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;SAC/B,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7D,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW;YAChC,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,MAAM,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CACV,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;aACxB,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;aAC7D,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;aAC/B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;QACF,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,OAAO;YACvC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,MAAM,SAAS,GACb,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO;YAClC,CAAC,CAAC,MAAM,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC;YACtE,CAAC,CAAC,EAAE,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;QACrC,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1F,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,cAAc;YAC1C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAChF,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAC3B,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,IAAI,SAAS,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CACpF,CAAC;YACF,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YAClE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5C,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;YACvB,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC;YACzB,KAAK,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACpD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI;oBAClD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBAC/C,MAAM,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAClD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC1C,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;gBACvB,MAAM,qBAAqB,CACzB,OAAO,EACP,SAAS,EACT,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CACjD,CAAC;gBACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC,IAAI,CAC/E,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EACpC,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;gBACF,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;gBAC5C,MAAM,QAAQ,CAAC;oBACb,KAAK;oBACL,GAAG;oBACH,KAAK;oBACL,KAAK;oBACL,QAAQ;oBACR,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;oBAC7B,GAAG,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;oBAC3D,OAAO;oBACP,MAAM;oBACN,OAAO;iBACR,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;gBACxB,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;YACjF,CAAC;YACD,KAAK,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;IAC9F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;QACtB,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC;IACnF,CAAC;YAAS,CAAC;QACT,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC5D,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;gBACxB,KAAK,CAAC,KAAK,GAAG,8CAA8C,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,GAAG,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;AACH,CAAC;AACD,KAAK,UAAU,QAAQ,CAAC,OAYvB;IACC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,GACxF,OAAO,CAAC;IACV,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAChC,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,MAAM,OAAO,GAAG,KAAK,EAAE,OAAgB,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAC/B,IAAI;YACJ,MAAM,CAAC,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;SACzE,CAAC,CACH,CAAC;QACF,MAAM,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAClF,MAAM,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACjE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,KAAK,CAAC,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;aAAM,IAAI,EAAE;YAAE,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,KAAK,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,KAAK,CAAC,GAAG,GAAG;YACV,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAChD,GAAG,CAAC,GAAG,KAAK,CAAC;iBACV,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC7C,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,SAAS,IAAI,SAAS,IAAI,EAAE,CAAC;SACvD;aACE,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;gBACjC,MAAM,YAAY,GAAG,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;gBACxC,MAAM,KAAK,CAAC,GAAG,CACb,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,aAAa,YAAY,EAAE,EAClE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EACjB,YAAY,CACb,CAAC;gBACF,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrC,CAAC;YACD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACvF,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACtE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC;YAC/D,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,WAAW,CAChC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAC5B,CAAC,IAAI,EAA4B,EAAE,CACjC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAC1D,CACF,CAAC;YACF,IAAI,MAAM,CAAC,cAAc;gBACvB,MAAM,IAAI,KAAK,CACb,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,iBAAiB,IAAI,qBAAqB,CAAC,CACjF,CAAC;YACJ,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC9B,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACjC,YAAY,GAAG,QAAQ,CAAC;QAC1B,CAAC;IACH,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;QACtB,IAAI,UAAU;YAAE,OAAO;QACvB,UAAU,GAAG,IAAI,CAAC;QAClB,OAAO,GAAG,OAAO;aACd,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;aAC1B,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;aACtB,OAAO,CAAC,GAAG,EAAE;YACZ,UAAU,GAAG,KAAK,CAAC;QACrB,CAAC,CAAC,CAAC;IACP,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAC1B,QAAQ,EACR,eAAe,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,EAChF;YACE,GAAG,EAAE,KAAK;YACV,GAAG,EAAE,QAAQ;YACb,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YAC7B,YAAY,EAAE,IAAI;YAClB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,QAAQ,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBAC3B,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YAC7E,CAAC;SACF,CACF,CAAC;QACF,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,OAAO,CAAC;QACd,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3F,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;IAC7B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,OAAO,CAAC;QACd,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,OAAO,CAAC;QACd,KAAK,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}

package/dist/evaluation/store.d.ts

@@ -0,0 +1,8 @@
+import type { ArtifactStore } from "../artifacts.js";
+import type { EvaluationInput, EvaluationRun } from "./types.js";
+export declare function evaluationPrefix(repoId: number, id?: string): string;
+export declare function initialEvaluation(input: EvaluationInput, modelLabel: string): EvaluationRun;
+export declare function saveEvaluation(store: ArtifactStore, run: EvaluationRun): Promise<void>;
+export declare function getEvaluation(store: ArtifactStore, repoId: number, id: string): Promise<EvaluationRun | undefined>;
+export declare function listEvaluations(store: ArtifactStore, repoId: number): Promise<EvaluationRun[]>;
+//# sourceMappingURL=store.d.ts.map

package/dist/evaluation/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/evaluation/store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEjE,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,SAAK,GAAG,MAAM,CAIhE;AACD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,CA6B3F;AACD,wBAAsB,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAO5F;AACD,wBAAsB,aAAa,CACjC,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAKpC;AACD,wBAAsB,eAAe,CACnC,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,aAAa,EAAE,CAAC,CAoB1B"}