self-bench 0.3.6 → 0.3.8

package/dist/evaluation/comparisons.js

@@ -0,0 +1,161 @@
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import { readAccount, updateAccount } from "./account.js";
+import { catalog, hostedSandboxes } from "./catalog.js";
+import { harnessIds } from "./harnesses.js";
+import { routeFor, thinkingLevels, thinkingOptions } from "./model-options.js";
+import { modelIdPattern } from "./providers.js";
+import { getEvaluation } from "./store.js";
+export const comparisonSchema = z
+    .object({
+    id: z.uuid(),
+    tasks: z
+        .array(z
+        .object({ runId: z.string().min(1).max(100), taskId: z.string().min(1).max(200) })
+        .strict())
+        .min(1),
+    models: z
+        .array(z
+        .object({
+        catalogId: z.string().max(80),
+        customModel: z.string().regex(modelIdPattern).optional(),
+        credentialId: z.uuid(),
+        thinking: z.enum(thinkingLevels).optional(),
+        harnesses: z.array(z.enum(harnessIds)).min(1).max(harnessIds.length),
+    })
+        .strict())
+        .min(1)
+        .max(12),
+    sandbox: z.enum(hostedSandboxes),
+    sandboxCredentialId: z.uuid(),
+})
+    .strict();
+export async function createComparison(records, tasks, scope, draft) {
+    const selection = comparisonSchema.parse(draft);
+    const signature = JSON.stringify(selection);
+    const previous = (await readAccount(records, scope.ownerId)).comparisons.find((entry) => entry.id === selection.id);
+    if (previous) {
+        if (previous.repoId !== scope.repoId || previous.signature !== signature)
+            throw new Error("Comparison ID belongs to a different selection");
+        return previous;
+    }
+    if (new Set(selection.tasks.map((task) => `${task.runId}/${task.taskId}`)).size !==
+        selection.tasks.length)
+        throw new Error("Duplicate task selection");
+    const bundles = await Promise.all(selection.tasks.map((task) => tasks.find(scope.repoId, task.runId, task.taskId)));
+    if (bundles.some((task) => !task?.bundleKey ||
+        task.pipelineStatus !== "accepted" ||
+        task.review?.decision !== "approve"))
+        throw new Error("Every task must be human-approved in this repository");
+    const frozen = bundles.map((task) => {
+        if (!task?.bundleKey)
+            throw new Error("Task bundle unavailable");
+        return { runId: task.runId, taskId: task.taskId, bundleKey: task.bundleKey };
+    });
+    return updateAccount(records, scope.ownerId, async (account) => {
+        const existing = account.comparisons.find((entry) => entry.id === selection.id);
+        if (existing) {
+            if (existing.repoId !== scope.repoId || existing.signature !== signature)
+                throw new Error("Comparison ID belongs to a different selection");
+            return existing;
+        }
+        if (account.comparisons.length >= 500)
+            throw new Error("Comparison retention limit reached; contact an operator");
+        const sandbox = account.credentials.find((entry) => entry.id === selection.sandboxCredentialId && !entry.deleted);
+        if (sandbox?.kind !== selection.sandbox)
+            throw new Error("Select your saved sandbox credential");
+        const seen = new Set();
+        const createdAt = new Date().toISOString();
+        const inputs = selection.models.map((selected) => {
+            const model = selected.catalogId === "custom" && selected.customModel
+                ? {
+                    id: "custom",
+                    label: "Custom model",
+                    source: "",
+                    provider: "custom",
+                    model: selected.customModel,
+                    harnesses: ["pi"],
+                }
+                : catalog.find((entry) => entry.id === selected.catalogId);
+            if (!model || (selected.catalogId !== "custom" && selected.customModel))
+                throw new Error("Unknown model");
+            const credential = account.credentials.find((entry) => entry.id === selected.credentialId && !entry.deleted);
+            const route = credential ? routeFor(model, credential.kind) : undefined;
+            if (!credential || !route)
+                throw new Error("Select your matching provider credential");
+            if (credential.auth === "codex-login" &&
+                selected.harnesses.some((harness) => harness !== "codex"))
+                throw new Error("Codex sign-in can only be used with the Codex harness");
+            if (selected.harnesses.some((harness) => !route.harnesses.includes(harness)) ||
+                new Set(selected.harnesses).size !== selected.harnesses.length)
+                throw new Error("Unsupported or repeated harness");
+            const levels = thinkingOptions(model, selected.harnesses);
+            const thinking = selected.thinking ?? (levels.includes("high") ? "high" : "default");
+            if (!levels.includes(thinking))
+                throw new Error("Unsupported thinking level for this model and harness");
+            const identity = JSON.stringify([route.provider, route.model, credential.id, thinking]);
+            for (const harness of selected.harnesses) {
+                const pair = `${identity}/${harness}`;
+                if (seen.has(pair))
+                    throw new Error("Select each model, harness, credential, and thinking configuration once");
+                seen.add(pair);
+            }
+            return {
+                id: randomUUID(),
+                model: selected.catalogId,
+                modelName: `${route.provider === "custom" ? "openai" : route.provider}/${route.model}`,
+                thinking,
+                harnesses: selected.harnesses,
+                sandbox: selection.sandbox,
+                tasks: frozen,
+                repoId: scope.repoId,
+                tenant: scope.tenant,
+                startedBy: scope.login,
+                createdAt,
+                ...(route.pricing ? { pricing: route.pricing } : {}),
+                credentialOwnerId: scope.ownerId,
+                ...(scope.credentialOrgId ? { credentialOrgId: scope.credentialOrgId } : {}),
+                comparisonId: selection.id,
+                credentials: {
+                    modelCredentialId: credential.id,
+                    sandboxCredentialId: sandbox.id,
+                    provider: route.provider,
+                },
+            };
+        });
+        const record = {
+            id: selection.id,
+            repoId: scope.repoId,
+            ownerId: scope.ownerId,
+            createdAt,
+            signature,
+            inputs,
+        };
+        account.comparisons.push(record);
+        return record;
+    });
+}
+export async function comparisonStatus(store, record) {
+    const runs = await Promise.all(record.inputs.map(async (input) => {
+        const run = await getEvaluation(store, record.repoId, input.id);
+        return {
+            id: input.id,
+            model: input.modelName,
+            thinking: input.thinking,
+            harnesses: input.harnesses,
+            status: run?.status ?? "pending",
+            completed: run?.trials.filter((trial) => trial.status === "completed" || trial.status === "failed")
+                .length ?? 0,
+            trials: input.tasks.length * input.harnesses.length,
+        };
+    }));
+    return { id: record.id, createdAt: record.createdAt, runs };
+}
+export async function dispatchComparison(store, record, start) {
+    for (const input of record.inputs) {
+        const run = await getEvaluation(store, record.repoId, input.id);
+        if (!run || run.status === "queued")
+            await start(input);
+    }
+}
+//# sourceMappingURL=comparisons.js.map

package/dist/evaluation/comparisons.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparisons.js","sourceRoot":"","sources":["../../src/evaluation/comparisons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAyB,WAAW,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAqB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE3E,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC;KAC9B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;IACZ,KAAK,EAAE,CAAC;SACL,KAAK,CACJ,CAAC;SACE,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;SACjF,MAAM,EAAE,CACZ;SACA,GAAG,CAAC,CAAC,CAAC;IACT,MAAM,EAAE,CAAC;SACN,KAAK,CACJ,CAAC;SACE,MAAM,CAAC;QACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;QACxD,YAAY,EAAE,CAAC,CAAC,IAAI,EAAE;QACtB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;QAC3C,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;KACrE,CAAC;SACD,MAAM,EAAE,CACZ;SACA,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;IACV,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC;IAChC,mBAAmB,EAAE,CAAC,CAAC,IAAI,EAAE;CAC9B,CAAC;KACD,MAAM,EAAE,CAAC;AASZ,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAA6B,EAC7B,KAAgB,EAChB,KAAsB,EACtB,KAAsB;IAEtB,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,CAAC,MAAM,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAC3E,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE,CACrC,CAAC;IACF,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;YACtE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IACE,IAAI,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI;QAC3E,SAAS,CAAC,KAAK,CAAC,MAAM;QAEtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CACjF,CAAC;IACF,IACE,OAAO,CAAC,IAAI,CACV,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,IAAI,EAAE,SAAS;QAChB,IAAI,CAAC,cAAc,KAAK,UAAU;QAClC,IAAI,CAAC,MAAM,EAAE,QAAQ,KAAK,SAAS,CACtC;QAED,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAClC,IAAI,CAAC,IAAI,EAAE,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACjE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;IAC/E,CAAC,CAAC,CAAC;IACH,OAAO,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC;QAChF,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;gBACtE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YACnC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CACtC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,mBAAmB,IAAI,CAAC,KAAK,CAAC,OAAO,CACxE,CAAC;QACF,IAAI,OAAO,EAAE,IAAI,KAAK,SAAS,CAAC,OAAO;YACrC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAsB,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAClE,MAAM,KAAK,GACT,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,WAAW;gBACrD,CAAC,CAAC;oBACE,EAAE,EAAE,QAAQ;oBACZ,KAAK,EAAE,cAAc;oBACrB,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,QAAQ;oBAClB,KAAK,EAAE,QAAQ,CAAC,WAAW;oBAC3B,SAAS,EAAE,CAAC,IAAI,CAAC;iBAClB;gBACH,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,QAAQ,CAAC,WAAW,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CACzC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,YAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAChE,CAAC;YACF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACxE,IAAI,CAAC,UAAU,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACvF,IACE,UAAU,CAAC,IAAI,KAAK,aAAa;gBACjC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,OAAO,CAAC;gBAEzD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;YAC3E,IACE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACxE,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,SAAS,CAAC,MAAM;gBAE9D,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACrF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;YAC3E,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;YACxF,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACzC,MAAM,IAAI,GAAG,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;oBAChB,MAAM,IAAI,KAAK,CACb,yEAAyE,CAC1E,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;YACD,OAAO;gBACL,EAAE,EAAE,UAAU,EAAE;gBAChB,KAAK,EAAE,QAAQ,CAAC,SAAS;gBACzB,SAAS,EAAE,GAAG,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE;gBACtF,QAAQ;gBACR,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,SAAS,EAAE,KAAK,CAAC,KAAK;gBACtB,SAAS;gBACT,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,iBAAiB,EAAE,KAAK,CAAC,OAAO;gBAChC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5E,YAAY,EAAE,SAAS,CAAC,EAAE;gBAC1B,WAAW,EAAE;oBACX,iBAAiB,EAAE,UAAU,CAAC,EAAE;oBAChC,mBAAmB,EAAE,OAAO,CAAC,EAAE;oBAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ;iBACzB;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAqB;YAC/B,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS;YACT,SAAS;YACT,MAAM;SACP,CAAC;QACF,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAoB,EAAE,MAAwB;IACnF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO;YACL,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,SAAS;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,GAAG,EAAE,MAAM,IAAI,SAAS;YAChC,SAAS,EACP,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC;iBACrF,MAAM,IAAI,CAAC;YAChB,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM;SACpD,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IACF,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC;AAC9D,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAoB,EACpB,MAAwB,EACxB,KAAgD;IAEhD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC"}

package/dist/evaluation/config.d.ts

@@ -0,0 +1,43 @@
+import type { EvaluationChoices, EvaluationInput } from "./types.js";
+export declare const HARBOR_VERSION = "0.23.0";
+export declare const evaluationSandboxes: readonly ["docker", "modal", "e2b", "daytona"];
+export declare function evaluationConfig(env?: NodeJS.ProcessEnv): {
+    profiles: {
+        id: string;
+        label: string;
+        model: string;
+        harnesses: ("codex" | "claude-code" | "pi" | "mini-swe-agent" | "terminus-2")[];
+        tenants: string[];
+        credentialEnv: string;
+        pricing?: {
+            input: number;
+            output: number;
+            cacheRead: number;
+            cacheWrite: number;
+            source: string;
+            asOf: string;
+        } | undefined;
+    }[];
+    sandboxes: ("docker" | "modal" | "e2b" | "daytona")[];
+};
+export declare function evaluationChoices(tenant: string, env?: NodeJS.ProcessEnv): EvaluationChoices;
+export declare function solverEnvironment(input: EvaluationInput, home: string, env?: NodeJS.ProcessEnv): {
+    profile: {
+        id: string;
+        label: string;
+        model: string;
+        harnesses: ("codex" | "claude-code" | "pi" | "mini-swe-agent" | "terminus-2")[];
+        tenants: string[];
+        credentialEnv: string;
+        pricing?: {
+            input: number;
+            output: number;
+            cacheRead: number;
+            cacheWrite: number;
+            source: string;
+            asOf: string;
+        } | undefined;
+    };
+    child: NodeJS.ProcessEnv;
+};
+//# sourceMappingURL=config.d.ts.map

package/dist/evaluation/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/evaluation/config.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAErE,eAAO,MAAM,cAAc,WAAW,CAAC;AACvC,eAAO,MAAM,mBAAmB,gDAAiD,CAAC;AAqClF,wBAAgB,gBAAgB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB;;;;;;;;;;;;;;;;;;EAapE;AACD,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,iBAAiB,CAiBnB;AACD,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,eAAe,EACtB,IAAI,EAAE,MAAM,EACZ,GAAG,GAAE,MAAM,CAAC,UAAwB;;;;;;;;;;;;;;;;;;EAiDrC"}

package/dist/evaluation/config.js

@@ -0,0 +1,116 @@
+import { z } from "zod";
+import { harnessIds } from "./harnesses.js";
+import { modelIdPattern, modelProvider, providers } from "./providers.js";
+export const HARBOR_VERSION = "0.23.0";
+export const evaluationSandboxes = ["docker", "modal", "e2b", "daytona"];
+const profileSchema = z
+    .object({
+    id: z.string().regex(/^[a-z0-9-]{1,60}$/),
+    label: z.string().min(1).max(100),
+    model: z
+        .string()
+        .refine((value) => !!modelProvider(value) &&
+        modelIdPattern.test(value.slice(value.indexOf("/") + 1)) &&
+        value.includes("/")),
+    harnesses: z.array(z.enum(harnessIds)).min(1).max(harnessIds.length),
+    tenants: z.array(z.string().regex(/^[A-Za-z0-9_.-]+$/)).min(1),
+    credentialEnv: z.string().regex(/^SELFBENCH_EVAL_SECRET_[A-Z0-9_]+$/),
+    pricing: z
+        .object({
+        input: z.number().nonnegative(),
+        output: z.number().nonnegative(),
+        cacheRead: z.number().nonnegative(),
+        cacheWrite: z.number().nonnegative(),
+        source: z.url(),
+        asOf: z.iso.date(),
+    })
+        .strict()
+        .optional(),
+})
+    .strict()
+    .superRefine((profile, context) => {
+    if (profile.harnesses.includes("codex") && !profile.model.startsWith("openai/")) {
+        context.addIssue({ code: "custom", message: "Codex requires an OpenAI model" });
+    }
+    if (profile.harnesses.includes("claude-code") && !profile.model.startsWith("anthropic/")) {
+        context.addIssue({ code: "custom", message: "Claude Code requires an Anthropic model" });
+    }
+});
+export function evaluationConfig(env = process.env) {
+    const profiles = z
+        .array(profileSchema)
+        .max(50)
+        .parse(JSON.parse(env.SELFBENCH_EVAL_MODELS ?? "[]"));
+    if (new Set(profiles.map((profile) => profile.id)).size !== profiles.length) {
+        throw new Error("Duplicate evaluation model IDs");
+    }
+    const sandboxes = z
+        .array(z.enum(evaluationSandboxes))
+        .max(4)
+        .parse(JSON.parse(env.SELFBENCH_EVAL_SANDBOXES ?? "[]"));
+    return { profiles, sandboxes };
+}
+export function evaluationChoices(tenant, env = process.env) {
+    const config = evaluationConfig(env);
+    return {
+        harborVersion: HARBOR_VERSION,
+        sandboxes: [...new Set(config.sandboxes)],
+        models: config.profiles
+            .filter((profile) => profile.tenants.some((allowed) => allowed.toLowerCase() === tenant.toLowerCase()))
+            .map(({ id, label, model, harnesses, pricing }) => ({
+            id,
+            label,
+            model,
+            harnesses: [...new Set(harnesses)],
+            ...(pricing ? { pricing } : {}),
+        })),
+    };
+}
+export function solverEnvironment(input, home, env = process.env) {
+    const config = evaluationConfig(env);
+    const profile = config.profiles.find((candidate) => candidate.id === input.model &&
+        candidate.tenants.some((tenant) => tenant.toLowerCase() === input.tenant.toLowerCase()));
+    if (!profile ||
+        profile.model !== input.modelName ||
+        !config.sandboxes.includes(input.sandbox) ||
+        input.harnesses.some((harness) => !profile.harnesses.includes(harness))) {
+        throw new Error("Evaluation configuration is no longer available");
+    }
+    const secret = env[profile.credentialEnv];
+    if (!secret)
+        throw new Error("Evaluation model credential is not configured on the worker");
+    const child = {
+        PATH: env.PATH,
+        HOME: home,
+        TMPDIR: home,
+        LANG: "C.UTF-8",
+        PYTHONUNBUFFERED: "1",
+    };
+    const provider = modelProvider(profile.model);
+    if (!provider)
+        throw new Error("Unsupported model provider");
+    child[providers[provider].credential] = secret;
+    if (input.sandbox === "docker" && env.DOCKER_HOST)
+        child.DOCKER_HOST = env.DOCKER_HOST;
+    if (input.sandbox === "modal") {
+        if (!env.SELFBENCH_EVAL_SECRET_MODAL_TOKEN_ID ||
+            !env.SELFBENCH_EVAL_SECRET_MODAL_TOKEN_SECRET) {
+            throw new Error("Evaluation Modal credentials are not configured on the worker");
+        }
+        child.MODAL_TOKEN_ID = env.SELFBENCH_EVAL_SECRET_MODAL_TOKEN_ID;
+        child.MODAL_TOKEN_SECRET = env.SELFBENCH_EVAL_SECRET_MODAL_TOKEN_SECRET;
+    }
+    for (const [sandbox, name] of [
+        ["e2b", "E2B_API_KEY"],
+        ["daytona", "DAYTONA_API_KEY"],
+    ]) {
+        if (input.sandbox !== sandbox)
+            continue;
+        const value = env[`SELFBENCH_EVAL_SECRET_${name}`];
+        if (!value)
+            throw new Error(`Evaluation ${sandbox} API key is missing`);
+        child[name] = value;
+    }
+    return { profile, child };
+}
+//# sourceMappingURL=config.js.map

package/dist/evaluation/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/evaluation/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG1E,MAAM,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAC;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,CAAU,CAAC;AAClF,MAAM,aAAa,GAAG,CAAC;KACpB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACjC,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;QACtB,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACxD,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CACtB;IACH,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;IACpE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,oCAAoC,CAAC;IACrE,OAAO,EAAE,CAAC;SACP,MAAM,CAAC;QACN,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;QAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;QAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;QACpC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE;QACf,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE;KACnB,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,MAAM,EAAE;KACR,WAAW,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;IAChC,IAAI,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAChF,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACzF,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,yCAAyC,EAAE,CAAC,CAAC;IAC3F,CAAC;AACH,CAAC,CAAC,CAAC;AACL,MAAM,UAAU,gBAAgB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACnE,MAAM,QAAQ,GAAG,CAAC;SACf,KAAK,CAAC,aAAa,CAAC;SACpB,GAAG,CAAC,EAAE,CAAC;SACP,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,IAAI,IAAI,CAAC,CAAC,CAAC;IACxD,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,SAAS,GAAG,CAAC;SAChB,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC;SACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,wBAAwB,IAAI,IAAI,CAAC,CAAC,CAAC;IAC3D,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AACD,MAAM,UAAU,iBAAiB,CAC/B,MAAc,EACd,MAAyB,OAAO,CAAC,GAAG;IAEpC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACrC,OAAO;QACL,aAAa,EAAE,cAAc;QAC7B,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,EAAE,MAAM,CAAC,QAAQ;aACpB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAClB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC,CAClF;aACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YAClD,EAAE;YACF,KAAK;YACL,KAAK;YACL,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAClC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC,CAAC;KACN,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,iBAAiB,CAC/B,KAAsB,EACtB,IAAY,EACZ,MAAyB,OAAO,CAAC,GAAG;IAEpC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAClC,CAAC,SAAS,EAAE,EAAE,CACZ,SAAS,CAAC,EAAE,KAAK,KAAK,CAAC,KAAK;QAC5B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAC1F,CAAC;IACF,IACE,CAAC,OAAO;QACR,OAAO,CAAC,KAAK,KAAK,KAAK,CAAC,SAAS;QACjC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;QACzC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACvE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAsB;QAC/B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,IAAI;QACZ,IAAI,EAAE,SAAS;QACf,gBAAgB,EAAE,GAAG;KACtB,CAAC;IACF,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC7D,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC;IAC/C,IAAI,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW;QAAE,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,CAAC;IACvF,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAC9B,IACE,CAAC,GAAG,CAAC,oCAAoC;YACzC,CAAC,GAAG,CAAC,wCAAwC,EAC7C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QACD,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC,oCAAoC,CAAC;QAChE,KAAK,CAAC,kBAAkB,GAAG,GAAG,CAAC,wCAAwC,CAAC;IAC1E,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI;QAC5B,CAAC,KAAK,EAAE,aAAa,CAAC;QACtB,CAAC,SAAS,EAAE,iBAAiB,CAAC;KACtB,EAAE,CAAC;QACX,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO;YAAE,SAAS;QACxC,MAAM,KAAK,GAAG,GAAG,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,OAAO,qBAAqB,CAAC,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC"}

package/dist/evaluation/cost.d.ts

@@ -0,0 +1,3 @@
+import type { EvaluationRun, EvaluationTrial } from "./types.js";
+export declare function trialCost(run: EvaluationRun, harness: string, files: Map<string, string>, result: unknown): Pick<EvaluationTrial, "modelVerified" | "apiCostUsd" | "tokenUsage" | "costSource">;
+//# sourceMappingURL=cost.d.ts.map

package/dist/evaluation/cost.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../../src/evaluation/cost.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAIjE,wBAAgB,SAAS,CACvB,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1B,MAAM,EAAE,OAAO,GACd,IAAI,CAAC,eAAe,EAAE,eAAe,GAAG,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC,CAuFrF"}

package/dist/evaluation/cost.js

@@ -0,0 +1,95 @@
+import { harborCost } from "./harbor-cost.js";
+import { record } from "./output.js";
+const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+export function trialCost(run, harness, files, result) {
+    const pricing = run.pricing;
+    const model = run.modelName.split("/").slice(1).join("/");
+    const matches = (value) => value === model || value === run.modelName;
+    let usage;
+    let verified = false;
+    let reportedCost;
+    if (harness === "pi") {
+        const text = [...files].find(([name]) => name.endsWith("/pi.txt"))?.[1];
+        if (!text)
+            return {};
+        const messages = [];
+        for (const line of text.split("\n").filter(Boolean)) {
+            let event;
+            try {
+                event = record(JSON.parse(line));
+            }
+            catch {
+                return {};
+            }
+            const message = record(event.message);
+            if (event.type === "message_end" && message.role === "assistant")
+                messages.push(message);
+        }
+        verified =
+            messages.length > 0 &&
+                messages.every((message) => `${message.provider}/${message.model}` === run.modelName);
+        const totals = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+        for (const message of messages) {
+            const tokens = record(message.usage);
+            for (const key of ["input", "output", "cacheRead", "cacheWrite"]) {
+                const value = tokens[key];
+                if (!count(value))
+                    return { modelVerified: verified };
+                totals[key] += value;
+            }
+        }
+        usage = totals;
+    }
+    else if (harness === "codex") {
+        const text = [...files].find(([name]) => name.endsWith("/trajectory.json"))?.[1];
+        if (!text)
+            return {};
+        let trajectory;
+        try {
+            trajectory = record(JSON.parse(text));
+        }
+        catch {
+            return {};
+        }
+        const steps = Array.isArray(trajectory.steps)
+            ? trajectory.steps.map(record).filter((step) => step.source === "agent")
+            : [];
+        verified =
+            steps.length > 0 &&
+                steps.every((step) => matches(step.model_name ?? record(trajectory.agent).model_name));
+        const tokens = record(record(result).agent_result);
+        const cacheWrite = record(record(trajectory.final_metrics).extra).total_cache_write_input_tokens ?? 0;
+        if (count(tokens.n_input_tokens) &&
+            count(tokens.n_output_tokens) &&
+            count(tokens.n_cache_tokens) &&
+            count(cacheWrite) &&
+            tokens.n_cache_tokens + cacheWrite <= tokens.n_input_tokens) {
+            usage = {
+                input: tokens.n_input_tokens - tokens.n_cache_tokens - cacheWrite,
+                output: tokens.n_output_tokens,
+                cacheRead: tokens.n_cache_tokens,
+                cacheWrite,
+            };
+            reportedCost = harborCost(trajectory, usage, tokens.cost_usd);
+        }
+    }
+    const measured = { modelVerified: verified, ...(usage ? { tokenUsage: usage } : {}) };
+    if (!verified || !usage)
+        return measured;
+    if (reportedCost !== undefined)
+        return { ...measured, apiCostUsd: reportedCost, costSource: "harbor" };
+    if (!pricing)
+        return measured;
+    if (pricing.maxInputTokens &&
+        usage.input + usage.cacheRead + usage.cacheWrite > pricing.maxInputTokens)
+        return measured;
+    const apiCostUsd = (usage.input * pricing.input +
+        usage.output * pricing.output +
+        usage.cacheRead * pricing.cacheRead +
+        usage.cacheWrite * pricing.cacheWrite) /
+        1_000_000;
+    return Number.isFinite(apiCostUsd)
+        ? { ...measured, apiCostUsd, costSource: "reference-rates" }
+        : measured;
+}
+//# sourceMappingURL=cost.js.map

package/dist/evaluation/cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.js","sourceRoot":"","sources":["../../src/evaluation/cost.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,KAAK,GAAG,CAAC,KAAc,EAAmB,EAAE,CAChD,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACzE,MAAM,UAAU,SAAS,CACvB,GAAkB,EAClB,OAAe,EACf,KAA0B,EAC1B,MAAe;IAEf,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,CAAC,KAAc,EAAE,EAAE,CAAC,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,GAAG,CAAC,SAAS,CAAC;IAC/E,IAAI,KAA2F,CAAC;IAChG,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,YAAgC,CAAC;IACrC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,MAAM,QAAQ,GAA8B,EAAE,CAAC;QAC/C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACpD,IAAI,KAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW;gBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3F,CAAC;QACD,QAAQ;YACN,QAAQ,CAAC,MAAM,GAAG,CAAC;gBACnB,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QACpE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrC,KAAK,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,CAAU,EAAE,CAAC;gBAC1E,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;oBAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;gBACtD,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;QACD,KAAK,GAAG,MAAM,CAAC;IACjB,CAAC;SAAM,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,IAAI,UAAmC,CAAC;QACxC,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YAC3C,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC;YACxE,CAAC,CAAC,EAAE,CAAC;QACP,QAAQ;YACN,KAAK,CAAC,MAAM,GAAG,CAAC;gBAChB,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,UAAU,GACd,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,8BAA8B,IAAI,CAAC,CAAC;QACrF,IACE,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC;YAC5B,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC;YAC7B,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC;YAC5B,KAAK,CAAC,UAAU,CAAC;YACjB,MAAM,CAAC,cAAc,GAAG,UAAU,IAAI,MAAM,CAAC,cAAc,EAC3D,CAAC;YACD,KAAK,GAAG;gBACN,KAAK,EAAE,MAAM,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,GAAG,UAAU;gBACjE,MAAM,EAAE,MAAM,CAAC,eAAe;gBAC9B,SAAS,EAAE,MAAM,CAAC,cAAc;gBAChC,UAAU;aACX,CAAC;YACF,YAAY,GAAG,UAAU,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACtF,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,YAAY,KAAK,SAAS;QAC5B,OAAO,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;IACzE,IAAI,CAAC,OAAO;QAAE,OAAO,QAAQ,CAAC;IAC9B,IACE,OAAO,CAAC,cAAc;QACtB,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,cAAc;QAEzE,OAAO,QAAQ,CAAC;IAClB,MAAM,UAAU,GACd,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK;QAC1B,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QAC7B,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS;QACnC,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACxC,SAAS,CAAC;IACZ,OAAO,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;QAChC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,iBAAiB,EAAE;QAC5D,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC"}

package/dist/evaluation/credential-execution.d.ts

@@ -0,0 +1,10 @@
+import type { EncryptedRecordStore } from "./encrypted-records.js";
+import type { EvaluationInput } from "./types.js";
+export declare function credentialExecution(input: EvaluationInput, home: string, env: NodeJS.ProcessEnv, records: EncryptedRecordStore): Promise<{
+    profile: {
+        model: string;
+    };
+    child: NodeJS.ProcessEnv;
+    secrets: string[];
+}>;
+//# sourceMappingURL=credential-execution.d.ts.map

package/dist/evaluation/credential-execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"credential-execution.d.ts","sourceRoot":"","sources":["../../src/evaluation/credential-execution.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,eAAe,EACtB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,OAAO,EAAE,oBAAoB;;;;;;GAsE9B"}

package/dist/evaluation/credential-execution.js

@@ -0,0 +1,65 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { readAccount, secretPath } from "./account.js";
+import { validateEndpoint } from "./credentials.js";
+import { orgRecords } from "./org-records.js";
+export async function credentialExecution(input, home, env, records) {
+    if (!input.credentials || !input.credentialOwnerId || !input.comparisonId)
+        throw new Error("Credential references missing");
+    records = orgRecords(records, input.credentialOrgId);
+    const account = await readAccount(records, input.credentialOwnerId);
+    const saved = account.comparisons
+        .find((entry) => entry.id === input.comparisonId && entry.repoId === input.repoId)
+        ?.inputs.find((entry) => entry.id === input.id);
+    if (!saved || !isDeepStrictEqual(saved, input))
+        throw new Error("Execution does not match the reserved comparison");
+    const info = account.credentials.find((entry) => entry.id === input.credentials?.modelCredentialId && !entry.deleted);
+    const sandbox = account.credentials.find((entry) => entry.id === input.credentials?.sandboxCredentialId && !entry.deleted);
+    if (!info ||
+        !sandbox ||
+        sandbox.kind !== input.sandbox ||
+        info.kind !== input.credentials.provider)
+        throw new Error("Credential unavailable");
+    const modelSecret = (await records.read(secretPath(input.credentialOwnerId, info.id)))?.value;
+    const sandboxSecret = (await records.read(secretPath(input.credentialOwnerId, sandbox.id)))?.value;
+    if (!modelSecret || !sandboxSecret)
+        throw new Error("Credential unavailable");
+    const child = {
+        PATH: env.PATH,
+        HOME: home,
+        TMPDIR: home,
+        LANG: "C.UTF-8",
+        PYTHONUNBUFFERED: "1",
+    };
+    const secrets = [modelSecret.value, sandboxSecret.value, sandboxSecret.tokenId ?? ""].filter(Boolean);
+    if (info.auth === "codex-login") {
+        if (input.harnesses.some((harness) => harness !== "codex"))
+            throw new Error("Codex login requires Codex harness");
+        const authPath = join(home, "codex-auth.json");
+        await writeFile(authPath, modelSecret.value, { mode: 0o600 });
+        child.CODEX_AUTH_JSON_PATH = authPath;
+        const auth = JSON.parse(modelSecret.value);
+        secrets.push(...Object.values(auth.tokens).filter((value) => typeof value === "string"));
+    }
+    else {
+        const name = info.kind === "anthropic"
+            ? "ANTHROPIC_API_KEY"
+            : info.kind === "openrouter"
+                ? "OPENROUTER_API_KEY"
+                : "OPENAI_API_KEY";
+        child[name] = modelSecret.value;
+    }
+    if (info.kind === "custom")
+        child.OPENAI_BASE_URL = await validateEndpoint(info.endpoint ?? "", env);
+    if (sandbox.kind === "modal") {
+        child.MODAL_TOKEN_ID = sandboxSecret.tokenId;
+        child.MODAL_TOKEN_SECRET = sandboxSecret.value;
+    }
+    else if (sandbox.kind === "e2b")
+        child.E2B_API_KEY = sandboxSecret.value;
+    else if (sandbox.kind === "daytona")
+        child.DAYTONA_API_KEY = sandboxSecret.value;
+    return { profile: { model: input.modelName }, child, secrets };
+}
+//# sourceMappingURL=credential-execution.js.map

package/dist/evaluation/credential-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"credential-execution.js","sourceRoot":"","sources":["../../src/evaluation/credential-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAG9C,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAsB,EACtB,IAAY,EACZ,GAAsB,EACtB,OAA6B;IAE7B,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,KAAK,CAAC,YAAY;QACvE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW;SAC9B,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC;QAClF,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,KAAK,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CACnC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,iBAAiB,IAAI,CAAC,KAAK,CAAC,OAAO,CAC/E,CAAC;IACF,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CACtC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,mBAAmB,IAAI,CAAC,KAAK,CAAC,OAAO,CACjF,CAAC;IACF,IACE,CAAC,IAAI;QACL,CAAC,OAAO;QACR,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO;QAC9B,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,WAAW,CAAC,QAAQ;QAExC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,CAClB,MAAM,OAAO,CAAC,IAAI,CAAoB,UAAU,CAAC,KAAK,CAAC,iBAAiB,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CACpF,EAAE,KAAK,CAAC;IACT,MAAM,aAAa,GAAG,CACpB,MAAM,OAAO,CAAC,IAAI,CAChB,UAAU,CAAC,KAAK,CAAC,iBAAiB,EAAE,OAAO,CAAC,EAAE,CAAC,CAChD,CACF,EAAE,KAAK,CAAC;IACT,IAAI,CAAC,WAAW,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAsB;QAC/B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,IAAI;QACZ,IAAI,EAAE,SAAS;QACf,gBAAgB,EAAE,GAAG;KACtB,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAC1F,OAAO,CACR,CAAC;IACF,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,OAAO,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAC/C,MAAM,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,KAAK,CAAC,oBAAoB,GAAG,QAAQ,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,CAAC,IAAI,CACV,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAC5F,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GACR,IAAI,CAAC,IAAI,KAAK,WAAW;YACvB,CAAC,CAAC,mBAAmB;YACrB,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,YAAY;gBAC1B,CAAC,CAAC,oBAAoB;gBACtB,CAAC,CAAC,gBAAgB,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC;IAClC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QACxB,KAAK,CAAC,eAAe,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7B,KAAK,CAAC,cAAc,GAAG,aAAa,CAAC,OAAO,CAAC;QAC7C,KAAK,CAAC,kBAAkB,GAAG,aAAa,CAAC,KAAK,CAAC;IACjD,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK;QAAE,KAAK,CAAC,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC;SACtE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;QAAE,KAAK,CAAC,eAAe,GAAG,aAAa,CAAC,KAAK,CAAC;IACjF,OAAO,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AACjE,CAAC"}

package/dist/evaluation/credentials.d.ts

@@ -0,0 +1,40 @@
+import { z } from "zod";
+import type { ArtifactStore } from "../artifacts.js";
+import { type CredentialInfo } from "./account.js";
+import type { EncryptedRecordStore } from "./encrypted-records.js";
+export declare const credentialSchema: z.ZodObject<{
+    name: z.ZodString;
+    kind: z.ZodEnum<{
+        modal: "modal";
+        vercel: "vercel";
+        e2b: "e2b";
+        daytona: "daytona";
+        custom: "custom";
+        openai: "openai";
+        anthropic: "anthropic";
+        openrouter: "openrouter";
+    }>;
+    auth: z.ZodDefault<z.ZodEnum<{
+        "api-key": "api-key";
+        "codex-login": "codex-login";
+    }>>;
+    value: z.ZodString;
+    tokenId: z.ZodOptional<z.ZodString>;
+    teamId: z.ZodOptional<z.ZodString>;
+    projectId: z.ZodOptional<z.ZodString>;
+    endpoint: z.ZodOptional<z.ZodURL>;
+}, z.core.$strict>;
+export type CredentialDraft = z.infer<typeof credentialSchema>;
+export declare function validateEndpoint(endpoint: string, env: NodeJS.ProcessEnv): Promise<string>;
+export declare function saveCredential(records: EncryptedRecordStore, ownerId: number, draft: CredentialDraft, env: NodeJS.ProcessEnv, migratedFrom?: string): Promise<CredentialInfo>;
+export declare function deleteCredential(records: EncryptedRecordStore, store: ArtifactStore, ownerId: number, id: string): Promise<void>;
+export declare function listCredentials(records: EncryptedRecordStore, ownerId: number): Promise<{
+    id: string;
+    name: string;
+    kind: import("./catalog.js").CatalogProvider | import("./catalog.js").HostedSandbox | "vercel";
+    auth: "api-key" | "codex-login";
+    createdAt: string;
+    endpoint?: string;
+    deleted?: boolean;
+}[]>;
+//# sourceMappingURL=credentials.d.ts.map

package/dist/evaluation/credentials.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../src/evaluation/credentials.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,KAAK,cAAc,EAA0C,MAAM,cAAc,CAAC;AAC3F,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAInE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;kBA+DzB,CAAC;AACL,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC/D,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,UAAU,mBAc9E;AACD,wBAAsB,cAAc,CAClC,OAAO,EAAE,oBAAoB,EAC7B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,eAAe,EACtB,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,cAAc,CAAC,CAyCzB;AACD,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,MAAM,iBAoBX;AACD,wBAAsB,eAAe,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM;;;;;;;;KAInF"}