self-bench 0.3.5 → 0.3.8

package/dist/evaluation/codex-login-process.d.ts

@@ -0,0 +1,12 @@
+export interface CodexDeviceCode {
+    verificationUrl: string;
+    userCode: string;
+}
+export interface CodexLoginProcess {
+    instructions: Promise<CodexDeviceCode>;
+    auth: Promise<string>;
+    close(): Promise<void>;
+}
+/** Only the authentication RPCs run here. Each attempt has its own private Codex home. */
+export declare function startCodexLogin(): Promise<CodexLoginProcess>;
+//# sourceMappingURL=codex-login-process.d.ts.map

package/dist/evaluation/codex-login-process.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-login-process.d.ts","sourceRoot":"","sources":["../../src/evaluation/codex-login-process.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AACD,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACvC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACtB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,0FAA0F;AAC1F,wBAAsB,eAAe,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAoHlE"}

package/dist/evaluation/codex-login-process.js

@@ -0,0 +1,121 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+/** Only the authentication RPCs run here. Each attempt has its own private Codex home. */
+export async function startCodexLogin() {
+    const directory = await mkdtemp(join(tmpdir(), "selfbench-codex-login-"));
+    let executable;
+    try {
+        executable = createRequire(import.meta.url).resolve("@openai/codex/bin/codex.js");
+    }
+    catch {
+        await rm(directory, { recursive: true, force: true });
+        throw new Error("Codex sign-in is unavailable on this server. Import an auth file instead.");
+    }
+    const child = spawn(process.execPath, [executable, "app-server", "-c", 'cli_auth_credentials_store="file"'], {
+        cwd: directory,
+        env: {
+            PATH: process.env.PATH,
+            CODEX_HOME: directory,
+            ...(process.env.SSL_CERT_FILE ? { SSL_CERT_FILE: process.env.SSL_CERT_FILE } : {}),
+            ...(process.env.CODEX_CA_CERTIFICATE
+                ? { CODEX_CA_CERTIFICATE: process.env.CODEX_CA_CERTIFICATE }
+                : {}),
+        },
+        stdio: ["pipe", "pipe", "ignore"],
+    });
+    let resolveInstructions;
+    let rejectInstructions;
+    let resolveAuth;
+    let rejectAuth;
+    const instructions = new Promise((resolve, reject) => {
+        resolveInstructions = resolve;
+        rejectInstructions = reject;
+    });
+    const auth = new Promise((resolve, reject) => {
+        resolveAuth = resolve;
+        rejectAuth = reject;
+    });
+    // Callers can await instructions before auth without an unhandled rejection.
+    void instructions.catch(() => undefined);
+    void auth.catch(() => undefined);
+    let closed;
+    const exited = new Promise((resolve) => child.once("close", () => resolve()));
+    const lines = createInterface({ input: child.stdout });
+    const fail = (message) => {
+        const error = new Error(message);
+        rejectInstructions(error);
+        rejectAuth(error);
+    };
+    const close = () => {
+        if (closed)
+            return closed;
+        closed = (async () => {
+            clearTimeout(startupTimeout);
+            lines.close();
+            fail("Sign-in was cancelled. Start again when you’re ready.");
+            child.kill("SIGTERM");
+            const force = setTimeout(() => child.kill("SIGKILL"), 2_000);
+            force.unref();
+            await exited;
+            clearTimeout(force);
+            await rm(directory, { recursive: true, force: true });
+        })();
+        return closed;
+    };
+    const startupTimeout = setTimeout(() => {
+        fail("Codex took too long to start. Try signing in again.");
+        void close();
+    }, 30_000);
+    startupTimeout.unref();
+    child.once("error", () => fail("Could not start Codex sign-in. Try again or import an auth file."));
+    child.once("close", () => fail("Codex sign-in ended. Try again or import an auth file."));
+    child.stdin.on("error", () => fail("Could not communicate with Codex. Try signing in again."));
+    const send = (value) => child.stdin.write(`${JSON.stringify(value)}\n`);
+    lines.on("line", (line) => {
+        try {
+            const message = JSON.parse(line);
+            if (message.error) {
+                fail("Could not start device sign-in. Check that device code login is enabled in your ChatGPT security settings, then try again.");
+                void close();
+            }
+            else if (message.id === 1) {
+                send({ method: "initialized" });
+                send({ id: 2, method: "account/login/start", params: { type: "chatgptDeviceCode" } });
+            }
+            else if (message.id === 2) {
+                const result = message.result;
+                if (result?.type !== "chatgptDeviceCode" ||
+                    result.verificationUrl !== "https://auth.openai.com/codex/device" ||
+                    typeof result.userCode !== "string") {
+                    fail("Codex returned unexpected sign-in instructions. Try again later.");
+                    void close();
+                    return;
+                }
+                clearTimeout(startupTimeout);
+                resolveInstructions({ verificationUrl: result.verificationUrl, userCode: result.userCode });
+            }
+            else if (message.method === "account/login/completed") {
+                if (message.params?.success) {
+                    void readFile(join(directory, "auth.json"), "utf8").then(resolveAuth, () => rejectAuth(new Error("Could not read the completed sign-in. Try again.")));
+                }
+                else
+                    fail("Sign-in was not completed. The code may have expired; try again.");
+            }
+        }
+        catch {
+            fail("Codex returned an invalid sign-in response. Try again.");
+            void close();
+        }
+    });
+    send({
+        id: 1,
+        method: "initialize",
+        params: { clientInfo: { name: "selfbench", title: "SelfBench", version: "0.3.6" } },
+    });
+    return { instructions, auth, close };
+}
+//# sourceMappingURL=codex-login-process.js.map

package/dist/evaluation/codex-login-process.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-login-process.js","sourceRoot":"","sources":["../../src/evaluation/codex-login-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAYhD,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAC1E,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACH,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CACjB,OAAO,CAAC,QAAQ,EAChB,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE,mCAAmC,CAAC,EACrE;QACE,GAAG,EAAE,SAAS;QACd,GAAG,EAAE;YACH,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI;YACtB,UAAU,EAAE,SAAS;YACrB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB;gBAClC,CAAC,CAAC,EAAE,oBAAoB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE;gBAC5D,CAAC,CAAC,EAAE,CAAC;SACR;QACD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;KAClC,CACF,CAAC;IACF,IAAI,mBAAsD,CAAC;IAC3D,IAAI,kBAA2C,CAAC;IAChD,IAAI,WAAqC,CAAC;IAC1C,IAAI,UAAmC,CAAC;IACxC,MAAM,YAAY,GAAG,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACpE,mBAAmB,GAAG,OAAO,CAAC;QAC9B,kBAAkB,GAAG,MAAM,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnD,WAAW,GAAG,OAAO,CAAC;QACtB,UAAU,GAAG,MAAM,CAAC;IACtB,CAAC,CAAC,CAAC;IACH,6EAA6E;IAC7E,KAAK,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACjC,IAAI,MAAiC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,EAAE;QAC/B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QACjC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC1B,UAAU,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,MAAM,GAAG,CAAC,KAAK,IAAI,EAAE;YACnB,YAAY,CAAC,cAAc,CAAC,CAAC;YAC7B,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,uDAAuD,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;YAC7D,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,MAAM,MAAM,CAAC;YACb,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;QACrC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QAC5D,KAAK,KAAK,EAAE,CAAC;IACf,CAAC,EAAE,MAAM,CAAC,CAAC;IACX,cAAc,CAAC,KAAK,EAAE,CAAC;IACvB,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CACvB,IAAI,CAAC,kEAAkE,CAAC,CACzE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC,CAAC;IAC1F,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC,CAAC;IAC/F,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,IAAI,CACF,4HAA4H,CAC7H,CAAC;gBACF,KAAK,KAAK,EAAE,CAAC;YACf,CAAC;iBAAM,IAAI,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;gBAChC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;YACxF,CAAC;iBAAM,IAAI,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;gBAC9B,IACE,MAAM,EAAE,IAAI,KAAK,mBAAmB;oBACpC,MAAM,CAAC,eAAe,KAAK,sCAAsC;oBACjE,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EACnC,CAAC;oBACD,IAAI,CAAC,kEAAkE,CAAC,CAAC;oBACzE,KAAK,KAAK,EAAE,CAAC;oBACb,OAAO;gBACT,CAAC;gBACD,YAAY,CAAC,cAAc,CAAC,CAAC;gBAC7B,mBAAmB,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,eAAe,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9F,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,KAAK,yBAAyB,EAAE,CAAC;gBACxD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBAC5B,KAAK,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CACzE,UAAU,CAAC,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC,CAC1E,CAAC;gBACJ,CAAC;;oBAAM,IAAI,CAAC,kEAAkE,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,wDAAwD,CAAC,CAAC;YAC/D,KAAK,KAAK,EAAE,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,EAAE,EAAE,CAAC;QACL,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE;KACpF,CAAC,CAAC;IACH,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACvC,CAAC"}

package/dist/evaluation/codex-login-routes.d.ts

@@ -0,0 +1,5 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { User } from "../auth/users.js";
+import type { EvaluationRoutesOptions } from "./routes.js";
+export declare function codexLoginRoutes(options: EvaluationRoutesOptions, request: IncomingMessage, url: URL, response: ServerResponse, user: User): Promise<boolean>;
+//# sourceMappingURL=codex-login-routes.d.ts.map

package/dist/evaluation/codex-login-routes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-login-routes.d.ts","sourceRoot":"","sources":["../../src/evaluation/codex-login-routes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGjE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAI7C,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAM3D,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,uBAAuB,EAChC,OAAO,EAAE,eAAe,EACxB,GAAG,EAAE,GAAG,EACR,QAAQ,EAAE,cAAc,EACxB,IAAI,EAAE,IAAI,oBAyCX"}

package/dist/evaluation/codex-login-routes.js

@@ -0,0 +1,55 @@
+import { z } from "zod";
+import { readBody, sendJson, trustedMutation } from "../api/http.js";
+import { tenantFor } from "../site/tenant.js";
+import { codexLogins } from "./codex-login.js";
+import { RecordStoreError } from "./encrypted-records.js";
+const pattern = /^\/api\/orgs\/([A-Za-z0-9_.-]+)\/credentials\/codex-login(?:\/([a-f0-9-]{36})(?:\/(complete|cancel))?)?$/;
+const startSchema = z.object({ name: z.string().trim().min(1).max(80) }).strict();
+export async function codexLoginRoutes(options, request, url, response, user) {
+    const match = pattern.exec(url.pathname);
+    if (!match)
+        return false;
+    response.setHeader("cache-control", "no-store");
+    const org = await tenantFor(options.users, user, match[1] ?? "");
+    if (!org) {
+        sendJson(response, 404, { error: "Organization not found" });
+        return true;
+    }
+    if (org.role !== "admin" ||
+        (request.method !== "GET" &&
+            (request.method !== "POST" || !trustedMutation(request, options.publicUrl, user)))) {
+        sendJson(response, 403, { error: "Organization admin and same-origin JSON request required" });
+        return true;
+    }
+    const logins = options.codexLogins ?? codexLogins;
+    try {
+        if (!options.records)
+            throw new RecordStoreError(503);
+        const id = match[2];
+        const action = match[3];
+        if (request.method === "GET" && id && !action) {
+            sendJson(response, 200, logins.status(org.id, user.id, id));
+        }
+        else if (request.method === "POST" && !id) {
+            const { name } = startSchema.parse(JSON.parse((await readBody(request, 1024)).toString()));
+            sendJson(response, 202, await logins.start(org.id, user.id, name));
+        }
+        else if (request.method === "POST" && id && action === "complete") {
+            sendJson(response, 200, await logins.complete(org.id, user.id, id, options.records));
+        }
+        else if (request.method === "POST" && id && action === "cancel") {
+            await logins.cancel(org.id, user.id, id);
+            sendJson(response, 200, { cancelled: true });
+        }
+        else
+            sendJson(response, 405, { error: "Method not allowed" });
+    }
+    catch (error) {
+        sendJson(response, error instanceof RecordStoreError ? error.status : 400, {
+            error: error instanceof RecordStoreError ? error.message : "Invalid sign-in request",
+        });
+    }
+    request.resume();
+    return true;
+}
+//# sourceMappingURL=codex-login-routes.js.map

package/dist/evaluation/codex-login-routes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-login-routes.js","sourceRoot":"","sources":["../../src/evaluation/codex-login-routes.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAErE,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAG1D,MAAM,OAAO,GACX,0GAA0G,CAAC;AAC7G,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAElF,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAgC,EAChC,OAAwB,EACxB,GAAQ,EACR,QAAwB,EACxB,IAAU;IAEV,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,GAAG,CAAC,IAAI,KAAK,OAAO;QACpB,CAAC,OAAO,CAAC,MAAM,KAAK,KAAK;YACvB,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EACpF,CAAC;QACD,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,0DAA0D,EAAE,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,IAAI,WAAW,CAAC;IAClD,IAAI,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,OAAO;YAAE,MAAM,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACtD,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9C,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC3F,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,EAAE,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YACpE,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACvF,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,EAAE,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YAClE,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACzC,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC;;YAAM,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,QAAQ,EAAE,KAAK,YAAY,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE;YACzE,KAAK,EAAE,KAAK,YAAY,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB;SACrF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,CAAC;IACjB,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/evaluation/codex-login.d.ts

@@ -0,0 +1,42 @@
+import type { CredentialInfo } from "./account.js";
+import { type CodexDeviceCode, startCodexLogin } from "./codex-login-process.js";
+import { type EncryptedRecordStore } from "./encrypted-records.js";
+export interface CodexLoginStatus {
+    id: string;
+    status: "starting" | "waiting" | "ready" | "saving" | "saved" | "failed";
+    expiresAt: string;
+    instructions?: CodexDeviceCode;
+    error?: string;
+    credential?: CredentialInfo;
+}
+/** Short-lived, user-scoped ceremonies. No tokens are returned to the browser. */
+export declare function createCodexLogins(start?: typeof startCodexLogin, lifetimeMs?: number): {
+    start(orgId: number, userId: number, name: string): Promise<{
+        id: string;
+        status: "starting" | "waiting" | "ready" | "saving" | "saved" | "failed";
+        expiresAt: string;
+        instructions?: CodexDeviceCode;
+        error?: string;
+        credential?: CredentialInfo;
+    }>;
+    status(orgId: number, userId: number, id: string): CodexLoginStatus;
+    complete(orgId: number, userId: number, id: string, records: EncryptedRecordStore): Promise<CredentialInfo>;
+    cancel(orgId: number, userId: number, id: string): Promise<void>;
+    close(): Promise<void>;
+};
+export type CodexLogins = ReturnType<typeof createCodexLogins>;
+export declare const codexLogins: {
+    start(orgId: number, userId: number, name: string): Promise<{
+        id: string;
+        status: "starting" | "waiting" | "ready" | "saving" | "saved" | "failed";
+        expiresAt: string;
+        instructions?: CodexDeviceCode;
+        error?: string;
+        credential?: CredentialInfo;
+    }>;
+    status(orgId: number, userId: number, id: string): CodexLoginStatus;
+    complete(orgId: number, userId: number, id: string, records: EncryptedRecordStore): Promise<CredentialInfo>;
+    cancel(orgId: number, userId: number, id: string): Promise<void>;
+    close(): Promise<void>;
+};
+//# sourceMappingURL=codex-login.d.ts.map

package/dist/evaluation/codex-login.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-login.d.ts","sourceRoot":"","sources":["../../src/evaluation/codex-login.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EACL,KAAK,eAAe,EAEpB,eAAe,EAChB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,oBAAoB,EAAoB,MAAM,wBAAwB,CAAC;AAGrF,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,cAAc,CAAC;CAC7B;AAYD,kFAAkF;AAClF,wBAAgB,iBAAiB,CAAC,KAAK,yBAAkB,EAAE,UAAU,SAAc;iBAkB5D,MAAM,UAAU,MAAM,QAAQ,MAAM;YArCrD,MAAM;gBACF,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ;mBAC7D,MAAM;uBACF,eAAe;gBACtB,MAAM;qBACD,cAAc;;kBAmGX,MAAM,UAAU,MAAM,MAAM,MAAM,GAAG,gBAAgB;oBAG7C,MAAM,UAAU,MAAM,MAAM,MAAM,WAAW,oBAAoB;kBAsCnE,MAAM,UAAU,MAAM,MAAM,MAAM;;EASzD;AACD,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC/D,eAAO,MAAM,WAAW;iBAvHD,MAAM,UAAU,MAAM,QAAQ,MAAM;YArCrD,MAAM;gBACF,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ;mBAC7D,MAAM;uBACF,eAAe;gBACtB,MAAM;qBACD,cAAc;;kBAmGX,MAAM,UAAU,MAAM,MAAM,MAAM,GAAG,gBAAgB;oBAG7C,MAAM,UAAU,MAAM,MAAM,MAAM,WAAW,oBAAoB;kBAsCnE,MAAM,UAAU,MAAM,MAAM,MAAM;;CAWZ,CAAC"}

package/dist/evaluation/codex-login.js

@@ -0,0 +1,143 @@
+import { randomUUID } from "node:crypto";
+import { startCodexLogin, } from "./codex-login-process.js";
+import { credentialSchema, saveCredential } from "./credentials.js";
+import { RecordStoreError } from "./encrypted-records.js";
+import { orgRecords } from "./org-records.js";
+/** Short-lived, user-scoped ceremonies. No tokens are returned to the browser. */
+export function createCodexLogins(start = startCodexLogin, lifetimeMs = 15 * 60_000) {
+    const attempts = new Map();
+    const startingUsers = new Set();
+    const cancel = async (id) => {
+        const attempt = attempts.get(id);
+        if (!attempt)
+            return;
+        attempts.delete(id);
+        clearTimeout(attempt.timer);
+        delete attempt.auth;
+        await attempt.process?.close();
+    };
+    const get = (orgId, userId, id) => {
+        const attempt = attempts.get(id);
+        if (!attempt || attempt.orgId !== orgId || attempt.userId !== userId)
+            throw new RecordStoreError(410, "This sign-in has expired. Start a new sign-in.");
+        return attempt;
+    };
+    return {
+        async start(orgId, userId, name) {
+            if (startingUsers.has(userId))
+                throw new RecordStoreError(429, "A sign-in is already starting. Please wait.");
+            startingUsers.add(userId);
+            try {
+                const normalized = name.trim();
+                if (!normalized || normalized.length > 80)
+                    throw new RecordStoreError(400, "Enter a credential name of 1–80 characters.");
+                const previous = [...attempts.values()].find((attempt) => attempt.userId === userId);
+                if (previous) {
+                    if (previous.view.status === "saving")
+                        throw new RecordStoreError(409, "Your sign-in is being saved. Please wait.");
+                    await cancel(previous.view.id);
+                }
+                if (attempts.size >= 32)
+                    throw new RecordStoreError(429, "Too many sign-ins in progress. Try again shortly.");
+                const id = randomUUID();
+                const view = {
+                    id,
+                    status: "starting",
+                    expiresAt: new Date(Date.now() + lifetimeMs).toISOString(),
+                };
+                const timer = setTimeout(() => {
+                    void cancel(id);
+                }, lifetimeMs);
+                timer.unref();
+                const attempt = { orgId, userId, name: normalized, view, timer };
+                attempts.set(id, attempt);
+                void (async () => {
+                    try {
+                        const process = await start();
+                        attempt.process = process;
+                        if (!attempts.has(id)) {
+                            await process.close();
+                            return;
+                        }
+                        view.instructions = await process.instructions;
+                        view.status = "waiting";
+                        const auth = await process.auth;
+                        if (!attempts.has(id))
+                            return;
+                        const draft = credentialSchema.parse({
+                            name: normalized,
+                            kind: "openai",
+                            auth: "codex-login",
+                            value: auth,
+                        });
+                        attempt.auth = draft.value;
+                        view.status = "ready";
+                    }
+                    catch (cause) {
+                        if (attempts.has(id)) {
+                            view.status = "failed";
+                            view.error =
+                                cause instanceof Error && cause.name !== "ZodError"
+                                    ? cause.message
+                                    : "Codex did not return a valid ChatGPT sign-in. Try again.";
+                        }
+                    }
+                    finally {
+                        const process = attempt.process;
+                        delete attempt.process;
+                        await process?.close();
+                    }
+                })();
+                return { ...view };
+            }
+            finally {
+                startingUsers.delete(userId);
+            }
+        },
+        status(orgId, userId, id) {
+            return { ...get(orgId, userId, id).view };
+        },
+        async complete(orgId, userId, id, records) {
+            const attempt = get(orgId, userId, id);
+            if (attempt.view.credential)
+                return attempt.view.credential;
+            if (attempt.saving)
+                return attempt.saving;
+            if (!attempt.auth || attempt.view.status !== "ready")
+                throw new RecordStoreError(409, "Finish signing in with ChatGPT first.");
+            attempt.view.status = "saving";
+            // A migration key also makes retries safe after a partial storage write.
+            attempt.saving = saveCredential(orgRecords(records, orgId), orgId, {
+                name: attempt.name,
+                kind: "openai",
+                auth: "codex-login",
+                value: attempt.auth,
+            }, {}, `codex-login:${id}`)
+                .then((credential) => {
+                delete attempt.auth;
+                attempt.view.credential = credential;
+                attempt.view.status = "saved";
+                return credential;
+            })
+                .catch(() => {
+                attempt.view.status = "ready";
+                throw new RecordStoreError(503, "Sign-in completed, but saving failed. Retry saving your credential.");
+            })
+                .finally(() => {
+                delete attempt.saving;
+            });
+            return attempt.saving;
+        },
+        async cancel(orgId, userId, id) {
+            const attempt = get(orgId, userId, id);
+            if (attempt.saving)
+                await attempt.saving;
+            await cancel(id);
+        },
+        async close() {
+            await Promise.all([...attempts.keys()].map(cancel));
+        },
+    };
+}
+export const codexLogins = createCodexLogins();
+//# sourceMappingURL=codex-login.js.map

package/dist/evaluation/codex-login.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex-login.js","sourceRoot":"","sources":["../../src/evaluation/codex-login.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAGL,eAAe,GAChB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAA6B,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAqB9C,kFAAkF;AAClF,MAAM,UAAU,iBAAiB,CAAC,KAAK,GAAG,eAAe,EAAE,UAAU,GAAG,EAAE,GAAG,MAAM;IACjF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,EAAE,EAAU,EAAE,EAAE;QAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpB,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,OAAO,CAAC,IAAI,CAAC;QACpB,MAAM,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;IACjC,CAAC,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAc,EAAE,EAAU,EAAE,EAAE;QACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAClE,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,gDAAgD,CAAC,CAAC;QACpF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,KAAa,EAAE,MAAc,EAAE,IAAY;YACrD,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;gBAC3B,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,6CAA6C,CAAC,CAAC;YACjF,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC/B,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,EAAE;oBACvC,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,6CAA6C,CAAC,CAAC;gBACjF,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;gBACrF,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;wBACnC,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,2CAA2C,CAAC,CAAC;oBAC/E,MAAM,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjC,CAAC;gBACD,IAAI,QAAQ,CAAC,IAAI,IAAI,EAAE;oBACrB,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,mDAAmD,CAAC,CAAC;gBACvF,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAqB;oBAC7B,EAAE;oBACF,MAAM,EAAE,UAAU;oBAClB,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,WAAW,EAAE;iBAC3D,CAAC;gBACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC;gBAClB,CAAC,EAAE,UAAU,CAAC,CAAC;gBACf,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,MAAM,OAAO,GAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAC1E,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBAC1B,KAAK,CAAC,KAAK,IAAI,EAAE;oBACf,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,KAAK,EAAE,CAAC;wBAC9B,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;wBAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;4BACtB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;4BACtB,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC;wBAC/C,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;wBACxB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;wBAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;4BAAE,OAAO;wBAC9B,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC;4BACnC,IAAI,EAAE,UAAU;4BAChB,IAAI,EAAE,QAAQ;4BACd,IAAI,EAAE,aAAa;4BACnB,KAAK,EAAE,IAAI;yBACZ,CAAC,CAAC;wBACH,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;wBAC3B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;oBACxB,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;4BACrB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;4BACvB,IAAI,CAAC,KAAK;gCACR,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;oCACjD,CAAC,CAAC,KAAK,CAAC,OAAO;oCACf,CAAC,CAAC,0DAA0D,CAAC;wBACnE,CAAC;oBACH,CAAC;4BAAS,CAAC;wBACT,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;wBAChC,OAAO,OAAO,CAAC,OAAO,CAAC;wBACvB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;oBACzB,CAAC;gBACH,CAAC,CAAC,EAAE,CAAC;gBACL,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC;YACrB,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,MAAM,CAAC,KAAa,EAAE,MAAc,EAAE,EAAU;YAC9C,OAAO,EAAE,GAAG,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,MAAc,EAAE,EAAU,EAAE,OAA6B;YACrF,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU;gBAAE,OAAO,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;YAC5D,IAAI,OAAO,CAAC,MAAM;gBAAE,OAAO,OAAO,CAAC,MAAM,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO;gBAClD,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,uCAAuC,CAAC,CAAC;YAC3E,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;YAC/B,yEAAyE;YACzE,OAAO,CAAC,MAAM,GAAG,cAAc,CAC7B,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,EAC1B,KAAK,EACL;gBACE,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,OAAO,CAAC,IAAI;aACpB,EACD,EAAE,EACF,eAAe,EAAE,EAAE,CACpB;iBACE,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE;gBACnB,OAAO,OAAO,CAAC,IAAI,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;gBAC9B,OAAO,UAAU,CAAC;YACpB,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE;gBACV,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;gBAC9B,MAAM,IAAI,gBAAgB,CACxB,GAAG,EACH,qEAAqE,CACtE,CAAC;YACJ,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,OAAO,OAAO,CAAC,MAAM,CAAC;YACxB,CAAC,CAAC,CAAC;YACL,OAAO,OAAO,CAAC,MAAM,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAc,EAAE,EAAU;YACpD,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YACvC,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,OAAO,CAAC,MAAM,CAAC;YACzC,MAAM,MAAM,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACtD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAC"}

package/dist/evaluation/comparisons.d.ts

@@ -0,0 +1,65 @@
+import { z } from "zod";
+import type { ArtifactStore } from "../artifacts.js";
+import type { TaskStore } from "../site/task-store.js";
+import { type ComparisonRecord } from "./account.js";
+import type { EncryptedRecordStore } from "./encrypted-records.js";
+import type { EvaluationInput } from "./types.js";
+export declare const comparisonSchema: z.ZodObject<{
+    id: z.ZodUUID;
+    tasks: z.ZodArray<z.ZodObject<{
+        runId: z.ZodString;
+        taskId: z.ZodString;
+    }, z.core.$strict>>;
+    models: z.ZodArray<z.ZodObject<{
+        catalogId: z.ZodString;
+        customModel: z.ZodOptional<z.ZodString>;
+        credentialId: z.ZodUUID;
+        thinking: z.ZodOptional<z.ZodEnum<{
+            default: "default";
+            medium: "medium";
+            low: "low";
+            high: "high";
+            max: "max";
+            off: "off";
+            minimal: "minimal";
+            xhigh: "xhigh";
+        }>>;
+        harnesses: z.ZodArray<z.ZodEnum<{
+            codex: "codex";
+            "claude-code": "claude-code";
+            pi: "pi";
+            "mini-swe-agent": "mini-swe-agent";
+            "terminus-2": "terminus-2";
+        }>>;
+    }, z.core.$strict>>;
+    sandbox: z.ZodEnum<{
+        modal: "modal";
+        e2b: "e2b";
+        daytona: "daytona";
+    }>;
+    sandboxCredentialId: z.ZodUUID;
+}, z.core.$strict>;
+export type ComparisonDraft = z.infer<typeof comparisonSchema>;
+export interface ComparisonScope {
+    repoId: number;
+    ownerId: number;
+    credentialOrgId?: number;
+    tenant: string;
+    login: string;
+}
+export declare function createComparison(records: EncryptedRecordStore, tasks: TaskStore, scope: ComparisonScope, draft: ComparisonDraft): Promise<ComparisonRecord>;
+export declare function comparisonStatus(store: ArtifactStore, record: ComparisonRecord): Promise<{
+    id: string;
+    createdAt: string;
+    runs: {
+        id: string;
+        model: string;
+        thinking: "default" | "medium" | "low" | "high" | "max" | "off" | "minimal" | "xhigh" | undefined;
+        harnesses: ("codex" | "claude-code" | "pi" | "mini-swe-agent" | "terminus-2")[];
+        status: string;
+        completed: number;
+        trials: number;
+    }[];
+}>;
+export declare function dispatchComparison(store: ArtifactStore, record: ComparisonRecord, start: (input: EvaluationInput) => Promise<void>): Promise<void>;
+//# sourceMappingURL=comparisons.d.ts.map

package/dist/evaluation/comparisons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparisons.d.ts","sourceRoot":"","sources":["../../src/evaluation/comparisons.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,KAAK,gBAAgB,EAA8B,MAAM,cAAc,CAAC;AAEjF,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAKnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BlB,CAAC;AACZ,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC/D,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AACD,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,EAAE,SAAS,EAChB,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,gBAAgB,CAAC,CA4H3B;AACD,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,gBAAgB;;;;;;;;;;;;GAkBpF;AACD,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,gBAAgB,EACxB,KAAK,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,iBAMjD"}