self-bench 0.3.5 → 0.3.8

package/dist/codex-review.d.ts

@@ -1,41 +0,0 @@
-import { z } from "zod";
-import type { CouplingEvidence } from "./coupling.js";
-export declare const COUPLING_REVIEW_MODEL = "gpt-5.6-sol";
-export declare const couplingReviewSchema: z.ZodObject<{
-    verdict: z.ZodEnum<{
-        clean: "clean";
-        coupled: "coupled";
-    }>;
-    reason: z.ZodString;
-    findings: z.ZodArray<z.ZodObject<{
-        artifact: z.ZodString;
-        category: z.ZodEnum<{
-            endpoint_path: "endpoint_path";
-            field_name: "field_name";
-            header_name: "header_name";
-            media_type: "media_type";
-            other: "other";
-        }>;
-        disposition: z.ZodEnum<{
-            base_contract: "base_contract";
-            prompt_contract: "prompt_contract";
-            external_contract: "external_contract";
-            gold_only: "gold_only";
-            not_contract: "not_contract";
-        }>;
-        evidence: z.ZodString;
-    }, z.core.$strip>>;
-    counterexample: z.ZodString;
-}, z.core.$strip>;
-export type CouplingReview = z.infer<typeof couplingReviewSchema>;
-export declare function reviewCouplingWithCodex(input: {
-    readonly apiKey?: string;
-    readonly authJson?: string;
-    readonly prompt: string;
-    readonly signal?: AbortSignal;
-    readonly fetch?: typeof fetch;
-}): Promise<CouplingReview>;
-export declare function parseCodexReviewEvents(value: string): CouplingReview;
-export declare function couplingReviewPrompt(): string;
-export declare function couplingReviewInput(prompt: string, testPatch: string, goldPatch: string, evidence: CouplingEvidence): string;
-//# sourceMappingURL=codex-review.d.ts.map

package/dist/codex-review.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"codex-review.d.ts","sourceRoot":"","sources":["../src/codex-review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AAEnD,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;iBAoB/B,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,wBAAsB,uBAAuB,CAAC,KAAK,EAAE;IACnD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CAC/B,GAAG,OAAO,CAAC,cAAc,CAAC,CAkD1B;AAuBD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CA4BpE;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAQ7C;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,gBAAgB,GACzB,MAAM,CAyBR"}

package/dist/codex-review.js

@@ -1,193 +0,0 @@
-import { z } from "zod";
-export const COUPLING_REVIEW_MODEL = "gpt-5.6-sol";
-export const couplingReviewSchema = z.object({
-    verdict: z.enum(["clean", "coupled"]),
-    reason: z.string().min(1),
-    findings: z
-        .array(z.object({
-        artifact: z.string().min(1),
-        category: z.enum(["endpoint_path", "field_name", "header_name", "media_type", "other"]),
-        disposition: z.enum([
-            "base_contract",
-            "prompt_contract",
-            "external_contract",
-            "gold_only",
-            "not_contract",
-        ]),
-        evidence: z.string().min(1),
-    }))
-        .min(1),
-    counterexample: z.string().min(1),
-});
-export async function reviewCouplingWithCodex(input) {
-    const subscription = !input.apiKey && input.authJson ? parseCredential(input.authJson) : undefined;
-    if (!input.apiKey && !subscription) {
-        throw new Error("OpenAI model authentication is required");
-    }
-    const request = input.fetch ?? fetch;
-    const response = await request(subscription
-        ? "https://chatgpt.com/backend-api/codex/responses"
-        : "https://api.openai.com/v1/responses", {
-        method: "POST",
-        headers: {
-            Authorization: `Bearer ${subscription?.access ?? input.apiKey}`,
-            ...(subscription ? { "ChatGPT-Account-Id": accountIdFromToken(subscription.access) } : {}),
-            Originator: "selfbench",
-            "OpenAI-Beta": "responses=experimental",
-            Accept: "text/event-stream",
-            "Content-Type": "application/json",
-        },
-        body: JSON.stringify({
-            model: COUPLING_REVIEW_MODEL,
-            store: false,
-            stream: true,
-            instructions: "You are an independent benchmark-quality reviewer. Analyze the supplied evidence and call submit_review exactly once.",
-            input: [
-                {
-                    role: "user",
-                    content: [{ type: "input_text", text: input.prompt }],
-                },
-            ],
-            reasoning: { effort: "high", summary: "auto" },
-            text: { verbosity: "low" },
-            include: ["reasoning.encrypted_content"],
-            tools: [reviewTool()],
-            tool_choice: { type: "function", name: "submit_review" },
-            parallel_tool_calls: false,
-        }),
-        ...(input.signal ? { signal: input.signal } : {}),
-    });
-    const responseText = await response.text();
-    if (!response.ok) {
-        throw new Error(`Codex coupling review failed (${response.status}): ${responseText.slice(0, 500)}`);
-    }
-    return parseCodexReviewEvents(responseText);
-}
-function parseCredential(value) {
-    const parsed = JSON.parse(value);
-    const credential = isRecord(parsed) ? parsed["openai-codex"] : undefined;
-    if (!isRecord(credential) || typeof credential.access !== "string") {
-        throw new Error("Pi auth does not contain an OpenAI Codex subscription access token");
-    }
-    return { access: credential.access };
-}
-function accountIdFromToken(token) {
-    const payload = JSON.parse(Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8"));
-    const auth = isRecord(payload) ? payload["https://api.openai.com/auth"] : undefined;
-    const accountId = isRecord(auth) ? auth.chatgpt_account_id : undefined;
-    if (typeof accountId !== "string" || !accountId) {
-        throw new Error("OpenAI Codex subscription token has no ChatGPT account ID");
-    }
-    return accountId;
-}
-export function parseCodexReviewEvents(value) {
-    for (const line of value.split("\n")) {
-        if (!line.startsWith("data: ")) {
-            continue;
-        }
-        const data = line.slice("data: ".length);
-        if (data === "[DONE]") {
-            continue;
-        }
-        const event = JSON.parse(data);
-        if (!isRecord(event)) {
-            continue;
-        }
-        const item = event.item;
-        if (event.type === "response.output_item.done" &&
-            isRecord(item) &&
-            item.type === "function_call" &&
-            item.name === "submit_review" &&
-            typeof item.arguments === "string") {
-            return couplingReviewSchema.parse(JSON.parse(item.arguments));
-        }
-        if (event.type === "error") {
-            throw new Error(`Codex coupling review stream failed: ${JSON.stringify(event)}`);
-        }
-    }
-    throw new Error("Codex coupling review returned no submit_review tool call");
-}
-export function couplingReviewPrompt() {
-    return `Independently review this hard benchmark for test-to-gold coupling.
-
-For every exact endpoint path, response/request field, header, media type, helper/module, error string, schema/index name, UI copy/order, or other implementation artifact asserted by the held-out tests, cite either the authentic prompt text that requires it or deterministic evidence that the exact base repository already establishes it. A name merely appearing in the gold patch is not evidence. Reject a test that would fail a coherent implementation of the request using different names, file boundaries, payload presentation, or internal structure.
-
-Resolve every artifact listed in couplingEvidence.blockers with a finding whose artifact exactly matches it. Use external_contract only when the authentic request names a standard protocol that fixes the exact artifact independently of the gold patch, and cite that protocol. Use not_contract only for incidental test-language or framework syntax that is not an asserted product contract. A clean verdict with a missing blocker finding or a gold_only finding is rejected automatically.
-
-Your findings must identify the artifact, classify it, give its disposition, and cite concrete evidence. The counterexample must describe a plausible alternative correct implementation and whether the tests would accept it. Also reject prompt leakage of PRs, commits, test names, or the solution.`;
-}
-export function couplingReviewInput(prompt, testPatch, goldPatch, evidence) {
-    return `${couplingReviewPrompt()}
-
-# Authentic request
-
-${prompt}
-
-# Deterministic base-contract evidence
-
-The exact-artifact scan compared the held-out tests with the authentic request, gold additions, and exact base snapshot. Every blocker below is asserted by tests and introduced by gold, but absent verbatim from both request and base. Resolve each blocker explicitly.
-
-${JSON.stringify(evidence, null, 2)}
-
-# Held-out test patch
-
-\`\`\`diff
-${testPatch}
-\`\`\`
-
-# Gold implementation patch
-
-\`\`\`diff
-${goldPatch}
-\`\`\`
-`;
-}
-function reviewTool() {
-    return {
-        type: "function",
-        name: "submit_review",
-        description: "Submit the final independent test-coupling verdict exactly once.",
-        strict: true,
-        parameters: {
-            type: "object",
-            additionalProperties: false,
-            required: ["verdict", "reason", "findings", "counterexample"],
-            properties: {
-                verdict: { type: "string", enum: ["clean", "coupled"] },
-                reason: { type: "string", minLength: 1 },
-                findings: {
-                    type: "array",
-                    minItems: 1,
-                    items: {
-                        type: "object",
-                        additionalProperties: false,
-                        required: ["artifact", "category", "disposition", "evidence"],
-                        properties: {
-                            artifact: { type: "string", minLength: 1 },
-                            category: {
-                                type: "string",
-                                enum: ["endpoint_path", "field_name", "header_name", "media_type", "other"],
-                            },
-                            disposition: {
-                                type: "string",
-                                enum: [
-                                    "base_contract",
-                                    "prompt_contract",
-                                    "external_contract",
-                                    "gold_only",
-                                    "not_contract",
-                                ],
-                            },
-                            evidence: { type: "string", minLength: 1 },
-                        },
-                    },
-                },
-                counterexample: { type: "string", minLength: 1 },
-            },
-        },
-    };
-}
-function isRecord(value) {
-    return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-//# sourceMappingURL=codex-review.js.map

package/dist/codex-review.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"codex-review.js","sourceRoot":"","sources":["../src/codex-review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAEnD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACrC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,QAAQ,EAAE,CAAC;SACR,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3B,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QACvF,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC;YAClB,eAAe;YACf,iBAAiB;YACjB,mBAAmB;YACnB,WAAW;YACX,cAAc;SACf,CAAC;QACF,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;KAC5B,CAAC,CACH;SACA,GAAG,CAAC,CAAC,CAAC;IACT,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAM7C;IACC,MAAM,YAAY,GAChB,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChF,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC;IACrC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAC5B,YAAY;QACV,CAAC,CAAC,iDAAiD;QACnD,CAAC,CAAC,qCAAqC,EACzC;QACE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,aAAa,EAAE,UAAU,YAAY,EAAE,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAC/D,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,kBAAkB,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1F,UAAU,EAAE,WAAW;YACvB,aAAa,EAAE,wBAAwB;YACvC,MAAM,EAAE,mBAAmB;YAC3B,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK,EAAE,qBAAqB;YAC5B,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,IAAI;YACZ,YAAY,EACV,uHAAuH;YACzH,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;iBACtD;aACF;YACD,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;YAC9C,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;YAC1B,OAAO,EAAE,CAAC,6BAA6B,CAAC;YACxC,KAAK,EAAE,CAAC,UAAU,EAAE,CAAC;YACrB,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE;YACxD,mBAAmB,EAAE,KAAK;SAC3B,CAAC;QACF,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CACF,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACb,iCAAiC,QAAQ,CAAC,MAAM,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACnF,CAAC;IACJ,CAAC;IACD,OAAO,sBAAsB,CAAC,YAAY,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAY,CAAC;IAC5C,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,UAAU,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC1D,CAAC;IACb,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpF,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa;IAClD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACxB,IACE,KAAK,CAAC,IAAI,KAAK,2BAA2B;YAC1C,QAAQ,CAAC,IAAI,CAAC;YACd,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,EAClC,CAAC;YACD,OAAO,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,OAAO;;;;;;ySAMgS,CAAC;AAC1S,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,SAAiB,EACjB,SAAiB,EACjB,QAA0B;IAE1B,OAAO,GAAG,oBAAoB,EAAE;;;;EAIhC,MAAM;;;;;;EAMN,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;;;;;EAKjC,SAAS;;;;;;EAMT,SAAS;;CAEV,CAAC;AACF,CAAC;AAED,SAAS,UAAU;IACjB,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,kEAAkE;QAC/E,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,oBAAoB,EAAE,KAAK;YAC3B,QAAQ,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,CAAC;YAC7D,UAAU,EAAE;gBACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE;gBACvD,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE;gBACxC,QAAQ,EAAE;oBACR,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,CAAC;oBACX,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,oBAAoB,EAAE,KAAK;wBAC3B,QAAQ,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,CAAC;wBAC7D,UAAU,EAAE;4BACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE;4BAC1C,QAAQ,EAAE;gCACR,IAAI,EAAE,QAAQ;gCACd,IAAI,EAAE,CAAC,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,OAAO,CAAC;6BAC5E;4BACD,WAAW,EAAE;gCACX,IAAI,EAAE,QAAQ;gCACd,IAAI,EAAE;oCACJ,eAAe;oCACf,iBAAiB;oCACjB,mBAAmB;oCACnB,WAAW;oCACX,cAAc;iCACf;6BACF;4BACD,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE;yBAC3C;qBACF;iBACF;gBACD,cAAc,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE;aACjD;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/docker-executor.d.ts

@@ -1,11 +0,0 @@
-import type { SelfBenchConfig } from "./config.js";
-import type { SandboxExecutor, SandboxRequest, SandboxResult, SandboxRunOptions } from "./sandbox.js";
-export declare class DockerSandboxExecutor implements SandboxExecutor {
-    #private;
-    constructor(config: Extract<SelfBenchConfig["execution"], {
-        kind: "docker";
-    }>);
-    run(request: SandboxRequest, options?: SandboxRunOptions): Promise<SandboxResult>;
-    close(): void;
-}
-//# sourceMappingURL=docker-executor.d.ts.map

package/dist/docker-executor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"docker-executor.d.ts","sourceRoot":"","sources":["../src/docker-executor.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,aAAa,EACb,iBAAiB,EAClB,MAAM,cAAc,CAAC;AAEtB,qBAAa,qBAAsB,YAAW,eAAe;;gBAG/C,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,EAAE;QAAE,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;IAIvE,GAAG,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;IA4E3F,KAAK,IAAI,IAAI;CACd"}

package/dist/docker-executor.js

@@ -1,103 +0,0 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { dirname, join, relative, resolve } from "node:path";
-import { runCommand } from "./process.js";
-export class DockerSandboxExecutor {
-    #config;
-    constructor(config) {
-        this.#config = config;
-    }
-    async run(request, options = {}) {
-        const root = await mkdtemp(join(tmpdir(), "selfbench-docker-"));
-        const sandboxId = sandboxName(request.runId, request.stage);
-        try {
-            for (const file of request.files ?? []) {
-                const destination = hostPath(root, file.path);
-                await mkdir(dirname(destination), { recursive: true });
-                await writeFile(destination, file.contents);
-            }
-            const environment = { ...request.environment, ...request.secrets };
-            const args = [
-                "create",
-                "--name",
-                sandboxId,
-                "--cpus",
-                String(request.cpu ?? 4),
-                "--memory",
-                `${request.memoryMiB ?? 8192}m`,
-                "--volume",
-                `${sandboxId}:/work`,
-                "--workdir",
-                "/work",
-            ];
-            for (const key of Object.keys(environment)) {
-                args.push("--env", key);
-            }
-            args.push(this.#config.image, ...request.command);
-            await runCommand("docker", ["volume", "create", sandboxId]);
-            await runCommand("docker", args, { env: { ...process.env, ...environment } });
-            await runCommand("docker", ["cp", `${root}/.`, `${sandboxId}:/work/`]);
-            const abort = () => {
-                void runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
-            };
-            options.signal?.addEventListener("abort", abort, { once: true });
-            try {
-                const result = await runCommand("docker", ["start", "--attach", sandboxId], {
-                    allowFailure: true,
-                    timeoutMs: request.timeoutMs,
-                    ...(request.inactivityTimeoutMs
-                        ? { inactivityTimeoutMs: request.inactivityTimeoutMs }
-                        : {}),
-                    onOutput: (stream, chunk) => options.onProgress?.({ stream, bytes: chunk.byteLength }),
-                });
-                const outputs = {};
-                for (const path of request.outputPaths ?? []) {
-                    const destination = hostPath(root, path);
-                    await mkdir(dirname(destination), { recursive: true });
-                    const copied = await runCommand("docker", ["cp", `${sandboxId}:${path}`, destination], {
-                        allowFailure: true,
-                    });
-                    if (copied.exitCode === 0) {
-                        outputs[path] = await readFile(destination);
-                    }
-                    else if (result.exitCode === 0) {
-                        throw new Error(`sandbox ${sandboxId} exited successfully without output ${path}`);
-                    }
-                }
-                return { sandboxId, ...result, outputs };
-            }
-            finally {
-                options.signal?.removeEventListener("abort", abort);
-                await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
-                await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
-                    allowFailure: true,
-                });
-            }
-        }
-        finally {
-            await runCommand("docker", ["rm", "--force", sandboxId], { allowFailure: true });
-            await runCommand("docker", ["volume", "rm", "--force", sandboxId], {
-                allowFailure: true,
-            });
-            await rm(root, { recursive: true, force: true });
-        }
-    }
-    close() { }
-}
-function hostPath(root, containerPath) {
-    if (!containerPath.startsWith("/work/")) {
-        throw new Error(`sandbox path must be beneath /work: ${containerPath}`);
-    }
-    const path = resolve(root, relative("/work", containerPath));
-    if (path !== root && !path.startsWith(`${root}/`)) {
-        throw new Error(`sandbox path escapes /work: ${containerPath}`);
-    }
-    return path;
-}
-function sandboxName(runId, stage) {
-    const suffix = crypto.randomUUID().slice(0, 8);
-    return `selfbench-${runId.slice(0, 20)}-${stage.slice(0, 16)}-${suffix}`
-        .toLowerCase()
-        .replace(/[^a-z0-9_.-]/g, "-");
-}
-//# sourceMappingURL=docker-executor.js.map

package/dist/docker-executor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"docker-executor.js","sourceRoot":"","sources":["../src/docker-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAQ1C,MAAM,OAAO,qBAAqB;IACvB,OAAO,CAA4D;IAE5E,YAAY,MAAiE;QAC3E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAuB,EAAE,UAA6B,EAAE;QAChE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5D,IAAI,CAAC;YACH,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBACvC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG;gBACX,QAAQ;gBACR,QAAQ;gBACR,SAAS;gBACT,QAAQ;gBACR,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;gBACxB,UAAU;gBACV,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,GAAG;gBAC/B,UAAU;gBACV,GAAG,SAAS,QAAQ;gBACpB,WAAW;gBACX,OAAO;aACR,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAElD,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAC5D,MAAM,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,EAAE,CAAC,CAAC;YAC9E,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,SAAS,SAAS,CAAC,CAAC,CAAC;YAEvE,MAAM,KAAK,GAAG,GAAG,EAAE;gBACjB,KAAK,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAClF,CAAC,CAAC;YACF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE;oBAC1E,YAAY,EAAE,IAAI;oBAClB,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,GAAG,CAAC,OAAO,CAAC,mBAAmB;wBAC7B,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE;wBACtD,CAAC,CAAC,EAAE,CAAC;oBACP,QAAQ,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;iBACvF,CAAC,CAAC;gBACH,MAAM,OAAO,GAA+B,EAAE,CAAC;gBAC/C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;oBAC7C,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBACzC,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACvD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,SAAS,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,EAAE;wBACrF,YAAY,EAAE,IAAI;qBACnB,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAC;oBAC9C,CAAC;yBAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;wBACjC,MAAM,IAAI,KAAK,CAAC,WAAW,SAAS,uCAAuC,IAAI,EAAE,CAAC,CAAC;oBACrF,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC;YAC3C,CAAC;oBAAS,CAAC;gBACT,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACpD,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;gBACjF,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE;oBACjE,YAAY,EAAE,IAAI;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE;gBACjE,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,KAAK,KAAU,CAAC;CACjB;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,aAAqB;IACnD,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,aAAa,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;IAC7D,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,+BAA+B,aAAa,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,KAAa;IAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,OAAO,aAAa,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE;SACrE,WAAW,EAAE;SACb,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;AACnC,CAAC"}

package/dist/eval-main.d.ts

@@ -1,3 +0,0 @@
-#!/usr/bin/env node
-export {};
-//# sourceMappingURL=eval-main.d.ts.map

package/dist/eval-main.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"eval-main.d.ts","sourceRoot":"","sources":["../src/eval-main.ts"],"names":[],"mappings":""}

package/dist/eval-main.js

@@ -1,81 +0,0 @@
-#!/usr/bin/env node
-import { parseArgs } from "node:util";
-import { MATRIX_MODELS, runMatrix } from "./evaluate.js";
-import { isHarborEnvironment } from "./providers.js";
-const parsed = parseArgs({
-    options: {
-        export: { type: "string" },
-        tasks: { type: "string" },
-        jobs: { type: "string" },
-        harbor: { type: "string" },
-        environment: { type: "string", default: "modal" },
-        concurrency: { type: "string", default: "3" },
-        auth: { type: "string" },
-        model: { type: "string", multiple: true },
-        help: { type: "boolean", short: "h" },
-    },
-    strict: true,
-});
-if (parsed.values.help) {
-    console.log(`Run Harbor tasks through the fixed Codex model matrix.
-
-Usage:
-  self-bench-eval (--export FILE.tar.gz | --tasks DIRECTORY) --jobs DIRECTORY [options]
-
-Options:
-  --tasks DIRECTORY          Expanded Harbor tasks (one or more)
-  --harbor PATH              Harbor executable (default: harbor)
-  --environment docker|modal Execution environment (default: modal)
-  --concurrency N            Concurrent trials (default: 3)
-  --model MODEL              Run only this model; may be repeated
-  --auth FILE                Optional Codex ChatGPT auth.json fallback
-  -h, --help                 Show this help`);
-    process.exit(0);
-}
-const environment = parsed.values.environment ?? "modal";
-if (!isHarborEnvironment(environment)) {
-    throw new Error("--environment must be docker or modal");
-}
-const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
-const models = evaluationModels(parsed.values.model);
-const summaries = await runMatrix({
-    ...(parsed.values.export ? { exportPath: parsed.values.export } : {}),
-    ...(parsed.values.tasks ? { tasksPath: parsed.values.tasks } : {}),
-    jobsDirectory: parsed.values.jobs ?? fail("--jobs is required"),
-    environment,
-    concurrency,
-    ...(parsed.values.harbor ? { harborPath: parsed.values.harbor } : {}),
-    ...(parsed.values.auth ? { authPath: parsed.values.auth } : {}),
-    ...(models ? { models } : {}),
-    onTrialComplete: (summary, completed, total) => {
-        const status = summary.passed ? "passed" : summary.exception ? "error" : "failed";
-        console.error(`[${completed}/${total}] ${summary.model} ${summary.taskId}: ${status}`);
-    },
-});
-console.log(JSON.stringify({
-    trials: summaries.length,
-    passed: summaries.filter((summary) => summary.passed).length,
-    failed: summaries.filter((summary) => !summary.passed).length,
-}, null, 2));
-function fail(message) {
-    throw new Error(message);
-}
-function evaluationModels(values) {
-    if (!values) {
-        return undefined;
-    }
-    const allowed = new Set(MATRIX_MODELS);
-    const invalid = values.filter((value) => !allowed.has(value));
-    if (invalid.length > 0) {
-        throw new Error(`--model must be one of ${MATRIX_MODELS.join(", ")}; got ${invalid.join(", ")}`);
-    }
-    return [...new Set(values)];
-}
-function positiveInteger(value, label) {
-    const parsed = Number(value);
-    if (!Number.isInteger(parsed) || parsed < 1) {
-        throw new Error(`${label} must be a positive integer`);
-    }
-    return parsed;
-}
-//# sourceMappingURL=eval-main.js.map

package/dist/eval-main.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"eval-main.js","sourceRoot":"","sources":["../src/eval-main.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,aAAa,EAAoB,SAAS,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,MAAM,GAAG,SAAS,CAAC;IACvB,OAAO,EAAE;QACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE;QACjD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;QAC7C,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACtC;IACD,MAAM,EAAE,IAAI;CACb,CAAC,CAAC;AACH,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;4CAY8B,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC;AACzD,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC;IACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;AAC3D,CAAC;AACD,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;AAChF,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACrD,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC;IAChC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC;IAC/D,WAAW;IACX,WAAW;IACX,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7B,eAAe,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;QAClF,OAAO,CAAC,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,KAAK,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IACzF,CAAC;CACF,CAAC,CAAC;AACH,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CACZ;IACE,MAAM,EAAE,SAAS,CAAC,MAAM;IACxB,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM;IAC5D,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM;CAC9D,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;AAEF,SAAS,IAAI,CAAC,OAAe;IAC3B,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,gBAAgB,CAAC,MAA4B;IACpD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,aAAa,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,0BAA0B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAkB,CAAC;AAC/C,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB,EAAE,KAAa;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/evaluate.d.ts

@@ -1,25 +0,0 @@
-import type { HarborEnvironment } from "./providers.js";
-export declare const MATRIX_MODELS: readonly ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"];
-export type MatrixModel = (typeof MATRIX_MODELS)[number];
-export interface MatrixOptions {
-    readonly exportPath?: string;
-    readonly tasksPath?: string;
-    readonly jobsDirectory: string;
-    readonly harborPath?: string;
-    readonly environment?: HarborEnvironment;
-    readonly concurrency?: number;
-    readonly authPath?: string;
-    readonly models?: readonly MatrixModel[];
-    readonly onTrialComplete?: (summary: MatrixTrialSummary, completed: number, total: number) => void;
-}
-export interface MatrixTrialSummary {
-    readonly taskId: string;
-    readonly model: MatrixModel;
-    readonly jobName: string;
-    readonly passed: boolean;
-    readonly rewards: Readonly<Record<string, number>>;
-    readonly exception?: string;
-}
-export declare function runMatrix(options: MatrixOptions): Promise<readonly MatrixTrialSummary[]>;
-export declare function summarizeResult(taskId: string, model: MatrixModel, jobName: string, result: unknown): MatrixTrialSummary;
-//# sourceMappingURL=evaluate.d.ts.map

package/dist/evaluate.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGxD,eAAO,MAAM,aAAa,2DAA4D,CAAC;AACvF,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAGzD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,eAAe,CAAC,EAAE,CACzB,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,KACV,IAAI,CAAC;CACX;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACnD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAsB,SAAS,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAgD9F;AAgGD,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,OAAO,GACd,kBAAkB,CA6BpB"}