sparta_worlds 0.6.0

package/dist/cli/judge-command.js

@@ -0,0 +1,184 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { chargeLookupFor, judgeShift, judgeWorld, parseTranscriptLines, planTranscriptJudge, policyAmountsCents, summarizeJudge, } from "./judge.js";
+import { createAdminCall, judgeLines, judgeSummaryLine, PasskConfigError, requireBeforeImages, resolveJudge, } from "./passk.js";
+import { gradeShift, loadRubric } from "./rubric.js";
+import { stdioDead } from "./stdio.js";
+export async function runJudgeCommand(opts) {
+    let rubric;
+    try {
+        rubric = opts.rubric ? loadRubric(opts.rubric) : undefined;
+    }
+    catch (err) {
+        throw new PasskConfigError(err instanceof Error ? err.message : String(err), { cause: err });
+    }
+    const judge = await resolveJudge({
+        policyFile: opts.policyFile,
+        policyText: opts.policyText,
+        model: opts.model,
+        runs: opts.runs,
+        budget: opts.budget,
+        client: opts.client,
+    });
+    const log = opts.log ??
+        ((line) => {
+            if (!stdioDead(process.stderr))
+                console.error(line);
+        });
+    const admin = opts.admin ?? createAdminCall(opts.adminUrl, opts.adminToken);
+    const { worldId, mark } = opts;
+    const diff = await admin("GET", `/admin/worlds/${worldId}/diff?from=${encodeURIComponent(mark)}`);
+    requireBeforeImages(diff);
+    const shift = rubric
+        ? await gradeShift(rubric, diff, chargeLookupFor(admin, worldId))
+        : undefined;
+    const evidenceDir = opts.evidence ? join(resolve(opts.evidence), worldId, "judge") : undefined;
+    log(`judge: world ${worldId} since mark ${mark} · ${judge.model}${judge.runs > 1 ? ` × ${judge.runs}` : ""}${rubric ? ` · against rubric ${rubric.name}` : ""}`);
+    const result = await judgeWorld({
+        admin,
+        worldId,
+        diff,
+        policy: judge.policy,
+        model: judge.model,
+        runs: judge.runs,
+        client: judge.client,
+        budget: judge.budget ?? rubric?.budget?.credits_total_max ?? null,
+        evidenceDir,
+        log,
+    });
+    const summary = summarizeJudge([result], shift ? [shift] : undefined);
+    const payload = {
+        world_id: worldId,
+        mark,
+        judge: result,
+        summary,
+        judge_config: {
+            model: judge.model,
+            runs: judge.runs,
+            policy_sha256: judge.policy_sha256,
+            ...(opts.policyFile ? { policy_file: resolve(opts.policyFile) } : {}),
+        },
+        ...(shift ? { shift } : {}),
+    };
+    if (opts.jsonFile) {
+        const file = resolve(opts.jsonFile);
+        mkdirSync(dirname(file), { recursive: true });
+        writeFileSync(file, `${JSON.stringify(payload, null, 2)}\n`);
+    }
+    if (opts.json) {
+        console.log(JSON.stringify(payload, null, 2));
+    }
+    else {
+        console.log("");
+        for (const line of judgeLines(result))
+            console.log(line);
+        console.log("");
+        console.log(judgeSummaryLine(judge.model, summary));
+        if (evidenceDir)
+            console.log(`evidence: ${evidenceDir}`);
+    }
+    return 0;
+}
+export async function runTranscriptJudgeCommand(opts) {
+    const judge = await resolveJudge({
+        policyFile: opts.policyFile,
+        policyText: opts.policyText,
+        model: opts.model,
+        runs: opts.runs,
+        budget: opts.budget,
+        client: opts.client,
+    });
+    let clockAtMark = null;
+    if (opts.clock !== undefined) {
+        const ms = Date.parse(opts.clock);
+        if (!Number.isFinite(ms))
+            throw new PasskConfigError(`--clock must be an ISO timestamp (got '${opts.clock}')`);
+        clockAtMark = Math.floor(ms / 1000);
+    }
+    const file = resolve(opts.transcript);
+    let records;
+    try {
+        records = parseTranscriptLines(readFileSync(file, "utf8"));
+    }
+    catch (err) {
+        throw new PasskConfigError(`--transcript ${file}: ${err instanceof Error ? err.message : String(err)}`, {
+            cause: err,
+        });
+    }
+    if (records.length === 0)
+        throw new PasskConfigError(`--transcript ${file}: no records`);
+    const log = opts.log ??
+        ((line) => {
+            if (!stdioDead(process.stderr))
+                console.error(line);
+        });
+    const byWorld = new Map();
+    for (const r of records) {
+        const list = byWorld.get(r.world_id) ?? [];
+        list.push(r);
+        byWorld.set(r.world_id, list);
+    }
+    log(`judge: transcript ${file} · ${records.length} ticket(s) in ${byWorld.size} world(s) · ${judge.model}${judge.runs > 1 ? ` × ${judge.runs}` : ""}`);
+    const worlds = [];
+    for (const [worldId, list] of byWorld) {
+        const plan = planTranscriptJudge(list, {
+            clockAtMark,
+            budget: judge.budget ?? null,
+            policyAmounts: policyAmountsCents(judge.policy),
+        });
+        const evidenceDir = opts.evidence
+            ? join(resolve(opts.evidence), worldId ?? "no-world", "transcript-judge")
+            : undefined;
+        if (evidenceDir)
+            mkdirSync(evidenceDir, { recursive: true });
+        log(`  ⚖ judge (${judge.model}${judge.runs > 1 ? ` × ${judge.runs}` : ""}, transcript): ${plan.tickets.length} tickets${worldId ? ` · world ${worldId}` : ""}`);
+        const result = await judgeShift(plan, judge.policy, {
+            model: judge.model,
+            client: judge.client,
+            samples: judge.runs,
+            log,
+            ...(evidenceDir
+                ? {
+                    onTicket: (input, raw, verdict) => writeFileSync(join(evidenceDir, `${verdict.ticket_id}.json`), `${JSON.stringify({ input, raw, verdict }, null, 2)}\n`),
+                }
+                : {}),
+        });
+        worlds.push({ world_id: worldId, tickets: list.length, judge: result });
+    }
+    const summary = summarizeJudge(worlds.map((w) => w.judge));
+    const payload = {
+        transcript: file,
+        clock_at_mark_iso: clockAtMark === null ? null : new Date(clockAtMark * 1000).toISOString().replace(/\.\d{3}Z$/, "Z"),
+        worlds,
+        summary,
+        judge_config: {
+            model: judge.model,
+            runs: judge.runs,
+            policy_sha256: judge.policy_sha256,
+            ...(opts.policyFile ? { policy_file: resolve(opts.policyFile) } : {}),
+            evidence: "transcript",
+        },
+    };
+    if (opts.jsonFile) {
+        const out = resolve(opts.jsonFile);
+        mkdirSync(dirname(out), { recursive: true });
+        writeFileSync(out, `${JSON.stringify(payload, null, 2)}\n`);
+    }
+    if (opts.json) {
+        console.log(JSON.stringify(payload, null, 2));
+    }
+    else {
+        for (const w of worlds) {
+            console.log("");
+            if (worlds.length > 1)
+                console.log(`world ${w.world_id ?? "(none)"}:`);
+            for (const line of judgeLines(w.judge))
+                console.log(line);
+        }
+        console.log("");
+        console.log(judgeSummaryLine(judge.model, summary));
+        if (opts.evidence)
+            console.log(`evidence: ${resolve(opts.evidence)}`);
+    }
+    return 0;
+}