stageflow 0.42.0 → 0.46.0

package/dist/evals/runner/internalAgentCaseRunner.js

@@ -0,0 +1,250 @@
+import { randomUUID } from "node:crypto";
+import { createOperatorEvalGate } from "../../operatorAgent/evalGate.js";
+import { buildConversationSubject } from "./conversationSubject.js";
+import { createDriver, DriverScenarioError } from "./userDriver.js";
+const ZERO_USAGE = {
+    cost_usd: 0,
+    agent_cost_usd: 0,
+    simulator_cost_usd: 0,
+    tokens_in: 0,
+    tokens_out: 0,
+    simulator_tokens_in: 0,
+    simulator_tokens_out: 0,
+    simulator_estimated: false,
+    agent_calls: 0,
+    simulator_calls: 0,
+};
+function failure(code, message, extra = {}) {
+    return { ok: false, code, message, ...extra };
+}
+function parseMocks(value) {
+    const out = {};
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return out;
+    for (const [tool, mock] of Object.entries(value)) {
+        if (typeof mock !== "object" || mock === null)
+            continue;
+        const { result, error } = mock;
+        if (typeof error === "string")
+            out[tool] = { error };
+        else if ("result" in mock)
+            out[tool] = { result };
+    }
+    return out;
+}
+function detailFrom(sessionId, profileId, messages) {
+    const now = new Date().toISOString();
+    return {
+        session: {
+            id: sessionId,
+            agentId: profileId,
+            revision: 0,
+            backend: "pi",
+            status: "closed",
+            source: "eval",
+            sourceRef: null,
+            threadKey: null,
+            title: null,
+            createdAt: now,
+            updatedAt: now,
+        },
+        messages,
+        turns: [],
+        spans: [],
+        approvals: [],
+        pendingApproval: null,
+        evalScores: [],
+        deliveries: [],
+    };
+}
+export function createInternalAgentCaseRunner(deps) {
+    const clock = deps.now ?? Date.now;
+    const states = new Map();
+    const stateFor = (id) => {
+        let state = states.get(id);
+        if (!state) {
+            state = { cancelled: false, abort: new AbortController() };
+            states.set(id, state);
+        }
+        return state;
+    };
+    async function runCase(plan, req) {
+        const state = stateFor(req.evalRunId);
+        if (state.cancelled)
+            return failure("EVAL_CANCELLED", "Eval run was cancelled");
+        if (plan.target.kind !== "internal_agent" || !plan.conversation) {
+            return failure("EVAL_ENGINE_ERROR", "Not an internal-agent plan");
+        }
+        if (plan.conversation.tools === "real") {
+            return failure("EVAL_ENGINE_ERROR", "Internal agents cannot run an eval with real tools");
+        }
+        const host = deps.host();
+        if (!host)
+            return failure("EVAL_ENGINE_ERROR", "Internal agents are unavailable on this host");
+        const planCase = plan.cases.find((c) => c.id === req.caseId);
+        if (!planCase)
+            return failure("EVAL_ENGINE_ERROR", `Case "${req.caseId}" is not in the plan`);
+        const scenario = planCase.input;
+        const mode = scenario.mode === "script" || scenario.mode === "simulated" ? scenario.mode : plan.conversation.mode;
+        let driver;
+        try {
+            driver = createDriver(mode, scenario, {});
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof DriverScenarioError ? err.message : "Scenario could not be prepared");
+        }
+        if (mode === "simulated") {
+            return failure("EVAL_ENGINE_ERROR", "A simulated internal-agent case needs a user model, which this runner does not call");
+        }
+        const conversation = plan.conversation;
+        const sessionId = `op_eval_${randomUUID()}`;
+        let turn = 0;
+        const gate = createOperatorEvalGate({
+            policy: { tools: conversation.tools, mocks: parseMocks(scenario.tool_mocks) },
+            turn: () => turn,
+        });
+        let session;
+        try {
+            session = host.openEvalSession({
+                profileId: plan.target.profileId,
+                context: { kind: "stageflow_guide" },
+                gate,
+            });
+        }
+        catch (err) {
+            return failure("EVAL_ENGINE_ERROR", err instanceof Error ? err.message : "The eval session could not be opened");
+        }
+        const startedAt = clock();
+        const messages = [];
+        const transcript = [];
+        let end = {
+            kind: "done",
+            stopReason: "max_turns",
+        };
+        const caseCtl = new AbortController();
+        let wallExpired = false;
+        let turnTimedOut = false;
+        const stopCase = () => {
+            caseCtl.abort();
+            void session.abort().catch(() => undefined);
+        };
+        const onRunAbort = () => stopCase();
+        state.abort.signal.addEventListener("abort", onRunAbort, { once: true });
+        const wallMs = conversation.maxTurns * conversation.turnTimeoutMs + 60_000;
+        const wall = setTimeout(() => {
+            wallExpired = true;
+            stopCase();
+        }, wallMs);
+        const stopped = new Promise((resolve) => {
+            if (caseCtl.signal.aborted)
+                resolve("stopped");
+            else
+                caseCtl.signal.addEventListener("abort", () => resolve("stopped"), { once: true });
+        });
+        try {
+            for (turn = 1; turn <= conversation.maxTurns; turn += 1) {
+                if (state.cancelled) {
+                    end = { kind: "cancelled" };
+                    break;
+                }
+                if (wallExpired || turnTimedOut || caseCtl.signal.aborted) {
+                    end = {
+                        kind: "failed",
+                        code: "EVAL_AGENT_TIMEOUT",
+                        message: turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit",
+                        stopReason: "timeout",
+                    };
+                    break;
+                }
+                const step = await driver.next({ turn, transcript, signal: caseCtl.signal });
+                if (step.kind === "end") {
+                    end = { kind: "done", stopReason: step.reason };
+                    break;
+                }
+                const now = new Date().toISOString();
+                messages.push({ id: randomUUID(), sessionId, seq: messages.length, turnId: String(turn), role: "user", author: null, text: step.text, createdAt: now });
+                const turnTimer = setTimeout(() => {
+                    turnTimedOut = true;
+                    stopCase();
+                }, conversation.turnTimeoutMs);
+                const sent = session.send(step.text, plan.modelOverride ? { modelId: plan.modelOverride } : {}).then((events) => ({ ok: true, events }), (err) => ({ ok: false, err }));
+                const outcome = await Promise.race([sent, stopped]);
+                clearTimeout(turnTimer);
+                if (state.cancelled) {
+                    end = { kind: "cancelled" };
+                    break;
+                }
+                if (outcome === "stopped" || wallExpired || turnTimedOut) {
+                    end = {
+                        kind: "failed",
+                        code: "EVAL_AGENT_TIMEOUT",
+                        message: turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit",
+                        stopReason: "timeout",
+                    };
+                    break;
+                }
+                if (!outcome.ok) {
+                    end = { kind: "failed", code: "EVAL_ENGINE_ERROR", message: outcome.err instanceof Error ? outcome.err.message : "Conversation failed", stopReason: "error" };
+                    break;
+                }
+                const text = outcome.events
+                    .filter((event) => event.type === "message" && event.role === "assistant")
+                    .map((event) => (event.type === "message" ? event.text : ""))
+                    .filter(Boolean)
+                    .join("\n\n");
+                if (!text && gate.records().every((call) => call.turn !== turn)) {
+                    end = { kind: "failed", code: "EVAL_AGENT_EMPTY_REPLY", message: "The agent returned an empty reply", stopReason: "error" };
+                    break;
+                }
+                messages.push({ id: randomUUID(), sessionId, seq: messages.length, turnId: String(turn), role: "assistant", author: null, text, createdAt: now });
+                transcript.push({ role: "user", text: step.text }, { role: "assistant", text });
+                if (step.endAfter !== undefined) {
+                    end = { kind: "done", stopReason: step.endAfter };
+                    break;
+                }
+                if (turn === conversation.maxTurns)
+                    end = { kind: "done", stopReason: "max_turns" };
+            }
+        }
+        catch (err) {
+            end = state.cancelled
+                ? { kind: "cancelled" }
+                : wallExpired || turnTimedOut
+                    ? { kind: "failed", code: "EVAL_AGENT_TIMEOUT", message: turnTimedOut ? `Turn ${turn} exceeded ${conversation.turnTimeoutMs} ms` : "Conversation exceeded its time limit", stopReason: "timeout" }
+                    : { kind: "failed", code: "EVAL_ENGINE_ERROR", message: err instanceof Error ? err.message : "Conversation failed", stopReason: "error" };
+        }
+        finally {
+            clearTimeout(wall);
+            state.abort.signal.removeEventListener("abort", onRunAbort);
+            session.close();
+        }
+        const usage = { ...ZERO_USAGE, agent_calls: messages.filter((m) => m.role === "assistant").length };
+        const subject = (stopReason, failed) => buildConversationSubject({
+            detail: detailFrom(sessionId, plan.target.kind === "internal_agent" ? plan.target.profileId : "", messages),
+            calls: gate.records(),
+            stopReason,
+            usage,
+            turns: messages.filter((m) => m.role === "user").length,
+            ...(failed !== undefined ? { failure: failed } : {}),
+        });
+        const latencyMs = Math.max(0, clock() - startedAt);
+        if (end.kind === "done") {
+            return { ok: true, subjectRef: sessionId, output: subject(end.stopReason), latencyMs, costUsd: 0, tokensIn: 0, tokensOut: 0 };
+        }
+        if (end.kind === "cancelled")
+            return failure("EVAL_CANCELLED", "Eval run was cancelled", { subjectRef: sessionId, latencyMs, costUsd: 0 });
+        return failure(end.code, end.message, { subjectRef: sessionId, output: subject(end.stopReason, end.message), latencyMs, costUsd: 0 });
+    }
+    return {
+        runCase,
+        async cancelAll(evalRunId) {
+            const state = states.get(evalRunId);
+            if (!state)
+                return;
+            state.cancelled = true;
+            state.abort.abort();
+        },
+        spend: () => 0,
+        addSpend: () => undefined,
+    };
+}

package/dist/evals/snapshot.d.ts

@@ -74,6 +74,14 @@ export type ConversationSettings = {
     user_model_from_judge: boolean;
     simulator_prompt_version: number;
 };
+export type InternalAgentSnapshotTarget = {
+    kind: "internal_agent";
+    profile_id: string;
+    title: string;
+    playbook_hash: string;
+    tools: string[];
+    model: string;
+};
 export type ConfigSnapshot = {
     snapshot_version: number;
     eval: {
@@ -83,7 +91,7 @@ export type ConfigSnapshot = {
         definition: Record<string, unknown>;
     };
     overrides: SnapshotOverrides;
-    target: StageSnapshotTarget | AgentSnapshotTarget;
+    target: StageSnapshotTarget | AgentSnapshotTarget | InternalAgentSnapshotTarget;
     dataset: {
         path: string;
         hash: string;
@@ -169,7 +177,7 @@ export type SnapshotInput = {
     evalHash: string;
     evalDefinition: Record<string, unknown>;
     overrides: SnapshotOverrides;
-    target: Omit<StageSnapshotTarget, "kind"> | Omit<AgentSnapshotTarget, "kind">;
+    target: Omit<StageSnapshotTarget, "kind"> | Omit<AgentSnapshotTarget, "kind"> | Omit<InternalAgentSnapshotTarget, "kind">;
     dataset: ConfigSnapshot["dataset"];
     graders: SnapshotGrader[];
     settings: ConfigSnapshot["settings"];

package/dist/evals/snapshot.js

@@ -40,7 +40,8 @@ export function computeFingerprint(components) {
     return sha256Hex(canonicalJson(payload)).slice(0, 8);
 }
 export function buildSnapshot(input) {
-    const agentTarget = "agent_id" in input.target ? input.target : null;
+    const internalTarget = "profile_id" in input.target ? input.target : null;
+    const agentTarget = !internalTarget && "agent_id" in input.target ? input.target : null;
     return {
         snapshot_version: CONFIG_SNAPSHOT_VERSION,
         eval: {
@@ -50,14 +51,18 @@ export function buildSnapshot(input) {
             definition: input.evalDefinition,
         },
         overrides: input.overrides,
-        target: agentTarget ? { kind: "agent", ...agentTarget } : { kind: "stage", ...input.target },
+        target: internalTarget
+            ? { kind: "internal_agent", ...internalTarget }
+            : agentTarget
+                ? { kind: "agent", ...agentTarget }
+                : { kind: "stage", ...input.target },
         dataset: input.dataset,
         graders: input.graders,
         settings: input.settings,
         engine: {
             name: input.engine.name,
             version: input.engine.version,
-            rubric_prompt_version: agentTarget ? CONVERSATION_RUBRIC_PROMPT_VERSION : RUBRIC_PROMPT_VERSION,
+            rubric_prompt_version: agentTarget || internalTarget ? CONVERSATION_RUBRIC_PROMPT_VERSION : RUBRIC_PROMPT_VERSION,
             node_version: input.engine.nodeVersion,
         },
         stageflow_version: input.stageflowVersion,

package/dist/evals/stats.d.ts

@@ -93,3 +93,43 @@ export declare function clusteredCi(center: number, perCase: number[], lo: numbe
 export declare function buildCaseStats(results: ResultRecord[]): CaseStat[];
 export declare function computeByGrader(graders: GraderRef[], grades: GradeRecord[], previousRun: PreviousRunLite | null | undefined): GraderStat[];
 export declare function computeStats(input: ComputeStatsInput): Stats;
+export declare const CALIBRATION_MIN_LABELS = 20;
+export declare const CALIBRATION_MINORITY_SHARE = 0.2;
+export interface CalibrationSample {
+    caseId: string;
+    repeatIndex: number;
+    label: boolean;
+    judge: boolean;
+}
+export interface CalibrationStats {
+    n: number;
+    agreement: number | null;
+    cohen_kappa: number | null;
+    tpr: number | null;
+    tnr: number | null;
+    confusion: {
+        tp: number;
+        fp: number;
+        tn: number;
+        fn: number;
+    };
+    warnings: Array<"low_n" | "imbalanced" | "no_variance">;
+}
+/** Judge pass is the positive class; the human label is the truth. */
+export declare function computeCalibration(samples: CalibrationSample[]): CalibrationStats;
+/** A strict majority of valid votes passes; a tie fails. */
+export declare function majorityPasses(passes: number, valid: number): boolean;
+export declare function meanOf(values: readonly number[]): number;
+/** Deterministic 70/30 split by a stable hash of the case id, so every repeat of a case lands on the same side. */
+export declare function calibrationSplitOf(caseId: string): "dev" | "held_out";
+export interface StabilityStats {
+    /** Judge calls per output. */
+    k: number;
+    /** Outputs with at least two valid votes. */
+    n: number;
+    /** Outputs whose valid votes were not all the same verdict. */
+    unstable: number;
+    flip_rate: number | null;
+}
+/** How often the judge changes its mind on the same output: the share of outputs with a split vote. */
+export declare function computeStability(voteSets: boolean[][], k: number): StabilityStats;

package/dist/evals/stats.js

@@ -218,3 +218,72 @@ export function computeStats(input) {
         ...(hasAgentData ? { mean_turns: mean(turnValues), mean_tool_calls: mean(toolCallValues) } : {}),
     };
 }
+export const CALIBRATION_MIN_LABELS = 20;
+export const CALIBRATION_MINORITY_SHARE = 0.2;
+/** Judge pass is the positive class; the human label is the truth. */
+export function computeCalibration(samples) {
+    let tp = 0;
+    let fp = 0;
+    let tn = 0;
+    let fn = 0;
+    for (const s of samples) {
+        if (s.judge && s.label)
+            tp++;
+        else if (s.judge && !s.label)
+            fp++;
+        else if (!s.judge && !s.label)
+            tn++;
+        else
+            fn++;
+    }
+    const n = tp + fp + tn + fn;
+    const warnings = [];
+    if (n < CALIBRATION_MIN_LABELS)
+        warnings.push("low_n");
+    const labelPass = tp + fn;
+    const labelFail = fp + tn;
+    if (n > 0 && Math.min(labelPass, labelFail) / n < CALIBRATION_MINORITY_SHARE)
+        warnings.push("imbalanced");
+    const agreement = n > 0 ? (tp + tn) / n : null;
+    const judgePass = tp + fp;
+    const judgeFail = fn + tn;
+    const expected = n > 0 ? (judgePass * labelPass + judgeFail * labelFail) / (n * n) : 0;
+    let kappa = null;
+    if (n > 0 && agreement !== null) {
+        if (expected >= 1)
+            warnings.push("no_variance");
+        else
+            kappa = (agreement - expected) / (1 - expected);
+    }
+    return {
+        n,
+        agreement,
+        cohen_kappa: kappa,
+        tpr: labelPass > 0 ? tp / labelPass : null,
+        tnr: labelFail > 0 ? tn / labelFail : null,
+        confusion: { tp, fp, tn, fn },
+        warnings,
+    };
+}
+/** A strict majority of valid votes passes; a tie fails. */
+export function majorityPasses(passes, valid) {
+    return passes * 2 > valid;
+}
+export function meanOf(values) {
+    return values.reduce((total, v) => total + v, 0) / values.length;
+}
+/** Deterministic 70/30 split by a stable hash of the case id, so every repeat of a case lands on the same side. */
+export function calibrationSplitOf(caseId) {
+    let h = 2166136261;
+    for (let i = 0; i < caseId.length; i++) {
+        h ^= caseId.charCodeAt(i);
+        h = Math.imul(h, 16777619) >>> 0;
+    }
+    return h % 10 < 7 ? "dev" : "held_out";
+}
+/** How often the judge changes its mind on the same output: the share of outputs with a split vote. */
+export function computeStability(voteSets, k) {
+    const counted = voteSets.filter((votes) => votes.length >= 2);
+    const unstable = counted.filter((votes) => votes.some((v) => v !== votes[0])).length;
+    return { k, n: counted.length, unstable, flip_rate: counted.length > 0 ? unstable / counted.length : null };
+}

package/dist/evals/storeTypes.d.ts

@@ -205,3 +205,45 @@ export type EvalGraderVersionInput = {
     graderId: string;
     graderHash: string;
 };
+export type EvalGraderLabelRow = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+    graderId: string;
+    label: EvalReviewLabel;
+    note: string | null;
+    createdAt: string;
+    updatedAt: string;
+};
+export type EvalGraderLabelPatch = {
+    label: EvalReviewLabel | null;
+    note?: string | null;
+};
+export type EvalCalibrationSplit = "all" | "dev" | "held_out";
+export type EvalCalibrationDisagreement = {
+    /** The run the labeled result came from; absent on calibrations stored before this field existed. */
+    evalRunId?: string;
+    caseId: string;
+    repeatIndex: number;
+    label: EvalReviewLabel;
+    judge: EvalReviewLabel;
+    reason: string | null;
+};
+export type EvalCalibrationInput = {
+    projectRoot: string;
+    evalId: string;
+    graderId: string;
+    graderHash: string;
+    judge: string;
+    promptVersion: number;
+    split: EvalCalibrationSplit;
+    candidate: boolean;
+    stats: EvalJson;
+    disagreements: EvalCalibrationDisagreement[];
+    costUsd: number;
+};
+export type EvalCalibrationRow = Omit<EvalCalibrationInput, "stats"> & {
+    id: string;
+    stats: EvalJson;
+    createdAt: string;
+};

package/dist/evals/types.d.ts

@@ -24,7 +24,13 @@ export type AgentEvalTarget = {
     kind: "agent";
     agent: string;
 };
-export type EvalTarget = StageEvalTarget | AgentEvalTarget;
+export type InternalAgentEvalTarget = {
+    kind: "internal_agent";
+    profile: string;
+};
+export type EvalTarget = StageEvalTarget | AgentEvalTarget | InternalAgentEvalTarget;
+/** A conversation eval: a project agent or a built-in operator agent. */
+export declare function isConversationTarget(kind: string): boolean;
 export type ConversationMode = "script" | "simulated";
 export type ConversationTools = "real" | "read_only" | "mocked";
 export type ConversationApprovals = "deny" | "auto_approve_read" | "record_only";
@@ -51,9 +57,18 @@ export type EvalTask = {
     context?: string;
     constraints?: string;
 };
+export type JudgeSettings = {
+    /** Sampling temperature for the judge call; unset leaves the provider default. */
+    temperature?: number;
+    /** The judge writes its reason before the verdict. */
+    reasoning?: boolean;
+    /** Judge calls per grade (odd, 1 to 5); the majority verdict wins. */
+    votes?: number;
+};
 export type EvalDefaults = {
     repeats?: number;
     judge?: string;
+    judge_settings?: JudgeSettings;
     pass_threshold?: number;
     max_cost_usd?: number;
     concurrency?: number;
@@ -97,6 +112,7 @@ export type GraderConfig = (GraderCommon & {
     scale?: "pass_fail" | "1_to_5";
     pass_at?: number;
     judge?: string;
+    judge_settings?: JudgeSettings;
     examples?: RubricExample[];
 }) | (GraderCommon & {
     type: "code";
@@ -116,6 +132,7 @@ export type GraderConfig = (GraderCommon & {
 }) | (GraderCommon & {
     type: "goal_reached";
     judge?: string;
+    judge_settings?: JudgeSettings;
     guidance?: string;
 });
 export type EvalFile = {
@@ -143,7 +160,11 @@ export type AgentTargetRef = {
     agentId: string;
     revision?: number;
 };
-export type TargetRef = StageTargetRef | AgentTargetRef;
+export type InternalAgentTargetRef = {
+    kind: "internal_agent";
+    profileId: string;
+};
+export type TargetRef = StageTargetRef | AgentTargetRef | InternalAgentTargetRef;
 type ResolvedGraderCommon = {
     id: string;
     weight: number;
@@ -168,6 +189,7 @@ export type ResolvedGrader = (ResolvedGraderCommon & {
     scale: "pass_fail" | "1_to_5";
     passAt: number;
     judge: string;
+    judgeSettings?: JudgeSettings;
     examples: RubricExample[];
 }) | (ResolvedGraderCommon & {
     type: "code";
@@ -187,6 +209,7 @@ export type ResolvedGrader = (ResolvedGraderCommon & {
 }) | (ResolvedGraderCommon & {
     type: "goal_reached";
     judge: string;
+    judgeSettings?: JudgeSettings;
     guidance?: string;
 });
 export type PlanCase = {

package/dist/evals/types.js

@@ -1 +1,4 @@
-export {};
+/** A conversation eval: a project agent or a built-in operator agent. */
+export function isConversationTarget(kind) {
+    return kind === "agent" || kind === "internal_agent";
+}

package/dist/evals/votes.d.ts

@@ -0,0 +1,10 @@
+import type { NormalizedGrade } from "./types.js";
+export declare const JUDGE_ERROR_CODE = "EVAL_JUDGE_ERROR";
+/**
+ * One grade from several votes: the majority verdict for pass/fail (a tie fails), the mean raw score for 1 to 5.
+ * Errored votes are left out; if every vote errored the grade is an error. Tokens and cost add up over all votes.
+ */
+export declare function combineVotes(grader: {
+    scale: "pass_fail" | "1_to_5";
+    passAt: number;
+}, votes: readonly NormalizedGrade[]): NormalizedGrade;

package/dist/evals/votes.js

@@ -0,0 +1,60 @@
+import { normalizeOneToFive } from "./scoring.js";
+import { majorityPasses, meanOf } from "./stats.js";
+export const JUDGE_ERROR_CODE = "EVAL_JUDGE_ERROR";
+/**
+ * One grade from several votes: the majority verdict for pass/fail (a tie fails), the mean raw score for 1 to 5.
+ * Errored votes are left out; if every vote errored the grade is an error. Tokens and cost add up over all votes.
+ */
+export function combineVotes(grader, votes) {
+    const first = votes[0];
+    if (votes.length === 1)
+        return first;
+    const valid = votes.filter((g) => g.status === "graded" && g.passed !== undefined);
+    const sum = (pick) => votes.reduce((total, g) => total + (pick(g) ?? 0), 0);
+    const totals = {
+        judgeTokensIn: sum((g) => g.judgeTokensIn),
+        judgeTokensOut: sum((g) => g.judgeTokensOut),
+        judgeCostUsd: sum((g) => g.judgeCostUsd),
+    };
+    const priceUnknown = votes.some((g) => g.details?.judgeCostUnknown);
+    const record = votes.map((g) => ({
+        status: g.status,
+        ...(g.passed !== undefined ? { passed: g.passed } : {}),
+        ...(g.rawValue !== undefined ? { raw_value: g.rawValue } : {}),
+        ...(g.reason !== undefined ? { reason: g.reason } : {}),
+    }));
+    const baseDetails = {
+        ...(first.details ?? {}),
+        ...(priceUnknown ? { judgeCostUnknown: true } : {}),
+        votes: record,
+    };
+    if (valid.length === 0) {
+        return { ...first, ...totals, status: "error", errorCode: first.errorCode ?? JUDGE_ERROR_CODE, details: baseDetails };
+    }
+    let passed;
+    let rawValue;
+    let score;
+    if (grader.scale === "1_to_5") {
+        rawValue = meanOf(valid.map((g) => g.rawValue ?? 0));
+        score = normalizeOneToFive(rawValue);
+        passed = rawValue >= grader.passAt;
+    }
+    else {
+        passed = majorityPasses(valid.filter((g) => g.passed === true).length, valid.length);
+        rawValue = passed ? 1 : 0;
+        score = rawValue;
+    }
+    const lead = valid.find((g) => g.passed === passed) ?? valid[0];
+    return {
+        graderId: first.graderId,
+        graderType: first.graderType,
+        status: "graded",
+        passed,
+        score,
+        rawValue,
+        ...(lead.reason !== undefined ? { reason: lead.reason } : {}),
+        ...(first.judgeModel !== undefined ? { judgeModel: first.judgeModel } : {}),
+        ...totals,
+        details: baseDetails,
+    };
+}