stageflow 0.40.0 → 0.44.0

package/dist/evals/snapshot.d.ts

@@ -0,0 +1,191 @@
+export type SnapshotOverrides = {
+    repeats: number | null;
+    judge: string | null;
+    model: string | null;
+    max_cost_usd: number | null;
+    case_ids: string[] | null;
+    tags: string[] | null;
+    only_failed_from: string | null;
+    user_model?: string | null;
+    tools?: string | null;
+    approvals?: string | null;
+};
+export type SnapshotGrader = {
+    id: string;
+    type: string;
+    hash: string;
+    version_ordinal: number;
+    weight: number;
+    must_pass: boolean;
+};
+export type StageSnapshotTarget = {
+    kind: "stage";
+    pipeline_path: string | null;
+    stage_id: string | null;
+    file: string | null;
+    stage_version_hash: string;
+    stage_definition: Record<string, unknown>;
+    model: string;
+    model_tier: string;
+    input_schema_hash: string | null;
+    output_schema_hash: string | null;
+};
+export type AgentLimitSet = {
+    max_turns: number | null;
+    session_minutes: number | null;
+    cost_usd: number | null;
+    always_on: boolean;
+};
+export type AgentSnapshotTarget = {
+    kind: "agent";
+    agent_id: string;
+    revision: number;
+    definition_hash: string;
+    definition: Record<string, unknown>;
+    model: string;
+    model_tier: string;
+    tools: Array<{
+        id: string;
+        risk: string;
+        class: string;
+    }>;
+    ask_first: string[];
+    sub_agents: Array<{
+        id: string;
+        revision: number;
+    }>;
+    limits_override: {
+        agent: AgentLimitSet;
+        effective: AgentLimitSet;
+        capture: string;
+        otlp: boolean;
+        groundedness: boolean;
+    };
+};
+export type ConversationSettings = {
+    mode: string;
+    max_turns: number;
+    turn_timeout_ms: number;
+    tools: string;
+    approvals: string;
+    read_only_tools: string[];
+    real_tools_confirmed: boolean;
+    user_model: string;
+    user_model_from_judge: boolean;
+    simulator_prompt_version: number;
+};
+export type InternalAgentSnapshotTarget = {
+    kind: "internal_agent";
+    profile_id: string;
+    title: string;
+    playbook_hash: string;
+    tools: string[];
+    model: string;
+};
+export type ConfigSnapshot = {
+    snapshot_version: number;
+    eval: {
+        id: string;
+        path: string;
+        hash: string;
+        definition: Record<string, unknown>;
+    };
+    overrides: SnapshotOverrides;
+    target: StageSnapshotTarget | AgentSnapshotTarget | InternalAgentSnapshotTarget;
+    dataset: {
+        path: string;
+        hash: string;
+        format: string;
+        case_count: number;
+        cases_run: number;
+        cases_skipped: number;
+        skipped_case_ids: string[];
+    };
+    graders: SnapshotGrader[];
+    settings: {
+        repeats: number;
+        pass_threshold: number;
+        judge: string;
+        judge_same_as_stage: boolean;
+        max_cost_usd: number;
+        concurrency: number;
+        suite_type: string;
+        conversation?: ConversationSettings;
+    };
+    engine: {
+        name: string;
+        version: string;
+        rubric_prompt_version: number;
+        node_version: string;
+    };
+    stageflow_version: string;
+};
+/** Every fingerprint input, as the exact value fingerprinted, so two runs can be diffed by key. */
+export type ConfigComponents = {
+    stage: string;
+    model: string;
+    dataset: string;
+    case_ids: string[] | null;
+    graders: Record<string, string>;
+    judge: string;
+    repeats: number;
+    pass_threshold: number;
+    engine: {
+        name: string;
+        version: string;
+    };
+    rubric_prompt_version: number;
+    agent?: AgentComponent;
+};
+/** Agent-only fingerprint inputs; absent for stage runs so stage payloads hash identically. */
+export type AgentComponent = {
+    revision: number;
+    mode: string;
+    max_turns: number;
+    turn_timeout_ms: number;
+    tools: string;
+    approvals: string;
+    read_only_tools: string[];
+    user_model: string | null;
+    simulator_prompt_version: number;
+};
+export type ComponentsInput = {
+    stageVersionHash: string;
+    model: string;
+    datasetHash: string;
+    caseIds: readonly string[] | null;
+    graders: ReadonlyArray<{
+        id: string;
+        hash: string;
+    }>;
+    judge: string;
+    repeats: number;
+    passThreshold: number;
+    engine: {
+        name: string;
+        version: string;
+    };
+    rubricPromptVersion?: number;
+    agent?: AgentComponent;
+};
+export declare function buildComponents(input: ComponentsInput): ConfigComponents;
+/** First 8 hex of sha256 over the canonical fingerprint payload. */
+export declare function computeFingerprint(components: ConfigComponents): string;
+export type SnapshotInput = {
+    evalId: string;
+    evalPath: string;
+    evalHash: string;
+    evalDefinition: Record<string, unknown>;
+    overrides: SnapshotOverrides;
+    target: Omit<StageSnapshotTarget, "kind"> | Omit<AgentSnapshotTarget, "kind"> | Omit<InternalAgentSnapshotTarget, "kind">;
+    dataset: ConfigSnapshot["dataset"];
+    graders: SnapshotGrader[];
+    settings: ConfigSnapshot["settings"];
+    engine: {
+        name: string;
+        version: string;
+        nodeVersion: string;
+    };
+    stageflowVersion: string;
+};
+export declare function buildSnapshot(input: SnapshotInput): ConfigSnapshot;

package/dist/evals/snapshot.js

@@ -0,0 +1,70 @@
+import { canonicalJson, sha256Hex } from "./canonical.js";
+import { CONFIG_SNAPSHOT_VERSION, CONVERSATION_RUBRIC_PROMPT_VERSION, RUBRIC_PROMPT_VERSION } from "./constants.js";
+export function buildComponents(input) {
+    const graders = {};
+    for (const grader of [...input.graders].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))) {
+        graders[grader.id] = grader.hash;
+    }
+    const agent = input.agent;
+    return {
+        stage: input.stageVersionHash,
+        model: input.model,
+        dataset: input.datasetHash,
+        case_ids: input.caseIds === null ? null : [...input.caseIds].sort(),
+        graders,
+        judge: input.judge,
+        repeats: input.repeats,
+        pass_threshold: input.passThreshold,
+        engine: { name: input.engine.name, version: input.engine.version },
+        rubric_prompt_version: input.rubricPromptVersion ?? (agent ? CONVERSATION_RUBRIC_PROMPT_VERSION : RUBRIC_PROMPT_VERSION),
+        ...(agent ? { agent: { ...agent, read_only_tools: [...agent.read_only_tools].sort() } } : {}),
+    };
+}
+/** First 8 hex of sha256 over the canonical fingerprint payload. */
+export function computeFingerprint(components) {
+    const payload = {
+        stage_version_hash: components.stage,
+        model: components.model,
+        dataset_hash: components.dataset,
+        case_ids: components.case_ids,
+        graders: Object.entries(components.graders)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([id, hash]) => ({ id, hash })),
+        repeats: components.repeats,
+        pass_threshold: components.pass_threshold,
+        judge: components.judge,
+        engine: components.engine,
+        rubric_prompt_version: components.rubric_prompt_version,
+        ...(components.agent ? { agent: components.agent } : {}),
+    };
+    return sha256Hex(canonicalJson(payload)).slice(0, 8);
+}
+export function buildSnapshot(input) {
+    const internalTarget = "profile_id" in input.target ? input.target : null;
+    const agentTarget = !internalTarget && "agent_id" in input.target ? input.target : null;
+    return {
+        snapshot_version: CONFIG_SNAPSHOT_VERSION,
+        eval: {
+            id: input.evalId,
+            path: input.evalPath,
+            hash: input.evalHash,
+            definition: input.evalDefinition,
+        },
+        overrides: input.overrides,
+        target: internalTarget
+            ? { kind: "internal_agent", ...internalTarget }
+            : agentTarget
+                ? { kind: "agent", ...agentTarget }
+                : { kind: "stage", ...input.target },
+        dataset: input.dataset,
+        graders: input.graders,
+        settings: input.settings,
+        engine: {
+            name: input.engine.name,
+            version: input.engine.version,
+            rubric_prompt_version: agentTarget || internalTarget ? CONVERSATION_RUBRIC_PROMPT_VERSION : RUBRIC_PROMPT_VERSION,
+            node_version: input.engine.nodeVersion,
+        },
+        stageflow_version: input.stageflowVersion,
+    };
+}

package/dist/evals/stageDiff.d.ts

@@ -0,0 +1,8 @@
+import { type DiffLine } from "./lineDiff.js";
+export interface StageDiff {
+    changed_keys: string[];
+    system_prompt_diff: DiffLine[] | null;
+}
+type Json = unknown;
+export declare function stageDiff(defA: Record<string, Json>, defB: Record<string, Json>): StageDiff;
+export {};

package/dist/evals/stageDiff.js

@@ -0,0 +1,48 @@
+import { lineDiff } from "./lineDiff.js";
+function canonical(value) {
+    if (value === undefined)
+        return "undefined";
+    return JSON.stringify(sortKeys(value));
+}
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (value && typeof value === "object") {
+        const obj = value;
+        return Object.fromEntries(Object.keys(obj)
+            .sort()
+            .map((k) => [k, sortKeys(obj[k])]));
+    }
+    return value;
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : {};
+}
+export function stageDiff(defA, defB) {
+    const keys = [...new Set([...Object.keys(defA), ...Object.keys(defB)])].sort();
+    const changed = [];
+    for (const key of keys) {
+        if (canonical(defA[key]) === canonical(defB[key]))
+            continue;
+        if (key === "io") {
+            const ioA = asRecord(defA.io);
+            const ioB = asRecord(defB.io);
+            const parts = ["input", "output"].filter((p) => canonical(ioA[p]) !== canonical(ioB[p]));
+            if (parts.length === 0)
+                changed.push("io");
+            else
+                for (const p of parts)
+                    changed.push(`io.${p}`);
+            continue;
+        }
+        changed.push(key);
+    }
+    const promptChanged = changed.includes("system_prompt");
+    const text = (v) => (typeof v === "string" ? v : v === undefined ? "" : canonical(v));
+    return {
+        changed_keys: changed,
+        system_prompt_diff: promptChanged ? lineDiff(text(defA.system_prompt), text(defB.system_prompt)) : null,
+    };
+}

package/dist/evals/stats.d.ts

@@ -0,0 +1,95 @@
+import { type GradeRecord, type GraderRef, type PreviousRunLite, type ResultRecord } from "./statsTypes.js";
+export interface ComputeStatsInput {
+    results: ResultRecord[];
+    grades: GradeRecord[];
+    graders: GraderRef[];
+    repeats: number;
+    passThreshold: number;
+    reviewsTags?: Map<string, string[]>;
+    previousRun?: PreviousRunLite | null;
+    targetKind?: "stage" | "agent";
+}
+export interface CaseStat {
+    caseId: string;
+    executed: number;
+    passed: number;
+    p: number;
+    flags: boolean[];
+    meanScore: number | null;
+}
+export interface Interval {
+    low: number;
+    high: number;
+}
+export declare const LOWEST_CASES_LIMIT = 5;
+export interface LowestCase {
+    case_id: string;
+    mean_score: number;
+    worst_repeat_index: number;
+}
+export interface GraderStat {
+    grader_id: string;
+    type: string;
+    hash: string;
+    graded: number;
+    passed: number;
+    pass_rate: number | null;
+    mean_raw: number | null;
+    min_raw: number | null;
+    max_raw: number | null;
+    errors: number;
+    delta: number | null;
+    grader_changed: boolean;
+}
+export interface FailingCase {
+    case_id: string;
+    pass_rate: number;
+    mean_score: number | null;
+    passes: boolean[];
+}
+export interface Stats {
+    cases_counted: number;
+    results_executed: number;
+    results_passed: number;
+    errored: number;
+    pass_rate: number | null;
+    pass_rate_ci: Interval | null;
+    mean_score: number | null;
+    score_spread: number | null;
+    reliability: number | null;
+    pass_any: number | null;
+    flaky_cases: string[];
+    latency_p50: number | null;
+    latency_p95: number | null;
+    latency_max: number | null;
+    cost_stage: number;
+    cost_judge: number;
+    cost_total: number;
+    cost_per_result: number | null;
+    tokens_in: number;
+    tokens_out: number;
+    judge_tokens_in: number;
+    judge_tokens_out: number;
+    by_grader: GraderStat[];
+    score_histogram: {
+        bins: number[];
+        threshold: number;
+    };
+    failure_types: {
+        by_tag: Record<string, number>;
+        untagged: number;
+    };
+    failing_cases: FailingCase[];
+    mean_turns?: number | null;
+    mean_tool_calls?: number | null;
+}
+export declare function isExecuted(result: ResultRecord): boolean;
+export declare function mean(values: number[]): number | null;
+export declare function sampleSd(values: number[]): number | null;
+export declare function passFraction(passed: number, executed: number): number | null;
+export declare function lowestCases(grades: GradeRecord[], limit?: number): LowestCase[];
+export declare function percentile(sorted: number[], q: number): number | null;
+export declare function clusteredCi(center: number, perCase: number[], lo: number, hi: number): Interval | null;
+export declare function buildCaseStats(results: ResultRecord[]): CaseStat[];
+export declare function computeByGrader(graders: GraderRef[], grades: GradeRecord[], previousRun: PreviousRunLite | null | undefined): GraderStat[];
+export declare function computeStats(input: ComputeStatsInput): Stats;

package/dist/evals/stats.js

@@ -0,0 +1,220 @@
+import { tQuantile } from "./tTable.js";
+import { EXECUTED_RESULT_STATUSES, resultKey, } from "./statsTypes.js";
+export const LOWEST_CASES_LIMIT = 5;
+const EPS = 1e-9;
+export function isExecuted(result) {
+    return EXECUTED_RESULT_STATUSES.includes(result.status);
+}
+export function mean(values) {
+    if (values.length === 0)
+        return null;
+    return values.reduce((a, b) => a + b, 0) / values.length;
+}
+export function sampleSd(values) {
+    if (values.length < 2)
+        return null;
+    const m = values.reduce((a, b) => a + b, 0) / values.length;
+    const ss = values.reduce((a, b) => a + (b - m) ** 2, 0);
+    return Math.sqrt(ss / (values.length - 1));
+}
+export function passFraction(passed, executed) {
+    return executed > 0 ? passed / executed : null;
+}
+export function lowestCases(grades, limit = LOWEST_CASES_LIMIT) {
+    const byCase = new Map();
+    for (const g of grades) {
+        if (g.status !== "graded" || g.score === null)
+            continue;
+        const list = byCase.get(g.caseId);
+        if (list)
+            list.push({ repeat: g.repeatIndex, score: g.score });
+        else
+            byCase.set(g.caseId, [{ repeat: g.repeatIndex, score: g.score }]);
+    }
+    const rows = [...byCase.entries()].map(([caseId, list]) => {
+        const worst = [...list].sort((a, b) => a.score - b.score || a.repeat - b.repeat)[0];
+        return {
+            case_id: caseId,
+            mean_score: mean(list.map((item) => item.score)),
+            worst_repeat_index: worst.repeat,
+        };
+    });
+    rows.sort((a, b) => a.mean_score - b.mean_score || (a.case_id < b.case_id ? -1 : a.case_id > b.case_id ? 1 : 0));
+    return rows.slice(0, limit);
+}
+export function percentile(sorted, q) {
+    if (sorted.length === 0)
+        return null;
+    const rank = Math.ceil(Math.round(q * sorted.length * 1e9) / 1e9);
+    return sorted[Math.min(sorted.length, Math.max(1, rank)) - 1];
+}
+export function clusteredCi(center, perCase, lo, hi) {
+    const n = perCase.length;
+    if (n < 2)
+        return null;
+    const pBar = perCase.reduce((a, b) => a + b, 0) / n;
+    const ss = perCase.reduce((a, b) => a + (b - pBar) ** 2, 0);
+    const se = Math.sqrt(ss / (n * (n - 1)));
+    const half = tQuantile(n - 1) * se;
+    return { low: Math.max(lo, center - half), high: Math.min(hi, center + half) };
+}
+export function buildCaseStats(results) {
+    const byCase = new Map();
+    for (const r of results) {
+        if (!isExecuted(r))
+            continue;
+        const list = byCase.get(r.caseId);
+        if (list)
+            list.push(r);
+        else
+            byCase.set(r.caseId, [r]);
+    }
+    const out = [];
+    for (const [caseId, list] of byCase) {
+        list.sort((a, b) => a.repeatIndex - b.repeatIndex);
+        const passed = list.filter((r) => r.passed === 1).length;
+        const scores = list
+            .filter((r) => r.status === "completed" && r.score !== null)
+            .map((r) => r.score);
+        out.push({
+            caseId,
+            executed: list.length,
+            passed,
+            p: passFraction(passed, list.length),
+            flags: list.map((r) => r.passed === 1),
+            meanScore: mean(scores),
+        });
+    }
+    return out;
+}
+function sum(values) {
+    let total = 0;
+    for (const v of values)
+        total += v ?? 0;
+    return total;
+}
+export function computeByGrader(graders, grades, previousRun) {
+    return graders.map((g) => {
+        const own = grades.filter((x) => x.graderId === g.id);
+        const graded = own.filter((x) => x.status === "graded");
+        const passed = graded.filter((x) => x.passed === 1).length;
+        const raws = graded.map((x) => x.rawValue).filter((v) => v !== null);
+        const passRate = graded.length > 0 ? passed / graded.length : null;
+        const prev = previousRun?.graders.find((p) => p.graderId === g.id);
+        const sameHash = prev !== undefined && prev.hash === g.hash;
+        const delta = sameHash && passRate !== null && prev.passRate !== null
+            ? (passRate - prev.passRate) * 100
+            : null;
+        return {
+            grader_id: g.id,
+            type: g.type,
+            hash: g.hash,
+            graded: graded.length,
+            passed,
+            pass_rate: passRate,
+            mean_raw: mean(raws),
+            min_raw: raws.length > 0 ? Math.min(...raws) : null,
+            max_raw: raws.length > 0 ? Math.max(...raws) : null,
+            errors: own.length - graded.length,
+            delta,
+            grader_changed: prev !== undefined && !sameHash,
+        };
+    });
+}
+export function computeStats(input) {
+    const executed = input.results.filter(isExecuted);
+    const executedKeys = new Set(executed.map((r) => resultKey(r.caseId, r.repeatIndex)));
+    const grades = input.grades.filter((g) => executedKeys.has(resultKey(g.caseId, g.repeatIndex)));
+    const cases = buildCaseStats(executed);
+    const completed = executed.filter((r) => r.status === "completed");
+    const resultsExecuted = executed.length;
+    const resultsPassed = executed.filter((r) => r.passed === 1).length;
+    const passRate = resultsExecuted > 0 ? resultsPassed / resultsExecuted : null;
+    const passRateCi = passRate === null
+        ? null
+        : clusteredCi(passRate, cases.map((c) => c.p), 0, 1);
+    const scores = completed.filter((r) => r.score !== null).map((r) => r.score);
+    const spreads = [];
+    for (const c of cases) {
+        const caseScores = completed
+            .filter((r) => r.caseId === c.caseId && r.score !== null)
+            .map((r) => r.score);
+        const sd = sampleSd(caseScores);
+        if (sd !== null)
+            spreads.push(sd);
+    }
+    const repeatedCases = cases.filter((c) => c.executed >= 2);
+    const reliability = input.repeats > 1 && repeatedCases.length > 0
+        ? repeatedCases.filter((c) => c.passed === c.executed).length / repeatedCases.length
+        : null;
+    const passAny = input.repeats > 1 && repeatedCases.length > 0
+        ? repeatedCases.filter((c) => c.passed >= 1).length / repeatedCases.length
+        : null;
+    const flaky = repeatedCases.filter((c) => c.passed > 0 && c.passed < c.executed).map((c) => c.caseId);
+    const latencies = completed
+        .map((r) => r.latencyMs)
+        .filter((v) => v !== null)
+        .sort((a, b) => a - b);
+    const costStage = sum(executed.map((r) => r.costUsd));
+    const costJudge = sum(grades.map((g) => g.judgeCostUsd));
+    const costTotal = costStage + costJudge;
+    const bins = new Array(10).fill(0);
+    for (const s of scores) {
+        const idx = Math.min(9, Math.max(0, Math.floor(s * 10 + EPS)));
+        bins[idx] += 1;
+    }
+    const byTag = {};
+    let untagged = 0;
+    for (const r of executed) {
+        if (r.passed === 1)
+            continue;
+        const tags = [...new Set(input.reviewsTags?.get(resultKey(r.caseId, r.repeatIndex)) ?? [])];
+        if (tags.length === 0)
+            untagged += 1;
+        for (const tag of tags)
+            byTag[tag] = (byTag[tag] ?? 0) + 1;
+    }
+    const failing = cases
+        .filter((c) => c.passed < c.executed)
+        .sort((a, b) => a.p - b.p ||
+        (a.meanScore ?? 0) - (b.meanScore ?? 0) ||
+        (a.caseId < b.caseId ? -1 : a.caseId > b.caseId ? 1 : 0))
+        .map((c) => ({
+        case_id: c.caseId,
+        pass_rate: c.p,
+        mean_score: c.meanScore,
+        passes: c.flags,
+    }));
+    const turnValues = executed.flatMap((r) => (r.turns != null ? [r.turns] : []));
+    const toolCallValues = executed.flatMap((r) => (r.toolCalls != null ? [r.toolCalls] : []));
+    const hasAgentData = input.targetKind === "agent" || input.results.some((r) => r.turns != null || r.toolCalls != null);
+    return {
+        cases_counted: cases.length,
+        results_executed: resultsExecuted,
+        results_passed: resultsPassed,
+        errored: executed.filter((r) => r.status === "error").length,
+        pass_rate: passRate,
+        pass_rate_ci: passRateCi,
+        mean_score: mean(scores),
+        score_spread: mean(spreads),
+        reliability,
+        pass_any: passAny,
+        flaky_cases: flaky,
+        latency_p50: percentile(latencies, 0.5),
+        latency_p95: percentile(latencies, 0.95),
+        latency_max: latencies.length > 0 ? latencies[latencies.length - 1] : null,
+        cost_stage: costStage,
+        cost_judge: costJudge,
+        cost_total: costTotal,
+        cost_per_result: resultsExecuted > 0 ? costTotal / resultsExecuted : null,
+        tokens_in: sum(executed.map((r) => r.tokensIn)),
+        tokens_out: sum(executed.map((r) => r.tokensOut)),
+        judge_tokens_in: sum(grades.map((g) => g.judgeTokensIn)),
+        judge_tokens_out: sum(grades.map((g) => g.judgeTokensOut)),
+        by_grader: computeByGrader(input.graders, grades, input.previousRun),
+        score_histogram: { bins, threshold: input.passThreshold },
+        failure_types: { by_tag: byTag, untagged },
+        failing_cases: failing,
+        ...(hasAgentData ? { mean_turns: mean(turnValues), mean_tool_calls: mean(toolCallValues) } : {}),
+    };
+}

package/dist/evals/statsAdapter.d.ts

@@ -0,0 +1,4 @@
+import type { EvalResultRow, EvalResultWithGrades } from "./storeTypes.js";
+import type { GradeRecord, ResultRecord } from "./statsTypes.js";
+export declare function toResultRecord(row: EvalResultRow): ResultRecord;
+export declare function toGradeRecords(row: EvalResultWithGrades): GradeRecord[];

package/dist/evals/statsAdapter.js

@@ -0,0 +1,30 @@
+export function toResultRecord(row) {
+    return {
+        caseId: row.caseId,
+        repeatIndex: row.repeatIndex,
+        status: row.status,
+        latencyMs: row.latencyMs,
+        costUsd: row.costUsd,
+        tokensIn: row.tokensIn,
+        tokensOut: row.tokensOut,
+        score: row.score,
+        passed: row.passed ? 1 : 0,
+        ...(row.turns !== null && row.turns !== undefined ? { turns: row.turns } : {}),
+        ...(row.toolCallCount !== null && row.toolCallCount !== undefined ? { toolCalls: row.toolCallCount } : {}),
+    };
+}
+export function toGradeRecords(row) {
+    return row.grades.map((g) => ({
+        caseId: g.caseId,
+        repeatIndex: g.repeatIndex,
+        graderId: g.graderId,
+        graderType: g.graderType,
+        status: g.status,
+        passed: g.passed === null ? null : g.passed ? 1 : 0,
+        score: g.score,
+        rawValue: g.rawValue,
+        judgeCostUsd: g.judgeCostUsd,
+        judgeTokensIn: g.judgeTokensIn,
+        judgeTokensOut: g.judgeTokensOut,
+    }));
+}