stageflow 0.40.0 → 0.42.0

package/dist/evals/statsTypes.d.ts

@@ -0,0 +1,51 @@
+export declare const RESULT_STATUSES: readonly ["completed", "stage_failed", "needs_input", "error", "skipped_budget", "cancelled"];
+export type ResultStatus = (typeof RESULT_STATUSES)[number];
+export declare const EXECUTED_RESULT_STATUSES: readonly ResultStatus[];
+export type GradeStatus = "graded" | "error";
+export type EvalType = "capability" | "regression";
+export declare const EVAL_COMPARE_MISMATCH = "EVAL_COMPARE_MISMATCH";
+export interface ResultRecord {
+    caseId: string;
+    repeatIndex: number;
+    status: ResultStatus;
+    latencyMs: number | null;
+    costUsd: number | null;
+    tokensIn: number | null;
+    tokensOut: number | null;
+    score: number | null;
+    passed: 0 | 1;
+    turns?: number | null;
+    toolCalls?: number | null;
+}
+export interface GradeRecord {
+    caseId: string;
+    repeatIndex: number;
+    graderId: string;
+    graderType: string;
+    status: GradeStatus;
+    passed: 0 | 1 | null;
+    score: number | null;
+    rawValue: number | null;
+    judgeCostUsd: number | null;
+    judgeTokensIn: number | null;
+    judgeTokensOut: number | null;
+}
+export interface GraderRef {
+    id: string;
+    type: string;
+    hash: string;
+}
+export interface PreviousRunGraderStat {
+    graderId: string;
+    hash: string;
+    passRate: number | null;
+}
+export interface PreviousRunLite {
+    graders: PreviousRunGraderStat[];
+}
+export interface RunBudget {
+    latencyMs?: number | null;
+    costUsd?: number | null;
+}
+export type BudgetWarning = "latency" | "cost";
+export declare function resultKey(caseId: string, repeatIndex: number): string;

package/dist/evals/statsTypes.js

@@ -0,0 +1,18 @@
+export const RESULT_STATUSES = [
+    "completed",
+    "stage_failed",
+    "needs_input",
+    "error",
+    "skipped_budget",
+    "cancelled",
+];
+export const EXECUTED_RESULT_STATUSES = [
+    "completed",
+    "stage_failed",
+    "needs_input",
+    "error",
+];
+export const EVAL_COMPARE_MISMATCH = "EVAL_COMPARE_MISMATCH";
+export function resultKey(caseId, repeatIndex) {
+    return `${caseId}:${repeatIndex}`;
+}

package/dist/evals/storeError.d.ts

@@ -0,0 +1,5 @@
+import type { EvalErrorCode } from "./constants.js";
+export declare class EvalStoreError extends Error {
+    readonly code: EvalErrorCode;
+    constructor(code: EvalErrorCode, message: string);
+}

package/dist/evals/storeError.js

@@ -0,0 +1,8 @@
+export class EvalStoreError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "EvalStoreError";
+    }
+}

package/dist/evals/storeTypes.d.ts

@@ -0,0 +1,207 @@
+import type { EvalBudgetWarning, EvalGradeStatus, EvalResultStatus, EvalReviewLabel, EvalRunStatus, EvalStartedVia, EvalTargetKind } from "./constants.js";
+export type EvalTargetRef = {
+    pipeline_path?: string;
+    stage_id?: string;
+    file?: string;
+    agent_id?: string;
+    revision?: number | string;
+};
+export type EvalJson = Record<string, unknown>;
+export type EvalRunRow = {
+    evalRunId: string;
+    projectRoot: string;
+    evalId: string;
+    runNumber: number;
+    evalPath: string;
+    evalHash: string;
+    targetKind: EvalTargetKind;
+    targetRef: EvalTargetRef;
+    targetVersionHash: string;
+    datasetPath: string;
+    datasetHash: string;
+    config: EvalJson;
+    configFingerprint: string;
+    configComponents: EvalJson;
+    status: EvalRunStatus;
+    statusReason: string | null;
+    passed: boolean | null;
+    repeats: number;
+    passThreshold: number;
+    maxCostUsd: number;
+    casesTotal: number;
+    resultsTotal: number;
+    resultsDone: number;
+    costUsd: number;
+    engineName: string;
+    engineVersion: string;
+    stageflowVersion: string;
+    label: string | null;
+    startedVia: EvalStartedVia;
+    callerId: string | null;
+    summary: EvalJson | null;
+    createdAt: string;
+    startedAt: string | null;
+    finishedAt: string | null;
+};
+export type CreateEvalRunInput = {
+    projectRoot: string;
+    evalId: string;
+    evalPath: string;
+    evalHash: string;
+    targetKind: EvalTargetKind;
+    targetRef: EvalTargetRef;
+    targetVersionHash: string;
+    datasetPath: string;
+    datasetHash: string;
+    config: EvalJson;
+    configFingerprint: string;
+    configComponents: EvalJson;
+    status?: EvalRunStatus;
+    repeats: number;
+    passThreshold: number;
+    maxCostUsd: number;
+    casesTotal: number;
+    resultsTotal: number;
+    engineName: string;
+    engineVersion: string;
+    stageflowVersion: string;
+    label?: string;
+    startedVia: EvalStartedVia;
+    callerId?: string;
+};
+export type EvalRunProgress = {
+    resultsDone?: number;
+    costUsd?: number;
+    status?: EvalRunStatus;
+    statusReason?: string | null;
+    startedAt?: string;
+};
+export type FinishEvalRunInput = {
+    status: Extract<EvalRunStatus, "completed" | "failed" | "cancelled" | "budget_stopped">;
+    statusReason?: string | null;
+    passed: boolean;
+    summary: EvalJson;
+    finishedAt: string;
+};
+export type ListEvalRunsQuery = {
+    projectRoot?: string;
+    evalId?: string;
+    status?: EvalRunStatus;
+    stageRunId?: string;
+    subjectRef?: string;
+    limit: number;
+    cursor?: string;
+};
+export type EvalRunPage = {
+    items: EvalRunRow[];
+    nextCursor: string | null;
+};
+export type EvalCaseRow = {
+    evalRunId: string;
+    caseId: string;
+    position: number;
+    case: EvalJson;
+};
+export type EvalCaseInput = Omit<EvalCaseRow, "evalRunId">;
+export type EvalGradeRow = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+    graderId: string;
+    graderType: string;
+    status: EvalGradeStatus;
+    passed: boolean | null;
+    score: number | null;
+    rawValue: number | null;
+    reason: string | null;
+    details: EvalJson | null;
+    errorCode: string | null;
+    judgeModel: string | null;
+    judgeCostUsd: number | null;
+    judgeTokensIn: number | null;
+    judgeTokensOut: number | null;
+};
+export type EvalGradeInput = {
+    graderId: string;
+    graderType: string;
+    status: EvalGradeStatus;
+    passed?: boolean | null;
+    score?: number | null;
+    rawValue?: number | null;
+    reason?: string | null;
+    details?: EvalJson | null;
+    errorCode?: string | null;
+    judgeModel?: string | null;
+    judgeCostUsd?: number | null;
+    judgeTokensIn?: number | null;
+    judgeTokensOut?: number | null;
+};
+export type EvalResultRow = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+    status: EvalResultStatus;
+    errorCode: string | null;
+    errorMessage: string | null;
+    stageRunId: string | null;
+    subjectRef: string | null;
+    turns: number | null;
+    toolCallCount: number | null;
+    output: EvalJson | null;
+    latencyMs: number | null;
+    costUsd: number | null;
+    tokensIn: number | null;
+    tokensOut: number | null;
+    score: number | null;
+    passed: boolean;
+    hasGraderErrors: boolean;
+    budgetWarnings: EvalBudgetWarning[] | null;
+    startedAt: string | null;
+    finishedAt: string | null;
+};
+export type EvalResultWithGrades = EvalResultRow & {
+    grades: EvalGradeRow[];
+};
+export type EvalResultInput = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+    status: EvalResultStatus;
+    errorCode?: string | null;
+    errorMessage?: string | null;
+    stageRunId?: string | null;
+    subjectRef?: string | null;
+    turns?: number | null;
+    toolCallCount?: number | null;
+    output?: EvalJson | null;
+    latencyMs?: number | null;
+    costUsd?: number | null;
+    tokensIn?: number | null;
+    tokensOut?: number | null;
+    score?: number | null;
+    passed?: boolean;
+    hasGraderErrors?: boolean;
+    budgetWarnings?: EvalBudgetWarning[] | null;
+    startedAt?: string | null;
+    finishedAt?: string | null;
+    grades?: EvalGradeInput[];
+};
+export type EvalReviewPatch = {
+    label?: EvalReviewLabel;
+    note?: string | null;
+    tags?: string[];
+};
+export type EvalReviewRow = {
+    evalRunId: string;
+    caseId: string;
+    repeatIndex: number;
+    label: EvalReviewLabel | null;
+    note: string | null;
+    tags: string[];
+    createdAt: string | null;
+    updatedAt: string | null;
+};
+export type EvalGraderVersionInput = {
+    graderId: string;
+    graderHash: string;
+};

package/dist/evals/storeTypes.js

@@ -0,0 +1 @@
+export {};

package/dist/evals/tTable.d.ts

@@ -0,0 +1,3 @@
+export declare const T_TABLE_MAX_DF: 30;
+export declare const T_LARGE_DF_QUANTILE = 1.96;
+export declare function tQuantile(df: number): number;

package/dist/evals/tTable.js

@@ -0,0 +1,15 @@
+const T_95_TWO_SIDED = [
+    12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228,
+    2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086,
+    2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042,
+];
+export const T_TABLE_MAX_DF = T_95_TWO_SIDED.length;
+export const T_LARGE_DF_QUANTILE = 1.96;
+export function tQuantile(df) {
+    if (!Number.isInteger(df) || df < 1) {
+        throw new RangeError(`degrees of freedom must be an integer >= 1, got ${df}`);
+    }
+    if (df > T_TABLE_MAX_DF)
+        return T_LARGE_DF_QUANTILE;
+    return T_95_TWO_SIDED[df - 1];
+}

package/dist/evals/triage/constants.d.ts

@@ -0,0 +1,26 @@
+export declare const TRIAGE_BUCKETS: readonly ["agent_wrong", "grader_wrong", "case_wrong", "unclear"];
+export type TriageBucket = (typeof TRIAGE_BUCKETS)[number];
+export declare const TRIAGE_SOURCES: readonly ["jev", "judge"];
+export type TriageSource = (typeof TRIAGE_SOURCES)[number];
+export declare const SUGGESTION_STATUSES: readonly ["pending", "accepted", "dismissed"];
+export type SuggestionStatus = (typeof SUGGESTION_STATUSES)[number];
+export declare const FAILURE_TYPES_BY_KIND: {
+    readonly agent: readonly ["missing_tool_call", "unneeded_tool_call", "wrong_or_missing_fact", "policy_violation", "tone", "ungrounded_claim", "off_topic", "no_handoff", "other"];
+    readonly stage: readonly ["wrong_value", "missing_field", "schema_invalid", "policy_violation", "ungrounded_claim", "tone", "other"];
+};
+/** Pinned System One model id; the alias `jev-latest` moves and is never used. Recorded on every suggestion. */
+export declare const JEV_MODEL_ID = "jev-1.13.0";
+export declare const JEV_BASE_URL = "https://api.typesafe.ai";
+export declare const JEV_PRICE_INPUT_PER_MILLION_USD = 0.042;
+/** Below this bucket confidence (or for `unclear`) a suggestion is low confidence and excluded from bulk accept. */
+export declare const LOW_CONFIDENCE_THRESHOLD = 0.7;
+export declare const TRIAGE_STATE_MAX_CHARS = 24000;
+export declare const TRIAGE_CONCURRENCY = 4;
+export declare const TRIAGE_DEFAULT_LIMIT = 200;
+export declare const TRIAGE_MAX_LIMIT = 1000;
+export declare const JEV_MAX_RETRIES = 4;
+export declare const JEV_BACKOFF_BASE_MS = 500;
+export declare const BUCKET_TAG_PREFIX = "bucket:";
+export declare const TYPESAFE_PROVIDER_ID = "typesafe";
+/** Env names tried after the stored `typesafe` provider credential, in order; the first one set wins. */
+export declare const TYPESAFE_KEY_ENV_NAMES: readonly ["STAGEFLOW_PROVIDER_TYPESAFE_API_KEY", "TYPESAFE_API_KEY", "JEV_API_KEY"];

package/dist/evals/triage/constants.js

@@ -0,0 +1,33 @@
+export const TRIAGE_BUCKETS = ["agent_wrong", "grader_wrong", "case_wrong", "unclear"];
+export const TRIAGE_SOURCES = ["jev", "judge"];
+export const SUGGESTION_STATUSES = ["pending", "accepted", "dismissed"];
+export const FAILURE_TYPES_BY_KIND = {
+    agent: [
+        "missing_tool_call",
+        "unneeded_tool_call",
+        "wrong_or_missing_fact",
+        "policy_violation",
+        "tone",
+        "ungrounded_claim",
+        "off_topic",
+        "no_handoff",
+        "other",
+    ],
+    stage: ["wrong_value", "missing_field", "schema_invalid", "policy_violation", "ungrounded_claim", "tone", "other"],
+};
+/** Pinned System One model id; the alias `jev-latest` moves and is never used. Recorded on every suggestion. */
+export const JEV_MODEL_ID = "jev-1.13.0";
+export const JEV_BASE_URL = "https://api.typesafe.ai";
+export const JEV_PRICE_INPUT_PER_MILLION_USD = 0.042;
+/** Below this bucket confidence (or for `unclear`) a suggestion is low confidence and excluded from bulk accept. */
+export const LOW_CONFIDENCE_THRESHOLD = 0.7;
+export const TRIAGE_STATE_MAX_CHARS = 24_000;
+export const TRIAGE_CONCURRENCY = 4;
+export const TRIAGE_DEFAULT_LIMIT = 200;
+export const TRIAGE_MAX_LIMIT = 1_000;
+export const JEV_MAX_RETRIES = 4;
+export const JEV_BACKOFF_BASE_MS = 500;
+export const BUCKET_TAG_PREFIX = "bucket:";
+export const TYPESAFE_PROVIDER_ID = "typesafe";
+/** Env names tried after the stored `typesafe` provider credential, in order; the first one set wins. */
+export const TYPESAFE_KEY_ENV_NAMES = ["STAGEFLOW_PROVIDER_TYPESAFE_API_KEY", "TYPESAFE_API_KEY", "JEV_API_KEY"];

package/dist/evals/triage/credentials.d.ts

@@ -0,0 +1,13 @@
+export declare const STORED_KEY_SOURCE = "stored:typesafe";
+export type TypesafeKeyLookup = {
+    /** The key. Never log it, return it from an API or put it in an error. */
+    key: string | undefined;
+    /** Names (never values) of the sources that hold a key, in lookup order. */
+    sources: string[];
+};
+export type TypesafeKeyResolver = (projectRoot: string) => Promise<TypesafeKeyLookup> | TypesafeKeyLookup;
+/**
+ * Lookup order: the stored `typesafe` provider credential (Settings, Providers, or boot from
+ * STAGEFLOW_PROVIDER_TYPESAFE_API_KEY), then the env names in TYPESAFE_KEY_ENV_NAMES. The first one set wins.
+ */
+export declare function resolveTypesafeKey(projectRoot: string, env?: NodeJS.ProcessEnv): TypesafeKeyLookup;

package/dist/evals/triage/credentials.js

@@ -0,0 +1,20 @@
+import { resolveCredentialBinding } from "../../runtime/credentialBinding.js";
+import { readAuthKey } from "../providerKeys.js";
+import { TYPESAFE_KEY_ENV_NAMES, TYPESAFE_PROVIDER_ID } from "./constants.js";
+export const STORED_KEY_SOURCE = `stored:${TYPESAFE_PROVIDER_ID}`;
+/**
+ * Lookup order: the stored `typesafe` provider credential (Settings, Providers, or boot from
+ * STAGEFLOW_PROVIDER_TYPESAFE_API_KEY), then the env names in TYPESAFE_KEY_ENV_NAMES. The first one set wins.
+ */
+export function resolveTypesafeKey(projectRoot, env = process.env) {
+    const found = [];
+    const stored = readAuthKey(resolveCredentialBinding(projectRoot).authPath, TYPESAFE_PROVIDER_ID);
+    if (stored !== undefined)
+        found.push({ source: STORED_KEY_SOURCE, key: stored });
+    for (const name of TYPESAFE_KEY_ENV_NAMES) {
+        const value = env[name]?.trim();
+        if (value !== undefined && value.length > 0)
+            found.push({ source: name, key: value });
+    }
+    return { key: found[0]?.key, sources: found.map((entry) => entry.source) };
+}

package/dist/evals/triage/jevClassifier.d.ts

@@ -0,0 +1,14 @@
+import { type TriageBucket } from "./constants.js";
+import { type FailureClassifier } from "./types.js";
+export type JevClassifierOptions = {
+    apiKey: string;
+    fetch?: typeof fetch;
+    baseUrl?: string;
+    model?: string;
+    maxRetries?: number;
+    backoffBaseMs?: number;
+    sleep?: (ms: number) => Promise<void>;
+};
+export declare function normalizeBucketProbabilities(raw: unknown, choice: TriageBucket, confidence: number): Record<TriageBucket, number>;
+/** System One (TypeSafe Jev) backend. The key goes in one header and is never logged, returned or put in an error. */
+export declare function createJevClassifier(options: JevClassifierOptions): FailureClassifier;

package/dist/evals/triage/jevClassifier.js

@@ -0,0 +1,124 @@
+import { JEV_BACKOFF_BASE_MS, JEV_BASE_URL, JEV_MAX_RETRIES, JEV_MODEL_ID, JEV_PRICE_INPUT_PER_MILLION_USD, TRIAGE_BUCKETS, } from "./constants.js";
+import { BUCKET_QUESTION, bucketCriteriaMap, FAILURE_TYPE_QUESTION, failureTypeCriteriaMap, failureTypesFor, falseAlarmQuestion, } from "./prompts.js";
+import { ClassifierError } from "./types.js";
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function num(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+function clamp01(value) {
+    return Math.min(1, Math.max(0, value));
+}
+export function normalizeBucketProbabilities(raw, choice, confidence) {
+    const out = { agent_wrong: 0, grader_wrong: 0, case_wrong: 0, unclear: 0 };
+    const rec = typeof raw === "object" && raw !== null ? raw : {};
+    let any = false;
+    for (const bucket of TRIAGE_BUCKETS) {
+        const p = num(rec[bucket]);
+        if (p !== undefined) {
+            out[bucket] = clamp01(p);
+            any = true;
+        }
+    }
+    if (!any) {
+        const rest = (1 - confidence) / (TRIAGE_BUCKETS.length - 1);
+        for (const bucket of TRIAGE_BUCKETS)
+            out[bucket] = bucket === choice ? confidence : rest;
+    }
+    return out;
+}
+/** System One (TypeSafe Jev) backend. The key goes in one header and is never logged, returned or put in an error. */
+export function createJevClassifier(options) {
+    const model = options.model ?? JEV_MODEL_ID;
+    const doFetch = options.fetch ?? fetch;
+    const url = `${(options.baseUrl ?? JEV_BASE_URL).replace(/\/+$/, "")}/v1/systemone`;
+    const maxRetries = options.maxRetries ?? JEV_MAX_RETRIES;
+    const backoff = options.backoffBaseMs ?? JEV_BACKOFF_BASE_MS;
+    const sleep = options.sleep ?? defaultSleep;
+    const { apiKey } = options;
+    async function post(body, signal) {
+        for (let attempt = 0;; attempt++) {
+            let res;
+            try {
+                res = await doFetch(url, {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
+                    body: JSON.stringify(body),
+                    ...(signal !== undefined ? { signal } : {}),
+                });
+            }
+            catch {
+                if (signal?.aborted)
+                    throw new ClassifierError("CLASSIFIER_FAILED", "Classifier request was cancelled");
+                throw new ClassifierError("CLASSIFIER_FAILED", "Could not reach the classifier");
+            }
+            if (res.status === 429) {
+                if (attempt >= maxRetries)
+                    throw new ClassifierError("CLASSIFIER_RATE_LIMITED", "Classifier rate limit; try again shortly");
+                const retryAfter = Number(res.headers.get("retry-after"));
+                const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : backoff * 2 ** attempt;
+                await res.body?.cancel().catch(() => undefined);
+                await sleep(wait);
+                continue;
+            }
+            if (res.status === 422 || res.status === 400) {
+                await res.body?.cancel().catch(() => undefined);
+                throw new ClassifierError("CLASSIFIER_REJECTED", `Classifier rejected the request (HTTP ${res.status})`);
+            }
+            if (!res.ok) {
+                await res.body?.cancel().catch(() => undefined);
+                throw new ClassifierError("CLASSIFIER_FAILED", `Classifier request failed (HTTP ${res.status})`);
+            }
+            try {
+                return (await res.json());
+            }
+            catch {
+                throw new ClassifierError("CLASSIFIER_BAD_REPLY", "Classifier reply was not JSON");
+            }
+        }
+    }
+    return {
+        source: "jev",
+        model,
+        async classify(input) {
+            const types = failureTypesFor(input.targetKind);
+            const questions = {
+                bucket: { type: "choice", instructions: BUCKET_QUESTION, criteria: bucketCriteriaMap() },
+                failure_type: {
+                    type: "choice",
+                    instructions: FAILURE_TYPE_QUESTION,
+                    criteria: failureTypeCriteriaMap(input.targetKind),
+                },
+            };
+            input.graderQuestions.forEach((q, index) => {
+                questions[`false_alarm_${index}`] = { type: "noul", instructions: falseAlarmQuestion(q.graderId) };
+            });
+            const reply = await post({ model, state: input.state, questions }, input.signal);
+            const answers = reply.answers ?? {};
+            const bucketAnswer = answers.bucket;
+            const choice = bucketAnswer?.choice;
+            if (typeof choice !== "string" || !TRIAGE_BUCKETS.includes(choice)) {
+                throw new ClassifierError("CLASSIFIER_BAD_REPLY", "Classifier gave no valid bucket");
+            }
+            const bucket = choice;
+            const probsRaw = bucketAnswer?.probabilities;
+            const fromProbs = num(probsRaw?.[bucket]);
+            const confidence = clamp01(num(bucketAnswer?.confidence) ?? fromProbs ?? 0);
+            const failureChoice = answers.failure_type?.choice;
+            const failureType = typeof failureChoice === "string" && types.includes(failureChoice) ? failureChoice : null;
+            const graderFalseAlarms = input.graderQuestions.flatMap((q, index) => {
+                const p = num(answers[`false_alarm_${index}`]?.noul);
+                return p === undefined ? [] : [{ graderId: q.graderId, probability: clamp01(p) }];
+            });
+            const tokens = num(reply.usage?.input_tokens) ?? 0;
+            return {
+                bucket,
+                confidence,
+                bucketProbabilities: normalizeBucketProbabilities(probsRaw, bucket, confidence),
+                failureType,
+                graderFalseAlarms,
+                costUsd: (tokens * JEV_PRICE_INPUT_PER_MILLION_USD) / 1_000_000,
+                ...(typeof reply.model === "string" && reply.model.length > 0 ? { answeredModel: reply.model } : {}),
+            };
+        },
+    };
+}

package/dist/evals/triage/judgeClassifier.d.ts

@@ -0,0 +1,10 @@
+import type { SimulatorModel } from "../runner/simulatorModel.js";
+import { type FailureClassifier } from "./types.js";
+/**
+ * Judge-model backend: the same one-shot model call the agent simulator uses, with a fixed prompt and a JSON reply.
+ * `model` is any `SimulatorModel`; the Host builds it from the agent backend.
+ */
+export declare function createJudgeClassifier(options: {
+    model: SimulatorModel;
+    modelId: string;
+}): FailureClassifier;

package/dist/evals/triage/judgeClassifier.js

@@ -0,0 +1,76 @@
+import { TRIAGE_BUCKETS } from "./constants.js";
+import { normalizeBucketProbabilities } from "./jevClassifier.js";
+import { failureTypesFor, judgeSystemPrompt, judgeUserPrompt } from "./prompts.js";
+import { ClassifierError } from "./types.js";
+const JUDGE_CALL_TIMEOUT_MS = 120_000;
+function extractJson(text) {
+    const start = text.indexOf("{");
+    const end = text.lastIndexOf("}");
+    if (start < 0 || end <= start)
+        return null;
+    try {
+        const parsed = JSON.parse(text.slice(start, end + 1));
+        return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+function num(value) {
+    return typeof value === "number" && Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : undefined;
+}
+/**
+ * Judge-model backend: the same one-shot model call the agent simulator uses, with a fixed prompt and a JSON reply.
+ * `model` is any `SimulatorModel`; the Host builds it from the agent backend.
+ */
+export function createJudgeClassifier(options) {
+    return {
+        source: "judge",
+        model: options.modelId,
+        async classify(input) {
+            const signal = input.signal ?? AbortSignal.timeout(JUDGE_CALL_TIMEOUT_MS);
+            let reply;
+            try {
+                reply = await options.model.complete({
+                    system: judgeSystemPrompt(),
+                    user: judgeUserPrompt({
+                        targetKind: input.targetKind,
+                        state: input.state,
+                        graderIds: input.graderQuestions.map((q) => q.graderId),
+                    }),
+                    signal,
+                });
+            }
+            catch {
+                throw new ClassifierError("CLASSIFIER_FAILED", "The judge model call failed");
+            }
+            const parsed = extractJson(reply.text);
+            const bucketRaw = parsed?.bucket;
+            if (parsed === null || typeof bucketRaw !== "string" || !TRIAGE_BUCKETS.includes(bucketRaw)) {
+                throw new ClassifierError("CLASSIFIER_BAD_REPLY", "The judge model did not return a valid bucket");
+            }
+            const bucket = bucketRaw;
+            const confidence = num(parsed.confidence) ?? 0.5;
+            const types = failureTypesFor(input.targetKind);
+            const failureType = typeof parsed.failure_type === "string" && types.includes(parsed.failure_type) ? parsed.failure_type : null;
+            const known = new Set(input.graderQuestions.map((q) => q.graderId));
+            const alarms = Array.isArray(parsed.grader_false_alarms) ? parsed.grader_false_alarms : [];
+            const graderFalseAlarms = alarms.flatMap((entry) => {
+                const rec = typeof entry === "object" && entry !== null ? entry : {};
+                const p = num(rec.probability);
+                return typeof rec.grader_id === "string" && known.has(rec.grader_id) && p !== undefined
+                    ? [{ graderId: rec.grader_id, probability: p }]
+                    : [];
+            });
+            return {
+                bucket,
+                confidence,
+                bucketProbabilities: normalizeBucketProbabilities(undefined, bucket, confidence),
+                failureType,
+                graderFalseAlarms,
+                costUsd: reply.usage?.costUsd ?? 0,
+                ...(reply.usage?.model !== undefined ? { answeredModel: reply.usage.model } : {}),
+            };
+        },
+    };
+}

package/dist/evals/triage/prompts.d.ts

@@ -0,0 +1,16 @@
+import { type TriageBucket } from "./constants.js";
+/** Changing any text here requires bumping TRIAGE_PROMPT_VERSION in src/evals/constants.ts. */
+export declare const BUCKET_CRITERIA: Record<TriageBucket, string>;
+export declare const FAILURE_TYPE_CRITERIA: Record<string, string>;
+export declare function failureTypesFor(kind: "stage" | "agent"): readonly string[];
+export declare function bucketCriteriaMap(): Record<string, string>;
+export declare function failureTypeCriteriaMap(kind: "stage" | "agent"): Record<string, string>;
+export declare const BUCKET_QUESTION = "Why did this eval result fail? Choose who is at fault: the target, a grader, or the test case. Choose unclear when unsure.";
+export declare const FAILURE_TYPE_QUESTION = "Which kind of failure is this? Choose the closest type.";
+export declare const falseAlarmQuestion: (graderId: string) => string;
+export declare function judgeSystemPrompt(): string;
+export declare function judgeUserPrompt(input: {
+    targetKind: "stage" | "agent";
+    state: string;
+    graderIds: string[];
+}): string;