vitest-evals 0.9.0 → 0.10.0

package/dist/harness.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/harness.ts"],"sourcesContent":["/** Primitive scalar values allowed in normalized JSON-safe eval data. */\nexport type JsonPrimitive = string | number | boolean | null;\n\n/** JSON-safe value shape used by normalized sessions, artifacts, and errors. */\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\n/**\n * Normalized record for one tool call observed during a harness run.\n *\n * @example\n * ```ts\n * const call: ToolCallRecord = {\n *   name: \"lookupInvoice\",\n *   arguments: { invoiceId: \"inv_123\" },\n *   result: { refundable: true },\n * };\n * ```\n */\nexport type ToolCallRecord = {\n  /** Provider or runtime tool-call id when one is available. */\n  id?: string;\n  /** Tool name as exposed to the agent or application runtime. */\n  name: string;\n  /** JSON-safe tool arguments after provider/runtime normalization. */\n  arguments?: Record<string, JsonValue>;\n  /** JSON-safe tool result returned by the application tool. */\n  result?: JsonValue;\n  /** Normalized tool error when execution failed. */\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  /** ISO timestamp for the start of tool execution. */\n  startedAt?: string;\n  /** ISO timestamp for the end of tool execution. */\n  finishedAt?: string;\n  /** Tool execution duration in milliseconds. */\n  durationMs?: number;\n  /** Extra JSON-safe tool metadata for reporters and custom judges. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * Normalized message recorded in a harness session transcript.\n *\n * @example\n * ```ts\n * const message: NormalizedMessage = {\n *   role: \"assistant\",\n *   content: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\" }],\n * };\n * ```\n */\nexport type NormalizedMessage = {\n  /** Transcript role for the normalized message. */\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  /** JSON-safe message content. */\n  content?: JsonValue;\n  /** Tool calls associated with this message. */\n  toolCalls?: ToolCallRecord[];\n  /** Extra JSON-safe message metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * Provider usage summary attached to a normalized harness run.\n *\n * @example\n * ```ts\n * const usage: UsageSummary = {\n *   provider: \"openai\",\n *   model: \"gpt-4o-mini\",\n *   inputTokens: 212,\n *   outputTokens: 48,\n *   totalTokens: 260,\n * };\n * ```\n */\nexport type UsageSummary = {\n  /** Provider that served the application run. */\n  provider?: string;\n  /** Model used for the application run. */\n  model?: string;\n  /** Input, prompt, or request tokens consumed by the run. */\n  inputTokens?: number;\n  /** Output or completion tokens produced by the run. */\n  outputTokens?: number;\n  /** Reasoning tokens reported by providers that expose them. */\n  reasoningTokens?: number;\n  /** Total token count reported by the provider or adapter. */\n  totalTokens?: number;\n  /** Count of tool calls observed during the run. */\n  toolCalls?: number;\n  /** Retry count observed during the run. */\n  retries?: number;\n  /** Provider-specific JSON-safe usage details. Cost estimates belong here. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/** Timing summary attached to a normalized harness run. */\nexport type TimingSummary = {\n  /** End-to-end run duration in milliseconds. */\n  totalMs?: number;\n  /** Extra JSON-safe timing metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * JSON-serializable transcript produced by the system under test.\n *\n * @example\n * ```ts\n * const session: NormalizedSession = {\n *   provider: \"openai\",\n *   model: \"gpt-4o-mini\",\n *   messages: [\n *     { role: \"user\", content: \"Refund invoice inv_123\" },\n *     { role: \"assistant\", content: { status: \"approved\" } },\n *   ],\n * };\n * ```\n */\nexport type NormalizedSession = {\n  /** Ordered normalized transcript messages. */\n  messages: NormalizedMessage[];\n  /** Provider that produced the session when known. */\n  provider?: string;\n  /** Model that produced the session when known. */\n  model?: string;\n  /** Extra JSON-safe session metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\ntype OutputField<TOutput extends JsonValue | undefined> =\n  undefined extends TOutput ? { output?: TOutput } : { output: TOutput };\n\n/**\n * Normalized result returned by every harness execution.\n *\n * @example\n * ```ts\n * const run: HarnessRun<{ status: \"approved\" }> = {\n *   output: { status: \"approved\" },\n *   session: {\n *     messages: [\n *       { role: \"user\", content: \"Refund invoice inv_123\" },\n *       { role: \"assistant\", content: { status: \"approved\" } },\n *     ],\n *   },\n *   usage: { totalTokens: 260 },\n *   errors: [],\n * };\n * ```\n */\nexport type HarnessRun<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = OutputField<TOutput> & {\n  /** Normalized transcript and provider/session metadata. */\n  session: NormalizedSession;\n  /** Stable provider usage units such as tokens, tools, and retries. */\n  usage: UsageSummary;\n  /** Optional timing summary for the run. */\n  timings?: TimingSummary;\n  /** JSON-safe run artifacts captured by the harness or test context. */\n  artifacts?: Record<string, JsonValue>;\n  /** Normalized errors captured during execution. */\n  errors: Array<Record<string, JsonValue>>;\n};\n\n/** Error value with an attached partial or complete normalized harness run. */\nexport type HarnessRunError = Error & {\n  /** Attached normalized harness run recovered by `getHarnessRunFromError(...)`. */\n  vitestEvalsRun: HarnessRun;\n};\n\n/** Per-run metadata shape accepted by harnesses and eval tests. */\nexport type HarnessMetadata = Record<string, unknown>;\n\n/**\n * Runtime context passed from the eval fixture into a harness run.\n *\n * @example\n * ```ts\n * const harness: Harness<string> = {\n *   name: \"refund-agent\",\n *   async run(input, context) {\n *     context.setArtifact(\"inputLength\", input.length);\n *\n *     return {\n *       output: undefined,\n *       session: { messages: [{ role: \"user\", content: input }] },\n *       usage: {},\n *       errors: [],\n *     };\n *   },\n * };\n * ```\n */\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Per-run metadata passed through `run(input, { metadata })`. */\n  metadata: Readonly<TMetadata>;\n  /** Abort signal from Vitest when available. */\n  signal?: AbortSignal;\n  /** Mutable JSON-safe artifact bag shared with the harness. */\n  artifacts: Record<string, JsonValue>;\n  /** Stores one JSON-safe artifact on the current run. */\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\n/**\n * Adapter that executes the system under test and returns a normalized run.\n *\n * @example\n * ```ts\n * const harness: Harness<string, { status: \"approved\" | \"denied\" }> = {\n *   name: \"refund-agent\",\n *   async run(input, context) {\n *     return normalizeHarnessRun(input, await runRefundFlow(input), context);\n *   },\n * };\n * ```\n */\nexport type Harness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Stable harness name used in reports. */\n  name: string;\n  /** Executes the system under test and returns a normalized run. */\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun<TOutput>>;\n};\n\n/** Value or promise accepted by lightweight harness callbacks. */\nexport type MaybePromise<T> = T | Promise<T>;\n\n/** Lightweight tool-call record accepted by `createHarness(...)` results. */\nexport type SimpleToolCallRecord = Omit<\n  ToolCallRecord,\n  \"arguments\" | \"result\" | \"error\" | \"metadata\"\n> & {\n  /** Raw tool arguments accepted by `createHarness(...)` before normalization. */\n  arguments?: unknown;\n  /** Raw tool result accepted by `createHarness(...)` before normalization. */\n  result?: unknown;\n  /** Raw tool error accepted by `createHarness(...)` before normalization. */\n  error?: unknown;\n  /** Raw tool metadata accepted by `createHarness(...)` before normalization. */\n  metadata?: Record<string, unknown>;\n};\n\n/**\n * Lightweight result shape normalized by `createHarness(...)`.\n *\n * @example\n * ```ts\n * const result: SimpleHarnessResult<{ status: \"approved\" }> = {\n *   output: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\", arguments: { invoiceId: \"inv_123\" } }],\n *   usage: { totalTokens: 260 },\n * };\n * ```\n */\nexport type SimpleHarnessResult<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = OutputField<TOutput> & {\n  /** Pre-normalized transcript messages. When omitted, a default user/assistant transcript is created. */\n  messages?: NormalizedMessage[];\n  /** Lightweight tool-call records to normalize into the session. */\n  toolCalls?: SimpleToolCallRecord[];\n  /** Usage summary to attach to the run. */\n  usage?: UsageSummary;\n  /** Timing summary to attach to the run. */\n  timings?: TimingSummary;\n  /** Raw artifact values to normalize and merge into the run. */\n  artifacts?: Record<string, unknown>;\n  /** Raw session metadata to normalize into the session. */\n  metadata?: Record<string, unknown>;\n  /** Raw errors to normalize into the run. */\n  errors?: unknown[];\n};\n\n/** Either a complete normalized run or a lightweight result to normalize. */\nexport type HarnessResultLike<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = HarnessRun<TOutput> | SimpleHarnessResult<TOutput>;\n\n/** Arguments passed to the `createHarness(...)` convenience callback. */\nexport type CreateHarnessRunArgs<TInput, TMetadata extends HarnessMetadata> = {\n  /** Original input passed to `run(input)`. */\n  input: TInput;\n  /** Read-only metadata passed to `run(input, { metadata })`. */\n  metadata: Readonly<TMetadata>;\n  /** Abort signal from Vitest when available. */\n  signal?: AbortSignal;\n  /** Mutable run artifact bag. */\n  artifacts: HarnessContext<TMetadata>[\"artifacts\"];\n  /** Stores one JSON-safe artifact on the current run. */\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n};\n\n/**\n * Options for creating a lightweight custom application harness.\n *\n * @example\n * ```ts\n * const options: CreateHarnessOptions<string, { status: \"approved\" }> = {\n *   name: \"refund-agent\",\n *   run: async ({ input }) => ({\n *     output: await classifyRefund(input),\n *   }),\n * };\n * ```\n */\nexport type CreateHarnessOptions<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Stable harness name used in reports. */\n  name: string;\n  /** Executes application code and returns either a lightweight result or full `HarnessRun`. */\n  run: (\n    args: CreateHarnessRunArgs<TInput, TMetadata>,\n  ) => MaybePromise<HarnessResultLike<TOutput>>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  const normalized = toJsonValue(value);\n  return normalized !== undefined ? normalized : String(value);\n}\n\n/**\n * Creates a harness from the common \"run app code and return output\" shape.\n *\n * @param options - Harness name plus the callback that executes app code.\n *\n * @example\n * ```ts\n * import { createHarness } from \"vitest-evals\";\n *\n * export const refundHarness = createHarness<\n *   string,\n *   { status: \"approved\" | \"denied\" },\n *   { expected: { status: \"approved\" | \"denied\" } }\n * >({\n *   name: \"refund-agent\",\n *   run: async ({ input, metadata, setArtifact }) => {\n *     const result = await runRefundFlow(input, metadata);\n *     const output = { status: result.status };\n *\n *     setArtifact(\"case\", { expected: metadata.expected.status });\n *\n *     return {\n *       output,\n *       toolCalls: result.toolCalls,\n *       usage: { provider: \"openai\", model: \"gpt-4o-mini\" },\n *     };\n *   },\n * });\n * ```\n */\nexport function createHarness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n>(\n  options: CreateHarnessOptions<TInput, TOutput, TMetadata>,\n): Harness<TInput, TOutput, TMetadata>;\nexport function createHarness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n>(\n  options: CreateHarnessOptions<TInput, TOutput, TMetadata>,\n): Harness<TInput, TOutput, TMetadata> {\n  const harness: Harness<TInput, TOutput, TMetadata> = {\n    name: options.name,\n    run: async (input, context) => {\n      const result = await options.run({\n        input,\n        metadata: context.metadata,\n        signal: context.signal,\n        artifacts: context.artifacts,\n        setArtifact: context.setArtifact,\n      });\n\n      return normalizeHarnessRun(input, result, context);\n    },\n  };\n\n  return harness;\n}\n\n/**\n * Normalizes a lightweight harness result into the reporter-facing run shape.\n *\n * @param input - Original input passed to the harness.\n * @param result - Lightweight result or pre-normalized harness run.\n * @param context - Optional per-run context used to merge artifacts.\n *\n * @example\n * ```ts\n * const run = normalizeHarnessRun(\"Refund invoice inv_123\", {\n *   output: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\", arguments: { invoiceId: \"inv_123\" } }],\n *   usage: { provider: \"openai\", model: \"gpt-4o-mini\" },\n * });\n *\n * expect(toolCalls(run.session)).toHaveLength(1);\n * ```\n */\nexport function normalizeHarnessRun<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n>(\n  input: TInput,\n  result: HarnessResultLike<TOutput>,\n  context?: HarnessContext<TMetadata>,\n): HarnessRun<TOutput> {\n  if (isHarnessRun(result)) {\n    if (\n      context &&\n      Object.keys(context.artifacts).length > 0 &&\n      !result.artifacts\n    ) {\n      return {\n        ...result,\n        artifacts: context.artifacts,\n      };\n    }\n\n    return result;\n  }\n\n  const output = result.output;\n  const toolCalls = normalizeSimpleToolCalls(result.toolCalls);\n  const usage = result.usage ?? {};\n  const messages =\n    result.messages ??\n    createDefaultSessionMessages({\n      input,\n      output,\n      toolCalls,\n    });\n  const metadata = result.metadata\n    ? normalizeMetadata(result.metadata)\n    : undefined;\n  const artifacts = normalizeMergedArtifacts(\n    context?.artifacts,\n    result.artifacts,\n  );\n\n  return {\n    session: {\n      messages,\n      ...(usage.provider ? { provider: usage.provider } : {}),\n      ...(usage.model ? { model: usage.model } : {}),\n      ...(metadata ? { metadata } : {}),\n    },\n    ...(output !== undefined ? { output } : {}),\n    usage,\n    ...(result.timings ? { timings: result.timings } : {}),\n    ...(artifacts ? { artifacts } : {}),\n    errors: normalizeSimpleErrors(result.errors),\n  } as HarnessRun<TOutput>;\n}\n\nfunction createDefaultSessionMessages<TInput>({\n  input,\n  output,\n  toolCalls: normalizedToolCalls,\n}: {\n  input: TInput;\n  output: JsonValue | undefined;\n  toolCalls: ToolCallRecord[];\n}): NormalizedMessage[] {\n  const messages: NormalizedMessage[] = [\n    {\n      role: \"user\",\n      content: normalizeContent(input),\n    },\n  ];\n\n  if (output !== undefined || normalizedToolCalls.length > 0) {\n    messages.push({\n      role: \"assistant\",\n      ...(output !== undefined ? { content: normalizeContent(output) } : {}),\n      ...(normalizedToolCalls.length > 0\n        ? { toolCalls: normalizedToolCalls }\n        : {}),\n    });\n  }\n\n  return messages;\n}\n\nfunction normalizeSimpleToolCalls(\n  calls: SimpleToolCallRecord[] | undefined,\n): ToolCallRecord[] {\n  return (calls ?? []).map((call) => {\n    const {\n      arguments: rawArguments,\n      result: rawResult,\n      error: rawError,\n      metadata: rawMetadata,\n      ...toolCall\n    } = call;\n    const args = normalizeToolCallArguments(rawArguments);\n    const result = toJsonValue(rawResult);\n    const error = normalizeToolCallError(rawError);\n    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : undefined;\n\n    return {\n      ...toolCall,\n      ...(args ? { arguments: args } : {}),\n      ...(result !== undefined ? { result } : {}),\n      ...(error ? { error } : {}),\n      ...(metadata ? { metadata } : {}),\n    };\n  });\n}\n\nfunction normalizeToolCallArguments(\n  value: unknown,\n): Record<string, JsonValue> | undefined {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  const normalized = toJsonValue(value);\n  return normalized &&\n    typeof normalized === \"object\" &&\n    !Array.isArray(normalized)\n    ? normalized\n    : undefined;\n}\n\nfunction normalizeToolCallError(\n  value: unknown,\n): ToolCallRecord[\"error\"] | undefined {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  const serialized = serializeError(value);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction normalizeMergedArtifacts(\n  contextArtifacts: Record<string, JsonValue> | undefined,\n  resultArtifacts: Record<string, unknown> | undefined,\n) {\n  const artifacts = {\n    ...(contextArtifacts ?? {}),\n    ...(resultArtifacts ? normalizeRecord(resultArtifacts) : {}),\n  };\n\n  return Object.keys(artifacts).length > 0 ? artifacts : undefined;\n}\n\nfunction normalizeSimpleErrors(\n  errors: unknown[] | undefined,\n): Array<Record<string, JsonValue>> {\n  return (errors ?? []).map((error) => {\n    const normalized = toJsonValue(error);\n\n    if (\n      normalized &&\n      typeof normalized === \"object\" &&\n      !Array.isArray(normalized) &&\n      Object.keys(normalized).length > 0\n    ) {\n      return normalized;\n    }\n\n    return serializeError(error);\n  });\n}\n\n/**\n * Flattens every recorded tool call from a normalized session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const names = toolCalls(result.session).map((call) => call.name);\n *\n * expect(names).toEqual([\"lookupInvoice\", \"createRefund\"]);\n * ```\n */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/**\n * Filters normalized session messages by role.\n *\n * @param session - Normalized session produced by a harness run.\n * @param role - Message role to keep.\n *\n * @example\n * ```ts\n * const assistantText = messagesByRole(result.session, \"assistant\")\n *   .map((message) => message.content)\n *   .join(\"\\n\");\n * ```\n */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\n/**\n * Returns every normalized system message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const systemPrompts = systemMessages(result.session);\n * ```\n */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/**\n * Returns every normalized user message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const firstPrompt = userMessages(result.session)[0]?.content;\n * ```\n */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/**\n * Returns every normalized assistant message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const finalAnswer = assistantMessages(result.session).at(-1)?.content;\n * ```\n */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/**\n * Returns every normalized tool message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const toolOutputs = toolMessages(result.session).map((message) => message.content);\n * ```\n */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/**\n * Attaches a partial or complete harness run to an arbitrary thrown error.\n *\n * @param error - Thrown value to wrap.\n * @param run - Partial or complete normalized harness run to preserve.\n *\n * @example\n * ```ts\n * try {\n *   return await runAgent(input);\n * } catch (error) {\n *   throw attachHarnessRunToError(error, partialRun);\n * }\n * ```\n */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/**\n * Reads an attached harness run back off a previously wrapped error value.\n *\n * @param error - Unknown thrown value that may contain a harness run.\n *\n * @example\n * ```ts\n * const partialRun = getHarnessRunFromError(error);\n *\n * if (partialRun) {\n *   console.log(toolCalls(partialRun.session));\n * }\n * ```\n */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";AAiVA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,kBAAkB,OAAgB,YAAoB;AACpE,SACE,UAAU,SACT,OAAO,UAAU,YAAY,OAAO,UAAU,eAC/C,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAGO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,QAAM,aAAa,YAAY,KAAK;AACpC,SAAO,eAAe,SAAY,aAAa,OAAO,KAAK;AAC7D;AAuCO,SAAS,cAKd,SACqC;AACrC,QAAM,UAA+C;AAAA,IACnD,MAAM,QAAQ;AAAA,IACd,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,SAAS,MAAM,QAAQ,IAAI;AAAA,QAC/B;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB,QAAQ,QAAQ;AAAA,QAChB,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,MACvB,CAAC;AAED,aAAO,oBAAoB,OAAO,QAAQ,OAAO;AAAA,IACnD;AAAA,EACF;AAEA,SAAO;AACT;AAoBO,SAAS,oBAKd,OACA,QACA,SACqB;AACrB,MAAI,aAAa,MAAM,GAAG;AACxB,QACE,WACA,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KACxC,CAAC,OAAO,WACR;AACA,aAAO;AAAA,QACL,GAAG;AAAA,QACH,WAAW,QAAQ;AAAA,MACrB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,OAAO;AACtB,QAAMA,aAAY,yBAAyB,OAAO,SAAS;AAC3D,QAAM,QAAQ,OAAO,SAAS,CAAC;AAC/B,QAAM,WACJ,OAAO,YACP,6BAA6B;AAAA,IAC3B;AAAA,IACA;AAAA,IACA,WAAAA;AAAA,EACF,CAAC;AACH,QAAM,WAAW,OAAO,WACpB,kBAAkB,OAAO,QAAQ,IACjC;AACJ,QAAM,YAAY;AAAA,IAChB,SAAS;AAAA,IACT,OAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,MACP;AAAA,MACA,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,MACrD,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;AAAA,MAC5C,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,IACA,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,IACzC;AAAA,IACA,GAAI,OAAO,UAAU,EAAE,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IACpD,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;AAAA,IACjC,QAAQ,sBAAsB,OAAO,MAAM;AAAA,EAC7C;AACF;AAEA,SAAS,6BAAqC;AAAA,EAC5C;AAAA,EACA;AAAA,EACA,WAAW;AACb,GAIwB;AACtB,QAAM,WAAgC;AAAA,IACpC;AAAA,MACE,MAAM;AAAA,MACN,SAAS,iBAAiB,KAAK;AAAA,IACjC;AAAA,EACF;AAEA,MAAI,WAAW,UAAa,oBAAoB,SAAS,GAAG;AAC1D,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,GAAI,WAAW,SAAY,EAAE,SAAS,iBAAiB,MAAM,EAAE,IAAI,CAAC;AAAA,MACpE,GAAI,oBAAoB,SAAS,IAC7B,EAAE,WAAW,oBAAoB,IACjC,CAAC;AAAA,IACP,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,OACkB;AAClB,UAAQ,SAAS,CAAC,GAAG,IAAI,CAAC,SAAS;AACjC,UAAM;AAAA,MACJ,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,UAAU;AAAA,MACV,GAAG;AAAA,IACL,IAAI;AACJ,UAAM,OAAO,2BAA2B,YAAY;AACpD,UAAM,SAAS,YAAY,SAAS;AACpC,UAAM,QAAQ,uBAAuB,QAAQ;AAC7C,UAAM,WAAW,cAAc,kBAAkB,WAAW,IAAI;AAEhE,WAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAI,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,MACzC,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;AAAA,MACzB,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,EACF,CAAC;AACH;AAEA,SAAS,2BACP,OACuC;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,YAAY,KAAK;AACpC,SAAO,cACL,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,IACvB,aACA;AACN;AAEA,SAAS,uBACP,OACqC;AACrC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,eAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,yBACP,kBACA,iBACA;AACA,QAAM,YAAY;AAAA,IAChB,GAAI,oBAAoB,CAAC;AAAA,IACzB,GAAI,kBAAkB,gBAAgB,eAAe,IAAI,CAAC;AAAA,EAC5D;AAEA,SAAO,OAAO,KAAK,SAAS,EAAE,SAAS,IAAI,YAAY;AACzD;AAEA,SAAS,sBACP,QACkC;AAClC,UAAQ,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU;AACnC,UAAM,aAAa,YAAY,KAAK;AAEpC,QACE,cACA,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,KACzB,OAAO,KAAK,UAAU,EAAE,SAAS,GACjC;AACA,aAAO;AAAA,IACT;AAEA,WAAO,eAAe,KAAK;AAAA,EAC7B,CAAC;AACH;AAcO,SAAS,UAAU,SAA8C;AACtE,SAAO,QAAQ,SAAS,QAAQ,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;AACtE;AAeO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAYO,SAAS,eAAe,SAA4B;AACzD,SAAO,eAAe,SAAS,QAAQ;AACzC;AAYO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAYO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAYO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAiBO,SAAS,wBACd,OACA,KACiB;AACjB,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,gBAAgB;AAAA,EAClB,CAAC;AACH;AAgBO,SAAS,uBAAuB,OAAwC;AAC7E,MACE,SACA,OAAO,UAAU,YACjB,oBAAoB,SACpB,aAAc,MAAuC,cAAc,GACnE;AACA,WAAQ,MAAyC;AAAA,EACnD;AAEA,SAAO;AACT;AAGO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAGO,SAAS,wBACd,QACkC;AAClC,MACE,UACA,OAAO,WAAW,YAClB,MAAM,QAAS,OAAmC,MAAM,GACxD;AACA,WAAQ,OAAwD;AAAA,EAClE;AAEA,SAAO,CAAC;AACV;AAGO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;","names":["toolCalls"]}
+{"version":3,"sources":["../src/harness.ts"],"sourcesContent":["/** Primitive scalar values allowed in normalized JSON-safe eval data. */\nexport type JsonPrimitive = string | number | boolean | null;\n\n/** JSON-safe value shape used by normalized sessions, artifacts, and errors. */\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\n/**\n * Normalized record for one tool call observed during a harness run.\n *\n * @example\n * ```ts\n * const call: ToolCallRecord = {\n *   name: \"lookupInvoice\",\n *   arguments: { invoiceId: \"inv_123\" },\n *   result: { refundable: true },\n * };\n * ```\n */\nexport type ToolCallRecord = {\n  /** Provider or runtime tool-call id when one is available. */\n  id?: string;\n  /** Tool name as exposed to the agent or application runtime. */\n  name: string;\n  /** JSON-safe tool arguments after provider/runtime normalization. */\n  arguments?: Record<string, JsonValue>;\n  /** JSON-safe tool result returned by the application tool. */\n  result?: JsonValue;\n  /** Normalized tool error when execution failed. */\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  /** ISO timestamp for the start of tool execution. */\n  startedAt?: string;\n  /** ISO timestamp for the end of tool execution. */\n  finishedAt?: string;\n  /** Tool execution duration in milliseconds. */\n  durationMs?: number;\n  /** Extra JSON-safe tool metadata for reporters and custom judges. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * Normalized message recorded in a harness session transcript.\n *\n * @example\n * ```ts\n * const message: NormalizedMessage = {\n *   role: \"assistant\",\n *   content: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\" }],\n * };\n * ```\n */\nexport type NormalizedMessage = {\n  /** Transcript role for the normalized message. */\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  /** JSON-safe message content. */\n  content?: JsonValue;\n  /** Tool calls associated with this message. */\n  toolCalls?: ToolCallRecord[];\n  /** Extra JSON-safe message metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * Provider usage summary attached to a normalized harness run.\n *\n * @example\n * ```ts\n * const usage: UsageSummary = {\n *   provider: \"openai\",\n *   model: \"gpt-4o-mini\",\n *   inputTokens: 212,\n *   outputTokens: 48,\n *   totalTokens: 260,\n * };\n * ```\n */\nexport type UsageSummary = {\n  /** Provider that served the application run. */\n  provider?: string;\n  /** Model used for the application run. */\n  model?: string;\n  /** Input, prompt, or request tokens consumed by the run. */\n  inputTokens?: number;\n  /** Output or completion tokens produced by the run. */\n  outputTokens?: number;\n  /** Reasoning tokens reported by providers that expose them. */\n  reasoningTokens?: number;\n  /** Total token count reported by the provider or adapter. */\n  totalTokens?: number;\n  /** Count of tool calls observed during the run. */\n  toolCalls?: number;\n  /** Retry count observed during the run. */\n  retries?: number;\n  /** Provider-specific JSON-safe usage details. Cost estimates belong here. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/** Timing summary attached to a normalized harness run. */\nexport type TimingSummary = {\n  /** End-to-end run duration in milliseconds. */\n  totalMs?: number;\n  /** Extra JSON-safe timing metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * JSON-serializable transcript produced by the system under test.\n *\n * @example\n * ```ts\n * const session: NormalizedSession = {\n *   provider: \"openai\",\n *   model: \"gpt-4o-mini\",\n *   messages: [\n *     { role: \"user\", content: \"Refund invoice inv_123\" },\n *     { role: \"assistant\", content: { status: \"approved\" } },\n *   ],\n * };\n * ```\n */\nexport type NormalizedSession = {\n  /** Ordered normalized transcript messages. */\n  messages: NormalizedMessage[];\n  /** Provider that produced the session when known. */\n  provider?: string;\n  /** Model that produced the session when known. */\n  model?: string;\n  /** Extra JSON-safe session metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\ntype OutputField<TOutput extends JsonValue | undefined> =\n  undefined extends TOutput ? { output?: TOutput } : { output: TOutput };\n\n/**\n * Normalized result returned by every harness execution.\n *\n * @example\n * ```ts\n * const run: HarnessRun<{ status: \"approved\" }> = {\n *   output: { status: \"approved\" },\n *   session: {\n *     messages: [\n *       { role: \"user\", content: \"Refund invoice inv_123\" },\n *       { role: \"assistant\", content: { status: \"approved\" } },\n *     ],\n *   },\n *   usage: { totalTokens: 260 },\n *   errors: [],\n * };\n * ```\n */\nexport type HarnessRun<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = OutputField<TOutput> & {\n  /** Normalized transcript and provider/session metadata. */\n  session: NormalizedSession;\n  /** Stable provider usage units such as tokens, tools, and retries. */\n  usage: UsageSummary;\n  /** Optional timing summary for the run. */\n  timings?: TimingSummary;\n  /** JSON-safe run artifacts captured by the harness or test context. */\n  artifacts?: Record<string, JsonValue>;\n  /** Normalized errors captured during execution. */\n  errors: Array<Record<string, JsonValue>>;\n};\n\n/** Error value with an attached partial or complete normalized harness run. */\nexport type HarnessRunError = Error & {\n  /** Attached normalized harness run recovered by `getHarnessRunFromError(...)`. */\n  vitestEvalsRun: HarnessRun;\n};\n\n/** Per-run metadata shape accepted by harnesses and eval tests. */\nexport type HarnessMetadata = Record<string, unknown>;\n\n/**\n * Runtime context passed from the eval fixture into a harness run.\n *\n * @example\n * ```ts\n * const harness: Harness<string> = {\n *   name: \"refund-agent\",\n *   async run(input, context) {\n *     context.setArtifact(\"inputLength\", input.length);\n *\n *     return {\n *       output: undefined,\n *       session: { messages: [{ role: \"user\", content: input }] },\n *       usage: {},\n *       errors: [],\n *     };\n *   },\n * };\n * ```\n */\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Per-run metadata passed through `run(input, { metadata })`. */\n  metadata: Readonly<TMetadata>;\n  /** Abort signal from Vitest when available. */\n  signal?: AbortSignal;\n  /** Mutable JSON-safe artifact bag shared with the harness. */\n  artifacts: Record<string, JsonValue>;\n  /** Stores one JSON-safe artifact on the current run. */\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\n/**\n * Adapter that executes the system under test and returns a normalized run.\n *\n * @example\n * ```ts\n * const harness: Harness<string, { status: \"approved\" | \"denied\" }> = {\n *   name: \"refund-agent\",\n *   async run(input, context) {\n *     return normalizeHarnessRun(input, await runRefundFlow(input), context);\n *   },\n * };\n * ```\n */\nexport type Harness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Stable harness name used in reports. */\n  name: string;\n  /** Executes the system under test and returns a normalized run. */\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun<TOutput>>;\n};\n\n/** Value or promise accepted by lightweight harness callbacks. */\nexport type MaybePromise<T> = T | Promise<T>;\n\n/** Lightweight tool-call record accepted by `createHarness(...)` results. */\nexport type SimpleToolCallRecord = Omit<\n  ToolCallRecord,\n  \"arguments\" | \"result\" | \"error\" | \"metadata\"\n> & {\n  /** Raw tool arguments accepted by `createHarness(...)` before normalization. */\n  arguments?: unknown;\n  /** Raw tool result accepted by `createHarness(...)` before normalization. */\n  result?: unknown;\n  /** Raw tool error accepted by `createHarness(...)` before normalization. */\n  error?: unknown;\n  /** Raw tool metadata accepted by `createHarness(...)` before normalization. */\n  metadata?: Record<string, unknown>;\n};\n\n/**\n * Lightweight result shape normalized by `createHarness(...)`.\n *\n * @example\n * ```ts\n * const result: SimpleHarnessResult<{ status: \"approved\" }> = {\n *   output: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\", arguments: { invoiceId: \"inv_123\" } }],\n *   usage: { totalTokens: 260 },\n * };\n * ```\n */\nexport type SimpleHarnessResult<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = OutputField<TOutput> & {\n  /** Pre-normalized transcript messages. When omitted, a default user/assistant transcript is created. */\n  messages?: NormalizedMessage[];\n  /** Lightweight tool-call records to normalize into the session. */\n  toolCalls?: SimpleToolCallRecord[];\n  /** Usage summary to attach to the run. */\n  usage?: UsageSummary;\n  /** Timing summary to attach to the run. */\n  timings?: TimingSummary;\n  /** Raw artifact values to normalize and merge into the run. */\n  artifacts?: Record<string, unknown>;\n  /** Raw session metadata to normalize into the session. */\n  metadata?: Record<string, unknown>;\n  /** Raw errors to normalize into the run. */\n  errors?: unknown[];\n};\n\n/** Either a complete normalized run or a lightweight result to normalize. */\nexport type HarnessResultLike<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = HarnessRun<TOutput> | SimpleHarnessResult<TOutput>;\n\n/** Arguments passed to the `createHarness(...)` convenience callback. */\nexport type CreateHarnessRunArgs<TInput, TMetadata extends HarnessMetadata> = {\n  /** Original input passed to `run(input)`. */\n  input: TInput;\n  /** Read-only metadata passed to `run(input, { metadata })`. */\n  metadata: Readonly<TMetadata>;\n  /** Abort signal from Vitest when available. */\n  signal?: AbortSignal;\n  /** Mutable run artifact bag. */\n  artifacts: HarnessContext<TMetadata>[\"artifacts\"];\n  /** Stores one JSON-safe artifact on the current run. */\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n};\n\n/**\n * Options for creating a lightweight custom application harness.\n *\n * @example\n * ```ts\n * const options: CreateHarnessOptions<string, { status: \"approved\" }> = {\n *   name: \"refund-agent\",\n *   run: async ({ input }) => ({\n *     output: await classifyRefund(input),\n *   }),\n * };\n * ```\n */\nexport type CreateHarnessOptions<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Stable harness name used in reports. */\n  name: string;\n  /** Executes application code and returns either a lightweight result or full `HarnessRun`. */\n  run: (\n    args: CreateHarnessRunArgs<TInput, TMetadata>,\n  ) => MaybePromise<HarnessResultLike<TOutput>>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  const normalized = toJsonValue(value);\n  return normalized !== undefined ? normalized : String(value);\n}\n\n/**\n * Creates a harness from the common \"run app code and return output\" shape.\n *\n * @param options - Harness name plus the callback that executes app code.\n *\n * @example\n * ```ts\n * import { createHarness } from \"vitest-evals\";\n *\n * export const refundHarness = createHarness<\n *   string,\n *   { status: \"approved\" | \"denied\" },\n *   { expected: { status: \"approved\" | \"denied\" } }\n * >({\n *   name: \"refund-agent\",\n *   run: async ({ input, metadata, setArtifact }) => {\n *     const result = await runRefundFlow(input, metadata);\n *     const output = { status: result.status };\n *\n *     setArtifact(\"case\", { expected: metadata.expected.status });\n *\n *     return {\n *       output,\n *       toolCalls: result.toolCalls,\n *       usage: { provider: \"openai\", model: \"gpt-4o-mini\" },\n *     };\n *   },\n * });\n * ```\n */\nexport function createHarness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n>(\n  options: CreateHarnessOptions<TInput, TOutput, TMetadata>,\n): Harness<TInput, TOutput, TMetadata>;\nexport function createHarness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n>(\n  options: CreateHarnessOptions<TInput, TOutput, TMetadata>,\n): Harness<TInput, TOutput, TMetadata> {\n  const harness: Harness<TInput, TOutput, TMetadata> = {\n    name: options.name,\n    run: async (input, context) => {\n      const result = await options.run({\n        input,\n        metadata: context.metadata,\n        signal: context.signal,\n        artifacts: context.artifacts,\n        setArtifact: context.setArtifact,\n      });\n\n      return normalizeHarnessRun(input, result, context);\n    },\n  };\n\n  return harness;\n}\n\n/**\n * Normalizes a lightweight harness result into the reporter-facing run shape.\n *\n * @param input - Original input passed to the harness.\n * @param result - Lightweight result or pre-normalized harness run.\n * @param context - Optional per-run context used to merge artifacts.\n *\n * @example\n * ```ts\n * const run = normalizeHarnessRun(\"Refund invoice inv_123\", {\n *   output: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\", arguments: { invoiceId: \"inv_123\" } }],\n *   usage: { provider: \"openai\", model: \"gpt-4o-mini\" },\n * });\n *\n * expect(toolCalls(run.session)).toHaveLength(1);\n * ```\n */\nexport function normalizeHarnessRun<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n>(\n  input: TInput,\n  result: HarnessResultLike<TOutput>,\n  context?: HarnessContext<TMetadata>,\n): HarnessRun<TOutput> {\n  if (isHarnessRun(result)) {\n    if (\n      context &&\n      Object.keys(context.artifacts).length > 0 &&\n      !result.artifacts\n    ) {\n      return {\n        ...result,\n        artifacts: context.artifacts,\n      };\n    }\n\n    return result;\n  }\n\n  const output = result.output;\n  const toolCalls = normalizeSimpleToolCalls(result.toolCalls);\n  const usage = result.usage ?? {};\n  const messages =\n    result.messages ??\n    createDefaultSessionMessages({\n      input,\n      output,\n      toolCalls,\n    });\n  const metadata = result.metadata\n    ? normalizeMetadata(result.metadata)\n    : undefined;\n  const artifacts = normalizeMergedArtifacts(\n    context?.artifacts,\n    result.artifacts,\n  );\n\n  return {\n    session: {\n      messages,\n      ...(usage.provider ? { provider: usage.provider } : {}),\n      ...(usage.model ? { model: usage.model } : {}),\n      ...(metadata ? { metadata } : {}),\n    },\n    ...(output !== undefined ? { output } : {}),\n    usage,\n    ...(result.timings ? { timings: result.timings } : {}),\n    ...(artifacts ? { artifacts } : {}),\n    errors: normalizeSimpleErrors(result.errors),\n  } as HarnessRun<TOutput>;\n}\n\nfunction createDefaultSessionMessages<TInput>({\n  input,\n  output,\n  toolCalls: normalizedToolCalls,\n}: {\n  input: TInput;\n  output: JsonValue | undefined;\n  toolCalls: ToolCallRecord[];\n}): NormalizedMessage[] {\n  const messages: NormalizedMessage[] = [\n    {\n      role: \"user\",\n      content: normalizeContent(input),\n    },\n  ];\n\n  if (output !== undefined || normalizedToolCalls.length > 0) {\n    messages.push({\n      role: \"assistant\",\n      ...(output !== undefined ? { content: normalizeContent(output) } : {}),\n      ...(normalizedToolCalls.length > 0\n        ? { toolCalls: normalizedToolCalls }\n        : {}),\n    });\n  }\n\n  return messages;\n}\n\nfunction normalizeSimpleToolCalls(\n  calls: SimpleToolCallRecord[] | undefined,\n): ToolCallRecord[] {\n  return (calls ?? []).map((call) => {\n    const {\n      arguments: rawArguments,\n      result: rawResult,\n      error: rawError,\n      metadata: rawMetadata,\n      ...toolCall\n    } = call;\n    const args = normalizeToolCallArguments(rawArguments);\n    const result = toJsonValue(rawResult);\n    const error = normalizeToolCallError(rawError);\n    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : undefined;\n\n    return {\n      ...toolCall,\n      ...(args ? { arguments: args } : {}),\n      ...(result !== undefined ? { result } : {}),\n      ...(error ? { error } : {}),\n      ...(metadata ? { metadata } : {}),\n    };\n  });\n}\n\nfunction normalizeToolCallArguments(\n  value: unknown,\n): Record<string, JsonValue> | undefined {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  const normalized = toJsonValue(value);\n  return normalized &&\n    typeof normalized === \"object\" &&\n    !Array.isArray(normalized)\n    ? normalized\n    : undefined;\n}\n\nfunction normalizeToolCallError(\n  value: unknown,\n): ToolCallRecord[\"error\"] | undefined {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  const serialized = serializeError(value);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction normalizeMergedArtifacts(\n  contextArtifacts: Record<string, JsonValue> | undefined,\n  resultArtifacts: Record<string, unknown> | undefined,\n) {\n  const artifacts = {\n    ...(contextArtifacts ?? {}),\n    ...(resultArtifacts ? normalizeRecord(resultArtifacts) : {}),\n  };\n\n  return Object.keys(artifacts).length > 0 ? artifacts : undefined;\n}\n\nfunction normalizeSimpleErrors(\n  errors: unknown[] | undefined,\n): Array<Record<string, JsonValue>> {\n  return (errors ?? []).map((error) => {\n    const normalized = toJsonValue(error);\n\n    if (\n      normalized &&\n      typeof normalized === \"object\" &&\n      !Array.isArray(normalized) &&\n      Object.keys(normalized).length > 0\n    ) {\n      return normalized;\n    }\n\n    return serializeError(error);\n  });\n}\n\n/**\n * Flattens every recorded tool call from a normalized session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const names = toolCalls(result.session).map((call) => call.name);\n *\n * expect(names).toEqual([\"lookupInvoice\", \"createRefund\"]);\n * ```\n */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/**\n * Filters normalized session messages by role.\n *\n * @param session - Normalized session produced by a harness run.\n * @param role - Message role to keep.\n *\n * @example\n * ```ts\n * const assistantText = messagesByRole(result.session, \"assistant\")\n *   .map((message) => message.content)\n *   .join(\"\\n\");\n * ```\n */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\nfunction hasNonEmptyMessageContent(message: NormalizedMessage) {\n  return (\n    message.content !== undefined &&\n    (typeof message.content !== \"string\" || message.content.trim().length > 0)\n  );\n}\n\n/**\n * Returns every normalized system message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const systemPrompts = systemMessages(result.session);\n * ```\n */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/**\n * Returns every normalized user message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const firstPrompt = userMessages(result.session)[0]?.content;\n * ```\n */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/**\n * Returns every normalized assistant message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const finalAnswer = assistantMessages(result.session).at(-1)?.content;\n * ```\n */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/**\n * Returns the latest assistant message content, ignoring empty text messages.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const finalAnswer = latestAssistantMessageContent(result.session);\n * ```\n */\nexport function latestAssistantMessageContent(session: NormalizedSession) {\n  return [...assistantMessages(session)]\n    .reverse()\n    .find(hasNonEmptyMessageContent)?.content;\n}\n\n/**\n * Returns every normalized tool message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const toolOutputs = toolMessages(result.session).map((message) => message.content);\n * ```\n */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/**\n * Attaches a partial or complete harness run to an arbitrary thrown error.\n *\n * @param error - Thrown value to wrap.\n * @param run - Partial or complete normalized harness run to preserve.\n *\n * @example\n * ```ts\n * try {\n *   return await runAgent(input);\n * } catch (error) {\n *   throw attachHarnessRunToError(error, partialRun);\n * }\n * ```\n */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/**\n * Reads an attached harness run back off a previously wrapped error value.\n *\n * @param error - Unknown thrown value that may contain a harness run.\n *\n * @example\n * ```ts\n * const partialRun = getHarnessRunFromError(error);\n *\n * if (partialRun) {\n *   console.log(toolCalls(partialRun.session));\n * }\n * ```\n */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";AAiVA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,kBAAkB,OAAgB,YAAoB;AACpE,SACE,UAAU,SACT,OAAO,UAAU,YAAY,OAAO,UAAU,eAC/C,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAGO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,QAAM,aAAa,YAAY,KAAK;AACpC,SAAO,eAAe,SAAY,aAAa,OAAO,KAAK;AAC7D;AAuCO,SAAS,cAKd,SACqC;AACrC,QAAM,UAA+C;AAAA,IACnD,MAAM,QAAQ;AAAA,IACd,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,SAAS,MAAM,QAAQ,IAAI;AAAA,QAC/B;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB,QAAQ,QAAQ;AAAA,QAChB,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,MACvB,CAAC;AAED,aAAO,oBAAoB,OAAO,QAAQ,OAAO;AAAA,IACnD;AAAA,EACF;AAEA,SAAO;AACT;AAoBO,SAAS,oBAKd,OACA,QACA,SACqB;AACrB,MAAI,aAAa,MAAM,GAAG;AACxB,QACE,WACA,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KACxC,CAAC,OAAO,WACR;AACA,aAAO;AAAA,QACL,GAAG;AAAA,QACH,WAAW,QAAQ;AAAA,MACrB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,OAAO;AACtB,QAAMA,aAAY,yBAAyB,OAAO,SAAS;AAC3D,QAAM,QAAQ,OAAO,SAAS,CAAC;AAC/B,QAAM,WACJ,OAAO,YACP,6BAA6B;AAAA,IAC3B;AAAA,IACA;AAAA,IACA,WAAAA;AAAA,EACF,CAAC;AACH,QAAM,WAAW,OAAO,WACpB,kBAAkB,OAAO,QAAQ,IACjC;AACJ,QAAM,YAAY;AAAA,IAChB,SAAS;AAAA,IACT,OAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,MACP;AAAA,MACA,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,MACrD,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;AAAA,MAC5C,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,IACA,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,IACzC;AAAA,IACA,GAAI,OAAO,UAAU,EAAE,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IACpD,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;AAAA,IACjC,QAAQ,sBAAsB,OAAO,MAAM;AAAA,EAC7C;AACF;AAEA,SAAS,6BAAqC;AAAA,EAC5C;AAAA,EACA;AAAA,EACA,WAAW;AACb,GAIwB;AACtB,QAAM,WAAgC;AAAA,IACpC;AAAA,MACE,MAAM;AAAA,MACN,SAAS,iBAAiB,KAAK;AAAA,IACjC;AAAA,EACF;AAEA,MAAI,WAAW,UAAa,oBAAoB,SAAS,GAAG;AAC1D,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,GAAI,WAAW,SAAY,EAAE,SAAS,iBAAiB,MAAM,EAAE,IAAI,CAAC;AAAA,MACpE,GAAI,oBAAoB,SAAS,IAC7B,EAAE,WAAW,oBAAoB,IACjC,CAAC;AAAA,IACP,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,OACkB;AAClB,UAAQ,SAAS,CAAC,GAAG,IAAI,CAAC,SAAS;AACjC,UAAM;AAAA,MACJ,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,UAAU;AAAA,MACV,GAAG;AAAA,IACL,IAAI;AACJ,UAAM,OAAO,2BAA2B,YAAY;AACpD,UAAM,SAAS,YAAY,SAAS;AACpC,UAAM,QAAQ,uBAAuB,QAAQ;AAC7C,UAAM,WAAW,cAAc,kBAAkB,WAAW,IAAI;AAEhE,WAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAI,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,MACzC,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;AAAA,MACzB,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,EACF,CAAC;AACH;AAEA,SAAS,2BACP,OACuC;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,YAAY,KAAK;AACpC,SAAO,cACL,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,IACvB,aACA;AACN;AAEA,SAAS,uBACP,OACqC;AACrC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,eAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,yBACP,kBACA,iBACA;AACA,QAAM,YAAY;AAAA,IAChB,GAAI,oBAAoB,CAAC;AAAA,IACzB,GAAI,kBAAkB,gBAAgB,eAAe,IAAI,CAAC;AAAA,EAC5D;AAEA,SAAO,OAAO,KAAK,SAAS,EAAE,SAAS,IAAI,YAAY;AACzD;AAEA,SAAS,sBACP,QACkC;AAClC,UAAQ,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU;AACnC,UAAM,aAAa,YAAY,KAAK;AAEpC,QACE,cACA,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,KACzB,OAAO,KAAK,UAAU,EAAE,SAAS,GACjC;AACA,aAAO;AAAA,IACT;AAEA,WAAO,eAAe,KAAK;AAAA,EAC7B,CAAC;AACH;AAcO,SAAS,UAAU,SAA8C;AACtE,SAAO,QAAQ,SAAS,QAAQ,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;AACtE;AAeO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAEA,SAAS,0BAA0B,SAA4B;AAC7D,SACE,QAAQ,YAAY,WACnB,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,KAAK,EAAE,SAAS;AAE5E;AAYO,SAAS,eAAe,SAA4B;AACzD,SAAO,eAAe,SAAS,QAAQ;AACzC;AAYO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAYO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAYO,SAAS,8BAA8B,SAA4B;AACxE,SAAO,CAAC,GAAG,kBAAkB,OAAO,CAAC,EAClC,QAAQ,EACR,KAAK,yBAAyB,GAAG;AACtC;AAYO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAiBO,SAAS,wBACd,OACA,KACiB;AACjB,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,gBAAgB;AAAA,EAClB,CAAC;AACH;AAgBO,SAAS,uBAAuB,OAAwC;AAC7E,MACE,SACA,OAAO,UAAU,YACjB,oBAAoB,SACpB,aAAc,MAAuC,cAAc,GACnE;AACA,WAAQ,MAAyC;AAAA,EACnD;AAEA,SAAO;AACT;AAGO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAGO,SAAS,wBACd,QACkC;AAClC,MACE,UACA,OAAO,WAAW,YAClB,MAAM,QAAS,OAAmC,MAAM,GACxD;AACA,WAAQ,OAAwD;AAAA,EAClE;AAEA,SAAO,CAAC;AACV;AAGO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;","names":["toolCalls"]}

package/dist/index.d.mts

@@ -1,10 +1,13 @@
 import * as vitest from 'vitest';
 import { TestAPI } from 'vitest';
 import { JsonValue, HarnessMetadata, Harness, HarnessRun, NormalizedSession, ToolCallRecord } from './harness.mjs';
-export { CreateHarnessOptions, CreateHarnessRunArgs, HarnessContext, HarnessResultLike, HarnessRunError, JsonPrimitive, MaybePromise, NormalizedMessage, SimpleHarnessResult, SimpleToolCallRecord, TimingSummary, UsageSummary, assistantMessages, attachHarnessRunToError, createHarness, getHarnessRunFromError, messagesByRole, normalizeHarnessRun, systemMessages, toolCalls, toolMessages, userMessages } from './harness.mjs';
+export { CreateHarnessOptions, CreateHarnessRunArgs, HarnessContext, HarnessResultLike, HarnessRunError, JsonPrimitive, MaybePromise, NormalizedMessage, SimpleHarnessResult, SimpleToolCallRecord, TimingSummary, UsageSummary, assistantMessages, attachHarnessRunToError, createHarness, getHarnessRunFromError, latestAssistantMessageContent, messagesByRole, normalizeHarnessRun, systemMessages, toolCalls, toolMessages, userMessages } from './harness.mjs';
 import { JudgeContext, Judge, JudgeResult, JudgeAssessFn, JudgeAssessor, JudgeAssessWithAssessorFn } from './judges/types.mjs';
 export { BoundJudgeAssessor, JudgeAssessorOptions, JudgeOptions } from './judges/types.mjs';
+import { JudgeHarness } from './judges/judgeHarness.mjs';
+export { CreateJudgeHarnessOptions, CreateJudgeHarnessRunOptions, JudgeHarnessInput, JudgeHarnessOutput, RunJudge, RunJudgeOptions, createJudgeHarness, runJudgeHarness } from './judges/judgeHarness.mjs';
 export { wrapText } from './wrapText.mjs';
+export { FactualityJudge, FactualityJudgeChoice, FactualityJudgeConfig, FactualityJudgeExpected, FactualityJudgeOptions, FactualityJudgePrompt, FactualityJudgeVerdict } from './judges/factualityJudge.mjs';
 export { StructuredOutputJudge, StructuredOutputJudgeConfig, StructuredOutputJudgeExpected, StructuredOutputJudgeOptions } from './judges/structuredOutputJudge.mjs';
 export { ToolCallJudge, ToolCallJudgeConfig, ToolCallJudgeExpectedTool, ToolCallJudgeOptions } from './judges/toolCallJudge.mjs';
 export { BaseMatcherConfig, FuzzyMatchOptions, MatchStrategy } from './internal/matchers.mjs';
@@ -30,6 +33,11 @@ type EvalTaskMeta = {
 type HarnessInput<THarness extends Harness<any, any, any>> = THarness extends Harness<infer TInput, any, any> ? TInput : unknown;
 type HarnessMetadataFor<THarness extends Harness<any, any, any>> = THarness extends Harness<any, any, infer TMetadata> ? TMetadata : HarnessMetadata;
 type HarnessOutput<THarness extends Harness<any, any, any>> = THarness extends Harness<any, infer TOutput, any> ? TOutput : JsonValue | undefined;
+type CreateJudgeConfig<TOptions extends JudgeContext<any, any, any, any> = JudgeContext> = {
+    name: string;
+    judgeHarness?: JudgeHarness;
+    assess: JudgeAssessFn<TOptions>;
+};
 declare const evalHarnessRunBrand: unique symbol;
 /**
  * Harness run returned by the fixture-backed `run(...)` API.
@@ -119,6 +127,8 @@ interface DescribeEvalOptions<TInput = unknown, TOutput extends JsonValue | unde
     harness: THarness;
     /** Automatic judges applied after each successful `run(...)`. */
     judges?: Array<Judge<JudgeContext<TInput, TOutput, TMetadata, THarness>>>;
+    /** Optional judge-side harness used only by judges that call `ctx.runJudge(...)`. */
+    judgeHarness?: JudgeHarness;
     /** Passing threshold for automatic suite-level judges. `null` disables fail-on-score. */
     judgeThreshold?: number | null;
     /** Skips the entire eval suite when the predicate returns true. */
@@ -136,7 +146,7 @@ type JudgeAssertionMetadata<TJudgeOptions extends JudgeContext<any, any, any, an
 type JudgeAssertionHarness<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
     harness: infer THarness;
 } ? Exclude<THarness, undefined> : Harness<JudgeAssertionInput<TJudgeOptions>, JudgeAssertionOutput<TJudgeOptions>, JudgeAssertionMetadata<TJudgeOptions>>;
-type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any, any> | "signal" | "threshold";
+type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any, any> | "judgeHarness" | "signal" | "threshold";
 type JudgeAssertionParams<TJudgeOptions extends JudgeContext<any, any, any, any>> = Omit<TJudgeOptions, JudgeAssertionReservedKey>;
 type RequiredKeys<T> = {
     [K in keyof T]-?: Record<string, never> extends Pick<T, K> ? never : K;
@@ -169,6 +179,8 @@ type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any, any
     session?: HarnessRun["session"];
     /** Override or provide the harness associated with the judge context. */
     harness?: JudgeAssertionHarness<TJudgeOptions>;
+    /** Override or provide the judge harness for judges that call `ctx.runJudge(...)`. */
+    judgeHarness?: JudgeHarness;
     /** Passing threshold for the explicit matcher. `null` records the score without failing. */
     threshold?: number | null;
 };
@@ -202,18 +214,42 @@ declare module "vitest" {
  *
  * @example
  * ```ts
+ * import { piAiHarness } from "@vitest-evals/harness-pi-ai";
+ * import { getModel } from "@mariozechner/pi-ai";
+ * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+ * import { expect } from "vitest";
+ * import {
+ *   describeEval,
+ *   FactualityJudge,
+ *   ToolCallJudge,
+ *   toolCalls,
+ * } from "vitest-evals";
+ * import { createRefundAgent } from "../src/refundAgent";
+ *
+ * const judgeHarness = piAiJudgeHarness({
+ *   model: getModel("anthropic", "claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ *
  * describeEval("refund agent", {
  *   harness: piAiHarness({
  *     agent: () => createRefundAgent(),
  *   }),
+ *   judgeHarness,
  *   judges: [ToolCallJudge()],
  * }, (it) => {
  *   it("approves a refundable invoice", async ({ run }) => {
- *     const result = await run("Refund invoice inv_123");
+ *     const result = await run("Refund invoice inv_123", {
+ *       metadata: {
+ *         expected: "Invoice inv_123 should be refunded.",
+ *       },
+ *     });
  *
  *     expect(result.output).toMatchObject({ status: "approved" });
  *     expect(toolCalls(result.session)).toHaveLength(2);
- *     await expect(result).toSatisfyJudge(FactualityJudge);
+ *     await expect(result).toSatisfyJudge(FactualityJudge(), {
+ *       threshold: 0.6,
+ *     });
  *   });
  * });
  * ```
@@ -262,23 +298,15 @@ declare function formatScores(scores: (JudgeResult & {
  * );
  * ```
  *
- * @example
- * ```ts
- * const ModelRubricJudge = createJudge(
- *   "ModelRubricJudge",
- *   openAiRubricAssessor,
- *   async ({ output }, assessor) => {
- *     const verdict = await assessor.assess(JSON.stringify(output));
- *
- *     return {
- *       score: verdict.passed ? 1 : 0,
- *       metadata: { rationale: verdict.rationale },
- *     };
- *   },
- * );
- * ```
+ * For LLM-backed judges, prefer the object form with `ctx.runJudge(...)` so
+ * provider-specific model configuration stays in the judge harness.
  */
 declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(name: string, assess: JudgeAssessFn<TOptions>): Judge<TOptions>;
+declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(config: CreateJudgeConfig<TOptions>): Judge<TOptions>;
+/**
+ * @deprecated Prefer `createJudge({ name, judgeHarness, assess })` and call
+ * `ctx.runJudge(...)` from LLM-backed judges.
+ */
 declare function createJudge<TOptions extends JudgeContext<any, any, any, any>, TInput, TOutput>(name: string, assessor: JudgeAssessor<TInput, TOutput>, assess: JudgeAssessWithAssessorFn<TOptions, TInput, TOutput>): Judge<TOptions>;
 
-export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, HarnessRun, JsonValue, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeResult, NormalizedSession, type ToSatisfyJudge, ToolCallRecord, createJudge, describeEval, formatScores };
+export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, HarnessRun, JsonValue, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeHarness, JudgeResult, NormalizedSession, type ToSatisfyJudge, ToolCallRecord, createJudge, describeEval, formatScores };

package/dist/index.d.ts

@@ -1,10 +1,13 @@
 import * as vitest from 'vitest';
 import { TestAPI } from 'vitest';
 import { JsonValue, HarnessMetadata, Harness, HarnessRun, NormalizedSession, ToolCallRecord } from './harness.js';
-export { CreateHarnessOptions, CreateHarnessRunArgs, HarnessContext, HarnessResultLike, HarnessRunError, JsonPrimitive, MaybePromise, NormalizedMessage, SimpleHarnessResult, SimpleToolCallRecord, TimingSummary, UsageSummary, assistantMessages, attachHarnessRunToError, createHarness, getHarnessRunFromError, messagesByRole, normalizeHarnessRun, systemMessages, toolCalls, toolMessages, userMessages } from './harness.js';
+export { CreateHarnessOptions, CreateHarnessRunArgs, HarnessContext, HarnessResultLike, HarnessRunError, JsonPrimitive, MaybePromise, NormalizedMessage, SimpleHarnessResult, SimpleToolCallRecord, TimingSummary, UsageSummary, assistantMessages, attachHarnessRunToError, createHarness, getHarnessRunFromError, latestAssistantMessageContent, messagesByRole, normalizeHarnessRun, systemMessages, toolCalls, toolMessages, userMessages } from './harness.js';
 import { JudgeContext, Judge, JudgeResult, JudgeAssessFn, JudgeAssessor, JudgeAssessWithAssessorFn } from './judges/types.js';
 export { BoundJudgeAssessor, JudgeAssessorOptions, JudgeOptions } from './judges/types.js';
+import { JudgeHarness } from './judges/judgeHarness.js';
+export { CreateJudgeHarnessOptions, CreateJudgeHarnessRunOptions, JudgeHarnessInput, JudgeHarnessOutput, RunJudge, RunJudgeOptions, createJudgeHarness, runJudgeHarness } from './judges/judgeHarness.js';
 export { wrapText } from './wrapText.js';
+export { FactualityJudge, FactualityJudgeChoice, FactualityJudgeConfig, FactualityJudgeExpected, FactualityJudgeOptions, FactualityJudgePrompt, FactualityJudgeVerdict } from './judges/factualityJudge.js';
 export { StructuredOutputJudge, StructuredOutputJudgeConfig, StructuredOutputJudgeExpected, StructuredOutputJudgeOptions } from './judges/structuredOutputJudge.js';
 export { ToolCallJudge, ToolCallJudgeConfig, ToolCallJudgeExpectedTool, ToolCallJudgeOptions } from './judges/toolCallJudge.js';
 export { BaseMatcherConfig, FuzzyMatchOptions, MatchStrategy } from './internal/matchers.js';
@@ -30,6 +33,11 @@ type EvalTaskMeta = {
 type HarnessInput<THarness extends Harness<any, any, any>> = THarness extends Harness<infer TInput, any, any> ? TInput : unknown;
 type HarnessMetadataFor<THarness extends Harness<any, any, any>> = THarness extends Harness<any, any, infer TMetadata> ? TMetadata : HarnessMetadata;
 type HarnessOutput<THarness extends Harness<any, any, any>> = THarness extends Harness<any, infer TOutput, any> ? TOutput : JsonValue | undefined;
+type CreateJudgeConfig<TOptions extends JudgeContext<any, any, any, any> = JudgeContext> = {
+    name: string;
+    judgeHarness?: JudgeHarness;
+    assess: JudgeAssessFn<TOptions>;
+};
 declare const evalHarnessRunBrand: unique symbol;
 /**
  * Harness run returned by the fixture-backed `run(...)` API.
@@ -119,6 +127,8 @@ interface DescribeEvalOptions<TInput = unknown, TOutput extends JsonValue | unde
     harness: THarness;
     /** Automatic judges applied after each successful `run(...)`. */
     judges?: Array<Judge<JudgeContext<TInput, TOutput, TMetadata, THarness>>>;
+    /** Optional judge-side harness used only by judges that call `ctx.runJudge(...)`. */
+    judgeHarness?: JudgeHarness;
     /** Passing threshold for automatic suite-level judges. `null` disables fail-on-score. */
     judgeThreshold?: number | null;
     /** Skips the entire eval suite when the predicate returns true. */
@@ -136,7 +146,7 @@ type JudgeAssertionMetadata<TJudgeOptions extends JudgeContext<any, any, any, an
 type JudgeAssertionHarness<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
     harness: infer THarness;
 } ? Exclude<THarness, undefined> : Harness<JudgeAssertionInput<TJudgeOptions>, JudgeAssertionOutput<TJudgeOptions>, JudgeAssertionMetadata<TJudgeOptions>>;
-type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any, any> | "signal" | "threshold";
+type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any, any> | "judgeHarness" | "signal" | "threshold";
 type JudgeAssertionParams<TJudgeOptions extends JudgeContext<any, any, any, any>> = Omit<TJudgeOptions, JudgeAssertionReservedKey>;
 type RequiredKeys<T> = {
     [K in keyof T]-?: Record<string, never> extends Pick<T, K> ? never : K;
@@ -169,6 +179,8 @@ type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any, any
     session?: HarnessRun["session"];
     /** Override or provide the harness associated with the judge context. */
     harness?: JudgeAssertionHarness<TJudgeOptions>;
+    /** Override or provide the judge harness for judges that call `ctx.runJudge(...)`. */
+    judgeHarness?: JudgeHarness;
     /** Passing threshold for the explicit matcher. `null` records the score without failing. */
     threshold?: number | null;
 };
@@ -202,18 +214,42 @@ declare module "vitest" {
  *
  * @example
  * ```ts
+ * import { piAiHarness } from "@vitest-evals/harness-pi-ai";
+ * import { getModel } from "@mariozechner/pi-ai";
+ * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+ * import { expect } from "vitest";
+ * import {
+ *   describeEval,
+ *   FactualityJudge,
+ *   ToolCallJudge,
+ *   toolCalls,
+ * } from "vitest-evals";
+ * import { createRefundAgent } from "../src/refundAgent";
+ *
+ * const judgeHarness = piAiJudgeHarness({
+ *   model: getModel("anthropic", "claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ *
  * describeEval("refund agent", {
  *   harness: piAiHarness({
  *     agent: () => createRefundAgent(),
  *   }),
+ *   judgeHarness,
  *   judges: [ToolCallJudge()],
  * }, (it) => {
  *   it("approves a refundable invoice", async ({ run }) => {
- *     const result = await run("Refund invoice inv_123");
+ *     const result = await run("Refund invoice inv_123", {
+ *       metadata: {
+ *         expected: "Invoice inv_123 should be refunded.",
+ *       },
+ *     });
  *
  *     expect(result.output).toMatchObject({ status: "approved" });
  *     expect(toolCalls(result.session)).toHaveLength(2);
- *     await expect(result).toSatisfyJudge(FactualityJudge);
+ *     await expect(result).toSatisfyJudge(FactualityJudge(), {
+ *       threshold: 0.6,
+ *     });
  *   });
  * });
  * ```
@@ -262,23 +298,15 @@ declare function formatScores(scores: (JudgeResult & {
  * );
  * ```
  *
- * @example
- * ```ts
- * const ModelRubricJudge = createJudge(
- *   "ModelRubricJudge",
- *   openAiRubricAssessor,
- *   async ({ output }, assessor) => {
- *     const verdict = await assessor.assess(JSON.stringify(output));
- *
- *     return {
- *       score: verdict.passed ? 1 : 0,
- *       metadata: { rationale: verdict.rationale },
- *     };
- *   },
- * );
- * ```
+ * For LLM-backed judges, prefer the object form with `ctx.runJudge(...)` so
+ * provider-specific model configuration stays in the judge harness.
  */
 declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(name: string, assess: JudgeAssessFn<TOptions>): Judge<TOptions>;
+declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(config: CreateJudgeConfig<TOptions>): Judge<TOptions>;
+/**
+ * @deprecated Prefer `createJudge({ name, judgeHarness, assess })` and call
+ * `ctx.runJudge(...)` from LLM-backed judges.
+ */
 declare function createJudge<TOptions extends JudgeContext<any, any, any, any>, TInput, TOutput>(name: string, assessor: JudgeAssessor<TInput, TOutput>, assess: JudgeAssessWithAssessorFn<TOptions, TInput, TOutput>): Judge<TOptions>;
 
-export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, HarnessRun, JsonValue, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeResult, NormalizedSession, type ToSatisfyJudge, ToolCallRecord, createJudge, describeEval, formatScores };
+export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, HarnessRun, JsonValue, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeHarness, JudgeResult, NormalizedSession, type ToSatisfyJudge, ToolCallRecord, createJudge, describeEval, formatScores };