vitest-evals 0.9.0 → 0.10.0

package/dist/judges/judgeHarness.js

@@ -0,0 +1,303 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/judges/judgeHarness.ts
+var judgeHarness_exports = {};
+__export(judgeHarness_exports, {
+  createJudgeHarness: () => createJudgeHarness,
+  createRunJudge: () => createRunJudge,
+  runJudgeHarness: () => runJudgeHarness
+});
+module.exports = __toCommonJS(judgeHarness_exports);
+
+// src/harness.ts
+function isJsonPrimitive(value) {
+  return value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+}
+function isJsonRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function normalizeJsonArray(value) {
+  return value.map((item) => {
+    const normalized = toJsonValue(item);
+    return normalized === void 0 ? null : normalized;
+  });
+}
+function normalizeJsonObject(value) {
+  const normalized = {};
+  for (const [key, entryValue] of Object.entries(value)) {
+    const entry = toJsonValue(entryValue);
+    if (entry !== void 0) {
+      normalized[key] = entry;
+    }
+  }
+  return normalized;
+}
+function toJsonValue(value) {
+  if (isJsonPrimitive(value)) {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return normalizeJsonArray(value);
+  }
+  if (isJsonRecord(value)) {
+    return normalizeJsonObject(value);
+  }
+  return void 0;
+}
+function normalizeRecord(value) {
+  return normalizeJsonObject(value);
+}
+function normalizeMetadata(value) {
+  const normalized = normalizeRecord(value);
+  return Object.keys(normalized).length > 0 ? normalized : void 0;
+}
+function normalizeContent(value) {
+  const normalized = toJsonValue(value);
+  return normalized !== void 0 ? normalized : String(value);
+}
+function createHarness(options) {
+  const harness = {
+    name: options.name,
+    run: async (input, context) => {
+      const result = await options.run({
+        input,
+        metadata: context.metadata,
+        signal: context.signal,
+        artifacts: context.artifacts,
+        setArtifact: context.setArtifact
+      });
+      return normalizeHarnessRun(input, result, context);
+    }
+  };
+  return harness;
+}
+function normalizeHarnessRun(input, result, context) {
+  if (isHarnessRun(result)) {
+    if (context && Object.keys(context.artifacts).length > 0 && !result.artifacts) {
+      return {
+        ...result,
+        artifacts: context.artifacts
+      };
+    }
+    return result;
+  }
+  const output = result.output;
+  const toolCalls = normalizeSimpleToolCalls(result.toolCalls);
+  const usage = result.usage ?? {};
+  const messages = result.messages ?? createDefaultSessionMessages({
+    input,
+    output,
+    toolCalls
+  });
+  const metadata = result.metadata ? normalizeMetadata(result.metadata) : void 0;
+  const artifacts = normalizeMergedArtifacts(
+    context?.artifacts,
+    result.artifacts
+  );
+  return {
+    session: {
+      messages,
+      ...usage.provider ? { provider: usage.provider } : {},
+      ...usage.model ? { model: usage.model } : {},
+      ...metadata ? { metadata } : {}
+    },
+    ...output !== void 0 ? { output } : {},
+    usage,
+    ...result.timings ? { timings: result.timings } : {},
+    ...artifacts ? { artifacts } : {},
+    errors: normalizeSimpleErrors(result.errors)
+  };
+}
+function createDefaultSessionMessages({
+  input,
+  output,
+  toolCalls: normalizedToolCalls
+}) {
+  const messages = [
+    {
+      role: "user",
+      content: normalizeContent(input)
+    }
+  ];
+  if (output !== void 0 || normalizedToolCalls.length > 0) {
+    messages.push({
+      role: "assistant",
+      ...output !== void 0 ? { content: normalizeContent(output) } : {},
+      ...normalizedToolCalls.length > 0 ? { toolCalls: normalizedToolCalls } : {}
+    });
+  }
+  return messages;
+}
+function normalizeSimpleToolCalls(calls) {
+  return (calls ?? []).map((call) => {
+    const {
+      arguments: rawArguments,
+      result: rawResult,
+      error: rawError,
+      metadata: rawMetadata,
+      ...toolCall
+    } = call;
+    const args = normalizeToolCallArguments(rawArguments);
+    const result = toJsonValue(rawResult);
+    const error = normalizeToolCallError(rawError);
+    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : void 0;
+    return {
+      ...toolCall,
+      ...args ? { arguments: args } : {},
+      ...result !== void 0 ? { result } : {},
+      ...error ? { error } : {},
+      ...metadata ? { metadata } : {}
+    };
+  });
+}
+function normalizeToolCallArguments(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  const normalized = toJsonValue(value);
+  return normalized && typeof normalized === "object" && !Array.isArray(normalized) ? normalized : void 0;
+}
+function normalizeToolCallError(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  const serialized = serializeError(value);
+  const { message, type, ...details } = serialized;
+  return {
+    ...details,
+    message: typeof message === "string" ? message : String(message),
+    ...typeof type === "string" ? { type } : {}
+  };
+}
+function normalizeMergedArtifacts(contextArtifacts, resultArtifacts) {
+  const artifacts = {
+    ...contextArtifacts ?? {},
+    ...resultArtifacts ? normalizeRecord(resultArtifacts) : {}
+  };
+  return Object.keys(artifacts).length > 0 ? artifacts : void 0;
+}
+function normalizeSimpleErrors(errors) {
+  return (errors ?? []).map((error) => {
+    const normalized = toJsonValue(error);
+    if (normalized && typeof normalized === "object" && !Array.isArray(normalized) && Object.keys(normalized).length > 0) {
+      return normalized;
+    }
+    return serializeError(error);
+  });
+}
+function messagesByRole(session, role) {
+  return session.messages.filter((message) => message.role === role);
+}
+function hasNonEmptyMessageContent(message) {
+  return message.content !== void 0 && (typeof message.content !== "string" || message.content.trim().length > 0);
+}
+function assistantMessages(session) {
+  return messagesByRole(session, "assistant");
+}
+function latestAssistantMessageContent(session) {
+  return [...assistantMessages(session)].reverse().find(hasNonEmptyMessageContent)?.content;
+}
+function isHarnessRun(value) {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const candidate = value;
+  return isNormalizedSession(candidate.session) && Boolean(candidate.usage) && typeof candidate.usage === "object" && !Array.isArray(candidate.usage) && Array.isArray(candidate.errors);
+}
+function isNormalizedSession(value) {
+  return Boolean(value) && typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
+}
+function serializeError(error) {
+  if (error instanceof Error) {
+    return {
+      type: error.name,
+      message: error.message
+    };
+  }
+  return {
+    type: "Error",
+    message: String(error)
+  };
+}
+
+// src/judges/judgeHarness.ts
+function createJudgeHarness(options) {
+  return createHarness({
+    name: options.name ?? "judge-harness",
+    run: async ({ input, signal, metadata }) => {
+      return normalizeJudgeHarnessResult(
+        await options.run(input, { signal, metadata })
+      );
+    }
+  });
+}
+async function runJudgeHarness(judgeHarness, input, options = {}) {
+  const artifacts = {};
+  const run = await judgeHarness.run(input, {
+    metadata: options.metadata ?? {},
+    signal: options.signal,
+    artifacts,
+    setArtifact: (name, value) => {
+      artifacts[name] = value;
+    }
+  });
+  return run.output !== void 0 ? run.output : resolveJudgeHarnessAssistantOutput(run);
+}
+function createRunJudge(judgeHarness, signal) {
+  if (!judgeHarness) {
+    return void 0;
+  }
+  return (input, options) => runJudgeHarness(judgeHarness, input, {
+    metadata: options?.metadata,
+    signal
+  });
+}
+function normalizeJudgeHarnessResult(result) {
+  if (isHarnessRun(result)) {
+    return result;
+  }
+  if (hasOutputField(result)) {
+    return {
+      output: normalizeJudgeHarnessOutput(result.output)
+    };
+  }
+  return {
+    output: normalizeJudgeHarnessOutput(result)
+  };
+}
+function hasOutputField(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 1 && "output" in value;
+}
+function normalizeJudgeHarnessOutput(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  return normalizeContent(value);
+}
+function resolveJudgeHarnessAssistantOutput(run) {
+  return latestAssistantMessageContent(run.session) ?? "";
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createJudgeHarness,
+  createRunJudge,
+  runJudgeHarness
+});
+//# sourceMappingURL=judgeHarness.js.map

package/dist/judges/judgeHarness.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/judges/judgeHarness.ts","../../src/harness.ts"],"sourcesContent":["import {\n  createHarness,\n  type Harness,\n  type HarnessContext,\n  type HarnessMetadata,\n  type HarnessResultLike,\n  type HarnessRun,\n  isHarnessRun,\n  latestAssistantMessageContent,\n  type JsonValue,\n  type MaybePromise,\n  normalizeContent,\n} from \"../harness\";\n\n/**\n * Provider-neutral prompt request issued by an LLM-backed judge.\n *\n * @example\n * ```ts\n * const input: JudgeHarnessInput = {\n *   system: \"Grade factual consistency.\",\n *   prompt: \"Compare the submitted answer with the reference answer.\",\n *   responseFormat: {\n *     type: \"json\",\n *   },\n * };\n * ```\n */\nexport type JudgeHarnessInput = {\n  /** Optional system prompt for the judge model. */\n  system?: string;\n  /** User prompt or instruction payload for the judge model. */\n  prompt: string;\n  /** Optional response-format hint for adapters that support structured output. */\n  responseFormat?: {\n    /** Requests a JSON-compatible response. */\n    type: \"json\";\n    /** Optional JSON Schema passed through to provider-specific adapters. */\n    schema?: JsonValue;\n  };\n};\n\n/** JSON-safe output returned by a judge harness. */\nexport type JudgeHarnessOutput = JsonValue | undefined;\n\n/**\n * Harness used by LLM-backed judges to issue judge-side prompts.\n *\n * This is separate from the application harness under test.\n *\n * @example\n * ```ts\n * const judgeHarness: JudgeHarness = createJudgeHarness({\n *   name: \"judge-model\",\n *   run: async ({ prompt }, { signal }) => {\n *     return callJudgeModel({ prompt, signal });\n *   },\n * });\n * ```\n */\nexport type JudgeHarness = Harness<\n  JudgeHarnessInput,\n  JudgeHarnessOutput,\n  HarnessMetadata\n>;\n\n/** Runtime options supplied when a judge calls `runJudge(...)`. */\nexport type RunJudgeOptions = {\n  /** Optional metadata forwarded to the judge harness run. */\n  metadata?: HarnessMetadata;\n};\n\n/**\n * Curried judge-harness runner available inside `JudgeContext`.\n *\n * @example\n * ```ts\n * const verdict = await ctx.runJudge?.({\n *   prompt: \"Return a JSON verdict.\",\n *   responseFormat: { type: \"json\" },\n * });\n * ```\n */\nexport type RunJudge = (\n  input: JudgeHarnessInput,\n  options?: RunJudgeOptions,\n) => Promise<JudgeHarnessOutput>;\n\n/** Runtime options passed to `createJudgeHarness(...)` callbacks. */\nexport type CreateJudgeHarnessRunOptions = {\n  /** Abort signal from the current eval run when available. */\n  signal?: AbortSignal;\n  /** Metadata for this judge-harness run. */\n  metadata: Readonly<HarnessMetadata>;\n};\n\n/**\n * Configuration for `createJudgeHarness(...)`.\n *\n * @example\n * ```ts\n * const judgeHarness = createJudgeHarness({\n *   name: \"custom-judge\",\n *   run: async ({ system, prompt }, { signal }) => {\n *     return callProvider({ system, prompt, signal });\n *   },\n * });\n * ```\n */\nexport type CreateJudgeHarnessOptions = {\n  /** Stable harness name used in diagnostics. */\n  name?: string;\n  /**\n   * Runs one provider-specific judge prompt.\n   *\n   * Return a JSON-safe value, a raw provider value to normalize, a lightweight\n   * `{ output }` result, or a full normalized `HarnessRun`.\n   */\n  run: (\n    input: JudgeHarnessInput,\n    options: CreateJudgeHarnessRunOptions,\n  ) => MaybePromise<unknown>;\n};\n\n/**\n * Creates a judge harness from a provider-specific prompt callback.\n *\n * @param options - Harness name plus the callback that issues the judge prompt.\n *\n * @example\n * ```ts\n * const judgeHarness = createJudgeHarness({\n *   run: async ({ prompt }) => callJudgeModel(prompt),\n * });\n * ```\n */\nexport function createJudgeHarness(\n  options: CreateJudgeHarnessOptions,\n): JudgeHarness {\n  return createHarness({\n    name: options.name ?? \"judge-harness\",\n    run: async ({ input, signal, metadata }) => {\n      return normalizeJudgeHarnessResult(\n        await options.run(input, { signal, metadata }),\n      );\n    },\n  });\n}\n\n/**\n * Runs a judge harness with eval-scoped context already supplied.\n *\n * @param judgeHarness - Judge-side harness configured on the matcher, judge, or suite.\n * @param input - Provider-neutral judge prompt request.\n * @param options - Run-scoped metadata and abort signal.\n */\nexport async function runJudgeHarness(\n  judgeHarness: JudgeHarness,\n  input: JudgeHarnessInput,\n  options: RunJudgeOptions & { signal?: AbortSignal } = {},\n): Promise<JudgeHarnessOutput> {\n  const artifacts: HarnessContext[\"artifacts\"] = {};\n  const run = await judgeHarness.run(input, {\n    metadata: options.metadata ?? {},\n    signal: options.signal,\n    artifacts,\n    setArtifact: (name, value) => {\n      artifacts[name] = value;\n    },\n  });\n\n  return run.output !== undefined\n    ? run.output\n    : resolveJudgeHarnessAssistantOutput(run);\n}\n\n/** Binds a judge harness to the current eval run context. */\nexport function createRunJudge(\n  judgeHarness: JudgeHarness | undefined,\n  signal?: AbortSignal,\n): RunJudge | undefined {\n  if (!judgeHarness) {\n    return undefined;\n  }\n\n  return (input, options) =>\n    runJudgeHarness(judgeHarness, input, {\n      metadata: options?.metadata,\n      signal,\n    });\n}\n\nfunction normalizeJudgeHarnessResult(\n  result: Awaited<ReturnType<CreateJudgeHarnessOptions[\"run\"]>>,\n): HarnessResultLike<JudgeHarnessOutput> {\n  if (isHarnessRun(result)) {\n    return result as HarnessRun<JudgeHarnessOutput>;\n  }\n\n  if (hasOutputField(result)) {\n    return {\n      output: normalizeJudgeHarnessOutput(result.output),\n    };\n  }\n\n  return {\n    output: normalizeJudgeHarnessOutput(result),\n  };\n}\n\nfunction hasOutputField(value: unknown): value is { output?: unknown } {\n  return (\n    value !== null &&\n    typeof value === \"object\" &&\n    !Array.isArray(value) &&\n    Object.keys(value).length === 1 &&\n    \"output\" in value\n  );\n}\n\nfunction normalizeJudgeHarnessOutput(value: unknown): JudgeHarnessOutput {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  return normalizeContent(value);\n}\n\nfunction resolveJudgeHarnessAssistantOutput(\n  run: HarnessRun<JudgeHarnessOutput>,\n): JudgeHarnessOutput {\n  return latestAssistantMessageContent(run.session) ?? \"\";\n}\n","/** Primitive scalar values allowed in normalized JSON-safe eval data. */\nexport type JsonPrimitive = string | number | boolean | null;\n\n/** JSON-safe value shape used by normalized sessions, artifacts, and errors. */\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\n/**\n * Normalized record for one tool call observed during a harness run.\n *\n * @example\n * ```ts\n * const call: ToolCallRecord = {\n *   name: \"lookupInvoice\",\n *   arguments: { invoiceId: \"inv_123\" },\n *   result: { refundable: true },\n * };\n * ```\n */\nexport type ToolCallRecord = {\n  /** Provider or runtime tool-call id when one is available. */\n  id?: string;\n  /** Tool name as exposed to the agent or application runtime. */\n  name: string;\n  /** JSON-safe tool arguments after provider/runtime normalization. */\n  arguments?: Record<string, JsonValue>;\n  /** JSON-safe tool result returned by the application tool. */\n  result?: JsonValue;\n  /** Normalized tool error when execution failed. */\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  /** ISO timestamp for the start of tool execution. */\n  startedAt?: string;\n  /** ISO timestamp for the end of tool execution. */\n  finishedAt?: string;\n  /** Tool execution duration in milliseconds. */\n  durationMs?: number;\n  /** Extra JSON-safe tool metadata for reporters and custom judges. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * Normalized message recorded in a harness session transcript.\n *\n * @example\n * ```ts\n * const message: NormalizedMessage = {\n *   role: \"assistant\",\n *   content: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\" }],\n * };\n * ```\n */\nexport type NormalizedMessage = {\n  /** Transcript role for the normalized message. */\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  /** JSON-safe message content. */\n  content?: JsonValue;\n  /** Tool calls associated with this message. */\n  toolCalls?: ToolCallRecord[];\n  /** Extra JSON-safe message metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * Provider usage summary attached to a normalized harness run.\n *\n * @example\n * ```ts\n * const usage: UsageSummary = {\n *   provider: \"openai\",\n *   model: \"gpt-4o-mini\",\n *   inputTokens: 212,\n *   outputTokens: 48,\n *   totalTokens: 260,\n * };\n * ```\n */\nexport type UsageSummary = {\n  /** Provider that served the application run. */\n  provider?: string;\n  /** Model used for the application run. */\n  model?: string;\n  /** Input, prompt, or request tokens consumed by the run. */\n  inputTokens?: number;\n  /** Output or completion tokens produced by the run. */\n  outputTokens?: number;\n  /** Reasoning tokens reported by providers that expose them. */\n  reasoningTokens?: number;\n  /** Total token count reported by the provider or adapter. */\n  totalTokens?: number;\n  /** Count of tool calls observed during the run. */\n  toolCalls?: number;\n  /** Retry count observed during the run. */\n  retries?: number;\n  /** Provider-specific JSON-safe usage details. Cost estimates belong here. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/** Timing summary attached to a normalized harness run. */\nexport type TimingSummary = {\n  /** End-to-end run duration in milliseconds. */\n  totalMs?: number;\n  /** Extra JSON-safe timing metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\n/**\n * JSON-serializable transcript produced by the system under test.\n *\n * @example\n * ```ts\n * const session: NormalizedSession = {\n *   provider: \"openai\",\n *   model: \"gpt-4o-mini\",\n *   messages: [\n *     { role: \"user\", content: \"Refund invoice inv_123\" },\n *     { role: \"assistant\", content: { status: \"approved\" } },\n *   ],\n * };\n * ```\n */\nexport type NormalizedSession = {\n  /** Ordered normalized transcript messages. */\n  messages: NormalizedMessage[];\n  /** Provider that produced the session when known. */\n  provider?: string;\n  /** Model that produced the session when known. */\n  model?: string;\n  /** Extra JSON-safe session metadata. */\n  metadata?: Record<string, JsonValue>;\n};\n\ntype OutputField<TOutput extends JsonValue | undefined> =\n  undefined extends TOutput ? { output?: TOutput } : { output: TOutput };\n\n/**\n * Normalized result returned by every harness execution.\n *\n * @example\n * ```ts\n * const run: HarnessRun<{ status: \"approved\" }> = {\n *   output: { status: \"approved\" },\n *   session: {\n *     messages: [\n *       { role: \"user\", content: \"Refund invoice inv_123\" },\n *       { role: \"assistant\", content: { status: \"approved\" } },\n *     ],\n *   },\n *   usage: { totalTokens: 260 },\n *   errors: [],\n * };\n * ```\n */\nexport type HarnessRun<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = OutputField<TOutput> & {\n  /** Normalized transcript and provider/session metadata. */\n  session: NormalizedSession;\n  /** Stable provider usage units such as tokens, tools, and retries. */\n  usage: UsageSummary;\n  /** Optional timing summary for the run. */\n  timings?: TimingSummary;\n  /** JSON-safe run artifacts captured by the harness or test context. */\n  artifacts?: Record<string, JsonValue>;\n  /** Normalized errors captured during execution. */\n  errors: Array<Record<string, JsonValue>>;\n};\n\n/** Error value with an attached partial or complete normalized harness run. */\nexport type HarnessRunError = Error & {\n  /** Attached normalized harness run recovered by `getHarnessRunFromError(...)`. */\n  vitestEvalsRun: HarnessRun;\n};\n\n/** Per-run metadata shape accepted by harnesses and eval tests. */\nexport type HarnessMetadata = Record<string, unknown>;\n\n/**\n * Runtime context passed from the eval fixture into a harness run.\n *\n * @example\n * ```ts\n * const harness: Harness<string> = {\n *   name: \"refund-agent\",\n *   async run(input, context) {\n *     context.setArtifact(\"inputLength\", input.length);\n *\n *     return {\n *       output: undefined,\n *       session: { messages: [{ role: \"user\", content: input }] },\n *       usage: {},\n *       errors: [],\n *     };\n *   },\n * };\n * ```\n */\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Per-run metadata passed through `run(input, { metadata })`. */\n  metadata: Readonly<TMetadata>;\n  /** Abort signal from Vitest when available. */\n  signal?: AbortSignal;\n  /** Mutable JSON-safe artifact bag shared with the harness. */\n  artifacts: Record<string, JsonValue>;\n  /** Stores one JSON-safe artifact on the current run. */\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\n/**\n * Adapter that executes the system under test and returns a normalized run.\n *\n * @example\n * ```ts\n * const harness: Harness<string, { status: \"approved\" | \"denied\" }> = {\n *   name: \"refund-agent\",\n *   async run(input, context) {\n *     return normalizeHarnessRun(input, await runRefundFlow(input), context);\n *   },\n * };\n * ```\n */\nexport type Harness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Stable harness name used in reports. */\n  name: string;\n  /** Executes the system under test and returns a normalized run. */\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun<TOutput>>;\n};\n\n/** Value or promise accepted by lightweight harness callbacks. */\nexport type MaybePromise<T> = T | Promise<T>;\n\n/** Lightweight tool-call record accepted by `createHarness(...)` results. */\nexport type SimpleToolCallRecord = Omit<\n  ToolCallRecord,\n  \"arguments\" | \"result\" | \"error\" | \"metadata\"\n> & {\n  /** Raw tool arguments accepted by `createHarness(...)` before normalization. */\n  arguments?: unknown;\n  /** Raw tool result accepted by `createHarness(...)` before normalization. */\n  result?: unknown;\n  /** Raw tool error accepted by `createHarness(...)` before normalization. */\n  error?: unknown;\n  /** Raw tool metadata accepted by `createHarness(...)` before normalization. */\n  metadata?: Record<string, unknown>;\n};\n\n/**\n * Lightweight result shape normalized by `createHarness(...)`.\n *\n * @example\n * ```ts\n * const result: SimpleHarnessResult<{ status: \"approved\" }> = {\n *   output: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\", arguments: { invoiceId: \"inv_123\" } }],\n *   usage: { totalTokens: 260 },\n * };\n * ```\n */\nexport type SimpleHarnessResult<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = OutputField<TOutput> & {\n  /** Pre-normalized transcript messages. When omitted, a default user/assistant transcript is created. */\n  messages?: NormalizedMessage[];\n  /** Lightweight tool-call records to normalize into the session. */\n  toolCalls?: SimpleToolCallRecord[];\n  /** Usage summary to attach to the run. */\n  usage?: UsageSummary;\n  /** Timing summary to attach to the run. */\n  timings?: TimingSummary;\n  /** Raw artifact values to normalize and merge into the run. */\n  artifacts?: Record<string, unknown>;\n  /** Raw session metadata to normalize into the session. */\n  metadata?: Record<string, unknown>;\n  /** Raw errors to normalize into the run. */\n  errors?: unknown[];\n};\n\n/** Either a complete normalized run or a lightweight result to normalize. */\nexport type HarnessResultLike<\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n> = HarnessRun<TOutput> | SimpleHarnessResult<TOutput>;\n\n/** Arguments passed to the `createHarness(...)` convenience callback. */\nexport type CreateHarnessRunArgs<TInput, TMetadata extends HarnessMetadata> = {\n  /** Original input passed to `run(input)`. */\n  input: TInput;\n  /** Read-only metadata passed to `run(input, { metadata })`. */\n  metadata: Readonly<TMetadata>;\n  /** Abort signal from Vitest when available. */\n  signal?: AbortSignal;\n  /** Mutable run artifact bag. */\n  artifacts: HarnessContext<TMetadata>[\"artifacts\"];\n  /** Stores one JSON-safe artifact on the current run. */\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n};\n\n/**\n * Options for creating a lightweight custom application harness.\n *\n * @example\n * ```ts\n * const options: CreateHarnessOptions<string, { status: \"approved\" }> = {\n *   name: \"refund-agent\",\n *   run: async ({ input }) => ({\n *     output: await classifyRefund(input),\n *   }),\n * };\n * ```\n */\nexport type CreateHarnessOptions<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  /** Stable harness name used in reports. */\n  name: string;\n  /** Executes application code and returns either a lightweight result or full `HarnessRun`. */\n  run: (\n    args: CreateHarnessRunArgs<TInput, TMetadata>,\n  ) => MaybePromise<HarnessResultLike<TOutput>>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  const normalized = toJsonValue(value);\n  return normalized !== undefined ? normalized : String(value);\n}\n\n/**\n * Creates a harness from the common \"run app code and return output\" shape.\n *\n * @param options - Harness name plus the callback that executes app code.\n *\n * @example\n * ```ts\n * import { createHarness } from \"vitest-evals\";\n *\n * export const refundHarness = createHarness<\n *   string,\n *   { status: \"approved\" | \"denied\" },\n *   { expected: { status: \"approved\" | \"denied\" } }\n * >({\n *   name: \"refund-agent\",\n *   run: async ({ input, metadata, setArtifact }) => {\n *     const result = await runRefundFlow(input, metadata);\n *     const output = { status: result.status };\n *\n *     setArtifact(\"case\", { expected: metadata.expected.status });\n *\n *     return {\n *       output,\n *       toolCalls: result.toolCalls,\n *       usage: { provider: \"openai\", model: \"gpt-4o-mini\" },\n *     };\n *   },\n * });\n * ```\n */\nexport function createHarness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n>(\n  options: CreateHarnessOptions<TInput, TOutput, TMetadata>,\n): Harness<TInput, TOutput, TMetadata>;\nexport function createHarness<\n  TInput = unknown,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n>(\n  options: CreateHarnessOptions<TInput, TOutput, TMetadata>,\n): Harness<TInput, TOutput, TMetadata> {\n  const harness: Harness<TInput, TOutput, TMetadata> = {\n    name: options.name,\n    run: async (input, context) => {\n      const result = await options.run({\n        input,\n        metadata: context.metadata,\n        signal: context.signal,\n        artifacts: context.artifacts,\n        setArtifact: context.setArtifact,\n      });\n\n      return normalizeHarnessRun(input, result, context);\n    },\n  };\n\n  return harness;\n}\n\n/**\n * Normalizes a lightweight harness result into the reporter-facing run shape.\n *\n * @param input - Original input passed to the harness.\n * @param result - Lightweight result or pre-normalized harness run.\n * @param context - Optional per-run context used to merge artifacts.\n *\n * @example\n * ```ts\n * const run = normalizeHarnessRun(\"Refund invoice inv_123\", {\n *   output: { status: \"approved\" },\n *   toolCalls: [{ name: \"lookupInvoice\", arguments: { invoiceId: \"inv_123\" } }],\n *   usage: { provider: \"openai\", model: \"gpt-4o-mini\" },\n * });\n *\n * expect(toolCalls(run.session)).toHaveLength(1);\n * ```\n */\nexport function normalizeHarnessRun<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TOutput extends JsonValue | undefined = JsonValue | undefined,\n>(\n  input: TInput,\n  result: HarnessResultLike<TOutput>,\n  context?: HarnessContext<TMetadata>,\n): HarnessRun<TOutput> {\n  if (isHarnessRun(result)) {\n    if (\n      context &&\n      Object.keys(context.artifacts).length > 0 &&\n      !result.artifacts\n    ) {\n      return {\n        ...result,\n        artifacts: context.artifacts,\n      };\n    }\n\n    return result;\n  }\n\n  const output = result.output;\n  const toolCalls = normalizeSimpleToolCalls(result.toolCalls);\n  const usage = result.usage ?? {};\n  const messages =\n    result.messages ??\n    createDefaultSessionMessages({\n      input,\n      output,\n      toolCalls,\n    });\n  const metadata = result.metadata\n    ? normalizeMetadata(result.metadata)\n    : undefined;\n  const artifacts = normalizeMergedArtifacts(\n    context?.artifacts,\n    result.artifacts,\n  );\n\n  return {\n    session: {\n      messages,\n      ...(usage.provider ? { provider: usage.provider } : {}),\n      ...(usage.model ? { model: usage.model } : {}),\n      ...(metadata ? { metadata } : {}),\n    },\n    ...(output !== undefined ? { output } : {}),\n    usage,\n    ...(result.timings ? { timings: result.timings } : {}),\n    ...(artifacts ? { artifacts } : {}),\n    errors: normalizeSimpleErrors(result.errors),\n  } as HarnessRun<TOutput>;\n}\n\nfunction createDefaultSessionMessages<TInput>({\n  input,\n  output,\n  toolCalls: normalizedToolCalls,\n}: {\n  input: TInput;\n  output: JsonValue | undefined;\n  toolCalls: ToolCallRecord[];\n}): NormalizedMessage[] {\n  const messages: NormalizedMessage[] = [\n    {\n      role: \"user\",\n      content: normalizeContent(input),\n    },\n  ];\n\n  if (output !== undefined || normalizedToolCalls.length > 0) {\n    messages.push({\n      role: \"assistant\",\n      ...(output !== undefined ? { content: normalizeContent(output) } : {}),\n      ...(normalizedToolCalls.length > 0\n        ? { toolCalls: normalizedToolCalls }\n        : {}),\n    });\n  }\n\n  return messages;\n}\n\nfunction normalizeSimpleToolCalls(\n  calls: SimpleToolCallRecord[] | undefined,\n): ToolCallRecord[] {\n  return (calls ?? []).map((call) => {\n    const {\n      arguments: rawArguments,\n      result: rawResult,\n      error: rawError,\n      metadata: rawMetadata,\n      ...toolCall\n    } = call;\n    const args = normalizeToolCallArguments(rawArguments);\n    const result = toJsonValue(rawResult);\n    const error = normalizeToolCallError(rawError);\n    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : undefined;\n\n    return {\n      ...toolCall,\n      ...(args ? { arguments: args } : {}),\n      ...(result !== undefined ? { result } : {}),\n      ...(error ? { error } : {}),\n      ...(metadata ? { metadata } : {}),\n    };\n  });\n}\n\nfunction normalizeToolCallArguments(\n  value: unknown,\n): Record<string, JsonValue> | undefined {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  const normalized = toJsonValue(value);\n  return normalized &&\n    typeof normalized === \"object\" &&\n    !Array.isArray(normalized)\n    ? normalized\n    : undefined;\n}\n\nfunction normalizeToolCallError(\n  value: unknown,\n): ToolCallRecord[\"error\"] | undefined {\n  if (value === undefined) {\n    return undefined;\n  }\n\n  const serialized = serializeError(value);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction normalizeMergedArtifacts(\n  contextArtifacts: Record<string, JsonValue> | undefined,\n  resultArtifacts: Record<string, unknown> | undefined,\n) {\n  const artifacts = {\n    ...(contextArtifacts ?? {}),\n    ...(resultArtifacts ? normalizeRecord(resultArtifacts) : {}),\n  };\n\n  return Object.keys(artifacts).length > 0 ? artifacts : undefined;\n}\n\nfunction normalizeSimpleErrors(\n  errors: unknown[] | undefined,\n): Array<Record<string, JsonValue>> {\n  return (errors ?? []).map((error) => {\n    const normalized = toJsonValue(error);\n\n    if (\n      normalized &&\n      typeof normalized === \"object\" &&\n      !Array.isArray(normalized) &&\n      Object.keys(normalized).length > 0\n    ) {\n      return normalized;\n    }\n\n    return serializeError(error);\n  });\n}\n\n/**\n * Flattens every recorded tool call from a normalized session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const names = toolCalls(result.session).map((call) => call.name);\n *\n * expect(names).toEqual([\"lookupInvoice\", \"createRefund\"]);\n * ```\n */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/**\n * Filters normalized session messages by role.\n *\n * @param session - Normalized session produced by a harness run.\n * @param role - Message role to keep.\n *\n * @example\n * ```ts\n * const assistantText = messagesByRole(result.session, \"assistant\")\n *   .map((message) => message.content)\n *   .join(\"\\n\");\n * ```\n */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\nfunction hasNonEmptyMessageContent(message: NormalizedMessage) {\n  return (\n    message.content !== undefined &&\n    (typeof message.content !== \"string\" || message.content.trim().length > 0)\n  );\n}\n\n/**\n * Returns every normalized system message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const systemPrompts = systemMessages(result.session);\n * ```\n */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/**\n * Returns every normalized user message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const firstPrompt = userMessages(result.session)[0]?.content;\n * ```\n */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/**\n * Returns every normalized assistant message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const finalAnswer = assistantMessages(result.session).at(-1)?.content;\n * ```\n */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/**\n * Returns the latest assistant message content, ignoring empty text messages.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const finalAnswer = latestAssistantMessageContent(result.session);\n * ```\n */\nexport function latestAssistantMessageContent(session: NormalizedSession) {\n  return [...assistantMessages(session)]\n    .reverse()\n    .find(hasNonEmptyMessageContent)?.content;\n}\n\n/**\n * Returns every normalized tool message from a session.\n *\n * @param session - Normalized session produced by a harness run.\n *\n * @example\n * ```ts\n * const toolOutputs = toolMessages(result.session).map((message) => message.content);\n * ```\n */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/**\n * Attaches a partial or complete harness run to an arbitrary thrown error.\n *\n * @param error - Thrown value to wrap.\n * @param run - Partial or complete normalized harness run to preserve.\n *\n * @example\n * ```ts\n * try {\n *   return await runAgent(input);\n * } catch (error) {\n *   throw attachHarnessRunToError(error, partialRun);\n * }\n * ```\n */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/**\n * Reads an attached harness run back off a previously wrapped error value.\n *\n * @param error - Unknown thrown value that may contain a harness run.\n *\n * @example\n * ```ts\n * const partialRun = getHarnessRunFromError(error);\n *\n * if (partialRun) {\n *   console.log(toolCalls(partialRun.session));\n * }\n * ```\n */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACiVA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAaO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,QAAM,aAAa,YAAY,KAAK;AACpC,SAAO,eAAe,SAAY,aAAa,OAAO,KAAK;AAC7D;AAuCO,SAAS,cAKd,SACqC;AACrC,QAAM,UAA+C;AAAA,IACnD,MAAM,QAAQ;AAAA,IACd,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,SAAS,MAAM,QAAQ,IAAI;AAAA,QAC/B;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB,QAAQ,QAAQ;AAAA,QAChB,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,MACvB,CAAC;AAED,aAAO,oBAAoB,OAAO,QAAQ,OAAO;AAAA,IACnD;AAAA,EACF;AAEA,SAAO;AACT;AAoBO,SAAS,oBAKd,OACA,QACA,SACqB;AACrB,MAAI,aAAa,MAAM,GAAG;AACxB,QACE,WACA,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KACxC,CAAC,OAAO,WACR;AACA,aAAO;AAAA,QACL,GAAG;AAAA,QACH,WAAW,QAAQ;AAAA,MACrB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,OAAO;AACtB,QAAM,YAAY,yBAAyB,OAAO,SAAS;AAC3D,QAAM,QAAQ,OAAO,SAAS,CAAC;AAC/B,QAAM,WACJ,OAAO,YACP,6BAA6B;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACH,QAAM,WAAW,OAAO,WACpB,kBAAkB,OAAO,QAAQ,IACjC;AACJ,QAAM,YAAY;AAAA,IAChB,SAAS;AAAA,IACT,OAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,MACP;AAAA,MACA,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,MACrD,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;AAAA,MAC5C,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,IACA,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,IACzC;AAAA,IACA,GAAI,OAAO,UAAU,EAAE,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IACpD,GAAI,YAAY,EAAE,UAAU,IAAI,CAAC;AAAA,IACjC,QAAQ,sBAAsB,OAAO,MAAM;AAAA,EAC7C;AACF;AAEA,SAAS,6BAAqC;AAAA,EAC5C;AAAA,EACA;AAAA,EACA,WAAW;AACb,GAIwB;AACtB,QAAM,WAAgC;AAAA,IACpC;AAAA,MACE,MAAM;AAAA,MACN,SAAS,iBAAiB,KAAK;AAAA,IACjC;AAAA,EACF;AAEA,MAAI,WAAW,UAAa,oBAAoB,SAAS,GAAG;AAC1D,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN,GAAI,WAAW,SAAY,EAAE,SAAS,iBAAiB,MAAM,EAAE,IAAI,CAAC;AAAA,MACpE,GAAI,oBAAoB,SAAS,IAC7B,EAAE,WAAW,oBAAoB,IACjC,CAAC;AAAA,IACP,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,OACkB;AAClB,UAAQ,SAAS,CAAC,GAAG,IAAI,CAAC,SAAS;AACjC,UAAM;AAAA,MACJ,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,UAAU;AAAA,MACV,GAAG;AAAA,IACL,IAAI;AACJ,UAAM,OAAO,2BAA2B,YAAY;AACpD,UAAM,SAAS,YAAY,SAAS;AACpC,UAAM,QAAQ,uBAAuB,QAAQ;AAC7C,UAAM,WAAW,cAAc,kBAAkB,WAAW,IAAI;AAEhE,WAAO;AAAA,MACL,GAAG;AAAA,MACH,GAAI,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,WAAW,SAAY,EAAE,OAAO,IAAI,CAAC;AAAA,MACzC,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;AAAA,MACzB,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,EACF,CAAC;AACH;AAEA,SAAS,2BACP,OACuC;AACvC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,YAAY,KAAK;AACpC,SAAO,cACL,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,IACvB,aACA;AACN;AAEA,SAAS,uBACP,OACqC;AACrC,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,eAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,yBACP,kBACA,iBACA;AACA,QAAM,YAAY;AAAA,IAChB,GAAI,oBAAoB,CAAC;AAAA,IACzB,GAAI,kBAAkB,gBAAgB,eAAe,IAAI,CAAC;AAAA,EAC5D;AAEA,SAAO,OAAO,KAAK,SAAS,EAAE,SAAS,IAAI,YAAY;AACzD;AAEA,SAAS,sBACP,QACkC;AAClC,UAAQ,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU;AACnC,UAAM,aAAa,YAAY,KAAK;AAEpC,QACE,cACA,OAAO,eAAe,YACtB,CAAC,MAAM,QAAQ,UAAU,KACzB,OAAO,KAAK,UAAU,EAAE,SAAS,GACjC;AACA,aAAO;AAAA,IACT;AAEA,WAAO,eAAe,KAAK;AAAA,EAC7B,CAAC;AACH;AA+BO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAEA,SAAS,0BAA0B,SAA4B;AAC7D,SACE,QAAQ,YAAY,WACnB,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,KAAK,EAAE,SAAS;AAE5E;AAwCO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAYO,SAAS,8BAA8B,SAA4B;AACxE,SAAO,CAAC,GAAG,kBAAkB,OAAO,CAAC,EAClC,QAAQ,EACR,KAAK,yBAAyB,GAAG;AACtC;AAwEO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAkBO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;;;ADnwBO,SAAS,mBACd,SACc;AACd,SAAO,cAAc;AAAA,IACnB,MAAM,QAAQ,QAAQ;AAAA,IACtB,KAAK,OAAO,EAAE,OAAO,QAAQ,SAAS,MAAM;AAC1C,aAAO;AAAA,QACL,MAAM,QAAQ,IAAI,OAAO,EAAE,QAAQ,SAAS,CAAC;AAAA,MAC/C;AAAA,IACF;AAAA,EACF,CAAC;AACH;AASA,eAAsB,gBACpB,cACA,OACA,UAAsD,CAAC,GAC1B;AAC7B,QAAM,YAAyC,CAAC;AAChD,QAAM,MAAM,MAAM,aAAa,IAAI,OAAO;AAAA,IACxC,UAAU,QAAQ,YAAY,CAAC;AAAA,IAC/B,QAAQ,QAAQ;AAAA,IAChB;AAAA,IACA,aAAa,CAAC,MAAM,UAAU;AAC5B,gBAAU,IAAI,IAAI;AAAA,IACpB;AAAA,EACF,CAAC;AAED,SAAO,IAAI,WAAW,SAClB,IAAI,SACJ,mCAAmC,GAAG;AAC5C;AAGO,SAAS,eACd,cACA,QACsB;AACtB,MAAI,CAAC,cAAc;AACjB,WAAO;AAAA,EACT;AAEA,SAAO,CAAC,OAAO,YACb,gBAAgB,cAAc,OAAO;AAAA,IACnC,UAAU,SAAS;AAAA,IACnB;AAAA,EACF,CAAC;AACL;AAEA,SAAS,4BACP,QACuC;AACvC,MAAI,aAAa,MAAM,GAAG;AACxB,WAAO;AAAA,EACT;AAEA,MAAI,eAAe,MAAM,GAAG;AAC1B,WAAO;AAAA,MACL,QAAQ,4BAA4B,OAAO,MAAM;AAAA,IACnD;AAAA,EACF;AAEA,SAAO;AAAA,IACL,QAAQ,4BAA4B,MAAM;AAAA,EAC5C;AACF;AAEA,SAAS,eAAe,OAA+C;AACrE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,CAAC,MAAM,QAAQ,KAAK,KACpB,OAAO,KAAK,KAAK,EAAE,WAAW,KAC9B,YAAY;AAEhB;AAEA,SAAS,4BAA4B,OAAoC;AACvE,MAAI,UAAU,QAAW;AACvB,WAAO;AAAA,EACT;AAEA,SAAO,iBAAiB,KAAK;AAC/B;AAEA,SAAS,mCACP,KACoB;AACpB,SAAO,8BAA8B,IAAI,OAAO,KAAK;AACvD;","names":[]}

package/dist/judges/judgeHarness.mjs

@@ -0,0 +1,274 @@
+// src/harness.ts
+function isJsonPrimitive(value) {
+  return value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+}
+function isJsonRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function normalizeJsonArray(value) {
+  return value.map((item) => {
+    const normalized = toJsonValue(item);
+    return normalized === void 0 ? null : normalized;
+  });
+}
+function normalizeJsonObject(value) {
+  const normalized = {};
+  for (const [key, entryValue] of Object.entries(value)) {
+    const entry = toJsonValue(entryValue);
+    if (entry !== void 0) {
+      normalized[key] = entry;
+    }
+  }
+  return normalized;
+}
+function toJsonValue(value) {
+  if (isJsonPrimitive(value)) {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return normalizeJsonArray(value);
+  }
+  if (isJsonRecord(value)) {
+    return normalizeJsonObject(value);
+  }
+  return void 0;
+}
+function normalizeRecord(value) {
+  return normalizeJsonObject(value);
+}
+function normalizeMetadata(value) {
+  const normalized = normalizeRecord(value);
+  return Object.keys(normalized).length > 0 ? normalized : void 0;
+}
+function normalizeContent(value) {
+  const normalized = toJsonValue(value);
+  return normalized !== void 0 ? normalized : String(value);
+}
+function createHarness(options) {
+  const harness = {
+    name: options.name,
+    run: async (input, context) => {
+      const result = await options.run({
+        input,
+        metadata: context.metadata,
+        signal: context.signal,
+        artifacts: context.artifacts,
+        setArtifact: context.setArtifact
+      });
+      return normalizeHarnessRun(input, result, context);
+    }
+  };
+  return harness;
+}
+function normalizeHarnessRun(input, result, context) {
+  if (isHarnessRun(result)) {
+    if (context && Object.keys(context.artifacts).length > 0 && !result.artifacts) {
+      return {
+        ...result,
+        artifacts: context.artifacts
+      };
+    }
+    return result;
+  }
+  const output = result.output;
+  const toolCalls = normalizeSimpleToolCalls(result.toolCalls);
+  const usage = result.usage ?? {};
+  const messages = result.messages ?? createDefaultSessionMessages({
+    input,
+    output,
+    toolCalls
+  });
+  const metadata = result.metadata ? normalizeMetadata(result.metadata) : void 0;
+  const artifacts = normalizeMergedArtifacts(
+    context?.artifacts,
+    result.artifacts
+  );
+  return {
+    session: {
+      messages,
+      ...usage.provider ? { provider: usage.provider } : {},
+      ...usage.model ? { model: usage.model } : {},
+      ...metadata ? { metadata } : {}
+    },
+    ...output !== void 0 ? { output } : {},
+    usage,
+    ...result.timings ? { timings: result.timings } : {},
+    ...artifacts ? { artifacts } : {},
+    errors: normalizeSimpleErrors(result.errors)
+  };
+}
+function createDefaultSessionMessages({
+  input,
+  output,
+  toolCalls: normalizedToolCalls
+}) {
+  const messages = [
+    {
+      role: "user",
+      content: normalizeContent(input)
+    }
+  ];
+  if (output !== void 0 || normalizedToolCalls.length > 0) {
+    messages.push({
+      role: "assistant",
+      ...output !== void 0 ? { content: normalizeContent(output) } : {},
+      ...normalizedToolCalls.length > 0 ? { toolCalls: normalizedToolCalls } : {}
+    });
+  }
+  return messages;
+}
+function normalizeSimpleToolCalls(calls) {
+  return (calls ?? []).map((call) => {
+    const {
+      arguments: rawArguments,
+      result: rawResult,
+      error: rawError,
+      metadata: rawMetadata,
+      ...toolCall
+    } = call;
+    const args = normalizeToolCallArguments(rawArguments);
+    const result = toJsonValue(rawResult);
+    const error = normalizeToolCallError(rawError);
+    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : void 0;
+    return {
+      ...toolCall,
+      ...args ? { arguments: args } : {},
+      ...result !== void 0 ? { result } : {},
+      ...error ? { error } : {},
+      ...metadata ? { metadata } : {}
+    };
+  });
+}
+function normalizeToolCallArguments(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  const normalized = toJsonValue(value);
+  return normalized && typeof normalized === "object" && !Array.isArray(normalized) ? normalized : void 0;
+}
+function normalizeToolCallError(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  const serialized = serializeError(value);
+  const { message, type, ...details } = serialized;
+  return {
+    ...details,
+    message: typeof message === "string" ? message : String(message),
+    ...typeof type === "string" ? { type } : {}
+  };
+}
+function normalizeMergedArtifacts(contextArtifacts, resultArtifacts) {
+  const artifacts = {
+    ...contextArtifacts ?? {},
+    ...resultArtifacts ? normalizeRecord(resultArtifacts) : {}
+  };
+  return Object.keys(artifacts).length > 0 ? artifacts : void 0;
+}
+function normalizeSimpleErrors(errors) {
+  return (errors ?? []).map((error) => {
+    const normalized = toJsonValue(error);
+    if (normalized && typeof normalized === "object" && !Array.isArray(normalized) && Object.keys(normalized).length > 0) {
+      return normalized;
+    }
+    return serializeError(error);
+  });
+}
+function messagesByRole(session, role) {
+  return session.messages.filter((message) => message.role === role);
+}
+function hasNonEmptyMessageContent(message) {
+  return message.content !== void 0 && (typeof message.content !== "string" || message.content.trim().length > 0);
+}
+function assistantMessages(session) {
+  return messagesByRole(session, "assistant");
+}
+function latestAssistantMessageContent(session) {
+  return [...assistantMessages(session)].reverse().find(hasNonEmptyMessageContent)?.content;
+}
+function isHarnessRun(value) {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const candidate = value;
+  return isNormalizedSession(candidate.session) && Boolean(candidate.usage) && typeof candidate.usage === "object" && !Array.isArray(candidate.usage) && Array.isArray(candidate.errors);
+}
+function isNormalizedSession(value) {
+  return Boolean(value) && typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
+}
+function serializeError(error) {
+  if (error instanceof Error) {
+    return {
+      type: error.name,
+      message: error.message
+    };
+  }
+  return {
+    type: "Error",
+    message: String(error)
+  };
+}
+
+// src/judges/judgeHarness.ts
+function createJudgeHarness(options) {
+  return createHarness({
+    name: options.name ?? "judge-harness",
+    run: async ({ input, signal, metadata }) => {
+      return normalizeJudgeHarnessResult(
+        await options.run(input, { signal, metadata })
+      );
+    }
+  });
+}
+async function runJudgeHarness(judgeHarness, input, options = {}) {
+  const artifacts = {};
+  const run = await judgeHarness.run(input, {
+    metadata: options.metadata ?? {},
+    signal: options.signal,
+    artifacts,
+    setArtifact: (name, value) => {
+      artifacts[name] = value;
+    }
+  });
+  return run.output !== void 0 ? run.output : resolveJudgeHarnessAssistantOutput(run);
+}
+function createRunJudge(judgeHarness, signal) {
+  if (!judgeHarness) {
+    return void 0;
+  }
+  return (input, options) => runJudgeHarness(judgeHarness, input, {
+    metadata: options?.metadata,
+    signal
+  });
+}
+function normalizeJudgeHarnessResult(result) {
+  if (isHarnessRun(result)) {
+    return result;
+  }
+  if (hasOutputField(result)) {
+    return {
+      output: normalizeJudgeHarnessOutput(result.output)
+    };
+  }
+  return {
+    output: normalizeJudgeHarnessOutput(result)
+  };
+}
+function hasOutputField(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 1 && "output" in value;
+}
+function normalizeJudgeHarnessOutput(value) {
+  if (value === void 0) {
+    return void 0;
+  }
+  return normalizeContent(value);
+}
+function resolveJudgeHarnessAssistantOutput(run) {
+  return latestAssistantMessageContent(run.session) ?? "";
+}
+export {
+  createJudgeHarness,
+  createRunJudge,
+  runJudgeHarness
+};
+//# sourceMappingURL=judgeHarness.mjs.map