vitest-evals 0.9.0-beta.0 → 0.9.0-beta.2

package/README.md

@@ -14,6 +14,8 @@ Install a first-party harness package for the runtime you want to test:
 npm install -D @vitest-evals/harness-pi-ai
 # or
 npm install -D @vitest-evals/harness-ai-sdk
+# or
+npm install -D @vitest-evals/harness-openai-agents
 ```
 
 ## Core Model
@@ -25,9 +27,13 @@ npm install -D @vitest-evals/harness-ai-sdk
 - the returned `result.output` is the app-facing value you assert on directly
 - the returned `result.session` is the canonical JSON-serializable trace for
   reporting, replay, tool assertions, and judges
-- per-run judge inputs should usually live under `metadata`
+- scenario-specific judge criteria can live in `inputValue`; use `metadata` for
+  per-run expectations or harness configuration that are not part of the
+  scenario payload
 - suite-level `judges` are optional and run automatically after each `run(...)`
 - suite-level `judgeThreshold` controls fail-on-score for those automatic judges
+- every judge receives `JudgeContext`, including the configured `harness` with
+  its required `prompt` function
 - explicit judge assertions use
   `await expect(result).toSatisfyJudge(judge, context)`
 
@@ -42,7 +48,7 @@ import {
   toolCalls,
   type JudgeContext,
 } from "vitest-evals";
-import { createRefundAgent } from "../src/refundAgent";
+import { createRefundAgent, judgePrompt } from "../src/refundAgent";
 
 type RefundEvalMetadata = {
   expectedStatus: "approved" | "denied";
@@ -76,6 +82,7 @@ describeEval(
   {
     harness: piAiHarness({
       createAgent: () => createRefundAgent(),
+      prompt: judgePrompt,
     }),
     judges: [FactualityJudge],
   },
@@ -141,10 +148,173 @@ The harness owns normalization, diagnostics, tool capture, replay plumbing, and
 reporter-facing artifacts. Your app just needs one runtime seam where those
 wrapped pieces can be injected.
 
+Replay opt-in belongs on the harness, via `toolReplay`, while replay mode and
+recording directory can live in Vitest environment config. Tool definitions
+should stay free of VCR policy.
+
 For the Pi-specific harness, output/session/usage normalization should usually
 be inferred automatically. Treat low-level normalization callbacks as an escape
 hatch, not part of the primary authoring path.
 
+For OpenAI Agents SDK apps, use
+`@vitest-evals/harness-openai-agents` with an existing `Agent` or
+`createAgent()` factory and a `Runner` / `createRunner()` callback. The harness
+calls `Runner.run(agent, input, options)` by default and exposes the same
+normalization and replay hooks when the app needs a custom entrypoint or
+structured domain output mapping.
+
+## Custom App Harnesses
+
+First-party harness packages are conveniences, not the only supported path. If
+you need to test a full application flow, define a harness that runs your app
+through its normal entrypoint and returns a normalized `HarnessRun`. The same
+harness should also expose `prompt`, which LLM-backed judges can reuse through
+`JudgeContext.harness.prompt`.
+
+```ts
+import {
+  describeEval,
+  namedJudge,
+  type JudgeContext,
+} from "vitest-evals";
+import {
+  normalizeContent,
+  normalizeMetadata,
+  toJsonValue,
+  type Harness,
+  type HarnessRun,
+} from "vitest-evals/harness";
+
+type AppEvent = {
+  type: string;
+  payload: Record<string, unknown>;
+};
+
+type AppEvalInput = {
+  events: AppEvent[];
+  criteria: {
+    contract: string;
+    pass: string[];
+    fail?: string[];
+  };
+};
+
+const appHarness: Harness<AppEvalInput> = {
+  name: "custom-app",
+  prompt: (input, options) => promptJudgeModel(input, options),
+  run: async (input, context): Promise<HarnessRun> => {
+    const result = await replayAppEvents(input.events, {
+      signal: context.signal,
+    });
+    const output = {
+      replies: result.replies,
+      sideEffects: result.sideEffects,
+    };
+
+    return {
+      output: toJsonValue(output),
+      session: {
+        messages: [
+          ...input.events.map((event) => ({
+            role: "user" as const,
+            content: normalizeContent(event),
+          })),
+          ...result.replies.map((reply) => ({
+            role: "assistant" as const,
+            content: normalizeContent(reply.text),
+            metadata: normalizeMetadata({
+              target: reply.target,
+            }),
+          })),
+        ],
+        outputText: result.replies.map((reply) => reply.text).join("\n\n"),
+        metadata: normalizeMetadata({
+          replyCount: result.replies.length,
+        }),
+      },
+      usage: {},
+      artifacts:
+        Object.keys(context.artifacts).length > 0
+          ? context.artifacts
+          : undefined,
+      errors: [],
+    };
+  },
+};
+
+const AppRubricJudge = namedJudge(
+  "AppRubricJudge",
+  async (
+    ctx: JudgeContext<AppEvalInput, Record<string, unknown>, typeof appHarness>,
+  ) => {
+    const verdict = await ctx.harness.prompt(
+      formatRubricPrompt({
+        output: ctx.output,
+        criteria: ctx.inputValue.criteria,
+      }),
+      {
+        metadata: {
+          judge: "AppRubricJudge",
+        },
+      },
+    );
+
+    return parseRubricVerdict(verdict);
+  },
+);
+
+describeEval(
+  "app behavior",
+  {
+    harness: appHarness,
+    judges: [AppRubricJudge],
+    judgeThreshold: 0.75,
+  },
+  (it) => {
+    it("handles an event flow", async ({ run }) => {
+      await run({
+        events: [
+          {
+            type: "message.created",
+            payload: {
+              text: "Summarize the current incident.",
+            },
+          },
+        ],
+        criteria: {
+          contract: "The app posts one user-visible incident summary.",
+          pass: ["The reply names the incident status."],
+          fail: ["The reply exposes internal metadata."],
+        },
+      });
+    });
+  },
+);
+```
+
+Use `Harness.run(...)` for the application under test and `Harness.prompt(...)`
+for judge model calls. Calling `ctx.harness.run(...)` from inside a judge runs
+the application a second time, so reserve that for judges that intentionally
+need a second execution. Put criteria on `inputValue` when they are part of the
+scenario itself; use per-run `metadata` for harness configuration or
+expectations that are not part of the scenario payload. `session.outputText` is
+the canonical text sent to judges, so define it deliberately when your app
+returns structured artifacts.
+
+Provider setup and rubric parsing stay in your harness and judge. The core
+package only requires the judge to return a `JudgeResult` with a score and
+optional metadata.
+
+Automatic suite-level judges are a good fit when every `run(...)` should get
+the same scoring. For cases where only some runs need an LLM judge, keep the
+suite free of automatic judges and use an explicit matcher:
+
+```ts
+await expect(result).toSatisfyJudge(AppRubricJudge, {
+  threshold: 0.75,
+});
+```
+
 ## Judge Matchers
 
 Use the matcher when a judge should behave like a normal Vitest assertion.
@@ -186,12 +356,24 @@ const FactualityJudge = namedJudge(
 );
 ```
 
-For a `HarnessRun`, `toSatisfyJudge(...)` passes `result.output` as `output`.
-For raw values or normalized sessions, the matcher infers the best available
-output from the received value. Structured or programmatic result checks should
-usually assert on `result.output` directly. When a judge needs richer context,
-type it with `JudgeContext` and read `inputValue`, `metadata`, `toolCalls`, or
-`session` from there.
+LLM-backed judges can reuse the suite harness prompt by calling
+`harness.prompt(...)`. `vitest-evals` does not prescribe a rubric schema,
+scoring scale, model provider, or parser; those stay in the judge. Calling
+`harness.run(...)` from a judge executes the application again, so use that
+only when a second run is intentional.
+
+For an `EvalHarnessRun` returned by fixture `run(...)`,
+`toSatisfyJudge(...)` uses the run's canonical text output and reuses the
+registered input, metadata, and harness prompt. Inside an eval test,
+matcher calls on registered raw output or session objects reuse that exact run
+context; raw output values are serialized as the judge `output`, so
+`expect(result.output).toSatisfyJudge(judge)` stays concise. Other raw values
+fall back to the current test's most recent `run(...)` context. For
+manually-created runs or values outside an eval context, pass any required
+`inputValue`, `metadata`, or `harness` in matcher options. Structured or
+programmatic result checks should usually assert on `result.output` directly.
+When a judge needs richer normalized context or the configured suite harness,
+type it with `JudgeContext`.
 
 When you only need deterministic contract checks, built-ins such as
 `StructuredOutputJudge()` and `ToolCallJudge()` are still available. The primary

package/dist/harness.d.mts

@@ -54,14 +54,13 @@ type HarnessRun = {
     artifacts?: Record<string, JsonValue>;
     errors: Array<Record<string, JsonValue>>;
 };
+/** Optional provider-facing hints for harness prompt calls. */
 type HarnessPromptOptions = {
     system?: string;
     metadata?: Record<string, JsonValue>;
 };
+/** Provider-agnostic prompt seam that judges can reuse from a harness. */
 type HarnessPrompt = (input: string, options?: HarnessPromptOptions) => Promise<string>;
-type HarnessRuntime = {
-    prompt: HarnessPrompt;
-};
 type HarnessRunError = Error & {
     vitestEvalsRun: HarnessRun;
 };
@@ -77,7 +76,8 @@ type HarnessContext<TMetadata extends HarnessMetadata = HarnessMetadata> = {
 };
 type Harness<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = {
     name: string;
-    prompt?: HarnessPrompt;
+    /** Prompt seam reused by LLM-backed judges. */
+    prompt: HarnessPrompt;
     run: (input: TInput, context: HarnessContext<TMetadata>) => Promise<HarnessRun>;
 };
 /** Returns true when a value exposes a callable method with the given name. */
@@ -115,4 +115,4 @@ declare function resolveHarnessRunErrors(result: unknown): Array<Record<string,
 /** Serializes an arbitrary thrown value into the normalized error shape. */
 declare function serializeError(error: unknown): Record<string, JsonValue>;
 
-export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type HarnessRuntime, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };
+export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };

package/dist/harness.d.ts

@@ -54,14 +54,13 @@ type HarnessRun = {
     artifacts?: Record<string, JsonValue>;
     errors: Array<Record<string, JsonValue>>;
 };
+/** Optional provider-facing hints for harness prompt calls. */
 type HarnessPromptOptions = {
     system?: string;
     metadata?: Record<string, JsonValue>;
 };
+/** Provider-agnostic prompt seam that judges can reuse from a harness. */
 type HarnessPrompt = (input: string, options?: HarnessPromptOptions) => Promise<string>;
-type HarnessRuntime = {
-    prompt: HarnessPrompt;
-};
 type HarnessRunError = Error & {
     vitestEvalsRun: HarnessRun;
 };
@@ -77,7 +76,8 @@ type HarnessContext<TMetadata extends HarnessMetadata = HarnessMetadata> = {
 };
 type Harness<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = {
     name: string;
-    prompt?: HarnessPrompt;
+    /** Prompt seam reused by LLM-backed judges. */
+    prompt: HarnessPrompt;
     run: (input: TInput, context: HarnessContext<TMetadata>) => Promise<HarnessRun>;
 };
 /** Returns true when a value exposes a callable method with the given name. */
@@ -115,4 +115,4 @@ declare function resolveHarnessRunErrors(result: unknown): Array<Record<string,
 /** Serializes an arbitrary thrown value into the normalized error shape. */
 declare function serializeError(error: unknown): Record<string, JsonValue>;
 
-export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type HarnessRuntime, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };
+export { type Harness, type HarnessContext, type HarnessMetadata, type HarnessPrompt, type HarnessPromptOptions, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type NormalizedMessage, type NormalizedSession, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };

package/dist/harness.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/harness.ts"],"sourcesContent":["export type JsonPrimitive = string | number | boolean | null;\n\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\nexport type ToolCallRecord = {\n  id?: string;\n  name: string;\n  arguments?: Record<string, JsonValue>;\n  result?: JsonValue;\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  startedAt?: string;\n  finishedAt?: string;\n  durationMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedMessage = {\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  content?: JsonValue;\n  toolCalls?: ToolCallRecord[];\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type UsageSummary = {\n  provider?: string;\n  model?: string;\n  inputTokens?: number;\n  outputTokens?: number;\n  reasoningTokens?: number;\n  totalTokens?: number;\n  estimatedCost?: number;\n  toolCalls?: number;\n  retries?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type TimingSummary = {\n  totalMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedSession = {\n  messages: NormalizedMessage[];\n  outputText?: string;\n  provider?: string;\n  model?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type HarnessRun = {\n  session: NormalizedSession;\n  output?: JsonValue;\n  usage: UsageSummary;\n  timings?: TimingSummary;\n  artifacts?: Record<string, JsonValue>;\n  errors: Array<Record<string, JsonValue>>;\n};\n\nexport type HarnessPromptOptions = {\n  system?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type HarnessPrompt = (\n  input: string,\n  options?: HarnessPromptOptions,\n) => Promise<string>;\n\nexport type HarnessRuntime = {\n  prompt: HarnessPrompt;\n};\n\nexport type HarnessRunError = Error & {\n  vitestEvalsRun: HarnessRun;\n};\n\nexport type HarnessMetadata = Record<string, unknown>;\n\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  metadata: Readonly<TMetadata>;\n  task: {\n    meta: Record<string, unknown>;\n  };\n  signal?: AbortSignal;\n  artifacts: Record<string, JsonValue>;\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\nexport type Harness<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  prompt?: HarnessPrompt;\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  return toJsonValue(value) ?? String(value);\n}\n\n/** Flattens every recorded tool call from a normalized session. */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/** Filters normalized session messages by role. */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\n/** Returns every normalized system message from a session. */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/** Returns every normalized user message from a session. */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/** Returns every normalized assistant message from a session. */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/** Returns every normalized tool message from a session. */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/** Attaches a partial or complete harness run to an arbitrary thrown error. */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/** Reads an attached harness run back off a previously wrapped error value. */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA6GA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,kBAAkB,OAAgB,YAAoB;AACpE,SACE,UAAU,SACT,OAAO,UAAU,YAAY,OAAO,UAAU,eAC/C,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAGO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,SAAO,YAAY,KAAK,KAAK,OAAO,KAAK;AAC3C;AAGO,SAAS,UAAU,SAA8C;AACtE,SAAO,QAAQ,SAAS,QAAQ,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;AACtE;AAGO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAGO,SAAS,eAAe,SAA4B;AACzD,SAAO,eAAe,SAAS,QAAQ;AACzC;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,wBACd,OACA,KACiB;AACjB,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,gBAAgB;AAAA,EAClB,CAAC;AACH;AAGO,SAAS,uBAAuB,OAAwC;AAC7E,MACE,SACA,OAAO,UAAU,YACjB,oBAAoB,SACpB,aAAc,MAAuC,cAAc,GACnE;AACA,WAAQ,MAAyC;AAAA,EACnD;AAEA,SAAO;AACT;AAGO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAGO,SAAS,wBACd,QACkC;AAClC,MACE,UACA,OAAO,WAAW,YAClB,MAAM,QAAS,OAAmC,MAAM,GACxD;AACA,WAAQ,OAAwD;AAAA,EAClE;AAEA,SAAO,CAAC;AACV;AAGO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../src/harness.ts"],"sourcesContent":["export type JsonPrimitive = string | number | boolean | null;\n\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\nexport type ToolCallRecord = {\n  id?: string;\n  name: string;\n  arguments?: Record<string, JsonValue>;\n  result?: JsonValue;\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  startedAt?: string;\n  finishedAt?: string;\n  durationMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedMessage = {\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  content?: JsonValue;\n  toolCalls?: ToolCallRecord[];\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type UsageSummary = {\n  provider?: string;\n  model?: string;\n  inputTokens?: number;\n  outputTokens?: number;\n  reasoningTokens?: number;\n  totalTokens?: number;\n  estimatedCost?: number;\n  toolCalls?: number;\n  retries?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type TimingSummary = {\n  totalMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedSession = {\n  messages: NormalizedMessage[];\n  outputText?: string;\n  provider?: string;\n  model?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type HarnessRun = {\n  session: NormalizedSession;\n  output?: JsonValue;\n  usage: UsageSummary;\n  timings?: TimingSummary;\n  artifacts?: Record<string, JsonValue>;\n  errors: Array<Record<string, JsonValue>>;\n};\n\n/** Optional provider-facing hints for harness prompt calls. */\nexport type HarnessPromptOptions = {\n  system?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\n/** Provider-agnostic prompt seam that judges can reuse from a harness. */\nexport type HarnessPrompt = (\n  input: string,\n  options?: HarnessPromptOptions,\n) => Promise<string>;\n\nexport type HarnessRunError = Error & {\n  vitestEvalsRun: HarnessRun;\n};\n\nexport type HarnessMetadata = Record<string, unknown>;\n\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  metadata: Readonly<TMetadata>;\n  task: {\n    meta: Record<string, unknown>;\n  };\n  signal?: AbortSignal;\n  artifacts: Record<string, JsonValue>;\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\nexport type Harness<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  /** Prompt seam reused by LLM-backed judges. */\n  prompt: HarnessPrompt;\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  return toJsonValue(value) ?? String(value);\n}\n\n/** Flattens every recorded tool call from a normalized session. */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/** Filters normalized session messages by role. */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\n/** Returns every normalized system message from a session. */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/** Returns every normalized user message from a session. */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/** Returns every normalized assistant message from a session. */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/** Returns every normalized tool message from a session. */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/** Attaches a partial or complete harness run to an arbitrary thrown error. */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/** Reads an attached harness run back off a previously wrapped error value. */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA4GA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,kBAAkB,OAAgB,YAAoB;AACpE,SACE,UAAU,SACT,OAAO,UAAU,YAAY,OAAO,UAAU,eAC/C,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAGO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,SAAO,YAAY,KAAK,KAAK,OAAO,KAAK;AAC3C;AAGO,SAAS,UAAU,SAA8C;AACtE,SAAO,QAAQ,SAAS,QAAQ,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;AACtE;AAGO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAGO,SAAS,eAAe,SAA4B;AACzD,SAAO,eAAe,SAAS,QAAQ;AACzC;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,wBACd,OACA,KACiB;AACjB,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,gBAAgB;AAAA,EAClB,CAAC;AACH;AAGO,SAAS,uBAAuB,OAAwC;AAC7E,MACE,SACA,OAAO,UAAU,YACjB,oBAAoB,SACpB,aAAc,MAAuC,cAAc,GACnE;AACA,WAAQ,MAAyC;AAAA,EACnD;AAEA,SAAO;AACT;AAGO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAGO,SAAS,wBACd,QACkC;AAClC,MACE,UACA,OAAO,WAAW,YAClB,MAAM,QAAS,OAAmC,MAAM,GACxD;AACA,WAAQ,OAAwD;AAAA,EAClE;AAEA,SAAO,CAAC;AACV;AAGO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;","names":[]}

package/dist/harness.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/harness.ts"],"sourcesContent":["export type JsonPrimitive = string | number | boolean | null;\n\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\nexport type ToolCallRecord = {\n  id?: string;\n  name: string;\n  arguments?: Record<string, JsonValue>;\n  result?: JsonValue;\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  startedAt?: string;\n  finishedAt?: string;\n  durationMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedMessage = {\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  content?: JsonValue;\n  toolCalls?: ToolCallRecord[];\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type UsageSummary = {\n  provider?: string;\n  model?: string;\n  inputTokens?: number;\n  outputTokens?: number;\n  reasoningTokens?: number;\n  totalTokens?: number;\n  estimatedCost?: number;\n  toolCalls?: number;\n  retries?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type TimingSummary = {\n  totalMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedSession = {\n  messages: NormalizedMessage[];\n  outputText?: string;\n  provider?: string;\n  model?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type HarnessRun = {\n  session: NormalizedSession;\n  output?: JsonValue;\n  usage: UsageSummary;\n  timings?: TimingSummary;\n  artifacts?: Record<string, JsonValue>;\n  errors: Array<Record<string, JsonValue>>;\n};\n\nexport type HarnessPromptOptions = {\n  system?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type HarnessPrompt = (\n  input: string,\n  options?: HarnessPromptOptions,\n) => Promise<string>;\n\nexport type HarnessRuntime = {\n  prompt: HarnessPrompt;\n};\n\nexport type HarnessRunError = Error & {\n  vitestEvalsRun: HarnessRun;\n};\n\nexport type HarnessMetadata = Record<string, unknown>;\n\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  metadata: Readonly<TMetadata>;\n  task: {\n    meta: Record<string, unknown>;\n  };\n  signal?: AbortSignal;\n  artifacts: Record<string, JsonValue>;\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\nexport type Harness<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  prompt?: HarnessPrompt;\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  return toJsonValue(value) ?? String(value);\n}\n\n/** Flattens every recorded tool call from a normalized session. */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/** Filters normalized session messages by role. */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\n/** Returns every normalized system message from a session. */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/** Returns every normalized user message from a session. */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/** Returns every normalized assistant message from a session. */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/** Returns every normalized tool message from a session. */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/** Attaches a partial or complete harness run to an arbitrary thrown error. */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/** Reads an attached harness run back off a previously wrapped error value. */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";AA6GA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,kBAAkB,OAAgB,YAAoB;AACpE,SACE,UAAU,SACT,OAAO,UAAU,YAAY,OAAO,UAAU,eAC/C,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAGO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,SAAO,YAAY,KAAK,KAAK,OAAO,KAAK;AAC3C;AAGO,SAAS,UAAU,SAA8C;AACtE,SAAO,QAAQ,SAAS,QAAQ,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;AACtE;AAGO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAGO,SAAS,eAAe,SAA4B;AACzD,SAAO,eAAe,SAAS,QAAQ;AACzC;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,wBACd,OACA,KACiB;AACjB,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,gBAAgB;AAAA,EAClB,CAAC;AACH;AAGO,SAAS,uBAAuB,OAAwC;AAC7E,MACE,SACA,OAAO,UAAU,YACjB,oBAAoB,SACpB,aAAc,MAAuC,cAAc,GACnE;AACA,WAAQ,MAAyC;AAAA,EACnD;AAEA,SAAO;AACT;AAGO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAGO,SAAS,wBACd,QACkC;AAClC,MACE,UACA,OAAO,WAAW,YAClB,MAAM,QAAS,OAAmC,MAAM,GACxD;AACA,WAAQ,OAAwD;AAAA,EAClE;AAEA,SAAO,CAAC;AACV;AAGO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../src/harness.ts"],"sourcesContent":["export type JsonPrimitive = string | number | boolean | null;\n\nexport type JsonValue =\n  | JsonPrimitive\n  | JsonValue[]\n  | { [key: string]: JsonValue };\n\nexport type ToolCallRecord = {\n  id?: string;\n  name: string;\n  arguments?: Record<string, JsonValue>;\n  result?: JsonValue;\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  startedAt?: string;\n  finishedAt?: string;\n  durationMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedMessage = {\n  role: \"system\" | \"user\" | \"assistant\" | \"tool\";\n  content?: JsonValue;\n  toolCalls?: ToolCallRecord[];\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type UsageSummary = {\n  provider?: string;\n  model?: string;\n  inputTokens?: number;\n  outputTokens?: number;\n  reasoningTokens?: number;\n  totalTokens?: number;\n  estimatedCost?: number;\n  toolCalls?: number;\n  retries?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type TimingSummary = {\n  totalMs?: number;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type NormalizedSession = {\n  messages: NormalizedMessage[];\n  outputText?: string;\n  provider?: string;\n  model?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\nexport type HarnessRun = {\n  session: NormalizedSession;\n  output?: JsonValue;\n  usage: UsageSummary;\n  timings?: TimingSummary;\n  artifacts?: Record<string, JsonValue>;\n  errors: Array<Record<string, JsonValue>>;\n};\n\n/** Optional provider-facing hints for harness prompt calls. */\nexport type HarnessPromptOptions = {\n  system?: string;\n  metadata?: Record<string, JsonValue>;\n};\n\n/** Provider-agnostic prompt seam that judges can reuse from a harness. */\nexport type HarnessPrompt = (\n  input: string,\n  options?: HarnessPromptOptions,\n) => Promise<string>;\n\nexport type HarnessRunError = Error & {\n  vitestEvalsRun: HarnessRun;\n};\n\nexport type HarnessMetadata = Record<string, unknown>;\n\nexport type HarnessContext<\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  metadata: Readonly<TMetadata>;\n  task: {\n    meta: Record<string, unknown>;\n  };\n  signal?: AbortSignal;\n  artifacts: Record<string, JsonValue>;\n  setArtifact: (name: string, value: JsonValue) => void;\n};\n\nexport type Harness<\n  TInput = unknown,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  /** Prompt seam reused by LLM-backed judges. */\n  prompt: HarnessPrompt;\n  run: (\n    input: TInput,\n    context: HarnessContext<TMetadata>,\n  ) => Promise<HarnessRun>;\n};\n\nfunction isJsonPrimitive(value: unknown): value is JsonPrimitive {\n  return (\n    value === null ||\n    typeof value === \"string\" ||\n    typeof value === \"number\" ||\n    typeof value === \"boolean\"\n  );\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction normalizeJsonArray(value: unknown[]): JsonValue[] {\n  return value.map((item) => {\n    const normalized = toJsonValue(item);\n    return normalized === undefined ? null : normalized;\n  });\n}\n\nfunction normalizeJsonObject(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  const normalized: Record<string, JsonValue> = {};\n\n  for (const [key, entryValue] of Object.entries(value)) {\n    const entry = toJsonValue(entryValue);\n    if (entry !== undefined) {\n      normalized[key] = entry;\n    }\n  }\n\n  return normalized;\n}\n\n/** Returns true when a value exposes a callable method with the given name. */\nexport function hasCallableMethod(value: unknown, methodName: string) {\n  return (\n    value !== null &&\n    (typeof value === \"object\" || typeof value === \"function\") &&\n    methodName in value &&\n    typeof (value as Record<string, unknown>)[methodName] === \"function\"\n  );\n}\n\n/** Normalizes an unknown value into the JSON-safe shape used by harness runs. */\nexport function toJsonValue(value: unknown): JsonValue | undefined {\n  if (isJsonPrimitive(value)) {\n    return value;\n  }\n\n  if (Array.isArray(value)) {\n    return normalizeJsonArray(value);\n  }\n\n  if (isJsonRecord(value)) {\n    return normalizeJsonObject(value);\n  }\n\n  return undefined;\n}\n\n/** Drops non-JSON properties from a record while preserving valid values. */\nexport function normalizeRecord(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> {\n  return normalizeJsonObject(value);\n}\n\n/** Normalizes metadata and omits the field entirely when nothing survives. */\nexport function normalizeMetadata(\n  value: Record<string, unknown>,\n): Record<string, JsonValue> | undefined {\n  const normalized = normalizeRecord(value);\n  return Object.keys(normalized).length > 0 ? normalized : undefined;\n}\n\n/** Converts arbitrary content into the JSON-safe message content shape. */\nexport function normalizeContent(value: unknown): JsonValue {\n  return toJsonValue(value) ?? String(value);\n}\n\n/** Flattens every recorded tool call from a normalized session. */\nexport function toolCalls(session: NormalizedSession): ToolCallRecord[] {\n  return session.messages.flatMap((message) => message.toolCalls ?? []);\n}\n\n/** Filters normalized session messages by role. */\nexport function messagesByRole(\n  session: NormalizedSession,\n  role: NormalizedMessage[\"role\"],\n): NormalizedMessage[] {\n  return session.messages.filter((message) => message.role === role);\n}\n\n/** Returns every normalized system message from a session. */\nexport function systemMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"system\");\n}\n\n/** Returns every normalized user message from a session. */\nexport function userMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"user\");\n}\n\n/** Returns every normalized assistant message from a session. */\nexport function assistantMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"assistant\");\n}\n\n/** Returns every normalized tool message from a session. */\nexport function toolMessages(session: NormalizedSession) {\n  return messagesByRole(session, \"tool\");\n}\n\n/** Attaches a partial or complete harness run to an arbitrary thrown error. */\nexport function attachHarnessRunToError(\n  error: unknown,\n  run: HarnessRun,\n): HarnessRunError {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsRun: run,\n  });\n}\n\n/** Reads an attached harness run back off a previously wrapped error value. */\nexport function getHarnessRunFromError(error: unknown): HarnessRun | undefined {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsRun\" in error &&\n    isHarnessRun((error as { vitestEvalsRun?: unknown }).vitestEvalsRun)\n  ) {\n    return (error as { vitestEvalsRun: HarnessRun }).vitestEvalsRun;\n  }\n\n  return undefined;\n}\n\n/** Returns true when a value matches the normalized `HarnessRun` contract. */\nexport function isHarnessRun(value: unknown): value is HarnessRun {\n  if (!value || typeof value !== \"object\") {\n    return false;\n  }\n\n  const candidate = value as {\n    session?: unknown;\n    usage?: unknown;\n    errors?: unknown;\n  };\n\n  return (\n    isNormalizedSession(candidate.session) &&\n    Boolean(candidate.usage) &&\n    typeof candidate.usage === \"object\" &&\n    !Array.isArray(candidate.usage) &&\n    Array.isArray(candidate.errors)\n  );\n}\n\n/** Returns true when a value matches the normalized session contract. */\nexport function isNormalizedSession(\n  value: unknown,\n): value is NormalizedSession {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"messages\" in value &&\n    Array.isArray((value as { messages?: unknown }).messages)\n  );\n}\n\n/** Reuses pre-normalized harness errors when a runtime already returns them. */\nexport function resolveHarnessRunErrors(\n  result: unknown,\n): Array<Record<string, JsonValue>> {\n  if (\n    result &&\n    typeof result === \"object\" &&\n    Array.isArray((result as Record<string, unknown>).errors)\n  ) {\n    return (result as { errors: Array<Record<string, JsonValue>> }).errors;\n  }\n\n  return [];\n}\n\n/** Serializes an arbitrary thrown value into the normalized error shape. */\nexport function serializeError(error: unknown): Record<string, JsonValue> {\n  if (error instanceof Error) {\n    return {\n      type: error.name,\n      message: error.message,\n    };\n  }\n\n  return {\n    type: \"Error\",\n    message: String(error),\n  };\n}\n"],"mappings":";AA4GA,SAAS,gBAAgB,OAAwC;AAC/D,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU;AAErB;AAEA,SAAS,aAAa,OAAkD;AACtE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,mBAAmB,OAA+B;AACzD,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,UAAM,aAAa,YAAY,IAAI;AACnC,WAAO,eAAe,SAAY,OAAO;AAAA,EAC3C,CAAC;AACH;AAEA,SAAS,oBACP,OAC2B;AAC3B,QAAM,aAAwC,CAAC;AAE/C,aAAW,CAAC,KAAK,UAAU,KAAK,OAAO,QAAQ,KAAK,GAAG;AACrD,UAAM,QAAQ,YAAY,UAAU;AACpC,QAAI,UAAU,QAAW;AACvB,iBAAW,GAAG,IAAI;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,kBAAkB,OAAgB,YAAoB;AACpE,SACE,UAAU,SACT,OAAO,UAAU,YAAY,OAAO,UAAU,eAC/C,cAAc,SACd,OAAQ,MAAkC,UAAU,MAAM;AAE9D;AAGO,SAAS,YAAY,OAAuC;AACjE,MAAI,gBAAgB,KAAK,GAAG;AAC1B,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,mBAAmB,KAAK;AAAA,EACjC;AAEA,MAAI,aAAa,KAAK,GAAG;AACvB,WAAO,oBAAoB,KAAK;AAAA,EAClC;AAEA,SAAO;AACT;AAGO,SAAS,gBACd,OAC2B;AAC3B,SAAO,oBAAoB,KAAK;AAClC;AAGO,SAAS,kBACd,OACuC;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,OAAO,KAAK,UAAU,EAAE,SAAS,IAAI,aAAa;AAC3D;AAGO,SAAS,iBAAiB,OAA2B;AAC1D,SAAO,YAAY,KAAK,KAAK,OAAO,KAAK;AAC3C;AAGO,SAAS,UAAU,SAA8C;AACtE,SAAO,QAAQ,SAAS,QAAQ,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;AACtE;AAGO,SAAS,eACd,SACA,MACqB;AACrB,SAAO,QAAQ,SAAS,OAAO,CAAC,YAAY,QAAQ,SAAS,IAAI;AACnE;AAGO,SAAS,eAAe,SAA4B;AACzD,SAAO,eAAe,SAAS,QAAQ;AACzC;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,kBAAkB,SAA4B;AAC5D,SAAO,eAAe,SAAS,WAAW;AAC5C;AAGO,SAAS,aAAa,SAA4B;AACvD,SAAO,eAAe,SAAS,MAAM;AACvC;AAGO,SAAS,wBACd,OACA,KACiB;AACjB,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,gBAAgB;AAAA,EAClB,CAAC;AACH;AAGO,SAAS,uBAAuB,OAAwC;AAC7E,MACE,SACA,OAAO,UAAU,YACjB,oBAAoB,SACpB,aAAc,MAAuC,cAAc,GACnE;AACA,WAAQ,MAAyC;AAAA,EACnD;AAEA,SAAO;AACT;AAGO,SAAS,aAAa,OAAqC;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,QAAM,YAAY;AAMlB,SACE,oBAAoB,UAAU,OAAO,KACrC,QAAQ,UAAU,KAAK,KACvB,OAAO,UAAU,UAAU,YAC3B,CAAC,MAAM,QAAQ,UAAU,KAAK,KAC9B,MAAM,QAAQ,UAAU,MAAM;AAElC;AAGO,SAAS,oBACd,OAC4B;AAC5B,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,MAAM,QAAS,MAAiC,QAAQ;AAE5D;AAGO,SAAS,wBACd,QACkC;AAClC,MACE,UACA,OAAO,WAAW,YAClB,MAAM,QAAS,OAAmC,MAAM,GACxD;AACA,WAAQ,OAAwD;AAAA,EAClE;AAEA,SAAO,CAAC;AACV;AAGO,SAAS,eAAe,OAA2C;AACxE,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS,OAAO,KAAK;AAAA,EACvB;AACF;","names":[]}

package/dist/index.d.mts

@@ -1,8 +1,9 @@
 import * as vitest from 'vitest';
 import { TestAPI } from 'vitest';
 import { HarnessMetadata, ToolCallRecord, HarnessRun, Harness } from './harness.mjs';
-export { HarnessContext, HarnessRunError, JsonPrimitive, JsonValue, NormalizedMessage, NormalizedSession, TimingSummary, UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, messagesByRole, systemMessages, toolCalls, toolMessages, userMessages } from './harness.mjs';
-import { BaseJudgeOptions, JudgeFn, JudgeResult, JudgeContext } from './judges/types.mjs';
+export { HarnessContext, HarnessPrompt, HarnessPromptOptions, HarnessRunError, JsonPrimitive, JsonValue, NormalizedMessage, NormalizedSession, TimingSummary, UsageSummary, assistantMessages, attachHarnessRunToError, getHarnessRunFromError, messagesByRole, systemMessages, toolCalls, toolMessages, userMessages } from './harness.mjs';
+import { JudgeContext, JudgeFn, JudgeResult } from './judges/types.mjs';
+export { JudgeOptions } from './judges/types.mjs';
 export { wrapText } from './wrapText.mjs';
 export { StructuredOutputJudge, StructuredOutputJudgeConfig, StructuredOutputJudgeOptions } from './judges/structuredOutputJudge.mjs';
 export { ToolCallJudge, ToolCallJudgeConfig, ToolCallJudgeOptions } from './judges/toolCallJudge.mjs';
@@ -26,52 +27,60 @@ type EvalTaskMeta = {
         run: HarnessRun;
     };
 };
+type HarnessInput<THarness extends Harness<any, any>> = THarness extends Harness<infer TInput, any> ? TInput : unknown;
+type HarnessMetadataFor<THarness extends Harness<any, any>> = THarness extends Harness<any, infer TMetadata> ? TMetadata : HarnessMetadata;
+declare const evalHarnessRunBrand: unique symbol;
+/** Harness run returned by the fixture-backed `run(...)` API. */
+type EvalHarnessRun<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> = Harness<TInput, TMetadata>> = HarnessRun & {
+    readonly [evalHarnessRunBrand]: {
+        readonly input: TInput;
+        readonly metadata: TMetadata;
+        readonly harness: THarness;
+    };
+};
 /** Per-run metadata forwarded to the harness alongside the test input. */
 interface EvalRunOptions<TMetadata extends HarnessMetadata = HarnessMetadata> {
     metadata?: TMetadata;
 }
 /** Explicit harness execution primitive exposed to each eval test. */
-type EvalRun<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = (input: TInput, options?: EvalRunOptions<TMetadata>) => Promise<HarnessRun>;
+type EvalRun<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> = Harness<TInput, TMetadata>> = (input: TInput, options?: EvalRunOptions<TMetadata>) => Promise<EvalHarnessRun<TInput, TMetadata, THarness>>;
 /** Fixture-backed Vitest context exposed inside `describeEval(...)` tests. */
-interface EvalTestContext<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> {
-    run: EvalRun<TInput, TMetadata>;
+interface EvalTestContext<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> = Harness<TInput, TMetadata>> {
+    run: EvalRun<TInput, TMetadata, THarness>;
 }
-type EvalTestAPI<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = TestAPI<EvalTestContext<TInput, TMetadata>>;
-/**
- * Compatibility alias for harness-backed judge inputs.
- *
- * New custom judges should prefer `JudgeContext` directly. This alias remains
- * for older imports that were already using the harness-backed judge shape.
- */
-type HarnessJudgeOptions<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> = JudgeContext<TInput, TMetadata>;
+type EvalTestAPI<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> = Harness<TInput, TMetadata>> = TestAPI<EvalTestContext<TInput, TMetadata, THarness>>;
 /** Suite-level configuration for a harness-backed eval block. */
-interface DescribeEvalOptions<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata> {
+interface DescribeEvalOptions<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TMetadata> = Harness<TInput, TMetadata>> {
     /** Harness used for every explicit `run(...)` call in the suite. */
-    harness: Harness<TInput, TMetadata>;
+    harness: THarness;
     /** Automatic judges applied after each successful `run(...)`. */
-    judges?: Array<JudgeFn<HarnessJudgeOptions<TInput, TMetadata>>>;
+    judges?: Array<JudgeFn<JudgeContext<TInput, TMetadata, THarness>>>;
     /** Passing threshold for automatic suite-level judges. `null` disables fail-on-score. */
     judgeThreshold?: number | null;
     skipIf?: () => boolean;
 }
-type JudgeAssertionInputValue<TJudgeOptions extends BaseJudgeOptions> = TJudgeOptions extends {
+type JudgeAssertionInputValue<TJudgeOptions extends JudgeContext<any, any, any>> = TJudgeOptions extends {
     inputValue: infer TInput;
 } ? TInput : unknown;
-type JudgeAssertionMetadata<TJudgeOptions extends BaseJudgeOptions> = TJudgeOptions extends {
+type JudgeAssertionMetadata<TJudgeOptions extends JudgeContext<any, any, any>> = TJudgeOptions extends {
     metadata: infer TMetadata;
 } ? TMetadata : HarnessMetadata;
+type JudgeAssertionHarness<TJudgeOptions extends JudgeContext<any, any, any>> = TJudgeOptions extends {
+    harness: infer THarness;
+} ? Exclude<THarness, undefined> : Harness<JudgeAssertionInputValue<TJudgeOptions>, JudgeAssertionMetadata<TJudgeOptions>>;
 /** Optional overrides passed to `expect(...).toSatisfyJudge(...)`. */
-type JudgeAssertionOptions<TJudgeOptions extends BaseJudgeOptions = BaseJudgeOptions> = Partial<Omit<TJudgeOptions, "input" | "output" | "metadata" | "run" | "session">> & {
+type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any> = JudgeContext> = Partial<Omit<TJudgeOptions, "input" | "output" | "inputValue" | "metadata" | "toolCalls" | "run" | "session" | "harness">> & {
     input?: string;
     inputValue?: JudgeAssertionInputValue<TJudgeOptions>;
     metadata?: JudgeAssertionMetadata<TJudgeOptions>;
     toolCalls?: ToolCallRecord[];
     run?: HarnessRun;
     session?: HarnessRun["session"];
+    harness?: JudgeAssertionHarness<TJudgeOptions>;
     /** Passing threshold for the explicit matcher. `null` records the score without failing. */
     threshold?: number | null;
 };
-type ToSatisfyJudge<R = unknown> = <TJudgeOptions extends BaseJudgeOptions = BaseJudgeOptions>(judge: JudgeFn<TJudgeOptions>, options?: JudgeAssertionOptions<TJudgeOptions>) => Promise<R>;
+type ToSatisfyJudge<TReceived = unknown> = <TJudgeOptions extends JudgeContext<any, any, any> = JudgeContext>(judge: JudgeFn<TJudgeOptions>, options?: JudgeAssertionOptions<TJudgeOptions>) => Promise<TReceived>;
 interface EvalMatchers<R = unknown> {
     toSatisfyJudge: ToSatisfyJudge<R>;
 }
@@ -91,6 +100,7 @@ declare module "vitest" {
  * describeEval("refund agent", {
  *   harness: piAiHarness({
  *     createAgent: () => createRefundAgent(),
+ *     prompt: judgePrompt,
  *   }),
  *   judges: [ToolCallJudge()],
  * }, (it) => {
@@ -104,12 +114,12 @@ declare module "vitest" {
  * });
  * ```
  */
-declare function describeEval<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata>(name: string, options: DescribeEvalOptions<TInput, TMetadata>, define: (it: EvalTestAPI<TInput, TMetadata>) => void): vitest.SuiteCollector<object>;
+declare function describeEval<THarness extends Harness<any, any>>(name: string, options: DescribeEvalOptions<HarnessInput<THarness>, HarnessMetadataFor<THarness>, THarness>, define: (it: EvalTestAPI<HarnessInput<THarness>, HarnessMetadataFor<THarness>, THarness>) => void): vitest.SuiteCollector<object>;
 /** Formats judge results for reporter and assertion output. */
 declare function formatScores(scores: (JudgeResult & {
     name: string;
 })[]): string;
 /** Applies a stable display name to a custom judge function. */
-declare function namedJudge<TOptions extends BaseJudgeOptions>(name: string, judge: JudgeFn<TOptions>): JudgeFn<TOptions>;
+declare function namedJudge<TOptions extends JudgeContext<any, any, any>>(name: string, judge: JudgeFn<TOptions>): JudgeFn<TOptions>;
 
-export { BaseJudgeOptions, type DescribeEvalOptions, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, type HarnessJudgeOptions, HarnessMetadata, HarnessRun, type JudgeAssertionOptions, JudgeContext, JudgeFn, JudgeResult, type ToSatisfyJudge, ToolCallRecord, describeEval, formatScores, namedJudge };
+export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, HarnessRun, type JudgeAssertionOptions, JudgeContext, JudgeFn, JudgeResult, type ToSatisfyJudge, ToolCallRecord, describeEval, formatScores, namedJudge };