vitest-evals 0.9.0 → 0.11.0

package/README.md

@@ -2,6 +2,10 @@
 
 Harness-backed AI testing on top of Vitest.
 
+Use this package README for the core authoring model. For a guided setup path,
+runtime-specific harness examples, replay, and GitHub Actions reporting, start
+with the docs site: `https://vitest-evals.sentry.dev/docs`.
+
 ## Install
 
 ```sh
@@ -29,8 +33,14 @@ workflow.
 - `run(input, { metadata? })` executes the harness explicitly and returns a
   normalized `HarnessRun`
 - the returned `result.output` is the app-facing value you assert on directly
-- the returned `result.session` is the canonical JSON-serializable trace for
+- the returned `result.session` is the canonical JSON-serializable transcript for
   reporting, replay, tool assertions, and judges
+- the returned `result.traces` contains JSON-serializable operation spans; the
+  first-party harnesses attach run, model, and tool spans automatically, while
+  `createHarness(...)` attaches fallback run and tool spans for custom harnesses
+  that do not return traces themselves. Span attributes include typed
+  OpenTelemetry GenAI semantic keys while still allowing provider-specific
+  metadata
 - scenario-specific judge criteria can live in `input`; use `metadata` for
   per-run expectations or harness configuration that are not part of the
   scenario payload
@@ -40,55 +50,28 @@ workflow.
 - every judge receives `JudgeContext` with typed `input`, typed `output`, the
   normalized run/session, tool calls, and metadata; `output` is only optional
   when the harness output type includes `undefined`
-- judges own their prompt, rubric, model call, and parsing; use
-  `createJudge(...)` for custom judges and its provider-helper overload only
-  when multiple judges share setup
+- judges own their prompt, rubric, and parsing; LLM-backed judges use
+  `ctx.runJudge(...)` from a configured `judgeHarness`
 - explicit judge assertions use
   `await expect(result).toSatisfyJudge(judge, context)`
 
 ## Explicit Run Example
 
 ```ts
+import { getModel } from "@mariozechner/pi-ai";
 import { expect } from "vitest";
-import { piAiHarness } from "@vitest-evals/harness-pi-ai";
+import { piAiHarness, piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
 import {
-  createJudge,
   describeEval,
+  FactualityJudge,
   toolCalls,
-  type JudgeContext,
 } from "vitest-evals";
 import { createRefundAgent } from "../src/refundAgent";
 
-type RefundEvalMetadata = {
-  expectedStatus: "approved" | "denied";
-  expectedTools: string[];
-};
-
-type RefundOutput = {
-  status: "approved" | "denied";
-};
-
-const FactualityJudge = createJudge(
-  "FactualityJudge",
-  async ({
-    input,
-    output,
-    metadata,
-  }: JudgeContext<string, RefundOutput, RefundEvalMetadata>) => {
-    const verdict = await judgeFactuality({
-      question: input,
-      answer: output,
-      expectedStatus: metadata.expectedStatus,
-    });
-
-    return {
-      score: verdict.score,
-      metadata: {
-        rationale: verdict.rationale,
-      },
-    };
-  },
-);
+const judgeHarness = piAiJudgeHarness({
+  model: getModel("anthropic", "claude-sonnet-4-5"),
+  temperature: 0,
+});
 
 describeEval(
   "refund agent",
@@ -96,12 +79,15 @@ describeEval(
     harness: piAiHarness({
       agent: () => createRefundAgent(),
     }),
-    judges: [FactualityJudge],
+    judgeHarness,
+    judges: [FactualityJudge()],
+    judgeThreshold: 0.6,
   },
   (it) => {
     it("approves a refundable invoice", async ({ run }) => {
       const result = await run("Refund invoice inv_123", {
         metadata: {
+          expected: "The refund request is approved.",
           expectedStatus: "approved",
           expectedTools: ["lookupInvoice", "createRefund"],
         },
@@ -147,6 +133,17 @@ describeEval("refund agent", { harness }, (it) => {
 });
 ```
 
+## Terminal Reporting
+
+The terminal reporter has two eval report levels. Normal mode prints compact
+test, score, usage, and tool-count summaries. Info mode adds per-tool summaries,
+arguments, timing/size metadata, replay status, and final output summaries.
+Set `VITEST_EVALS_REPORT_LEVEL=info`, or pass `--info` through the workspace
+eval scripts, to enable it. `--verbose` and `-v` remain aliases for
+compatibility.
+
+Full transcripts and spans are preserved in the Vitest JSON report metadata.
+
 ## GitHub Actions Reporting
 
 Use Vitest JSON as the eval report artifact. It preserves the `meta` field that
@@ -210,6 +207,7 @@ When generics are needed, use `createHarness<Input, Output, Metadata>(...)`.
 import {
   createHarness,
   createJudge,
+  createJudgeHarness,
   describeEval,
   type JudgeContext,
 } from "vitest-evals";
@@ -255,14 +253,25 @@ const appHarness = createHarness<AppEvalInput, AppOutput, AppEvalMetadata>({
   },
 });
 
+const judgeHarness = createJudgeHarness({
+  name: "app-rubric-judge-model",
+  run: async ({ prompt }, { signal }) =>
+    promptJudgeModel({ prompt, signal }),
+});
+
 const AppRubricJudge = createJudge(
   "AppRubricJudge",
   async (ctx: JudgeContext<AppEvalInput, AppOutput, AppEvalMetadata>) => {
-    const verdict = await promptJudgeModel({
+    if (!ctx.runJudge) {
+      throw new Error("AppRubricJudge requires a configured judgeHarness.");
+    }
+
+    const verdict = await ctx.runJudge({
       prompt: formatRubricPrompt({
         output: ctx.output,
         criteria: ctx.input.criteria,
       }),
+      responseFormat: { type: "json" },
     });
 
     return parseRubricVerdict(verdict);
@@ -273,6 +282,7 @@ describeEval(
   "app behavior",
   {
     harness: appHarness,
+    judgeHarness,
     judges: [AppRubricJudge],
     judgeThreshold: 0.75,
   },
@@ -328,11 +338,26 @@ In practice, this is usually most useful for factuality, rubric, or grounded
 answer checks:
 
 ```ts
-await expect(result).toSatisfyJudge(FactualityJudge);
+import { openai } from "@ai-sdk/openai";
+import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+import { expect } from "vitest";
+import { FactualityJudge } from "vitest-evals";
+
+const judgeHarness = aiSdkJudgeHarness({
+  model: openai("gpt-4.1-mini"),
+  temperature: 0,
+});
+const factualityJudge = FactualityJudge({ judgeHarness });
+
+await expect(result).toSatisfyJudge(factualityJudge, {
+  expected: "Paris is the capital of France.",
+  threshold: 0.6,
+});
 ```
 
 For lower-level cases, the matcher also accepts raw values and synthetic judge
-context:
+context. Pass every context field the judge needs when the value did not come
+from eval fixture `run(...)`:
 
 ```ts
 await expect({ status: "approved" }).toSatisfyJudge(MyJudge, {
@@ -340,35 +365,75 @@ await expect({ status: "approved" }).toSatisfyJudge(MyJudge, {
 });
 ```
 
-Use `createJudge(...)` for custom judges so reporter output gets a stable
-label:
+Use the built-in factuality judge when you want a model-backed factuality grade
+over the normalized run:
 
 ```ts
-import { createJudge } from "vitest-evals";
+import { openai } from "@ai-sdk/openai";
+import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+import { FactualityJudge } from "vitest-evals";
 
-const FactualityJudge = createJudge(
-  "FactualityJudge",
-  async ({ output }) => {
-    const answer = output;
-    const verdict = await judgeFactuality(answer);
+export const judgeHarness = aiSdkJudgeHarness({
+  model: openai("gpt-4.1-mini"),
+  temperature: 0,
+});
+export const factualityJudge = FactualityJudge({ judgeHarness });
+```
 
-    return {
-      score: verdict.score,
-      metadata: {
-        rationale: verdict.rationale,
-      },
-    };
-  },
-);
+For custom judge providers, create a dedicated judge harness with the same
+prompt contract:
+
+```ts
+import {
+  createJudgeHarness,
+  FactualityJudge,
+  type JudgeHarness,
+} from "vitest-evals";
+import { callJudgeModel } from "./judgeModel";
+
+export const judgeHarness: JudgeHarness = createJudgeHarness({
+  name: "factuality-judge-model",
+  run: async ({ system, prompt }, { signal }) =>
+    callJudgeModel({ system, prompt, signal }),
+});
+
+export const factualityJudge = FactualityJudge({ judgeHarness });
+```
+
+Configure that judge harness once and reuse the same judge with any app
+harness:
+
+```ts
+import { describeEval } from "vitest-evals";
+import { aiSdkRefundHarness } from "./aiSdkRefundHarness";
+import { piRefundHarness } from "./piRefundHarness";
+import { factualityJudge } from "./sharedJudges";
+
+describeEval("ai sdk refund agent", {
+  harness: aiSdkRefundHarness,
+  judges: [factualityJudge],
+});
+
+describeEval("pi refund agent", {
+  harness: piRefundHarness,
+  judges: [factualityJudge],
+});
 ```
 
-LLM-backed judges should provide their own judge prompt and rubric text.
-`vitest-evals` does not prescribe a rubric schema, scoring scale, model
-provider, or parser; those stay in the judge. When multiple judges share a
-reusable judge-side provider helper, use the provider-helper overload of
-`createJudge(...)` so run-scoped options such as abort signals stay curried.
-Calling `harness.run(...)` from a judge executes the application again, so use
-that only when a second run is intentional.
+Use `createJudge(...)` for custom judges so reporter output gets a stable
+label. Custom LLM-backed judges should provide their own judge prompt, rubric
+text, and parser, then call `ctx.runJudge(...)` for the provider-specific model
+request. Bind a reusable default with `createJudge({ name, judgeHarness,
+assess })` or pass `judgeHarness` on the matcher or suite. Core curries the
+matcher, judge, or explicit suite `judgeHarness` into that function with the
+current run's abort signal. Matcher options win over a judge default, and a
+judge default wins over the suite default. Explicit matcher calls can also
+reuse a single unambiguous judge-level harness from the suite's automatic
+judges, but automatic judges do not inherit inferred harnesses from sibling
+judges. That inference requires those judges to share the same judge harness
+instance. Leave `judgeHarness` unset for suites that only use deterministic
+judges. Calling `harness.run(...)` from a judge executes the application again,
+so use that only when a second run is intentional.
 
 For an `EvalHarnessRun` returned by fixture `run(...)`,
 `toSatisfyJudge(...)` uses the run's typed `output` and reuses the registered
@@ -386,6 +451,4 @@ When a judge needs richer normalized context or the configured suite harness,
 type it with `JudgeContext`.
 
 When you only need deterministic contract checks, built-ins such as
-`StructuredOutputJudge()` and `ToolCallJudge()` are still available. The primary
-documentation examples intentionally use factuality/rubric judges because those
-match the product's LLM-as-a-judge direction.
+`StructuredOutputJudge()` and `ToolCallJudge()` are still available.

package/dist/harness.d.mts

@@ -4,6 +4,169 @@ type JsonPrimitive = string | number | boolean | null;
 type JsonValue = JsonPrimitive | JsonValue[] | {
     [key: string]: JsonValue;
 };
+/** Well-known OpenTelemetry GenAI operation names. */
+type GenAiOperationName = "chat" | "create_agent" | "embeddings" | "execute_tool" | "generate_content" | "invoke_agent" | "invoke_workflow" | "retrieval" | "text_completion" | (string & {});
+/** Well-known OpenTelemetry GenAI output content types. */
+type GenAiOutputType = "image" | "json" | "speech" | "text" | (string & {});
+/** Well-known OpenTelemetry GenAI provider names. */
+type GenAiProviderName = "anthropic" | "aws.bedrock" | "azure.ai.inference" | "azure.ai.openai" | "cohere" | "deepseek" | "gcp.gemini" | "gcp.gen_ai" | "gcp.vertex_ai" | "groq" | "ibm.watsonx.ai" | "mistral_ai" | "openai" | "perplexity" | "x_ai" | (string & {});
+/** Well-known OpenTelemetry GenAI token types. */
+type GenAiTokenType = "input" | "output" | (string & {});
+/** Well-known OpenTelemetry GenAI tool execution types. */
+type GenAiToolType = "datastore" | "extension" | "function" | (string & {});
+/** Typed subset of OpenTelemetry GenAI semantic attributes. */
+type GenAiSemanticAttributes = {
+    "gen_ai.agent.description"?: string;
+    "gen_ai.agent.id"?: string;
+    "gen_ai.agent.name"?: string;
+    "gen_ai.agent.version"?: string;
+    "gen_ai.conversation.id"?: string;
+    "gen_ai.data_source.id"?: string;
+    "gen_ai.embeddings.dimension.count"?: number;
+    "gen_ai.evaluation.explanation"?: string;
+    "gen_ai.evaluation.name"?: string;
+    "gen_ai.evaluation.score.label"?: string;
+    "gen_ai.evaluation.score.value"?: number;
+    "gen_ai.input.messages"?: JsonValue;
+    "gen_ai.operation.name"?: GenAiOperationName;
+    "gen_ai.output.messages"?: JsonValue;
+    "gen_ai.output.type"?: GenAiOutputType;
+    "gen_ai.prompt.name"?: string;
+    "gen_ai.provider.name"?: GenAiProviderName;
+    "gen_ai.request.choice.count"?: number;
+    "gen_ai.request.encoding_formats"?: string[];
+    "gen_ai.request.frequency_penalty"?: number;
+    "gen_ai.request.max_tokens"?: number;
+    "gen_ai.request.model"?: string;
+    "gen_ai.request.presence_penalty"?: number;
+    "gen_ai.request.seed"?: number;
+    "gen_ai.request.stop_sequences"?: string[];
+    "gen_ai.request.stream"?: boolean;
+    "gen_ai.request.temperature"?: number;
+    "gen_ai.request.top_k"?: number;
+    "gen_ai.request.top_p"?: number;
+    "gen_ai.response.finish_reasons"?: string[];
+    "gen_ai.response.id"?: string;
+    "gen_ai.response.model"?: string;
+    "gen_ai.response.time_to_first_chunk"?: number;
+    "gen_ai.retrieval.documents"?: JsonValue;
+    "gen_ai.retrieval.query.text"?: string;
+    "gen_ai.system_instructions"?: JsonValue;
+    "gen_ai.token.type"?: GenAiTokenType;
+    "gen_ai.tool.call.arguments"?: JsonValue;
+    "gen_ai.tool.call.id"?: string;
+    "gen_ai.tool.call.result"?: JsonValue;
+    "gen_ai.tool.definitions"?: JsonValue;
+    "gen_ai.tool.description"?: string;
+    "gen_ai.tool.name"?: string;
+    "gen_ai.tool.type"?: GenAiToolType;
+    "gen_ai.usage.cache_creation.input_tokens"?: number;
+    "gen_ai.usage.cache_read.input_tokens"?: number;
+    "gen_ai.usage.input_tokens"?: number;
+    "gen_ai.usage.output_tokens"?: number;
+    "gen_ai.usage.reasoning.output_tokens"?: number;
+    "gen_ai.workflow.name"?: string;
+};
+/** Attribute keys defined by the OpenTelemetry GenAI semantic conventions. */
+type GenAiSemanticAttributeKey = keyof GenAiSemanticAttributes;
+/** Typed OpenTelemetry semantic attributes accepted on normalized spans. */
+type OpenTelemetrySemanticAttributes = GenAiSemanticAttributes & {
+    "error.type"?: string;
+    "server.address"?: string;
+    "server.port"?: number;
+};
+/** Known OpenTelemetry semantic attribute keys accepted on normalized spans. */
+type OpenTelemetrySemanticAttributeKey = keyof OpenTelemetrySemanticAttributes;
+/** Attribute keys accepted on normalized spans. */
+type NormalizedSpanAttributeKey = OpenTelemetrySemanticAttributeKey | (string & {});
+/**
+ * JSON-safe span attributes. Known OpenTelemetry GenAI keys are typed while
+ * custom provider and application keys remain allowed.
+ */
+type NormalizedSpanAttributes = OpenTelemetrySemanticAttributes & {
+    [key: string]: JsonValue | undefined;
+};
+/** Event attached to one normalized span. */
+type NormalizedSpanEvent = {
+    /** Event name emitted by the runtime or harness. */
+    name: string;
+    /** ISO timestamp for the event when available. */
+    timestamp?: string;
+    /** JSON-safe event attributes. */
+    attributes?: NormalizedSpanAttributes;
+};
+/** Normalized operation span captured during a harness run. */
+type NormalizedSpan = {
+    /** Runtime or provider span id when one is available. */
+    id?: string;
+    /** Trace id this span belongs to. */
+    traceId?: string;
+    /** Parent span id when the runtime exposes hierarchy. */
+    parentId?: string;
+    /** Human-readable operation name. */
+    name: string;
+    /** Coarse operation kind used by reporters and judges. */
+    kind?: "run" | "agent" | "model" | "tool" | "guardrail" | "handoff" | "custom";
+    /** ISO timestamp for the start of the span. */
+    startedAt?: string;
+    /** ISO timestamp for the end of the span. */
+    finishedAt?: string;
+    /** Span duration in milliseconds. */
+    durationMs?: number;
+    /** Success or failure status for the span. */
+    status?: "ok" | "error";
+    /** Normalized error when the span failed. */
+    error?: {
+        message: string;
+        type?: string;
+        [key: string]: JsonValue | undefined;
+    };
+    /** JSON-safe operation attributes. */
+    attributes?: NormalizedSpanAttributes;
+    /** Events observed inside this span. */
+    events?: NormalizedSpanEvent[];
+};
+/** Normalized trace captured during a harness run. */
+type NormalizedTrace = {
+    /** Runtime or provider trace id when one is available. */
+    id?: string;
+    /** Human-readable trace or workflow name. */
+    name?: string;
+    /** ISO timestamp for the start of the trace. */
+    startedAt?: string;
+    /** ISO timestamp for the end of the trace. */
+    finishedAt?: string;
+    /** Trace duration in milliseconds. */
+    durationMs?: number;
+    /** Extra JSON-safe trace metadata. */
+    metadata?: Record<string, JsonValue>;
+    /** Spans that make up this trace. */
+    spans: NormalizedSpan[];
+};
+/** Options for converting normalized tool calls into trace spans. */
+type CreateToolCallSpansOptions = {
+    /** Trace id to attach to each generated tool span. */
+    traceId?: string;
+    /** Parent span id to attach to each generated tool span. */
+    parentId?: string;
+    /** Prefix used to create internal span ids instead of reusing tool-call ids. */
+    spanIdPrefix?: string;
+};
+/** Options for attaching a fallback run trace to a harness result. */
+type EnsureRunTraceOptions = {
+    /** Human-readable run or harness name. */
+    name: string;
+    /** Wall-clock start time for the harness run. */
+    startedAt: Date;
+    /** Wall-clock finish time for the harness run. */
+    finishedAt: Date;
+    /** Optional trace id. A generated id is used when omitted. */
+    id?: string;
+    /** GenAI operation name to place on the root run span. */
+    operationName?: GenAiOperationName;
+    /** Optional JSON-safe source marker for the trace metadata. */
+    source?: string;
+};
 /**
  * Normalized record for one tool call observed during a harness run.
  *
@@ -160,6 +323,8 @@ type HarnessRun<TOutput extends JsonValue | undefined = JsonValue | undefined> =
     timings?: TimingSummary;
     /** JSON-safe run artifacts captured by the harness or test context. */
     artifacts?: Record<string, JsonValue>;
+    /** Normalized traces and spans captured during execution. */
+    traces?: NormalizedTrace[];
     /** Normalized errors captured during execution. */
     errors: Array<Record<string, JsonValue>>;
 };
@@ -232,6 +397,27 @@ type SimpleToolCallRecord = Omit<ToolCallRecord, "arguments" | "result" | "error
     /** Raw tool metadata accepted by `createHarness(...)` before normalization. */
     metadata?: Record<string, unknown>;
 };
+/** Lightweight span event accepted by `createHarness(...)` results. */
+type SimpleSpanEvent = Omit<NormalizedSpanEvent, "attributes"> & {
+    /** Raw event attributes accepted by `createHarness(...)` before normalization. */
+    attributes?: Record<string, unknown>;
+};
+/** Lightweight span record accepted by `createHarness(...)` results. */
+type SimpleSpanRecord = Omit<NormalizedSpan, "attributes" | "error" | "events"> & {
+    /** Raw span attributes accepted by `createHarness(...)` before normalization. */
+    attributes?: Record<string, unknown>;
+    /** Raw span error accepted by `createHarness(...)` before normalization. */
+    error?: unknown;
+    /** Raw span events accepted by `createHarness(...)` before normalization. */
+    events?: SimpleSpanEvent[];
+};
+/** Lightweight trace record accepted by `createHarness(...)` results. */
+type SimpleTraceRecord = Omit<NormalizedTrace, "metadata" | "spans"> & {
+    /** Raw trace metadata accepted by `createHarness(...)` before normalization. */
+    metadata?: Record<string, unknown>;
+    /** Lightweight spans to normalize into the trace. */
+    spans: SimpleSpanRecord[];
+};
 /**
  * Lightweight result shape normalized by `createHarness(...)`.
  *
@@ -255,6 +441,8 @@ type SimpleHarnessResult<TOutput extends JsonValue | undefined = JsonValue | und
     timings?: TimingSummary;
     /** Raw artifact values to normalize and merge into the run. */
     artifacts?: Record<string, unknown>;
+    /** Lightweight traces and spans to normalize into the run. */
+    traces?: SimpleTraceRecord[];
     /** Raw session metadata to normalize into the session. */
     metadata?: Record<string, unknown>;
     /** Raw errors to normalize into the run. */
@@ -354,6 +542,31 @@ declare function createHarness<TInput = unknown, TOutput extends JsonValue | und
  * ```
  */
 declare function normalizeHarnessRun<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, TOutput extends JsonValue | undefined = JsonValue | undefined>(input: TInput, result: HarnessResultLike<TOutput>, context?: HarnessContext<TMetadata>): HarnessRun<TOutput>;
+/**
+ * Builds a JSON-safe failed run for errors that happen before a harness can return.
+ *
+ * @param input - Original input passed to the harness.
+ * @param error - Error thrown by setup or execution.
+ * @param options - Optional artifacts to preserve on the failed run.
+ */
+declare function createFailedHarnessRun(input: unknown, error: unknown, options?: {
+    artifacts?: Record<string, JsonValue>;
+}): HarnessRun;
+/** Normalizes arbitrary span errors while preserving object-shaped messages. */
+declare function normalizeSpanError(error: unknown): NormalizedSpan["error"] | undefined;
+/** Normalizes raw span attributes into the JSON-safe span attribute shape. */
+declare function normalizeSpanAttributes(attributes: Record<string, unknown>): NormalizedSpanAttributes | undefined;
+/** Builds common OpenTelemetry GenAI usage attributes from a usage summary. */
+declare function createGenAiUsageAttributes(usage: UsageSummary | undefined, options?: {
+    provider?: string;
+}): {
+    "gen_ai.provider.name": string | undefined;
+    "gen_ai.request.model": string | undefined;
+    "gen_ai.response.model": string | undefined;
+    "gen_ai.usage.input_tokens": number | undefined;
+    "gen_ai.usage.output_tokens": number | undefined;
+    "gen_ai.usage.reasoning.output_tokens": number | undefined;
+};
 /**
  * Flattens every recorded tool call from a normalized session.
  *
@@ -367,6 +580,44 @@ declare function normalizeHarnessRun<TInput = unknown, TMetadata extends Harness
  * ```
  */
 declare function toolCalls(session: NormalizedSession): ToolCallRecord[];
+/**
+ * Converts normalized tool-call records into trace spans.
+ *
+ * Tool-call ids are preserved as GenAI attributes. Pass `spanIdPrefix` when the
+ * spans belong to a known trace so span ids stay internally unique.
+ */
+declare function createToolCallSpans(calls: ToolCallRecord[], options?: CreateToolCallSpansOptions): NormalizedSpan[];
+/**
+ * Attaches a fallback run trace when a harness result does not already contain spans.
+ *
+ * This keeps custom harnesses inspectable while first-party harness packages
+ * remain free to attach richer native traces.
+ */
+declare function ensureRunTrace(run: HarnessRun, options: EnsureRunTraceOptions): NormalizedTrace | undefined;
+/**
+ * Flattens every recorded span from a normalized harness run.
+ *
+ * @param run - Normalized harness run produced by a harness.
+ *
+ * @example
+ * ```ts
+ * const modelSpans = spans(result).filter((span) => span.kind === "model");
+ * ```
+ */
+declare function spans(run: HarnessRun): NormalizedSpan[];
+/**
+ * Returns spans of one coarse operation kind from a normalized run.
+ *
+ * @param run - Normalized harness run produced by a harness.
+ * @param kind - Span kind to keep.
+ */
+declare function spansByKind(run: HarnessRun, kind: NonNullable<NormalizedSpan["kind"]>): NormalizedSpan[];
+/**
+ * Returns every span that explicitly failed or carries a normalized error.
+ *
+ * @param run - Normalized harness run produced by a harness.
+ */
+declare function failedSpans(run: HarnessRun): NormalizedSpan[];
 /**
  * Filters normalized session messages by role.
  *
@@ -414,6 +665,17 @@ declare function userMessages(session: NormalizedSession): NormalizedMessage[];
  * ```
  */
 declare function assistantMessages(session: NormalizedSession): NormalizedMessage[];
+/**
+ * Returns the latest assistant message content, ignoring empty text messages.
+ *
+ * @param session - Normalized session produced by a harness run.
+ *
+ * @example
+ * ```ts
+ * const finalAnswer = latestAssistantMessageContent(result.session);
+ * ```
+ */
+declare function latestAssistantMessageContent(session: NormalizedSession): JsonValue | undefined;
 /**
  * Returns every normalized tool message from a session.
  *
@@ -465,4 +727,4 @@ declare function resolveHarnessRunErrors(result: unknown): Array<Record<string,
 /** Serializes an arbitrary thrown value into the normalized error shape. */
 declare function serializeError(error: unknown): Record<string, JsonValue>;
 
-export { type CreateHarnessOptions, type CreateHarnessRunArgs, type Harness, type HarnessContext, type HarnessMetadata, type HarnessResultLike, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type MaybePromise, type NormalizedMessage, type NormalizedSession, type SimpleHarnessResult, type SimpleToolCallRecord, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, createHarness, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, messagesByRole, normalizeContent, normalizeHarnessRun, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };
+export { type CreateHarnessOptions, type CreateHarnessRunArgs, type CreateToolCallSpansOptions, type EnsureRunTraceOptions, type GenAiOperationName, type GenAiOutputType, type GenAiProviderName, type GenAiSemanticAttributeKey, type GenAiSemanticAttributes, type GenAiTokenType, type GenAiToolType, type Harness, type HarnessContext, type HarnessMetadata, type HarnessResultLike, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type MaybePromise, type NormalizedMessage, type NormalizedSession, type NormalizedSpan, type NormalizedSpanAttributeKey, type NormalizedSpanAttributes, type NormalizedSpanEvent, type NormalizedTrace, type OpenTelemetrySemanticAttributeKey, type OpenTelemetrySemanticAttributes, type SimpleHarnessResult, type SimpleSpanEvent, type SimpleSpanRecord, type SimpleToolCallRecord, type SimpleTraceRecord, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, failedSpans, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, latestAssistantMessageContent, messagesByRole, normalizeContent, normalizeHarnessRun, normalizeMetadata, normalizeRecord, normalizeSpanAttributes, normalizeSpanError, resolveHarnessRunErrors, serializeError, spans, spansByKind, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };