vitest-evals 0.10.0 → 0.12.0

package/README.md

@@ -33,8 +33,14 @@ workflow.
 - `run(input, { metadata? })` executes the harness explicitly and returns a
   normalized `HarnessRun`
 - the returned `result.output` is the app-facing value you assert on directly
-- the returned `result.session` is the canonical JSON-serializable trace for
+- the returned `result.session` is the canonical JSON-serializable transcript for
   reporting, replay, tool assertions, and judges
+- the returned `result.traces` contains JSON-serializable operation spans; the
+  first-party harnesses attach run, model, and tool spans automatically, while
+  `createHarness(...)` attaches fallback run and tool spans for custom harnesses
+  that do not return traces themselves. Span attributes include typed
+  OpenTelemetry GenAI semantic keys while still allowing provider-specific
+  metadata
 - scenario-specific judge criteria can live in `input`; use `metadata` for
   per-run expectations or harness configuration that are not part of the
   scenario payload
@@ -127,6 +133,29 @@ describeEval("refund agent", { harness }, (it) => {
 });
 ```
 
+## Terminal Reporting
+
+The terminal reporter has two eval report levels. Normal mode prints compact
+test, score, usage, and tool-count summaries. Info mode adds per-tool summaries,
+arguments, timing/size metadata, replay status, and final output summaries.
+Set `VITEST_EVALS_REPORT_LEVEL=info`, or pass `--info` through the workspace
+eval scripts, to enable it. `--verbose` and `-v` remain aliases for
+compatibility.
+
+Full transcripts and spans are preserved in the Vitest JSON report metadata.
+
+## Local Report UI
+
+The local report UI reads the same Vitest JSON artifacts and serves a React SPA
+for drilling into runs, eval cases, harness output, sessions, tool calls,
+scores, and trace spans.
+
+```sh
+pnpm exec vitest-evals serve vitest-results.json
+pnpm exec vitest-evals serve "eval-results/*.json"
+pnpm exec vitest-evals serve eval-results/
+```
+
 ## GitHub Actions Reporting
 
 Use Vitest JSON as the eval report artifact. It preserves the `meta` field that

package/bin/vitest-evals.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+import("../dist/cli.mjs")
+  .then(({ runVitestEvalsCli }) => runVitestEvalsCli(process.argv.slice(2)))
+  .catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });

package/dist/cli.d.mts

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+/** Output streams used by the `vitest-evals` CLI runner. */
+type VitestEvalsCliIo = {
+    stdout?: Pick<NodeJS.WriteStream, "write">;
+};
+/** Options for running the `vitest-evals` CLI. */
+type RunVitestEvalsCliOptions = VitestEvalsCliIo & {
+    cwd?: string;
+};
+/** Runs the product-facing `vitest-evals` CLI. */
+declare function runVitestEvalsCli(args?: string[], options?: RunVitestEvalsCliOptions): Promise<void>;
+
+export { type RunVitestEvalsCliOptions, type VitestEvalsCliIo, runVitestEvalsCli };

package/dist/cli.d.ts

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+/** Output streams used by the `vitest-evals` CLI runner. */
+type VitestEvalsCliIo = {
+    stdout?: Pick<NodeJS.WriteStream, "write">;
+};
+/** Options for running the `vitest-evals` CLI. */
+type RunVitestEvalsCliOptions = VitestEvalsCliIo & {
+    cwd?: string;
+};
+/** Runs the product-facing `vitest-evals` CLI. */
+declare function runVitestEvalsCli(args?: string[], options?: RunVitestEvalsCliOptions): Promise<void>;
+
+export { type RunVitestEvalsCliOptions, type VitestEvalsCliIo, runVitestEvalsCli };

package/dist/cli.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/cli.ts
+var cli_exports = {};
+__export(cli_exports, {
+  runVitestEvalsCli: () => runVitestEvalsCli
+});
+module.exports = __toCommonJS(cli_exports);
+async function runVitestEvalsCli(args = process.argv.slice(2), options = {}) {
+  const [command, ...commandArgs] = args;
+  if (!command || command === "help" || command === "--help" || command === "-h") {
+    writeLine(options.stdout, usage());
+    return;
+  }
+  switch (command) {
+    case "serve": {
+      const { runReportUiCli } = await import("@vitest-evals/report-ui");
+      await runReportUiCli(commandArgs, {
+        commandName: "vitest-evals serve",
+        cwd: options.cwd,
+        stdout: options.stdout
+      });
+      return;
+    }
+    default:
+      throw new Error(`Unknown command: ${command}
+
+${usage()}`);
+  }
+}
+function usage() {
+  return [
+    "Usage: vitest-evals <command>",
+    "",
+    "Commands:",
+    "  serve [json | dir | glob]   Serve the local report UI",
+    "",
+    "Run `vitest-evals serve --help` for report UI options."
+  ].join("\n");
+}
+function writeLine(stdout, message) {
+  (stdout ?? process.stdout).write(`${message}
+`);
+}
+if (typeof require !== "undefined" && typeof module !== "undefined" && require.main === module) {
+  runVitestEvalsCli().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  runVitestEvalsCli
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n\n/** Output streams used by the `vitest-evals` CLI runner. */\nexport type VitestEvalsCliIo = {\n  stdout?: Pick<NodeJS.WriteStream, \"write\">;\n};\n\n/** Options for running the `vitest-evals` CLI. */\nexport type RunVitestEvalsCliOptions = VitestEvalsCliIo & {\n  cwd?: string;\n};\n\n/** Runs the product-facing `vitest-evals` CLI. */\nexport async function runVitestEvalsCli(\n  args = process.argv.slice(2),\n  options: RunVitestEvalsCliOptions = {},\n) {\n  const [command, ...commandArgs] = args;\n\n  if (\n    !command ||\n    command === \"help\" ||\n    command === \"--help\" ||\n    command === \"-h\"\n  ) {\n    writeLine(options.stdout, usage());\n    return;\n  }\n\n  switch (command) {\n    case \"serve\": {\n      const { runReportUiCli } = await import(\"@vitest-evals/report-ui\");\n      await runReportUiCli(commandArgs, {\n        commandName: \"vitest-evals serve\",\n        cwd: options.cwd,\n        stdout: options.stdout,\n      });\n      return;\n    }\n    default:\n      throw new Error(`Unknown command: ${command}\\n\\n${usage()}`);\n  }\n}\n\nfunction usage() {\n  return [\n    \"Usage: vitest-evals <command>\",\n    \"\",\n    \"Commands:\",\n    \"  serve [json | dir | glob]   Serve the local report UI\",\n    \"\",\n    \"Run `vitest-evals serve --help` for report UI options.\",\n  ].join(\"\\n\");\n}\n\nfunction writeLine(\n  stdout: Pick<NodeJS.WriteStream, \"write\"> | undefined,\n  message: string,\n) {\n  (stdout ?? process.stdout).write(`${message}\\n`);\n}\n\ndeclare const require: NodeJS.Require | undefined;\ndeclare const module: NodeJS.Module | undefined;\n\nif (\n  typeof require !== \"undefined\" &&\n  typeof module !== \"undefined\" &&\n  require.main === module\n) {\n  runVitestEvalsCli().catch((error) => {\n    console.error(error instanceof Error ? error.message : String(error));\n    process.exitCode = 1;\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAaA,eAAsB,kBACpB,OAAO,QAAQ,KAAK,MAAM,CAAC,GAC3B,UAAoC,CAAC,GACrC;AACA,QAAM,CAAC,SAAS,GAAG,WAAW,IAAI;AAElC,MACE,CAAC,WACD,YAAY,UACZ,YAAY,YACZ,YAAY,MACZ;AACA,cAAU,QAAQ,QAAQ,MAAM,CAAC;AACjC;AAAA,EACF;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,EAAE,eAAe,IAAI,MAAM,OAAO,yBAAyB;AACjE,YAAM,eAAe,aAAa;AAAA,QAChC,aAAa;AAAA,QACb,KAAK,QAAQ;AAAA,QACb,QAAQ,QAAQ;AAAA,MAClB,CAAC;AACD;AAAA,IACF;AAAA,IACA;AACE,YAAM,IAAI,MAAM,oBAAoB,OAAO;AAAA;AAAA,EAAO,MAAM,CAAC,EAAE;AAAA,EAC/D;AACF;AAEA,SAAS,QAAQ;AACf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UACP,QACA,SACA;AACA,GAAC,UAAU,QAAQ,QAAQ,MAAM,GAAG,OAAO;AAAA,CAAI;AACjD;AAKA,IACE,OAAO,YAAY,eACnB,OAAO,WAAW,eAClB,QAAQ,SAAS,QACjB;AACA,oBAAkB,EAAE,MAAM,CAAC,UAAU;AACnC,YAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AACpE,YAAQ,WAAW;AAAA,EACrB,CAAC;AACH;","names":[]}

package/dist/cli.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+
+// src/cli.ts
+async function runVitestEvalsCli(args = process.argv.slice(2), options = {}) {
+  const [command, ...commandArgs] = args;
+  if (!command || command === "help" || command === "--help" || command === "-h") {
+    writeLine(options.stdout, usage());
+    return;
+  }
+  switch (command) {
+    case "serve": {
+      const { runReportUiCli } = await import("@vitest-evals/report-ui");
+      await runReportUiCli(commandArgs, {
+        commandName: "vitest-evals serve",
+        cwd: options.cwd,
+        stdout: options.stdout
+      });
+      return;
+    }
+    default:
+      throw new Error(`Unknown command: ${command}
+
+${usage()}`);
+  }
+}
+function usage() {
+  return [
+    "Usage: vitest-evals <command>",
+    "",
+    "Commands:",
+    "  serve [json | dir | glob]   Serve the local report UI",
+    "",
+    "Run `vitest-evals serve --help` for report UI options."
+  ].join("\n");
+}
+function writeLine(stdout, message) {
+  (stdout ?? process.stdout).write(`${message}
+`);
+}
+if (typeof __require !== "undefined" && typeof module !== "undefined" && __require.main === module) {
+  runVitestEvalsCli().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  });
+}
+export {
+  runVitestEvalsCli
+};
+//# sourceMappingURL=cli.mjs.map

package/dist/cli.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n\n/** Output streams used by the `vitest-evals` CLI runner. */\nexport type VitestEvalsCliIo = {\n  stdout?: Pick<NodeJS.WriteStream, \"write\">;\n};\n\n/** Options for running the `vitest-evals` CLI. */\nexport type RunVitestEvalsCliOptions = VitestEvalsCliIo & {\n  cwd?: string;\n};\n\n/** Runs the product-facing `vitest-evals` CLI. */\nexport async function runVitestEvalsCli(\n  args = process.argv.slice(2),\n  options: RunVitestEvalsCliOptions = {},\n) {\n  const [command, ...commandArgs] = args;\n\n  if (\n    !command ||\n    command === \"help\" ||\n    command === \"--help\" ||\n    command === \"-h\"\n  ) {\n    writeLine(options.stdout, usage());\n    return;\n  }\n\n  switch (command) {\n    case \"serve\": {\n      const { runReportUiCli } = await import(\"@vitest-evals/report-ui\");\n      await runReportUiCli(commandArgs, {\n        commandName: \"vitest-evals serve\",\n        cwd: options.cwd,\n        stdout: options.stdout,\n      });\n      return;\n    }\n    default:\n      throw new Error(`Unknown command: ${command}\\n\\n${usage()}`);\n  }\n}\n\nfunction usage() {\n  return [\n    \"Usage: vitest-evals <command>\",\n    \"\",\n    \"Commands:\",\n    \"  serve [json | dir | glob]   Serve the local report UI\",\n    \"\",\n    \"Run `vitest-evals serve --help` for report UI options.\",\n  ].join(\"\\n\");\n}\n\nfunction writeLine(\n  stdout: Pick<NodeJS.WriteStream, \"write\"> | undefined,\n  message: string,\n) {\n  (stdout ?? process.stdout).write(`${message}\\n`);\n}\n\ndeclare const require: NodeJS.Require | undefined;\ndeclare const module: NodeJS.Module | undefined;\n\nif (\n  typeof require !== \"undefined\" &&\n  typeof module !== \"undefined\" &&\n  require.main === module\n) {\n  runVitestEvalsCli().catch((error) => {\n    console.error(error instanceof Error ? error.message : String(error));\n    process.exitCode = 1;\n  });\n}\n"],"mappings":";;;;;;;;;AAaA,eAAsB,kBACpB,OAAO,QAAQ,KAAK,MAAM,CAAC,GAC3B,UAAoC,CAAC,GACrC;AACA,QAAM,CAAC,SAAS,GAAG,WAAW,IAAI;AAElC,MACE,CAAC,WACD,YAAY,UACZ,YAAY,YACZ,YAAY,MACZ;AACA,cAAU,QAAQ,QAAQ,MAAM,CAAC;AACjC;AAAA,EACF;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,EAAE,eAAe,IAAI,MAAM,OAAO,yBAAyB;AACjE,YAAM,eAAe,aAAa;AAAA,QAChC,aAAa;AAAA,QACb,KAAK,QAAQ;AAAA,QACb,QAAQ,QAAQ;AAAA,MAClB,CAAC;AACD;AAAA,IACF;AAAA,IACA;AACE,YAAM,IAAI,MAAM,oBAAoB,OAAO;AAAA;AAAA,EAAO,MAAM,CAAC,EAAE;AAAA,EAC/D;AACF;AAEA,SAAS,QAAQ;AACf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UACP,QACA,SACA;AACA,GAAC,UAAU,QAAQ,QAAQ,MAAM,GAAG,OAAO;AAAA,CAAI;AACjD;AAKA,IACE,OAAO,cAAY,eACnB,OAAO,WAAW,eAClB,UAAQ,SAAS,QACjB;AACA,oBAAkB,EAAE,MAAM,CAAC,UAAU;AACnC,YAAQ,MAAM,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AACpE,YAAQ,WAAW;AAAA,EACrB,CAAC;AACH;","names":[]}

package/dist/harness.d.mts

@@ -1,173 +1,35 @@
-/** Primitive scalar values allowed in normalized JSON-safe eval data. */
-type JsonPrimitive = string | number | boolean | null;
-/** JSON-safe value shape used by normalized sessions, artifacts, and errors. */
-type JsonValue = JsonPrimitive | JsonValue[] | {
-    [key: string]: JsonValue;
+import { JsonValue, HarnessRun, GenAiOperationName, ToolCallRecord, NormalizedSpanEvent, NormalizedSpan, NormalizedTrace, NormalizedMessage, UsageSummary, TimingSummary, NormalizedSpanAttributes, HarnessRunError, NormalizedSession } from '@vitest-evals/core';
+export { GenAiOperationName, GenAiOutputType, GenAiProviderName, GenAiSemanticAttributeKey, GenAiSemanticAttributes, GenAiTokenType, GenAiToolType, HarnessRun, HarnessRunError, JsonPrimitive, JsonValue, NormalizedMessage, NormalizedSession, NormalizedSpan, NormalizedSpanAttributeKey, NormalizedSpanAttributes, NormalizedSpanEvent, NormalizedTrace, OpenTelemetrySemanticAttributeKey, OpenTelemetrySemanticAttributes, TimingSummary, ToolCallRecord, UsageSummary, assistantMessages, failedSpans, latestAssistantMessageContent, messagesByRole, spans, spansByKind, systemMessages, toolCalls, toolMessages, userMessages } from '@vitest-evals/core';
+
+/** Options for converting normalized tool calls into trace spans. */
+type CreateToolCallSpansOptions = {
+    /** Trace id to attach to each generated tool span. */
+    traceId?: string;
+    /** Parent span id to attach to each generated tool span. */
+    parentId?: string;
+    /** Prefix used to create internal span ids instead of reusing tool-call ids. */
+    spanIdPrefix?: string;
 };
-/**
- * Normalized record for one tool call observed during a harness run.
- *
- * @example
- * ```ts
- * const call: ToolCallRecord = {
- *   name: "lookupInvoice",
- *   arguments: { invoiceId: "inv_123" },
- *   result: { refundable: true },
- * };
- * ```
- */
-type ToolCallRecord = {
-    /** Provider or runtime tool-call id when one is available. */
-    id?: string;
-    /** Tool name as exposed to the agent or application runtime. */
+/** Options for attaching a fallback run trace to a harness result. */
+type EnsureRunTraceOptions = {
+    /** Human-readable run or harness name. */
     name: string;
-    /** JSON-safe tool arguments after provider/runtime normalization. */
-    arguments?: Record<string, JsonValue>;
-    /** JSON-safe tool result returned by the application tool. */
-    result?: JsonValue;
-    /** Normalized tool error when execution failed. */
-    error?: {
-        message: string;
-        type?: string;
-        [key: string]: JsonValue | undefined;
-    };
-    /** ISO timestamp for the start of tool execution. */
-    startedAt?: string;
-    /** ISO timestamp for the end of tool execution. */
-    finishedAt?: string;
-    /** Tool execution duration in milliseconds. */
-    durationMs?: number;
-    /** Extra JSON-safe tool metadata for reporters and custom judges. */
-    metadata?: Record<string, JsonValue>;
-};
-/**
- * Normalized message recorded in a harness session transcript.
- *
- * @example
- * ```ts
- * const message: NormalizedMessage = {
- *   role: "assistant",
- *   content: { status: "approved" },
- *   toolCalls: [{ name: "lookupInvoice" }],
- * };
- * ```
- */
-type NormalizedMessage = {
-    /** Transcript role for the normalized message. */
-    role: "system" | "user" | "assistant" | "tool";
-    /** JSON-safe message content. */
-    content?: JsonValue;
-    /** Tool calls associated with this message. */
-    toolCalls?: ToolCallRecord[];
-    /** Extra JSON-safe message metadata. */
-    metadata?: Record<string, JsonValue>;
-};
-/**
- * Provider usage summary attached to a normalized harness run.
- *
- * @example
- * ```ts
- * const usage: UsageSummary = {
- *   provider: "openai",
- *   model: "gpt-4o-mini",
- *   inputTokens: 212,
- *   outputTokens: 48,
- *   totalTokens: 260,
- * };
- * ```
- */
-type UsageSummary = {
-    /** Provider that served the application run. */
-    provider?: string;
-    /** Model used for the application run. */
-    model?: string;
-    /** Input, prompt, or request tokens consumed by the run. */
-    inputTokens?: number;
-    /** Output or completion tokens produced by the run. */
-    outputTokens?: number;
-    /** Reasoning tokens reported by providers that expose them. */
-    reasoningTokens?: number;
-    /** Total token count reported by the provider or adapter. */
-    totalTokens?: number;
-    /** Count of tool calls observed during the run. */
-    toolCalls?: number;
-    /** Retry count observed during the run. */
-    retries?: number;
-    /** Provider-specific JSON-safe usage details. Cost estimates belong here. */
-    metadata?: Record<string, JsonValue>;
-};
-/** Timing summary attached to a normalized harness run. */
-type TimingSummary = {
-    /** End-to-end run duration in milliseconds. */
-    totalMs?: number;
-    /** Extra JSON-safe timing metadata. */
-    metadata?: Record<string, JsonValue>;
-};
-/**
- * JSON-serializable transcript produced by the system under test.
- *
- * @example
- * ```ts
- * const session: NormalizedSession = {
- *   provider: "openai",
- *   model: "gpt-4o-mini",
- *   messages: [
- *     { role: "user", content: "Refund invoice inv_123" },
- *     { role: "assistant", content: { status: "approved" } },
- *   ],
- * };
- * ```
- */
-type NormalizedSession = {
-    /** Ordered normalized transcript messages. */
-    messages: NormalizedMessage[];
-    /** Provider that produced the session when known. */
-    provider?: string;
-    /** Model that produced the session when known. */
-    model?: string;
-    /** Extra JSON-safe session metadata. */
-    metadata?: Record<string, JsonValue>;
+    /** Wall-clock start time for the harness run. */
+    startedAt: Date;
+    /** Wall-clock finish time for the harness run. */
+    finishedAt: Date;
+    /** Optional trace id. A generated id is used when omitted. */
+    id?: string;
+    /** GenAI operation name to place on the root run span. */
+    operationName?: GenAiOperationName;
+    /** Optional JSON-safe source marker for the trace metadata. */
+    source?: string;
 };
 type OutputField<TOutput extends JsonValue | undefined> = undefined extends TOutput ? {
     output?: TOutput;
 } : {
     output: TOutput;
 };
-/**
- * Normalized result returned by every harness execution.
- *
- * @example
- * ```ts
- * const run: HarnessRun<{ status: "approved" }> = {
- *   output: { status: "approved" },
- *   session: {
- *     messages: [
- *       { role: "user", content: "Refund invoice inv_123" },
- *       { role: "assistant", content: { status: "approved" } },
- *     ],
- *   },
- *   usage: { totalTokens: 260 },
- *   errors: [],
- * };
- * ```
- */
-type HarnessRun<TOutput extends JsonValue | undefined = JsonValue | undefined> = OutputField<TOutput> & {
-    /** Normalized transcript and provider/session metadata. */
-    session: NormalizedSession;
-    /** Stable provider usage units such as tokens, tools, and retries. */
-    usage: UsageSummary;
-    /** Optional timing summary for the run. */
-    timings?: TimingSummary;
-    /** JSON-safe run artifacts captured by the harness or test context. */
-    artifacts?: Record<string, JsonValue>;
-    /** Normalized errors captured during execution. */
-    errors: Array<Record<string, JsonValue>>;
-};
-/** Error value with an attached partial or complete normalized harness run. */
-type HarnessRunError = Error & {
-    /** Attached normalized harness run recovered by `getHarnessRunFromError(...)`. */
-    vitestEvalsRun: HarnessRun;
-};
 /** Per-run metadata shape accepted by harnesses and eval tests. */
 type HarnessMetadata = Record<string, unknown>;
 /**
@@ -232,6 +94,27 @@ type SimpleToolCallRecord = Omit<ToolCallRecord, "arguments" | "result" | "error
     /** Raw tool metadata accepted by `createHarness(...)` before normalization. */
     metadata?: Record<string, unknown>;
 };
+/** Lightweight span event accepted by `createHarness(...)` results. */
+type SimpleSpanEvent = Omit<NormalizedSpanEvent, "attributes"> & {
+    /** Raw event attributes accepted by `createHarness(...)` before normalization. */
+    attributes?: Record<string, unknown>;
+};
+/** Lightweight span record accepted by `createHarness(...)` results. */
+type SimpleSpanRecord = Omit<NormalizedSpan, "attributes" | "error" | "events"> & {
+    /** Raw span attributes accepted by `createHarness(...)` before normalization. */
+    attributes?: Record<string, unknown>;
+    /** Raw span error accepted by `createHarness(...)` before normalization. */
+    error?: unknown;
+    /** Raw span events accepted by `createHarness(...)` before normalization. */
+    events?: SimpleSpanEvent[];
+};
+/** Lightweight trace record accepted by `createHarness(...)` results. */
+type SimpleTraceRecord = Omit<NormalizedTrace, "metadata" | "spans"> & {
+    /** Raw trace metadata accepted by `createHarness(...)` before normalization. */
+    metadata?: Record<string, unknown>;
+    /** Lightweight spans to normalize into the trace. */
+    spans: SimpleSpanRecord[];
+};
 /**
  * Lightweight result shape normalized by `createHarness(...)`.
  *
@@ -255,6 +138,8 @@ type SimpleHarnessResult<TOutput extends JsonValue | undefined = JsonValue | und
     timings?: TimingSummary;
     /** Raw artifact values to normalize and merge into the run. */
     artifacts?: Record<string, unknown>;
+    /** Lightweight traces and spans to normalize into the run. */
+    traces?: SimpleTraceRecord[];
     /** Raw session metadata to normalize into the session. */
     metadata?: Record<string, unknown>;
     /** Raw errors to normalize into the run. */
@@ -355,87 +240,44 @@ declare function createHarness<TInput = unknown, TOutput extends JsonValue | und
  */
 declare function normalizeHarnessRun<TInput = unknown, TMetadata extends HarnessMetadata = HarnessMetadata, TOutput extends JsonValue | undefined = JsonValue | undefined>(input: TInput, result: HarnessResultLike<TOutput>, context?: HarnessContext<TMetadata>): HarnessRun<TOutput>;
 /**
- * Flattens every recorded tool call from a normalized session.
- *
- * @param session - Normalized session produced by a harness run.
- *
- * @example
- * ```ts
- * const names = toolCalls(result.session).map((call) => call.name);
- *
- * expect(names).toEqual(["lookupInvoice", "createRefund"]);
- * ```
- */
-declare function toolCalls(session: NormalizedSession): ToolCallRecord[];
-/**
- * Filters normalized session messages by role.
- *
- * @param session - Normalized session produced by a harness run.
- * @param role - Message role to keep.
- *
- * @example
- * ```ts
- * const assistantText = messagesByRole(result.session, "assistant")
- *   .map((message) => message.content)
- *   .join("\n");
- * ```
- */
-declare function messagesByRole(session: NormalizedSession, role: NormalizedMessage["role"]): NormalizedMessage[];
-/**
- * Returns every normalized system message from a session.
- *
- * @param session - Normalized session produced by a harness run.
- *
- * @example
- * ```ts
- * const systemPrompts = systemMessages(result.session);
- * ```
- */
-declare function systemMessages(session: NormalizedSession): NormalizedMessage[];
-/**
- * Returns every normalized user message from a session.
+ * Builds a JSON-safe failed run for errors that happen before a harness can return.
  *
- * @param session - Normalized session produced by a harness run.
- *
- * @example
- * ```ts
- * const firstPrompt = userMessages(result.session)[0]?.content;
- * ```
- */
-declare function userMessages(session: NormalizedSession): NormalizedMessage[];
-/**
- * Returns every normalized assistant message from a session.
- *
- * @param session - Normalized session produced by a harness run.
- *
- * @example
- * ```ts
- * const finalAnswer = assistantMessages(result.session).at(-1)?.content;
- * ```
+ * @param input - Original input passed to the harness.
+ * @param error - Error thrown by setup or execution.
+ * @param options - Optional artifacts to preserve on the failed run.
  */
-declare function assistantMessages(session: NormalizedSession): NormalizedMessage[];
+declare function createFailedHarnessRun(input: unknown, error: unknown, options?: {
+    artifacts?: Record<string, JsonValue>;
+}): HarnessRun;
+/** Normalizes arbitrary span errors while preserving object-shaped messages. */
+declare function normalizeSpanError(error: unknown): NormalizedSpan["error"] | undefined;
+/** Normalizes raw span attributes into the JSON-safe span attribute shape. */
+declare function normalizeSpanAttributes(attributes: Record<string, unknown>): NormalizedSpanAttributes | undefined;
+/** Builds common OpenTelemetry GenAI usage attributes from a usage summary. */
+declare function createGenAiUsageAttributes(usage: UsageSummary | undefined, options?: {
+    provider?: string;
+}): {
+    "gen_ai.provider.name": string | undefined;
+    "gen_ai.request.model": string | undefined;
+    "gen_ai.response.model": string | undefined;
+    "gen_ai.usage.input_tokens": number | undefined;
+    "gen_ai.usage.output_tokens": number | undefined;
+    "gen_ai.usage.reasoning.output_tokens": number | undefined;
+};
 /**
- * Returns the latest assistant message content, ignoring empty text messages.
- *
- * @param session - Normalized session produced by a harness run.
+ * Converts normalized tool-call records into trace spans.
  *
- * @example
- * ```ts
- * const finalAnswer = latestAssistantMessageContent(result.session);
- * ```
+ * Tool-call ids are preserved as GenAI attributes. Pass `spanIdPrefix` when the
+ * spans belong to a known trace so span ids stay internally unique.
  */
-declare function latestAssistantMessageContent(session: NormalizedSession): JsonValue | undefined;
+declare function createToolCallSpans(calls: ToolCallRecord[], options?: CreateToolCallSpansOptions): NormalizedSpan[];
 /**
- * Returns every normalized tool message from a session.
- *
- * @param session - Normalized session produced by a harness run.
+ * Attaches a fallback run trace when a harness result does not already contain spans.
  *
- * @example
- * ```ts
- * const toolOutputs = toolMessages(result.session).map((message) => message.content);
- * ```
+ * This keeps custom harnesses inspectable while first-party harness packages
+ * remain free to attach richer native traces.
  */
-declare function toolMessages(session: NormalizedSession): NormalizedMessage[];
+declare function ensureRunTrace(run: HarnessRun, options: EnsureRunTraceOptions): NormalizedTrace | undefined;
 /**
  * Attaches a partial or complete harness run to an arbitrary thrown error.
  *
@@ -476,4 +318,4 @@ declare function resolveHarnessRunErrors(result: unknown): Array<Record<string,
 /** Serializes an arbitrary thrown value into the normalized error shape. */
 declare function serializeError(error: unknown): Record<string, JsonValue>;
 
-export { type CreateHarnessOptions, type CreateHarnessRunArgs, type Harness, type HarnessContext, type HarnessMetadata, type HarnessResultLike, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type MaybePromise, type NormalizedMessage, type NormalizedSession, type SimpleHarnessResult, type SimpleToolCallRecord, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, createHarness, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, latestAssistantMessageContent, messagesByRole, normalizeContent, normalizeHarnessRun, normalizeMetadata, normalizeRecord, resolveHarnessRunErrors, serializeError, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };
+export { type CreateHarnessOptions, type CreateHarnessRunArgs, type CreateToolCallSpansOptions, type EnsureRunTraceOptions, type Harness, type HarnessContext, type HarnessMetadata, type HarnessResultLike, type MaybePromise, type SimpleHarnessResult, type SimpleSpanEvent, type SimpleSpanRecord, type SimpleToolCallRecord, type SimpleTraceRecord, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, normalizeContent, normalizeHarnessRun, normalizeMetadata, normalizeRecord, normalizeSpanAttributes, normalizeSpanError, resolveHarnessRunErrors, serializeError, toJsonValue };