vitest-evals 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +12 -0
  2. package/bin/vitest-evals.js +8 -0
  3. package/dist/cli.d.mts +13 -0
  4. package/dist/cli.d.ts +13 -0
  5. package/dist/cli.js +83 -0
  6. package/dist/cli.js.map +1 -0
  7. package/dist/cli.mjs +55 -0
  8. package/dist/cli.mjs.map +1 -0
  9. package/dist/harness.d.mts +4 -413
  10. package/dist/harness.d.ts +4 -413
  11. package/dist/harness.js +19 -50
  12. package/dist/harness.js.map +1 -1
  13. package/dist/harness.mjs +31 -48
  14. package/dist/harness.mjs.map +1 -1
  15. package/dist/index.d.mts +5 -3
  16. package/dist/index.d.ts +5 -3
  17. package/dist/index.js +25 -56
  18. package/dist/index.js.map +1 -1
  19. package/dist/index.mjs +37 -54
  20. package/dist/index.mjs.map +1 -1
  21. package/dist/internal/scoring.d.mts +1 -1
  22. package/dist/internal/scoring.d.ts +1 -1
  23. package/dist/internal/structuredOutputScorer.d.mts +1 -1
  24. package/dist/internal/structuredOutputScorer.d.ts +1 -1
  25. package/dist/internal/toolCallScorer.d.mts +1 -1
  26. package/dist/internal/toolCallScorer.d.ts +1 -1
  27. package/dist/internal/toolCallScorer.js +2 -0
  28. package/dist/internal/toolCallScorer.js.map +1 -1
  29. package/dist/internal/toolCallScorer.mjs +16 -0
  30. package/dist/internal/toolCallScorer.mjs.map +1 -1
  31. package/dist/judges/factualityJudge.d.mts +2 -1
  32. package/dist/judges/factualityJudge.d.ts +2 -1
  33. package/dist/judges/factualityJudge.js +4 -14
  34. package/dist/judges/factualityJudge.js.map +1 -1
  35. package/dist/judges/factualityJudge.mjs +18 -14
  36. package/dist/judges/factualityJudge.mjs.map +1 -1
  37. package/dist/judges/index.d.mts +1 -0
  38. package/dist/judges/index.d.ts +1 -0
  39. package/dist/judges/index.js +11 -27
  40. package/dist/judges/index.js.map +1 -1
  41. package/dist/judges/index.mjs +23 -25
  42. package/dist/judges/index.mjs.map +1 -1
  43. package/dist/judges/judgeHarness.d.mts +2 -1
  44. package/dist/judges/judgeHarness.d.ts +2 -1
  45. package/dist/judges/judgeHarness.js +10 -26
  46. package/dist/judges/judgeHarness.js.map +1 -1
  47. package/dist/judges/judgeHarness.mjs +22 -24
  48. package/dist/judges/judgeHarness.mjs.map +1 -1
  49. package/dist/judges/structuredOutputJudge.d.mts +1 -0
  50. package/dist/judges/structuredOutputJudge.d.ts +1 -0
  51. package/dist/judges/toolCallJudge.d.mts +1 -0
  52. package/dist/judges/toolCallJudge.d.ts +1 -0
  53. package/dist/judges/toolCallJudge.js +2 -0
  54. package/dist/judges/toolCallJudge.js.map +1 -1
  55. package/dist/judges/toolCallJudge.mjs +16 -0
  56. package/dist/judges/toolCallJudge.mjs.map +1 -1
  57. package/dist/judges/types.d.mts +2 -1
  58. package/dist/judges/types.d.ts +2 -1
  59. package/dist/legacy/scorers/index.js +2 -0
  60. package/dist/legacy/scorers/index.js.map +1 -1
  61. package/dist/legacy/scorers/index.mjs +16 -0
  62. package/dist/legacy/scorers/index.mjs.map +1 -1
  63. package/dist/legacy/scorers/toolCallScorer.js +2 -0
  64. package/dist/legacy/scorers/toolCallScorer.js.map +1 -1
  65. package/dist/legacy/scorers/toolCallScorer.mjs +16 -0
  66. package/dist/legacy/scorers/toolCallScorer.mjs.map +1 -1
  67. package/dist/legacy.js +7 -5
  68. package/dist/legacy.js.map +1 -1
  69. package/dist/legacy.mjs +21 -5
  70. package/dist/legacy.mjs.map +1 -1
  71. package/dist/replay.d.mts +1 -1
  72. package/dist/replay.d.ts +1 -1
  73. package/dist/reporter.js +4 -5
  74. package/dist/reporter.js.map +1 -1
  75. package/dist/reporter.mjs +18 -5
  76. package/dist/reporter.mjs.map +1 -1
  77. package/package.json +9 -1
package/dist/harness.d.ts CHANGED
@@ -1,148 +1,6 @@
1
- /** Primitive scalar values allowed in normalized JSON-safe eval data. */
2
- type JsonPrimitive = string | number | boolean | null;
3
- /** JSON-safe value shape used by normalized sessions, artifacts, and errors. */
4
- type JsonValue = JsonPrimitive | JsonValue[] | {
5
- [key: string]: JsonValue;
6
- };
7
- /** Well-known OpenTelemetry GenAI operation names. */
8
- type GenAiOperationName = "chat" | "create_agent" | "embeddings" | "execute_tool" | "generate_content" | "invoke_agent" | "invoke_workflow" | "retrieval" | "text_completion" | (string & {});
9
- /** Well-known OpenTelemetry GenAI output content types. */
10
- type GenAiOutputType = "image" | "json" | "speech" | "text" | (string & {});
11
- /** Well-known OpenTelemetry GenAI provider names. */
12
- type GenAiProviderName = "anthropic" | "aws.bedrock" | "azure.ai.inference" | "azure.ai.openai" | "cohere" | "deepseek" | "gcp.gemini" | "gcp.gen_ai" | "gcp.vertex_ai" | "groq" | "ibm.watsonx.ai" | "mistral_ai" | "openai" | "perplexity" | "x_ai" | (string & {});
13
- /** Well-known OpenTelemetry GenAI token types. */
14
- type GenAiTokenType = "input" | "output" | (string & {});
15
- /** Well-known OpenTelemetry GenAI tool execution types. */
16
- type GenAiToolType = "datastore" | "extension" | "function" | (string & {});
17
- /** Typed subset of OpenTelemetry GenAI semantic attributes. */
18
- type GenAiSemanticAttributes = {
19
- "gen_ai.agent.description"?: string;
20
- "gen_ai.agent.id"?: string;
21
- "gen_ai.agent.name"?: string;
22
- "gen_ai.agent.version"?: string;
23
- "gen_ai.conversation.id"?: string;
24
- "gen_ai.data_source.id"?: string;
25
- "gen_ai.embeddings.dimension.count"?: number;
26
- "gen_ai.evaluation.explanation"?: string;
27
- "gen_ai.evaluation.name"?: string;
28
- "gen_ai.evaluation.score.label"?: string;
29
- "gen_ai.evaluation.score.value"?: number;
30
- "gen_ai.input.messages"?: JsonValue;
31
- "gen_ai.operation.name"?: GenAiOperationName;
32
- "gen_ai.output.messages"?: JsonValue;
33
- "gen_ai.output.type"?: GenAiOutputType;
34
- "gen_ai.prompt.name"?: string;
35
- "gen_ai.provider.name"?: GenAiProviderName;
36
- "gen_ai.request.choice.count"?: number;
37
- "gen_ai.request.encoding_formats"?: string[];
38
- "gen_ai.request.frequency_penalty"?: number;
39
- "gen_ai.request.max_tokens"?: number;
40
- "gen_ai.request.model"?: string;
41
- "gen_ai.request.presence_penalty"?: number;
42
- "gen_ai.request.seed"?: number;
43
- "gen_ai.request.stop_sequences"?: string[];
44
- "gen_ai.request.stream"?: boolean;
45
- "gen_ai.request.temperature"?: number;
46
- "gen_ai.request.top_k"?: number;
47
- "gen_ai.request.top_p"?: number;
48
- "gen_ai.response.finish_reasons"?: string[];
49
- "gen_ai.response.id"?: string;
50
- "gen_ai.response.model"?: string;
51
- "gen_ai.response.time_to_first_chunk"?: number;
52
- "gen_ai.retrieval.documents"?: JsonValue;
53
- "gen_ai.retrieval.query.text"?: string;
54
- "gen_ai.system_instructions"?: JsonValue;
55
- "gen_ai.token.type"?: GenAiTokenType;
56
- "gen_ai.tool.call.arguments"?: JsonValue;
57
- "gen_ai.tool.call.id"?: string;
58
- "gen_ai.tool.call.result"?: JsonValue;
59
- "gen_ai.tool.definitions"?: JsonValue;
60
- "gen_ai.tool.description"?: string;
61
- "gen_ai.tool.name"?: string;
62
- "gen_ai.tool.type"?: GenAiToolType;
63
- "gen_ai.usage.cache_creation.input_tokens"?: number;
64
- "gen_ai.usage.cache_read.input_tokens"?: number;
65
- "gen_ai.usage.input_tokens"?: number;
66
- "gen_ai.usage.output_tokens"?: number;
67
- "gen_ai.usage.reasoning.output_tokens"?: number;
68
- "gen_ai.workflow.name"?: string;
69
- };
70
- /** Attribute keys defined by the OpenTelemetry GenAI semantic conventions. */
71
- type GenAiSemanticAttributeKey = keyof GenAiSemanticAttributes;
72
- /** Typed OpenTelemetry semantic attributes accepted on normalized spans. */
73
- type OpenTelemetrySemanticAttributes = GenAiSemanticAttributes & {
74
- "error.type"?: string;
75
- "server.address"?: string;
76
- "server.port"?: number;
77
- };
78
- /** Known OpenTelemetry semantic attribute keys accepted on normalized spans. */
79
- type OpenTelemetrySemanticAttributeKey = keyof OpenTelemetrySemanticAttributes;
80
- /** Attribute keys accepted on normalized spans. */
81
- type NormalizedSpanAttributeKey = OpenTelemetrySemanticAttributeKey | (string & {});
82
- /**
83
- * JSON-safe span attributes. Known OpenTelemetry GenAI keys are typed while
84
- * custom provider and application keys remain allowed.
85
- */
86
- type NormalizedSpanAttributes = OpenTelemetrySemanticAttributes & {
87
- [key: string]: JsonValue | undefined;
88
- };
89
- /** Event attached to one normalized span. */
90
- type NormalizedSpanEvent = {
91
- /** Event name emitted by the runtime or harness. */
92
- name: string;
93
- /** ISO timestamp for the event when available. */
94
- timestamp?: string;
95
- /** JSON-safe event attributes. */
96
- attributes?: NormalizedSpanAttributes;
97
- };
98
- /** Normalized operation span captured during a harness run. */
99
- type NormalizedSpan = {
100
- /** Runtime or provider span id when one is available. */
101
- id?: string;
102
- /** Trace id this span belongs to. */
103
- traceId?: string;
104
- /** Parent span id when the runtime exposes hierarchy. */
105
- parentId?: string;
106
- /** Human-readable operation name. */
107
- name: string;
108
- /** Coarse operation kind used by reporters and judges. */
109
- kind?: "run" | "agent" | "model" | "tool" | "guardrail" | "handoff" | "custom";
110
- /** ISO timestamp for the start of the span. */
111
- startedAt?: string;
112
- /** ISO timestamp for the end of the span. */
113
- finishedAt?: string;
114
- /** Span duration in milliseconds. */
115
- durationMs?: number;
116
- /** Success or failure status for the span. */
117
- status?: "ok" | "error";
118
- /** Normalized error when the span failed. */
119
- error?: {
120
- message: string;
121
- type?: string;
122
- [key: string]: JsonValue | undefined;
123
- };
124
- /** JSON-safe operation attributes. */
125
- attributes?: NormalizedSpanAttributes;
126
- /** Events observed inside this span. */
127
- events?: NormalizedSpanEvent[];
128
- };
129
- /** Normalized trace captured during a harness run. */
130
- type NormalizedTrace = {
131
- /** Runtime or provider trace id when one is available. */
132
- id?: string;
133
- /** Human-readable trace or workflow name. */
134
- name?: string;
135
- /** ISO timestamp for the start of the trace. */
136
- startedAt?: string;
137
- /** ISO timestamp for the end of the trace. */
138
- finishedAt?: string;
139
- /** Trace duration in milliseconds. */
140
- durationMs?: number;
141
- /** Extra JSON-safe trace metadata. */
142
- metadata?: Record<string, JsonValue>;
143
- /** Spans that make up this trace. */
144
- spans: NormalizedSpan[];
145
- };
1
+ import { JsonValue, HarnessRun, GenAiOperationName, ToolCallRecord, NormalizedSpanEvent, NormalizedSpan, NormalizedTrace, NormalizedMessage, UsageSummary, TimingSummary, NormalizedSpanAttributes, HarnessRunError, NormalizedSession } from '@vitest-evals/core';
2
+ export { GenAiOperationName, GenAiOutputType, GenAiProviderName, GenAiSemanticAttributeKey, GenAiSemanticAttributes, GenAiTokenType, GenAiToolType, HarnessRun, HarnessRunError, JsonPrimitive, JsonValue, NormalizedMessage, NormalizedSession, NormalizedSpan, NormalizedSpanAttributeKey, NormalizedSpanAttributes, NormalizedSpanEvent, NormalizedTrace, OpenTelemetrySemanticAttributeKey, OpenTelemetrySemanticAttributes, TimingSummary, ToolCallRecord, UsageSummary, assistantMessages, failedSpans, latestAssistantMessageContent, messagesByRole, spans, spansByKind, systemMessages, toolCalls, toolMessages, userMessages } from '@vitest-evals/core';
3
+
146
4
  /** Options for converting normalized tool calls into trace spans. */
147
5
  type CreateToolCallSpansOptions = {
148
6
  /** Trace id to attach to each generated tool span. */
@@ -167,172 +25,11 @@ type EnsureRunTraceOptions = {
167
25
  /** Optional JSON-safe source marker for the trace metadata. */
168
26
  source?: string;
169
27
  };
170
- /**
171
- * Normalized record for one tool call observed during a harness run.
172
- *
173
- * @example
174
- * ```ts
175
- * const call: ToolCallRecord = {
176
- * name: "lookupInvoice",
177
- * arguments: { invoiceId: "inv_123" },
178
- * result: { refundable: true },
179
- * };
180
- * ```
181
- */
182
- type ToolCallRecord = {
183
- /** Provider or runtime tool-call id when one is available. */
184
- id?: string;
185
- /** Tool name as exposed to the agent or application runtime. */
186
- name: string;
187
- /** JSON-safe tool arguments after provider/runtime normalization. */
188
- arguments?: Record<string, JsonValue>;
189
- /** JSON-safe tool result returned by the application tool. */
190
- result?: JsonValue;
191
- /** Normalized tool error when execution failed. */
192
- error?: {
193
- message: string;
194
- type?: string;
195
- [key: string]: JsonValue | undefined;
196
- };
197
- /** ISO timestamp for the start of tool execution. */
198
- startedAt?: string;
199
- /** ISO timestamp for the end of tool execution. */
200
- finishedAt?: string;
201
- /** Tool execution duration in milliseconds. */
202
- durationMs?: number;
203
- /** Extra JSON-safe tool metadata for reporters and custom judges. */
204
- metadata?: Record<string, JsonValue>;
205
- };
206
- /**
207
- * Normalized message recorded in a harness session transcript.
208
- *
209
- * @example
210
- * ```ts
211
- * const message: NormalizedMessage = {
212
- * role: "assistant",
213
- * content: { status: "approved" },
214
- * toolCalls: [{ name: "lookupInvoice" }],
215
- * };
216
- * ```
217
- */
218
- type NormalizedMessage = {
219
- /** Transcript role for the normalized message. */
220
- role: "system" | "user" | "assistant" | "tool";
221
- /** JSON-safe message content. */
222
- content?: JsonValue;
223
- /** Tool calls associated with this message. */
224
- toolCalls?: ToolCallRecord[];
225
- /** Extra JSON-safe message metadata. */
226
- metadata?: Record<string, JsonValue>;
227
- };
228
- /**
229
- * Provider usage summary attached to a normalized harness run.
230
- *
231
- * @example
232
- * ```ts
233
- * const usage: UsageSummary = {
234
- * provider: "openai",
235
- * model: "gpt-4o-mini",
236
- * inputTokens: 212,
237
- * outputTokens: 48,
238
- * totalTokens: 260,
239
- * };
240
- * ```
241
- */
242
- type UsageSummary = {
243
- /** Provider that served the application run. */
244
- provider?: string;
245
- /** Model used for the application run. */
246
- model?: string;
247
- /** Input, prompt, or request tokens consumed by the run. */
248
- inputTokens?: number;
249
- /** Output or completion tokens produced by the run. */
250
- outputTokens?: number;
251
- /** Reasoning tokens reported by providers that expose them. */
252
- reasoningTokens?: number;
253
- /** Total token count reported by the provider or adapter. */
254
- totalTokens?: number;
255
- /** Count of tool calls observed during the run. */
256
- toolCalls?: number;
257
- /** Retry count observed during the run. */
258
- retries?: number;
259
- /** Provider-specific JSON-safe usage details. Cost estimates belong here. */
260
- metadata?: Record<string, JsonValue>;
261
- };
262
- /** Timing summary attached to a normalized harness run. */
263
- type TimingSummary = {
264
- /** End-to-end run duration in milliseconds. */
265
- totalMs?: number;
266
- /** Extra JSON-safe timing metadata. */
267
- metadata?: Record<string, JsonValue>;
268
- };
269
- /**
270
- * JSON-serializable transcript produced by the system under test.
271
- *
272
- * @example
273
- * ```ts
274
- * const session: NormalizedSession = {
275
- * provider: "openai",
276
- * model: "gpt-4o-mini",
277
- * messages: [
278
- * { role: "user", content: "Refund invoice inv_123" },
279
- * { role: "assistant", content: { status: "approved" } },
280
- * ],
281
- * };
282
- * ```
283
- */
284
- type NormalizedSession = {
285
- /** Ordered normalized transcript messages. */
286
- messages: NormalizedMessage[];
287
- /** Provider that produced the session when known. */
288
- provider?: string;
289
- /** Model that produced the session when known. */
290
- model?: string;
291
- /** Extra JSON-safe session metadata. */
292
- metadata?: Record<string, JsonValue>;
293
- };
294
28
  type OutputField<TOutput extends JsonValue | undefined> = undefined extends TOutput ? {
295
29
  output?: TOutput;
296
30
  } : {
297
31
  output: TOutput;
298
32
  };
299
- /**
300
- * Normalized result returned by every harness execution.
301
- *
302
- * @example
303
- * ```ts
304
- * const run: HarnessRun<{ status: "approved" }> = {
305
- * output: { status: "approved" },
306
- * session: {
307
- * messages: [
308
- * { role: "user", content: "Refund invoice inv_123" },
309
- * { role: "assistant", content: { status: "approved" } },
310
- * ],
311
- * },
312
- * usage: { totalTokens: 260 },
313
- * errors: [],
314
- * };
315
- * ```
316
- */
317
- type HarnessRun<TOutput extends JsonValue | undefined = JsonValue | undefined> = OutputField<TOutput> & {
318
- /** Normalized transcript and provider/session metadata. */
319
- session: NormalizedSession;
320
- /** Stable provider usage units such as tokens, tools, and retries. */
321
- usage: UsageSummary;
322
- /** Optional timing summary for the run. */
323
- timings?: TimingSummary;
324
- /** JSON-safe run artifacts captured by the harness or test context. */
325
- artifacts?: Record<string, JsonValue>;
326
- /** Normalized traces and spans captured during execution. */
327
- traces?: NormalizedTrace[];
328
- /** Normalized errors captured during execution. */
329
- errors: Array<Record<string, JsonValue>>;
330
- };
331
- /** Error value with an attached partial or complete normalized harness run. */
332
- type HarnessRunError = Error & {
333
- /** Attached normalized harness run recovered by `getHarnessRunFromError(...)`. */
334
- vitestEvalsRun: HarnessRun;
335
- };
336
33
  /** Per-run metadata shape accepted by harnesses and eval tests. */
337
34
  type HarnessMetadata = Record<string, unknown>;
338
35
  /**
@@ -567,19 +264,6 @@ declare function createGenAiUsageAttributes(usage: UsageSummary | undefined, opt
567
264
  "gen_ai.usage.output_tokens": number | undefined;
568
265
  "gen_ai.usage.reasoning.output_tokens": number | undefined;
569
266
  };
570
- /**
571
- * Flattens every recorded tool call from a normalized session.
572
- *
573
- * @param session - Normalized session produced by a harness run.
574
- *
575
- * @example
576
- * ```ts
577
- * const names = toolCalls(result.session).map((call) => call.name);
578
- *
579
- * expect(names).toEqual(["lookupInvoice", "createRefund"]);
580
- * ```
581
- */
582
- declare function toolCalls(session: NormalizedSession): ToolCallRecord[];
583
267
  /**
584
268
  * Converts normalized tool-call records into trace spans.
585
269
  *
@@ -594,99 +278,6 @@ declare function createToolCallSpans(calls: ToolCallRecord[], options?: CreateTo
594
278
  * remain free to attach richer native traces.
595
279
  */
596
280
  declare function ensureRunTrace(run: HarnessRun, options: EnsureRunTraceOptions): NormalizedTrace | undefined;
597
- /**
598
- * Flattens every recorded span from a normalized harness run.
599
- *
600
- * @param run - Normalized harness run produced by a harness.
601
- *
602
- * @example
603
- * ```ts
604
- * const modelSpans = spans(result).filter((span) => span.kind === "model");
605
- * ```
606
- */
607
- declare function spans(run: HarnessRun): NormalizedSpan[];
608
- /**
609
- * Returns spans of one coarse operation kind from a normalized run.
610
- *
611
- * @param run - Normalized harness run produced by a harness.
612
- * @param kind - Span kind to keep.
613
- */
614
- declare function spansByKind(run: HarnessRun, kind: NonNullable<NormalizedSpan["kind"]>): NormalizedSpan[];
615
- /**
616
- * Returns every span that explicitly failed or carries a normalized error.
617
- *
618
- * @param run - Normalized harness run produced by a harness.
619
- */
620
- declare function failedSpans(run: HarnessRun): NormalizedSpan[];
621
- /**
622
- * Filters normalized session messages by role.
623
- *
624
- * @param session - Normalized session produced by a harness run.
625
- * @param role - Message role to keep.
626
- *
627
- * @example
628
- * ```ts
629
- * const assistantText = messagesByRole(result.session, "assistant")
630
- * .map((message) => message.content)
631
- * .join("\n");
632
- * ```
633
- */
634
- declare function messagesByRole(session: NormalizedSession, role: NormalizedMessage["role"]): NormalizedMessage[];
635
- /**
636
- * Returns every normalized system message from a session.
637
- *
638
- * @param session - Normalized session produced by a harness run.
639
- *
640
- * @example
641
- * ```ts
642
- * const systemPrompts = systemMessages(result.session);
643
- * ```
644
- */
645
- declare function systemMessages(session: NormalizedSession): NormalizedMessage[];
646
- /**
647
- * Returns every normalized user message from a session.
648
- *
649
- * @param session - Normalized session produced by a harness run.
650
- *
651
- * @example
652
- * ```ts
653
- * const firstPrompt = userMessages(result.session)[0]?.content;
654
- * ```
655
- */
656
- declare function userMessages(session: NormalizedSession): NormalizedMessage[];
657
- /**
658
- * Returns every normalized assistant message from a session.
659
- *
660
- * @param session - Normalized session produced by a harness run.
661
- *
662
- * @example
663
- * ```ts
664
- * const finalAnswer = assistantMessages(result.session).at(-1)?.content;
665
- * ```
666
- */
667
- declare function assistantMessages(session: NormalizedSession): NormalizedMessage[];
668
- /**
669
- * Returns the latest assistant message content, ignoring empty text messages.
670
- *
671
- * @param session - Normalized session produced by a harness run.
672
- *
673
- * @example
674
- * ```ts
675
- * const finalAnswer = latestAssistantMessageContent(result.session);
676
- * ```
677
- */
678
- declare function latestAssistantMessageContent(session: NormalizedSession): JsonValue | undefined;
679
- /**
680
- * Returns every normalized tool message from a session.
681
- *
682
- * @param session - Normalized session produced by a harness run.
683
- *
684
- * @example
685
- * ```ts
686
- * const toolOutputs = toolMessages(result.session).map((message) => message.content);
687
- * ```
688
- */
689
- declare function toolMessages(session: NormalizedSession): NormalizedMessage[];
690
281
  /**
691
282
  * Attaches a partial or complete harness run to an arbitrary thrown error.
692
283
  *
@@ -727,4 +318,4 @@ declare function resolveHarnessRunErrors(result: unknown): Array<Record<string,
727
318
  /** Serializes an arbitrary thrown value into the normalized error shape. */
728
319
  declare function serializeError(error: unknown): Record<string, JsonValue>;
729
320
 
730
- export { type CreateHarnessOptions, type CreateHarnessRunArgs, type CreateToolCallSpansOptions, type EnsureRunTraceOptions, type GenAiOperationName, type GenAiOutputType, type GenAiProviderName, type GenAiSemanticAttributeKey, type GenAiSemanticAttributes, type GenAiTokenType, type GenAiToolType, type Harness, type HarnessContext, type HarnessMetadata, type HarnessResultLike, type HarnessRun, type HarnessRunError, type JsonPrimitive, type JsonValue, type MaybePromise, type NormalizedMessage, type NormalizedSession, type NormalizedSpan, type NormalizedSpanAttributeKey, type NormalizedSpanAttributes, type NormalizedSpanEvent, type NormalizedTrace, type OpenTelemetrySemanticAttributeKey, type OpenTelemetrySemanticAttributes, type SimpleHarnessResult, type SimpleSpanEvent, type SimpleSpanRecord, type SimpleToolCallRecord, type SimpleTraceRecord, type TimingSummary, type ToolCallRecord, type UsageSummary, assistantMessages, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, failedSpans, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, latestAssistantMessageContent, messagesByRole, normalizeContent, normalizeHarnessRun, normalizeMetadata, normalizeRecord, normalizeSpanAttributes, normalizeSpanError, resolveHarnessRunErrors, serializeError, spans, spansByKind, systemMessages, toJsonValue, toolCalls, toolMessages, userMessages };
321
+ export { type CreateHarnessOptions, type CreateHarnessRunArgs, type CreateToolCallSpansOptions, type EnsureRunTraceOptions, type Harness, type HarnessContext, type HarnessMetadata, type HarnessResultLike, type MaybePromise, type SimpleHarnessResult, type SimpleSpanEvent, type SimpleSpanRecord, type SimpleToolCallRecord, type SimpleTraceRecord, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, getHarnessRunFromError, hasCallableMethod, isHarnessRun, isNormalizedSession, normalizeContent, normalizeHarnessRun, normalizeMetadata, normalizeRecord, normalizeSpanAttributes, normalizeSpanError, resolveHarnessRunErrors, serializeError, toJsonValue };
package/dist/harness.js CHANGED
@@ -20,20 +20,20 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/harness.ts
21
21
  var harness_exports = {};
22
22
  __export(harness_exports, {
23
- assistantMessages: () => assistantMessages,
23
+ assistantMessages: () => import_core2.assistantMessages,
24
24
  attachHarnessRunToError: () => attachHarnessRunToError,
25
25
  createFailedHarnessRun: () => createFailedHarnessRun,
26
26
  createGenAiUsageAttributes: () => createGenAiUsageAttributes,
27
27
  createHarness: () => createHarness,
28
28
  createToolCallSpans: () => createToolCallSpans,
29
29
  ensureRunTrace: () => ensureRunTrace,
30
- failedSpans: () => failedSpans,
30
+ failedSpans: () => import_core2.failedSpans,
31
31
  getHarnessRunFromError: () => getHarnessRunFromError,
32
32
  hasCallableMethod: () => hasCallableMethod,
33
33
  isHarnessRun: () => isHarnessRun,
34
34
  isNormalizedSession: () => isNormalizedSession,
35
- latestAssistantMessageContent: () => latestAssistantMessageContent,
36
- messagesByRole: () => messagesByRole,
35
+ latestAssistantMessageContent: () => import_core2.latestAssistantMessageContent,
36
+ messagesByRole: () => import_core2.messagesByRole,
37
37
  normalizeContent: () => normalizeContent,
38
38
  normalizeHarnessRun: () => normalizeHarnessRun,
39
39
  normalizeMetadata: () => normalizeMetadata,
@@ -42,15 +42,17 @@ __export(harness_exports, {
42
42
  normalizeSpanError: () => normalizeSpanError,
43
43
  resolveHarnessRunErrors: () => resolveHarnessRunErrors,
44
44
  serializeError: () => serializeError,
45
- spans: () => spans,
46
- spansByKind: () => spansByKind,
47
- systemMessages: () => systemMessages,
45
+ spans: () => import_core2.spans,
46
+ spansByKind: () => import_core2.spansByKind,
47
+ systemMessages: () => import_core2.systemMessages,
48
48
  toJsonValue: () => toJsonValue,
49
- toolCalls: () => toolCalls,
50
- toolMessages: () => toolMessages,
51
- userMessages: () => userMessages
49
+ toolCalls: () => import_core2.toolCalls,
50
+ toolMessages: () => import_core2.toolMessages,
51
+ userMessages: () => import_core2.userMessages
52
52
  });
53
53
  module.exports = __toCommonJS(harness_exports);
54
+ var import_core = require("@vitest-evals/core");
55
+ var import_core2 = require("@vitest-evals/core");
54
56
  function isJsonPrimitive(value) {
55
57
  return value === null || typeof value === "string" || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value);
56
58
  }
@@ -177,12 +179,12 @@ function normalizeHarnessRun(input, result, context) {
177
179
  return result;
178
180
  }
179
181
  const output = result.output;
180
- const toolCalls2 = normalizeSimpleToolCalls(result.toolCalls);
182
+ const toolCalls3 = normalizeSimpleToolCalls(result.toolCalls);
181
183
  const usage = result.usage ?? {};
182
184
  const messages = result.messages ?? createDefaultSessionMessages({
183
185
  input,
184
186
  output,
185
- toolCalls: toolCalls2
187
+ toolCalls: toolCalls3
186
188
  });
187
189
  const metadata = result.metadata ? normalizeMetadata(result.metadata) : void 0;
188
190
  const artifacts = normalizeMergedArtifacts(
@@ -314,15 +316,15 @@ function normalizeSimpleTrace(trace) {
314
316
  spans: rawSpans,
315
317
  ...traceFields
316
318
  } = trace;
317
- const spans2 = (Array.isArray(rawSpans) ? rawSpans : []).map((span) => normalizeSimpleSpan(span)).filter((span) => Boolean(span));
319
+ const spans3 = (Array.isArray(rawSpans) ? rawSpans : []).map((span) => normalizeSimpleSpan(span)).filter((span) => Boolean(span));
318
320
  const metadata = isJsonRecord(rawMetadata) ? normalizeMetadata(rawMetadata) : void 0;
319
- if (spans2.length === 0 && !traceFields.id && !traceFields.name) {
321
+ if (spans3.length === 0 && !traceFields.id && !traceFields.name) {
320
322
  return void 0;
321
323
  }
322
324
  return {
323
325
  ...traceFields,
324
326
  ...metadata ? { metadata } : {},
325
- spans: spans2
327
+ spans: spans3
326
328
  };
327
329
  }
328
330
  function normalizeSimpleSpan(span) {
@@ -407,9 +409,6 @@ function createGenAiUsageAttributes(usage, options = {}) {
407
409
  "gen_ai.usage.reasoning.output_tokens": usage?.reasoningTokens
408
410
  };
409
411
  }
410
- function toolCalls(session) {
411
- return session.messages.flatMap((message) => message.toolCalls ?? []);
412
- }
413
412
  function createToolCallSpans(calls, options = {}) {
414
413
  return calls.map((call, index) => {
415
414
  const spanError = call.error ? normalizeSpanError(call.error) : void 0;
@@ -437,7 +436,7 @@ function createToolCallSpans(calls, options = {}) {
437
436
  });
438
437
  }
439
438
  function ensureRunTrace(run, options) {
440
- if (spans(run).length > 0) {
439
+ if ((0, import_core.spans)(run).length > 0) {
441
440
  return void 0;
442
441
  }
443
442
  const traceId = options.id ?? createGeneratedTraceId();
@@ -460,7 +459,7 @@ function ensureRunTrace(run, options) {
460
459
  ...createGenAiUsageAttributes(run.usage)
461
460
  })
462
461
  };
463
- const toolSpans = createToolCallSpans(toolCalls(run.session), {
462
+ const toolSpans = createToolCallSpans((0, import_core.toolCalls)(run.session), {
464
463
  traceId,
465
464
  parentId: rootSpanId,
466
465
  spanIdPrefix: `${traceId}:tool`
@@ -482,36 +481,6 @@ function createGeneratedTraceId() {
482
481
  nextGeneratedTraceId += 1;
483
482
  return `trace_${nextGeneratedTraceId}`;
484
483
  }
485
- function spans(run) {
486
- return (run.traces ?? []).flatMap((trace) => trace.spans);
487
- }
488
- function spansByKind(run, kind) {
489
- return spans(run).filter((span) => span.kind === kind);
490
- }
491
- function failedSpans(run) {
492
- return spans(run).filter((span) => span.status === "error" || span.error);
493
- }
494
- function messagesByRole(session, role) {
495
- return session.messages.filter((message) => message.role === role);
496
- }
497
- function hasNonEmptyMessageContent(message) {
498
- return message.content !== void 0 && (typeof message.content !== "string" || message.content.trim().length > 0);
499
- }
500
- function systemMessages(session) {
501
- return messagesByRole(session, "system");
502
- }
503
- function userMessages(session) {
504
- return messagesByRole(session, "user");
505
- }
506
- function assistantMessages(session) {
507
- return messagesByRole(session, "assistant");
508
- }
509
- function latestAssistantMessageContent(session) {
510
- return [...assistantMessages(session)].reverse().find(hasNonEmptyMessageContent)?.content;
511
- }
512
- function toolMessages(session) {
513
- return messagesByRole(session, "tool");
514
- }
515
484
  function attachHarnessRunToError(error, run) {
516
485
  const baseError = error instanceof Error ? error : new Error(String(error ?? "Unknown error"));
517
486
  return Object.assign(baseError, {