vitest-evals 0.13.0 → 0.14.0

package/dist/index.mjs

@@ -4,14 +4,17 @@ import "vitest";
 
 // src/harness.ts
 import {
+  messagesToTranscriptEvents,
+  NormalizedSessionSchema,
   spans,
-  toolCalls
+  TranscriptEventSchema
 } from "@vitest-evals/core";
 import {
   assistantMessages as assistantMessages2,
   failedSpans as failedSpans2,
   latestAssistantMessageContent as latestAssistantMessageContent2,
   messagesByRole as messagesByRole2,
+  messagesToTranscriptEvents as messagesToTranscriptEvents2,
   spans as spans2,
   spansByKind as spansByKind2,
   systemMessages as systemMessages2,
@@ -140,14 +143,24 @@ function normalizeHarnessRun(input, result, context) {
     }
     return result;
   }
+  if ("toolCalls" in result) {
+    throw new TypeError(
+      'createHarness results do not accept top-level toolCalls. Return ordered session events with type: "tool_call" and type: "tool_result" entries instead.'
+    );
+  }
   const output = result.output;
-  const toolCalls3 = normalizeSimpleToolCalls(result.toolCalls);
   const usage = result.usage ?? {};
-  const messages = result.messages ?? createDefaultSessionMessages({
-    input,
-    output,
-    toolCalls: toolCalls3
-  });
+  const events = normalizeTranscriptInput(result);
+  if (!events) {
+    throw new TypeError(
+      "createHarness results must include ordered events or messages. Return a full HarnessRun or a lightweight result with events/messages."
+    );
+  }
+  if (events.length === 0) {
+    throw new TypeError(
+      "createHarness results must include at least one transcript event. Return ordered events or message transport inputs that normalize into events."
+    );
+  }
   const metadata = result.metadata ? normalizeMetadata(result.metadata) : void 0;
   const artifacts = normalizeMergedArtifacts(
     context?.artifacts,
@@ -156,7 +169,7 @@ function normalizeHarnessRun(input, result, context) {
   const traces = normalizeSimpleTraces(result.traces);
   return {
     session: {
-      messages,
+      events,
       ...usage.provider ? { provider: usage.provider } : {},
       ...usage.model ? { model: usage.model } : {},
       ...metadata ? { metadata } : {}
@@ -169,12 +182,24 @@ function normalizeHarnessRun(input, result, context) {
     errors: normalizeSimpleErrors(result.errors)
   };
 }
+function normalizeTranscriptInput(result) {
+  if ("events" in result && Array.isArray(result.events)) {
+    return result.events.map((event) => TranscriptEventSchema.parse(event));
+  }
+  if ("messages" in result && Array.isArray(result.messages)) {
+    return messagesToTranscriptEvents(result.messages).map(
+      (event) => TranscriptEventSchema.parse(event)
+    );
+  }
+  return void 0;
+}
 function createFailedHarnessRun(input, error, options = {}) {
   const artifacts = options.artifacts;
   return {
     session: {
-      messages: [
+      events: [
         {
+          type: "message",
           role: "user",
           content: normalizeContent(input)
         }
@@ -185,67 +210,6 @@ function createFailedHarnessRun(input, error, options = {}) {
     errors: [serializeError(error)]
   };
 }
-function createDefaultSessionMessages({
-  input,
-  output,
-  toolCalls: normalizedToolCalls
-}) {
-  const messages = [
-    {
-      role: "user",
-      content: normalizeContent(input)
-    }
-  ];
-  if (output !== void 0 || normalizedToolCalls.length > 0) {
-    messages.push({
-      role: "assistant",
-      ...output !== void 0 ? { content: normalizeContent(output) } : {},
-      ...normalizedToolCalls.length > 0 ? { toolCalls: normalizedToolCalls } : {}
-    });
-  }
-  return messages;
-}
-function normalizeSimpleToolCalls(calls) {
-  return (calls ?? []).map((call) => {
-    const {
-      arguments: rawArguments,
-      result: rawResult,
-      error: rawError,
-      metadata: rawMetadata,
-      ...toolCall
-    } = call;
-    const args = normalizeToolCallArguments(rawArguments);
-    const result = toJsonValue(rawResult);
-    const error = normalizeToolCallError(rawError);
-    const metadata = rawMetadata ? normalizeMetadata(rawMetadata) : void 0;
-    return {
-      ...toolCall,
-      ...args ? { arguments: args } : {},
-      ...result !== void 0 ? { result } : {},
-      ...error ? { error } : {},
-      ...metadata ? { metadata } : {}
-    };
-  });
-}
-function normalizeToolCallArguments(value) {
-  if (value === void 0) {
-    return void 0;
-  }
-  const normalized = toJsonValue(value);
-  return normalized && typeof normalized === "object" && !Array.isArray(normalized) ? normalized : void 0;
-}
-function normalizeToolCallError(value) {
-  if (value === void 0) {
-    return void 0;
-  }
-  const serialized = serializeError(value);
-  const { message, type, ...details } = serialized;
-  return {
-    ...details,
-    message: typeof message === "string" ? message : String(message),
-    ...typeof type === "string" ? { type } : {}
-  };
-}
 function normalizeMergedArtifacts(contextArtifacts, resultArtifacts) {
   const artifacts = {
     ...contextArtifacts ?? {},
@@ -371,32 +335,6 @@ function createGenAiUsageAttributes(usage, options = {}) {
     "gen_ai.usage.reasoning.output_tokens": usage?.reasoningTokens
   };
 }
-function createToolCallSpans(calls, options = {}) {
-  return calls.map((call, index) => {
-    const spanError = call.error ? normalizeSpanError(call.error) : void 0;
-    const spanId = options.spanIdPrefix ? `${options.spanIdPrefix}:${index + 1}` : call.id;
-    return {
-      ...spanId ? { id: spanId } : {},
-      ...options.traceId ? { traceId: options.traceId } : {},
-      ...options.parentId ? { parentId: options.parentId } : {},
-      name: call.name,
-      kind: "tool",
-      ...call.startedAt ? { startedAt: call.startedAt } : {},
-      ...call.finishedAt ? { finishedAt: call.finishedAt } : {},
-      ...call.durationMs !== void 0 ? { durationMs: call.durationMs } : {},
-      status: spanError ? "error" : "ok",
-      ...spanError ? { error: spanError } : {},
-      attributes: normalizeSpanAttributes({
-        "gen_ai.operation.name": "execute_tool",
-        "gen_ai.tool.name": call.name,
-        "gen_ai.tool.type": "function",
-        ...call.id ? { "gen_ai.tool.call.id": call.id } : {},
-        ...call.arguments !== void 0 ? { "gen_ai.tool.call.arguments": call.arguments } : {},
-        ...call.result !== void 0 ? { "gen_ai.tool.call.result": call.result } : {}
-      })
-    };
-  });
-}
 function ensureRunTrace(run, options) {
   if (spans(run).length > 0) {
     return void 0;
@@ -421,11 +359,6 @@ function ensureRunTrace(run, options) {
       ...createGenAiUsageAttributes(run.usage)
     })
   };
-  const toolSpans = createToolCallSpans(toolCalls(run.session), {
-    traceId,
-    parentId: rootSpanId,
-    spanIdPrefix: `${traceId}:tool`
-  });
   const trace = {
     id: traceId,
     name: options.name,
@@ -433,7 +366,7 @@ function ensureRunTrace(run, options) {
     finishedAt: options.finishedAt.toISOString(),
     durationMs,
     ...options.source ? { metadata: { source: options.source } } : {},
-    spans: [runSpan, ...toolSpans]
+    spans: [runSpan]
   };
   run.traces = [trace];
   return trace;
@@ -463,7 +396,7 @@ function isHarnessRun(value) {
   return isNormalizedSession(candidate.session) && Boolean(candidate.usage) && typeof candidate.usage === "object" && !Array.isArray(candidate.usage) && Array.isArray(candidate.errors);
 }
 function isNormalizedSession(value) {
-  return Boolean(value) && typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
+  return NormalizedSessionSchema.safeParse(value).success;
 }
 function serializeError(error) {
   if (error instanceof Error) {
@@ -483,7 +416,10 @@ function createJudgeHarness(options) {
   return createHarness({
     name: options.name ?? "judge-harness",
     run: async ({ input, signal }) => {
-      return normalizeJudgeHarnessResult(await options.run(input, { signal }));
+      return normalizeJudgeHarnessResult(
+        input,
+        await options.run(input, { signal })
+      );
     }
   });
 }
@@ -506,17 +442,14 @@ function createRunJudge(judgeHarness, signal) {
     signal: options?.signal ?? signal
   });
 }
-function normalizeJudgeHarnessResult(result) {
+function normalizeJudgeHarnessResult(input, result) {
   if (isHarnessRun(result)) {
     return result;
   }
-  if (hasOutputField(result)) {
-    return {
-      output: normalizeJudgeHarnessOutput(result.output)
-    };
-  }
+  const output = hasOutputField(result) ? normalizeJudgeHarnessOutput(result.output) : normalizeJudgeHarnessOutput(result);
   return {
-    output: normalizeJudgeHarnessOutput(result)
+    output,
+    messages: createJudgeHarnessMessages(input, output)
   };
 }
 function hasOutputField(value) {
@@ -528,6 +461,13 @@ function normalizeJudgeHarnessOutput(value) {
   }
   return normalizeContent(value);
 }
+function createJudgeHarnessMessages(input, output) {
+  return [
+    ...input.system ? [{ role: "system", content: input.system }] : [],
+    { role: "user", content: input.prompt },
+    ...output !== void 0 ? [{ role: "assistant", content: output }] : []
+  ];
+}
 function resolveJudgeHarnessAssistantOutput(run) {
   return latestAssistantMessageContent2(run.session) ?? "";
 }
@@ -1655,23 +1595,25 @@ function resolveJudgeAssertionOutput(received, run, explicitOutput) {
   return normalizeJudgeJsonValue(received);
 }
 function createSyntheticJudgeSession(received, options) {
-  const messages = [];
+  const events = [];
   const userContent = normalizeJudgeJsonValue(options.input);
   if (userContent !== void 0) {
-    messages.push({
+    events.push({
+      type: "message",
       role: "user",
       content: userContent
     });
   }
   const assistantContent = normalizeJudgeJsonValue(received);
   if (assistantContent !== void 0) {
-    messages.push({
+    events.push({
+      type: "message",
       role: "assistant",
       content: assistantContent
     });
   }
   return {
-    messages
+    events
   };
 }
 function inferJudgeOutputValue(received, session) {
@@ -1679,7 +1621,7 @@ function inferJudgeOutputValue(received, session) {
     return received.output;
   }
   if (isNormalizedSession(received)) {
-    return resolveAssistantOutput(session) ?? normalizeJudgeJsonValue(received.messages);
+    return resolveAssistantOutput(session) ?? normalizeJudgeJsonValue(received.events);
   }
   return normalizeJudgeJsonValue(received);
 }
@@ -1752,7 +1694,6 @@ export {
   createHarness,
   createJudge,
   createJudgeHarness,
-  createToolCallSpans,
   describeEval,
   ensureRunTrace,
   failedSpans2 as failedSpans,
@@ -1760,6 +1701,7 @@ export {
   getHarnessRunFromError,
   latestAssistantMessageContent2 as latestAssistantMessageContent,
   messagesByRole2 as messagesByRole,
+  messagesToTranscriptEvents2 as messagesToTranscriptEvents,
   normalizeHarnessRun,
   normalizeSpanAttributes,
   normalizeSpanError,