raindrop-ai 0.5.0 → 0.6.0

package/README.md

@@ -67,6 +67,41 @@ On runtimes that lack a usable `AsyncLocalStorage.enterWith` (notably Cloudflare
 
 ## Eval mode
 
+Run your agent locally and store eval results in hosted Raindrop. Workshop is not required. Define a reusable suite with `defineEvalSuite`, then run it with `runEvalSuite` or the `@raindrop-ai/vitest` adapter. Input-only datasets require the SDK release with optional-reference support and the matching Raindrop backend.
+
+```ts
+import Raindrop, { defineEvalSuite, runEvalSuite } from "raindrop-ai";
+import { supportAgent } from "./support-agent";
+
+const client = new Raindrop({
+  apiKey: process.env.RAINDROP_QUERY_API_KEY,
+  projectId: process.env.RAINDROP_PROJECT_ID,
+  localWorkshopUrl: false,
+});
+
+const suite = defineEvalSuite({
+  name: "Support responses",
+  dataset: "support-cases",
+  run: ({ input }) => supportAgent(input ?? ""),
+  evaluators: [{ evaluator: "task-completed", threshold: { equals: true } }],
+});
+
+try {
+  const result = await runEvalSuite(client, suite, {
+    destination: { kind: "raindrop" },
+  });
+  console.log(result);
+} finally {
+  await client.close();
+}
+```
+
+The dataset string names a published, project-scoped eval dataset, not an ordinary custom-upload dataset ID. Publish its immutable version with `publishEvalDataset`. Cases can contain only input and scenario properties; an optional reference must be a complete captured trace. Local evaluators can declare `requiresReference: true`, and hosted programs can export `const requiresReference = true`, to reject missing references before agent execution. A local `defineDataset` object does not publish itself and is not accepted by the hosted runner. See the [Vitest guide](../vitest/README.md) for publication, evaluator requirements, and credential setup.
+
+The Query SDK organization `apiKey` authenticates evals, dataset publication/reads, and replay trace uploads. Browser login and production `writeKey` credentials do not replace it. The agent's model credentials and test environment are separate. Running locally describes where your callback executes, not where results are stored.
+
+### Trace correlation
+
 A replay calls your agent once per row and needs every span and event that call
 emits to land on that row. `withEvalScope` binds one opaque **correlation id**
 to the execution context, so it does, with no change to the agent:
@@ -115,9 +150,9 @@ Both limits are demonstrated by tests in `packages/core/src/eval-scope.test.ts`.
 
 ### Replaying a dataset
 
-`replay` is the loop that uses the scope. It asks Raindrop for the dataset's
-rows, calls your agent once per row inside that row's scope, says which rows
-failed, and waits for the traces to land:
+`replay` is the lower-level loop used by hosted suite execution. It reads the
+published dataset's immutable version, calls your agent once per row inside
+that row's scope, reports failed rows, and waits for the traces to land:
 
 ```ts
 import Raindrop, { replay } from "raindrop-ai";

package/dist/{chunk-HFHHGRTW.mjs → chunk-CFGEE5ID.mjs}

@@ -707,7 +707,7 @@ var EvalCaseTracePairSchema = z2.object({
   }),
   reference: EvalCaseTraceSchema.extend({
     trace: TraceSchema.extend({ origin: z2.literal("dataset") })
-  })
+  }).optional()
 });
 function compareNanoseconds(left, right) {
   if (Number.isFinite(left) && Number.isFinite(right))
@@ -1007,7 +1007,7 @@ function traceOutput(trace) {
 function traceToolCalls(trace, name) {
   return trace.entries.filter((entry) => entry.type === "tool_call" && (name === void 0 || entry.name === name));
 }
-var TRACE_PROJECTION_SHA256 = "8dab83087aa2def3d076324021d5e08911e37967b68597b58b793a80d54b5874";
+var TRACE_PROJECTION_SHA256 = "7d12eb03150b8f203e4404622f047676f50b3aa1e5442f0ed42d77d6002a33a6";
 
 // src/evals/portable.ts
 import { z as z3 } from "zod";
@@ -1095,7 +1095,7 @@ async function evaluatePortableEvaluator(evaluator, input) {
   const artifact = await verifyPortableEvalArtifact(evaluator.artifact, {
     expectedSha256: evaluator.expectedSha256
   });
-  const { runPortableEvaluator } = await import("./portable-runtime-GBE3SHBZ.mjs");
+  const { runPortableEvaluator } = await import("./portable-runtime-QZKU2VXS.mjs");
   const cases = (_a = input.cases) != null ? _a : [];
   if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
     (entry, index) => {

package/dist/{chunk-AUNFORWV.mjs → chunk-ZVHQBBST.mjs}

@@ -1238,7 +1238,7 @@ installTracingSuppressionHook();
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.5.0",
+  version: "0.6.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",

package/dist/{index-CQj4bn2v.d.mts → index-APN1jN-i.d.mts}

@@ -1082,7 +1082,7 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
     name: z.ZodString;
     input: z.ZodNullable<z.ZodString>;
     properties: z.ZodRecord<z.ZodString, z.ZodString>;
-    referenceTrace: z.ZodEffects<z.ZodObject<{
+    referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
         event: z.ZodObject<{
             id: z.ZodString;
             _raindropDatabaseId: z.ZodString;
@@ -5470,13 +5470,13 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    }>;
+    }>>;
 }, "strict", z.ZodTypeAny, {
     properties: Record<string, string>;
     input: string | null;
     name: string;
     id: string;
-    referenceTrace: {
+    referenceTrace?: {
         event: {
             properties: Record<string, unknown> | null;
             timestamp: string;
@@ -5873,13 +5873,13 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    };
+    } | undefined;
 }, {
     properties: Record<string, string>;
     input: string | null;
     name: string;
     id: string;
-    referenceTrace: {
+    referenceTrace?: {
         event: {
             properties: Record<string, unknown> | null;
             timestamp: string;
@@ -6276,7 +6276,7 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    };
+    } | undefined;
 }>;
 type EvalDatasetCaseInput = z.infer<typeof EvalDatasetCaseInputSchema>;
 declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
@@ -6289,7 +6289,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         name: z.ZodString;
         input: z.ZodNullable<z.ZodString>;
         properties: z.ZodRecord<z.ZodString, z.ZodString>;
-        referenceTrace: z.ZodEffects<z.ZodObject<{
+        referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
             event: z.ZodObject<{
                 id: z.ZodString;
                 _raindropDatabaseId: z.ZodString;
@@ -10677,13 +10677,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        }>;
+        }>>;
     }, "strict", z.ZodTypeAny, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11080,13 +11080,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11483,13 +11483,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }>, "many">, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11886,13 +11886,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[], {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -12289,7 +12289,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[]>;
 }, "strict", z.ZodTypeAny, {
     name: string;
@@ -12301,7 +12301,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -12698,7 +12698,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[];
 }, {
     name: string;
@@ -12710,7 +12710,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -13107,7 +13107,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[];
 }>;
 type PublishEvalDatasetVersionInput = z.infer<typeof PublishEvalDatasetVersionInputSchema>;
@@ -13138,7 +13138,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
             name: z.ZodString;
             input: z.ZodNullable<z.ZodString>;
             properties: z.ZodRecord<z.ZodString, z.ZodString>;
-            referenceTrace: z.ZodEffects<z.ZodObject<{
+            referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
                 event: z.ZodObject<{
                     id: z.ZodString;
                     _raindropDatabaseId: z.ZodString;
@@ -17526,15 +17526,15 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            }>;
+            }>>;
         } & {
-            referenceTraceSha256: z.ZodString;
+            referenceTraceSha256: z.ZodOptional<z.ZodString>;
         }, "strict", z.ZodTypeAny, {
             properties: Record<string, string>;
             input: string | null;
             name: string;
             id: string;
-            referenceTrace: {
+            referenceTrace?: {
                 event: {
                     properties: Record<string, unknown> | null;
                     timestamp: string;
@@ -17931,14 +17931,14 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            };
-            referenceTraceSha256: string;
+            } | undefined;
+            referenceTraceSha256?: string | undefined;
         }, {
             properties: Record<string, string>;
             input: string | null;
             name: string;
             id: string;
-            referenceTrace: {
+            referenceTrace?: {
                 event: {
                     properties: Record<string, unknown> | null;
                     timestamp: string;
@@ -18335,8 +18335,8 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            };
-            referenceTraceSha256: string;
+            } | undefined;
+            referenceTraceSha256?: string | undefined;
         }>, "many">;
         createdAt: z.ZodString;
     }, "cases">, "strip", z.ZodTypeAny, {
@@ -18357,7 +18357,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         name: z.ZodString;
         input: z.ZodNullable<z.ZodString>;
         properties: z.ZodRecord<z.ZodString, z.ZodString>;
-        referenceTrace: z.ZodEffects<z.ZodObject<{
+        referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
             event: z.ZodObject<{
                 id: z.ZodString;
                 _raindropDatabaseId: z.ZodString;
@@ -22745,23 +22745,23 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        }>;
+        }>>;
     } & {
-        referenceTraceSha256: z.ZodString;
+        referenceTraceSha256: z.ZodOptional<z.ZodString>;
     }, "properties" | "input" | "name" | "id"> & {
-        referenceTraceSha256: z.ZodString;
+        referenceTraceSha256: z.ZodOptional<z.ZodString>;
     }, "strict", z.ZodTypeAny, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     dataset: {
@@ -22775,7 +22775,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }[];
     version: {
         id: string;
@@ -22796,7 +22796,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }[];
     version: {
         id: string;
@@ -43264,7 +43264,7 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
             origin: "replay";
         };
     }>;
-    reference: z$1.ZodObject<{
+    reference: z$1.ZodOptional<z$1.ZodObject<{
         snapshot: z$1.ZodEffects<z$1.ZodObject<{
             event: z$1.ZodObject<{
                 id: z$1.ZodString;
@@ -54793,9 +54793,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
             })[];
             origin: "dataset";
         };
-    }>;
+    }>>;
 }, "strip", z$1.ZodTypeAny, {
-    reference: {
+    caseId: string;
+    candidate: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -55818,11 +55819,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "dataset";
+            origin: "replay";
         };
     };
-    caseId: string;
-    candidate: {
+    reference?: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -56845,11 +56845,12 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "replay";
+            origin: "dataset";
         };
-    };
+    } | undefined;
 }, {
-    reference: {
+    caseId: string;
+    candidate: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -57872,11 +57873,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "dataset";
+            origin: "replay";
         };
     };
-    caseId: string;
-    candidate: {
+    reference?: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -58899,9 +58899,9 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "replay";
+            origin: "dataset";
         };
-    };
+    } | undefined;
 }>;
 type EvalCaseTracePair = z$1.infer<typeof EvalCaseTracePairSchema>;
 type TraceNanoseconds = number | string;
@@ -59769,6 +59769,8 @@ type LocalEvaluatorContext$1<AgentResult = unknown> = {
 type LocalJudge<AgentResult, Verdict> = (context: LocalEvaluatorContext$1<AgentResult>) => Verdict | Promise<Verdict>;
 /** A named evaluator whose implementation stays in the customer's process. */
 type LocalEvaluator<AgentResult = unknown> = {
+    readonly requiresReference?: boolean;
+} & ({
     readonly slug: string;
     readonly name: string;
     readonly output: "boolean";
@@ -59783,7 +59785,7 @@ type LocalEvaluator<AgentResult = unknown> = {
     readonly name: string;
     readonly output: "number";
     readonly judge: LocalJudge<AgentResult, LocalNumberVerdict>;
-};
+});
 type CreateReplayOptions = {
     /** A project-scoped dataset slug, or a manifest already pinned by discovery. */
     dataset: string | EvalDatasetManifest;
@@ -60113,6 +60115,8 @@ type LocalEvaluatorContext<AgentResult = unknown> = {
     readonly result: AgentResult;
 };
 type LocalEvaluatorBase<AgentResult, Output, Verdict> = {
+    /** Reject input-only cases before the agent runs when this evaluator needs a baseline. */
+    readonly requiresReference?: boolean;
     readonly slug: string;
     readonly name: string;
     readonly output: Output;
@@ -60176,7 +60180,7 @@ type PublishEvalDatasetOptions = {
     readonly slug: string;
     /** Display name. Publishing a new version does not rename an existing dataset. */
     readonly name: PublishEvalDatasetVersionInput["name"];
-    /** Ordered cases. Each reference is a complete captured trace snapshot. */
+    /** Ordered scenarios. Supplied references must be complete captured trace snapshots. */
     readonly cases: readonly EvalDatasetCaseInput[];
     /** Omit or pass null to create. Pass the discovered current version when changing evidence. */
     readonly expectedCurrentVersionId?: string | null;

package/dist/{index-CQj4bn2v.d.ts → index-APN1jN-i.d.ts}

@@ -1082,7 +1082,7 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
     name: z.ZodString;
     input: z.ZodNullable<z.ZodString>;
     properties: z.ZodRecord<z.ZodString, z.ZodString>;
-    referenceTrace: z.ZodEffects<z.ZodObject<{
+    referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
         event: z.ZodObject<{
             id: z.ZodString;
             _raindropDatabaseId: z.ZodString;
@@ -5470,13 +5470,13 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    }>;
+    }>>;
 }, "strict", z.ZodTypeAny, {
     properties: Record<string, string>;
     input: string | null;
     name: string;
     id: string;
-    referenceTrace: {
+    referenceTrace?: {
         event: {
             properties: Record<string, unknown> | null;
             timestamp: string;
@@ -5873,13 +5873,13 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    };
+    } | undefined;
 }, {
     properties: Record<string, string>;
     input: string | null;
     name: string;
     id: string;
-    referenceTrace: {
+    referenceTrace?: {
         event: {
             properties: Record<string, unknown> | null;
             timestamp: string;
@@ -6276,7 +6276,7 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    };
+    } | undefined;
 }>;
 type EvalDatasetCaseInput = z.infer<typeof EvalDatasetCaseInputSchema>;
 declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
@@ -6289,7 +6289,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         name: z.ZodString;
         input: z.ZodNullable<z.ZodString>;
         properties: z.ZodRecord<z.ZodString, z.ZodString>;
-        referenceTrace: z.ZodEffects<z.ZodObject<{
+        referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
             event: z.ZodObject<{
                 id: z.ZodString;
                 _raindropDatabaseId: z.ZodString;
@@ -10677,13 +10677,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        }>;
+        }>>;
     }, "strict", z.ZodTypeAny, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11080,13 +11080,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11483,13 +11483,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }>, "many">, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11886,13 +11886,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[], {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -12289,7 +12289,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[]>;
 }, "strict", z.ZodTypeAny, {
     name: string;
@@ -12301,7 +12301,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -12698,7 +12698,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[];
 }, {
     name: string;
@@ -12710,7 +12710,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -13107,7 +13107,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[];
 }>;
 type PublishEvalDatasetVersionInput = z.infer<typeof PublishEvalDatasetVersionInputSchema>;
@@ -13138,7 +13138,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
             name: z.ZodString;
             input: z.ZodNullable<z.ZodString>;
             properties: z.ZodRecord<z.ZodString, z.ZodString>;
-            referenceTrace: z.ZodEffects<z.ZodObject<{
+            referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
                 event: z.ZodObject<{
                     id: z.ZodString;
                     _raindropDatabaseId: z.ZodString;
@@ -17526,15 +17526,15 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            }>;
+            }>>;
         } & {
-            referenceTraceSha256: z.ZodString;
+            referenceTraceSha256: z.ZodOptional<z.ZodString>;
         }, "strict", z.ZodTypeAny, {
             properties: Record<string, string>;
             input: string | null;
             name: string;
             id: string;
-            referenceTrace: {
+            referenceTrace?: {
                 event: {
                     properties: Record<string, unknown> | null;
                     timestamp: string;
@@ -17931,14 +17931,14 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            };
-            referenceTraceSha256: string;
+            } | undefined;
+            referenceTraceSha256?: string | undefined;
         }, {
             properties: Record<string, string>;
             input: string | null;
             name: string;
             id: string;
-            referenceTrace: {
+            referenceTrace?: {
                 event: {
                     properties: Record<string, unknown> | null;
                     timestamp: string;
@@ -18335,8 +18335,8 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            };
-            referenceTraceSha256: string;
+            } | undefined;
+            referenceTraceSha256?: string | undefined;
         }>, "many">;
         createdAt: z.ZodString;
     }, "cases">, "strip", z.ZodTypeAny, {
@@ -18357,7 +18357,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         name: z.ZodString;
         input: z.ZodNullable<z.ZodString>;
         properties: z.ZodRecord<z.ZodString, z.ZodString>;
-        referenceTrace: z.ZodEffects<z.ZodObject<{
+        referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
             event: z.ZodObject<{
                 id: z.ZodString;
                 _raindropDatabaseId: z.ZodString;
@@ -22745,23 +22745,23 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        }>;
+        }>>;
     } & {
-        referenceTraceSha256: z.ZodString;
+        referenceTraceSha256: z.ZodOptional<z.ZodString>;
     }, "properties" | "input" | "name" | "id"> & {
-        referenceTraceSha256: z.ZodString;
+        referenceTraceSha256: z.ZodOptional<z.ZodString>;
     }, "strict", z.ZodTypeAny, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     dataset: {
@@ -22775,7 +22775,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }[];
     version: {
         id: string;
@@ -22796,7 +22796,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }[];
     version: {
         id: string;
@@ -43264,7 +43264,7 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
             origin: "replay";
         };
     }>;
-    reference: z$1.ZodObject<{
+    reference: z$1.ZodOptional<z$1.ZodObject<{
         snapshot: z$1.ZodEffects<z$1.ZodObject<{
             event: z$1.ZodObject<{
                 id: z$1.ZodString;
@@ -54793,9 +54793,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
             })[];
             origin: "dataset";
         };
-    }>;
+    }>>;
 }, "strip", z$1.ZodTypeAny, {
-    reference: {
+    caseId: string;
+    candidate: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -55818,11 +55819,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "dataset";
+            origin: "replay";
         };
     };
-    caseId: string;
-    candidate: {
+    reference?: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -56845,11 +56845,12 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "replay";
+            origin: "dataset";
         };
-    };
+    } | undefined;
 }, {
-    reference: {
+    caseId: string;
+    candidate: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -57872,11 +57873,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "dataset";
+            origin: "replay";
         };
     };
-    caseId: string;
-    candidate: {
+    reference?: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -58899,9 +58899,9 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "replay";
+            origin: "dataset";
         };
-    };
+    } | undefined;
 }>;
 type EvalCaseTracePair = z$1.infer<typeof EvalCaseTracePairSchema>;
 type TraceNanoseconds = number | string;
@@ -59769,6 +59769,8 @@ type LocalEvaluatorContext$1<AgentResult = unknown> = {
 type LocalJudge<AgentResult, Verdict> = (context: LocalEvaluatorContext$1<AgentResult>) => Verdict | Promise<Verdict>;
 /** A named evaluator whose implementation stays in the customer's process. */
 type LocalEvaluator<AgentResult = unknown> = {
+    readonly requiresReference?: boolean;
+} & ({
     readonly slug: string;
     readonly name: string;
     readonly output: "boolean";
@@ -59783,7 +59785,7 @@ type LocalEvaluator<AgentResult = unknown> = {
     readonly name: string;
     readonly output: "number";
     readonly judge: LocalJudge<AgentResult, LocalNumberVerdict>;
-};
+});
 type CreateReplayOptions = {
     /** A project-scoped dataset slug, or a manifest already pinned by discovery. */
     dataset: string | EvalDatasetManifest;
@@ -60113,6 +60115,8 @@ type LocalEvaluatorContext<AgentResult = unknown> = {
     readonly result: AgentResult;
 };
 type LocalEvaluatorBase<AgentResult, Output, Verdict> = {
+    /** Reject input-only cases before the agent runs when this evaluator needs a baseline. */
+    readonly requiresReference?: boolean;
     readonly slug: string;
     readonly name: string;
     readonly output: Output;
@@ -60176,7 +60180,7 @@ type PublishEvalDatasetOptions = {
     readonly slug: string;
     /** Display name. Publishing a new version does not rename an existing dataset. */
     readonly name: PublishEvalDatasetVersionInput["name"];
-    /** Ordered cases. Each reference is a complete captured trace snapshot. */
+    /** Ordered scenarios. Supplied references must be complete captured trace snapshots. */
     readonly cases: readonly EvalDatasetCaseInput[];
     /** Omit or pass null to create. Pass the discovered current version when changing evidence. */
     readonly expectedCurrentVersionId?: string | null;

package/dist/index.d.mts

@@ -1,5 +1,5 @@
 import '@opentelemetry/instrumentation';
-export { f as AiTrackEvent, A as AiTrackUsage, g as Attachment, h as AttachmentType, B as BeginInteractionOptions, i as BooleanThreshold, j as BooleanThresholdSchema, k as CreateReplayOptions, D as DEFAULT_EVAL_POLL_INTERVAL_MS, l as DEFAULT_EVAL_WAIT_MS, m as DEFAULT_QUERY_URL, n as DEFAULT_TRACE_WAIT_MS, o as DefinedEvalDataset, E as EvalCaseResult, p as EvalClient, q as EvalDataset, r as EvalDatasetCaseInput, s as EvalDatasetCaseInputSchema, t as EvalDatasetManifest, u as EvalDatasetPublishConflictError, v as EvalDestination, w as EvalEvaluator, x as EvalLocalEvaluator, y as EvalManifest, z as EvalProgram, F as EvalSuiteDefinition, G as EvalSuiteDefinitionInput, H as EvalSuiteResult, J as EvalSuiteRunSession, K as EvalVerdictAssessment, L as EvaluateReplayOptions, M as FinishInteractionOptions, N as HostedEvaluatorPin, O as IdentifyEvent, b as Interaction, Q as LocalBooleanVerdict, U as LocalBooleanVerdictSchema, V as LocalDebuggerLiveEventInput, W as LocalEvaluator, X as LocalEvaluatorContext, Y as LocalNumberVerdict, Z as LocalNumberVerdictSchema, _ as LocalScoreVerdict, $ as LocalScoreVerdictSchema, a0 as LocalVerdict, a1 as LocalVerdictSchema, a2 as MAX_EVAL_STATUS_RETRIES, a3 as MAX_INGEST_SIZE_BYTES, a4 as NumericThreshold, a5 as NumericThresholdSchema, P as PartialAiTrackEvent, a6 as PortableEvalArtifact, a7 as PortableEvalLimits, a8 as PortableEvalOutcome, a9 as PortableEvalResult, aa as PortableEvaluator, ab as PortableJudgeRequest, ac as PortableJudgeVerdict, ad as PublishEvalDatasetOptions, ae as PullEvalOptions, af as PulledEvalSnapshot, R as Raindrop, ag as RaindropConnection, ah as RaindropFeatureFlagValue, ai as RaindropFeatureFlags, aj as RaindropProperties, ak as RaindropPropertyLeaf, al as RaindropPropertyValue, a as RaindropSpanProcessor, am as ReplayClient, an as ReplayCounts, ao as ReplayCountsSchema, ap as ReplayEvaluator, aq as ReplayEvaluatorCounts, ar as ReplayEvaluatorCountsSchema, as as ReplayEvaluatorStats, at as ReplayEvaluatorStatsSchema, au as ReplayEvaluatorSummary, av as ReplayEvaluatorSummarySchema, aw as ReplayHandle, ax as ReplayLocalEvaluatorContext, ay as ReplayOptions, az as ReplayReading, aA as ReplayResult, aB as ReplayResultRow, aC as ReplayResultRowSchema, aD as ReplayRow, aE as ReplayRowOutcome, aF as ReplayVerdict, aG as ReplayVerdictSchema, aH as RunEvalSuiteOptions, aI as RunReplayOptions, aJ as SelfDiagnoseOptions, aK as SelfDiagnosticsExecuteContext, aL as SelfDiagnosticsExecuteResult, aM as SelfDiagnosticsSignalDefinition, aN as SelfDiagnosticsSignalDefinitions, aO as SelfDiagnosticsTool, aP as SelfDiagnosticsToolInputSchema, aQ as SelfDiagnosticsToolOptions, aR as SignalEvent, aS as SpanParams, aT as SubagentDispatch, aU as SubagentDispatchOptions, aV as SubagentFinishOptions, c as SubagentResumeOptions, d as SubagentRun, aW as ToolParams, aX as ToolSpan, aY as Trace, aZ as TraceContext, a_ as TraceSchema, e as Tracer, a$ as TrackToolParams, b0 as WorkflowParams, b1 as claimReplay, b2 as createEvalSuiteRun, b3 as createReplay, R as default, b4 as defineDataset, b5 as defineEvalSuite, b6 as defineEvaluatorProgram, b7 as defineLocalEvaluator, b8 as evaluatePortableEvaluator, b9 as evaluateReplay, ba as importPortableEvaluator, bb as isEvalDataset, bc as isEvalSuiteDefinition, bd as loadEvalSnapshot, be as projectWorkshopTrace, bf as publishEvalDataset, bg as pullEval, bh as readEvalDataset, bi as readEvalManifest, bj as readReplay, bk as replay, bl as resolveDisableBatching, bm as runEvalSuite, bn as runReplay, bo as traceOutput, bp as traceToolCalls, bq as verifyPortableEvalArtifact } from './index-CQj4bn2v.mjs';
+export { f as AiTrackEvent, A as AiTrackUsage, g as Attachment, h as AttachmentType, B as BeginInteractionOptions, i as BooleanThreshold, j as BooleanThresholdSchema, k as CreateReplayOptions, D as DEFAULT_EVAL_POLL_INTERVAL_MS, l as DEFAULT_EVAL_WAIT_MS, m as DEFAULT_QUERY_URL, n as DEFAULT_TRACE_WAIT_MS, o as DefinedEvalDataset, E as EvalCaseResult, p as EvalClient, q as EvalDataset, r as EvalDatasetCaseInput, s as EvalDatasetCaseInputSchema, t as EvalDatasetManifest, u as EvalDatasetPublishConflictError, v as EvalDestination, w as EvalEvaluator, x as EvalLocalEvaluator, y as EvalManifest, z as EvalProgram, F as EvalSuiteDefinition, G as EvalSuiteDefinitionInput, H as EvalSuiteResult, J as EvalSuiteRunSession, K as EvalVerdictAssessment, L as EvaluateReplayOptions, M as FinishInteractionOptions, N as HostedEvaluatorPin, O as IdentifyEvent, b as Interaction, Q as LocalBooleanVerdict, U as LocalBooleanVerdictSchema, V as LocalDebuggerLiveEventInput, W as LocalEvaluator, X as LocalEvaluatorContext, Y as LocalNumberVerdict, Z as LocalNumberVerdictSchema, _ as LocalScoreVerdict, $ as LocalScoreVerdictSchema, a0 as LocalVerdict, a1 as LocalVerdictSchema, a2 as MAX_EVAL_STATUS_RETRIES, a3 as MAX_INGEST_SIZE_BYTES, a4 as NumericThreshold, a5 as NumericThresholdSchema, P as PartialAiTrackEvent, a6 as PortableEvalArtifact, a7 as PortableEvalLimits, a8 as PortableEvalOutcome, a9 as PortableEvalResult, aa as PortableEvaluator, ab as PortableJudgeRequest, ac as PortableJudgeVerdict, ad as PublishEvalDatasetOptions, ae as PullEvalOptions, af as PulledEvalSnapshot, R as Raindrop, ag as RaindropConnection, ah as RaindropFeatureFlagValue, ai as RaindropFeatureFlags, aj as RaindropProperties, ak as RaindropPropertyLeaf, al as RaindropPropertyValue, a as RaindropSpanProcessor, am as ReplayClient, an as ReplayCounts, ao as ReplayCountsSchema, ap as ReplayEvaluator, aq as ReplayEvaluatorCounts, ar as ReplayEvaluatorCountsSchema, as as ReplayEvaluatorStats, at as ReplayEvaluatorStatsSchema, au as ReplayEvaluatorSummary, av as ReplayEvaluatorSummarySchema, aw as ReplayHandle, ax as ReplayLocalEvaluatorContext, ay as ReplayOptions, az as ReplayReading, aA as ReplayResult, aB as ReplayResultRow, aC as ReplayResultRowSchema, aD as ReplayRow, aE as ReplayRowOutcome, aF as ReplayVerdict, aG as ReplayVerdictSchema, aH as RunEvalSuiteOptions, aI as RunReplayOptions, aJ as SelfDiagnoseOptions, aK as SelfDiagnosticsExecuteContext, aL as SelfDiagnosticsExecuteResult, aM as SelfDiagnosticsSignalDefinition, aN as SelfDiagnosticsSignalDefinitions, aO as SelfDiagnosticsTool, aP as SelfDiagnosticsToolInputSchema, aQ as SelfDiagnosticsToolOptions, aR as SignalEvent, aS as SpanParams, aT as SubagentDispatch, aU as SubagentDispatchOptions, aV as SubagentFinishOptions, c as SubagentResumeOptions, d as SubagentRun, aW as ToolParams, aX as ToolSpan, aY as Trace, aZ as TraceContext, a_ as TraceSchema, e as Tracer, a$ as TrackToolParams, b0 as WorkflowParams, b1 as claimReplay, b2 as createEvalSuiteRun, b3 as createReplay, R as default, b4 as defineDataset, b5 as defineEvalSuite, b6 as defineEvaluatorProgram, b7 as defineLocalEvaluator, b8 as evaluatePortableEvaluator, b9 as evaluateReplay, ba as importPortableEvaluator, bb as isEvalDataset, bc as isEvalSuiteDefinition, bd as loadEvalSnapshot, be as projectWorkshopTrace, bf as publishEvalDataset, bg as pullEval, bh as readEvalDataset, bi as readEvalManifest, bj as readReplay, bk as replay, bl as resolveDisableBatching, bm as runEvalSuite, bn as runReplay, bo as traceOutput, bp as traceToolCalls, bq as verifyPortableEvalArtifact } from './index-APN1jN-i.mjs';
 export { E as EVAL_CORRELATION_ID_ATTRIBUTE, a as EvalScope, c as currentEvalScope, w as withEvalScope } from './eval-scope-BYkhP4uI.js';
 import '@opentelemetry/api';
 import 'zod';

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import '@opentelemetry/instrumentation';
-export { f as AiTrackEvent, A as AiTrackUsage, g as Attachment, h as AttachmentType, B as BeginInteractionOptions, i as BooleanThreshold, j as BooleanThresholdSchema, k as CreateReplayOptions, D as DEFAULT_EVAL_POLL_INTERVAL_MS, l as DEFAULT_EVAL_WAIT_MS, m as DEFAULT_QUERY_URL, n as DEFAULT_TRACE_WAIT_MS, o as DefinedEvalDataset, E as EvalCaseResult, p as EvalClient, q as EvalDataset, r as EvalDatasetCaseInput, s as EvalDatasetCaseInputSchema, t as EvalDatasetManifest, u as EvalDatasetPublishConflictError, v as EvalDestination, w as EvalEvaluator, x as EvalLocalEvaluator, y as EvalManifest, z as EvalProgram, F as EvalSuiteDefinition, G as EvalSuiteDefinitionInput, H as EvalSuiteResult, J as EvalSuiteRunSession, K as EvalVerdictAssessment, L as EvaluateReplayOptions, M as FinishInteractionOptions, N as HostedEvaluatorPin, O as IdentifyEvent, b as Interaction, Q as LocalBooleanVerdict, U as LocalBooleanVerdictSchema, V as LocalDebuggerLiveEventInput, W as LocalEvaluator, X as LocalEvaluatorContext, Y as LocalNumberVerdict, Z as LocalNumberVerdictSchema, _ as LocalScoreVerdict, $ as LocalScoreVerdictSchema, a0 as LocalVerdict, a1 as LocalVerdictSchema, a2 as MAX_EVAL_STATUS_RETRIES, a3 as MAX_INGEST_SIZE_BYTES, a4 as NumericThreshold, a5 as NumericThresholdSchema, P as PartialAiTrackEvent, a6 as PortableEvalArtifact, a7 as PortableEvalLimits, a8 as PortableEvalOutcome, a9 as PortableEvalResult, aa as PortableEvaluator, ab as PortableJudgeRequest, ac as PortableJudgeVerdict, ad as PublishEvalDatasetOptions, ae as PullEvalOptions, af as PulledEvalSnapshot, R as Raindrop, ag as RaindropConnection, ah as RaindropFeatureFlagValue, ai as RaindropFeatureFlags, aj as RaindropProperties, ak as RaindropPropertyLeaf, al as RaindropPropertyValue, a as RaindropSpanProcessor, am as ReplayClient, an as ReplayCounts, ao as ReplayCountsSchema, ap as ReplayEvaluator, aq as ReplayEvaluatorCounts, ar as ReplayEvaluatorCountsSchema, as as ReplayEvaluatorStats, at as ReplayEvaluatorStatsSchema, au as ReplayEvaluatorSummary, av as ReplayEvaluatorSummarySchema, aw as ReplayHandle, ax as ReplayLocalEvaluatorContext, ay as ReplayOptions, az as ReplayReading, aA as ReplayResult, aB as ReplayResultRow, aC as ReplayResultRowSchema, aD as ReplayRow, aE as ReplayRowOutcome, aF as ReplayVerdict, aG as ReplayVerdictSchema, aH as RunEvalSuiteOptions, aI as RunReplayOptions, aJ as SelfDiagnoseOptions, aK as SelfDiagnosticsExecuteContext, aL as SelfDiagnosticsExecuteResult, aM as SelfDiagnosticsSignalDefinition, aN as SelfDiagnosticsSignalDefinitions, aO as SelfDiagnosticsTool, aP as SelfDiagnosticsToolInputSchema, aQ as SelfDiagnosticsToolOptions, aR as SignalEvent, aS as SpanParams, aT as SubagentDispatch, aU as SubagentDispatchOptions, aV as SubagentFinishOptions, c as SubagentResumeOptions, d as SubagentRun, aW as ToolParams, aX as ToolSpan, aY as Trace, aZ as TraceContext, a_ as TraceSchema, e as Tracer, a$ as TrackToolParams, b0 as WorkflowParams, b1 as claimReplay, b2 as createEvalSuiteRun, b3 as createReplay, R as default, b4 as defineDataset, b5 as defineEvalSuite, b6 as defineEvaluatorProgram, b7 as defineLocalEvaluator, b8 as evaluatePortableEvaluator, b9 as evaluateReplay, ba as importPortableEvaluator, bb as isEvalDataset, bc as isEvalSuiteDefinition, bd as loadEvalSnapshot, be as projectWorkshopTrace, bf as publishEvalDataset, bg as pullEval, bh as readEvalDataset, bi as readEvalManifest, bj as readReplay, bk as replay, bl as resolveDisableBatching, bm as runEvalSuite, bn as runReplay, bo as traceOutput, bp as traceToolCalls, bq as verifyPortableEvalArtifact } from './index-CQj4bn2v.js';
+export { f as AiTrackEvent, A as AiTrackUsage, g as Attachment, h as AttachmentType, B as BeginInteractionOptions, i as BooleanThreshold, j as BooleanThresholdSchema, k as CreateReplayOptions, D as DEFAULT_EVAL_POLL_INTERVAL_MS, l as DEFAULT_EVAL_WAIT_MS, m as DEFAULT_QUERY_URL, n as DEFAULT_TRACE_WAIT_MS, o as DefinedEvalDataset, E as EvalCaseResult, p as EvalClient, q as EvalDataset, r as EvalDatasetCaseInput, s as EvalDatasetCaseInputSchema, t as EvalDatasetManifest, u as EvalDatasetPublishConflictError, v as EvalDestination, w as EvalEvaluator, x as EvalLocalEvaluator, y as EvalManifest, z as EvalProgram, F as EvalSuiteDefinition, G as EvalSuiteDefinitionInput, H as EvalSuiteResult, J as EvalSuiteRunSession, K as EvalVerdictAssessment, L as EvaluateReplayOptions, M as FinishInteractionOptions, N as HostedEvaluatorPin, O as IdentifyEvent, b as Interaction, Q as LocalBooleanVerdict, U as LocalBooleanVerdictSchema, V as LocalDebuggerLiveEventInput, W as LocalEvaluator, X as LocalEvaluatorContext, Y as LocalNumberVerdict, Z as LocalNumberVerdictSchema, _ as LocalScoreVerdict, $ as LocalScoreVerdictSchema, a0 as LocalVerdict, a1 as LocalVerdictSchema, a2 as MAX_EVAL_STATUS_RETRIES, a3 as MAX_INGEST_SIZE_BYTES, a4 as NumericThreshold, a5 as NumericThresholdSchema, P as PartialAiTrackEvent, a6 as PortableEvalArtifact, a7 as PortableEvalLimits, a8 as PortableEvalOutcome, a9 as PortableEvalResult, aa as PortableEvaluator, ab as PortableJudgeRequest, ac as PortableJudgeVerdict, ad as PublishEvalDatasetOptions, ae as PullEvalOptions, af as PulledEvalSnapshot, R as Raindrop, ag as RaindropConnection, ah as RaindropFeatureFlagValue, ai as RaindropFeatureFlags, aj as RaindropProperties, ak as RaindropPropertyLeaf, al as RaindropPropertyValue, a as RaindropSpanProcessor, am as ReplayClient, an as ReplayCounts, ao as ReplayCountsSchema, ap as ReplayEvaluator, aq as ReplayEvaluatorCounts, ar as ReplayEvaluatorCountsSchema, as as ReplayEvaluatorStats, at as ReplayEvaluatorStatsSchema, au as ReplayEvaluatorSummary, av as ReplayEvaluatorSummarySchema, aw as ReplayHandle, ax as ReplayLocalEvaluatorContext, ay as ReplayOptions, az as ReplayReading, aA as ReplayResult, aB as ReplayResultRow, aC as ReplayResultRowSchema, aD as ReplayRow, aE as ReplayRowOutcome, aF as ReplayVerdict, aG as ReplayVerdictSchema, aH as RunEvalSuiteOptions, aI as RunReplayOptions, aJ as SelfDiagnoseOptions, aK as SelfDiagnosticsExecuteContext, aL as SelfDiagnosticsExecuteResult, aM as SelfDiagnosticsSignalDefinition, aN as SelfDiagnosticsSignalDefinitions, aO as SelfDiagnosticsTool, aP as SelfDiagnosticsToolInputSchema, aQ as SelfDiagnosticsToolOptions, aR as SignalEvent, aS as SpanParams, aT as SubagentDispatch, aU as SubagentDispatchOptions, aV as SubagentFinishOptions, c as SubagentResumeOptions, d as SubagentRun, aW as ToolParams, aX as ToolSpan, aY as Trace, aZ as TraceContext, a_ as TraceSchema, e as Tracer, a$ as TrackToolParams, b0 as WorkflowParams, b1 as claimReplay, b2 as createEvalSuiteRun, b3 as createReplay, R as default, b4 as defineDataset, b5 as defineEvalSuite, b6 as defineEvaluatorProgram, b7 as defineLocalEvaluator, b8 as evaluatePortableEvaluator, b9 as evaluateReplay, ba as importPortableEvaluator, bb as isEvalDataset, bc as isEvalSuiteDefinition, bd as loadEvalSnapshot, be as projectWorkshopTrace, bf as publishEvalDataset, bg as pullEval, bh as readEvalDataset, bi as readEvalManifest, bj as readReplay, bk as replay, bl as resolveDisableBatching, bm as runEvalSuite, bn as runReplay, bo as traceOutput, bp as traceToolCalls, bq as verifyPortableEvalArtifact } from './index-APN1jN-i.js';
 export { E as EVAL_CORRELATION_ID_ATTRIBUTE, a as EvalScope, c as currentEvalScope, w as withEvalScope } from './eval-scope-BYkhP4uI.js';
 import '@opentelemetry/api';
 import 'zod';

package/dist/index.js

@@ -6718,7 +6718,7 @@ var init_trace = __esm({
       }),
       reference: EvalCaseTraceSchema.extend({
         trace: TraceSchema.extend({ origin: import_zod3.default.literal("dataset") })
-      })
+      }).optional()
     });
     MAX_CUSTOM_ATTRIBUTES = 25;
     MAX_ATTRIBUTE_VALUE_CHARS = 500;
@@ -6784,7 +6784,7 @@ var init_trace = __esm({
       "refresh token"
     ]);
     MAX_EVAL_PAYLOAD_RAW_CHARS = 16 * 1024;
-    TRACE_PROJECTION_SHA256 = "8dab83087aa2def3d076324021d5e08911e37967b68597b58b793a80d54b5874";
+    TRACE_PROJECTION_SHA256 = "7d12eb03150b8f203e4404622f047676f50b3aa1e5442f0ed42d77d6002a33a6";
   }
 });
 
@@ -12558,7 +12558,7 @@ var SignalEventSchema = external_exports.object({
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.5.0",
+  version: "0.6.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",
@@ -16438,10 +16438,10 @@ var EvalDatasetCaseInputSchema = import_zod4.z.object({
   name: import_zod4.z.string().trim().min(1).max(200),
   input: import_zod4.z.string().nullable(),
   properties: import_zod4.z.record(import_zod4.z.string(), import_zod4.z.string()),
-  referenceTrace: CapturedTraceSnapshotSchema
+  referenceTrace: CapturedTraceSnapshotSchema.optional()
 }).strict();
 var EvalDatasetVersionCaseSchema = EvalDatasetCaseInputSchema.extend({
-  referenceTraceSha256: EvalDatasetFingerprintSchema
+  referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
 });
 var EvalDatasetCaseInputsSchema = import_zod4.z.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context9) => {
   const seen = /* @__PURE__ */ new Set();
@@ -16480,8 +16480,13 @@ var EvalDatasetVersionSchema = import_zod4.z.object({
 var EvalDatasetManifestSchema = import_zod4.z.object({
   dataset: EvalDatasetIdentitySchema,
   version: EvalDatasetVersionSchema.omit({ cases: true }),
-  cases: import_zod4.z.array(EvalDatasetVersionCaseSchema.pick({ id: true, name: true, input: true, properties: true }).extend({
-    referenceTraceSha256: EvalDatasetFingerprintSchema
+  cases: import_zod4.z.array(EvalDatasetVersionCaseSchema.pick({
+    id: true,
+    name: true,
+    input: true,
+    properties: true
+  }).extend({
+    referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
   }))
 });
 function compareNanoseconds2(left, right) {
@@ -16496,10 +16501,10 @@ function orderSnapshot(snapshot) {
   };
 }
 function normalizeCasesForFingerprint(cases) {
-  return cases.map((entry) => ({
+  return cases.map((entry) => entry.referenceTrace === void 0 ? entry : {
     ...entry,
     referenceTrace: orderSnapshot(entry.referenceTrace)
-  }));
+  });
 }
 function stableStringify(value) {
   if (value === null || typeof value !== "object") return JSON.stringify(value);
@@ -17328,6 +17333,7 @@ async function createLocalEvalReplaySession(client, options) {
     throw new Error("Raindrop local eval replay requires at least one evaluator");
   }
   const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  await validateReferenceRequirements(client, options.dataset, options.rowIds, options.evaluators, queryUrl);
   const handle = await createReplay(client, {
     dataset: options.dataset,
     rowIds: options.rowIds,
@@ -17651,9 +17657,30 @@ function reportReplay(client, queryUrl, replayId, body) {
     maxRetries: MAX_REPLAY_REPORT_RETRIES
   });
 }
+async function validateReferenceRequirements(client, dataset, rowIds, evaluators, queryUrl) {
+  const selected = new Set(selectEvalDatasetRows(dataset, rowIds));
+  const missing = dataset.cases.filter((entry) => selected.has(entry.id) && entry.referenceTraceSha256 === void 0);
+  if (missing.length === 0) return;
+  for (const evaluator of evaluators) {
+    const requiresReference = typeof evaluator === "string" ? (await queryApi({
+      client,
+      queryUrl,
+      path: `/v1/evals/${encodeURIComponent(evaluator)}`,
+      method: "GET",
+      schema: import_zod5.z.object({ data: import_zod5.z.object({ requiresReference: import_zod5.z.boolean() }) })
+    })).data.requiresReference : evaluator.requiresReference === true;
+    if (requiresReference) {
+      const name = typeof evaluator === "string" ? evaluator : evaluator.slug;
+      throw new Error(`Raindrop evaluator ${name} requires a reference trace. Missing references for cases: ${missing.map((entry) => entry.id).join(", ")}`);
+    }
+  }
+}
 async function replay(client, options) {
-  var _a, _b;
+  var _a, _b, _c;
   const evaluators = normalizeEvaluators(options.evaluators);
+  const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  const dataset = typeof options.dataset === "string" ? await readEvalDataset(client, options.dataset, { queryUrl }) : options.dataset;
+  await validateReferenceRequirements(client, dataset, options.rowIds, evaluators, queryUrl);
   const local = evaluators.flatMap(
     (evaluator) => typeof evaluator === "string" ? [] : [{ evaluator, verdicts: [], durationMs: 0 }]
   );
@@ -17665,7 +17692,7 @@ async function replay(client, options) {
       }
     )
   );
-  const handle = await createReplay(client, options);
+  const handle = await createReplay(client, { ...options, dataset });
   const execution = await runReplayRows(client, handle, options);
   const { reading } = execution;
   const doneRowIds = new Set(
@@ -17678,11 +17705,11 @@ async function replay(client, options) {
   if (local.length > 0) {
     const pairs = await readEvalCaseTracePairs({
       client,
-      queryUrl: (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+      queryUrl: (_b = options.queryUrl) != null ? _b : DEFAULT_QUERY_URL,
       replayId: reading.replayId,
       datasetVersion: handle.datasetVersion,
       evidence: traceEvidence,
-      waitMs: (_b = options.traceWaitMs) != null ? _b : DEFAULT_TRACE_WAIT_MS
+      waitMs: (_c = options.traceWaitMs) != null ? _c : DEFAULT_TRACE_WAIT_MS
     });
     const rowsById = new Map(
       handle.rows.map(({ correlationId: _correlationId, ...row }) => [row.id, row])
@@ -18181,6 +18208,7 @@ async function publishEvalDataset(client, options) {
 }
 function validateUploadSizes(body) {
   for (const [index, entry] of body.cases.entries()) {
+    if (entry.referenceTrace === void 0) continue;
     const bytes2 = encodedBytes(entry.referenceTrace);
     if (bytes2 > EVAL_DATASET_MAX_REFERENCE_BYTES) {
       throw new Error(
@@ -19324,21 +19352,24 @@ function replayEvaluator(entry) {
         slug: entry.evaluator.slug,
         name: entry.evaluator.name,
         output: entry.evaluator.output,
-        judge: entry.evaluator.judge
+        judge: entry.evaluator.judge,
+        requiresReference: entry.evaluator.requiresReference
       };
     case "score":
       return {
         slug: entry.evaluator.slug,
         name: entry.evaluator.name,
         output: entry.evaluator.output,
-        judge: entry.evaluator.judge
+        judge: entry.evaluator.judge,
+        requiresReference: entry.evaluator.requiresReference
       };
     case "number":
       return {
         slug: entry.evaluator.slug,
         name: entry.evaluator.name,
         output: entry.evaluator.output,
-        judge: entry.evaluator.judge
+        judge: entry.evaluator.judge,
+        requiresReference: entry.evaluator.requiresReference
       };
   }
 }

package/dist/index.mjs

@@ -30,7 +30,7 @@ import {
   withReplayTraceDestination,
   withRoutingContext,
   withTask
-} from "./chunk-AUNFORWV.mjs";
+} from "./chunk-ZVHQBBST.mjs";
 import {
   BooleanThresholdSchema,
   CapturedTraceSnapshotSchema,
@@ -55,7 +55,7 @@ import {
   traceToolCalls,
   validateEvaluatorOutput,
   verifyPortableEvalArtifact
-} from "./chunk-HFHHGRTW.mjs";
+} from "./chunk-CFGEE5ID.mjs";
 import {
   __commonJS,
   __toESM
@@ -10034,10 +10034,10 @@ var EvalDatasetCaseInputSchema = z2.object({
   name: z2.string().trim().min(1).max(200),
   input: z2.string().nullable(),
   properties: z2.record(z2.string(), z2.string()),
-  referenceTrace: CapturedTraceSnapshotSchema
+  referenceTrace: CapturedTraceSnapshotSchema.optional()
 }).strict();
 var EvalDatasetVersionCaseSchema = EvalDatasetCaseInputSchema.extend({
-  referenceTraceSha256: EvalDatasetFingerprintSchema
+  referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
 });
 var EvalDatasetCaseInputsSchema = z2.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context3) => {
   const seen = /* @__PURE__ */ new Set();
@@ -10076,8 +10076,13 @@ var EvalDatasetVersionSchema = z2.object({
 var EvalDatasetManifestSchema = z2.object({
   dataset: EvalDatasetIdentitySchema,
   version: EvalDatasetVersionSchema.omit({ cases: true }),
-  cases: z2.array(EvalDatasetVersionCaseSchema.pick({ id: true, name: true, input: true, properties: true }).extend({
-    referenceTraceSha256: EvalDatasetFingerprintSchema
+  cases: z2.array(EvalDatasetVersionCaseSchema.pick({
+    id: true,
+    name: true,
+    input: true,
+    properties: true
+  }).extend({
+    referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
   }))
 });
 function compareNanoseconds(left, right) {
@@ -10092,10 +10097,10 @@ function orderSnapshot(snapshot) {
   };
 }
 function normalizeCasesForFingerprint(cases) {
-  return cases.map((entry) => ({
+  return cases.map((entry) => entry.referenceTrace === void 0 ? entry : {
     ...entry,
     referenceTrace: orderSnapshot(entry.referenceTrace)
-  }));
+  });
 }
 function stableStringify(value) {
   if (value === null || typeof value !== "object") return JSON.stringify(value);
@@ -10921,6 +10926,7 @@ async function createLocalEvalReplaySession(client, options) {
     throw new Error("Raindrop local eval replay requires at least one evaluator");
   }
   const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  await validateReferenceRequirements(client, options.dataset, options.rowIds, options.evaluators, queryUrl);
   const handle = await createReplay(client, {
     dataset: options.dataset,
     rowIds: options.rowIds,
@@ -11244,9 +11250,30 @@ function reportReplay(client, queryUrl, replayId, body) {
     maxRetries: MAX_REPLAY_REPORT_RETRIES
   });
 }
+async function validateReferenceRequirements(client, dataset, rowIds, evaluators, queryUrl) {
+  const selected = new Set(selectEvalDatasetRows(dataset, rowIds));
+  const missing = dataset.cases.filter((entry) => selected.has(entry.id) && entry.referenceTraceSha256 === void 0);
+  if (missing.length === 0) return;
+  for (const evaluator of evaluators) {
+    const requiresReference = typeof evaluator === "string" ? (await queryApi({
+      client,
+      queryUrl,
+      path: `/v1/evals/${encodeURIComponent(evaluator)}`,
+      method: "GET",
+      schema: z3.object({ data: z3.object({ requiresReference: z3.boolean() }) })
+    })).data.requiresReference : evaluator.requiresReference === true;
+    if (requiresReference) {
+      const name = typeof evaluator === "string" ? evaluator : evaluator.slug;
+      throw new Error(`Raindrop evaluator ${name} requires a reference trace. Missing references for cases: ${missing.map((entry) => entry.id).join(", ")}`);
+    }
+  }
+}
 async function replay(client, options) {
-  var _a, _b;
+  var _a, _b, _c;
   const evaluators = normalizeEvaluators(options.evaluators);
+  const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  const dataset = typeof options.dataset === "string" ? await readEvalDataset(client, options.dataset, { queryUrl }) : options.dataset;
+  await validateReferenceRequirements(client, dataset, options.rowIds, evaluators, queryUrl);
   const local = evaluators.flatMap(
     (evaluator) => typeof evaluator === "string" ? [] : [{ evaluator, verdicts: [], durationMs: 0 }]
   );
@@ -11258,7 +11285,7 @@ async function replay(client, options) {
       }
     )
   );
-  const handle = await createReplay(client, options);
+  const handle = await createReplay(client, { ...options, dataset });
   const execution = await runReplayRows(client, handle, options);
   const { reading } = execution;
   const doneRowIds = new Set(
@@ -11271,11 +11298,11 @@ async function replay(client, options) {
   if (local.length > 0) {
     const pairs = await readEvalCaseTracePairs({
       client,
-      queryUrl: (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+      queryUrl: (_b = options.queryUrl) != null ? _b : DEFAULT_QUERY_URL,
       replayId: reading.replayId,
       datasetVersion: handle.datasetVersion,
       evidence: traceEvidence,
-      waitMs: (_b = options.traceWaitMs) != null ? _b : DEFAULT_TRACE_WAIT_MS
+      waitMs: (_c = options.traceWaitMs) != null ? _c : DEFAULT_TRACE_WAIT_MS
     });
     const rowsById = new Map(
       handle.rows.map(({ correlationId: _correlationId, ...row }) => [row.id, row])
@@ -11771,6 +11798,7 @@ async function publishEvalDataset(client, options) {
 }
 function validateUploadSizes(body) {
   for (const [index, entry] of body.cases.entries()) {
+    if (entry.referenceTrace === void 0) continue;
     const bytes2 = encodedBytes(entry.referenceTrace);
     if (bytes2 > EVAL_DATASET_MAX_REFERENCE_BYTES) {
       throw new Error(
@@ -12902,21 +12930,24 @@ function replayEvaluator(entry) {
         slug: entry.evaluator.slug,
         name: entry.evaluator.name,
         output: entry.evaluator.output,
-        judge: entry.evaluator.judge
+        judge: entry.evaluator.judge,
+        requiresReference: entry.evaluator.requiresReference
       };
     case "score":
       return {
         slug: entry.evaluator.slug,
         name: entry.evaluator.name,
         output: entry.evaluator.output,
-        judge: entry.evaluator.judge
+        judge: entry.evaluator.judge,
+        requiresReference: entry.evaluator.requiresReference
       };
     case "number":
       return {
         slug: entry.evaluator.slug,
         name: entry.evaluator.name,
         output: entry.evaluator.output,
-        judge: entry.evaluator.judge
+        judge: entry.evaluator.judge,
+        requiresReference: entry.evaluator.requiresReference
       };
   }
 }

package/dist/{portable-runtime-GBE3SHBZ.mjs → portable-runtime-QZKU2VXS.mjs}

@@ -1,6 +1,6 @@
 import {
   parsePortableEvalResult
-} from "./chunk-HFHHGRTW.mjs";
+} from "./chunk-CFGEE5ID.mjs";
 import "./chunk-UJCSKKID.mjs";
 
 // src/evals/portable-runtime.ts

package/dist/tracing/index.d.mts

@@ -1,5 +1,5 @@
 import { Instrumentation } from '@opentelemetry/instrumentation';
-import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-CQj4bn2v.mjs';
+import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-APN1jN-i.mjs';
 import '@opentelemetry/api';
 import 'zod';
 import '@opentelemetry/sdk-trace-base';

package/dist/tracing/index.d.ts

@@ -1,5 +1,5 @@
 import { Instrumentation } from '@opentelemetry/instrumentation';
-import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-CQj4bn2v.js';
+import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-APN1jN-i.js';
 import '@opentelemetry/api';
 import 'zod';
 import '@opentelemetry/sdk-trace-base';

package/dist/tracing/index.js

@@ -1384,7 +1384,7 @@ function base64ToHex2(base64) {
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.5.0",
+  version: "0.6.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",

package/dist/tracing/index.mjs

@@ -1,6 +1,6 @@
 import {
   tracing
-} from "../chunk-AUNFORWV.mjs";
+} from "../chunk-ZVHQBBST.mjs";
 import "../chunk-UJCSKKID.mjs";
 
 // src/tracing/index.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "raindrop-ai",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
   "types": "dist/index.d.ts",