raindrop-ai 0.5.0-otelv2 → 0.6.0

package/README.md

@@ -67,6 +67,41 @@ On runtimes that lack a usable `AsyncLocalStorage.enterWith` (notably Cloudflare
 
 ## Eval mode
 
+Run your agent locally and store eval results in hosted Raindrop. Workshop is not required. Define a reusable suite with `defineEvalSuite`, then run it with `runEvalSuite` or the `@raindrop-ai/vitest` adapter. Input-only datasets require the SDK release with optional-reference support and the matching Raindrop backend.
+
+```ts
+import Raindrop, { defineEvalSuite, runEvalSuite } from "raindrop-ai";
+import { supportAgent } from "./support-agent";
+
+const client = new Raindrop({
+  apiKey: process.env.RAINDROP_QUERY_API_KEY,
+  projectId: process.env.RAINDROP_PROJECT_ID,
+  localWorkshopUrl: false,
+});
+
+const suite = defineEvalSuite({
+  name: "Support responses",
+  dataset: "support-cases",
+  run: ({ input }) => supportAgent(input ?? ""),
+  evaluators: [{ evaluator: "task-completed", threshold: { equals: true } }],
+});
+
+try {
+  const result = await runEvalSuite(client, suite, {
+    destination: { kind: "raindrop" },
+  });
+  console.log(result);
+} finally {
+  await client.close();
+}
+```
+
+The dataset string names a published, project-scoped eval dataset, not an ordinary custom-upload dataset ID. Publish its immutable version with `publishEvalDataset`. Cases can contain only input and scenario properties; an optional reference must be a complete captured trace. Local evaluators can declare `requiresReference: true`, and hosted programs can export `const requiresReference = true`, to reject missing references before agent execution. A local `defineDataset` object does not publish itself and is not accepted by the hosted runner. See the [Vitest guide](../vitest/README.md) for publication, evaluator requirements, and credential setup.
+
+The Query SDK organization `apiKey` authenticates evals, dataset publication/reads, and replay trace uploads. Browser login and production `writeKey` credentials do not replace it. The agent's model credentials and test environment are separate. Running locally describes where your callback executes, not where results are stored.
+
+### Trace correlation
+
 A replay calls your agent once per row and needs every span and event that call
 emits to land on that row. `withEvalScope` binds one opaque **correlation id**
 to the execution context, so it does, with no change to the agent:
@@ -115,9 +150,9 @@ Both limits are demonstrated by tests in `packages/core/src/eval-scope.test.ts`.
 
 ### Replaying a dataset
 
-`replay` is the loop that uses the scope. It asks Raindrop for the dataset's
-rows, calls your agent once per row inside that row's scope, says which rows
-failed, and waits for the traces to land:
+`replay` is the lower-level loop used by hosted suite execution. It reads the
+published dataset's immutable version, calls your agent once per row inside
+that row's scope, reports failed rows, and waits for the traces to land:
 
 ```ts
 import Raindrop, { replay } from "raindrop-ai";

package/dist/{chunk-HFHHGRTW.mjs → chunk-CFGEE5ID.mjs}

@@ -707,7 +707,7 @@ var EvalCaseTracePairSchema = z2.object({
   }),
   reference: EvalCaseTraceSchema.extend({
     trace: TraceSchema.extend({ origin: z2.literal("dataset") })
-  })
+  }).optional()
 });
 function compareNanoseconds(left, right) {
   if (Number.isFinite(left) && Number.isFinite(right))
@@ -1007,7 +1007,7 @@ function traceOutput(trace) {
 function traceToolCalls(trace, name) {
   return trace.entries.filter((entry) => entry.type === "tool_call" && (name === void 0 || entry.name === name));
 }
-var TRACE_PROJECTION_SHA256 = "8dab83087aa2def3d076324021d5e08911e37967b68597b58b793a80d54b5874";
+var TRACE_PROJECTION_SHA256 = "7d12eb03150b8f203e4404622f047676f50b3aa1e5442f0ed42d77d6002a33a6";
 
 // src/evals/portable.ts
 import { z as z3 } from "zod";
@@ -1095,7 +1095,7 @@ async function evaluatePortableEvaluator(evaluator, input) {
   const artifact = await verifyPortableEvalArtifact(evaluator.artifact, {
     expectedSha256: evaluator.expectedSha256
   });
-  const { runPortableEvaluator } = await import("./portable-runtime-GBE3SHBZ.mjs");
+  const { runPortableEvaluator } = await import("./portable-runtime-QZKU2VXS.mjs");
   const cases = (_a = input.cases) != null ? _a : [];
   if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
     (entry, index) => {

package/dist/{chunk-AUNFORWV.mjs → chunk-ZVHQBBST.mjs}

@@ -1238,7 +1238,7 @@ installTracingSuppressionHook();
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.5.0",
+  version: "0.6.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",

package/dist/{index-CQj4bn2v.d.mts → index-APN1jN-i.d.mts}

@@ -1082,7 +1082,7 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
     name: z.ZodString;
     input: z.ZodNullable<z.ZodString>;
     properties: z.ZodRecord<z.ZodString, z.ZodString>;
-    referenceTrace: z.ZodEffects<z.ZodObject<{
+    referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
         event: z.ZodObject<{
             id: z.ZodString;
             _raindropDatabaseId: z.ZodString;
@@ -5470,13 +5470,13 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    }>;
+    }>>;
 }, "strict", z.ZodTypeAny, {
     properties: Record<string, string>;
     input: string | null;
     name: string;
     id: string;
-    referenceTrace: {
+    referenceTrace?: {
         event: {
             properties: Record<string, unknown> | null;
             timestamp: string;
@@ -5873,13 +5873,13 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    };
+    } | undefined;
 }, {
     properties: Record<string, string>;
     input: string | null;
     name: string;
     id: string;
-    referenceTrace: {
+    referenceTrace?: {
         event: {
             properties: Record<string, unknown> | null;
             timestamp: string;
@@ -6276,7 +6276,7 @@ declare const EvalDatasetCaseInputSchema: z.ZodObject<{
             end_timestamp: string;
             _inserted_at: string;
         }[];
-    };
+    } | undefined;
 }>;
 type EvalDatasetCaseInput = z.infer<typeof EvalDatasetCaseInputSchema>;
 declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
@@ -6289,7 +6289,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         name: z.ZodString;
         input: z.ZodNullable<z.ZodString>;
         properties: z.ZodRecord<z.ZodString, z.ZodString>;
-        referenceTrace: z.ZodEffects<z.ZodObject<{
+        referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
             event: z.ZodObject<{
                 id: z.ZodString;
                 _raindropDatabaseId: z.ZodString;
@@ -10677,13 +10677,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        }>;
+        }>>;
     }, "strict", z.ZodTypeAny, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11080,13 +11080,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11483,13 +11483,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }>, "many">, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -11886,13 +11886,13 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[], {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -12289,7 +12289,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[]>;
 }, "strict", z.ZodTypeAny, {
     name: string;
@@ -12301,7 +12301,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -12698,7 +12698,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[];
 }, {
     name: string;
@@ -12710,7 +12710,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTrace: {
+        referenceTrace?: {
             event: {
                 properties: Record<string, unknown> | null;
                 timestamp: string;
@@ -13107,7 +13107,7 @@ declare const PublishEvalDatasetVersionInputSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        };
+        } | undefined;
     }[];
 }>;
 type PublishEvalDatasetVersionInput = z.infer<typeof PublishEvalDatasetVersionInputSchema>;
@@ -13138,7 +13138,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
             name: z.ZodString;
             input: z.ZodNullable<z.ZodString>;
             properties: z.ZodRecord<z.ZodString, z.ZodString>;
-            referenceTrace: z.ZodEffects<z.ZodObject<{
+            referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
                 event: z.ZodObject<{
                     id: z.ZodString;
                     _raindropDatabaseId: z.ZodString;
@@ -17526,15 +17526,15 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            }>;
+            }>>;
         } & {
-            referenceTraceSha256: z.ZodString;
+            referenceTraceSha256: z.ZodOptional<z.ZodString>;
         }, "strict", z.ZodTypeAny, {
             properties: Record<string, string>;
             input: string | null;
             name: string;
             id: string;
-            referenceTrace: {
+            referenceTrace?: {
                 event: {
                     properties: Record<string, unknown> | null;
                     timestamp: string;
@@ -17931,14 +17931,14 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            };
-            referenceTraceSha256: string;
+            } | undefined;
+            referenceTraceSha256?: string | undefined;
         }, {
             properties: Record<string, string>;
             input: string | null;
             name: string;
             id: string;
-            referenceTrace: {
+            referenceTrace?: {
                 event: {
                     properties: Record<string, unknown> | null;
                     timestamp: string;
@@ -18335,8 +18335,8 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                     end_timestamp: string;
                     _inserted_at: string;
                 }[];
-            };
-            referenceTraceSha256: string;
+            } | undefined;
+            referenceTraceSha256?: string | undefined;
         }>, "many">;
         createdAt: z.ZodString;
     }, "cases">, "strip", z.ZodTypeAny, {
@@ -18357,7 +18357,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         name: z.ZodString;
         input: z.ZodNullable<z.ZodString>;
         properties: z.ZodRecord<z.ZodString, z.ZodString>;
-        referenceTrace: z.ZodEffects<z.ZodObject<{
+        referenceTrace: z.ZodOptional<z.ZodEffects<z.ZodObject<{
             event: z.ZodObject<{
                 id: z.ZodString;
                 _raindropDatabaseId: z.ZodString;
@@ -22745,23 +22745,23 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
                 end_timestamp: string;
                 _inserted_at: string;
             }[];
-        }>;
+        }>>;
     } & {
-        referenceTraceSha256: z.ZodString;
+        referenceTraceSha256: z.ZodOptional<z.ZodString>;
     }, "properties" | "input" | "name" | "id"> & {
-        referenceTraceSha256: z.ZodString;
+        referenceTraceSha256: z.ZodOptional<z.ZodString>;
     }, "strict", z.ZodTypeAny, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }, {
         properties: Record<string, string>;
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     dataset: {
@@ -22775,7 +22775,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }[];
     version: {
         id: string;
@@ -22796,7 +22796,7 @@ declare const EvalDatasetManifestSchema: z.ZodObject<{
         input: string | null;
         name: string;
         id: string;
-        referenceTraceSha256: string;
+        referenceTraceSha256?: string | undefined;
     }[];
     version: {
         id: string;
@@ -43264,7 +43264,7 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
             origin: "replay";
         };
     }>;
-    reference: z$1.ZodObject<{
+    reference: z$1.ZodOptional<z$1.ZodObject<{
         snapshot: z$1.ZodEffects<z$1.ZodObject<{
             event: z$1.ZodObject<{
                 id: z$1.ZodString;
@@ -54793,9 +54793,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
             })[];
             origin: "dataset";
         };
-    }>;
+    }>>;
 }, "strip", z$1.ZodTypeAny, {
-    reference: {
+    caseId: string;
+    candidate: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -55818,11 +55819,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "dataset";
+            origin: "replay";
         };
     };
-    caseId: string;
-    candidate: {
+    reference?: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -56845,11 +56845,12 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "replay";
+            origin: "dataset";
         };
-    };
+    } | undefined;
 }, {
-    reference: {
+    caseId: string;
+    candidate: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -57872,11 +57873,10 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "dataset";
+            origin: "replay";
         };
     };
-    caseId: string;
-    candidate: {
+    reference?: {
         snapshot: {
             event: {
                 properties: Record<string, unknown> | null;
@@ -58899,9 +58899,9 @@ declare const EvalCaseTracePairSchema: z$1.ZodObject<{
                 fromAgent: string;
                 toAgent: string;
             })[];
-            origin: "replay";
+            origin: "dataset";
         };
-    };
+    } | undefined;
 }>;
 type EvalCaseTracePair = z$1.infer<typeof EvalCaseTracePairSchema>;
 type TraceNanoseconds = number | string;
@@ -59769,6 +59769,8 @@ type LocalEvaluatorContext$1<AgentResult = unknown> = {
 type LocalJudge<AgentResult, Verdict> = (context: LocalEvaluatorContext$1<AgentResult>) => Verdict | Promise<Verdict>;
 /** A named evaluator whose implementation stays in the customer's process. */
 type LocalEvaluator<AgentResult = unknown> = {
+    readonly requiresReference?: boolean;
+} & ({
     readonly slug: string;
     readonly name: string;
     readonly output: "boolean";
@@ -59783,7 +59785,7 @@ type LocalEvaluator<AgentResult = unknown> = {
     readonly name: string;
     readonly output: "number";
     readonly judge: LocalJudge<AgentResult, LocalNumberVerdict>;
-};
+});
 type CreateReplayOptions = {
     /** A project-scoped dataset slug, or a manifest already pinned by discovery. */
     dataset: string | EvalDatasetManifest;
@@ -60113,6 +60115,8 @@ type LocalEvaluatorContext<AgentResult = unknown> = {
     readonly result: AgentResult;
 };
 type LocalEvaluatorBase<AgentResult, Output, Verdict> = {
+    /** Reject input-only cases before the agent runs when this evaluator needs a baseline. */
+    readonly requiresReference?: boolean;
     readonly slug: string;
     readonly name: string;
     readonly output: Output;
@@ -60176,7 +60180,7 @@ type PublishEvalDatasetOptions = {
     readonly slug: string;
     /** Display name. Publishing a new version does not rename an existing dataset. */
     readonly name: PublishEvalDatasetVersionInput["name"];
-    /** Ordered cases. Each reference is a complete captured trace snapshot. */
+    /** Ordered scenarios. Supplied references must be complete captured trace snapshots. */
     readonly cases: readonly EvalDatasetCaseInput[];
     /** Omit or pass null to create. Pass the discovered current version when changing evidence. */
     readonly expectedCurrentVersionId?: string | null;