raindrop-ai 0.6.0 → 0.7.0

package/dist/index.js

@@ -5767,17 +5767,38 @@ var require_dist = __commonJS({
 
 // src/evals/definition.ts
 function defineDataset(dataset) {
+  var _a, _b;
   const id = dataset.id.trim();
   const name = dataset.name.trim();
-  const version = dataset.version.trim();
+  const rows = dataset.rows.map((row) => {
+    var _a2;
+    const parsed = DatasetRowInputSchema.parse(row);
+    return {
+      id: parsed.id,
+      name: (_a2 = parsed.name) != null ? _a2 : parsed.id,
+      input: parsed.input,
+      output: parsed.output,
+      properties: parsed.properties
+    };
+  });
+  const version = (_b = (_a = dataset.version) == null ? void 0 : _a.trim()) != null ? _b : (0, import_node_crypto2.createHash)("sha256").update(
+    JSON.stringify(
+      rows.map((row) => ({
+        ...row,
+        properties: Object.fromEntries(
+          Object.keys(row.properties).sort().map((key) => [key, row.properties[key]])
+        )
+      }))
+    )
+  ).digest("hex");
   if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
   if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
   if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.cases.length === 0)
-    throw new Error("Raindrop eval datasets require at least one case");
-  const ids = dataset.cases.map((entry) => entry.id.trim());
-  if (ids.some((caseId) => !caseId)) throw new Error("Raindrop eval case ids cannot be empty");
-  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval case ids must be unique");
+  if (dataset.rows.length === 0)
+    throw new Error("Raindrop eval datasets require at least one row");
+  const ids = rows.map((entry) => entry.id);
+  if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
+  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
   if (dataset.remote) {
     if (!dataset.remote.reference.trim())
       throw new Error("Raindrop remote dataset reference cannot be empty");
@@ -5790,7 +5811,7 @@ function defineDataset(dataset) {
     id,
     name,
     version,
-    cases: dataset.cases.map((entry) => ({ ...entry, id: entry.id.trim() })),
+    rows,
     [EVAL_DATASET]: true
   };
   Object.defineProperty(defined2, EVAL_DATASET, { enumerable: false });
@@ -5800,7 +5821,7 @@ function isEvalDataset(value) {
   return typeof value === "object" && value !== null && EVAL_DATASET in value && value[EVAL_DATASET] === true;
 }
 function defineEvaluatorProgram(program) {
-  var _a, _b, _c;
+  var _a, _b, _c, _d;
   const slug = program.slug.trim();
   const name = program.name.trim();
   const source = program.source.trim();
@@ -5832,16 +5853,18 @@ function defineEvaluatorProgram(program) {
   return {
     ...program,
     kind: "program",
+    scope: import_zod2.z.literal("batch").parse((_c = program.scope) != null ? _c : "batch"),
     slug,
     name,
     source,
     intent,
-    description: ((_c = program.description) == null ? void 0 : _c.trim()) || null,
+    description: ((_d = program.description) == null ? void 0 : _d.trim()) || null,
     rules
   };
 }
 function defineLocalEvaluator(evaluator) {
-  return evaluator;
+  var _a;
+  return { ...evaluator, scope: import_zod2.z.literal("row").parse((_a = evaluator.scope) != null ? _a : "row") };
 }
 function defineEvalSuite(definition) {
   const name = definition.name.trim();
@@ -5850,6 +5873,11 @@ function defineEvalSuite(definition) {
   if (typeof dataset === "string" && dataset.length === 0) {
     throw new Error("Raindrop eval suite datasets cannot be empty");
   }
+  if (definition.datasetVersionId !== void 0) {
+    import_zod2.z.string().uuid().parse(definition.datasetVersionId);
+    if (typeof dataset !== "string")
+      throw new Error("A dataset version pin requires a published dataset slug");
+  }
   if (typeof definition.run !== "function") {
     throw new Error("Raindrop eval suite run must be a function");
   }
@@ -5878,6 +5906,11 @@ function defineEvalSuite(definition) {
 function isEvalSuiteDefinition(value) {
   return typeof value === "object" && value !== null && EVAL_DEFINITION in value && value[EVAL_DEFINITION] === true;
 }
+function isPublishedEvalSuite(definition) {
+  return typeof definition.dataset === "string" && definition.datasetVersionId !== void 0 && definition.evaluators.every(
+    ({ evaluator }) => typeof evaluator !== "string" && (!("kind" in evaluator) || evaluator.kind !== "program" || evaluator.expected !== void 0)
+  );
+}
 function evaluatorName(evaluator) {
   return typeof evaluator.evaluator === "string" ? evaluator.evaluator.trim() : evaluator.evaluator.slug.trim();
 }
@@ -5940,8 +5973,8 @@ function validateEvaluator(evaluator) {
   if (typeof evaluator.evaluator.judge !== "function") {
     throw new Error(`Raindrop local evaluator ${name} judge must be a function`);
   }
-  if (evaluator.evaluator.scope !== "case") {
-    throw new Error(`Raindrop local evaluator ${name} scope must be case`);
+  if (evaluator.evaluator.scope !== "row") {
+    throw new Error(`Raindrop local evaluator ${name} scope must be row`);
   }
   const output = EvalOutputSchema.parse(evaluator.evaluator.output);
   validateEvaluatorOutput(evaluator, output);
@@ -5968,10 +6001,11 @@ function parseNumericThreshold(name, output, threshold) {
   }
   return parsed;
 }
-var import_zod2, EvalScoreSchema, EVAL_DATASET, BooleanThresholdSchema, FiniteThresholdSchema, EvalOutputSchema, NumericThresholdSchema, EVAL_DEFINITION;
+var import_node_crypto2, import_zod2, EvalScoreSchema, EVAL_DATASET, DatasetRowInputSchema, BooleanThresholdSchema, FiniteThresholdSchema, EvalOutputSchema, NumericThresholdSchema, EVAL_DEFINITION;
 var init_definition = __esm({
   "src/evals/definition.ts"() {
     "use strict";
+    import_node_crypto2 = require("crypto");
     import_zod2 = require("zod");
     EvalScoreSchema = import_zod2.z.union([
       import_zod2.z.literal(1),
@@ -5981,6 +6015,13 @@ var init_definition = __esm({
       import_zod2.z.literal(5)
     ]);
     EVAL_DATASET = /* @__PURE__ */ Symbol.for("raindrop.evalDataset");
+    DatasetRowInputSchema = import_zod2.z.object({
+      id: import_zod2.z.string().trim().min(1),
+      name: import_zod2.z.string().optional(),
+      input: import_zod2.z.string().nullable(),
+      output: import_zod2.z.string().nullable().default(null),
+      properties: import_zod2.z.record(import_zod2.z.string()).default({})
+    });
     BooleanThresholdSchema = import_zod2.z.object({ equals: import_zod2.z.boolean() }).strict();
     FiniteThresholdSchema = import_zod2.z.number().finite();
     EvalOutputSchema = import_zod2.z.enum(["boolean", "score", "number"]);
@@ -6929,7 +6970,7 @@ async function resolveJudge(evaluator, rawRequest, remainingMs, tracesById, case
           var _a2;
           return (_a2 = evaluator.judge) == null ? void 0 : _a2.call(evaluator, {
             trace: trace8,
-            tracePair: casesByTraceId.get(request.traceId),
+            tracePair: rowTracePair(casesByTraceId.get(request.traceId)),
             rubric: request.rubric.slice(0, 4e4),
             signal: controller.signal
           });
@@ -6957,13 +6998,18 @@ async function resolveJudge(evaluator, rawRequest, remainingMs, tracesById, case
 function byteLength(value) {
   return new TextEncoder().encode(value).byteLength;
 }
-var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod6, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
+function rowTracePair(pair) {
+  if (!pair) return void 0;
+  const { caseId, ...evidence } = pair;
+  return { ...evidence, rowId: caseId };
+}
+var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod7, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
 var init_portable_runtime = __esm({
   "src/evals/portable-runtime.ts"() {
     "use strict";
     import_quickjs_singlefile_cjs_release_sync = __toESM(require("@jitl/quickjs-singlefile-cjs-release-sync"));
     import_quickjs_emscripten_core = require("quickjs-emscripten-core");
-    import_zod6 = require("zod");
+    import_zod7 = require("zod");
     init_portable();
     DEFAULT_MEMORY_BYTES = 32 * 1024 * 1024;
     DEFAULT_STACK_BYTES = 64 * 1024;
@@ -6973,11 +7019,11 @@ var init_portable_runtime = __esm({
     DEFAULT_JUDGE_CONCURRENCY = 4;
     DEFAULT_MAX_INPUT_BYTES = 4 * 1024 * 1024;
     DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
-    JudgeRequestSchema = import_zod6.z.object({
-      type: import_zod6.z.literal("judge_request"),
-      requestId: import_zod6.z.string(),
-      traceId: import_zod6.z.string(),
-      rubric: import_zod6.z.string()
+    JudgeRequestSchema = import_zod7.z.object({
+      type: import_zod7.z.literal("judge_request"),
+      requestId: import_zod7.z.string(),
+      traceId: import_zod7.z.string(),
+      rubric: import_zod7.z.string()
     });
   }
 });
@@ -7012,16 +7058,17 @@ async function evaluatePortableEvaluator(evaluator, input) {
     expectedSha256: evaluator.expectedSha256
   });
   const { runPortableEvaluator: runPortableEvaluator2 } = await Promise.resolve().then(() => (init_portable_runtime(), portable_runtime_exports));
-  const cases = (_a = input.cases) != null ? _a : [];
+  const cases = ((_a = input.rows) != null ? _a : []).map(({ rowId, ...pair }) => ({ ...pair, caseId: rowId }));
   if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
     (entry, index) => {
       var _a2;
       return entry.candidate.trace.event.id !== ((_a2 = input.traces[index]) == null ? void 0 : _a2.event.id);
     }
   ))) {
-    throw new Error("Raindrop portable evaluator cases must match traces exactly, in slice order");
+    throw new Error("Raindrop portable evaluator rows must match traces exactly, in slice order");
   }
-  const result = await runPortableEvaluator2(evaluator, { ...input, cases });
+  const { rows: _rows, ...runtimeInput } = input;
+  const result = await runPortableEvaluator2(evaluator, { ...runtimeInput, cases });
   validatePortableResult(artifact, input.traces, result);
   return result;
 }
@@ -7088,67 +7135,67 @@ function canonicalJson2(value) {
   if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
   return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson2(entry)}`).join(",")}}`;
 }
-var import_zod7, PortableEvalArtifactSchema, PortableEvalResultSchema;
+var import_zod8, PortableEvalArtifactSchema, PortableEvalResultSchema;
 var init_portable = __esm({
   "src/evals/portable.ts"() {
     "use strict";
-    import_zod7 = require("zod");
+    import_zod8 = require("zod");
     init_definition();
     init_trace();
-    PortableEvalArtifactSchema = import_zod7.z.object({
-      format: import_zod7.z.literal("raindrop.eval-program"),
-      formatVersion: import_zod7.z.literal(1),
-      runtimeAbi: import_zod7.z.literal("raindrop-eval-v2"),
-      evaluator: import_zod7.z.object({
-        id: import_zod7.z.string().uuid(),
-        slug: import_zod7.z.string().min(1),
-        programVersion: import_zod7.z.number().int().positive(),
-        executionMode: import_zod7.z.enum(["deterministic", "judge"]),
-        outputType: import_zod7.z.enum(["boolean", "score", "number"]),
-        scope: import_zod7.z.literal("batch")
+    PortableEvalArtifactSchema = import_zod8.z.object({
+      format: import_zod8.z.literal("raindrop.eval-program"),
+      formatVersion: import_zod8.z.literal(1),
+      runtimeAbi: import_zod8.z.literal("raindrop-eval-v2"),
+      evaluator: import_zod8.z.object({
+        id: import_zod8.z.string().uuid(),
+        slug: import_zod8.z.string().min(1),
+        programVersion: import_zod8.z.number().int().positive(),
+        executionMode: import_zod8.z.enum(["deterministic", "judge"]),
+        outputType: import_zod8.z.enum(["boolean", "score", "number"]),
+        scope: import_zod8.z.literal("batch")
       }).strict(),
-      traceProjectionVersion: import_zod7.z.literal(1),
-      traceProjectionSha256: import_zod7.z.string().regex(/^[a-f0-9]{64}$/),
-      source: import_zod7.z.string().min(1),
-      sha256: import_zod7.z.string().regex(/^[a-f0-9]{64}$/)
+      traceProjectionVersion: import_zod8.z.literal(1),
+      traceProjectionSha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
+      source: import_zod8.z.string().min(1),
+      sha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/)
     }).strict();
-    PortableEvalResultSchema = import_zod7.z.object({
-      outcomes: import_zod7.z.array(
-        import_zod7.z.union([
-          import_zod7.z.object({
-            state: import_zod7.z.literal("graded"),
-            traceId: import_zod7.z.string(),
-            pass: import_zod7.z.boolean(),
-            note: import_zod7.z.string().optional()
+    PortableEvalResultSchema = import_zod8.z.object({
+      outcomes: import_zod8.z.array(
+        import_zod8.z.union([
+          import_zod8.z.object({
+            state: import_zod8.z.literal("graded"),
+            traceId: import_zod8.z.string(),
+            pass: import_zod8.z.boolean(),
+            note: import_zod8.z.string().optional()
           }),
-          import_zod7.z.object({
-            state: import_zod7.z.literal("graded"),
-            traceId: import_zod7.z.string(),
+          import_zod8.z.object({
+            state: import_zod8.z.literal("graded"),
+            traceId: import_zod8.z.string(),
             score: EvalScoreSchema,
-            note: import_zod7.z.string().optional()
+            note: import_zod8.z.string().optional()
           }),
-          import_zod7.z.object({
-            state: import_zod7.z.literal("graded"),
-            traceId: import_zod7.z.string(),
-            value: import_zod7.z.number().finite(),
-            note: import_zod7.z.string().optional()
+          import_zod8.z.object({
+            state: import_zod8.z.literal("graded"),
+            traceId: import_zod8.z.string(),
+            value: import_zod8.z.number().finite(),
+            note: import_zod8.z.string().optional()
           }),
-          import_zod7.z.object({ state: import_zod7.z.literal("errored"), traceId: import_zod7.z.string(), message: import_zod7.z.string() }),
-          import_zod7.z.object({
-            state: import_zod7.z.literal("ungraded"),
-            traceId: import_zod7.z.string(),
-            reason: import_zod7.z.enum(["not sampled", "trace unavailable", "content unreadable"])
+          import_zod8.z.object({ state: import_zod8.z.literal("errored"), traceId: import_zod8.z.string(), message: import_zod8.z.string() }),
+          import_zod8.z.object({
+            state: import_zod8.z.literal("ungraded"),
+            traceId: import_zod8.z.string(),
+            reason: import_zod8.z.enum(["not sampled", "trace unavailable", "content unreadable"])
           })
         ])
       ),
-      failures: import_zod7.z.array(import_zod7.z.string()),
-      stats: import_zod7.z.object({
-        traceCount: import_zod7.z.number().int().nonnegative(),
-        hydratedCount: import_zod7.z.number().int().nonnegative(),
-        judgeCalls: import_zod7.z.number().int().nonnegative(),
-        durationMs: import_zod7.z.number().nonnegative(),
-        skippedRows: import_zod7.z.number().int().nonnegative(),
-        skippedReason: import_zod7.z.literal("no trace").nullable()
+      failures: import_zod8.z.array(import_zod8.z.string()),
+      stats: import_zod8.z.object({
+        traceCount: import_zod8.z.number().int().nonnegative(),
+        hydratedCount: import_zod8.z.number().int().nonnegative(),
+        judgeCalls: import_zod8.z.number().int().nonnegative(),
+        durationMs: import_zod8.z.number().nonnegative(),
+        skippedRows: import_zod8.z.number().int().nonnegative(),
+        skippedReason: import_zod8.z.literal("no trace").nullable()
       })
     });
   }
@@ -7163,8 +7210,9 @@ __export(index_exports, {
   DEFAULT_QUERY_URL: () => DEFAULT_QUERY_URL,
   DEFAULT_TRACE_WAIT_MS: () => DEFAULT_TRACE_WAIT_MS,
   EVAL_CORRELATION_ID_ATTRIBUTE: () => EVAL_CORRELATION_ID_ATTRIBUTE,
-  EvalDatasetCaseInputSchema: () => EvalDatasetCaseInputSchema,
   EvalDatasetPublishConflictError: () => EvalDatasetPublishConflictError,
+  EvalDatasetRowInputSchema: () => EvalDatasetRowInputSchema,
+  EvalPublishError: () => EvalPublishError,
   LocalBooleanVerdictSchema: () => LocalBooleanVerdictSchema,
   LocalNumberVerdictSchema: () => LocalNumberVerdictSchema,
   LocalScoreVerdictSchema: () => LocalScoreVerdictSchema,
@@ -7181,6 +7229,7 @@ __export(index_exports, {
   ReplayVerdictSchema: () => ReplayVerdictSchema,
   TraceSchema: () => TraceSchema,
   claimReplay: () => claimReplay,
+  compareEvalRuns: () => compareEvalRuns,
   createEvalSuiteRun: () => createEvalSuiteRun,
   createReplay: () => createReplay,
   currentEvalScope: () => currentEvalScope,
@@ -7189,17 +7238,22 @@ __export(index_exports, {
   defineEvalSuite: () => defineEvalSuite,
   defineEvaluatorProgram: () => defineEvaluatorProgram,
   defineLocalEvaluator: () => defineLocalEvaluator,
+  evaluateEvalRun: () => evaluateEvalRun,
   evaluatePortableEvaluator: () => evaluatePortableEvaluator,
   evaluateReplay: () => evaluateReplay,
   importPortableEvaluator: () => importPortableEvaluator,
   isEvalDataset: () => isEvalDataset,
   isEvalSuiteDefinition: () => isEvalSuiteDefinition,
+  isPublishedEvalSuite: () => isPublishedEvalSuite,
   loadEvalSnapshot: () => loadEvalSnapshot,
   projectWorkshopTrace: () => projectWorkshopTrace,
   publishEvalDataset: () => publishEvalDataset,
+  publishEvalSuite: () => publishEvalSuite,
   pullEval: () => pullEval,
-  readEvalDataset: () => readEvalDataset,
+  readEvalDataset: () => readEvalDataset2,
   readEvalManifest: () => readEvalManifest,
+  readEvalRun: () => readEvalRun,
+  readEvaluator: () => readEvaluator,
   readReplay: () => readReplay,
   replay: () => replay,
   resolveDisableBatching: () => resolveDisableBatching,
@@ -7207,6 +7261,7 @@ __export(index_exports, {
   runReplay: () => runReplay,
   traceOutput: () => traceOutput,
   traceToolCalls: () => traceToolCalls,
+  traceTools: () => traceTools,
   verifyPortableEvalArtifact: () => verifyPortableEvalArtifact,
   withEvalScope: () => withEvalScope
 });
@@ -12558,10 +12613,13 @@ var SignalEventSchema = external_exports.object({
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.6.0",
+  version: "0.7.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",
+  bin: {
+    "raindrop-evals": "dist/evals/cli.js"
+  },
   license: "MIT",
   homepage: "https://www.raindrop.ai/docs/sdk/typescript/",
   bugs: {
@@ -12658,6 +12716,7 @@ var package_default = {
   tsup: {
     entry: [
       "src/index.ts",
+      "src/evals/cli.ts",
       "src/tracing/index.ts",
       "src/otel/index.ts"
     ],
@@ -16449,7 +16508,7 @@ var EvalDatasetCaseInputsSchema = import_zod4.z.array(EvalDatasetCaseInputSchema
     if (seen.has(entry.id)) {
       context9.addIssue({
         code: import_zod4.z.ZodIssueCode.custom,
-        message: `Duplicate case id ${entry.id}`,
+        message: `Duplicate row id ${entry.id}`,
         path: [index, "id"]
       });
     }
@@ -16829,6 +16888,34 @@ var WireReplayDetailSchema = import_zod5.z.object({
   replay: WireReplaySchema,
   rows: import_zod5.z.array(WireRowSchema)
 });
+var EvalRunDetailSchema = WireReplayDetailSchema.extend({
+  replay: WireReplaySchema.extend({
+    dataset_id: import_zod5.z.string().uuid(),
+    dataset_version_id: import_zod5.z.string().uuid().nullable()
+  }),
+  eval_runs: import_zod5.z.array(
+    import_zod5.z.object({
+      id: import_zod5.z.string().uuid(),
+      eval_id: import_zod5.z.string().uuid(),
+      program_version: import_zod5.z.number().int().positive(),
+      output_type: import_zod5.z.enum(["boolean", "score", "number"]),
+      executed_by: import_zod5.z.enum(["hosted", "local"]),
+      status: import_zod5.z.enum(["running", "completed", "failed"])
+    })
+  )
+});
+async function readReplayDetails(client, runId, options = {}) {
+  var _a;
+  const detail = await queryApi({
+    client,
+    queryUrl: (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+    path: `/v1/replays/${encodeURIComponent(runId)}`,
+    method: "GET",
+    schema: EvalRunDetailSchema
+  });
+  if (detail.replay.id !== runId) throw new Error("Raindrop returned a different run");
+  return detail;
+}
 var WireReportSchema = import_zod5.z.object({ status: import_zod5.z.string() });
 var WireAttemptStartedSchema = import_zod5.z.object({
   status: import_zod5.z.literal("pending"),
@@ -17007,7 +17094,7 @@ function selectEvalDatasetRows(manifest, requestedRowIds) {
   if (ordered.length !== selected.size) {
     const available = new Set(manifest.cases.map((entry) => entry.id));
     const unknown = rowIds.find((rowId) => !available.has(rowId));
-    throw new Error(`Raindrop replay rowIds contains unknown case ${unknown != null ? unknown : "unknown"}`);
+    throw new Error(`Raindrop replay rowIds contains unknown row ${unknown != null ? unknown : "unknown"}`);
   }
   return ordered;
 }
@@ -17075,7 +17162,7 @@ async function readEvalManifest(client, dataset, options) {
   });
   const caseIds = manifest.rows.map((row) => row.id);
   if (new Set(caseIds).size !== caseIds.length) {
-    throw new Error(`Raindrop eval manifest for ${normalized} contains duplicate case ids`);
+    throw new Error(`Raindrop eval manifest for ${normalized} contains duplicate row ids`);
   }
   return {
     datasetId: manifest.dataset_id,
@@ -17115,7 +17202,7 @@ function validateEvalDatasetManifest(value) {
   }
   const caseIds = manifest.cases.map((entry) => entry.id);
   if (new Set(caseIds).size !== caseIds.length) {
-    throw new Error(`Raindrop eval dataset ${manifest.dataset.slug} contains duplicate case ids`);
+    throw new Error(`Raindrop eval dataset ${manifest.dataset.slug} contains duplicate row ids`);
   }
   return manifest;
 }
@@ -17180,14 +17267,17 @@ async function runReplayRows(client, handle, options) {
                   var _a3;
                   const parent = (_a3 = import_api7.trace.getSpan(import_api7.context.active())) == null ? void 0 : _a3.spanContext();
                   if (!parent) return options.run(visible);
-                  return withReplayTraceDestination({
-                    ...replayDestination,
-                    parentSpanContext: {
-                      traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
-                      spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
-                      eventId: parent.traceId
-                    }
-                  }, () => options.run(visible));
+                  return withReplayTraceDestination(
+                    {
+                      ...replayDestination,
+                      parentSpanContext: {
+                        traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
+                        spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
+                        eventId: parent.traceId
+                      }
+                    },
+                    () => options.run(visible)
+                  );
                 }
               )
             )
@@ -17282,7 +17372,11 @@ async function readEvalCaseTracePairs(input) {
       while (reading.state === "pending" && Date.now() < deadlineAt) {
         await sleep(Math.min(TRACE_POLL_INTERVAL_MS, Math.max(0, deadlineAt - Date.now())));
         if (Date.now() >= deadlineAt) break;
-        reading = await readEvalCaseTracePair({ ...input, deadlineAt }, evidence, (_b = input.attempt) != null ? _b : 0);
+        reading = await readEvalCaseTracePair(
+          { ...input, deadlineAt },
+          evidence,
+          (_b = input.attempt) != null ? _b : 0
+        );
       }
       if (reading.state !== "ready") {
         const detail = reading.state === "pending" ? `did not become complete within ${input.waitMs}ms` : reading.state === "expired" ? `expired at ${reading.expiredAt}` : reading.reason;
@@ -17333,7 +17427,13 @@ async function createLocalEvalReplaySession(client, options) {
     throw new Error("Raindrop local eval replay requires at least one evaluator");
   }
   const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
-  await validateReferenceRequirements(client, options.dataset, options.rowIds, options.evaluators, queryUrl);
+  await validateReferenceRequirements(
+    client,
+    options.dataset,
+    options.rowIds,
+    options.evaluators,
+    queryUrl
+  );
   const handle = await createReplay(client, {
     dataset: options.dataset,
     rowIds: options.rowIds,
@@ -17426,14 +17526,17 @@ var LocalEvalReplaySessionImpl = class {
                 var _a2;
                 const parent = (_a2 = import_api7.trace.getSpan(import_api7.context.active())) == null ? void 0 : _a2.spanContext();
                 if (!parent) return input.run(row);
-                return withReplayTraceDestination({
-                  ...replayDestination,
-                  parentSpanContext: {
-                    traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
-                    spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
-                    eventId: parent.traceId
-                  }
-                }, () => input.run(row));
+                return withReplayTraceDestination(
+                  {
+                    ...replayDestination,
+                    parentSpanContext: {
+                      traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
+                      spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
+                      eventId: parent.traceId
+                    }
+                  },
+                  () => input.run(row)
+                );
               }
             )
           )
@@ -17659,7 +17762,9 @@ function reportReplay(client, queryUrl, replayId, body) {
 }
 async function validateReferenceRequirements(client, dataset, rowIds, evaluators, queryUrl) {
   const selected = new Set(selectEvalDatasetRows(dataset, rowIds));
-  const missing = dataset.cases.filter((entry) => selected.has(entry.id) && entry.referenceTraceSha256 === void 0);
+  const missing = dataset.cases.filter(
+    (entry) => selected.has(entry.id) && entry.referenceTraceSha256 === void 0
+  );
   if (missing.length === 0) return;
   for (const evaluator of evaluators) {
     const requiresReference = typeof evaluator === "string" ? (await queryApi({
@@ -17671,7 +17776,9 @@ async function validateReferenceRequirements(client, dataset, rowIds, evaluators
     })).data.requiresReference : evaluator.requiresReference === true;
     if (requiresReference) {
       const name = typeof evaluator === "string" ? evaluator : evaluator.slug;
-      throw new Error(`Raindrop evaluator ${name} requires a reference trace. Missing references for cases: ${missing.map((entry) => entry.id).join(", ")}`);
+      throw new Error(
+        `Raindrop evaluator ${name} requires a reference trace. Missing references for rows: ${missing.map((entry) => entry.id).join(", ")}`
+      );
     }
   }
 }
@@ -17680,7 +17787,8 @@ async function replay(client, options) {
   const evaluators = normalizeEvaluators(options.evaluators);
   const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
   const dataset = typeof options.dataset === "string" ? await readEvalDataset(client, options.dataset, { queryUrl }) : options.dataset;
-  await validateReferenceRequirements(client, dataset, options.rowIds, evaluators, queryUrl);
+  const rowIds = options.runId === void 0 ? options.rowIds : await queuedSuiteRows(client, options.runId, dataset, options.rowIds, queryUrl);
+  await validateReferenceRequirements(client, dataset, rowIds, evaluators, queryUrl);
   const local = evaluators.flatMap(
     (evaluator) => typeof evaluator === "string" ? [] : [{ evaluator, verdicts: [], durationMs: 0 }]
   );
@@ -17692,7 +17800,7 @@ async function replay(client, options) {
       }
     )
   );
-  const handle = await createReplay(client, { ...options, dataset });
+  const handle = options.runId === void 0 ? await createReplay(client, { ...options, dataset, rowIds }) : await claimSuiteRun(client, options.runId, dataset, rowIds != null ? rowIds : [], queryUrl);
   const execution = await runReplayRows(client, handle, options);
   const { reading } = execution;
   const doneRowIds = new Set(
@@ -17765,6 +17873,33 @@ async function replay(client, options) {
     evaluators: evaluatorResults.map(({ outcomes: _outcomes, ...evaluator }) => evaluator)
   };
 }
+async function queuedSuiteRows(client, runId, dataset, requestedRows, queryUrl) {
+  const detail = await readReplayDetails(client, runId, { queryUrl });
+  if (detail.replay.dataset_id !== dataset.dataset.id || detail.replay.dataset_version_id !== dataset.version.id) {
+    throw new Error("Queued run dataset version does not match the suite");
+  }
+  if (detail.replay.status !== "queued") throw new Error("Only queued runs can be claimed");
+  if (detail.replay.traces_expired) throw new Error("Queued run has expired");
+  const rowIds = selectEvalDatasetRows(
+    dataset,
+    detail.rows.map((row) => row.row_id)
+  );
+  const requested = requestedRows && selectEvalDatasetRows(dataset, requestedRows);
+  if (requested && (requested.length !== rowIds.length || requested.some((id, index) => id !== rowIds[index]))) {
+    throw new Error("Queued run rows do not match the requested selection");
+  }
+  return rowIds;
+}
+async function claimSuiteRun(client, runId, dataset, rowIds, queryUrl) {
+  const handle = await claimReplay(client, runId, { queryUrl });
+  if (handle.replayId !== runId) throw new Error("Raindrop claimed a different run");
+  if (!handle.ingestUrl) throw new Error("Run has already been claimed by another runner");
+  assertSelectedRows(handle, rowIds);
+  return {
+    ...handle,
+    datasetVersion: { id: dataset.version.id, fingerprint: dataset.version.fingerprint }
+  };
+}
 async function runLocalEvaluators(evaluators, evalCase, row, result) {
   await Promise.all(
     evaluators.map(async (state2) => {
@@ -17882,11 +18017,11 @@ async function runReplayEvaluators(client, replayId, options) {
   const deadlineAt = Date.now() + waitMs;
   return Promise.all(
     startedRuns.map(async (started) => {
-      let run = await readEvalRun(client, queryUrl, started.runId, deadlineAt);
+      let run = await readReplayEvaluatorRun(client, queryUrl, started.runId, deadlineAt);
       while (run.status === "running" && Date.now() < deadlineAt) {
         await sleep(Math.min(pollIntervalMs, Math.max(0, deadlineAt - Date.now())));
         if (Date.now() >= deadlineAt) break;
-        run = await readEvalRun(client, queryUrl, started.runId, deadlineAt);
+        run = await readReplayEvaluatorRun(client, queryUrl, started.runId, deadlineAt);
       }
       if (run.status === "running") {
         throw new Error(
@@ -17986,7 +18121,7 @@ function replayResultRows(handle, reading, evaluators) {
     return replayRow;
   });
 }
-async function readEvalRun(client, queryUrl, runId, deadlineAt) {
+async function readReplayEvaluatorRun(client, queryUrl, runId, deadlineAt) {
   const response = await queryApi({
     client,
     queryUrl,
@@ -18164,6 +18299,18 @@ function describeError(cause) {
 // src/evals/index.ts
 init_definition();
 
+// src/evals/rows.ts
+var EvalDatasetRowInputSchema = EvalDatasetCaseInputSchema;
+function datasetFromWire({ cases, ...dataset }) {
+  return { ...dataset, rows: cases };
+}
+function datasetToWire({ rows, ...dataset }) {
+  return { ...dataset, cases: rows };
+}
+async function readEvalDataset2(client, dataset, options) {
+  return datasetFromWire(await readEvalDataset(client, dataset, options));
+}
+
 // src/evals/dataset.ts
 var MAX_PUBLISH_RETRIES = 2;
 var RETRY_BASE_DELAY_MS = 100;
@@ -18177,21 +18324,14 @@ var EvalDatasetPublishConflictError = class extends Error {
   }
 };
 async function publishEvalDataset(client, options) {
-  var _a, _b, _c;
+  var _a, _b;
   if (!((_a = client.connection.apiKey) == null ? void 0 : _a.trim())) {
     throw new Error("Raindrop eval dataset publication requires a Query SDK API key (apiKey)");
   }
   const queryUrl = credentialEndpoint((_b = options.queryUrl) != null ? _b : DEFAULT_QUERY_URL).href;
   const slug = EvalDatasetSlugSchema.parse(options.slug.trim());
-  const fingerprint = await evalDatasetVersionFingerprint(options.cases);
-  const body = PublishEvalDatasetVersionInputSchema.parse({
-    name: options.name,
-    requestKey: randomUUID(),
-    expectedCurrentVersionId: (_c = options.expectedCurrentVersionId) != null ? _c : null,
-    fingerprint,
-    cases: options.cases
-  });
-  validateUploadSizes(body);
+  const body = await prepareEvalDatasetUpload(options);
+  const fingerprint = body.fingerprint;
   const path = `/v1/eval-datasets/${encodeURIComponent(slug)}`;
   const response = await publishRequest({
     client,
@@ -18204,7 +18344,20 @@ async function publishEvalDataset(client, options) {
   if (manifest.dataset.slug !== slug || manifest.dataset.id !== manifest.version.datasetId || manifest.version.fingerprint !== fingerprint) {
     throw new Error(`Raindrop eval dataset ${slug} returned mismatched publication identity`);
   }
-  return manifest;
+  return datasetFromWire(manifest);
+}
+async function prepareEvalDatasetUpload(options) {
+  var _a;
+  const fingerprint = await evalDatasetVersionFingerprint(options.rows);
+  const body = PublishEvalDatasetVersionInputSchema.parse({
+    name: options.name,
+    requestKey: randomUUID(),
+    expectedCurrentVersionId: (_a = options.expectedCurrentVersionId) != null ? _a : null,
+    fingerprint,
+    cases: options.rows
+  });
+  validateUploadSizes(body);
+  return body;
 }
 function validateUploadSizes(body) {
   for (const [index, entry] of body.cases.entries()) {
@@ -18212,7 +18365,7 @@ function validateUploadSizes(body) {
     const bytes2 = encodedBytes(entry.referenceTrace);
     if (bytes2 > EVAL_DATASET_MAX_REFERENCE_BYTES) {
       throw new Error(
-        `Raindrop eval dataset case ${entry.id} at index ${index} exceeds ${EVAL_DATASET_MAX_REFERENCE_BYTES} reference bytes`
+        `Raindrop eval dataset row ${entry.id} at index ${index} exceeds ${EVAL_DATASET_MAX_REFERENCE_BYTES} reference bytes`
       );
     }
   }
@@ -18295,39 +18448,278 @@ function sleep2(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+// src/evals/publish.ts
+var import_zod6 = require("zod");
+init_definition();
+
+// src/evals/runs.ts
+async function readEvalRun(client, runId, options = {}) {
+  const detail = await readReplayDetails(client, runId, options);
+  if (detail.eval_runs.length >= 200) {
+    throw new Error("Run evaluation history exceeds the read limit");
+  }
+  const evaluators = await Promise.all(
+    detail.eval_runs.map(async (evaluation) => {
+      var _a;
+      const result = await readReplayEvaluatorRun(
+        client,
+        (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+        evaluation.id
+      );
+      if (result.id !== evaluation.id) throw new Error("Raindrop returned a different evaluation");
+      return {
+        ...result,
+        evaluator: result.evalSlug,
+        evaluatorId: evaluation.eval_id,
+        programVersion: evaluation.program_version,
+        executedBy: evaluation.executed_by
+      };
+    })
+  );
+  return {
+    runId: detail.replay.id,
+    datasetId: detail.replay.dataset_id,
+    datasetVersionId: detail.replay.dataset_version_id,
+    status: detail.replay.status,
+    counts: detail.replay.counts,
+    tracesExpireAt: detail.replay.traces_expire_at,
+    tracesExpired: detail.replay.traces_expired,
+    rows: detail.rows.map((row) => ({
+      rowId: row.row_id,
+      attempt: row.attempt,
+      status: row.status,
+      traceId: row.trace_id,
+      error: row.error
+    })),
+    evaluators
+  };
+}
+async function evaluateEvalRun(client, runId, options) {
+  return { runId, evaluators: await evaluateReplay(client, runId, options) };
+}
+
+// src/evals/publish.ts
+var EvaluatorIdentitySchema = import_zod6.z.object({
+  id: import_zod6.z.string().uuid(),
+  slug: import_zod6.z.string(),
+  programVersion: import_zod6.z.number().int().positive(),
+  outputType: import_zod6.z.enum(["boolean", "score", "number"])
+});
+var EvaluatorSourceSchema = EvaluatorIdentitySchema.extend({
+  name: import_zod6.z.string(),
+  description: import_zod6.z.string().nullable(),
+  intent: import_zod6.z.string(),
+  kind: import_zod6.z.enum(["judge", "deterministic"]),
+  programSource: import_zod6.z.string(),
+  rules: import_zod6.z.array(import_zod6.z.string())
+});
+var EvalPublishError = class extends Error {
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+    this.name = "EvalPublishError";
+  }
+};
+var ComparisonRunSchema = import_zod6.z.object({
+  id: import_zod6.z.string().uuid(),
+  programVersion: import_zod6.z.number().int().positive()
+});
+var ComparisonLinkSchema = import_zod6.z.object({
+  url: import_zod6.z.string().url(),
+  first: ComparisonRunSchema,
+  second: ComparisonRunSchema
+});
+async function compareEvalRuns(client, options) {
+  const [first, second] = await Promise.all([
+    readEvalRun(client, options.beforeRunId, options),
+    readEvalRun(client, options.afterRunId, options)
+  ]);
+  const candidates = first.evaluators.filter(
+    (entry) => (!options.evaluator || entry.evaluator === options.evaluator) && second.evaluators.some((other) => other.evaluatorId === entry.evaluatorId)
+  );
+  const selected = candidates[0];
+  if (!selected || candidates.length !== 1) {
+    throw new Error(
+      "Comparison requires one shared evaluator; specify evaluator when comparing a suite"
+    );
+  }
+  const matches = second.evaluators.filter((entry) => entry.evaluatorId === selected.evaluatorId);
+  const counterpart = matches[0];
+  if (!counterpart || matches.length !== 1) {
+    throw new Error("Comparison is ambiguous because this evaluator graded the run more than once");
+  }
+  if (selected.status === "running" || counterpart.status === "running") {
+    throw new Error("Wait for both evaluations to finish before comparing");
+  }
+  const before = selected.id;
+  const after = counterpart.id;
+  const { data } = import_zod6.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
+  if (data.first.id !== before || data.second.id !== after) {
+    throw new Error("Raindrop returned a different comparison pair");
+  }
+  return {
+    ...data,
+    beforeRunId: first.runId,
+    afterRunId: second.runId,
+    evaluator: selected.evaluator,
+    sameEvaluatorVersion: data.first.programVersion === data.second.programVersion
+  };
+}
+async function readEvaluator(client, slug, options = {}) {
+  const { data } = import_zod6.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
+  if (data.slug !== slug) throw new Error("Raindrop returned a different evaluator slug");
+  return defineEvaluatorProgram({
+    slug: data.slug,
+    name: data.name,
+    output: data.outputType,
+    scope: "batch",
+    executionMode: data.kind,
+    source: data.programSource,
+    description: data.description,
+    intent: data.intent,
+    rules: data.rules,
+    expected: { evalId: data.id, programVersion: data.programVersion }
+  });
+}
+async function publishEvalSuite(client, definition, options = {}) {
+  const suite = defineEvalSuite(definition);
+  const localDataset = isEvalDataset(suite.dataset) ? suite.dataset : void 0;
+  const slug = EvalDatasetSlugSchema.parse(localDataset ? localDataset.id : suite.dataset);
+  const rows = localDataset == null ? void 0 : localDataset.rows.map((row) => {
+    if (row.output !== null) {
+      throw new Error(
+        "Publish reference evidence with publishEvalDataset first; a local row output is not a reference trace"
+      );
+    }
+    return EvalDatasetCaseInputSchema.parse({
+      id: row.id,
+      name: row.name,
+      input: row.input,
+      properties: row.properties
+    });
+  });
+  if (localDataset && rows) {
+    await prepareEvalDatasetUpload({
+      slug,
+      name: localDataset.name,
+      rows,
+      expectedCurrentVersionId: options.expectedCurrentDatasetVersionId
+    });
+  }
+  const entries = suite.evaluators.map((entry) => {
+    if (typeof entry.evaluator === "string") return entry;
+    if ("kind" in entry.evaluator && entry.evaluator.kind === "program") {
+      if (!entry.evaluator.expected) {
+        throw new Error(
+          "Evaluator source publication is not supported. Create the evaluator in Raindrop and reference its slug."
+        );
+      }
+      return { ...entry, evaluator: defineEvaluatorProgram(entry.evaluator) };
+    }
+    return entry;
+  });
+  const evaluators = [];
+  for (const entry of entries) {
+    if (typeof entry.evaluator === "string") {
+      evaluators.push({
+        ...entry,
+        evaluator: await readEvaluator(client, entry.evaluator, options)
+      });
+    } else {
+      evaluators.push(entry);
+    }
+  }
+  const dataset = localDataset && rows ? await publishEvalDataset(client, {
+    slug,
+    name: localDataset.name,
+    rows,
+    queryUrl: options.queryUrl,
+    expectedCurrentVersionId: options.expectedCurrentDatasetVersionId
+  }) : await readEvalDataset2(client, slug, {
+    queryUrl: options.queryUrl,
+    versionId: suite.datasetVersionId
+  });
+  return defineEvalSuite({
+    ...suite,
+    dataset: slug,
+    datasetVersionId: dataset.version.id,
+    evaluators
+  });
+}
+async function evalRequest(client, path, options) {
+  var _a, _b;
+  const apiKey = (_a = client.connection.apiKey) == null ? void 0 : _a.trim();
+  if (!apiKey) throw new Error("Raindrop evals require a Query SDK API key (apiKey)");
+  const base = credentialEndpoint((_b = options.queryUrl) != null ? _b : DEFAULT_QUERY_URL).href.replace(/\/+$/, "");
+  const headers = {
+    Authorization: `Bearer ${apiKey}`,
+    "Content-Type": "application/json"
+  };
+  if (client.connection.projectId !== void 0)
+    headers["X-Raindrop-Project-Id"] = client.connection.projectId;
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 3e4);
+  try {
+    const response = await runWithTracingSuppressed(
+      () => fetch(`${base}${path}`, {
+        method: "GET",
+        headers,
+        redirect: "error",
+        signal: controller.signal
+      })
+    );
+    const body = await response.text();
+    if (!response.ok)
+      throw new EvalPublishError(
+        response.status,
+        `Raindrop GET ${path}: ${response.status} ${body}`
+      );
+    return JSON.parse(body);
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+// src/evals/trace-tools.ts
+init_trace();
+var traceTools = Object.freeze({
+  getOutput: traceOutput,
+  getToolCalls: traceToolCalls
+});
+
 // src/evals/index.ts
 init_trace();
 init_portable();
 
 // src/evals/pull.ts
-var import_zod8 = require("zod");
+var import_zod9 = require("zod");
 init_definition();
 init_portable();
 var SNAPSHOT_VERSION = 1;
-var PulledEvaluatorSchema = import_zod8.z.object({
-  slug: import_zod8.z.string().min(1),
-  name: import_zod8.z.string().min(1),
-  expectedSha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
-  artifact: import_zod8.z.unknown()
+var PulledEvaluatorSchema = import_zod9.z.object({
+  slug: import_zod9.z.string().min(1),
+  name: import_zod9.z.string().min(1),
+  expectedSha256: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
+  artifact: import_zod9.z.unknown()
 });
-var EvalSnapshotSchema = import_zod8.z.object({
-  schemaVersion: import_zod8.z.literal(SNAPSHOT_VERSION),
-  pulledAt: import_zod8.z.string().datetime(),
-  dataset: import_zod8.z.object({
-    reference: import_zod8.z.string().min(1),
-    datasetId: import_zod8.z.string().uuid(),
-    fingerprint: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
-    rows: import_zod8.z.array(
-      import_zod8.z.object({
-        id: import_zod8.z.string(),
-        name: import_zod8.z.string(),
-        input: import_zod8.z.string().nullable(),
-        output: import_zod8.z.string().nullable(),
-        properties: import_zod8.z.record(import_zod8.z.string())
+var EvalSnapshotSchema = import_zod9.z.object({
+  schemaVersion: import_zod9.z.literal(SNAPSHOT_VERSION),
+  pulledAt: import_zod9.z.string().datetime(),
+  dataset: import_zod9.z.object({
+    reference: import_zod9.z.string().min(1),
+    datasetId: import_zod9.z.string().uuid(),
+    fingerprint: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
+    rows: import_zod9.z.array(
+      import_zod9.z.object({
+        id: import_zod9.z.string(),
+        name: import_zod9.z.string(),
+        input: import_zod9.z.string().nullable(),
+        output: import_zod9.z.string().nullable(),
+        properties: import_zod9.z.record(import_zod9.z.string())
       })
     )
   }),
-  evaluators: import_zod8.z.array(PulledEvaluatorSchema)
+  evaluators: import_zod9.z.array(PulledEvaluatorSchema)
 });
 async function pullEval(client, options) {
   var _a;
@@ -18392,11 +18784,11 @@ async function pullEvaluator(client, queryUrl, rawSlug) {
       `Raindrop evaluator ${rawSlug} pull failed with ${response.status}: ${await response.text()}`
     );
   }
-  const body = import_zod8.z.object({
-    data: import_zod8.z.object({
-      slug: import_zod8.z.string(),
-      name: import_zod8.z.string(),
-      artifact: import_zod8.z.unknown().nullable()
+  const body = import_zod9.z.object({
+    data: import_zod9.z.object({
+      slug: import_zod9.z.string(),
+      name: import_zod9.z.string(),
+      artifact: import_zod9.z.unknown().nullable()
     })
   }).parse(await response.json());
   if (!body.data.artifact)
@@ -18425,7 +18817,7 @@ async function materializeSnapshot(snapshotPath, raw, judges) {
     id: snapshot.dataset.datasetId,
     name: snapshot.dataset.reference,
     version: snapshot.dataset.fingerprint,
-    cases: snapshot.dataset.rows,
+    rows: snapshot.dataset.rows,
     remote: {
       reference: snapshot.dataset.reference,
       datasetId: snapshot.dataset.datasetId,
@@ -18533,46 +18925,46 @@ function outputMismatch(name, output) {
 init_portable();
 
 // src/evals/workshop.ts
-var import_zod9 = require("zod");
-var WorkshopRunSchema = import_zod9.z.object({
-  id: import_zod9.z.string(),
-  event_id: import_zod9.z.string().nullable(),
-  name: import_zod9.z.string().nullable(),
-  event_name: import_zod9.z.string().nullable(),
-  user_id: import_zod9.z.string().nullable(),
-  convo_id: import_zod9.z.string().nullable(),
-  started_at: import_zod9.z.number().nullable(),
-  last_updated_at: import_zod9.z.number().nullable(),
-  metadata: import_zod9.z.string().nullable(),
-  model: import_zod9.z.string().nullable(),
-  finished: import_zod9.z.number().nullable()
+var import_zod10 = require("zod");
+var WorkshopRunSchema = import_zod10.z.object({
+  id: import_zod10.z.string(),
+  event_id: import_zod10.z.string().nullable(),
+  name: import_zod10.z.string().nullable(),
+  event_name: import_zod10.z.string().nullable(),
+  user_id: import_zod10.z.string().nullable(),
+  convo_id: import_zod10.z.string().nullable(),
+  started_at: import_zod10.z.number().nullable(),
+  last_updated_at: import_zod10.z.number().nullable(),
+  metadata: import_zod10.z.string().nullable(),
+  model: import_zod10.z.string().nullable(),
+  finished: import_zod10.z.number().nullable()
 });
-var WorkshopSpanSchema = import_zod9.z.object({
-  id: import_zod9.z.string(),
-  run_id: import_zod9.z.string(),
-  parent_span_id: import_zod9.z.string().nullable(),
-  name: import_zod9.z.string(),
-  span_type: import_zod9.z.string().nullable(),
-  status: import_zod9.z.string().nullable(),
-  input_payload: import_zod9.z.string().nullable(),
-  output_payload: import_zod9.z.string().nullable(),
-  start_time_ms: import_zod9.z.number().nullable(),
-  end_time_ms: import_zod9.z.number().nullable(),
-  duration_ms: import_zod9.z.number().nullable(),
-  model: import_zod9.z.string().nullable(),
-  provider: import_zod9.z.string().nullable(),
-  input_tokens: import_zod9.z.number().nullable(),
-  output_tokens: import_zod9.z.number().nullable(),
-  attributes: import_zod9.z.string().nullable()
+var WorkshopSpanSchema = import_zod10.z.object({
+  id: import_zod10.z.string(),
+  run_id: import_zod10.z.string(),
+  parent_span_id: import_zod10.z.string().nullable(),
+  name: import_zod10.z.string(),
+  span_type: import_zod10.z.string().nullable(),
+  status: import_zod10.z.string().nullable(),
+  input_payload: import_zod10.z.string().nullable(),
+  output_payload: import_zod10.z.string().nullable(),
+  start_time_ms: import_zod10.z.number().nullable(),
+  end_time_ms: import_zod10.z.number().nullable(),
+  duration_ms: import_zod10.z.number().nullable(),
+  model: import_zod10.z.string().nullable(),
+  provider: import_zod10.z.string().nullable(),
+  input_tokens: import_zod10.z.number().nullable(),
+  output_tokens: import_zod10.z.number().nullable(),
+  attributes: import_zod10.z.string().nullable()
 });
-var WorkshopTraceEnvelopeSchema = import_zod9.z.object({
-  status: import_zod9.z.literal("complete"),
-  correlationId: import_zod9.z.string(),
-  traceId: import_zod9.z.string(),
-  rootSpanId: import_zod9.z.string(),
-  spanCount: import_zod9.z.number().int().positive(),
+var WorkshopTraceEnvelopeSchema = import_zod10.z.object({
+  status: import_zod10.z.literal("complete"),
+  correlationId: import_zod10.z.string(),
+  traceId: import_zod10.z.string(),
+  rootSpanId: import_zod10.z.string(),
+  spanCount: import_zod10.z.number().int().positive(),
   run: WorkshopRunSchema,
-  spans: import_zod9.z.array(WorkshopSpanSchema).min(1)
+  spans: import_zod10.z.array(WorkshopSpanSchema).min(1)
 });
 async function readWorkshopTrace(input) {
   if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
@@ -18658,9 +19050,12 @@ function serializableOutput(value) {
 var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
 function createEvalSuiteRun(client, definition, options) {
   const validated = validateDefinition(definition);
+  if (validated.datasetVersionId && "selection" in options && options.selection.dataset.version.id !== validated.datasetVersionId) {
+    throw new Error("Selected dataset version does not match the published suite");
+  }
   if (options.destination.kind === "raindrop") {
     if (!("selection" in options)) {
-      throw new Error("Raindrop remote eval sessions require selected cases");
+      throw new Error("Raindrop remote eval sessions require selected rows");
     }
     if (typeof validated.dataset !== "string") {
       throw new Error("Raindrop remote eval sessions require a dataset slug");
@@ -18675,16 +19070,16 @@ function createEvalSuiteRun(client, definition, options) {
       return typeof evaluator === "string" ? [] : [evaluator];
     });
     if (localEvaluators.length !== validated.evaluators.length) {
-      throw new Error("Raindrop case-scoped eval sessions support local evaluators only");
+      throw new Error("Raindrop row-scoped eval sessions support local evaluators only");
     }
     const selection = validateRemoteSelection(validated.dataset, options.selection);
-    if (!selection) throw new Error("Raindrop remote eval sessions require selected cases");
-    return new RaindropCaseEvalRun(
+    if (!selection) throw new Error("Raindrop remote eval sessions require selected rows");
+    return new RaindropRowEvalRun(
       client,
       validated,
       options.destination,
       options.selection.dataset,
-      selection.caseIds,
+      selection.rowIds,
       localEvaluators
     );
   }
@@ -18709,34 +19104,56 @@ function createEvalSuiteRun(client, definition, options) {
     options.destination
   );
 }
-async function runEvalSuite(client, definition, options) {
-  var _a, _b, _c, _d;
-  const validated = validateDefinition(definition);
-  if (options === void 0) throw new Error("Raindrop runEvalSuite requires an explicit destination");
-  if (options.destination.kind === "workshop") {
-    const session = createEvalSuiteRun(client, validated, { destination: options.destination });
+async function runEvalSuite(client, definition, options = {}) {
+  var _a, _b, _c, _d, _e, _f, _g, _h;
+  let validated = validateDefinition(definition);
+  const destination = (_a = options.destination) != null ? _a : { kind: "raindrop" };
+  if (validated.datasetVersionId && ((_b = options.selection) == null ? void 0 : _b.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
+    throw new Error("Selected dataset version does not match the published suite");
+  }
+  if (destination.kind === "workshop") {
+    if (options.runId !== void 0) throw new Error("Queued runs are only supported in Raindrop");
+    const session = createEvalSuiteRun(client, validated, { destination });
     await session.run({ selection: options.selection, attempt: options.attempt });
     return session.finish();
   }
-  if (isCaseScopedLocalEval(validated)) {
+  const alreadyPublished = isPublishedEvalSuite(validated);
+  if (typeof validated.dataset === "string" && ((_c = options.selection) == null ? void 0 : _c.dataset)) {
+    validateRemoteSelection(validated.dataset, options.selection);
+    validated = defineEvalSuite({
+      ...validated,
+      datasetVersionId: options.selection.dataset.version.id
+    });
+  }
+  if (!alreadyPublished) {
+    validated = await publishEvalSuite(client, validated, {
+      queryUrl: destination.queryUrl,
+      expectedCurrentDatasetVersionId: options.expectedCurrentDatasetVersionId
+    });
+  }
+  if (((_d = options.selection) == null ? void 0 : _d.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
+    throw new Error("Selected dataset version does not match the published suite");
+  }
+  if (isRowScopedLocalEval(validated) && options.runId === void 0) {
     if (typeof validated.dataset !== "string") {
       throw new Error("Raindrop remote execution requires an eval dataset slug");
     }
-    const dataset = (_b = (_a = options.selection) == null ? void 0 : _a.dataset) != null ? _b : await readEvalDataset(client, validated.dataset, {
-      queryUrl: options.destination.queryUrl
+    const dataset = (_f = (_e = options.selection) == null ? void 0 : _e.dataset) != null ? _f : await readEvalDataset2(client, validated.dataset, {
+      queryUrl: destination.queryUrl,
+      versionId: validated.datasetVersionId
     });
     const selection = validateRemoteSelection(validated.dataset, {
       dataset,
-      caseIds: (_d = (_c = options.selection) == null ? void 0 : _c.caseIds) != null ? _d : dataset.cases.map((entry) => entry.id)
+      rowIds: (_h = (_g = options.selection) == null ? void 0 : _g.rowIds) != null ? _h : dataset.rows.map((entry) => entry.id)
     });
-    if (!selection) throw new Error("Raindrop remote eval requires selected cases");
+    if (!selection) throw new Error("Raindrop remote eval requires selected rows");
     const session = createEvalSuiteRun(client, validated, {
-      destination: options.destination,
-      selection: { dataset, caseIds: selection.caseIds }
+      destination,
+      selection: { dataset, rowIds: selection.rowIds }
     });
     const attempts = await Promise.allSettled(
-      selection.caseIds.map(
-        (caseId) => session.run({ selection: { dataset, caseIds: [caseId] }, attempt: 0 })
+      selection.rowIds.map(
+        (rowId) => session.run({ selection: { dataset, rowIds: [rowId] }, attempt: 0 })
       )
     );
     const failures = attempts.filter(
@@ -18749,7 +19166,7 @@ async function runEvalSuite(client, definition, options) {
       if (failures.length > 0) {
         throw new AggregateError(
           [...failures.map((failure) => failure.reason), cause],
-          "Raindrop eval case execution and run finalization failed"
+          "Raindrop eval row execution and run finalization failed"
         );
       }
       throw cause;
@@ -18759,33 +19176,33 @@ async function runEvalSuite(client, definition, options) {
     if (failures.length > 1) {
       throw new AggregateError(
         failures.map((failure) => failure.reason),
-        "Multiple Raindrop eval cases failed"
+        "Multiple Raindrop eval rows failed"
       );
     }
     return finished;
   }
-  return runRaindropEval(client, validated, options.destination, options);
+  return runRaindropEval(client, validated, destination, options);
 }
-function isCaseScopedLocalEval(definition) {
+function isRowScopedLocalEval(definition) {
   return definition.evaluators.every(
-    (entry) => typeof entry.evaluator !== "string" && entry.evaluator.scope === "case" && !("kind" in entry.evaluator)
+    (entry) => typeof entry.evaluator !== "string" && entry.evaluator.scope === "row" && !("kind" in entry.evaluator)
   );
 }
-var RaindropCaseEvalRun = class {
-  constructor(client, definition, destination, dataset, caseIds, evaluators) {
+var RaindropRowEvalRun = class {
+  constructor(client, definition, destination, dataset, rowIds, evaluators) {
     this.client = client;
     this.definition = definition;
     this.destination = destination;
     this.dataset = dataset;
-    this.caseIds = caseIds;
+    this.rowIds = rowIds;
     this.evaluators = evaluators;
     this.id = randomId();
     this.active = /* @__PURE__ */ new Set();
     this.completed = [];
     this.attempts = /* @__PURE__ */ new Set();
     this.state = "open";
-    this.runningCases = 0;
-    this.caseWaiters = [];
+    this.runningRows = 0;
+    this.rowWaiters = [];
   }
   async run(options = {}) {
     if (this.state !== "open") throw new Error(`Raindrop eval run ${this.id} is already finishing`);
@@ -18805,24 +19222,24 @@ var RaindropCaseEvalRun = class {
   }
   async runOnce(options) {
     var _a, _b;
-    const selected = (_a = options.selection) == null ? void 0 : _a.caseIds;
+    const selected = (_a = options.selection) == null ? void 0 : _a.rowIds;
     if (!selected || selected.length !== 1) {
-      throw new Error("Raindrop case-scoped eval session run requires exactly one case");
+      throw new Error("Raindrop row-scoped eval session run requires exactly one row");
     }
-    const caseId = selected[0];
-    if (caseId === void 0 || !this.caseIds.includes(caseId)) {
-      throw new Error(`Raindrop eval ${this.definition.name} did not select a planned case`);
+    const rowId = selected[0];
+    if (rowId === void 0 || !this.rowIds.includes(rowId)) {
+      throw new Error(`Raindrop eval ${this.definition.name} did not select a planned row`);
     }
     const attempt = (_b = options.attempt) != null ? _b : 0;
-    const attemptKey = `${caseId}:${attempt}`;
+    const attemptKey = `${rowId}:${attempt}`;
     if (this.attempts.has(attemptKey)) {
-      throw new Error(`Raindrop eval case ${caseId} attempt ${attempt} already ran`);
+      throw new Error(`Raindrop eval row ${rowId} attempt ${attempt} already ran`);
     }
     this.attempts.add(attemptKey);
-    return this.withCasePermit(async () => {
+    return this.withRowPermit(async () => {
       const replay2 = await this.getReplay();
       const attemptResult = await replay2.run({
-        rowId: caseId,
+        rowId,
         attempt,
         run: this.definition.run,
         traceWaitMs: this.definition.traceWaitMs
@@ -18846,7 +19263,7 @@ var RaindropCaseEvalRun = class {
       };
       this.completed.push(result);
       return {
-        replayId: replay2.replayId,
+        runId: replay2.replayId,
         name: this.definition.name,
         status: "running",
         counts: {
@@ -18858,7 +19275,7 @@ var RaindropCaseEvalRun = class {
         tracesExpireAt: null,
         tracesExpired: false,
         evaluators: replayEvaluatorsFromVerdicts(attemptResult.verdicts),
-        cases: [result]
+        rows: [result]
       };
     });
   }
@@ -18866,20 +19283,21 @@ var RaindropCaseEvalRun = class {
     await Promise.allSettled([...this.active]);
     const replay2 = await this.getReplay();
     const finished = await replay2.finish();
-    const { rows: _rows, ...reading } = finished.reading;
+    const { rows: _rows, replayId, ...reading } = finished.reading;
     this.state = "finished";
     return {
       ...reading,
+      runId: replayId,
       name: this.definition.name,
       evaluators: finished.evaluators,
-      cases: this.sortedResults()
+      rows: this.sortedResults()
     };
   }
   getReplay() {
     if (!this.replayPromise) {
       this.replayPromise = createLocalEvalReplaySession(this.client, {
-        dataset: this.dataset,
-        rowIds: this.caseIds,
+        dataset: datasetToWire(this.dataset),
+        rowIds: this.rowIds,
         evaluators: this.evaluators,
         queryUrl: this.destination.queryUrl,
         replayIngestUrl: this.destination.replayIngestUrl
@@ -18887,28 +19305,28 @@ var RaindropCaseEvalRun = class {
     }
     return this.replayPromise;
   }
-  async withCasePermit(run) {
+  async withRowPermit(run) {
     var _a;
     const limit = (_a = this.definition.concurrency) != null ? _a : 4;
-    if (this.runningCases >= limit) {
-      await new Promise((resolve) => this.caseWaiters.push(resolve));
+    if (this.runningRows >= limit) {
+      await new Promise((resolve) => this.rowWaiters.push(resolve));
     } else {
-      this.runningCases += 1;
+      this.runningRows += 1;
     }
     try {
       return await run();
     } finally {
-      const next = this.caseWaiters.shift();
+      const next = this.rowWaiters.shift();
       if (next) next();
-      else this.runningCases -= 1;
+      else this.runningRows -= 1;
     }
   }
   sortedResults() {
-    const order = new Map(this.caseIds.map((caseId, index) => [caseId, index]));
+    const order = new Map(this.rowIds.map((rowId, index) => [rowId, index]));
     return [...this.completed].sort((left, right) => {
       var _a, _b, _c, _d;
-      const caseOrder = ((_a = order.get(left.id)) != null ? _a : Number.MAX_SAFE_INTEGER) - ((_b = order.get(right.id)) != null ? _b : Number.MAX_SAFE_INTEGER);
-      return caseOrder || ((_c = left.attempt) != null ? _c : 0) - ((_d = right.attempt) != null ? _d : 0);
+      const rowOrder = ((_a = order.get(left.id)) != null ? _a : Number.MAX_SAFE_INTEGER) - ((_b = order.get(right.id)) != null ? _b : Number.MAX_SAFE_INTEGER);
+      return rowOrder || ((_c = left.attempt) != null ? _c : 0) - ((_d = right.attempt) != null ? _d : 0);
     });
   }
 };
@@ -18937,8 +19355,8 @@ var WorkshopEvalRun = class {
     this.completed = [];
     this.active = /* @__PURE__ */ new Set();
     this.state = "open";
-    this.runningCases = 0;
-    this.caseWaiters = [];
+    this.runningRows = 0;
+    this.rowWaiters = [];
   }
   async run(options = {}) {
     if (this.state !== "open") throw new Error(`Raindrop eval run ${this.id} is already finishing`);
@@ -18952,11 +19370,11 @@ var WorkshopEvalRun = class {
   }
   async runOnce(options) {
     var _a;
-    const cases = selectLocalCases(this.definition, options.selection);
+    const rows = selectLocalRows(this.definition, options.selection);
     const completed = await Promise.all(
-      cases.map((row) => this.withCasePermit(() => {
+      rows.map((row) => this.withRowPermit(() => {
         var _a2;
-        return this.executeCase(row, (_a2 = options.attempt) != null ? _a2 : 0);
+        return this.executeRow(row, (_a2 = options.attempt) != null ? _a2 : 0);
       }))
     );
     const graded = await this.applyPortableEvaluators(completed, (_a = options.attempt) != null ? _a : 0);
@@ -19017,7 +19435,7 @@ var WorkshopEvalRun = class {
               name: entry.evaluator.name,
               output: entry.evaluator.output,
               execution: "local",
-              scope: entry.evaluator.scope,
+              scope: entry.evaluator.scope === "row" ? "case" : entry.evaluator.scope,
               threshold: entry.threshold
             };
           })
@@ -19029,7 +19447,7 @@ var WorkshopEvalRun = class {
     this.state = "finished";
     return result;
   }
-  async executeCase(row, attempt) {
+  async executeRow(row, attempt) {
     var _a, _b, _c;
     const correlationId = randomId();
     const exportedSpanIds = /* @__PURE__ */ new Set();
@@ -19171,20 +19589,20 @@ var WorkshopEvalRun = class {
     };
     return { result, packet, trace: trace8 };
   }
-  async withCasePermit(run) {
+  async withRowPermit(run) {
     var _a;
     const limit = (_a = this.definition.concurrency) != null ? _a : 4;
-    if (this.runningCases >= limit) {
-      await new Promise((resolve) => this.caseWaiters.push(resolve));
+    if (this.runningRows >= limit) {
+      await new Promise((resolve) => this.rowWaiters.push(resolve));
     } else {
-      this.runningCases += 1;
+      this.runningRows += 1;
     }
     try {
       return await run();
     } finally {
-      const next = this.caseWaiters.shift();
+      const next = this.rowWaiters.shift();
       if (next) next();
-      else this.runningCases -= 1;
+      else this.runningRows -= 1;
     }
   }
   async applyPortableEvaluators(completed, attempt) {
@@ -19249,14 +19667,17 @@ var WorkshopEvalRun = class {
   }
 };
 async function runRaindropEval(client, definition, destination, options) {
-  var _a, _b, _c;
+  var _a, _b, _c, _d;
   if (typeof definition.dataset !== "string") {
     throw new Error(
       "Raindrop remote execution requires an eval dataset slug; legacy pulled snapshots do not carry an immutable dataset version id"
     );
   }
   const datasetReference = definition.dataset;
-  const pinnedDataset = (_a = options.selection) == null ? void 0 : _a.dataset;
+  const pinnedDataset = (_b = (_a = options.selection) == null ? void 0 : _a.dataset) != null ? _b : definition.datasetVersionId ? await readEvalDataset2(client, datasetReference, {
+    queryUrl: destination.queryUrl,
+    versionId: definition.datasetVersionId
+  }) : void 0;
   if (pinnedDataset && pinnedDataset.dataset.slug !== datasetReference) {
     throw new Error(
       `Raindrop eval ${definition.name} pinned dataset ${pinnedDataset.dataset.slug} does not match ${datasetReference}`
@@ -19268,7 +19689,8 @@ async function runRaindropEval(client, definition, destination, options) {
     definition.evaluators.map((evaluator) => [evaluatorName(evaluator), evaluator])
   );
   const replayOptions = {
-    dataset: pinnedDataset != null ? pinnedDataset : datasetReference,
+    runId: options.runId,
+    dataset: pinnedDataset ? datasetToWire(pinnedDataset) : datasetReference,
     evaluators: definition.evaluators.map((entry) => replayEvaluator(entry)),
     evaluatorPins: hostedEvaluatorPins(definition.evaluators),
     run: definition.run,
@@ -19283,7 +19705,7 @@ async function runRaindropEval(client, definition, destination, options) {
     client,
     selection ? {
       ...replayOptions,
-      rowIds: selection.caseIds
+      rowIds: selection.rowIds
     } : replayOptions
   );
   const outputByEvaluator = /* @__PURE__ */ new Map();
@@ -19296,7 +19718,7 @@ async function runRaindropEval(client, definition, destination, options) {
     if (evaluatorResult.status === "failed") {
       continue;
     }
-    const output = (_c = (_b = evaluatorResult.summary) == null ? void 0 : _b.outputType) != null ? _c : evaluatorOutput(evaluator);
+    const output = (_d = (_c = evaluatorResult.summary) == null ? void 0 : _c.outputType) != null ? _d : evaluatorOutput(evaluator);
     if (output === void 0)
       throw new Error(
         `Raindrop evaluator ${evaluatorResult.evaluator} completed without an output type`
@@ -19304,11 +19726,12 @@ async function runRaindropEval(client, definition, destination, options) {
     validateEvaluatorOutput(evaluator, output);
     outputByEvaluator.set(evaluatorResult.evaluator, output);
   }
-  const { rows, ...summary } = result;
+  const { rows, replayId, ...summary } = result;
   return {
     ...summary,
+    runId: replayId,
     name: definition.name,
-    cases: rows.map((row) => ({
+    rows: rows.map((row) => ({
       ...row,
       verdicts: row.verdicts.map((verdict) => {
         const evaluator = configured.get(verdict.evaluator);
@@ -19344,32 +19767,38 @@ function replayEvaluator(entry) {
   if (typeof entry.evaluator === "string") return entry.evaluator;
   if (isPortableEvaluator(entry.evaluator)) return entry.evaluator.slug;
   if ("kind" in entry.evaluator && entry.evaluator.kind === "program") {
+    if (!entry.evaluator.expected) {
+      throw new Error(
+        `Create evaluator ${entry.evaluator.slug} in Raindrop and reference its slug; uploading evaluator source is not supported`
+      );
+    }
     return entry.evaluator.slug;
   }
-  switch (entry.evaluator.output) {
+  const evaluator = entry.evaluator;
+  switch (evaluator.output) {
     case "boolean":
       return {
-        slug: entry.evaluator.slug,
-        name: entry.evaluator.name,
-        output: entry.evaluator.output,
-        judge: entry.evaluator.judge,
-        requiresReference: entry.evaluator.requiresReference
+        slug: evaluator.slug,
+        name: evaluator.name,
+        output: evaluator.output,
+        judge: ({ trace: trace8, row, result, case: pair }) => evaluator.judge({ trace: trace8, row, result, reference: pair == null ? void 0 : pair.reference }),
+        requiresReference: evaluator.requiresReference
       };
     case "score":
       return {
-        slug: entry.evaluator.slug,
-        name: entry.evaluator.name,
-        output: entry.evaluator.output,
-        judge: entry.evaluator.judge,
-        requiresReference: entry.evaluator.requiresReference
+        slug: evaluator.slug,
+        name: evaluator.name,
+        output: evaluator.output,
+        judge: ({ trace: trace8, row, result, case: pair }) => evaluator.judge({ trace: trace8, row, result, reference: pair == null ? void 0 : pair.reference }),
+        requiresReference: evaluator.requiresReference
       };
     case "number":
       return {
-        slug: entry.evaluator.slug,
-        name: entry.evaluator.name,
-        output: entry.evaluator.output,
-        judge: entry.evaluator.judge,
-        requiresReference: entry.evaluator.requiresReference
+        slug: evaluator.slug,
+        name: evaluator.name,
+        output: evaluator.output,
+        judge: ({ trace: trace8, row, result, case: pair }) => evaluator.judge({ trace: trace8, row, result, reference: pair == null ? void 0 : pair.reference }),
+        requiresReference: evaluator.requiresReference
       };
   }
 }
@@ -19378,34 +19807,34 @@ function validateDefinition(definition) {
     throw new Error("Raindrop runEvalSuite requires a definition created by defineEvalSuite");
   return definition;
 }
-function selectLocalCases(definition, selection) {
-  if (!selection) return definition.dataset.cases;
-  if (selection.caseIds.length === 0)
+function selectLocalRows(definition, selection) {
+  if (!selection) return definition.dataset.rows;
+  if (selection.rowIds.length === 0)
     throw new Error(`Raindrop eval ${definition.name} selection cannot be empty`);
-  if (new Set(selection.caseIds).size !== selection.caseIds.length)
-    throw new Error(`Raindrop eval ${definition.name} selection case ids must be unique`);
-  const byId = new Map(definition.dataset.cases.map((row) => [row.id, row]));
-  return selection.caseIds.map((id) => {
+  if (new Set(selection.rowIds).size !== selection.rowIds.length)
+    throw new Error(`Raindrop eval ${definition.name} selection row ids must be unique`);
+  const byId = new Map(definition.dataset.rows.map((row) => [row.id, row]));
+  return selection.rowIds.map((id) => {
     const row = byId.get(id);
     if (!row)
-      throw new Error(`Raindrop eval ${definition.name} selection contains unknown case ${id}`);
+      throw new Error(`Raindrop eval ${definition.name} selection contains unknown row ${id}`);
     return row;
   });
 }
 function validateRemoteSelection(dataset, selection) {
-  var _a, _b, _c;
+  var _a;
   if (!selection) return void 0;
-  if (selection.caseIds.length === 0)
+  if (selection.rowIds.length === 0)
     throw new Error(`Raindrop eval ${dataset} selection cannot be empty`);
-  if (new Set(selection.caseIds).size !== selection.caseIds.length)
-    throw new Error(`Raindrop eval ${dataset} selection case ids must be unique`);
-  const cases = (_c = (_a = selection.dataset) == null ? void 0 : _a.cases) != null ? _c : (_b = selection.manifest) == null ? void 0 : _b.cases;
-  if (!cases) throw new Error(`Raindrop eval ${dataset} selection requires its dataset manifest`);
-  const available = new Set(cases.map((row) => row.id));
-  const unknown = selection.caseIds.find((caseId) => !available.has(caseId));
+  if (new Set(selection.rowIds).size !== selection.rowIds.length)
+    throw new Error(`Raindrop eval ${dataset} selection row ids must be unique`);
+  const rows = (_a = selection.dataset) == null ? void 0 : _a.rows;
+  if (!rows) throw new Error(`Raindrop eval ${dataset} selection requires its dataset manifest`);
+  const available = new Set(rows.map((row) => row.id));
+  const unknown = selection.rowIds.find((rowId) => !available.has(rowId));
   if (unknown)
-    throw new Error(`Raindrop eval ${dataset} selection contains unknown case ${unknown}`);
-  return { caseIds: selection.caseIds };
+    throw new Error(`Raindrop eval ${dataset} selection contains unknown row ${unknown}`);
+  return { rowIds: selection.rowIds };
 }
 function canonicalWorkshopTrace(envelope) {
   var _a, _b, _c, _d, _e, _f;
@@ -19531,18 +19960,18 @@ function portableVerdict(evaluator, runId, workshopUrl, outcome) {
     ...outcome.note ? { note: outcome.note } : {}
   };
 }
-function workshopResult(id, name, cases) {
-  const done = cases.filter((entry) => entry.status === "done").length;
-  const missing = cases.filter((entry) => entry.status === "missing").length;
+function workshopResult(id, name, rows) {
+  const done = rows.filter((entry) => entry.status === "done").length;
+  const missing = rows.filter((entry) => entry.status === "missing").length;
   return {
-    replayId: id,
+    runId: id,
     name,
     status: missing > 0 ? "failed" : "complete",
-    counts: { total: cases.length, pending: 0, done, missing },
+    counts: { total: rows.length, pending: 0, done, missing },
     tracesExpireAt: null,
     tracesExpired: false,
     evaluators: [],
-    cases
+    rows
   };
 }
 function workshopEvalUrl(baseUrl, id) {
@@ -20859,8 +21288,9 @@ var index_default = Raindrop;
   DEFAULT_QUERY_URL,
   DEFAULT_TRACE_WAIT_MS,
   EVAL_CORRELATION_ID_ATTRIBUTE,
-  EvalDatasetCaseInputSchema,
   EvalDatasetPublishConflictError,
+  EvalDatasetRowInputSchema,
+  EvalPublishError,
   LocalBooleanVerdictSchema,
   LocalNumberVerdictSchema,
   LocalScoreVerdictSchema,
@@ -20877,6 +21307,7 @@ var index_default = Raindrop;
   ReplayVerdictSchema,
   TraceSchema,
   claimReplay,
+  compareEvalRuns,
   createEvalSuiteRun,
   createReplay,
   currentEvalScope,
@@ -20884,17 +21315,22 @@ var index_default = Raindrop;
   defineEvalSuite,
   defineEvaluatorProgram,
   defineLocalEvaluator,
+  evaluateEvalRun,
   evaluatePortableEvaluator,
   evaluateReplay,
   importPortableEvaluator,
   isEvalDataset,
   isEvalSuiteDefinition,
+  isPublishedEvalSuite,
   loadEvalSnapshot,
   projectWorkshopTrace,
   publishEvalDataset,
+  publishEvalSuite,
   pullEval,
   readEvalDataset,
   readEvalManifest,
+  readEvalRun,
+  readEvaluator,
   readReplay,
   replay,
   resolveDisableBatching,
@@ -20902,6 +21338,7 @@ var index_default = Raindrop;
   runReplay,
   traceOutput,
   traceToolCalls,
+  traceTools,
   verifyPortableEvalArtifact,
   withEvalScope
 });