raindrop-ai 0.7.1 → 0.8.0

package/dist/{chunk-RMP6BZSY.mjs → chunk-EFATLUNL.mjs}

@@ -44,8 +44,7 @@ function defineDataset(dataset) {
   if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
   if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
   if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.rows.length === 0)
-    throw new Error("Raindrop eval datasets require at least one row");
+  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
   const ids = rows.map((entry) => entry.id);
   if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
   if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
@@ -139,7 +138,10 @@ function defineEvalSuite(definition) {
     if (typeof dataset !== "string")
       throw new Error("A dataset version pin requires a published dataset slug");
   }
-  if (typeof definition.run !== "function") {
+  if (definition.agent !== void 0 && definition.run !== void 0) {
+    throw new Error("An eval suite accepts either agent or run, not both");
+  }
+  if (definition.agent === void 0 && typeof definition.run !== "function") {
     throw new Error("Raindrop eval suite run must be a function");
   }
   if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
@@ -1136,7 +1138,7 @@ async function evaluatePortableEvaluator(evaluator, input) {
   const artifact = await verifyPortableEvalArtifact(evaluator.artifact, {
     expectedSha256: evaluator.expectedSha256
   });
-  const { runPortableEvaluator } = await import("./portable-runtime-MFVOVFPB.mjs");
+  const { runPortableEvaluator } = await import("./portable-runtime-6X2HHB4Y.mjs");
   const cases = ((_a = input.rows) != null ? _a : []).map(({ rowId, ...pair }) => ({ ...pair, caseId: rowId }));
   if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
     (entry, index) => {

package/dist/{chunk-SHH4SYVY.mjs → chunk-EWHJ2YBW.mjs}

@@ -1612,7 +1612,7 @@ installTracingSuppressionHook();
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.7.1",
+  version: "0.8.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",

package/dist/evals/cli.js

@@ -5794,8 +5794,7 @@ function defineDataset(dataset) {
   if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
   if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
   if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.rows.length === 0)
-    throw new Error("Raindrop eval datasets require at least one row");
+  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
   const ids = rows.map((entry) => entry.id);
   if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
   if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
@@ -5874,7 +5873,10 @@ function defineEvalSuite(definition) {
     if (typeof dataset !== "string")
       throw new Error("A dataset version pin requires a published dataset slug");
   }
-  if (typeof definition.run !== "function") {
+  if (definition.agent !== void 0 && definition.run !== void 0) {
+    throw new Error("An eval suite accepts either agent or run, not both");
+  }
+  if (definition.agent === void 0 && typeof definition.run !== "function") {
     throw new Error("Raindrop eval suite run must be a function");
   }
   if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
@@ -6999,13 +7001,13 @@ function rowTracePair(pair) {
   const { caseId, ...evidence } = pair;
   return { ...evidence, rowId: caseId };
 }
-var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod7, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
+var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod9, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
 var init_portable_runtime = __esm({
   "src/evals/portable-runtime.ts"() {
     "use strict";
     import_quickjs_singlefile_cjs_release_sync = __toESM(require("@jitl/quickjs-singlefile-cjs-release-sync"));
     import_quickjs_emscripten_core = require("quickjs-emscripten-core");
-    import_zod7 = require("zod");
+    import_zod9 = require("zod");
     init_portable();
     DEFAULT_MEMORY_BYTES = 32 * 1024 * 1024;
     DEFAULT_STACK_BYTES = 64 * 1024;
@@ -7015,11 +7017,11 @@ var init_portable_runtime = __esm({
     DEFAULT_JUDGE_CONCURRENCY = 4;
     DEFAULT_MAX_INPUT_BYTES = 4 * 1024 * 1024;
     DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
-    JudgeRequestSchema = import_zod7.z.object({
-      type: import_zod7.z.literal("judge_request"),
-      requestId: import_zod7.z.string(),
-      traceId: import_zod7.z.string(),
-      rubric: import_zod7.z.string()
+    JudgeRequestSchema = import_zod9.z.object({
+      type: import_zod9.z.literal("judge_request"),
+      requestId: import_zod9.z.string(),
+      traceId: import_zod9.z.string(),
+      rubric: import_zod9.z.string()
     });
   }
 });
@@ -7111,67 +7113,67 @@ function canonicalJson2(value) {
   if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
   return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson2(entry)}`).join(",")}}`;
 }
-var import_zod8, PortableEvalArtifactSchema, PortableEvalResultSchema;
+var import_zod10, PortableEvalArtifactSchema, PortableEvalResultSchema;
 var init_portable = __esm({
   "src/evals/portable.ts"() {
     "use strict";
-    import_zod8 = require("zod");
+    import_zod10 = require("zod");
     init_definition();
     init_trace();
-    PortableEvalArtifactSchema = import_zod8.z.object({
-      format: import_zod8.z.literal("raindrop.eval-program"),
-      formatVersion: import_zod8.z.literal(1),
-      runtimeAbi: import_zod8.z.literal("raindrop-eval-v2"),
-      evaluator: import_zod8.z.object({
-        id: import_zod8.z.string().uuid(),
-        slug: import_zod8.z.string().min(1),
-        programVersion: import_zod8.z.number().int().positive(),
-        executionMode: import_zod8.z.enum(["deterministic", "judge"]),
-        outputType: import_zod8.z.enum(["boolean", "score", "number"]),
-        scope: import_zod8.z.literal("batch")
+    PortableEvalArtifactSchema = import_zod10.z.object({
+      format: import_zod10.z.literal("raindrop.eval-program"),
+      formatVersion: import_zod10.z.literal(1),
+      runtimeAbi: import_zod10.z.literal("raindrop-eval-v2"),
+      evaluator: import_zod10.z.object({
+        id: import_zod10.z.string().uuid(),
+        slug: import_zod10.z.string().min(1),
+        programVersion: import_zod10.z.number().int().positive(),
+        executionMode: import_zod10.z.enum(["deterministic", "judge"]),
+        outputType: import_zod10.z.enum(["boolean", "score", "number"]),
+        scope: import_zod10.z.literal("batch")
       }).strict(),
-      traceProjectionVersion: import_zod8.z.literal(1),
-      traceProjectionSha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
-      source: import_zod8.z.string().min(1),
-      sha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/)
+      traceProjectionVersion: import_zod10.z.literal(1),
+      traceProjectionSha256: import_zod10.z.string().regex(/^[a-f0-9]{64}$/),
+      source: import_zod10.z.string().min(1),
+      sha256: import_zod10.z.string().regex(/^[a-f0-9]{64}$/)
     }).strict();
-    PortableEvalResultSchema = import_zod8.z.object({
-      outcomes: import_zod8.z.array(
-        import_zod8.z.union([
-          import_zod8.z.object({
-            state: import_zod8.z.literal("graded"),
-            traceId: import_zod8.z.string(),
-            pass: import_zod8.z.boolean(),
-            note: import_zod8.z.string().optional()
+    PortableEvalResultSchema = import_zod10.z.object({
+      outcomes: import_zod10.z.array(
+        import_zod10.z.union([
+          import_zod10.z.object({
+            state: import_zod10.z.literal("graded"),
+            traceId: import_zod10.z.string(),
+            pass: import_zod10.z.boolean(),
+            note: import_zod10.z.string().optional()
           }),
-          import_zod8.z.object({
-            state: import_zod8.z.literal("graded"),
-            traceId: import_zod8.z.string(),
+          import_zod10.z.object({
+            state: import_zod10.z.literal("graded"),
+            traceId: import_zod10.z.string(),
             score: EvalScoreSchema,
-            note: import_zod8.z.string().optional()
+            note: import_zod10.z.string().optional()
           }),
-          import_zod8.z.object({
-            state: import_zod8.z.literal("graded"),
-            traceId: import_zod8.z.string(),
-            value: import_zod8.z.number().finite(),
-            note: import_zod8.z.string().optional()
+          import_zod10.z.object({
+            state: import_zod10.z.literal("graded"),
+            traceId: import_zod10.z.string(),
+            value: import_zod10.z.number().finite(),
+            note: import_zod10.z.string().optional()
           }),
-          import_zod8.z.object({ state: import_zod8.z.literal("errored"), traceId: import_zod8.z.string(), message: import_zod8.z.string() }),
-          import_zod8.z.object({
-            state: import_zod8.z.literal("ungraded"),
-            traceId: import_zod8.z.string(),
-            reason: import_zod8.z.enum(["not sampled", "trace unavailable", "content unreadable"])
+          import_zod10.z.object({ state: import_zod10.z.literal("errored"), traceId: import_zod10.z.string(), message: import_zod10.z.string() }),
+          import_zod10.z.object({
+            state: import_zod10.z.literal("ungraded"),
+            traceId: import_zod10.z.string(),
+            reason: import_zod10.z.enum(["not sampled", "trace unavailable", "content unreadable"])
           })
         ])
       ),
-      failures: import_zod8.z.array(import_zod8.z.string()),
-      stats: import_zod8.z.object({
-        traceCount: import_zod8.z.number().int().nonnegative(),
-        hydratedCount: import_zod8.z.number().int().nonnegative(),
-        judgeCalls: import_zod8.z.number().int().nonnegative(),
-        durationMs: import_zod8.z.number().nonnegative(),
-        skippedRows: import_zod8.z.number().int().nonnegative(),
-        skippedReason: import_zod8.z.literal("no trace").nullable()
+      failures: import_zod10.z.array(import_zod10.z.string()),
+      stats: import_zod10.z.object({
+        traceCount: import_zod10.z.number().int().nonnegative(),
+        hydratedCount: import_zod10.z.number().int().nonnegative(),
+        judgeCalls: import_zod10.z.number().int().nonnegative(),
+        durationMs: import_zod10.z.number().nonnegative(),
+        skippedRows: import_zod10.z.number().int().nonnegative(),
+        skippedReason: import_zod10.z.literal("no trace").nullable()
       })
     });
   }
@@ -7181,7 +7183,7 @@ var init_portable = __esm({
 var import_node_path = require("path");
 var import_node_url = require("url");
 var import_node_util = require("util");
-var import_zod11 = require("zod");
+var import_zod13 = require("zod");
 
 // src/lib/inject-als.ts
 var import_node_async_hooks = require("async_hooks");
@@ -12911,7 +12913,7 @@ var SignalEventSchema = external_exports.object({
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.7.1",
+  version: "0.8.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",
@@ -16987,7 +16989,7 @@ function replayIngestEndpoint(value, trustedUrl) {
 
 // src/replay/index.ts
 var DEFAULT_QUERY_URL = "https://query.raindrop.ai";
-var DEFAULT_TRACE_WAIT_MS = 3e4;
+var DEFAULT_TRACE_WAIT_MS = 6e4;
 var DEFAULT_EVAL_WAIT_MS = 10 * 6e4;
 var DEFAULT_EVAL_POLL_INTERVAL_MS = 2e3;
 var MAX_EVAL_STATUS_RETRIES = 2;
@@ -17679,12 +17681,17 @@ async function runReplayRows(client, handle, options) {
       }
     }
   };
-  await Promise.all(
+  const workers = await Promise.allSettled(
     Array.from(
       { length: Math.min((_f = options.concurrency) != null ? _f : DEFAULT_CONCURRENCY, rows.length) },
       () => worker()
     )
   );
+  const failures = workers.flatMap(
+    (result) => result.status === "rejected" ? [result.reason] : []
+  );
+  if (failures.length === 1) throw failures[0];
+  if (failures.length > 1) throw new AggregateError(failures, "Replay row execution failed");
   await client.forceFlush();
   const traceEvidence = [];
   for (const row of rows) {
@@ -18664,6 +18671,34 @@ function describeError(cause) {
   return String(cause);
 }
 
+// src/evals/agent.ts
+var import_zod6 = require("zod");
+var AgentMetadataSchema = import_zod6.z.object({
+  slug: import_zod6.z.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
+  name: import_zod6.z.string().trim().min(1).max(120),
+  description: import_zod6.z.string().max(2e3).optional()
+});
+async function finishWith(execution, work) {
+  let result;
+  try {
+    result = await work();
+  } catch (error) {
+    await execution.finish().catch(() => void 0);
+    throw error;
+  }
+  await execution.finish();
+  return result;
+}
+
+// src/evals/server.ts
+var import_zod7 = require("zod");
+var InvocationSchema = import_zod7.z.object({
+  runId: import_zod7.z.string().uuid(),
+  agent: import_zod7.z.string().min(1).max(64),
+  parameters: import_zod7.z.record(import_zod7.z.unknown()).default({})
+});
+var MAX_BODY_BYTES = 64 * 1024;
+
 // src/evals/index.ts
 init_definition();
 
@@ -18816,7 +18851,7 @@ function sleep2(ms) {
 }
 
 // src/evals/publish.ts
-var import_zod6 = require("zod");
+var import_zod8 = require("zod");
 init_definition();
 
 // src/evals/runs.ts
@@ -18863,19 +18898,19 @@ async function readEvalRun(client, runId, options = {}) {
 }
 
 // src/evals/publish.ts
-var EvaluatorIdentitySchema = import_zod6.z.object({
-  id: import_zod6.z.string().uuid(),
-  slug: import_zod6.z.string(),
-  programVersion: import_zod6.z.number().int().positive(),
-  outputType: import_zod6.z.enum(["boolean", "score", "number"])
+var EvaluatorIdentitySchema = import_zod8.z.object({
+  id: import_zod8.z.string().uuid(),
+  slug: import_zod8.z.string(),
+  programVersion: import_zod8.z.number().int().positive(),
+  outputType: import_zod8.z.enum(["boolean", "score", "number"])
 });
 var EvaluatorSourceSchema = EvaluatorIdentitySchema.extend({
-  name: import_zod6.z.string(),
-  description: import_zod6.z.string().nullable(),
-  intent: import_zod6.z.string(),
-  kind: import_zod6.z.enum(["judge", "deterministic"]),
-  programSource: import_zod6.z.string(),
-  rules: import_zod6.z.array(import_zod6.z.string())
+  name: import_zod8.z.string(),
+  description: import_zod8.z.string().nullable(),
+  intent: import_zod8.z.string(),
+  kind: import_zod8.z.enum(["judge", "deterministic"]),
+  programSource: import_zod8.z.string(),
+  rules: import_zod8.z.array(import_zod8.z.string())
 });
 var EvalPublishError = class extends Error {
   constructor(status, message) {
@@ -18884,12 +18919,12 @@ var EvalPublishError = class extends Error {
     this.name = "EvalPublishError";
   }
 };
-var ComparisonRunSchema = import_zod6.z.object({
-  id: import_zod6.z.string().uuid(),
-  programVersion: import_zod6.z.number().int().positive()
+var ComparisonRunSchema = import_zod8.z.object({
+  id: import_zod8.z.string().uuid(),
+  programVersion: import_zod8.z.number().int().positive()
 });
-var ComparisonLinkSchema = import_zod6.z.object({
-  url: import_zod6.z.string().url(),
+var ComparisonLinkSchema = import_zod8.z.object({
+  url: import_zod8.z.string().url(),
   first: ComparisonRunSchema,
   second: ComparisonRunSchema
 });
@@ -18917,7 +18952,7 @@ async function compareEvalRuns(client, options) {
   }
   const before = selected.id;
   const after = counterpart.id;
-  const { data } = import_zod6.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
+  const { data } = import_zod8.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
   if (data.first.id !== before || data.second.id !== after) {
     throw new Error("Raindrop returned a different comparison pair");
   }
@@ -18930,7 +18965,7 @@ async function compareEvalRuns(client, options) {
   };
 }
 async function readEvaluator(client, slug, options = {}) {
-  const { data } = import_zod6.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
+  const { data } = import_zod8.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
   if (data.slug !== slug) throw new Error("Raindrop returned a different evaluator slug");
   return defineEvaluatorProgram({
     slug: data.slug,
@@ -19056,34 +19091,34 @@ init_trace();
 init_portable();
 
 // src/evals/pull.ts
-var import_zod9 = require("zod");
+var import_zod11 = require("zod");
 init_definition();
 init_portable();
 var SNAPSHOT_VERSION = 1;
-var PulledEvaluatorSchema = import_zod9.z.object({
-  slug: import_zod9.z.string().min(1),
-  name: import_zod9.z.string().min(1),
-  expectedSha256: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
-  artifact: import_zod9.z.unknown()
+var PulledEvaluatorSchema = import_zod11.z.object({
+  slug: import_zod11.z.string().min(1),
+  name: import_zod11.z.string().min(1),
+  expectedSha256: import_zod11.z.string().regex(/^[a-f0-9]{64}$/),
+  artifact: import_zod11.z.unknown()
 });
-var EvalSnapshotSchema = import_zod9.z.object({
-  schemaVersion: import_zod9.z.literal(SNAPSHOT_VERSION),
-  pulledAt: import_zod9.z.string().datetime(),
-  dataset: import_zod9.z.object({
-    reference: import_zod9.z.string().min(1),
-    datasetId: import_zod9.z.string().uuid(),
-    fingerprint: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
-    rows: import_zod9.z.array(
-      import_zod9.z.object({
-        id: import_zod9.z.string(),
-        name: import_zod9.z.string(),
-        input: import_zod9.z.string().nullable(),
-        output: import_zod9.z.string().nullable(),
-        properties: import_zod9.z.record(import_zod9.z.string())
+var EvalSnapshotSchema = import_zod11.z.object({
+  schemaVersion: import_zod11.z.literal(SNAPSHOT_VERSION),
+  pulledAt: import_zod11.z.string().datetime(),
+  dataset: import_zod11.z.object({
+    reference: import_zod11.z.string().min(1),
+    datasetId: import_zod11.z.string().uuid(),
+    fingerprint: import_zod11.z.string().regex(/^[a-f0-9]{64}$/),
+    rows: import_zod11.z.array(
+      import_zod11.z.object({
+        id: import_zod11.z.string(),
+        name: import_zod11.z.string(),
+        input: import_zod11.z.string().nullable(),
+        output: import_zod11.z.string().nullable(),
+        properties: import_zod11.z.record(import_zod11.z.string())
       })
     )
   }),
-  evaluators: import_zod9.z.array(PulledEvaluatorSchema)
+  evaluators: import_zod11.z.array(PulledEvaluatorSchema)
 });
 
 // src/evals/run.ts
@@ -19137,46 +19172,46 @@ function outputMismatch(name, output) {
 init_portable();
 
 // src/evals/workshop.ts
-var import_zod10 = require("zod");
-var WorkshopRunSchema = import_zod10.z.object({
-  id: import_zod10.z.string(),
-  event_id: import_zod10.z.string().nullable(),
-  name: import_zod10.z.string().nullable(),
-  event_name: import_zod10.z.string().nullable(),
-  user_id: import_zod10.z.string().nullable(),
-  convo_id: import_zod10.z.string().nullable(),
-  started_at: import_zod10.z.number().nullable(),
-  last_updated_at: import_zod10.z.number().nullable(),
-  metadata: import_zod10.z.string().nullable(),
-  model: import_zod10.z.string().nullable(),
-  finished: import_zod10.z.number().nullable()
+var import_zod12 = require("zod");
+var WorkshopRunSchema = import_zod12.z.object({
+  id: import_zod12.z.string(),
+  event_id: import_zod12.z.string().nullable(),
+  name: import_zod12.z.string().nullable(),
+  event_name: import_zod12.z.string().nullable(),
+  user_id: import_zod12.z.string().nullable(),
+  convo_id: import_zod12.z.string().nullable(),
+  started_at: import_zod12.z.number().nullable(),
+  last_updated_at: import_zod12.z.number().nullable(),
+  metadata: import_zod12.z.string().nullable(),
+  model: import_zod12.z.string().nullable(),
+  finished: import_zod12.z.number().nullable()
 });
-var WorkshopSpanSchema = import_zod10.z.object({
-  id: import_zod10.z.string(),
-  run_id: import_zod10.z.string(),
-  parent_span_id: import_zod10.z.string().nullable(),
-  name: import_zod10.z.string(),
-  span_type: import_zod10.z.string().nullable(),
-  status: import_zod10.z.string().nullable(),
-  input_payload: import_zod10.z.string().nullable(),
-  output_payload: import_zod10.z.string().nullable(),
-  start_time_ms: import_zod10.z.number().nullable(),
-  end_time_ms: import_zod10.z.number().nullable(),
-  duration_ms: import_zod10.z.number().nullable(),
-  model: import_zod10.z.string().nullable(),
-  provider: import_zod10.z.string().nullable(),
-  input_tokens: import_zod10.z.number().nullable(),
-  output_tokens: import_zod10.z.number().nullable(),
-  attributes: import_zod10.z.string().nullable()
+var WorkshopSpanSchema = import_zod12.z.object({
+  id: import_zod12.z.string(),
+  run_id: import_zod12.z.string(),
+  parent_span_id: import_zod12.z.string().nullable(),
+  name: import_zod12.z.string(),
+  span_type: import_zod12.z.string().nullable(),
+  status: import_zod12.z.string().nullable(),
+  input_payload: import_zod12.z.string().nullable(),
+  output_payload: import_zod12.z.string().nullable(),
+  start_time_ms: import_zod12.z.number().nullable(),
+  end_time_ms: import_zod12.z.number().nullable(),
+  duration_ms: import_zod12.z.number().nullable(),
+  model: import_zod12.z.string().nullable(),
+  provider: import_zod12.z.string().nullable(),
+  input_tokens: import_zod12.z.number().nullable(),
+  output_tokens: import_zod12.z.number().nullable(),
+  attributes: import_zod12.z.string().nullable()
 });
-var WorkshopTraceEnvelopeSchema = import_zod10.z.object({
-  status: import_zod10.z.literal("complete"),
-  correlationId: import_zod10.z.string(),
-  traceId: import_zod10.z.string(),
-  rootSpanId: import_zod10.z.string(),
-  spanCount: import_zod10.z.number().int().positive(),
+var WorkshopTraceEnvelopeSchema = import_zod12.z.object({
+  status: import_zod12.z.literal("complete"),
+  correlationId: import_zod12.z.string(),
+  traceId: import_zod12.z.string(),
+  rootSpanId: import_zod12.z.string(),
+  spanCount: import_zod12.z.number().int().positive(),
   run: WorkshopRunSchema,
-  spans: import_zod10.z.array(WorkshopSpanSchema).min(1)
+  spans: import_zod12.z.array(WorkshopSpanSchema).min(1)
 });
 async function readWorkshopTrace(input) {
   if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
@@ -19260,8 +19295,17 @@ function serializableOutput(value) {
 
 // src/evals/run.ts
 var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
-function createEvalSuiteRun(client, definition, options) {
-  const validated = validateDefinition(definition);
+function bindSuite(definition, parameters) {
+  if (definition.agent === void 0) {
+    const { run } = definition;
+    return {
+      run: (row) => Promise.resolve().then(() => run(row)),
+      finish: () => Promise.resolve()
+    };
+  }
+  return definition.agent.start(parameters);
+}
+function createSuiteSession(client, validated, run, options) {
   if (validated.datasetVersionId && "selection" in options && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
@@ -19289,6 +19333,7 @@ function createEvalSuiteRun(client, definition, options) {
     return new RaindropRowEvalRun(
       client,
       validated,
+      run,
       options.destination,
       options.selection.dataset,
       selection.rowIds,
@@ -19313,19 +19358,26 @@ function createEvalSuiteRun(client, definition, options) {
   return new WorkshopEvalRun(
     client,
     { ...validated, dataset: validated.dataset },
+    run,
     options.destination
   );
 }
 async function runEvalSuite(client, definition, options = {}) {
+  var _a;
+  const validated = validateDefinition(definition);
+  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  return finishWith(execution, () => runSuite(client, validated, execution.run, options));
+}
+async function runSuite(client, definition, run, options) {
   var _a, _b, _c, _d, _e, _f, _g, _h;
-  let validated = validateDefinition(definition);
+  let validated = definition;
   const destination = (_a = options.destination) != null ? _a : { kind: "raindrop" };
   if (validated.datasetVersionId && ((_b = options.selection) == null ? void 0 : _b.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
   if (destination.kind === "workshop") {
     if (options.runId !== void 0) throw new Error("Queued runs are only supported in Raindrop");
-    const session = createEvalSuiteRun(client, validated, { destination });
+    const session = createSuiteSession(client, validated, run, { destination });
     await session.run({ selection: options.selection, attempt: options.attempt });
     return session.finish();
   }
@@ -19359,7 +19411,7 @@ async function runEvalSuite(client, definition, options = {}) {
       rowIds: (_h = (_g = options.selection) == null ? void 0 : _g.rowIds) != null ? _h : dataset.rows.map((entry) => entry.id)
     });
     if (!selection) throw new Error("Raindrop remote eval requires selected rows");
-    const session = createEvalSuiteRun(client, validated, {
+    const session = createSuiteSession(client, validated, run, {
       destination,
       selection: { dataset, rowIds: selection.rowIds }
     });
@@ -19393,7 +19445,7 @@ async function runEvalSuite(client, definition, options = {}) {
     }
     return finished;
   }
-  return runRaindropEval(client, validated, destination, options);
+  return runRaindropEval(client, validated, run, destination, options);
 }
 function isRowScopedLocalEval(definition) {
   return definition.evaluators.every(
@@ -19401,9 +19453,10 @@ function isRowScopedLocalEval(definition) {
   );
 }
 var RaindropRowEvalRun = class {
-  constructor(client, definition, destination, dataset, rowIds, evaluators) {
+  constructor(client, definition, runRow, destination, dataset, rowIds, evaluators) {
     this.client = client;
     this.definition = definition;
+    this.runRow = runRow;
     this.destination = destination;
     this.dataset = dataset;
     this.rowIds = rowIds;
@@ -19453,7 +19506,7 @@ var RaindropRowEvalRun = class {
       const attemptResult = await replay2.run({
         rowId,
         attempt,
-        run: this.definition.run,
+        run: this.runRow,
         traceWaitMs: this.definition.traceWaitMs
       });
       const configured = new Map(
@@ -19558,9 +19611,10 @@ function replayEvaluatorsFromVerdicts(verdicts) {
   return [...evaluators.values()];
 }
 var WorkshopEvalRun = class {
-  constructor(client, definition, destination) {
+  constructor(client, definition, runRow, destination) {
     this.client = client;
     this.definition = definition;
+    this.runRow = runRow;
     this.destination = destination;
     this.id = randomId();
     this.startedAt = /* @__PURE__ */ new Date();
@@ -19690,7 +19744,7 @@ var WorkshopEvalRun = class {
             () => {
               var _a2;
               const parent = (_a2 = import_api8.trace.getSpan(import_api8.context.active())) == null ? void 0 : _a2.spanContext();
-              if (!parent) return this.definition.run(row);
+              if (!parent) return this.runRow(row);
               return withReplayTraceDestination(
                 {
                   ...destination,
@@ -19700,7 +19754,7 @@ var WorkshopEvalRun = class {
                     eventId: parent.traceId
                   }
                 },
-                () => this.definition.run(row)
+                () => this.runRow(row)
               );
             }
           )
@@ -19878,7 +19932,7 @@ var WorkshopEvalRun = class {
     return current;
   }
 };
-async function runRaindropEval(client, definition, destination, options) {
+async function runRaindropEval(client, definition, run, destination, options) {
   var _a, _b, _c, _d;
   if (typeof definition.dataset !== "string") {
     throw new Error(
@@ -19905,7 +19959,7 @@ async function runRaindropEval(client, definition, destination, options) {
     dataset: pinnedDataset ? datasetToWire(pinnedDataset) : datasetReference,
     evaluators: definition.evaluators.map((entry) => replayEvaluator(entry)),
     evaluatorPins: hostedEvaluatorPins(definition.evaluators),
-    run: definition.run,
+    run,
     concurrency: definition.concurrency,
     traceWaitMs: definition.traceWaitMs,
     evalWaitMs: definition.evalWaitMs,
@@ -21555,8 +21609,8 @@ var index_default = Raindrop;
 
 // src/evals/cli.ts
 init_definition();
-var SuiteModuleSchema = import_zod11.z.object({
-  default: import_zod11.z.custom(isEvalSuiteDefinition)
+var SuiteModuleSchema = import_zod13.z.object({
+  default: import_zod13.z.custom(isEvalSuiteDefinition)
 });
 async function main() {
   var _a, _b;