raindrop-ai 0.7.1 → 0.8.0

package/dist/{chunk-6WL655QL.mjs → chunk-22CNYHSA.mjs}

@@ -37,7 +37,7 @@ import {
   withReplayTraceDestination,
   withRoutingContext,
   withTask
-} from "./chunk-SHH4SYVY.mjs";
+} from "./chunk-EWHJ2YBW.mjs";
 import {
   CapturedTraceSnapshotSchema,
   EvalCaseTracePairSchema,
@@ -59,7 +59,7 @@ import {
   traceToolCalls,
   validateEvaluatorOutput,
   verifyPortableEvalArtifact
-} from "./chunk-RMP6BZSY.mjs";
+} from "./chunk-EFATLUNL.mjs";
 import {
   __commonJS,
   __toESM
@@ -3739,7 +3739,7 @@ var require_lodash = __commonJS({
           if (fromIndex < 0) {
             fromIndex = nativeMax(length + fromIndex, 0);
           }
-          return isString(collection) ? fromIndex <= length && collection.indexOf(value, fromIndex) > -1 : !!length && baseIndexOf(collection, value, fromIndex) > -1;
+          return isString2(collection) ? fromIndex <= length && collection.indexOf(value, fromIndex) > -1 : !!length && baseIndexOf(collection, value, fromIndex) > -1;
         }
         var invokeMap = baseRest(function(collection, path, args) {
           var index = -1, isFunc = typeof path == "function", result2 = isArrayLike(collection) ? Array2(collection.length) : [];
@@ -3807,7 +3807,7 @@ var require_lodash = __commonJS({
             return 0;
           }
           if (isArrayLike(collection)) {
-            return isString(collection) ? stringSize(collection) : collection.length;
+            return isString2(collection) ? stringSize(collection) : collection.length;
           }
           var tag = getTag(collection);
           if (tag == mapTag || tag == setTag) {
@@ -4133,7 +4133,7 @@ var require_lodash = __commonJS({
         function isArrayLikeObject(value) {
           return isObjectLike(value) && isArrayLike(value);
         }
-        function isBoolean(value) {
+        function isBoolean2(value) {
           return value === true || value === false || isObjectLike(value) && baseGetTag(value) == boolTag;
         }
         var isBuffer = nativeIsBuffer || stubFalse;
@@ -4209,7 +4209,7 @@ var require_lodash = __commonJS({
           return baseIsMatch(object, source, getMatchData(source), customizer);
         }
         function isNaN(value) {
-          return isNumber(value) && value != +value;
+          return isNumber2(value) && value != +value;
         }
         function isNative(value) {
           if (isMaskable(value)) {
@@ -4223,7 +4223,7 @@ var require_lodash = __commonJS({
         function isNil(value) {
           return value == null;
         }
-        function isNumber(value) {
+        function isNumber2(value) {
           return typeof value == "number" || isObjectLike(value) && baseGetTag(value) == numberTag;
         }
         function isPlainObject(value) {
@@ -4242,7 +4242,7 @@ var require_lodash = __commonJS({
           return isInteger(value) && value >= -MAX_SAFE_INTEGER && value <= MAX_SAFE_INTEGER;
         }
         var isSet = nodeIsSet ? baseUnary(nodeIsSet) : baseIsSet;
-        function isString(value) {
+        function isString2(value) {
           return typeof value == "string" || !isArray(value) && isObjectLike(value) && baseGetTag(value) == stringTag;
         }
         function isSymbol(value) {
@@ -4267,7 +4267,7 @@ var require_lodash = __commonJS({
             return [];
           }
           if (isArrayLike(value)) {
-            return isString(value) ? stringToArray(value) : copyArray(value);
+            return isString2(value) ? stringToArray(value) : copyArray(value);
           }
           if (symIterator && value[symIterator]) {
             return iteratorToArray(value[symIterator]());
@@ -5275,7 +5275,7 @@ var require_lodash = __commonJS({
         lodash.isArrayBuffer = isArrayBuffer;
         lodash.isArrayLike = isArrayLike;
         lodash.isArrayLikeObject = isArrayLikeObject;
-        lodash.isBoolean = isBoolean;
+        lodash.isBoolean = isBoolean2;
         lodash.isBuffer = isBuffer;
         lodash.isDate = isDate;
         lodash.isElement = isElement;
@@ -5294,14 +5294,14 @@ var require_lodash = __commonJS({
         lodash.isNative = isNative;
         lodash.isNil = isNil;
         lodash.isNull = isNull;
-        lodash.isNumber = isNumber;
+        lodash.isNumber = isNumber2;
         lodash.isObject = isObject;
         lodash.isObjectLike = isObjectLike;
         lodash.isPlainObject = isPlainObject;
         lodash.isRegExp = isRegExp;
         lodash.isSafeInteger = isSafeInteger;
         lodash.isSet = isSet;
-        lodash.isString = isString;
+        lodash.isString = isString2;
         lodash.isSymbol = isSymbol;
         lodash.isTypedArray = isTypedArray;
         lodash.isUndefined = isUndefined;
@@ -6283,15 +6283,15 @@ var makeIssue = (params) => {
       message: issueData.message
     };
   }
-  let errorMessage = "";
+  let errorMessage2 = "";
   const maps = errorMaps.filter((m) => !!m).slice().reverse();
   for (const map of maps) {
-    errorMessage = map(fullIssue, { data, defaultError: errorMessage }).message;
+    errorMessage2 = map(fullIssue, { data, defaultError: errorMessage2 }).message;
   }
   return {
     ...issueData,
     path: fullPath,
-    message: errorMessage
+    message: errorMessage2
   };
 };
 var EMPTY_PATH = [];
@@ -10161,7 +10161,7 @@ function replayIngestEndpoint(value, trustedUrl) {
 
 // src/replay/index.ts
 var DEFAULT_QUERY_URL = "https://query.raindrop.ai";
-var DEFAULT_TRACE_WAIT_MS = 3e4;
+var DEFAULT_TRACE_WAIT_MS = 6e4;
 var DEFAULT_EVAL_WAIT_MS = 10 * 6e4;
 var DEFAULT_EVAL_POLL_INTERVAL_MS = 2e3;
 var MAX_EVAL_STATUS_RETRIES = 2;
@@ -10773,6 +10773,9 @@ function validateEvalDatasetManifest(value) {
 async function runReplay(client, handle, options) {
   return (await runReplayRows(client, handle, options)).reading;
 }
+async function runEvalReplay(client, handle, options) {
+  return runReplayRows(client, handle, options);
+}
 async function runReplayRows(client, handle, options) {
   var _a, _b, _c, _d, _e, _f, _g, _h;
   if (!Number.isSafeInteger((_a = options.concurrency) != null ? _a : DEFAULT_CONCURRENCY) || ((_b = options.concurrency) != null ? _b : DEFAULT_CONCURRENCY) < 1) {
@@ -10877,12 +10880,17 @@ async function runReplayRows(client, handle, options) {
       }
     }
   };
-  await Promise.all(
+  const workers = await Promise.allSettled(
     Array.from(
       { length: Math.min((_f = options.concurrency) != null ? _f : DEFAULT_CONCURRENCY, rows.length) },
       () => worker()
     )
   );
+  const failures = workers.flatMap(
+    (result) => result.status === "rejected" ? [result.reason] : []
+  );
+  if (failures.length === 1) throw failures[0];
+  if (failures.length > 1) throw new AggregateError(failures, "Replay row execution failed");
   await client.forceFlush();
   const traceEvidence = [];
   for (const row of rows) {
@@ -11866,6 +11874,253 @@ function describeError(cause) {
   return String(cause);
 }
 
+// src/evals/agent.ts
+import { z as z4 } from "zod";
+var EvalParameterError = class extends Error {
+  constructor(agent, issues) {
+    super(
+      `Invalid parameters for agent ${agent}: ${issues.map((issue) => `${issue.path.join(".") || "parameters"} ${issue.message}`).join("; ")}`
+    );
+    this.name = "EvalParameterError";
+  }
+};
+var AgentMetadataSchema = z4.object({
+  slug: z4.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
+  name: z4.string().trim().min(1).max(120),
+  description: z4.string().max(2e3).optional()
+});
+var PARAMETER_KEY = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/;
+var MAX_PARAMETERS = 50;
+function unwrapField(schema) {
+  let inner = schema;
+  let description;
+  while (inner instanceof z4.ZodDefault || inner instanceof z4.ZodOptional) {
+    description != null ? description : description = inner.description;
+    inner = inner._def.innerType;
+  }
+  const omitted = schema.safeParse(void 0);
+  return {
+    inner,
+    required: !omitted.success,
+    defaultValue: omitted.success ? omitted.data : void 0,
+    description: description != null ? description : inner.description
+  };
+}
+function fieldDefault(key, value, guard) {
+  if (value === void 0) return void 0;
+  if (!guard(value)) throw new Error(`Parameter ${key} has a default of the wrong type`);
+  return value;
+}
+var isString = (value) => typeof value === "string";
+var isNumber = (value) => typeof value === "number";
+var isBoolean = (value) => typeof value === "boolean";
+function describeParameterField(key, schema) {
+  if (!PARAMETER_KEY.test(key)) throw new Error(`Parameter key ${key} must be an identifier`);
+  const { inner, required, defaultValue, description } = unwrapField(schema);
+  const base = { key, required, ...description === void 0 ? {} : { description } };
+  if (inner instanceof z4.ZodString) {
+    const max = inner._def.checks.find((check) => check.kind === "max");
+    return {
+      ...base,
+      type: "string",
+      default: fieldDefault(key, defaultValue, isString),
+      ...(max == null ? void 0 : max.kind) === "max" ? { maxLength: max.value } : {}
+    };
+  }
+  if (inner instanceof z4.ZodNumber) {
+    const { minValue, maxValue } = inner;
+    return {
+      ...base,
+      type: "number",
+      default: fieldDefault(key, defaultValue, isNumber),
+      integer: inner.isInt,
+      ...minValue === null ? {} : { min: minValue },
+      ...maxValue === null ? {} : { max: maxValue }
+    };
+  }
+  if (inner instanceof z4.ZodBoolean) {
+    return { ...base, type: "boolean", default: fieldDefault(key, defaultValue, isBoolean) };
+  }
+  if (inner instanceof z4.ZodEnum) {
+    const options = inner.options;
+    return { ...base, type: "enum", default: fieldDefault(key, defaultValue, isString), options };
+  }
+  throw new Error(
+    `Parameter ${key} must be a z.string(), z.number(), z.boolean(), or z.enum() schema`
+  );
+}
+function defineAgent(definition) {
+  var _a, _b;
+  const { slug, name, description } = AgentMetadataSchema.parse({
+    slug: definition.slug,
+    name: (_a = definition.name) != null ? _a : definition.slug,
+    description: definition.description
+  });
+  if (typeof definition.run !== "function") throw new Error("An agent requires a run function");
+  const fields = (_b = definition.parameters) != null ? _b : {};
+  const entries = Object.entries(fields);
+  if (entries.length > MAX_PARAMETERS)
+    throw new Error(`Agent ${slug} declares more than ${MAX_PARAMETERS} parameters`);
+  const parameters = entries.map(([key, schema2]) => describeParameterField(key, schema2));
+  const schema = z4.object(fields).strict();
+  return {
+    slug,
+    name,
+    description,
+    parameters,
+    start(input) {
+      const parsed = schema.safeParse(input);
+      if (!parsed.success) throw new EvalParameterError(slug, parsed.error.issues);
+      return startAgent(definition, parsed.data);
+    }
+  };
+}
+async function finishWith(execution, work) {
+  let result;
+  try {
+    result = await work();
+  } catch (error) {
+    await execution.finish().catch(() => void 0);
+    throw error;
+  }
+  await execution.finish();
+  return result;
+}
+function startAgent(definition, parameters) {
+  const active = /* @__PURE__ */ new Set();
+  let finishing;
+  const execution = (runRow, cleanup2) => ({
+    run(row) {
+      if (finishing !== void 0)
+        return Promise.reject(new Error(`Agent ${definition.slug} has already finished`));
+      const pending = runRow(row);
+      active.add(pending);
+      pending.catch(() => void 0).finally(() => active.delete(pending));
+      return pending;
+    },
+    finish: () => finishing != null ? finishing : finishing = Promise.allSettled(active).then(cleanup2)
+  });
+  if (definition.setup === void 0) {
+    const { run: run2, cleanup: cleanup2 } = definition;
+    return execution(
+      (row) => Promise.resolve().then(() => run2(row, { parameters })),
+      async () => {
+        await (cleanup2 == null ? void 0 : cleanup2());
+      }
+    );
+  }
+  const { setup, run, cleanup } = definition;
+  let environment;
+  const ready = () => {
+    if (environment === void 0) {
+      environment = Promise.resolve().then(() => setup(parameters));
+      environment.catch(() => void 0);
+    }
+    return environment;
+  };
+  return execution(
+    async (row) => run(row, { parameters, environment: await ready() }),
+    () => environment === void 0 ? Promise.resolve() : environment.then(
+      (built) => cleanup == null ? void 0 : cleanup(built),
+      () => void 0
+    )
+  );
+}
+
+// src/evals/server.ts
+import { z as z5 } from "zod";
+var InvocationSchema = z5.object({
+  runId: z5.string().uuid(),
+  agent: z5.string().min(1).max(64),
+  parameters: z5.record(z5.unknown()).default({})
+});
+var MAX_BODY_BYTES = 64 * 1024;
+async function parseInvocation(request) {
+  var _a;
+  if (Number((_a = request.headers.get("content-length")) != null ? _a : 0) > MAX_BODY_BYTES) return void 0;
+  const body = await request.text().catch(() => "");
+  if (new TextEncoder().encode(body).byteLength > MAX_BODY_BYTES) return void 0;
+  try {
+    const parsed = InvocationSchema.safeParse(JSON.parse(body));
+    return parsed.success ? parsed.data : void 0;
+  } catch (e) {
+    return void 0;
+  }
+}
+function startExecution(agent, parameters) {
+  try {
+    return agent.start(parameters);
+  } catch (error) {
+    if (error instanceof EvalParameterError)
+      return Response.json({ error: error.message }, { status: 400 });
+    throw error;
+  }
+}
+function errorMessage(error) {
+  return error instanceof Error ? error.message : "Eval execution failed";
+}
+function createEvalHandler(client, options) {
+  if (options.queryUrl !== void 0) credentialEndpoint(options.queryUrl);
+  if (options.replayIngestUrl !== void 0) credentialEndpoint(options.replayIngestUrl);
+  const agents = new Map(options.agents.map((agent) => [agent.slug, agent]));
+  if (agents.size !== options.agents.length)
+    throw new Error("Registered agent slugs must be unique");
+  const catalog = [...agents.values()].map(({ slug, name, description, parameters }) => ({
+    slug,
+    name,
+    description,
+    parameters
+  }));
+  return async (request) => {
+    if (request.method === "GET") return Response.json({ agents: catalog });
+    if (request.method !== "POST") {
+      return Response.json(
+        { error: "Method not allowed" },
+        { status: 405, headers: { Allow: "GET, POST" } }
+      );
+    }
+    const invocation = await parseInvocation(request);
+    if (invocation === void 0) {
+      return Response.json({ error: "Invalid eval request" }, { status: 400 });
+    }
+    const { runId, parameters } = invocation;
+    const agent = agents.get(invocation.agent);
+    if (agent === void 0) return Response.json({ error: "Unknown agent" }, { status: 404 });
+    const execution = startExecution(agent, parameters);
+    if (execution instanceof Response) return execution;
+    try {
+      const { replay: replay2 } = await readReplayDetails(client, runId, { queryUrl: options.queryUrl });
+      if (replay2.status === "complete" || replay2.status === "failed") {
+        return Response.json({ runId, status: replay2.status, counts: replay2.counts });
+      }
+      if (replay2.status !== "queued") {
+        return Response.json({ error: "Run is already claimed" }, { status: 409 });
+      }
+      const handle = await claimReplay(client, runId, { queryUrl: options.queryUrl });
+      if (handle.replayId !== runId) throw new Error("Raindrop claimed a different run");
+      if (handle.ingestUrl === null) {
+        return Response.json({ error: "Run is already claimed" }, { status: 409 });
+      }
+      const { reading, traceEvidence } = await finishWith(
+        execution,
+        () => runEvalReplay(client, handle, {
+          run: execution.run,
+          queryUrl: options.queryUrl,
+          replayIngestUrl: options.replayIngestUrl
+        })
+      );
+      return Response.json({
+        runId: reading.replayId,
+        status: reading.status,
+        counts: reading.counts,
+        traceEvidence
+      });
+    } catch (error) {
+      return Response.json({ error: errorMessage(error) }, { status: 500 });
+    }
+  };
+}
+
 // src/evals/rows.ts
 var EvalDatasetRowInputSchema = EvalDatasetCaseInputSchema;
 function datasetFromWire({ cases, ...dataset }) {
@@ -12016,7 +12271,7 @@ function sleep2(ms) {
 }
 
 // src/evals/publish.ts
-import { z as z4 } from "zod";
+import { z as z6 } from "zod";
 
 // src/evals/runs.ts
 async function readEvalRun(client, runId, options = {}) {
@@ -12065,19 +12320,19 @@ async function evaluateEvalRun(client, runId, options) {
 }
 
 // src/evals/publish.ts
-var EvaluatorIdentitySchema = z4.object({
-  id: z4.string().uuid(),
-  slug: z4.string(),
-  programVersion: z4.number().int().positive(),
-  outputType: z4.enum(["boolean", "score", "number"])
+var EvaluatorIdentitySchema = z6.object({
+  id: z6.string().uuid(),
+  slug: z6.string(),
+  programVersion: z6.number().int().positive(),
+  outputType: z6.enum(["boolean", "score", "number"])
 });
 var EvaluatorSourceSchema = EvaluatorIdentitySchema.extend({
-  name: z4.string(),
-  description: z4.string().nullable(),
-  intent: z4.string(),
-  kind: z4.enum(["judge", "deterministic"]),
-  programSource: z4.string(),
-  rules: z4.array(z4.string())
+  name: z6.string(),
+  description: z6.string().nullable(),
+  intent: z6.string(),
+  kind: z6.enum(["judge", "deterministic"]),
+  programSource: z6.string(),
+  rules: z6.array(z6.string())
 });
 var EvalPublishError = class extends Error {
   constructor(status, message) {
@@ -12086,12 +12341,12 @@ var EvalPublishError = class extends Error {
     this.name = "EvalPublishError";
   }
 };
-var ComparisonRunSchema = z4.object({
-  id: z4.string().uuid(),
-  programVersion: z4.number().int().positive()
+var ComparisonRunSchema = z6.object({
+  id: z6.string().uuid(),
+  programVersion: z6.number().int().positive()
 });
-var ComparisonLinkSchema = z4.object({
-  url: z4.string().url(),
+var ComparisonLinkSchema = z6.object({
+  url: z6.string().url(),
   first: ComparisonRunSchema,
   second: ComparisonRunSchema
 });
@@ -12119,7 +12374,7 @@ async function compareEvalRuns(client, options) {
   }
   const before = selected.id;
   const after = counterpart.id;
-  const { data } = z4.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
+  const { data } = z6.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
   if (data.first.id !== before || data.second.id !== after) {
     throw new Error("Raindrop returned a different comparison pair");
   }
@@ -12132,7 +12387,7 @@ async function compareEvalRuns(client, options) {
   };
 }
 async function readEvaluator(client, slug, options = {}) {
-  const { data } = z4.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
+  const { data } = z6.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
   if (data.slug !== slug) throw new Error("Raindrop returned a different evaluator slug");
   return defineEvaluatorProgram({
     slug: data.slug,
@@ -12253,32 +12508,32 @@ var traceTools = Object.freeze({
 });
 
 // src/evals/pull.ts
-import { z as z5 } from "zod";
+import { z as z7 } from "zod";
 var SNAPSHOT_VERSION = 1;
-var PulledEvaluatorSchema = z5.object({
-  slug: z5.string().min(1),
-  name: z5.string().min(1),
-  expectedSha256: z5.string().regex(/^[a-f0-9]{64}$/),
-  artifact: z5.unknown()
+var PulledEvaluatorSchema = z7.object({
+  slug: z7.string().min(1),
+  name: z7.string().min(1),
+  expectedSha256: z7.string().regex(/^[a-f0-9]{64}$/),
+  artifact: z7.unknown()
 });
-var EvalSnapshotSchema = z5.object({
-  schemaVersion: z5.literal(SNAPSHOT_VERSION),
-  pulledAt: z5.string().datetime(),
-  dataset: z5.object({
-    reference: z5.string().min(1),
-    datasetId: z5.string().uuid(),
-    fingerprint: z5.string().regex(/^[a-f0-9]{64}$/),
-    rows: z5.array(
-      z5.object({
-        id: z5.string(),
-        name: z5.string(),
-        input: z5.string().nullable(),
-        output: z5.string().nullable(),
-        properties: z5.record(z5.string())
+var EvalSnapshotSchema = z7.object({
+  schemaVersion: z7.literal(SNAPSHOT_VERSION),
+  pulledAt: z7.string().datetime(),
+  dataset: z7.object({
+    reference: z7.string().min(1),
+    datasetId: z7.string().uuid(),
+    fingerprint: z7.string().regex(/^[a-f0-9]{64}$/),
+    rows: z7.array(
+      z7.object({
+        id: z7.string(),
+        name: z7.string(),
+        input: z7.string().nullable(),
+        output: z7.string().nullable(),
+        properties: z7.record(z7.string())
       })
     )
   }),
-  evaluators: z5.array(PulledEvaluatorSchema)
+  evaluators: z7.array(PulledEvaluatorSchema)
 });
 async function pullEval(client, options) {
   var _a;
@@ -12343,11 +12598,11 @@ async function pullEvaluator(client, queryUrl, rawSlug) {
       `Raindrop evaluator ${rawSlug} pull failed with ${response.status}: ${await response.text()}`
     );
   }
-  const body = z5.object({
-    data: z5.object({
-      slug: z5.string(),
-      name: z5.string(),
-      artifact: z5.unknown().nullable()
+  const body = z7.object({
+    data: z7.object({
+      slug: z7.string(),
+      name: z7.string(),
+      artifact: z7.unknown().nullable()
     })
   }).parse(await response.json());
   if (!body.data.artifact)
@@ -12478,46 +12733,46 @@ function outputMismatch(name, output) {
 }
 
 // src/evals/workshop.ts
-import { z as z6 } from "zod";
-var WorkshopRunSchema = z6.object({
-  id: z6.string(),
-  event_id: z6.string().nullable(),
-  name: z6.string().nullable(),
-  event_name: z6.string().nullable(),
-  user_id: z6.string().nullable(),
-  convo_id: z6.string().nullable(),
-  started_at: z6.number().nullable(),
-  last_updated_at: z6.number().nullable(),
-  metadata: z6.string().nullable(),
-  model: z6.string().nullable(),
-  finished: z6.number().nullable()
+import { z as z8 } from "zod";
+var WorkshopRunSchema = z8.object({
+  id: z8.string(),
+  event_id: z8.string().nullable(),
+  name: z8.string().nullable(),
+  event_name: z8.string().nullable(),
+  user_id: z8.string().nullable(),
+  convo_id: z8.string().nullable(),
+  started_at: z8.number().nullable(),
+  last_updated_at: z8.number().nullable(),
+  metadata: z8.string().nullable(),
+  model: z8.string().nullable(),
+  finished: z8.number().nullable()
 });
-var WorkshopSpanSchema = z6.object({
-  id: z6.string(),
-  run_id: z6.string(),
-  parent_span_id: z6.string().nullable(),
-  name: z6.string(),
-  span_type: z6.string().nullable(),
-  status: z6.string().nullable(),
-  input_payload: z6.string().nullable(),
-  output_payload: z6.string().nullable(),
-  start_time_ms: z6.number().nullable(),
-  end_time_ms: z6.number().nullable(),
-  duration_ms: z6.number().nullable(),
-  model: z6.string().nullable(),
-  provider: z6.string().nullable(),
-  input_tokens: z6.number().nullable(),
-  output_tokens: z6.number().nullable(),
-  attributes: z6.string().nullable()
+var WorkshopSpanSchema = z8.object({
+  id: z8.string(),
+  run_id: z8.string(),
+  parent_span_id: z8.string().nullable(),
+  name: z8.string(),
+  span_type: z8.string().nullable(),
+  status: z8.string().nullable(),
+  input_payload: z8.string().nullable(),
+  output_payload: z8.string().nullable(),
+  start_time_ms: z8.number().nullable(),
+  end_time_ms: z8.number().nullable(),
+  duration_ms: z8.number().nullable(),
+  model: z8.string().nullable(),
+  provider: z8.string().nullable(),
+  input_tokens: z8.number().nullable(),
+  output_tokens: z8.number().nullable(),
+  attributes: z8.string().nullable()
 });
-var WorkshopTraceEnvelopeSchema = z6.object({
-  status: z6.literal("complete"),
-  correlationId: z6.string(),
-  traceId: z6.string(),
-  rootSpanId: z6.string(),
-  spanCount: z6.number().int().positive(),
+var WorkshopTraceEnvelopeSchema = z8.object({
+  status: z8.literal("complete"),
+  correlationId: z8.string(),
+  traceId: z8.string(),
+  rootSpanId: z8.string(),
+  spanCount: z8.number().int().positive(),
   run: WorkshopRunSchema,
-  spans: z6.array(WorkshopSpanSchema).min(1)
+  spans: z8.array(WorkshopSpanSchema).min(1)
 });
 async function readWorkshopTrace(input) {
   if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
@@ -12602,7 +12857,28 @@ function serializableOutput(value) {
 // src/evals/run.ts
 var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
 function createEvalSuiteRun(client, definition, options) {
+  var _a;
   const validated = validateDefinition(definition);
+  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  const session = createSuiteSession(client, validated, execution.run, options);
+  let finishing;
+  return {
+    id: session.id,
+    run: (selection) => session.run(selection),
+    finish: () => finishing != null ? finishing : finishing = finishWith(execution, () => session.finish())
+  };
+}
+function bindSuite(definition, parameters) {
+  if (definition.agent === void 0) {
+    const { run } = definition;
+    return {
+      run: (row) => Promise.resolve().then(() => run(row)),
+      finish: () => Promise.resolve()
+    };
+  }
+  return definition.agent.start(parameters);
+}
+function createSuiteSession(client, validated, run, options) {
   if (validated.datasetVersionId && "selection" in options && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
@@ -12630,6 +12906,7 @@ function createEvalSuiteRun(client, definition, options) {
     return new RaindropRowEvalRun(
       client,
       validated,
+      run,
       options.destination,
       options.selection.dataset,
       selection.rowIds,
@@ -12654,19 +12931,26 @@ function createEvalSuiteRun(client, definition, options) {
   return new WorkshopEvalRun(
     client,
     { ...validated, dataset: validated.dataset },
+    run,
     options.destination
   );
 }
 async function runEvalSuite(client, definition, options = {}) {
+  var _a;
+  const validated = validateDefinition(definition);
+  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  return finishWith(execution, () => runSuite(client, validated, execution.run, options));
+}
+async function runSuite(client, definition, run, options) {
   var _a, _b, _c, _d, _e, _f, _g, _h;
-  let validated = validateDefinition(definition);
+  let validated = definition;
   const destination = (_a = options.destination) != null ? _a : { kind: "raindrop" };
   if (validated.datasetVersionId && ((_b = options.selection) == null ? void 0 : _b.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
   if (destination.kind === "workshop") {
     if (options.runId !== void 0) throw new Error("Queued runs are only supported in Raindrop");
-    const session = createEvalSuiteRun(client, validated, { destination });
+    const session = createSuiteSession(client, validated, run, { destination });
     await session.run({ selection: options.selection, attempt: options.attempt });
     return session.finish();
   }
@@ -12700,7 +12984,7 @@ async function runEvalSuite(client, definition, options = {}) {
       rowIds: (_h = (_g = options.selection) == null ? void 0 : _g.rowIds) != null ? _h : dataset.rows.map((entry) => entry.id)
     });
     if (!selection) throw new Error("Raindrop remote eval requires selected rows");
-    const session = createEvalSuiteRun(client, validated, {
+    const session = createSuiteSession(client, validated, run, {
       destination,
       selection: { dataset, rowIds: selection.rowIds }
     });
@@ -12734,7 +13018,7 @@ async function runEvalSuite(client, definition, options = {}) {
     }
     return finished;
   }
-  return runRaindropEval(client, validated, destination, options);
+  return runRaindropEval(client, validated, run, destination, options);
 }
 function isRowScopedLocalEval(definition) {
   return definition.evaluators.every(
@@ -12742,9 +13026,10 @@ function isRowScopedLocalEval(definition) {
   );
 }
 var RaindropRowEvalRun = class {
-  constructor(client, definition, destination, dataset, rowIds, evaluators) {
+  constructor(client, definition, runRow, destination, dataset, rowIds, evaluators) {
     this.client = client;
     this.definition = definition;
+    this.runRow = runRow;
     this.destination = destination;
     this.dataset = dataset;
     this.rowIds = rowIds;
@@ -12794,7 +13079,7 @@ var RaindropRowEvalRun = class {
       const attemptResult = await replay2.run({
         rowId,
         attempt,
-        run: this.definition.run,
+        run: this.runRow,
         traceWaitMs: this.definition.traceWaitMs
       });
       const configured = new Map(
@@ -12899,9 +13184,10 @@ function replayEvaluatorsFromVerdicts(verdicts) {
   return [...evaluators.values()];
 }
 var WorkshopEvalRun = class {
-  constructor(client, definition, destination) {
+  constructor(client, definition, runRow, destination) {
     this.client = client;
     this.definition = definition;
+    this.runRow = runRow;
     this.destination = destination;
     this.id = randomId();
     this.startedAt = /* @__PURE__ */ new Date();
@@ -13031,7 +13317,7 @@ var WorkshopEvalRun = class {
             () => {
               var _a2;
               const parent = (_a2 = otelTrace.getSpan(context2.active())) == null ? void 0 : _a2.spanContext();
-              if (!parent) return this.definition.run(row);
+              if (!parent) return this.runRow(row);
               return withReplayTraceDestination(
                 {
                   ...destination,
@@ -13041,7 +13327,7 @@ var WorkshopEvalRun = class {
                     eventId: parent.traceId
                   }
                 },
-                () => this.definition.run(row)
+                () => this.runRow(row)
               );
             }
           )
@@ -13219,7 +13505,7 @@ var WorkshopEvalRun = class {
     return current;
   }
 };
-async function runRaindropEval(client, definition, destination, options) {
+async function runRaindropEval(client, definition, run, destination, options) {
   var _a, _b, _c, _d;
   if (typeof definition.dataset !== "string") {
     throw new Error(
@@ -13246,7 +13532,7 @@ async function runRaindropEval(client, definition, destination, options) {
     dataset: pinnedDataset ? datasetToWire(pinnedDataset) : datasetReference,
     evaluators: definition.evaluators.map((entry) => replayEvaluator(entry)),
     evaluatorPins: hostedEvaluatorPins(definition.evaluators),
-    run: definition.run,
+    run,
     concurrency: definition.concurrency,
     traceWaitMs: definition.traceWaitMs,
     evalWaitMs: definition.evalWaitMs,
@@ -14920,6 +15206,9 @@ export {
   runReplay,
   replay,
   evaluateReplay,
+  EvalParameterError,
+  defineAgent,
+  createEvalHandler,
   EvalDatasetRowInputSchema,
   readEvalDataset2 as readEvalDataset,
   EvalDatasetPublishConflictError,