raindrop-ai 0.7.1 → 0.8.0

package/dist/index.js

@@ -3707,7 +3707,7 @@ var require_lodash = __commonJS({
           if (fromIndex < 0) {
             fromIndex = nativeMax(length + fromIndex, 0);
           }
-          return isString(collection) ? fromIndex <= length && collection.indexOf(value, fromIndex) > -1 : !!length && baseIndexOf(collection, value, fromIndex) > -1;
+          return isString2(collection) ? fromIndex <= length && collection.indexOf(value, fromIndex) > -1 : !!length && baseIndexOf(collection, value, fromIndex) > -1;
         }
         var invokeMap = baseRest(function(collection, path, args) {
           var index = -1, isFunc = typeof path == "function", result2 = isArrayLike(collection) ? Array2(collection.length) : [];
@@ -3775,7 +3775,7 @@ var require_lodash = __commonJS({
             return 0;
           }
           if (isArrayLike(collection)) {
-            return isString(collection) ? stringSize(collection) : collection.length;
+            return isString2(collection) ? stringSize(collection) : collection.length;
           }
           var tag = getTag(collection);
           if (tag == mapTag || tag == setTag) {
@@ -4101,7 +4101,7 @@ var require_lodash = __commonJS({
         function isArrayLikeObject(value) {
           return isObjectLike(value) && isArrayLike(value);
         }
-        function isBoolean(value) {
+        function isBoolean2(value) {
           return value === true || value === false || isObjectLike(value) && baseGetTag(value) == boolTag;
         }
         var isBuffer = nativeIsBuffer || stubFalse;
@@ -4177,7 +4177,7 @@ var require_lodash = __commonJS({
           return baseIsMatch(object, source, getMatchData(source), customizer);
         }
         function isNaN(value) {
-          return isNumber(value) && value != +value;
+          return isNumber2(value) && value != +value;
         }
         function isNative(value) {
           if (isMaskable(value)) {
@@ -4191,7 +4191,7 @@ var require_lodash = __commonJS({
         function isNil(value) {
           return value == null;
         }
-        function isNumber(value) {
+        function isNumber2(value) {
           return typeof value == "number" || isObjectLike(value) && baseGetTag(value) == numberTag;
         }
         function isPlainObject(value) {
@@ -4210,7 +4210,7 @@ var require_lodash = __commonJS({
           return isInteger(value) && value >= -MAX_SAFE_INTEGER && value <= MAX_SAFE_INTEGER;
         }
         var isSet = nodeIsSet ? baseUnary(nodeIsSet) : baseIsSet;
-        function isString(value) {
+        function isString2(value) {
           return typeof value == "string" || !isArray(value) && isObjectLike(value) && baseGetTag(value) == stringTag;
         }
         function isSymbol(value) {
@@ -4235,7 +4235,7 @@ var require_lodash = __commonJS({
             return [];
           }
           if (isArrayLike(value)) {
-            return isString(value) ? stringToArray(value) : copyArray(value);
+            return isString2(value) ? stringToArray(value) : copyArray(value);
           }
           if (symIterator && value[symIterator]) {
             return iteratorToArray(value[symIterator]());
@@ -5243,7 +5243,7 @@ var require_lodash = __commonJS({
         lodash.isArrayBuffer = isArrayBuffer;
         lodash.isArrayLike = isArrayLike;
         lodash.isArrayLikeObject = isArrayLikeObject;
-        lodash.isBoolean = isBoolean;
+        lodash.isBoolean = isBoolean2;
         lodash.isBuffer = isBuffer;
         lodash.isDate = isDate;
         lodash.isElement = isElement;
@@ -5262,14 +5262,14 @@ var require_lodash = __commonJS({
         lodash.isNative = isNative;
         lodash.isNil = isNil;
         lodash.isNull = isNull;
-        lodash.isNumber = isNumber;
+        lodash.isNumber = isNumber2;
         lodash.isObject = isObject;
         lodash.isObjectLike = isObjectLike;
         lodash.isPlainObject = isPlainObject;
         lodash.isRegExp = isRegExp;
         lodash.isSafeInteger = isSafeInteger;
         lodash.isSet = isSet;
-        lodash.isString = isString;
+        lodash.isString = isString2;
         lodash.isSymbol = isSymbol;
         lodash.isTypedArray = isTypedArray;
         lodash.isUndefined = isUndefined;
@@ -5794,8 +5794,7 @@ function defineDataset(dataset) {
   if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
   if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
   if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.rows.length === 0)
-    throw new Error("Raindrop eval datasets require at least one row");
+  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
   const ids = rows.map((entry) => entry.id);
   if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
   if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
@@ -5878,7 +5877,10 @@ function defineEvalSuite(definition) {
     if (typeof dataset !== "string")
       throw new Error("A dataset version pin requires a published dataset slug");
   }
-  if (typeof definition.run !== "function") {
+  if (definition.agent !== void 0 && definition.run !== void 0) {
+    throw new Error("An eval suite accepts either agent or run, not both");
+  }
+  if (definition.agent === void 0 && typeof definition.run !== "function") {
     throw new Error("Raindrop eval suite run must be a function");
   }
   if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
@@ -7003,13 +7005,13 @@ function rowTracePair(pair) {
   const { caseId, ...evidence } = pair;
   return { ...evidence, rowId: caseId };
 }
-var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod7, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
+var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod9, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
 var init_portable_runtime = __esm({
   "src/evals/portable-runtime.ts"() {
     "use strict";
     import_quickjs_singlefile_cjs_release_sync = __toESM(require("@jitl/quickjs-singlefile-cjs-release-sync"));
     import_quickjs_emscripten_core = require("quickjs-emscripten-core");
-    import_zod7 = require("zod");
+    import_zod9 = require("zod");
     init_portable();
     DEFAULT_MEMORY_BYTES = 32 * 1024 * 1024;
     DEFAULT_STACK_BYTES = 64 * 1024;
@@ -7019,11 +7021,11 @@ var init_portable_runtime = __esm({
     DEFAULT_JUDGE_CONCURRENCY = 4;
     DEFAULT_MAX_INPUT_BYTES = 4 * 1024 * 1024;
     DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
-    JudgeRequestSchema = import_zod7.z.object({
-      type: import_zod7.z.literal("judge_request"),
-      requestId: import_zod7.z.string(),
-      traceId: import_zod7.z.string(),
-      rubric: import_zod7.z.string()
+    JudgeRequestSchema = import_zod9.z.object({
+      type: import_zod9.z.literal("judge_request"),
+      requestId: import_zod9.z.string(),
+      traceId: import_zod9.z.string(),
+      rubric: import_zod9.z.string()
     });
   }
 });
@@ -7135,67 +7137,67 @@ function canonicalJson2(value) {
   if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
   return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson2(entry)}`).join(",")}}`;
 }
-var import_zod8, PortableEvalArtifactSchema, PortableEvalResultSchema;
+var import_zod10, PortableEvalArtifactSchema, PortableEvalResultSchema;
 var init_portable = __esm({
   "src/evals/portable.ts"() {
     "use strict";
-    import_zod8 = require("zod");
+    import_zod10 = require("zod");
     init_definition();
     init_trace();
-    PortableEvalArtifactSchema = import_zod8.z.object({
-      format: import_zod8.z.literal("raindrop.eval-program"),
-      formatVersion: import_zod8.z.literal(1),
-      runtimeAbi: import_zod8.z.literal("raindrop-eval-v2"),
-      evaluator: import_zod8.z.object({
-        id: import_zod8.z.string().uuid(),
-        slug: import_zod8.z.string().min(1),
-        programVersion: import_zod8.z.number().int().positive(),
-        executionMode: import_zod8.z.enum(["deterministic", "judge"]),
-        outputType: import_zod8.z.enum(["boolean", "score", "number"]),
-        scope: import_zod8.z.literal("batch")
+    PortableEvalArtifactSchema = import_zod10.z.object({
+      format: import_zod10.z.literal("raindrop.eval-program"),
+      formatVersion: import_zod10.z.literal(1),
+      runtimeAbi: import_zod10.z.literal("raindrop-eval-v2"),
+      evaluator: import_zod10.z.object({
+        id: import_zod10.z.string().uuid(),
+        slug: import_zod10.z.string().min(1),
+        programVersion: import_zod10.z.number().int().positive(),
+        executionMode: import_zod10.z.enum(["deterministic", "judge"]),
+        outputType: import_zod10.z.enum(["boolean", "score", "number"]),
+        scope: import_zod10.z.literal("batch")
       }).strict(),
-      traceProjectionVersion: import_zod8.z.literal(1),
-      traceProjectionSha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
-      source: import_zod8.z.string().min(1),
-      sha256: import_zod8.z.string().regex(/^[a-f0-9]{64}$/)
+      traceProjectionVersion: import_zod10.z.literal(1),
+      traceProjectionSha256: import_zod10.z.string().regex(/^[a-f0-9]{64}$/),
+      source: import_zod10.z.string().min(1),
+      sha256: import_zod10.z.string().regex(/^[a-f0-9]{64}$/)
     }).strict();
-    PortableEvalResultSchema = import_zod8.z.object({
-      outcomes: import_zod8.z.array(
-        import_zod8.z.union([
-          import_zod8.z.object({
-            state: import_zod8.z.literal("graded"),
-            traceId: import_zod8.z.string(),
-            pass: import_zod8.z.boolean(),
-            note: import_zod8.z.string().optional()
+    PortableEvalResultSchema = import_zod10.z.object({
+      outcomes: import_zod10.z.array(
+        import_zod10.z.union([
+          import_zod10.z.object({
+            state: import_zod10.z.literal("graded"),
+            traceId: import_zod10.z.string(),
+            pass: import_zod10.z.boolean(),
+            note: import_zod10.z.string().optional()
           }),
-          import_zod8.z.object({
-            state: import_zod8.z.literal("graded"),
-            traceId: import_zod8.z.string(),
+          import_zod10.z.object({
+            state: import_zod10.z.literal("graded"),
+            traceId: import_zod10.z.string(),
             score: EvalScoreSchema,
-            note: import_zod8.z.string().optional()
+            note: import_zod10.z.string().optional()
           }),
-          import_zod8.z.object({
-            state: import_zod8.z.literal("graded"),
-            traceId: import_zod8.z.string(),
-            value: import_zod8.z.number().finite(),
-            note: import_zod8.z.string().optional()
+          import_zod10.z.object({
+            state: import_zod10.z.literal("graded"),
+            traceId: import_zod10.z.string(),
+            value: import_zod10.z.number().finite(),
+            note: import_zod10.z.string().optional()
           }),
-          import_zod8.z.object({ state: import_zod8.z.literal("errored"), traceId: import_zod8.z.string(), message: import_zod8.z.string() }),
-          import_zod8.z.object({
-            state: import_zod8.z.literal("ungraded"),
-            traceId: import_zod8.z.string(),
-            reason: import_zod8.z.enum(["not sampled", "trace unavailable", "content unreadable"])
+          import_zod10.z.object({ state: import_zod10.z.literal("errored"), traceId: import_zod10.z.string(), message: import_zod10.z.string() }),
+          import_zod10.z.object({
+            state: import_zod10.z.literal("ungraded"),
+            traceId: import_zod10.z.string(),
+            reason: import_zod10.z.enum(["not sampled", "trace unavailable", "content unreadable"])
           })
         ])
       ),
-      failures: import_zod8.z.array(import_zod8.z.string()),
-      stats: import_zod8.z.object({
-        traceCount: import_zod8.z.number().int().nonnegative(),
-        hydratedCount: import_zod8.z.number().int().nonnegative(),
-        judgeCalls: import_zod8.z.number().int().nonnegative(),
-        durationMs: import_zod8.z.number().nonnegative(),
-        skippedRows: import_zod8.z.number().int().nonnegative(),
-        skippedReason: import_zod8.z.literal("no trace").nullable()
+      failures: import_zod10.z.array(import_zod10.z.string()),
+      stats: import_zod10.z.object({
+        traceCount: import_zod10.z.number().int().nonnegative(),
+        hydratedCount: import_zod10.z.number().int().nonnegative(),
+        judgeCalls: import_zod10.z.number().int().nonnegative(),
+        durationMs: import_zod10.z.number().nonnegative(),
+        skippedRows: import_zod10.z.number().int().nonnegative(),
+        skippedReason: import_zod10.z.literal("no trace").nullable()
       })
     });
   }
@@ -7212,6 +7214,7 @@ __export(index_exports, {
   EVAL_CORRELATION_ID_ATTRIBUTE: () => EVAL_CORRELATION_ID_ATTRIBUTE2,
   EvalDatasetPublishConflictError: () => EvalDatasetPublishConflictError,
   EvalDatasetRowInputSchema: () => EvalDatasetRowInputSchema,
+  EvalParameterError: () => EvalParameterError,
   EvalPublishError: () => EvalPublishError,
   LocalBooleanVerdictSchema: () => LocalBooleanVerdictSchema,
   LocalNumberVerdictSchema: () => LocalNumberVerdictSchema,
@@ -7230,10 +7233,12 @@ __export(index_exports, {
   TraceSchema: () => TraceSchema,
   claimReplay: () => claimReplay,
   compareEvalRuns: () => compareEvalRuns,
+  createEvalHandler: () => createEvalHandler,
   createEvalSuiteRun: () => createEvalSuiteRun,
   createReplay: () => createReplay,
   currentEvalScope: () => currentEvalScope2,
   default: () => index_default,
+  defineAgent: () => defineAgent,
   defineDataset: () => defineDataset,
   defineEvalSuite: () => defineEvalSuite,
   defineEvaluatorProgram: () => defineEvaluatorProgram,
@@ -9364,15 +9369,15 @@ var makeIssue = (params) => {
       message: issueData.message
     };
   }
-  let errorMessage = "";
+  let errorMessage2 = "";
   const maps = errorMaps.filter((m) => !!m).slice().reverse();
   for (const map of maps) {
-    errorMessage = map(fullIssue, { data, defaultError: errorMessage }).message;
+    errorMessage2 = map(fullIssue, { data, defaultError: errorMessage2 }).message;
   }
   return {
     ...issueData,
     path: fullPath,
-    message: errorMessage
+    message: errorMessage2
   };
 };
 var EMPTY_PATH = [];
@@ -12995,7 +13000,7 @@ var SignalEventSchema = external_exports.object({
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.7.1",
+  version: "0.8.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",
@@ -15696,9 +15701,9 @@ var LiveInteraction = class {
       span.setAttribute("traceloop.entity.duration_ms", durationMs);
     }
     if (error) {
-      const errorMessage = error instanceof Error ? error.message : String(error);
-      span.setStatus({ code: import_api4.SpanStatusCode.ERROR, message: errorMessage });
-      span.recordException(error instanceof Error ? error : new Error(errorMessage));
+      const errorMessage2 = error instanceof Error ? error.message : String(error);
+      span.setStatus({ code: import_api4.SpanStatusCode.ERROR, message: errorMessage2 });
+      span.recordException(error instanceof Error ? error : new Error(errorMessage2));
     } else {
       span.setStatus({ code: import_api4.SpanStatusCode.OK });
     }
@@ -15857,9 +15862,9 @@ var LiveTracer = class {
       span.setAttribute("traceloop.entity.duration_ms", durationMs);
     }
     if (error) {
-      const errorMessage = error instanceof Error ? error.message : String(error);
-      span.setStatus({ code: import_api5.SpanStatusCode.ERROR, message: errorMessage });
-      span.recordException(error instanceof Error ? error : new Error(errorMessage));
+      const errorMessage2 = error instanceof Error ? error.message : String(error);
+      span.setStatus({ code: import_api5.SpanStatusCode.ERROR, message: errorMessage2 });
+      span.recordException(error instanceof Error ? error : new Error(errorMessage2));
     } else {
       span.setStatus({ code: import_api5.SpanStatusCode.OK });
     }
@@ -17071,7 +17076,7 @@ function replayIngestEndpoint(value, trustedUrl) {
 
 // src/replay/index.ts
 var DEFAULT_QUERY_URL = "https://query.raindrop.ai";
-var DEFAULT_TRACE_WAIT_MS = 3e4;
+var DEFAULT_TRACE_WAIT_MS = 6e4;
 var DEFAULT_EVAL_WAIT_MS = 10 * 6e4;
 var DEFAULT_EVAL_POLL_INTERVAL_MS = 2e3;
 var MAX_EVAL_STATUS_RETRIES = 2;
@@ -17683,6 +17688,9 @@ function validateEvalDatasetManifest(value) {
 async function runReplay(client, handle, options) {
   return (await runReplayRows(client, handle, options)).reading;
 }
+async function runEvalReplay(client, handle, options) {
+  return runReplayRows(client, handle, options);
+}
 async function runReplayRows(client, handle, options) {
   var _a, _b, _c, _d, _e, _f, _g, _h;
   if (!Number.isSafeInteger((_a = options.concurrency) != null ? _a : DEFAULT_CONCURRENCY) || ((_b = options.concurrency) != null ? _b : DEFAULT_CONCURRENCY) < 1) {
@@ -17787,12 +17795,17 @@ async function runReplayRows(client, handle, options) {
       }
     }
   };
-  await Promise.all(
+  const workers = await Promise.allSettled(
     Array.from(
       { length: Math.min((_f = options.concurrency) != null ? _f : DEFAULT_CONCURRENCY, rows.length) },
       () => worker()
     )
   );
+  const failures = workers.flatMap(
+    (result) => result.status === "rejected" ? [result.reason] : []
+  );
+  if (failures.length === 1) throw failures[0];
+  if (failures.length > 1) throw new AggregateError(failures, "Replay row execution failed");
   await client.forceFlush();
   const traceEvidence = [];
   for (const row of rows) {
@@ -18776,6 +18789,253 @@ function describeError(cause) {
   return String(cause);
 }
 
+// src/evals/agent.ts
+var import_zod6 = require("zod");
+var EvalParameterError = class extends Error {
+  constructor(agent, issues) {
+    super(
+      `Invalid parameters for agent ${agent}: ${issues.map((issue) => `${issue.path.join(".") || "parameters"} ${issue.message}`).join("; ")}`
+    );
+    this.name = "EvalParameterError";
+  }
+};
+var AgentMetadataSchema = import_zod6.z.object({
+  slug: import_zod6.z.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
+  name: import_zod6.z.string().trim().min(1).max(120),
+  description: import_zod6.z.string().max(2e3).optional()
+});
+var PARAMETER_KEY = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/;
+var MAX_PARAMETERS = 50;
+function unwrapField(schema) {
+  let inner = schema;
+  let description;
+  while (inner instanceof import_zod6.z.ZodDefault || inner instanceof import_zod6.z.ZodOptional) {
+    description != null ? description : description = inner.description;
+    inner = inner._def.innerType;
+  }
+  const omitted = schema.safeParse(void 0);
+  return {
+    inner,
+    required: !omitted.success,
+    defaultValue: omitted.success ? omitted.data : void 0,
+    description: description != null ? description : inner.description
+  };
+}
+function fieldDefault(key, value, guard) {
+  if (value === void 0) return void 0;
+  if (!guard(value)) throw new Error(`Parameter ${key} has a default of the wrong type`);
+  return value;
+}
+var isString = (value) => typeof value === "string";
+var isNumber = (value) => typeof value === "number";
+var isBoolean = (value) => typeof value === "boolean";
+function describeParameterField(key, schema) {
+  if (!PARAMETER_KEY.test(key)) throw new Error(`Parameter key ${key} must be an identifier`);
+  const { inner, required, defaultValue, description } = unwrapField(schema);
+  const base = { key, required, ...description === void 0 ? {} : { description } };
+  if (inner instanceof import_zod6.z.ZodString) {
+    const max = inner._def.checks.find((check) => check.kind === "max");
+    return {
+      ...base,
+      type: "string",
+      default: fieldDefault(key, defaultValue, isString),
+      ...(max == null ? void 0 : max.kind) === "max" ? { maxLength: max.value } : {}
+    };
+  }
+  if (inner instanceof import_zod6.z.ZodNumber) {
+    const { minValue, maxValue } = inner;
+    return {
+      ...base,
+      type: "number",
+      default: fieldDefault(key, defaultValue, isNumber),
+      integer: inner.isInt,
+      ...minValue === null ? {} : { min: minValue },
+      ...maxValue === null ? {} : { max: maxValue }
+    };
+  }
+  if (inner instanceof import_zod6.z.ZodBoolean) {
+    return { ...base, type: "boolean", default: fieldDefault(key, defaultValue, isBoolean) };
+  }
+  if (inner instanceof import_zod6.z.ZodEnum) {
+    const options = inner.options;
+    return { ...base, type: "enum", default: fieldDefault(key, defaultValue, isString), options };
+  }
+  throw new Error(
+    `Parameter ${key} must be a z.string(), z.number(), z.boolean(), or z.enum() schema`
+  );
+}
+function defineAgent(definition) {
+  var _a, _b;
+  const { slug, name, description } = AgentMetadataSchema.parse({
+    slug: definition.slug,
+    name: (_a = definition.name) != null ? _a : definition.slug,
+    description: definition.description
+  });
+  if (typeof definition.run !== "function") throw new Error("An agent requires a run function");
+  const fields = (_b = definition.parameters) != null ? _b : {};
+  const entries = Object.entries(fields);
+  if (entries.length > MAX_PARAMETERS)
+    throw new Error(`Agent ${slug} declares more than ${MAX_PARAMETERS} parameters`);
+  const parameters = entries.map(([key, schema2]) => describeParameterField(key, schema2));
+  const schema = import_zod6.z.object(fields).strict();
+  return {
+    slug,
+    name,
+    description,
+    parameters,
+    start(input) {
+      const parsed = schema.safeParse(input);
+      if (!parsed.success) throw new EvalParameterError(slug, parsed.error.issues);
+      return startAgent(definition, parsed.data);
+    }
+  };
+}
+async function finishWith(execution, work) {
+  let result;
+  try {
+    result = await work();
+  } catch (error) {
+    await execution.finish().catch(() => void 0);
+    throw error;
+  }
+  await execution.finish();
+  return result;
+}
+function startAgent(definition, parameters) {
+  const active = /* @__PURE__ */ new Set();
+  let finishing;
+  const execution = (runRow, cleanup2) => ({
+    run(row) {
+      if (finishing !== void 0)
+        return Promise.reject(new Error(`Agent ${definition.slug} has already finished`));
+      const pending = runRow(row);
+      active.add(pending);
+      pending.catch(() => void 0).finally(() => active.delete(pending));
+      return pending;
+    },
+    finish: () => finishing != null ? finishing : finishing = Promise.allSettled(active).then(cleanup2)
+  });
+  if (definition.setup === void 0) {
+    const { run: run2, cleanup: cleanup2 } = definition;
+    return execution(
+      (row) => Promise.resolve().then(() => run2(row, { parameters })),
+      async () => {
+        await (cleanup2 == null ? void 0 : cleanup2());
+      }
+    );
+  }
+  const { setup, run, cleanup } = definition;
+  let environment2;
+  const ready = () => {
+    if (environment2 === void 0) {
+      environment2 = Promise.resolve().then(() => setup(parameters));
+      environment2.catch(() => void 0);
+    }
+    return environment2;
+  };
+  return execution(
+    async (row) => run(row, { parameters, environment: await ready() }),
+    () => environment2 === void 0 ? Promise.resolve() : environment2.then(
+      (built) => cleanup == null ? void 0 : cleanup(built),
+      () => void 0
+    )
+  );
+}
+
+// src/evals/server.ts
+var import_zod7 = require("zod");
+var InvocationSchema = import_zod7.z.object({
+  runId: import_zod7.z.string().uuid(),
+  agent: import_zod7.z.string().min(1).max(64),
+  parameters: import_zod7.z.record(import_zod7.z.unknown()).default({})
+});
+var MAX_BODY_BYTES = 64 * 1024;
+async function parseInvocation(request) {
+  var _a;
+  if (Number((_a = request.headers.get("content-length")) != null ? _a : 0) > MAX_BODY_BYTES) return void 0;
+  const body = await request.text().catch(() => "");
+  if (new TextEncoder().encode(body).byteLength > MAX_BODY_BYTES) return void 0;
+  try {
+    const parsed = InvocationSchema.safeParse(JSON.parse(body));
+    return parsed.success ? parsed.data : void 0;
+  } catch (e) {
+    return void 0;
+  }
+}
+function startExecution(agent, parameters) {
+  try {
+    return agent.start(parameters);
+  } catch (error) {
+    if (error instanceof EvalParameterError)
+      return Response.json({ error: error.message }, { status: 400 });
+    throw error;
+  }
+}
+function errorMessage(error) {
+  return error instanceof Error ? error.message : "Eval execution failed";
+}
+function createEvalHandler(client, options) {
+  if (options.queryUrl !== void 0) credentialEndpoint(options.queryUrl);
+  if (options.replayIngestUrl !== void 0) credentialEndpoint(options.replayIngestUrl);
+  const agents = new Map(options.agents.map((agent) => [agent.slug, agent]));
+  if (agents.size !== options.agents.length)
+    throw new Error("Registered agent slugs must be unique");
+  const catalog = [...agents.values()].map(({ slug, name, description, parameters }) => ({
+    slug,
+    name,
+    description,
+    parameters
+  }));
+  return async (request) => {
+    if (request.method === "GET") return Response.json({ agents: catalog });
+    if (request.method !== "POST") {
+      return Response.json(
+        { error: "Method not allowed" },
+        { status: 405, headers: { Allow: "GET, POST" } }
+      );
+    }
+    const invocation = await parseInvocation(request);
+    if (invocation === void 0) {
+      return Response.json({ error: "Invalid eval request" }, { status: 400 });
+    }
+    const { runId, parameters } = invocation;
+    const agent = agents.get(invocation.agent);
+    if (agent === void 0) return Response.json({ error: "Unknown agent" }, { status: 404 });
+    const execution = startExecution(agent, parameters);
+    if (execution instanceof Response) return execution;
+    try {
+      const { replay: replay2 } = await readReplayDetails(client, runId, { queryUrl: options.queryUrl });
+      if (replay2.status === "complete" || replay2.status === "failed") {
+        return Response.json({ runId, status: replay2.status, counts: replay2.counts });
+      }
+      if (replay2.status !== "queued") {
+        return Response.json({ error: "Run is already claimed" }, { status: 409 });
+      }
+      const handle = await claimReplay(client, runId, { queryUrl: options.queryUrl });
+      if (handle.replayId !== runId) throw new Error("Raindrop claimed a different run");
+      if (handle.ingestUrl === null) {
+        return Response.json({ error: "Run is already claimed" }, { status: 409 });
+      }
+      const { reading, traceEvidence } = await finishWith(
+        execution,
+        () => runEvalReplay(client, handle, {
+          run: execution.run,
+          queryUrl: options.queryUrl,
+          replayIngestUrl: options.replayIngestUrl
+        })
+      );
+      return Response.json({
+        runId: reading.replayId,
+        status: reading.status,
+        counts: reading.counts,
+        traceEvidence
+      });
+    } catch (error) {
+      return Response.json({ error: errorMessage(error) }, { status: 500 });
+    }
+  };
+}
+
 // src/evals/index.ts
 init_definition();
 
@@ -18929,7 +19189,7 @@ function sleep2(ms) {
 }
 
 // src/evals/publish.ts
-var import_zod6 = require("zod");
+var import_zod8 = require("zod");
 init_definition();
 
 // src/evals/runs.ts
@@ -18979,19 +19239,19 @@ async function evaluateEvalRun(client, runId, options) {
 }
 
 // src/evals/publish.ts
-var EvaluatorIdentitySchema = import_zod6.z.object({
-  id: import_zod6.z.string().uuid(),
-  slug: import_zod6.z.string(),
-  programVersion: import_zod6.z.number().int().positive(),
-  outputType: import_zod6.z.enum(["boolean", "score", "number"])
+var EvaluatorIdentitySchema = import_zod8.z.object({
+  id: import_zod8.z.string().uuid(),
+  slug: import_zod8.z.string(),
+  programVersion: import_zod8.z.number().int().positive(),
+  outputType: import_zod8.z.enum(["boolean", "score", "number"])
 });
 var EvaluatorSourceSchema = EvaluatorIdentitySchema.extend({
-  name: import_zod6.z.string(),
-  description: import_zod6.z.string().nullable(),
-  intent: import_zod6.z.string(),
-  kind: import_zod6.z.enum(["judge", "deterministic"]),
-  programSource: import_zod6.z.string(),
-  rules: import_zod6.z.array(import_zod6.z.string())
+  name: import_zod8.z.string(),
+  description: import_zod8.z.string().nullable(),
+  intent: import_zod8.z.string(),
+  kind: import_zod8.z.enum(["judge", "deterministic"]),
+  programSource: import_zod8.z.string(),
+  rules: import_zod8.z.array(import_zod8.z.string())
 });
 var EvalPublishError = class extends Error {
   constructor(status, message) {
@@ -19000,12 +19260,12 @@ var EvalPublishError = class extends Error {
     this.name = "EvalPublishError";
   }
 };
-var ComparisonRunSchema = import_zod6.z.object({
-  id: import_zod6.z.string().uuid(),
-  programVersion: import_zod6.z.number().int().positive()
+var ComparisonRunSchema = import_zod8.z.object({
+  id: import_zod8.z.string().uuid(),
+  programVersion: import_zod8.z.number().int().positive()
 });
-var ComparisonLinkSchema = import_zod6.z.object({
-  url: import_zod6.z.string().url(),
+var ComparisonLinkSchema = import_zod8.z.object({
+  url: import_zod8.z.string().url(),
   first: ComparisonRunSchema,
   second: ComparisonRunSchema
 });
@@ -19033,7 +19293,7 @@ async function compareEvalRuns(client, options) {
   }
   const before = selected.id;
   const after = counterpart.id;
-  const { data } = import_zod6.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
+  const { data } = import_zod8.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
   if (data.first.id !== before || data.second.id !== after) {
     throw new Error("Raindrop returned a different comparison pair");
   }
@@ -19046,7 +19306,7 @@ async function compareEvalRuns(client, options) {
   };
 }
 async function readEvaluator(client, slug, options = {}) {
-  const { data } = import_zod6.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
+  const { data } = import_zod8.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
   if (data.slug !== slug) throw new Error("Raindrop returned a different evaluator slug");
   return defineEvaluatorProgram({
     slug: data.slug,
@@ -19172,34 +19432,34 @@ init_trace();
 init_portable();
 
 // src/evals/pull.ts
-var import_zod9 = require("zod");
+var import_zod11 = require("zod");
 init_definition();
 init_portable();
 var SNAPSHOT_VERSION = 1;
-var PulledEvaluatorSchema = import_zod9.z.object({
-  slug: import_zod9.z.string().min(1),
-  name: import_zod9.z.string().min(1),
-  expectedSha256: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
-  artifact: import_zod9.z.unknown()
+var PulledEvaluatorSchema = import_zod11.z.object({
+  slug: import_zod11.z.string().min(1),
+  name: import_zod11.z.string().min(1),
+  expectedSha256: import_zod11.z.string().regex(/^[a-f0-9]{64}$/),
+  artifact: import_zod11.z.unknown()
 });
-var EvalSnapshotSchema = import_zod9.z.object({
-  schemaVersion: import_zod9.z.literal(SNAPSHOT_VERSION),
-  pulledAt: import_zod9.z.string().datetime(),
-  dataset: import_zod9.z.object({
-    reference: import_zod9.z.string().min(1),
-    datasetId: import_zod9.z.string().uuid(),
-    fingerprint: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
-    rows: import_zod9.z.array(
-      import_zod9.z.object({
-        id: import_zod9.z.string(),
-        name: import_zod9.z.string(),
-        input: import_zod9.z.string().nullable(),
-        output: import_zod9.z.string().nullable(),
-        properties: import_zod9.z.record(import_zod9.z.string())
+var EvalSnapshotSchema = import_zod11.z.object({
+  schemaVersion: import_zod11.z.literal(SNAPSHOT_VERSION),
+  pulledAt: import_zod11.z.string().datetime(),
+  dataset: import_zod11.z.object({
+    reference: import_zod11.z.string().min(1),
+    datasetId: import_zod11.z.string().uuid(),
+    fingerprint: import_zod11.z.string().regex(/^[a-f0-9]{64}$/),
+    rows: import_zod11.z.array(
+      import_zod11.z.object({
+        id: import_zod11.z.string(),
+        name: import_zod11.z.string(),
+        input: import_zod11.z.string().nullable(),
+        output: import_zod11.z.string().nullable(),
+        properties: import_zod11.z.record(import_zod11.z.string())
       })
     )
   }),
-  evaluators: import_zod9.z.array(PulledEvaluatorSchema)
+  evaluators: import_zod11.z.array(PulledEvaluatorSchema)
 });
 async function pullEval(client, options) {
   var _a;
@@ -19264,11 +19524,11 @@ async function pullEvaluator(client, queryUrl, rawSlug) {
       `Raindrop evaluator ${rawSlug} pull failed with ${response.status}: ${await response.text()}`
     );
   }
-  const body = import_zod9.z.object({
-    data: import_zod9.z.object({
-      slug: import_zod9.z.string(),
-      name: import_zod9.z.string(),
-      artifact: import_zod9.z.unknown().nullable()
+  const body = import_zod11.z.object({
+    data: import_zod11.z.object({
+      slug: import_zod11.z.string(),
+      name: import_zod11.z.string(),
+      artifact: import_zod11.z.unknown().nullable()
     })
   }).parse(await response.json());
   if (!body.data.artifact)
@@ -19405,46 +19665,46 @@ function outputMismatch(name, output) {
 init_portable();
 
 // src/evals/workshop.ts
-var import_zod10 = require("zod");
-var WorkshopRunSchema = import_zod10.z.object({
-  id: import_zod10.z.string(),
-  event_id: import_zod10.z.string().nullable(),
-  name: import_zod10.z.string().nullable(),
-  event_name: import_zod10.z.string().nullable(),
-  user_id: import_zod10.z.string().nullable(),
-  convo_id: import_zod10.z.string().nullable(),
-  started_at: import_zod10.z.number().nullable(),
-  last_updated_at: import_zod10.z.number().nullable(),
-  metadata: import_zod10.z.string().nullable(),
-  model: import_zod10.z.string().nullable(),
-  finished: import_zod10.z.number().nullable()
+var import_zod12 = require("zod");
+var WorkshopRunSchema = import_zod12.z.object({
+  id: import_zod12.z.string(),
+  event_id: import_zod12.z.string().nullable(),
+  name: import_zod12.z.string().nullable(),
+  event_name: import_zod12.z.string().nullable(),
+  user_id: import_zod12.z.string().nullable(),
+  convo_id: import_zod12.z.string().nullable(),
+  started_at: import_zod12.z.number().nullable(),
+  last_updated_at: import_zod12.z.number().nullable(),
+  metadata: import_zod12.z.string().nullable(),
+  model: import_zod12.z.string().nullable(),
+  finished: import_zod12.z.number().nullable()
 });
-var WorkshopSpanSchema = import_zod10.z.object({
-  id: import_zod10.z.string(),
-  run_id: import_zod10.z.string(),
-  parent_span_id: import_zod10.z.string().nullable(),
-  name: import_zod10.z.string(),
-  span_type: import_zod10.z.string().nullable(),
-  status: import_zod10.z.string().nullable(),
-  input_payload: import_zod10.z.string().nullable(),
-  output_payload: import_zod10.z.string().nullable(),
-  start_time_ms: import_zod10.z.number().nullable(),
-  end_time_ms: import_zod10.z.number().nullable(),
-  duration_ms: import_zod10.z.number().nullable(),
-  model: import_zod10.z.string().nullable(),
-  provider: import_zod10.z.string().nullable(),
-  input_tokens: import_zod10.z.number().nullable(),
-  output_tokens: import_zod10.z.number().nullable(),
-  attributes: import_zod10.z.string().nullable()
+var WorkshopSpanSchema = import_zod12.z.object({
+  id: import_zod12.z.string(),
+  run_id: import_zod12.z.string(),
+  parent_span_id: import_zod12.z.string().nullable(),
+  name: import_zod12.z.string(),
+  span_type: import_zod12.z.string().nullable(),
+  status: import_zod12.z.string().nullable(),
+  input_payload: import_zod12.z.string().nullable(),
+  output_payload: import_zod12.z.string().nullable(),
+  start_time_ms: import_zod12.z.number().nullable(),
+  end_time_ms: import_zod12.z.number().nullable(),
+  duration_ms: import_zod12.z.number().nullable(),
+  model: import_zod12.z.string().nullable(),
+  provider: import_zod12.z.string().nullable(),
+  input_tokens: import_zod12.z.number().nullable(),
+  output_tokens: import_zod12.z.number().nullable(),
+  attributes: import_zod12.z.string().nullable()
 });
-var WorkshopTraceEnvelopeSchema = import_zod10.z.object({
-  status: import_zod10.z.literal("complete"),
-  correlationId: import_zod10.z.string(),
-  traceId: import_zod10.z.string(),
-  rootSpanId: import_zod10.z.string(),
-  spanCount: import_zod10.z.number().int().positive(),
+var WorkshopTraceEnvelopeSchema = import_zod12.z.object({
+  status: import_zod12.z.literal("complete"),
+  correlationId: import_zod12.z.string(),
+  traceId: import_zod12.z.string(),
+  rootSpanId: import_zod12.z.string(),
+  spanCount: import_zod12.z.number().int().positive(),
   run: WorkshopRunSchema,
-  spans: import_zod10.z.array(WorkshopSpanSchema).min(1)
+  spans: import_zod12.z.array(WorkshopSpanSchema).min(1)
 });
 async function readWorkshopTrace(input) {
   if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
@@ -19529,7 +19789,28 @@ function serializableOutput(value) {
 // src/evals/run.ts
 var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
 function createEvalSuiteRun(client, definition, options) {
+  var _a;
   const validated = validateDefinition(definition);
+  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  const session = createSuiteSession(client, validated, execution.run, options);
+  let finishing;
+  return {
+    id: session.id,
+    run: (selection) => session.run(selection),
+    finish: () => finishing != null ? finishing : finishing = finishWith(execution, () => session.finish())
+  };
+}
+function bindSuite(definition, parameters) {
+  if (definition.agent === void 0) {
+    const { run } = definition;
+    return {
+      run: (row) => Promise.resolve().then(() => run(row)),
+      finish: () => Promise.resolve()
+    };
+  }
+  return definition.agent.start(parameters);
+}
+function createSuiteSession(client, validated, run, options) {
   if (validated.datasetVersionId && "selection" in options && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
@@ -19557,6 +19838,7 @@ function createEvalSuiteRun(client, definition, options) {
     return new RaindropRowEvalRun(
       client,
       validated,
+      run,
       options.destination,
       options.selection.dataset,
       selection.rowIds,
@@ -19581,19 +19863,26 @@ function createEvalSuiteRun(client, definition, options) {
   return new WorkshopEvalRun(
     client,
     { ...validated, dataset: validated.dataset },
+    run,
     options.destination
   );
 }
 async function runEvalSuite(client, definition, options = {}) {
+  var _a;
+  const validated = validateDefinition(definition);
+  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  return finishWith(execution, () => runSuite(client, validated, execution.run, options));
+}
+async function runSuite(client, definition, run, options) {
   var _a, _b, _c, _d, _e, _f, _g, _h;
-  let validated = validateDefinition(definition);
+  let validated = definition;
   const destination = (_a = options.destination) != null ? _a : { kind: "raindrop" };
   if (validated.datasetVersionId && ((_b = options.selection) == null ? void 0 : _b.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
   if (destination.kind === "workshop") {
     if (options.runId !== void 0) throw new Error("Queued runs are only supported in Raindrop");
-    const session = createEvalSuiteRun(client, validated, { destination });
+    const session = createSuiteSession(client, validated, run, { destination });
     await session.run({ selection: options.selection, attempt: options.attempt });
     return session.finish();
   }
@@ -19627,7 +19916,7 @@ async function runEvalSuite(client, definition, options = {}) {
       rowIds: (_h = (_g = options.selection) == null ? void 0 : _g.rowIds) != null ? _h : dataset.rows.map((entry) => entry.id)
     });
     if (!selection) throw new Error("Raindrop remote eval requires selected rows");
-    const session = createEvalSuiteRun(client, validated, {
+    const session = createSuiteSession(client, validated, run, {
       destination,
       selection: { dataset, rowIds: selection.rowIds }
     });
@@ -19661,7 +19950,7 @@ async function runEvalSuite(client, definition, options = {}) {
     }
     return finished;
   }
-  return runRaindropEval(client, validated, destination, options);
+  return runRaindropEval(client, validated, run, destination, options);
 }
 function isRowScopedLocalEval(definition) {
   return definition.evaluators.every(
@@ -19669,9 +19958,10 @@ function isRowScopedLocalEval(definition) {
   );
 }
 var RaindropRowEvalRun = class {
-  constructor(client, definition, destination, dataset, rowIds, evaluators) {
+  constructor(client, definition, runRow, destination, dataset, rowIds, evaluators) {
     this.client = client;
     this.definition = definition;
+    this.runRow = runRow;
     this.destination = destination;
     this.dataset = dataset;
     this.rowIds = rowIds;
@@ -19721,7 +20011,7 @@ var RaindropRowEvalRun = class {
       const attemptResult = await replay2.run({
         rowId,
         attempt,
-        run: this.definition.run,
+        run: this.runRow,
         traceWaitMs: this.definition.traceWaitMs
       });
       const configured = new Map(
@@ -19826,9 +20116,10 @@ function replayEvaluatorsFromVerdicts(verdicts) {
   return [...evaluators.values()];
 }
 var WorkshopEvalRun = class {
-  constructor(client, definition, destination) {
+  constructor(client, definition, runRow, destination) {
     this.client = client;
     this.definition = definition;
+    this.runRow = runRow;
     this.destination = destination;
     this.id = randomId();
     this.startedAt = /* @__PURE__ */ new Date();
@@ -19958,7 +20249,7 @@ var WorkshopEvalRun = class {
             () => {
               var _a2;
               const parent = (_a2 = import_api8.trace.getSpan(import_api8.context.active())) == null ? void 0 : _a2.spanContext();
-              if (!parent) return this.definition.run(row);
+              if (!parent) return this.runRow(row);
               return withReplayTraceDestination(
                 {
                   ...destination,
@@ -19968,7 +20259,7 @@ var WorkshopEvalRun = class {
                     eventId: parent.traceId
                   }
                 },
-                () => this.definition.run(row)
+                () => this.runRow(row)
               );
             }
           )
@@ -20146,7 +20437,7 @@ var WorkshopEvalRun = class {
     return current;
   }
 };
-async function runRaindropEval(client, definition, destination, options) {
+async function runRaindropEval(client, definition, run, destination, options) {
   var _a, _b, _c, _d;
   if (typeof definition.dataset !== "string") {
     throw new Error(
@@ -20173,7 +20464,7 @@ async function runRaindropEval(client, definition, destination, options) {
     dataset: pinnedDataset ? datasetToWire(pinnedDataset) : datasetReference,
     evaluators: definition.evaluators.map((entry) => replayEvaluator(entry)),
     evaluatorPins: hostedEvaluatorPins(definition.evaluators),
-    run: definition.run,
+    run,
     concurrency: definition.concurrency,
     traceWaitMs: definition.traceWaitMs,
     evalWaitMs: definition.evalWaitMs,
@@ -21833,6 +22124,7 @@ var withEvalScope2 = withEvalScope;
   EVAL_CORRELATION_ID_ATTRIBUTE,
   EvalDatasetPublishConflictError,
   EvalDatasetRowInputSchema,
+  EvalParameterError,
   EvalPublishError,
   LocalBooleanVerdictSchema,
   LocalNumberVerdictSchema,
@@ -21851,9 +22143,11 @@ var withEvalScope2 = withEvalScope;
   TraceSchema,
   claimReplay,
   compareEvalRuns,
+  createEvalHandler,
   createEvalSuiteRun,
   createReplay,
   currentEvalScope,
+  defineAgent,
   defineDataset,
   defineEvalSuite,
   defineEvaluatorProgram,