raindrop-ai 0.8.0 → 0.9.0

package/dist/evals/cli.js

@@ -12,10 +12,6 @@ var __esm = (fn, res) => function __init() {
 var __commonJS = (cb, mod) => function __require() {
   return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
 };
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
 var __copyProps = (to, from, except, desc) => {
   if (from && typeof from === "object" || typeof from === "function") {
     for (let key of __getOwnPropNames(from))
@@ -723,11 +719,11 @@ var require_lodash = __commonJS({
       function unicodeWords(string) {
         return string.match(reUnicodeWord) || [];
       }
-      var runInContext = (function runInContext2(context9) {
-        context9 = context9 == null ? root : _.defaults(root.Object(), context9, _.pick(root, contextProps));
-        var Array2 = context9.Array, Date2 = context9.Date, Error2 = context9.Error, Function2 = context9.Function, Math2 = context9.Math, Object2 = context9.Object, RegExp2 = context9.RegExp, String2 = context9.String, TypeError2 = context9.TypeError;
+      var runInContext = (function runInContext2(context8) {
+        context8 = context8 == null ? root : _.defaults(root.Object(), context8, _.pick(root, contextProps));
+        var Array2 = context8.Array, Date2 = context8.Date, Error2 = context8.Error, Function2 = context8.Function, Math2 = context8.Math, Object2 = context8.Object, RegExp2 = context8.RegExp, String2 = context8.String, TypeError2 = context8.TypeError;
         var arrayProto = Array2.prototype, funcProto = Function2.prototype, objectProto = Object2.prototype;
-        var coreJsData = context9["__core-js_shared__"];
+        var coreJsData = context8["__core-js_shared__"];
         var funcToString = funcProto.toString;
         var hasOwnProperty = objectProto.hasOwnProperty;
         var idCounter = 0;
@@ -741,7 +737,7 @@ var require_lodash = __commonJS({
         var reIsNative = RegExp2(
           "^" + funcToString.call(hasOwnProperty).replace(reRegExpChar, "\\$&").replace(/hasOwnProperty|(function).*?(?=\\\()| for .+?(?=\\\])/g, "$1.*?") + "$"
         );
-        var Buffer2 = moduleExports ? context9.Buffer : undefined2, Symbol2 = context9.Symbol, Uint8Array2 = context9.Uint8Array, allocUnsafe = Buffer2 ? Buffer2.allocUnsafe : undefined2, getPrototype = overArg(Object2.getPrototypeOf, Object2), objectCreate = Object2.create, propertyIsEnumerable = objectProto.propertyIsEnumerable, splice = arrayProto.splice, spreadableSymbol = Symbol2 ? Symbol2.isConcatSpreadable : undefined2, symIterator = Symbol2 ? Symbol2.iterator : undefined2, symToStringTag = Symbol2 ? Symbol2.toStringTag : undefined2;
+        var Buffer2 = moduleExports ? context8.Buffer : undefined2, Symbol2 = context8.Symbol, Uint8Array2 = context8.Uint8Array, allocUnsafe = Buffer2 ? Buffer2.allocUnsafe : undefined2, getPrototype = overArg(Object2.getPrototypeOf, Object2), objectCreate = Object2.create, propertyIsEnumerable = objectProto.propertyIsEnumerable, splice = arrayProto.splice, spreadableSymbol = Symbol2 ? Symbol2.isConcatSpreadable : undefined2, symIterator = Symbol2 ? Symbol2.iterator : undefined2, symToStringTag = Symbol2 ? Symbol2.toStringTag : undefined2;
         var defineProperty = (function() {
           try {
             var func = getNative(Object2, "defineProperty");
@@ -750,9 +746,9 @@ var require_lodash = __commonJS({
           } catch (e) {
           }
         })();
-        var ctxClearTimeout = context9.clearTimeout !== root.clearTimeout && context9.clearTimeout, ctxNow = Date2 && Date2.now !== root.Date.now && Date2.now, ctxSetTimeout = context9.setTimeout !== root.setTimeout && context9.setTimeout;
-        var nativeCeil = Math2.ceil, nativeFloor = Math2.floor, nativeGetSymbols = Object2.getOwnPropertySymbols, nativeIsBuffer = Buffer2 ? Buffer2.isBuffer : undefined2, nativeIsFinite = context9.isFinite, nativeJoin = arrayProto.join, nativeKeys = overArg(Object2.keys, Object2), nativeMax = Math2.max, nativeMin = Math2.min, nativeNow = Date2.now, nativeParseInt = context9.parseInt, nativeRandom = Math2.random, nativeReverse = arrayProto.reverse;
-        var DataView = getNative(context9, "DataView"), Map2 = getNative(context9, "Map"), Promise2 = getNative(context9, "Promise"), Set2 = getNative(context9, "Set"), WeakMap2 = getNative(context9, "WeakMap"), nativeCreate = getNative(Object2, "create");
+        var ctxClearTimeout = context8.clearTimeout !== root.clearTimeout && context8.clearTimeout, ctxNow = Date2 && Date2.now !== root.Date.now && Date2.now, ctxSetTimeout = context8.setTimeout !== root.setTimeout && context8.setTimeout;
+        var nativeCeil = Math2.ceil, nativeFloor = Math2.floor, nativeGetSymbols = Object2.getOwnPropertySymbols, nativeIsBuffer = Buffer2 ? Buffer2.isBuffer : undefined2, nativeIsFinite = context8.isFinite, nativeJoin = arrayProto.join, nativeKeys = overArg(Object2.keys, Object2), nativeMax = Math2.max, nativeMin = Math2.min, nativeNow = Date2.now, nativeParseInt = context8.parseInt, nativeRandom = Math2.random, nativeReverse = arrayProto.reverse;
+        var DataView = getNative(context8, "DataView"), Map2 = getNative(context8, "Map"), Promise2 = getNative(context8, "Promise"), Set2 = getNative(context8, "Set"), WeakMap2 = getNative(context8, "WeakMap"), nativeCreate = getNative(Object2, "create");
         var metaMap = WeakMap2 && new WeakMap2();
         var realNames = {};
         var dataViewCtorString = toSource(DataView), mapCtorString = toSource(Map2), promiseCtorString = toSource(Promise2), setCtorString = toSource(Set2), weakMapCtorString = toSource(WeakMap2);
@@ -5765,264 +5761,14 @@ var require_dist = __commonJS({
   }
 });
 
-// src/evals/definition.ts
-function defineDataset(dataset) {
-  var _a, _b;
-  const id = dataset.id.trim();
-  const name = dataset.name.trim();
-  const rows = dataset.rows.map((row) => {
-    var _a2;
-    const parsed = DatasetRowInputSchema.parse(row);
-    return {
-      id: parsed.id,
-      name: (_a2 = parsed.name) != null ? _a2 : parsed.id,
-      input: parsed.input,
-      output: parsed.output,
-      properties: parsed.properties
-    };
-  });
-  const version = (_b = (_a = dataset.version) == null ? void 0 : _a.trim()) != null ? _b : (0, import_node_crypto2.createHash)("sha256").update(
-    JSON.stringify(
-      rows.map((row) => ({
-        ...row,
-        properties: Object.fromEntries(
-          Object.keys(row.properties).sort().map((key) => [key, row.properties[key]])
-        )
-      }))
-    )
-  ).digest("hex");
-  if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
-  if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
-  if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
-  const ids = rows.map((entry) => entry.id);
-  if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
-  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
-  if (dataset.remote) {
-    if (!dataset.remote.reference.trim())
-      throw new Error("Raindrop remote dataset reference cannot be empty");
-    if (!/^[a-f0-9]{64}$/.test(dataset.remote.fingerprint)) {
-      throw new Error("Raindrop remote dataset fingerprint must be a SHA-256 digest");
-    }
-  }
-  const defined2 = {
-    ...dataset,
-    id,
-    name,
-    version,
-    rows,
-    [EVAL_DATASET]: true
-  };
-  Object.defineProperty(defined2, EVAL_DATASET, { enumerable: false });
-  return defined2;
-}
-function isEvalDataset(value) {
-  return typeof value === "object" && value !== null && EVAL_DATASET in value && value[EVAL_DATASET] === true;
-}
-function defineEvaluatorProgram(program) {
-  var _a, _b, _c, _d;
-  const slug = program.slug.trim();
-  const name = program.name.trim();
-  const source = program.source.trim();
-  const intent = program.intent.trim();
-  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug) || slug.length > 64) {
-    throw new Error(
-      "Raindrop evaluator program slugs must use lowercase letters, numbers, and single hyphens"
-    );
-  }
-  if (!name || name.length > 120) {
-    throw new Error("Raindrop evaluator program names must be between 1 and 120 characters");
-  }
-  if (!source) throw new Error(`Raindrop evaluator program ${slug} source cannot be empty`);
-  if (!intent || intent.length > 1e4) {
-    throw new Error(
-      `Raindrop evaluator program ${slug} intent must be between 1 and 10000 characters`
-    );
-  }
-  if (program.description != null && program.description.trim().length > 2e3) {
-    throw new Error(`Raindrop evaluator program ${slug} description exceeds 2000 characters`);
-  }
-  const rules = (_b = (_a = program.rules) == null ? void 0 : _a.map((rule) => rule.trim())) != null ? _b : [];
-  if (rules.length > 12 || rules.some((rule) => !rule || rule.length > 2e3)) {
-    throw new Error(`Raindrop evaluator program ${slug} rules are invalid`);
-  }
-  if (program.expected && (!import_zod2.z.string().uuid().safeParse(program.expected.evalId).success || !Number.isInteger(program.expected.programVersion) || program.expected.programVersion <= 0)) {
-    throw new Error(`Raindrop evaluator program ${slug} expected identity is invalid`);
-  }
-  return {
-    ...program,
-    kind: "program",
-    scope: import_zod2.z.literal("batch").parse((_c = program.scope) != null ? _c : "batch"),
-    slug,
-    name,
-    source,
-    intent,
-    description: ((_d = program.description) == null ? void 0 : _d.trim()) || null,
-    rules
-  };
-}
-function defineEvalSuite(definition) {
-  const name = definition.name.trim();
-  if (name.length === 0) throw new Error("Raindrop eval suite names cannot be empty");
-  const dataset = typeof definition.dataset === "string" ? definition.dataset.trim() : defineDataset(definition.dataset);
-  if (typeof dataset === "string" && dataset.length === 0) {
-    throw new Error("Raindrop eval suite datasets cannot be empty");
-  }
-  if (definition.datasetVersionId !== void 0) {
-    import_zod2.z.string().uuid().parse(definition.datasetVersionId);
-    if (typeof dataset !== "string")
-      throw new Error("A dataset version pin requires a published dataset slug");
-  }
-  if (definition.agent !== void 0 && definition.run !== void 0) {
-    throw new Error("An eval suite accepts either agent or run, not both");
-  }
-  if (definition.agent === void 0 && typeof definition.run !== "function") {
-    throw new Error("Raindrop eval suite run must be a function");
-  }
-  if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
-    throw new Error("Raindrop eval suites require at least one evaluator");
-  }
-  validateOptionalNumber("concurrency", definition.concurrency, true, false);
-  validateOptionalNumber("traceWaitMs", definition.traceWaitMs, false, true);
-  validateOptionalNumber("evalWaitMs", definition.evalWaitMs, false, true);
-  validateOptionalNumber("evalPollIntervalMs", definition.evalPollIntervalMs, false, false);
-  const evaluators = definition.evaluators.map((evaluator) => validateEvaluator(evaluator));
-  const evaluatorNames = evaluators.map((entry) => evaluatorName(entry));
-  if (new Set(evaluatorNames).size !== evaluatorNames.length) {
-    throw new Error("Raindrop eval suite evaluators must be unique");
-  }
-  const branded = {
-    ...definition,
-    [EVAL_DEFINITION]: true,
-    name,
-    dataset,
-    evaluators
-  };
-  Object.defineProperty(branded, EVAL_DEFINITION, { enumerable: false });
-  return branded;
-}
-function isEvalSuiteDefinition(value) {
-  return typeof value === "object" && value !== null && EVAL_DEFINITION in value && value[EVAL_DEFINITION] === true;
-}
-function isPublishedEvalSuite(definition) {
-  return typeof definition.dataset === "string" && definition.datasetVersionId !== void 0 && definition.evaluators.every(
-    ({ evaluator }) => typeof evaluator !== "string" && (!("kind" in evaluator) || evaluator.kind !== "program" || evaluator.expected !== void 0)
-  );
-}
-function evaluatorName(evaluator) {
-  return typeof evaluator.evaluator === "string" ? evaluator.evaluator.trim() : evaluator.evaluator.slug.trim();
-}
-function evaluatorOutput(evaluator) {
-  return typeof evaluator.evaluator === "string" ? void 0 : evaluator.evaluator.output;
-}
-function validateEvaluatorOutput(evaluator, rawOutput) {
-  const name = evaluatorName(evaluator);
-  const output = EvalOutputSchema.parse(rawOutput);
-  const localOutput = evaluatorOutput(evaluator);
-  if (localOutput !== void 0 && localOutput !== output) {
-    throw new Error(`Raindrop local evaluator ${name} expected ${localOutput}, received ${output}`);
-  }
-  switch (output) {
-    case "boolean":
-      parseBooleanThreshold(evaluator.threshold);
-      break;
-    case "score":
-      parseNumericThreshold(name, "score", evaluator.threshold);
-      break;
-    case "number":
-      parseNumericThreshold(name, "number", evaluator.threshold);
-      break;
-  }
-}
-function validateEvaluator(evaluator) {
-  if (typeof evaluator !== "object" || evaluator === null || !("evaluator" in evaluator)) {
-    throw new Error("Raindrop eval evaluator entries must name an evaluator");
-  }
-  if ("output" in evaluator) {
-    throw new Error(
-      "Raindrop eval evaluator entries do not accept output; the evaluator definition owns it"
-    );
-  }
-  if (typeof evaluator.evaluator !== "string" && (typeof evaluator.evaluator !== "object" || evaluator.evaluator === null || typeof evaluator.evaluator.slug !== "string" || typeof evaluator.evaluator.name !== "string")) {
-    throw new Error("Raindrop local eval evaluators require a slug, name, output, and judge");
-  }
-  const name = evaluatorName(evaluator);
-  if (name.length === 0) throw new Error("Raindrop eval evaluator names cannot be empty");
-  if (typeof evaluator.evaluator === "string") {
-    const threshold = evaluator.threshold;
-    if (threshold !== void 0) {
-      const boolean = BooleanThresholdSchema.safeParse(threshold);
-      const numeric = NumericThresholdSchema.safeParse(threshold);
-      if (!boolean.success && !numeric.success) {
-        throw new Error(`Raindrop hosted evaluator ${name} has an invalid threshold`);
-      }
-    }
-    return { ...evaluator, evaluator: name };
-  }
-  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "portable") {
-    validateEvaluatorOutput(evaluator, evaluator.evaluator.output);
-    return evaluator;
-  }
-  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "program") {
-    const program = defineEvaluatorProgram(evaluator.evaluator);
-    validateEvaluatorOutput({ ...evaluator, evaluator: program }, program.output);
-    return { ...evaluator, evaluator: program };
-  }
-  if (typeof evaluator.evaluator.judge !== "function") {
-    throw new Error(`Raindrop local evaluator ${name} judge must be a function`);
-  }
-  if (evaluator.evaluator.scope !== "row") {
-    throw new Error(`Raindrop local evaluator ${name} scope must be row`);
-  }
-  const output = EvalOutputSchema.parse(evaluator.evaluator.output);
-  validateEvaluatorOutput(evaluator, output);
-  return evaluator;
-}
-function validateOptionalNumber(name, value, integer, allowZero) {
-  if (value === void 0) return;
-  const valid = Number.isFinite(value) && (!integer || Number.isInteger(value)) && (allowZero ? value >= 0 : value > 0);
-  if (!valid) {
-    const kind = integer ? "a positive integer" : allowZero ? "non-negative" : "positive";
-    throw new Error(`Raindrop eval ${name} must be ${kind}`);
-  }
-}
-function parseBooleanThreshold(threshold) {
-  return threshold === void 0 ? void 0 : BooleanThresholdSchema.parse(threshold);
-}
-function parseNumericThreshold(name, output, threshold) {
-  const parsed = threshold === void 0 ? void 0 : NumericThresholdSchema.parse(threshold);
-  if (output === "score" && parsed) {
-    const lower = "gte" in parsed ? parsed.gte : void 0;
-    const inRange = (lower === void 0 || lower >= 1 && lower <= 5) && (parsed.lte === void 0 || parsed.lte >= 1 && parsed.lte <= 5);
-    if (!inRange)
-      throw new Error(`Raindrop score evaluator ${name} thresholds must be from 1 to 5`);
-  }
-  return parsed;
-}
-var import_node_crypto2, import_zod2, EvalScoreSchema, EVAL_DATASET, DatasetRowInputSchema, BooleanThresholdSchema, FiniteThresholdSchema, EvalOutputSchema, NumericThresholdSchema, EVAL_DEFINITION;
-var init_definition = __esm({
-  "src/evals/definition.ts"() {
+// src/evals/generated/manifest.ts
+var import_zod2, BooleanThresholdSchema, FiniteThresholdSchema, NumericThresholdSchema, EvalSuiteManifestSchema;
+var init_manifest = __esm({
+  "src/evals/generated/manifest.ts"() {
     "use strict";
-    import_node_crypto2 = require("crypto");
     import_zod2 = require("zod");
-    EvalScoreSchema = import_zod2.z.union([
-      import_zod2.z.literal(1),
-      import_zod2.z.literal(2),
-      import_zod2.z.literal(3),
-      import_zod2.z.literal(4),
-      import_zod2.z.literal(5)
-    ]);
-    EVAL_DATASET = /* @__PURE__ */ Symbol.for("raindrop.evalDataset");
-    DatasetRowInputSchema = import_zod2.z.object({
-      id: import_zod2.z.string().trim().min(1),
-      name: import_zod2.z.string().optional(),
-      input: import_zod2.z.string().nullable(),
-      output: import_zod2.z.string().nullable().default(null),
-      properties: import_zod2.z.record(import_zod2.z.string()).default({})
-    });
     BooleanThresholdSchema = import_zod2.z.object({ equals: import_zod2.z.boolean() }).strict();
     FiniteThresholdSchema = import_zod2.z.number().finite();
-    EvalOutputSchema = import_zod2.z.enum(["boolean", "score", "number"]);
     NumericThresholdSchema = import_zod2.z.union([
       import_zod2.z.object({ gte: FiniteThresholdSchema, lte: FiniteThresholdSchema.optional() }).strict(),
       import_zod2.z.object({ lte: FiniteThresholdSchema }).strict()
@@ -6030,250 +5776,45 @@ var init_definition = __esm({
       (threshold) => !("gte" in threshold) || threshold.lte === void 0 || threshold.gte <= threshold.lte,
       { message: "Lower threshold cannot exceed upper threshold" }
     );
-    EVAL_DEFINITION = /* @__PURE__ */ Symbol.for("raindrop.evalDefinition");
+    EvalSuiteManifestSchema = import_zod2.z.object({
+      name: import_zod2.z.string().trim().min(1),
+      dataset: import_zod2.z.string().trim().min(1),
+      datasetVersionId: import_zod2.z.string().uuid().optional(),
+      evaluators: import_zod2.z.array(
+        import_zod2.z.object({
+          evaluator: import_zod2.z.string().trim().min(1),
+          threshold: import_zod2.z.union([BooleanThresholdSchema, NumericThresholdSchema]).optional()
+        }).strict()
+      ).min(1),
+      concurrency: import_zod2.z.number().int().positive().optional(),
+      traceWaitMs: import_zod2.z.number().nonnegative().optional(),
+      evalWaitMs: import_zod2.z.number().nonnegative().optional(),
+      evalPollIntervalMs: import_zod2.z.number().positive().optional()
+    }).strict();
   }
 });
 
 // src/evals/generated/trace.ts
-function compareNanoseconds(left, right) {
-  if (Number.isFinite(left) && Number.isFinite(right))
-    return Number(left) - Number(right);
-  const leftNs = BigInt(left != null ? left : 0);
-  const rightNs = BigInt(right != null ? right : 0);
-  return leftNs < rightNs ? -1 : leftNs > rightNs ? 1 : 0;
-}
-function nanosecondsToRoundedMilliseconds(value) {
-  if (Number.isFinite(value))
-    return Math.round(Number(value) / 1e6);
-  return Number((BigInt(value != null ? value : 0) + BigInt(5e5)) / BigInt(1e6));
-}
-function isKnownAttributeKey(key) {
-  return KNOWN_ATTRIBUTE_PREFIXES.some((prefix) => key.startsWith(prefix));
-}
-function tokenizeKey(key) {
-  return key.replace(/([a-z0-9])([A-Z])/g, "$1 $2").split(/[^a-zA-Z0-9]+/).filter(Boolean).map((token) => token.toLowerCase());
-}
-function isSensitiveAttributeKey(key) {
-  const tokens = tokenizeKey(key);
-  if (tokens.some((token) => SENSITIVE_KEY_TOKENS.has(token)))
-    return true;
-  for (let index = 0; index < tokens.length - 1; index += 1) {
-    if (SENSITIVE_KEY_TOKEN_PAIRS.has(`${tokens[index]} ${tokens[index + 1]}`))
-      return true;
-  }
-  return false;
-}
-function extractCustomSpanAttributes(span) {
-  var _a, _b;
-  const attributes = {};
-  const strings = (_a = span.attributes_string) != null ? _a : {};
-  for (const [key, value] of Object.entries(strings)) {
-    if (Object.keys(attributes).length >= MAX_CUSTOM_ATTRIBUTES)
-      break;
-    if (isKnownAttributeKey(key) || isSensitiveAttributeKey(key)) {
-      continue;
-    }
-    attributes[key] = value.length > MAX_ATTRIBUTE_VALUE_CHARS ? `${value.slice(0, MAX_ATTRIBUTE_VALUE_CHARS)}\u2026` : value;
-  }
-  const numbers = (_b = span.attributes_num) != null ? _b : {};
-  for (const [key, value] of Object.entries(numbers)) {
-    if (Object.keys(attributes).length >= MAX_CUSTOM_ATTRIBUTES)
-      break;
-    if (isKnownAttributeKey(key) || isSensitiveAttributeKey(key)) {
-      continue;
-    }
-    attributes[key] = value;
-  }
-  return Object.keys(attributes).length > 0 ? attributes : void 0;
-}
-function extractFinishReason(span) {
-  var _a, _b, _c;
-  const strings = (_a = span.attributes_string) != null ? _a : {};
-  const value = (_c = (_b = strings["ai.response.finishReason"]) != null ? _b : strings["gen_ai.response.finish_reasons"]) != null ? _c : strings["llm.response.finish_reason"];
-  return value && value.length > 0 ? value : void 0;
-}
-function normalizePayload(value) {
-  return value != null ? value : void 0;
-}
-function buildTraceSteps(rawSpans, options = {}) {
-  var _a;
-  const includePayloads = (_a = options.includePayloads) != null ? _a : false;
-  const sorted = [...rawSpans].sort((left, right) => compareNanoseconds(left.start_unix_ns, right.start_unix_ns));
-  return sorted.map((span, index) => {
-    var _a2, _b;
-    const status = span.status === "OK" || span.status === "ERROR" ? span.status : "UNSET";
-    const step = {
-      step: index + 1,
-      type: span.span_type || "INTERNAL",
-      name: span.span_name || "unknown",
-      status,
-      duration_ms: nanosecondsToRoundedMilliseconds(span.duration_ns)
-    };
-    if (span.span_id)
-      step.span_id = span.span_id;
-    if (span.status === "ERROR" && span.output_payload) {
-      step.error = span.output_payload.slice(0, 500);
-    }
-    if (span.model)
-      step.model = span.model;
-    const finishReason = extractFinishReason(span);
-    if (finishReason)
-      step.finish_reason = finishReason;
-    if (((_a2 = span.input_tokens) != null ? _a2 : 0) > 0)
-      step.input_tokens = span.input_tokens;
-    if (((_b = span.output_tokens) != null ? _b : 0) > 0)
-      step.output_tokens = span.output_tokens;
-    if (includePayloads && step.type === "TOOL_CALL") {
-      const inputPayload = normalizePayload(span.input_payload);
-      const outputPayload = normalizePayload(span.output_payload);
-      if (inputPayload)
-        step.input_payload = inputPayload;
-      if (outputPayload)
-        step.output_payload = outputPayload;
-    }
-    const attributes = extractCustomSpanAttributes(span);
-    if (attributes)
-      step.attributes = attributes;
-    return step;
-  });
-}
-function boundEvalPayload(payload) {
-  var _a;
-  if (payload.raw === null)
-    return payload;
-  const sizeChars = (_a = payload.sizeChars) != null ? _a : payload.raw.length;
-  return {
-    ...payload,
-    raw: payload.raw.slice(0, MAX_EVAL_PAYLOAD_RAW_CHARS),
-    sizeChars,
-    truncated: payload.truncated || sizeChars > MAX_EVAL_PAYLOAD_RAW_CHARS
-  };
-}
-function boundEvalEntry(entry) {
-  switch (entry.type) {
-    case "system":
-    case "reasoning":
-      return {
-        ...entry,
-        content: {
-          ...entry.content,
-          text: entry.content.text.slice(0, MAX_EVAL_PAYLOAD_RAW_CHARS)
-        }
-      };
-    case "user_message":
-    case "assistant_text":
-      return {
-        ...entry,
-        content: entry.content.map((content) => content.type === "text" ? { ...content, text: content.text.slice(0, MAX_EVAL_PAYLOAD_RAW_CHARS) } : content.type === "json" && content.raw !== null ? { ...content, raw: content.raw.slice(0, MAX_EVAL_PAYLOAD_RAW_CHARS) } : content)
-      };
-    case "tool_call":
-      return {
-        ...entry,
-        input: boundEvalPayload(entry.input),
-        output: entry.output === null ? null : boundEvalPayload(entry.output),
-        error: entry.error === null ? null : boundEvalPayload(entry.error)
-      };
-    default:
-      return entry;
-  }
-}
-function createTrace(origin, event, entries) {
-  return TraceSchema.parse({ origin, event, entries });
-}
-function traceStepsToEntries(steps) {
-  return steps.map((step, index) => {
-    var _a;
-    const source = {
-      adapter: "raw_spans",
-      confidence: "medium",
-      spanIds: step.span_id ? [step.span_id] : [],
-      parentSpanIds: [],
-      attributeKeys: []
-    };
-    const common = {
-      id: (_a = step.span_id) != null ? _a : `raw-step-${index}`,
-      order: index,
-      startMs: null,
-      endMs: null,
-      source
-    };
-    if (step.type === "TOOL_CALL") {
-      const payload = (raw) => {
-        var _a2;
-        return boundEvalPayload({
-          kind: raw ? "text" : "empty",
-          raw: raw != null ? raw : null,
-          truncated: false,
-          sizeChars: (_a2 = raw == null ? void 0 : raw.length) != null ? _a2 : null
-        });
-      };
-      return {
-        ...common,
-        type: "tool_call",
-        callId: null,
-        name: step.name,
-        status: step.status === "ERROR" ? "error" : step.status === "OK" ? "completed" : "unknown",
-        durationMs: step.duration_ms,
-        input: payload(step.input_payload),
-        output: step.output_payload ? payload(step.output_payload) : null,
-        error: step.error ? payload(step.error) : null
-      };
-    }
-    return {
-      ...common,
-      type: "unknown_span",
-      spanType: step.type,
-      name: step.name,
-      hiddenByDefault: true
-    };
-  });
-}
-function projectWorkshopTrace(input) {
-  return createTrace(input.origin, input.event, traceStepsToEntries(buildTraceSteps([...input.spans], { includePayloads: true })).map(boundEvalEntry));
-}
-function projectCapturedEvent(input) {
-  var _a, _b, _c, _d, _e, _f;
-  const ordered = [...input.spans].sort((left, right) => compareNanoseconds(left.start_unix_ns, right.start_unix_ns));
-  const eventSpan = ordered.find((span) => span.span_id === `evt_${input.eventId}`);
-  const llmSpans = ordered.filter((span) => span.span_type === "LLM_GENERATION" || span.span_type === "LLM_GENERATION_STREAM");
-  const inputSpan = ((_a = eventSpan == null ? void 0 : eventSpan.input_payload) == null ? void 0 : _a.trim()) ? eventSpan : llmSpans.find((span) => {
-    var _a2;
-    return Boolean((_a2 = span.input_payload) == null ? void 0 : _a2.trim());
-  });
-  const reverseLlmSpans = [...llmSpans].reverse();
-  const outputSpan = (_c = ((_b = eventSpan == null ? void 0 : eventSpan.output_payload) == null ? void 0 : _b.trim()) ? eventSpan : void 0) != null ? _c : reverseLlmSpans.find((span) => {
-    var _a2;
-    return Boolean((_a2 = span.output_payload) == null ? void 0 : _a2.trim()) && extractFinishReason(span) !== "tool-calls";
-  });
-  const modelSpan = (eventSpan == null ? void 0 : eventSpan.model) ? eventSpan : reverseLlmSpans.find((span) => Boolean(span.model));
-  return AIAnalyticsEventSchema.parse({
-    id: input.eventId,
-    _raindropDatabaseId: input.internalEventId,
-    name: input.name,
-    timestamp: input.timestamp,
-    receivedAt: input.receivedAt,
-    userId: input.userId,
-    aiData: {
-      input: (_d = inputSpan == null ? void 0 : inputSpan.input_payload) != null ? _d : null,
-      output: (_e = outputSpan == null ? void 0 : outputSpan.output_payload) != null ? _e : null,
-      model: (_f = modelSpan == null ? void 0 : modelSpan.model) != null ? _f : null,
-      convoId: input.convoId
-    },
-    properties: input.properties
-  });
-}
-function traceOutput(trace8) {
-  var _a;
-  return (_a = trace8.event.aiData.output) != null ? _a : null;
+function traceOutput(trace8) {
+  var _a;
+  return (_a = trace8.event.aiData.output) != null ? _a : null;
 }
 function traceToolCalls(trace8, name) {
   return trace8.entries.filter((entry) => entry.type === "tool_call" && (name === void 0 || entry.name === name));
 }
-var import_zod3, stringToJSONSchema, SignalPropertiesSchema, AIEventMetadataSchema, AnalyticsEventSchema, LensInfoSchema, AttachmentSchema2, SignalSchema, RichTextContentSchema, RichJsonContentSchema, RichMediaContentSchema, RichReferenceContentSchema, RichContentSchema, RichPayloadValueSchema, RichEntrySourceSchema, RichInteractionEntryBaseSchema, RichSystemEntrySchema, RichUserMessageEntrySchema, RichAssistantTextEntrySchema, RichReasoningEntrySchema, RichToolCallEntrySchema, RichUnknownSpanEntrySchema, RichSubAgentEntrySchema, RichDetachedSubAgentEntrySchema, RichAgentHandoffEntrySchema, RichInteractionEntrySchema, RichSpanSummarySchema, RichDiagnosticWarningSchema, RichInteractionStatsSchema, RichEventInteractionSchema, CoreEventSchema, AIAnalyticsEventSchema, SpanStatusSchema, SpanTypeSchema, TracesListPipe, UINT64_MAX, NANOSECONDS_PATTERN, NanosecondsStringSchema, ReplayTraceSnapshotSpanSchema, TraceSchema, CapturedTraceSpanSchema, EVAL_DATASET_MAX_SPANS_PER_TRACE, EvalDatasetCaseIdSchema, CapturedTraceSnapshotSchema, EvalCaseTraceSchema, EvalCaseTracePairSchema, MAX_CUSTOM_ATTRIBUTES, MAX_ATTRIBUTE_VALUE_CHARS, KNOWN_ATTRIBUTE_PREFIXES, SENSITIVE_KEY_TOKENS, SENSITIVE_KEY_TOKEN_PAIRS, MAX_EVAL_PAYLOAD_RAW_CHARS, TRACE_PROJECTION_SHA256;
+var import_zod3, JsonValueSchema, stringToJSONSchema, SignalPropertiesSchema, AIEventMetadataSchema, AnalyticsEventSchema, LensInfoSchema, AttachmentSchema2, SignalSchema, RichTextContentSchema, RichJsonContentSchema, RichMediaContentSchema, RichReferenceContentSchema, RichContentSchema, RichPayloadValueSchema, RichEntrySourceSchema, RichInteractionEntryBaseSchema, RichSystemEntrySchema, RichUserMessageEntrySchema, RichAssistantTextEntrySchema, RichReasoningEntrySchema, RichToolCallEntrySchema, RichUnknownSpanEntrySchema, RichSubAgentEntrySchema, RichDetachedSubAgentEntrySchema, RichAgentHandoffEntrySchema, RichInteractionEntrySchema, RichSpanSummarySchema, RichDiagnosticWarningSchema, RichInteractionStatsSchema, RichEventInteractionSchema, CoreEventSchema, AIAnalyticsEventSchema, SpanStatusSchema, SpanTypeSchema, TracesListPipe, UINT64_MAX, NANOSECONDS_PATTERN, NanosecondsStringSchema, ReplayTraceSnapshotSpanSchema, EvalDatasetCaseIdSchema, EvalCaseExpectationSchema, TraceSchema, CapturedTraceSpanSchema, EVAL_DATASET_MAX_SPANS_PER_TRACE, CapturedTraceSnapshotSchema, EvalCaseTraceSchema, EvalCaseTracePairSchema, MAX_EVAL_PAYLOAD_RAW_CHARS;
 var init_trace = __esm({
   "src/evals/generated/trace.ts"() {
     "use strict";
     import_zod3 = __toESM(require("zod"));
+    JsonValueSchema = import_zod3.default.lazy(() => import_zod3.default.union([
+      import_zod3.default.string(),
+      import_zod3.default.number(),
+      import_zod3.default.boolean(),
+      import_zod3.default.null(),
+      import_zod3.default.array(JsonValueSchema),
+      import_zod3.default.record(JsonValueSchema)
+    ]));
     stringToJSONSchema = import_zod3.default.string().transform((str, ctx) => {
       try {
         return JSON.parse(str);
@@ -6642,52 +6183,58 @@ var init_trace = __esm({
       duration_ns: NanosecondsStringSchema,
       _inserted_at: import_zod3.default.string().min(1)
     }).strict();
+    EvalDatasetCaseIdSchema = import_zod3.default.string().min(1).max(512).refine((value) => value === value.trim(), "Case id cannot have surrounding whitespace");
+    EvalCaseExpectationSchema = import_zod3.default.object({
+      description: import_zod3.default.string().trim().min(1).max(1e4),
+      basis: import_zod3.default.enum(["human", "policy", "code", "inferred"]).optional(),
+      reference: import_zod3.default.string().trim().min(1).max(2e3).optional()
+    }).strict();
     TraceSchema = import_zod3.default.object({
       origin: import_zod3.default.enum(["production", "dataset", "simulation", "replay"]),
       event: AIAnalyticsEventSchema,
-      entries: import_zod3.default.array(RichInteractionEntrySchema)
+      entries: import_zod3.default.array(RichInteractionEntrySchema),
+      caseContext: import_zod3.default.object({ caseId: EvalDatasetCaseIdSchema, expectation: EvalCaseExpectationSchema.optional() }).optional()
     });
     CapturedTraceSpanSchema = ReplayTraceSnapshotSpanSchema;
     EVAL_DATASET_MAX_SPANS_PER_TRACE = 500;
-    EvalDatasetCaseIdSchema = import_zod3.default.string().min(1).max(512).refine((value) => value === value.trim(), "Case id cannot have surrounding whitespace");
     CapturedTraceSnapshotSchema = import_zod3.default.object({
       event: AIAnalyticsEventSchema,
       spans: import_zod3.default.array(CapturedTraceSpanSchema).min(1).max(EVAL_DATASET_MAX_SPANS_PER_TRACE)
-    }).strict().superRefine(({ event, spans }, context9) => {
+    }).strict().superRefine(({ event, spans }, context8) => {
       var _a;
       const spanIds = /* @__PURE__ */ new Set();
       const traceIds = /* @__PURE__ */ new Set();
       for (const [index, span] of spans.entries()) {
         if (!/^[0-9a-f]{32}$/.test(span.trace_id)) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: "Captured trace id must be a lowercase 32-character hexadecimal value",
             path: ["spans", index, "trace_id"]
           });
         }
         if (!/^[0-9a-f]{16}$/.test(span.span_id)) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: "Captured span id must be a lowercase 16-character hexadecimal value",
             path: ["spans", index, "span_id"]
           });
         }
         if (span.parent_span_id !== "" && !/^[0-9a-f]{16}$/.test(span.parent_span_id)) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: "Captured parent span id must be empty or lowercase hexadecimal",
             path: ["spans", index, "parent_span_id"]
           });
         }
         if (span.event_id.length === 0) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: "Captured event id cannot be empty",
             path: ["spans", index, "event_id"]
           });
         }
         if (spanIds.has(span.span_id)) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: `Duplicate span id ${span.span_id}`,
             path: ["spans", index, "span_id"]
@@ -6696,7 +6243,7 @@ var init_trace = __esm({
         spanIds.add(span.span_id);
         traceIds.add(span.trace_id);
         if (span.event_id !== event._raindropDatabaseId) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: "Span event identity does not match the captured event",
             path: ["spans", index, "event_id"]
@@ -6705,7 +6252,7 @@ var init_trace = __esm({
         const validStart = /^\d{1,20}$/.test(span.start_unix_ns);
         const validEnd = /^\d{1,20}$/.test(span.end_unix_ns);
         if (validStart && validEnd && (BigInt(span.start_unix_ns) === BigInt(0) || BigInt(span.end_unix_ns) < BigInt(span.start_unix_ns))) {
-          context9.addIssue({
+          context8.addIssue({
             code: import_zod3.default.ZodIssueCode.custom,
             message: "A captured span must have an end time at or after its start time",
             path: ["spans", index, "end_unix_ns"]
@@ -6713,28 +6260,19 @@ var init_trace = __esm({
         }
       }
       if (traceIds.size !== 1) {
-        context9.addIssue({
+        context8.addIssue({
           code: import_zod3.default.ZodIssueCode.custom,
           message: "A reference snapshot must contain exactly one trace id",
           path: ["spans"]
         });
       }
-      for (const [index, span] of spans.entries()) {
-        if (span.parent_span_id && !spanIds.has(span.parent_span_id)) {
-          context9.addIssue({
-            code: import_zod3.default.ZodIssueCode.custom,
-            message: `Missing parent span ${span.parent_span_id}`,
-            path: ["spans", index, "parent_span_id"]
-          });
-        }
-      }
       const parentBySpanId = new Map(spans.map((span) => [span.span_id, span.parent_span_id]));
       for (const [index, span] of spans.entries()) {
         const path = /* @__PURE__ */ new Set();
         let spanId = span.span_id;
         while (spanId !== "") {
           if (path.has(spanId)) {
-            context9.addIssue({
+            context8.addIssue({
               code: import_zod3.default.ZodIssueCode.custom,
               message: "Captured span parent relationships cannot contain a cycle",
               path: ["spans", index, "parent_span_id"]
@@ -6752,367 +6290,289 @@ var init_trace = __esm({
     });
     EvalCaseTracePairSchema = import_zod3.default.object({
       caseId: EvalDatasetCaseIdSchema,
+      output: JsonValueSchema.optional(),
       candidate: EvalCaseTraceSchema.extend({
-        trace: TraceSchema.extend({ origin: import_zod3.default.literal("replay") })
+        trace: TraceSchema.extend({ origin: import_zod3.default.enum(["replay", "dataset"]) })
       }),
       reference: EvalCaseTraceSchema.extend({
         trace: TraceSchema.extend({ origin: import_zod3.default.literal("dataset") })
       }).optional()
     });
-    MAX_CUSTOM_ATTRIBUTES = 25;
-    MAX_ATTRIBUTE_VALUE_CHARS = 500;
-    KNOWN_ATTRIBUTE_PREFIXES = [
-      "ai.",
-      "gen_ai.",
-      "llm.",
-      "openinference.",
-      "traceloop.",
-      "raindrop.",
-      "otel.",
-      "telemetry.",
-      "service.",
-      "resource.",
-      "http.",
-      "net.",
-      "network.",
-      "server.",
-      "client.",
-      "url.",
-      "rpc.",
-      "db.",
-      "messaging.",
-      "code.",
-      "thread.",
-      "session.",
-      "user_agent.",
-      "process.",
-      "host.",
-      "os.",
-      "exception.",
-      "span."
-    ];
-    SENSITIVE_KEY_TOKENS = /* @__PURE__ */ new Set([
-      "secret",
-      "secrets",
-      "secretkey",
-      "password",
-      "passwd",
-      "pwd",
-      "token",
-      "tokens",
-      "auth",
-      "authorization",
-      "credential",
-      "credentials",
-      "bearer",
-      "cookie",
-      "cookies",
-      "apikey",
-      "accesskey",
-      "privatekey",
-      "sessionid"
-    ]);
-    SENSITIVE_KEY_TOKEN_PAIRS = /* @__PURE__ */ new Set([
-      "api key",
-      "access key",
-      "private key",
-      "secret key",
-      "session id",
-      "auth token",
-      "access token",
-      "refresh token"
-    ]);
     MAX_EVAL_PAYLOAD_RAW_CHARS = 16 * 1024;
-    TRACE_PROJECTION_SHA256 = "7d12eb03150b8f203e4404622f047676f50b3aa1e5442f0ed42d77d6002a33a6";
   }
 });
 
-// src/evals/portable-runtime.ts
-var portable_runtime_exports = {};
-__export(portable_runtime_exports, {
-  runPortableEvaluator: () => runPortableEvaluator
-});
-async function runPortableEvaluator(evaluator, input) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
-  const inputJson = JSON.stringify(input);
-  const maxInputBytes = (_b = (_a = evaluator.limits) == null ? void 0 : _a.maxInputBytes) != null ? _b : DEFAULT_MAX_INPUT_BYTES;
-  if (byteLength(inputJson) > maxInputBytes) {
-    throw new Error(`Raindrop portable evaluator input exceeds ${maxInputBytes} bytes`);
-  }
-  const QuickJS = await (0, import_quickjs_emscripten_core.newQuickJSWASMModuleFromVariant)(import_quickjs_singlefile_cjs_release_sync.default);
-  const runtime = QuickJS.newRuntime();
-  runtime.setMemoryLimit((_d = (_c = evaluator.limits) == null ? void 0 : _c.memoryBytes) != null ? _d : DEFAULT_MEMORY_BYTES);
-  runtime.setMaxStackSize((_f = (_e = evaluator.limits) == null ? void 0 : _e.stackBytes) != null ? _f : DEFAULT_STACK_BYTES);
-  const timeoutMs = (_h = (_g = evaluator.limits) == null ? void 0 : _g.timeoutMs) != null ? _h : evaluator.artifact.evaluator.executionMode === "judge" ? DEFAULT_JUDGE_TIMEOUT_MS : DEFAULT_TIMEOUT_MS;
-  const deadline = Date.now() + timeoutMs;
-  runtime.setInterruptHandler((0, import_quickjs_emscripten_core.shouldInterruptAfterDeadline)(deadline));
-  const context9 = runtime.newContext();
-  const deferreds = [];
-  const hostCalls = /* @__PURE__ */ new Set();
-  const tracesById = new Map(input.traces.map((trace8) => [trace8.event.id, trace8]));
-  const casesByTraceId = new Map(
-    input.cases.map((entry) => [entry.candidate.trace.event.id, entry])
-  );
-  let activeJudgeCalls = 0;
-  try {
-    const bridge = context9.newFunction("__raindrop_eval_judge", (requestHandle) => {
-      var _a2, _b2;
-      const deferred = context9.newPromise();
-      deferreds.push(deferred);
-      const requestJson = context9.getString(requestHandle);
-      const judgeConcurrency = (_b2 = (_a2 = evaluator.limits) == null ? void 0 : _a2.judgeConcurrency) != null ? _b2 : DEFAULT_JUDGE_CONCURRENCY;
-      let acquired = false;
-      const hostCall = resolveJudge(
-        evaluator,
-        requestJson,
-        Math.max(1, deadline - Date.now()),
-        tracesById,
-        casesByTraceId,
-        () => {
-          if (activeJudgeCalls >= judgeConcurrency) return false;
-          activeJudgeCalls += 1;
-          acquired = true;
-          return true;
-        }
-      ).then((responseJson) => {
-        const response = context9.newString(responseJson);
-        try {
-          deferred.resolve(response);
-        } finally {
-          response.dispose();
-        }
-      }).finally(() => {
-        if (acquired) activeJudgeCalls -= 1;
-        hostCalls.delete(hostCall);
-      });
-      hostCalls.add(hostCall);
-      return deferred.handle;
-    });
-    context9.setProp(context9.global, "__raindrop_eval_judge", bridge);
-    bridge.dispose();
-    const loaded = context9.evalCode(evaluator.artifact.source, "raindrop-evaluator.js");
-    context9.unwrapResult(loaded).dispose();
-    const entrypoint = context9.getProp(context9.global, "__raindrop_eval_run");
-    const inputHandle = context9.newString(inputJson);
-    let promiseHandle;
-    try {
-      promiseHandle = context9.unwrapResult(
-        context9.callFunction(entrypoint, context9.global, inputHandle)
-      );
-    } finally {
-      inputHandle.dispose();
-      entrypoint.dispose();
-    }
-    try {
-      const result = await drivePromise(runtime, context9, promiseHandle, deadline);
-      const outputHandle = context9.unwrapResult(result);
-      try {
-        const outputJson = context9.getString(outputHandle);
-        const maxOutputBytes = (_j = (_i = evaluator.limits) == null ? void 0 : _i.maxOutputBytes) != null ? _j : DEFAULT_MAX_OUTPUT_BYTES;
-        if (byteLength(outputJson) > maxOutputBytes) {
-          throw new Error(`Raindrop portable evaluator output exceeds ${maxOutputBytes} bytes`);
-        }
-        const envelope = JSON.parse(outputJson);
-        if (typeof envelope !== "object" || envelope === null || !("type" in envelope) || envelope.type !== "result" || !("result" in envelope)) {
-          throw new Error("Raindrop portable evaluator returned an invalid result envelope");
-        }
-        return parsePortableEvalResult(envelope.result);
-      } finally {
-        outputHandle.dispose();
-      }
-    } finally {
-      promiseHandle.dispose();
-    }
-  } catch (cause) {
-    const message = cause instanceof Error ? cause.message : String(cause);
-    if (Date.now() >= deadline || message.includes("interrupted")) {
-      throw new Error(`Raindrop portable evaluator exceeded ${timeoutMs}ms`);
+// src/evals/definition.ts
+function defineDataset(dataset) {
+  var _a, _b;
+  const id = dataset.id.trim();
+  const name = dataset.name.trim();
+  const rows = dataset.rows.map((row) => {
+    var _a2;
+    const parsed = DatasetRowInputSchema.parse(row);
+    return {
+      id: parsed.id,
+      name: (_a2 = parsed.name) != null ? _a2 : parsed.id,
+      input: parsed.input,
+      output: parsed.output,
+      properties: parsed.properties
+    };
+  });
+  const version = (_b = (_a = dataset.version) == null ? void 0 : _a.trim()) != null ? _b : (0, import_node_crypto2.createHash)("sha256").update(
+    JSON.stringify(
+      rows.map((row) => ({
+        ...row,
+        properties: Object.fromEntries(
+          Object.keys(row.properties).sort().map((key) => [key, row.properties[key]])
+        )
+      }))
+    )
+  ).digest("hex");
+  if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
+  if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
+  if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
+  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
+  const ids = rows.map((entry) => entry.id);
+  if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
+  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
+  if (dataset.remote) {
+    if (!dataset.remote.reference.trim())
+      throw new Error("Raindrop remote dataset reference cannot be empty");
+    if (!/^[a-f0-9]{64}$/.test(dataset.remote.fingerprint)) {
+      throw new Error("Raindrop remote dataset fingerprint must be a SHA-256 digest");
     }
-    throw new Error(`Raindrop portable evaluator failed: ${message}`);
-  } finally {
-    await Promise.allSettled([...hostCalls]);
-    for (const deferred of deferreds) deferred.dispose();
-    context9.dispose();
-    runtime.dispose();
   }
+  const defined2 = {
+    ...dataset,
+    id,
+    name,
+    version,
+    rows,
+    [EVAL_DATASET]: true
+  };
+  Object.defineProperty(defined2, EVAL_DATASET, { enumerable: false });
+  return defined2;
 }
-async function drivePromise(runtime, context9, promise, deadline) {
-  while (true) {
-    runtime.executePendingJobs(100).unwrap();
-    const state2 = context9.getPromiseState(promise);
-    if (state2.type === "fulfilled") return { value: state2.value };
-    if (state2.type === "rejected") return { error: state2.error };
-    await new Promise((resolve3) => setTimeout(resolve3, 0));
-    if (Date.now() >= deadline) throw new Error("interrupted");
-  }
+function isEvalDataset(value) {
+  return typeof value === "object" && value !== null && EVAL_DATASET in value && value[EVAL_DATASET] === true;
 }
-async function resolveJudge(evaluator, rawRequest, remainingMs, tracesById, casesByTraceId, acquire) {
-  var _a, _b;
-  let requestId = "unknown";
-  try {
-    const request = JudgeRequestSchema.parse(JSON.parse(rawRequest));
-    requestId = request.requestId;
-    const trace8 = tracesById.get(request.traceId);
-    if (!trace8) throw new Error("Judge trace is outside this run");
-    if (!evaluator.judge) throw new Error("Judge unavailable");
-    if (!acquire()) throw new Error("Judge concurrency exceeded");
-    const controller = new AbortController();
-    const timeoutMs = Math.min(
-      (_b = (_a = evaluator.limits) == null ? void 0 : _a.judgeTimeoutMs) != null ? _b : DEFAULT_JUDGE_REQUEST_TIMEOUT_MS,
-      remainingMs
+function defineEvaluatorProgram(program) {
+  var _a, _b, _c, _d;
+  const slug = program.slug.trim();
+  const name = program.name.trim();
+  const source = program.source.trim();
+  const intent = program.intent.trim();
+  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug) || slug.length > 64) {
+    throw new Error(
+      "Raindrop evaluator program slugs must use lowercase letters, numbers, and single hyphens"
     );
-    const timeout = setTimeout(() => controller.abort(), timeoutMs);
-    const result = await Promise.race([
-      Promise.resolve().then(
-        () => {
-          var _a2;
-          return (_a2 = evaluator.judge) == null ? void 0 : _a2.call(evaluator, {
-            trace: trace8,
-            tracePair: rowTracePair(casesByTraceId.get(request.traceId)),
-            rubric: request.rubric.slice(0, 4e4),
-            signal: controller.signal
-          });
-        }
-      ),
-      new Promise((_resolve, reject) => {
-        controller.signal.addEventListener(
-          "abort",
-          () => reject(new Error(`Judge exceeded ${timeoutMs}ms`)),
-          { once: true }
-        );
-      })
-    ]).finally(() => clearTimeout(timeout));
-    if (typeof result !== "object" || result === null) throw new Error("invalid judge response");
-    return JSON.stringify({ type: "judge_response", requestId, status: "completed", result });
-  } catch (e) {
-    return JSON.stringify({
-      type: "judge_response",
-      requestId,
-      status: "unavailable",
-      message: "Judge unavailable"
-    });
   }
+  if (!name || name.length > 120) {
+    throw new Error("Raindrop evaluator program names must be between 1 and 120 characters");
+  }
+  if (!source) throw new Error(`Raindrop evaluator program ${slug} source cannot be empty`);
+  if (!intent || intent.length > 1e4) {
+    throw new Error(
+      `Raindrop evaluator program ${slug} intent must be between 1 and 10000 characters`
+    );
+  }
+  if (program.description != null && program.description.trim().length > 2e3) {
+    throw new Error(`Raindrop evaluator program ${slug} description exceeds 2000 characters`);
+  }
+  const rules = (_b = (_a = program.rules) == null ? void 0 : _a.map((rule) => rule.trim())) != null ? _b : [];
+  if (rules.length > 12 || rules.some((rule) => !rule || rule.length > 2e3)) {
+    throw new Error(`Raindrop evaluator program ${slug} rules are invalid`);
+  }
+  if (program.expected && (!import_zod4.z.string().uuid().safeParse(program.expected.evalId).success || !Number.isInteger(program.expected.programVersion) || program.expected.programVersion <= 0)) {
+    throw new Error(`Raindrop evaluator program ${slug} expected identity is invalid`);
+  }
+  return {
+    ...program,
+    kind: "program",
+    scope: import_zod4.z.literal("batch").parse((_c = program.scope) != null ? _c : "batch"),
+    slug,
+    name,
+    source,
+    intent,
+    description: ((_d = program.description) == null ? void 0 : _d.trim()) || null,
+    rules
+  };
 }
-function byteLength(value) {
-  return new TextEncoder().encode(value).byteLength;
-}
-function rowTracePair(pair) {
-  if (!pair) return void 0;
-  const { caseId, ...evidence } = pair;
-  return { ...evidence, rowId: caseId };
+function defineEvalSuite(definition, binding) {
+  if (binding) return createEvalSuite({ ...EvalSuiteManifestSchema.parse(definition), ...binding });
+  if (!("run" in definition) && !("agent" in definition)) {
+    throw new Error("Bind the eval manifest to your local run function or agent");
+  }
+  return createEvalSuite(definition);
 }
-var import_quickjs_singlefile_cjs_release_sync, import_quickjs_emscripten_core, import_zod9, DEFAULT_MEMORY_BYTES, DEFAULT_STACK_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_JUDGE_TIMEOUT_MS, DEFAULT_JUDGE_REQUEST_TIMEOUT_MS, DEFAULT_JUDGE_CONCURRENCY, DEFAULT_MAX_INPUT_BYTES, DEFAULT_MAX_OUTPUT_BYTES, JudgeRequestSchema;
-var init_portable_runtime = __esm({
-  "src/evals/portable-runtime.ts"() {
-    "use strict";
-    import_quickjs_singlefile_cjs_release_sync = __toESM(require("@jitl/quickjs-singlefile-cjs-release-sync"));
-    import_quickjs_emscripten_core = require("quickjs-emscripten-core");
-    import_zod9 = require("zod");
-    init_portable();
-    DEFAULT_MEMORY_BYTES = 32 * 1024 * 1024;
-    DEFAULT_STACK_BYTES = 64 * 1024;
-    DEFAULT_TIMEOUT_MS = 1e3;
-    DEFAULT_JUDGE_TIMEOUT_MS = 6e4;
-    DEFAULT_JUDGE_REQUEST_TIMEOUT_MS = 3e4;
-    DEFAULT_JUDGE_CONCURRENCY = 4;
-    DEFAULT_MAX_INPUT_BYTES = 4 * 1024 * 1024;
-    DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
-    JudgeRequestSchema = import_zod9.z.object({
-      type: import_zod9.z.literal("judge_request"),
-      requestId: import_zod9.z.string(),
-      traceId: import_zod9.z.string(),
-      rubric: import_zod9.z.string()
-    });
+function createEvalSuite(definition) {
+  const name = definition.name.trim();
+  if (name.length === 0) throw new Error("Raindrop eval suite names cannot be empty");
+  const dataset = typeof definition.dataset === "string" ? definition.dataset.trim() : defineDataset(definition.dataset);
+  if (typeof dataset === "string" && dataset.length === 0) {
+    throw new Error("Raindrop eval suite datasets cannot be empty");
   }
-});
-
-// src/evals/portable.ts
-function parsePortableEvalResult(value) {
-  return PortableEvalResultSchema.parse(value);
+  if (definition.datasetVersionId !== void 0) {
+    import_zod4.z.string().uuid().parse(definition.datasetVersionId);
+    if (typeof dataset !== "string")
+      throw new Error("A dataset version pin requires a published dataset slug");
+  }
+  if (definition.agent !== void 0 && definition.run !== void 0) {
+    throw new Error("An eval suite accepts either agent or run, not both");
+  }
+  if (definition.agent === void 0 && typeof definition.run !== "function") {
+    throw new Error("Raindrop eval suite run must be a function");
+  }
+  if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
+    throw new Error("Raindrop eval suites require at least one evaluator");
+  }
+  validateOptionalNumber("concurrency", definition.concurrency, true, false);
+  validateOptionalNumber("traceWaitMs", definition.traceWaitMs, false, true);
+  validateOptionalNumber("evalWaitMs", definition.evalWaitMs, false, true);
+  validateOptionalNumber("evalPollIntervalMs", definition.evalPollIntervalMs, false, false);
+  const evaluators = definition.evaluators.map((evaluator) => validateEvaluator(evaluator));
+  const evaluatorNames = evaluators.map((entry) => evaluatorName(entry));
+  if (new Set(evaluatorNames).size !== evaluatorNames.length) {
+    throw new Error("Raindrop eval suite evaluators must be unique");
+  }
+  const branded = {
+    ...definition,
+    [EVAL_DEFINITION]: true,
+    name,
+    dataset,
+    evaluators
+  };
+  Object.defineProperty(branded, EVAL_DEFINITION, { enumerable: false });
+  return branded;
 }
-async function evaluatePortableEvaluator(evaluator, input) {
-  var _a;
-  const artifact = await verifyPortableEvalArtifact(evaluator.artifact, {
-    expectedSha256: evaluator.expectedSha256
-  });
-  const { runPortableEvaluator: runPortableEvaluator2 } = await Promise.resolve().then(() => (init_portable_runtime(), portable_runtime_exports));
-  const cases = ((_a = input.rows) != null ? _a : []).map(({ rowId, ...pair }) => ({ ...pair, caseId: rowId }));
-  if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
-    (entry, index) => {
-      var _a2;
-      return entry.candidate.trace.event.id !== ((_a2 = input.traces[index]) == null ? void 0 : _a2.event.id);
-    }
-  ))) {
-    throw new Error("Raindrop portable evaluator rows must match traces exactly, in slice order");
+function isEvalSuiteDefinition(value) {
+  return typeof value === "object" && value !== null && EVAL_DEFINITION in value && value[EVAL_DEFINITION] === true;
+}
+function isPublishedEvalSuite(definition) {
+  return typeof definition.dataset === "string" && definition.datasetVersionId !== void 0 && definition.evaluators.every(
+    ({ evaluator }) => typeof evaluator !== "string" && (!("kind" in evaluator) || evaluator.kind !== "program" || evaluator.expected !== void 0)
+  );
+}
+function evaluatorName(evaluator) {
+  return typeof evaluator.evaluator === "string" ? evaluator.evaluator.trim() : evaluator.evaluator.slug.trim();
+}
+function evaluatorOutput(evaluator) {
+  return typeof evaluator.evaluator === "string" ? void 0 : evaluator.evaluator.output;
+}
+function validateEvaluatorOutput(evaluator, rawOutput) {
+  const name = evaluatorName(evaluator);
+  const output = EvalOutputSchema.parse(rawOutput);
+  const localOutput = evaluatorOutput(evaluator);
+  if (localOutput !== void 0 && localOutput !== output) {
+    throw new Error(`Raindrop local evaluator ${name} expected ${localOutput}, received ${output}`);
+  }
+  switch (output) {
+    case "boolean":
+      parseBooleanThreshold(evaluator.threshold);
+      break;
+    case "score":
+      parseNumericThreshold(name, "score", evaluator.threshold);
+      break;
+    case "number":
+      parseNumericThreshold(name, "number", evaluator.threshold);
+      break;
   }
-  const { rows: _rows, ...runtimeInput } = input;
-  const result = await runPortableEvaluator2(evaluator, { ...runtimeInput, cases });
-  validatePortableResult(artifact, input.traces, result);
-  return result;
 }
-async function verifyPortableEvalArtifact(rawArtifact, options = {}) {
-  const artifact = PortableEvalArtifactSchema.parse(rawArtifact);
-  if (artifact.traceProjectionSha256 !== TRACE_PROJECTION_SHA256) {
-    throw new Error(
-      `Raindrop portable evaluator ${artifact.evaluator.slug} targets a different trace projection`
-    );
+function validateEvaluator(evaluator) {
+  if (typeof evaluator !== "object" || evaluator === null || !("evaluator" in evaluator)) {
+    throw new Error("Raindrop eval evaluator entries must name an evaluator");
   }
-  const digest = await artifactDigest(artifact);
-  if (digest !== artifact.sha256) {
+  if ("output" in evaluator) {
     throw new Error(
-      `Raindrop portable evaluator ${artifact.evaluator.slug} content digest is invalid`
+      "Raindrop eval evaluator entries do not accept output; the evaluator definition owns it"
     );
   }
-  if (options.expectedSha256 && digest !== options.expectedSha256) {
-    throw new Error(
-      `Raindrop portable evaluator ${artifact.evaluator.slug} does not match expectedSha256`
-    );
+  if (typeof evaluator.evaluator !== "string" && (typeof evaluator.evaluator !== "object" || evaluator.evaluator === null || typeof evaluator.evaluator.slug !== "string" || typeof evaluator.evaluator.name !== "string")) {
+    throw new Error("Raindrop local eval evaluators require a slug, name, output, and judge");
   }
-  return artifact;
-}
-function validatePortableResult(artifact, traces, result) {
-  if (result.stats.traceCount !== traces.length) {
-    throw new Error("Raindrop portable evaluator returned an inconsistent trace count");
+  const name = evaluatorName(evaluator);
+  if (name.length === 0) throw new Error("Raindrop eval evaluator names cannot be empty");
+  if (typeof evaluator.evaluator === "string") {
+    const threshold = evaluator.threshold;
+    if (threshold !== void 0) {
+      const boolean = BooleanThresholdSchema.safeParse(threshold);
+      const numeric = NumericThresholdSchema.safeParse(threshold);
+      if (!boolean.success && !numeric.success) {
+        throw new Error(`Raindrop hosted evaluator ${name} has an invalid threshold`);
+      }
+    }
+    return { ...evaluator, evaluator: name };
   }
-  const expected = new Set(traces.map((trace8) => trace8.event.id));
-  if (expected.size !== traces.length) {
-    throw new Error("Raindrop portable evaluator input trace ids must be unique");
+  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "portable") {
+    validateEvaluatorOutput(evaluator, evaluator.evaluator.output);
+    return evaluator;
   }
-  const returned = /* @__PURE__ */ new Set();
-  for (const outcome of result.outcomes) {
-    if (!expected.has(outcome.traceId)) {
-      throw new Error(`Raindrop portable evaluator returned foreign trace ${outcome.traceId}`);
-    }
-    if (returned.has(outcome.traceId)) {
-      throw new Error(`Raindrop portable evaluator repeated trace ${outcome.traceId}`);
-    }
-    returned.add(outcome.traceId);
-    if (outcome.state !== "graded") continue;
-    const valid = artifact.evaluator.outputType === "boolean" ? "pass" in outcome : artifact.evaluator.outputType === "score" ? "score" in outcome : "value" in outcome;
-    if (!valid) {
-      throw new Error(
-        `Raindrop portable evaluator ${artifact.evaluator.slug} returned the wrong graded value`
-      );
-    }
+  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "program") {
+    const program = defineEvaluatorProgram(evaluator.evaluator);
+    validateEvaluatorOutput({ ...evaluator, evaluator: program }, program.output);
+    return { ...evaluator, evaluator: program };
+  }
+  if (typeof evaluator.evaluator.judge !== "function") {
+    throw new Error(`Raindrop local evaluator ${name} judge must be a function`);
+  }
+  if (evaluator.evaluator.scope !== "row") {
+    throw new Error(`Raindrop local evaluator ${name} scope must be row`);
   }
-  const missing = [...expected].find((traceId) => !returned.has(traceId));
-  if (missing) throw new Error(`Raindrop portable evaluator omitted trace ${missing}`);
+  const output = EvalOutputSchema.parse(evaluator.evaluator.output);
+  validateEvaluatorOutput(evaluator, output);
+  return evaluator;
 }
-async function artifactDigest(artifact) {
-  var _a;
-  if (!((_a = globalThis.crypto) == null ? void 0 : _a.subtle)) {
-    throw new Error("Raindrop portable evaluator requires Web Crypto SHA-256 support");
+function validateOptionalNumber(name, value, integer, allowZero) {
+  if (value === void 0) return;
+  const valid = Number.isFinite(value) && (!integer || Number.isInteger(value)) && (allowZero ? value >= 0 : value > 0);
+  if (!valid) {
+    const kind = integer ? "a positive integer" : allowZero ? "non-negative" : "positive";
+    throw new Error(`Raindrop eval ${name} must be ${kind}`);
   }
-  const { sha256: _digest, ...unsigned } = artifact;
-  const encoded = new TextEncoder().encode(canonicalJson2(unsigned));
-  const digest = await globalThis.crypto.subtle.digest("SHA-256", encoded);
-  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
 }
-function canonicalJson2(value) {
-  if (value === null || typeof value !== "object") return JSON.stringify(value);
-  if (Array.isArray(value)) return `[${value.map(canonicalJson2).join(",")}]`;
-  return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson2(entry)}`).join(",")}}`;
+function parseBooleanThreshold(threshold) {
+  return threshold === void 0 ? void 0 : BooleanThresholdSchema.parse(threshold);
 }
+function parseNumericThreshold(name, output, threshold) {
+  const parsed = threshold === void 0 ? void 0 : NumericThresholdSchema.parse(threshold);
+  if (output === "score" && parsed) {
+    const lower = "gte" in parsed ? parsed.gte : void 0;
+    const inRange = (lower === void 0 || lower >= 1 && lower <= 5) && (parsed.lte === void 0 || parsed.lte >= 1 && parsed.lte <= 5);
+    if (!inRange)
+      throw new Error(`Raindrop score evaluator ${name} thresholds must be from 1 to 5`);
+  }
+  return parsed;
+}
+var import_node_crypto2, import_zod4, EvalScoreSchema, EVAL_DATASET, DatasetRowInputSchema, EvalOutputSchema, EVAL_DEFINITION;
+var init_definition = __esm({
+  "src/evals/definition.ts"() {
+    "use strict";
+    import_node_crypto2 = require("crypto");
+    import_zod4 = require("zod");
+    init_manifest();
+    init_trace();
+    init_manifest();
+    EvalScoreSchema = import_zod4.z.union([
+      import_zod4.z.literal(1),
+      import_zod4.z.literal(2),
+      import_zod4.z.literal(3),
+      import_zod4.z.literal(4),
+      import_zod4.z.literal(5)
+    ]);
+    EVAL_DATASET = /* @__PURE__ */ Symbol.for("raindrop.evalDataset");
+    DatasetRowInputSchema = import_zod4.z.object({
+      id: import_zod4.z.string().trim().min(1),
+      name: import_zod4.z.string().optional(),
+      input: import_zod4.z.string().nullable(),
+      output: JsonValueSchema.default(null),
+      properties: import_zod4.z.record(import_zod4.z.string()).default({})
+    });
+    EvalOutputSchema = import_zod4.z.enum(["boolean", "score", "number"]);
+    EVAL_DEFINITION = /* @__PURE__ */ Symbol.for("raindrop.evalDefinition");
+  }
+});
+
+// src/evals/portable.ts
 var import_zod10, PortableEvalArtifactSchema, PortableEvalResultSchema;
 var init_portable = __esm({
   "src/evals/portable.ts"() {
@@ -7183,7 +6643,7 @@ var init_portable = __esm({
 var import_node_path = require("path");
 var import_node_url = require("url");
 var import_node_util = require("util");
-var import_zod13 = require("zod");
+var import_zod12 = require("zod");
 
 // src/lib/inject-als.ts
 var import_node_async_hooks = require("async_hooks");
@@ -8798,12 +8258,12 @@ installTracingSuppressionHook();
 
 // ../schemas/dist/chunk-HA2X6666.js
 var __defProp2 = Object.defineProperty;
-var __export2 = (target, all) => {
+var __export = (target, all) => {
   for (var name in all)
     __defProp2(target, name, { get: all[name], enumerable: true });
 };
 var external_exports = {};
-__export2(external_exports, {
+__export(external_exports, {
   BRAND: () => BRAND,
   DIRTY: () => DIRTY,
   EMPTY_PATH: () => EMPTY_PATH,
@@ -12913,7 +12373,7 @@ var SignalEventSchema = external_exports.object({
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.8.0",
+  version: "0.9.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",
@@ -13159,9 +12619,9 @@ function recordExportedSpan(destination, span) {
   if (typeof span !== "object" && typeof span !== "function" || span === null) return;
   const spanContext = span.spanContext;
   if (typeof spanContext !== "function") return;
-  const context9 = spanContext.call(span);
-  if (typeof (context9 == null ? void 0 : context9.spanId) === "string" && typeof context9.traceId === "string") {
-    recordReplayExportedSpan(destination, context9.spanId, context9.traceId);
+  const context8 = spanContext.call(span);
+  if (typeof (context8 == null ? void 0 : context8.spanId) === "string" && typeof context8.traceId === "string") {
+    recordReplayExportedSpan(destination, context8.spanId, context8.traceId);
   }
 }
 function createReplayExporter(destination) {
@@ -14758,16 +14218,16 @@ var LiveSubagentRun = class {
 };
 
 // src/tracing/liveInteraction.ts
-function getPropertiesFromContext(context9) {
+function getPropertiesFromContext(context8) {
   const properties = {
-    ...context9.userId && { user_id: context9.userId },
-    ...context9.convoId && { convo_id: context9.convoId },
-    ...context9.eventId && { event_id: context9.eventId },
-    ...context9.event && { event_name: context9.event },
-    ...context9.properties || {}
+    ...context8.userId && { user_id: context8.userId },
+    ...context8.convoId && { convo_id: context8.convoId },
+    ...context8.eventId && { event_id: context8.eventId },
+    ...context8.event && { event_name: context8.event },
+    ...context8.properties || {}
   };
-  if (context9.attachments && context9.attachments.length > 0) {
-    properties.attachments = stringifyBounded(context9.attachments);
+  if (context8.attachments && context8.attachments.length > 0) {
+    properties.attachments = stringifyBounded(context8.attachments);
   }
   return properties;
 }
@@ -14787,12 +14247,12 @@ function serializeAssociationProperties(properties) {
 function serializeSpanValue(value) {
   return stringifyBounded(value);
 }
-function getLocalDebuggerMetadata(context9) {
+function getLocalDebuggerMetadata(context8) {
   return {
-    ...context9.eventId ? { eventId: context9.eventId } : {},
-    ...context9.event ? { eventName: context9.event } : {},
-    ...context9.userId ? { userId: context9.userId } : {},
-    ...context9.convoId ? { convoId: context9.convoId } : {}
+    ...context8.eventId ? { eventId: context8.eventId } : {},
+    ...context8.event ? { eventName: context8.event } : {},
+    ...context8.userId ? { userId: context8.userId } : {},
+    ...context8.convoId ? { convoId: context8.convoId } : {}
   };
 }
 var LiveToolSpan = class {
@@ -14893,9 +14353,9 @@ var DirectToolSpan = class {
   }
 };
 var LiveInteraction = class {
-  constructor(context9, traceId, analytics, directShipper, routing) {
+  constructor(context8, traceId, analytics, directShipper, routing) {
     this.usageEmitted = false;
-    this.context = context9;
+    this.context = context8;
     this.analytics = analytics;
     this.tracer = import_api4.trace.getTracer("traceloop.tracer");
     this.traceId = traceId;
@@ -16855,33 +16315,49 @@ var AttachmentTypeSchema = import_zod.z.enum(["code", "text", "image", "iframe"]
 
 // src/replay/index.ts
 var import_api7 = require("@opentelemetry/api");
-var import_zod5 = require("zod");
+var import_zod6 = require("zod");
 init_definition();
 
 // src/evals/generated/dataset.ts
-var import_zod4 = require("zod");
+var import_zod5 = require("zod");
 init_trace();
 var EVAL_DATASET_MAX_CASES = 1e3;
 var EVAL_DATASET_MAX_REQUEST_BYTES = 4 * 1024 * 1024;
 var EVAL_DATASET_MAX_REFERENCE_BYTES = 1024 * 1024;
-var EvalDatasetSlugSchema = import_zod4.z.string().min(1).max(64).regex(/^[a-z0-9]+(-[a-z0-9]+)*$/);
-var EvalDatasetFingerprintSchema = import_zod4.z.string().regex(/^[0-9a-f]{64}$/);
-var EvalDatasetCaseInputSchema = import_zod4.z.object({
+var EvalDatasetSlugSchema = import_zod5.z.string().min(1).max(64).regex(/^[a-z0-9]+(-[a-z0-9]+)*$/);
+var EvalDatasetFingerprintSchema = import_zod5.z.string().regex(/^[0-9a-f]{64}$/);
+var EvalDatasetEventSourceSchema = import_zod5.z.object({
+  sourceEventId: import_zod5.z.string().trim().min(1).max(512),
+  sourceEventTimestamp: import_zod5.z.string().datetime({ offset: true }).optional()
+}).strict();
+var EvalDatasetCaptureRefSchema = EvalDatasetEventSourceSchema.extend({
+  id: EvalDatasetFingerprintSchema
+}).strict();
+var EvalReferenceVerdictSchema = import_zod5.z.union([
+  import_zod5.z.object({ pass: import_zod5.z.boolean() }).strict(),
+  import_zod5.z.object({ score: import_zod5.z.number().int().min(1).max(5) }).strict(),
+  import_zod5.z.object({ value: import_zod5.z.number().finite() }).strict()
+]);
+var EvalDatasetCaseInputSchema = import_zod5.z.object({
   id: EvalDatasetCaseIdSchema,
-  name: import_zod4.z.string().trim().min(1).max(200),
-  input: import_zod4.z.string().nullable(),
-  properties: import_zod4.z.record(import_zod4.z.string(), import_zod4.z.string()),
-  referenceTrace: CapturedTraceSnapshotSchema.optional()
+  name: import_zod5.z.string().trim().min(1).max(200),
+  input: import_zod5.z.string().nullable(),
+  output: JsonValueSchema.optional(),
+  properties: import_zod5.z.record(import_zod5.z.string(), import_zod5.z.string()),
+  referenceTrace: CapturedTraceSnapshotSchema.optional(),
+  referenceCapture: EvalDatasetCaptureRefSchema.optional(),
+  expectation: EvalCaseExpectationSchema.optional(),
+  expectedVerdict: EvalReferenceVerdictSchema.optional()
 }).strict();
 var EvalDatasetVersionCaseSchema = EvalDatasetCaseInputSchema.extend({
   referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
 });
-var EvalDatasetCaseInputsSchema = import_zod4.z.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context9) => {
+var EvalDatasetCaseInputsSchema = import_zod5.z.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context8) => {
   const seen = /* @__PURE__ */ new Set();
   for (const [index, entry] of cases.entries()) {
     if (seen.has(entry.id)) {
-      context9.addIssue({
-        code: import_zod4.z.ZodIssueCode.custom,
+      context8.addIssue({
+        code: import_zod5.z.ZodIssueCode.custom,
         message: `Duplicate row id ${entry.id}`,
         path: [index, "id"]
       });
@@ -16889,40 +16365,59 @@ var EvalDatasetCaseInputsSchema = import_zod4.z.array(EvalDatasetCaseInputSchema
     seen.add(entry.id);
   }
 });
-var PublishEvalDatasetVersionInputSchema = import_zod4.z.object({
-  name: import_zod4.z.string().trim().min(1).max(100),
-  requestKey: import_zod4.z.string().uuid(),
-  expectedCurrentVersionId: import_zod4.z.string().uuid().nullable(),
+var PublishEvalDatasetVersionInputSchema = import_zod5.z.object({
+  name: import_zod5.z.string().trim().min(1).max(100),
+  requestKey: import_zod5.z.string().uuid(),
+  expectedCurrentVersionId: import_zod5.z.string().uuid().nullable(),
   fingerprint: EvalDatasetFingerprintSchema,
   cases: EvalDatasetCaseInputsSchema
 }).strict();
-var EvalDatasetIdentitySchema = import_zod4.z.object({
-  id: import_zod4.z.string().uuid(),
+var CaptureEvalDatasetCaseSchema = EvalDatasetCaseInputSchema.pick({
+  id: true,
+  name: true,
+  properties: true,
+  expectation: true,
+  expectedVerdict: true
+}).merge(EvalDatasetEventSourceSchema).strict();
+var CaptureEvalDatasetVersionInputSchema = PublishEvalDatasetVersionInputSchema.omit({
+  cases: true,
+  fingerprint: true
+}).extend({
+  requestKey: import_zod5.z.string().uuid().optional(),
+  expectedCurrentVersionId: import_zod5.z.string().uuid().nullable().optional(),
+  cases: import_zod5.z.array(CaptureEvalDatasetCaseSchema).min(1).max(EVAL_DATASET_MAX_CASES).refine((cases) => new Set(cases.map((entry) => entry.id)).size === cases.length, "Case IDs must be unique")
+}).strict();
+var EvalDatasetIdentitySchema = import_zod5.z.object({
+  id: import_zod5.z.string().uuid(),
   slug: EvalDatasetSlugSchema,
-  name: import_zod4.z.string().min(1).max(100),
-  currentVersionId: import_zod4.z.string().uuid().nullable()
+  name: import_zod5.z.string().min(1).max(100),
+  currentVersionId: import_zod5.z.string().uuid().nullable()
 });
-var EvalDatasetVersionSchema = import_zod4.z.object({
-  id: import_zod4.z.string().uuid(),
-  datasetId: import_zod4.z.string().uuid(),
-  parentVersionId: import_zod4.z.string().uuid().nullable(),
+var EvalDatasetVersionSchema = import_zod5.z.object({
+  id: import_zod5.z.string().uuid(),
+  datasetId: import_zod5.z.string().uuid(),
+  parentVersionId: import_zod5.z.string().uuid().nullable(),
   fingerprint: EvalDatasetFingerprintSchema,
-  cases: import_zod4.z.array(EvalDatasetVersionCaseSchema).min(1).max(EVAL_DATASET_MAX_CASES),
-  createdAt: import_zod4.z.string().datetime({ offset: true })
+  cases: import_zod5.z.array(EvalDatasetVersionCaseSchema).min(1).max(EVAL_DATASET_MAX_CASES),
+  createdAt: import_zod5.z.string().datetime({ offset: true })
 });
-var EvalDatasetManifestSchema = import_zod4.z.object({
+var EvalDatasetManifestSchema = import_zod5.z.object({
   dataset: EvalDatasetIdentitySchema,
   version: EvalDatasetVersionSchema.omit({ cases: true }),
-  cases: import_zod4.z.array(EvalDatasetVersionCaseSchema.pick({
+  cases: import_zod5.z.array(EvalDatasetVersionCaseSchema.pick({
     id: true,
     name: true,
     input: true,
+    output: true,
     properties: true
   }).extend({
-    referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
+    referenceTraceSha256: EvalDatasetFingerprintSchema.optional(),
+    referenceCapture: EvalDatasetCaptureRefSchema.optional(),
+    expectation: EvalCaseExpectationSchema.optional(),
+    expectedVerdict: EvalReferenceVerdictSchema.optional()
   }))
 });
-function compareNanoseconds2(left, right) {
+function compareNanoseconds(left, right) {
   const leftValue = BigInt(left);
   const rightValue = BigInt(right);
   return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
@@ -16930,7 +16425,7 @@ function compareNanoseconds2(left, right) {
 function orderSnapshot(snapshot) {
   return {
     event: snapshot.event,
-    spans: [...snapshot.spans].sort((left, right) => compareNanoseconds2(left.start_unix_ns, right.start_unix_ns) || left.span_id.localeCompare(right.span_id))
+    spans: [...snapshot.spans].sort((left, right) => compareNanoseconds(left.start_unix_ns, right.start_unix_ns) || left.span_id.localeCompare(right.span_id))
   };
 }
 function normalizeCasesForFingerprint(cases) {
@@ -17007,115 +16502,116 @@ function retryDelayMs(response, retry) {
   }
   return EVAL_RETRY_BASE_DELAY_MS * 2 ** retry;
 }
-var ReplayRowSchema = import_zod5.z.object({
-  id: import_zod5.z.string(),
-  name: import_zod5.z.string(),
-  input: import_zod5.z.string().nullable(),
-  output: import_zod5.z.string().nullable(),
-  properties: import_zod5.z.record(import_zod5.z.string())
+var ReplayRowSchema = import_zod6.z.object({
+  id: import_zod6.z.string(),
+  name: import_zod6.z.string(),
+  input: import_zod6.z.string().nullable(),
+  output: JsonValueSchema,
+  properties: import_zod6.z.record(import_zod6.z.string())
 });
-var WireEvalManifestSchema = import_zod5.z.object({
-  dataset_id: import_zod5.z.string().uuid(),
-  fingerprint: import_zod5.z.string().regex(/^[a-f0-9]{64}$/),
-  rows: import_zod5.z.array(ReplayRowSchema.extend({ id: import_zod5.z.string().min(1) }))
+var WireEvalManifestSchema = import_zod6.z.object({
+  dataset_id: import_zod6.z.string().uuid(),
+  dataset_version_id: import_zod6.z.string().uuid().optional(),
+  fingerprint: import_zod6.z.string().regex(/^[a-f0-9]{64}$/),
+  rows: import_zod6.z.array(ReplayRowSchema.extend({ id: import_zod6.z.string().min(1) }))
 });
-var ReplayCountsSchema = import_zod5.z.object({
-  total: import_zod5.z.number(),
-  pending: import_zod5.z.number(),
-  done: import_zod5.z.number(),
-  missing: import_zod5.z.number()
+var ReplayCountsSchema = import_zod6.z.object({
+  total: import_zod6.z.number(),
+  pending: import_zod6.z.number(),
+  done: import_zod6.z.number(),
+  missing: import_zod6.z.number()
 });
-var ReplayEvaluatorCountsSchema = import_zod5.z.object({
-  traces: import_zod5.z.number(),
-  graded: import_zod5.z.number(),
-  errored: import_zod5.z.number(),
-  ungraded: import_zod5.z.number(),
-  pending: import_zod5.z.number()
+var ReplayEvaluatorCountsSchema = import_zod6.z.object({
+  traces: import_zod6.z.number(),
+  graded: import_zod6.z.number(),
+  errored: import_zod6.z.number(),
+  ungraded: import_zod6.z.number(),
+  pending: import_zod6.z.number()
 });
-var ReplayEvaluatorStatsSchema = import_zod5.z.object({
-  traceCount: import_zod5.z.number(),
-  hydratedCount: import_zod5.z.number(),
-  judgeCalls: import_zod5.z.number(),
-  durationMs: import_zod5.z.number(),
-  skippedRows: import_zod5.z.number(),
-  skippedReason: import_zod5.z.literal("no trace").nullable()
+var ReplayEvaluatorStatsSchema = import_zod6.z.object({
+  traceCount: import_zod6.z.number(),
+  hydratedCount: import_zod6.z.number(),
+  judgeCalls: import_zod6.z.number(),
+  durationMs: import_zod6.z.number(),
+  skippedRows: import_zod6.z.number(),
+  skippedReason: import_zod6.z.literal("no trace").nullable()
 });
-var ReplayEvaluatorSummarySchema = import_zod5.z.discriminatedUnion("outputType", [
-  import_zod5.z.object({
-    outputType: import_zod5.z.literal("boolean"),
+var ReplayEvaluatorSummarySchema = import_zod6.z.discriminatedUnion("outputType", [
+  import_zod6.z.object({
+    outputType: import_zod6.z.literal("boolean"),
     counts: ReplayEvaluatorCountsSchema,
     stats: ReplayEvaluatorStatsSchema,
-    passed: import_zod5.z.number(),
-    failed: import_zod5.z.number(),
-    passRate: import_zod5.z.number().nullable()
+    passed: import_zod6.z.number(),
+    failed: import_zod6.z.number(),
+    passRate: import_zod6.z.number().nullable()
   }),
-  import_zod5.z.object({
-    outputType: import_zod5.z.literal("score"),
+  import_zod6.z.object({
+    outputType: import_zod6.z.literal("score"),
     counts: ReplayEvaluatorCountsSchema,
     stats: ReplayEvaluatorStatsSchema,
-    mean: import_zod5.z.number().nullable(),
-    median: import_zod5.z.number().nullable(),
-    p10: import_zod5.z.number().nullable(),
-    p90: import_zod5.z.number().nullable(),
-    atLeast4Share: import_zod5.z.number().nullable(),
-    histogram: import_zod5.z.tuple([import_zod5.z.number(), import_zod5.z.number(), import_zod5.z.number(), import_zod5.z.number(), import_zod5.z.number()])
+    mean: import_zod6.z.number().nullable(),
+    median: import_zod6.z.number().nullable(),
+    p10: import_zod6.z.number().nullable(),
+    p90: import_zod6.z.number().nullable(),
+    atLeast4Share: import_zod6.z.number().nullable(),
+    histogram: import_zod6.z.tuple([import_zod6.z.number(), import_zod6.z.number(), import_zod6.z.number(), import_zod6.z.number(), import_zod6.z.number()])
   }),
-  import_zod5.z.object({
-    outputType: import_zod5.z.literal("number"),
+  import_zod6.z.object({
+    outputType: import_zod6.z.literal("number"),
     counts: ReplayEvaluatorCountsSchema,
     stats: ReplayEvaluatorStatsSchema,
-    median: import_zod5.z.number().nullable(),
-    p10: import_zod5.z.number().nullable(),
-    p90: import_zod5.z.number().nullable()
+    median: import_zod6.z.number().nullable(),
+    p10: import_zod6.z.number().nullable(),
+    p90: import_zod6.z.number().nullable()
   })
 ]);
-var ReplayVerdictIdentitySchema = import_zod5.z.object({
-  evaluator: import_zod5.z.string(),
-  runId: import_zod5.z.string(),
-  url: import_zod5.z.string()
+var ReplayVerdictIdentitySchema = import_zod6.z.object({
+  evaluator: import_zod6.z.string(),
+  runId: import_zod6.z.string(),
+  url: import_zod6.z.string()
 });
 var ReplayVerdictGradedSchema = ReplayVerdictIdentitySchema.extend({
-  state: import_zod5.z.literal("graded"),
-  note: import_zod5.z.string().optional()
+  state: import_zod6.z.literal("graded"),
+  note: import_zod6.z.string().optional()
 });
-var ReplayVerdictSchema = import_zod5.z.union([
-  ReplayVerdictGradedSchema.extend({ pass: import_zod5.z.boolean() }),
+var ReplayVerdictSchema = import_zod6.z.union([
+  ReplayVerdictGradedSchema.extend({ pass: import_zod6.z.boolean() }),
   ReplayVerdictGradedSchema.extend({ score: EvalScoreSchema }),
-  ReplayVerdictGradedSchema.extend({ value: import_zod5.z.number().finite() }),
+  ReplayVerdictGradedSchema.extend({ value: import_zod6.z.number().finite() }),
   ReplayVerdictIdentitySchema.extend({
-    state: import_zod5.z.literal("errored"),
-    message: import_zod5.z.string()
+    state: import_zod6.z.literal("errored"),
+    message: import_zod6.z.string()
   }),
   ReplayVerdictIdentitySchema.extend({
-    state: import_zod5.z.literal("ungraded"),
-    reason: import_zod5.z.enum(["not sampled", "trace unavailable", "content unreadable"])
+    state: import_zod6.z.literal("ungraded"),
+    reason: import_zod6.z.enum(["not sampled", "trace unavailable", "content unreadable"])
   })
 ]);
 var ReplayResultRowSchema = ReplayRowSchema.extend({
-  status: import_zod5.z.enum(["pending", "done", "missing"]),
-  traceId: import_zod5.z.string().nullable(),
-  error: import_zod5.z.string().nullable(),
-  verdicts: import_zod5.z.array(ReplayVerdictSchema)
+  status: import_zod6.z.enum(["pending", "done", "missing"]),
+  traceId: import_zod6.z.string().nullable(),
+  error: import_zod6.z.string().nullable(),
+  verdicts: import_zod6.z.array(ReplayVerdictSchema)
 });
-var ReplayTraceEvidenceEntrySchema = import_zod5.z.object({
-  rowId: import_zod5.z.string().min(1).max(512),
-  traceId: import_zod5.z.string().regex(/^[a-f0-9]{32}$/),
-  spanIds: import_zod5.z.array(import_zod5.z.string().regex(/^[a-f0-9]{16}$/)).min(1).max(500)
-}).superRefine((entry, context9) => {
+var ReplayTraceEvidenceEntrySchema = import_zod6.z.object({
+  rowId: import_zod6.z.string().min(1).max(512),
+  traceId: import_zod6.z.string().regex(/^[a-f0-9]{32}$/),
+  spanIds: import_zod6.z.array(import_zod6.z.string().regex(/^[a-f0-9]{16}$/)).min(1).max(500)
+}).superRefine((entry, context8) => {
   if (new Set(entry.spanIds).size !== entry.spanIds.length) {
-    context9.addIssue({
-      code: import_zod5.z.ZodIssueCode.custom,
+    context8.addIssue({
+      code: import_zod6.z.ZodIssueCode.custom,
       message: "Replay trace evidence span ids must be unique",
       path: ["spanIds"]
     });
   }
 });
-var ReplayTraceEvidenceSchema = import_zod5.z.array(ReplayTraceEvidenceEntrySchema).superRefine((entries, context9) => {
+var ReplayTraceEvidenceSchema = import_zod6.z.array(ReplayTraceEvidenceEntrySchema).superRefine((entries, context8) => {
   const seen = /* @__PURE__ */ new Set();
   for (const [index, entry] of entries.entries()) {
     if (seen.has(entry.rowId)) {
-      context9.addIssue({
-        code: import_zod5.z.ZodIssueCode.custom,
+      context8.addIssue({
+        code: import_zod6.z.ZodIssueCode.custom,
         message: "Replay trace evidence row ids must be unique",
         path: [index, "rowId"]
       });
@@ -17123,13 +16619,13 @@ var ReplayTraceEvidenceSchema = import_zod5.z.array(ReplayTraceEvidenceEntrySche
     seen.add(entry.rowId);
   }
 });
-var LocalBooleanVerdictSchema = import_zod5.z.object({ pass: import_zod5.z.boolean(), note: import_zod5.z.string().trim().min(1).max(600).optional() }).strict();
-var LocalScoreVerdictSchema = import_zod5.z.object({
+var LocalBooleanVerdictSchema = import_zod6.z.object({ pass: import_zod6.z.boolean(), note: import_zod6.z.string().trim().min(1).max(600).optional() }).strict();
+var LocalScoreVerdictSchema = import_zod6.z.object({
   score: EvalScoreSchema,
-  note: import_zod5.z.string().trim().min(1).max(600).optional()
+  note: import_zod6.z.string().trim().min(1).max(600).optional()
 }).strict();
-var LocalNumberVerdictSchema = import_zod5.z.object({ value: import_zod5.z.number().finite(), note: import_zod5.z.string().trim().min(1).max(600).optional() }).strict();
-var LocalVerdictSchema = import_zod5.z.union([
+var LocalNumberVerdictSchema = import_zod6.z.object({ value: import_zod6.z.number().finite(), note: import_zod6.z.string().trim().min(1).max(600).optional() }).strict();
+var LocalVerdictSchema = import_zod6.z.union([
   LocalBooleanVerdictSchema,
   LocalScoreVerdictSchema,
   LocalNumberVerdictSchema
@@ -17205,50 +16701,50 @@ async function queryApi(request) {
     return parsed.data;
   }
 }
-var ReplayStatusSchema = import_zod5.z.enum(["queued", "running", "complete", "failed"]);
-var ReplayRowStatusSchema = import_zod5.z.enum(["pending", "done", "missing"]);
-var WireReplaySchema = import_zod5.z.object({
-  id: import_zod5.z.string(),
+var ReplayStatusSchema = import_zod6.z.enum(["queued", "running", "complete", "failed"]);
+var ReplayRowStatusSchema = import_zod6.z.enum(["pending", "done", "missing"]);
+var WireReplaySchema = import_zod6.z.object({
+  id: import_zod6.z.string(),
   status: ReplayStatusSchema,
   counts: ReplayCountsSchema,
-  traces_expire_at: import_zod5.z.string(),
-  traces_expired: import_zod5.z.boolean()
+  traces_expire_at: import_zod6.z.string(),
+  traces_expired: import_zod6.z.boolean()
 });
-var WireRowSchema = import_zod5.z.object({
-  row_id: import_zod5.z.string(),
-  attempt: import_zod5.z.number().int().nonnegative(),
+var WireRowSchema = import_zod6.z.object({
+  row_id: import_zod6.z.string(),
+  attempt: import_zod6.z.number().int().nonnegative(),
   status: ReplayRowStatusSchema,
-  trace_id: import_zod5.z.string().nullable(),
-  error: import_zod5.z.string().nullable()
+  trace_id: import_zod6.z.string().nullable(),
+  error: import_zod6.z.string().nullable()
 });
-var WirePlanSchema = import_zod5.z.object({
+var WirePlanSchema = import_zod6.z.object({
   replay: WireReplaySchema,
-  ingest_url: import_zod5.z.string().min(1).nullable(),
-  rows: import_zod5.z.array(
+  ingest_url: import_zod6.z.string().min(1).nullable(),
+  rows: import_zod6.z.array(
     ReplayRowSchema.omit({ id: true }).extend({
-      row_id: import_zod5.z.string(),
-      attempt: import_zod5.z.literal(0),
-      correlation_id: import_zod5.z.string()
+      row_id: import_zod6.z.string(),
+      attempt: import_zod6.z.literal(0),
+      correlation_id: import_zod6.z.string()
     })
   )
 });
-var WireReplayDetailSchema = import_zod5.z.object({
+var WireReplayDetailSchema = import_zod6.z.object({
   replay: WireReplaySchema,
-  rows: import_zod5.z.array(WireRowSchema)
+  rows: import_zod6.z.array(WireRowSchema)
 });
 var EvalRunDetailSchema = WireReplayDetailSchema.extend({
   replay: WireReplaySchema.extend({
-    dataset_id: import_zod5.z.string().uuid(),
-    dataset_version_id: import_zod5.z.string().uuid().nullable()
+    dataset_id: import_zod6.z.string().uuid(),
+    dataset_version_id: import_zod6.z.string().uuid().nullable()
   }),
-  eval_runs: import_zod5.z.array(
-    import_zod5.z.object({
-      id: import_zod5.z.string().uuid(),
-      eval_id: import_zod5.z.string().uuid(),
-      program_version: import_zod5.z.number().int().positive(),
-      output_type: import_zod5.z.enum(["boolean", "score", "number"]),
-      executed_by: import_zod5.z.enum(["hosted", "local"]),
-      status: import_zod5.z.enum(["running", "completed", "failed"])
+  eval_runs: import_zod6.z.array(
+    import_zod6.z.object({
+      id: import_zod6.z.string().uuid(),
+      eval_id: import_zod6.z.string().uuid(),
+      program_version: import_zod6.z.number().int().positive(),
+      output_type: import_zod6.z.enum(["boolean", "score", "number"]),
+      executed_by: import_zod6.z.enum(["hosted", "local"]),
+      status: import_zod6.z.enum(["running", "completed", "failed"])
     })
   )
 });
@@ -17264,118 +16760,118 @@ async function readReplayDetails(client, runId, options = {}) {
   if (detail.replay.id !== runId) throw new Error("Raindrop returned a different run");
   return detail;
 }
-var WireReportSchema = import_zod5.z.object({ status: import_zod5.z.string() });
-var WireAttemptStartedSchema = import_zod5.z.object({
-  status: import_zod5.z.literal("pending"),
-  row: import_zod5.z.object({
-    row_id: import_zod5.z.string().min(1),
-    attempt: import_zod5.z.number().int().positive(),
-    correlation_id: import_zod5.z.string().min(1)
+var WireReportSchema = import_zod6.z.object({ status: import_zod6.z.string() });
+var WireAttemptStartedSchema = import_zod6.z.object({
+  status: import_zod6.z.literal("pending"),
+  row: import_zod6.z.object({
+    row_id: import_zod6.z.string().min(1),
+    attempt: import_zod6.z.number().int().positive(),
+    correlation_id: import_zod6.z.string().min(1)
   })
 });
-var WireLocalEvaluatorSchema = import_zod5.z.object({
-  data: import_zod5.z.object({
-    id: import_zod5.z.string().uuid(),
-    slug: import_zod5.z.string(),
-    name: import_zod5.z.string(),
-    output: import_zod5.z.enum(["boolean", "score", "number"]),
-    kind: import_zod5.z.literal("local"),
-    programVersion: import_zod5.z.number().int().positive()
+var WireLocalEvaluatorSchema = import_zod6.z.object({
+  data: import_zod6.z.object({
+    id: import_zod6.z.string().uuid(),
+    slug: import_zod6.z.string(),
+    name: import_zod6.z.string(),
+    output: import_zod6.z.enum(["boolean", "score", "number"]),
+    kind: import_zod6.z.literal("local"),
+    programVersion: import_zod6.z.number().int().positive()
   })
 });
-var WireStartedEvalRunSchema = import_zod5.z.object({
-  data: import_zod5.z.object({
-    runId: import_zod5.z.string(),
-    status: import_zod5.z.literal("running"),
-    url: import_zod5.z.string(),
-    evalId: import_zod5.z.string().uuid().optional(),
-    programVersion: import_zod5.z.number().int().positive().optional()
+var WireStartedEvalRunSchema = import_zod6.z.object({
+  data: import_zod6.z.object({
+    runId: import_zod6.z.string(),
+    status: import_zod6.z.literal("running"),
+    url: import_zod6.z.string(),
+    evalId: import_zod6.z.string().uuid().optional(),
+    programVersion: import_zod6.z.number().int().positive().optional()
   })
 });
-var WireEvalRunSchema = import_zod5.z.object({
-  data: import_zod5.z.object({
-    id: import_zod5.z.string(),
-    evalSlug: import_zod5.z.string(),
-    status: import_zod5.z.enum(["running", "completed", "failed"]),
+var WireEvalRunSchema = import_zod6.z.object({
+  data: import_zod6.z.object({
+    id: import_zod6.z.string(),
+    evalSlug: import_zod6.z.string(),
+    status: import_zod6.z.enum(["running", "completed", "failed"]),
     summary: ReplayEvaluatorSummarySchema.nullable(),
-    error: import_zod5.z.string().nullable(),
-    traces: import_zod5.z.array(
-      import_zod5.z.union([
-        import_zod5.z.object({
-          state: import_zod5.z.literal("graded"),
-          traceId: import_zod5.z.string(),
-          pass: import_zod5.z.boolean(),
-          note: import_zod5.z.string().optional()
+    error: import_zod6.z.string().nullable(),
+    traces: import_zod6.z.array(
+      import_zod6.z.union([
+        import_zod6.z.object({
+          state: import_zod6.z.literal("graded"),
+          traceId: import_zod6.z.string(),
+          pass: import_zod6.z.boolean(),
+          note: import_zod6.z.string().optional()
         }),
-        import_zod5.z.object({
-          state: import_zod5.z.literal("graded"),
-          traceId: import_zod5.z.string(),
+        import_zod6.z.object({
+          state: import_zod6.z.literal("graded"),
+          traceId: import_zod6.z.string(),
           score: EvalScoreSchema,
-          note: import_zod5.z.string().optional()
+          note: import_zod6.z.string().optional()
         }),
-        import_zod5.z.object({
-          state: import_zod5.z.literal("graded"),
-          traceId: import_zod5.z.string(),
-          value: import_zod5.z.number().finite(),
-          note: import_zod5.z.string().optional()
+        import_zod6.z.object({
+          state: import_zod6.z.literal("graded"),
+          traceId: import_zod6.z.string(),
+          value: import_zod6.z.number().finite(),
+          note: import_zod6.z.string().optional()
         }),
-        import_zod5.z.object({ state: import_zod5.z.literal("errored"), traceId: import_zod5.z.string(), message: import_zod5.z.string() }),
-        import_zod5.z.object({
-          state: import_zod5.z.literal("ungraded"),
-          traceId: import_zod5.z.string(),
-          reason: import_zod5.z.enum(["not sampled", "trace unavailable", "content unreadable"])
+        import_zod6.z.object({ state: import_zod6.z.literal("errored"), traceId: import_zod6.z.string(), message: import_zod6.z.string() }),
+        import_zod6.z.object({
+          state: import_zod6.z.literal("ungraded"),
+          traceId: import_zod6.z.string(),
+          reason: import_zod6.z.enum(["not sampled", "trace unavailable", "content unreadable"])
         })
       ])
     )
   })
 });
-var WireLocalEvalRunSchema = import_zod5.z.object({
-  data: WireEvalRunSchema.shape.data.extend({ status: import_zod5.z.literal("completed"), url: import_zod5.z.string() })
+var WireLocalEvalRunSchema = import_zod6.z.object({
+  data: WireEvalRunSchema.shape.data.extend({ status: import_zod6.z.literal("completed"), url: import_zod6.z.string() })
 });
-var WireLocalEvalRunStartedSchema = import_zod5.z.object({
-  data: WireEvalRunSchema.shape.data.omit({ traces: true }).extend({ status: import_zod5.z.literal("running"), url: import_zod5.z.string() })
+var WireLocalEvalRunStartedSchema = import_zod6.z.object({
+  data: WireEvalRunSchema.shape.data.omit({ traces: true }).extend({ status: import_zod6.z.literal("running"), url: import_zod6.z.string() })
 });
-var WireLocalEvalAttemptSchema = import_zod5.z.object({
-  data: import_zod5.z.object({
-    runId: import_zod5.z.string().uuid(),
-    requestId: import_zod5.z.string().uuid(),
-    rowId: import_zod5.z.string().min(1),
-    attempt: import_zod5.z.number().int().nonnegative(),
-    traceId: import_zod5.z.string().regex(/^[a-f0-9]{32}$/),
+var WireLocalEvalAttemptSchema = import_zod6.z.object({
+  data: import_zod6.z.object({
+    runId: import_zod6.z.string().uuid(),
+    requestId: import_zod6.z.string().uuid(),
+    rowId: import_zod6.z.string().min(1),
+    attempt: import_zod6.z.number().int().nonnegative(),
+    traceId: import_zod6.z.string().regex(/^[a-f0-9]{32}$/),
     outcome: WireEvalRunSchema.shape.data.shape.traces.element
   })
 });
-var WireEvalCaseTracePairReadSchema = import_zod5.z.discriminatedUnion("state", [
+var WireEvalCaseTracePairReadSchema = import_zod6.z.discriminatedUnion("state", [
   EvalCaseTracePairSchema.extend({
-    state: import_zod5.z.literal("ready"),
-    attempt: import_zod5.z.number().int().nonnegative(),
-    datasetVersion: import_zod5.z.object({
-      id: import_zod5.z.string().uuid(),
+    state: import_zod6.z.literal("ready"),
+    attempt: import_zod6.z.number().int().nonnegative(),
+    datasetVersion: import_zod6.z.object({
+      id: import_zod6.z.string().uuid(),
       fingerprint: EvalDatasetFingerprintSchema
     })
   }),
-  import_zod5.z.object({
-    state: import_zod5.z.literal("pending"),
-    caseId: import_zod5.z.string().min(1),
-    attempt: import_zod5.z.number().int().nonnegative()
+  import_zod6.z.object({
+    state: import_zod6.z.literal("pending"),
+    caseId: import_zod6.z.string().min(1),
+    attempt: import_zod6.z.number().int().nonnegative()
   }),
-  import_zod5.z.object({
-    state: import_zod5.z.literal("missing"),
-    caseId: import_zod5.z.string().min(1),
-    attempt: import_zod5.z.number().int().nonnegative(),
-    reason: import_zod5.z.string()
+  import_zod6.z.object({
+    state: import_zod6.z.literal("missing"),
+    caseId: import_zod6.z.string().min(1),
+    attempt: import_zod6.z.number().int().nonnegative(),
+    reason: import_zod6.z.string()
   }),
-  import_zod5.z.object({
-    state: import_zod5.z.literal("expired"),
-    caseId: import_zod5.z.string().min(1),
-    attempt: import_zod5.z.number().int().nonnegative(),
-    expiredAt: import_zod5.z.string().datetime({ offset: true })
+  import_zod6.z.object({
+    state: import_zod6.z.literal("expired"),
+    caseId: import_zod6.z.string().min(1),
+    attempt: import_zod6.z.number().int().nonnegative(),
+    expiredAt: import_zod6.z.string().datetime({ offset: true })
   }),
-  import_zod5.z.object({
-    state: import_zod5.z.literal("errored"),
-    caseId: import_zod5.z.string().min(1),
-    attempt: import_zod5.z.number().int().nonnegative(),
-    reason: import_zod5.z.string()
+  import_zod6.z.object({
+    state: import_zod6.z.literal("errored"),
+    caseId: import_zod6.z.string().min(1),
+    attempt: import_zod6.z.number().int().nonnegative(),
+    reason: import_zod6.z.string()
   })
 ]);
 var replayGitSnapshots = /* @__PURE__ */ new WeakMap();
@@ -17545,7 +17041,7 @@ async function readEvalDataset(client, dataset, options) {
   var _a;
   const slug = EvalDatasetSlugSchema.parse(dataset.trim());
   const versionId = options == null ? void 0 : options.versionId;
-  if (versionId !== void 0 && !import_zod5.z.string().uuid().safeParse(versionId).success) {
+  if (versionId !== void 0 && !import_zod6.z.string().uuid().safeParse(versionId).success) {
     throw new Error("Raindrop eval dataset versionId must be a UUID");
   }
   const path = versionId ? `/v1/eval-datasets/${encodeURIComponent(slug)}/versions/${encodeURIComponent(versionId)}` : `/v1/eval-datasets/${encodeURIComponent(slug)}`;
@@ -17558,7 +17054,7 @@ async function readEvalDataset(client, dataset, options) {
       schema: EvalDatasetManifestSchema
     })
   );
-  if (manifest.dataset.slug !== slug) {
+  if (manifest.dataset.slug !== slug && manifest.dataset.id !== slug) {
     throw new Error(`Raindrop eval dataset ${slug} returned mismatched dataset identity`);
   }
   if (versionId !== void 0 && manifest.version.id !== versionId) {
@@ -18151,7 +17647,7 @@ async function validateReferenceRequirements(client, dataset, rowIds, evaluators
       queryUrl,
       path: `/v1/evals/${encodeURIComponent(evaluator)}`,
       method: "GET",
-      schema: import_zod5.z.object({ data: import_zod5.z.object({ requiresReference: import_zod5.z.boolean() }) })
+      schema: import_zod6.z.object({ data: import_zod6.z.object({ requiresReference: import_zod6.z.boolean() }) })
     })).data.requiresReference : evaluator.requiresReference === true;
     if (requiresReference) {
       const name = typeof evaluator === "string" ? evaluator : evaluator.slug;
@@ -18616,7 +18112,7 @@ function validateEvaluatorPins(evaluators, raw) {
     if (!allowed.has(slug)) {
       throw new Error(`Raindrop evaluator pin names unknown evaluator ${slug}`);
     }
-    if (!import_zod5.z.string().uuid().safeParse(pin.expectedEvalId).success) {
+    if (!import_zod6.z.string().uuid().safeParse(pin.expectedEvalId).success) {
       throw new Error(`Raindrop evaluator ${slug} pin has an invalid evaluator id`);
     }
     if (!Number.isInteger(pin.expectedProgramVersion) || pin.expectedProgramVersion <= 0) {
@@ -18671,12 +18167,15 @@ function describeError(cause) {
   return String(cause);
 }
 
+// src/evals/index.ts
+init_manifest();
+
 // src/evals/agent.ts
-var import_zod6 = require("zod");
-var AgentMetadataSchema = import_zod6.z.object({
-  slug: import_zod6.z.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
-  name: import_zod6.z.string().trim().min(1).max(120),
-  description: import_zod6.z.string().max(2e3).optional()
+var import_zod7 = require("zod");
+var AgentMetadataSchema = import_zod7.z.object({
+  slug: import_zod7.z.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
+  name: import_zod7.z.string().trim().min(1).max(120),
+  description: import_zod7.z.string().max(2e3).optional()
 });
 async function finishWith(execution, work) {
   let result;
@@ -18691,11 +18190,11 @@ async function finishWith(execution, work) {
 }
 
 // src/evals/server.ts
-var import_zod7 = require("zod");
-var InvocationSchema = import_zod7.z.object({
-  runId: import_zod7.z.string().uuid(),
-  agent: import_zod7.z.string().min(1).max(64),
-  parameters: import_zod7.z.record(import_zod7.z.unknown()).default({})
+var import_zod8 = require("zod");
+var InvocationSchema = import_zod8.z.object({
+  runId: import_zod8.z.string().uuid(),
+  agent: import_zod8.z.string().min(1).max(64),
+  parameters: import_zod8.z.record(import_zod8.z.unknown()).default({})
 });
 var MAX_BODY_BYTES = 64 * 1024;
 
@@ -18851,7 +18350,7 @@ function sleep2(ms) {
 }
 
 // src/evals/publish.ts
-var import_zod8 = require("zod");
+var import_zod9 = require("zod");
 init_definition();
 
 // src/evals/runs.ts
@@ -18898,19 +18397,19 @@ async function readEvalRun(client, runId, options = {}) {
 }
 
 // src/evals/publish.ts
-var EvaluatorIdentitySchema = import_zod8.z.object({
-  id: import_zod8.z.string().uuid(),
-  slug: import_zod8.z.string(),
-  programVersion: import_zod8.z.number().int().positive(),
-  outputType: import_zod8.z.enum(["boolean", "score", "number"])
+var EvaluatorIdentitySchema = import_zod9.z.object({
+  id: import_zod9.z.string().uuid(),
+  slug: import_zod9.z.string(),
+  programVersion: import_zod9.z.number().int().positive(),
+  outputType: import_zod9.z.enum(["boolean", "score", "number"])
 });
 var EvaluatorSourceSchema = EvaluatorIdentitySchema.extend({
-  name: import_zod8.z.string(),
-  description: import_zod8.z.string().nullable(),
-  intent: import_zod8.z.string(),
-  kind: import_zod8.z.enum(["judge", "deterministic"]),
-  programSource: import_zod8.z.string(),
-  rules: import_zod8.z.array(import_zod8.z.string())
+  name: import_zod9.z.string(),
+  description: import_zod9.z.string().nullable(),
+  intent: import_zod9.z.string(),
+  kind: import_zod9.z.enum(["judge", "deterministic"]),
+  programSource: import_zod9.z.string(),
+  rules: import_zod9.z.array(import_zod9.z.string())
 });
 var EvalPublishError = class extends Error {
   constructor(status, message) {
@@ -18919,12 +18418,12 @@ var EvalPublishError = class extends Error {
     this.name = "EvalPublishError";
   }
 };
-var ComparisonRunSchema = import_zod8.z.object({
-  id: import_zod8.z.string().uuid(),
-  programVersion: import_zod8.z.number().int().positive()
+var ComparisonRunSchema = import_zod9.z.object({
+  id: import_zod9.z.string().uuid(),
+  programVersion: import_zod9.z.number().int().positive()
 });
-var ComparisonLinkSchema = import_zod8.z.object({
-  url: import_zod8.z.string().url(),
+var ComparisonLinkSchema = import_zod9.z.object({
+  url: import_zod9.z.string().url(),
   first: ComparisonRunSchema,
   second: ComparisonRunSchema
 });
@@ -18952,7 +18451,7 @@ async function compareEvalRuns(client, options) {
   }
   const before = selected.id;
   const after = counterpart.id;
-  const { data } = import_zod8.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
+  const { data } = import_zod9.z.object({ data: ComparisonLinkSchema }).parse(await evalRequest(client, `/v1/eval-runs/${before}/compare/${after}`, options));
   if (data.first.id !== before || data.second.id !== after) {
     throw new Error("Raindrop returned a different comparison pair");
   }
@@ -18965,7 +18464,7 @@ async function compareEvalRuns(client, options) {
   };
 }
 async function readEvaluator(client, slug, options = {}) {
-  const { data } = import_zod8.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
+  const { data } = import_zod9.z.object({ data: EvaluatorSourceSchema }).parse(await evalRequest(client, `/v1/evals/${encodeURIComponent(slug)}`, options));
   if (data.slug !== slug) throw new Error("Raindrop returned a different evaluator slug");
   return defineEvaluatorProgram({
     slug: data.slug,
@@ -18985,15 +18484,11 @@ async function publishEvalSuite(client, definition, options = {}) {
   const localDataset = isEvalDataset(suite.dataset) ? suite.dataset : void 0;
   const slug = EvalDatasetSlugSchema.parse(localDataset ? localDataset.id : suite.dataset);
   const rows = localDataset == null ? void 0 : localDataset.rows.map((row) => {
-    if (row.output !== null) {
-      throw new Error(
-        "Publish reference evidence with publishEvalDataset first; a local row output is not a reference trace"
-      );
-    }
     return EvalDatasetCaseInputSchema.parse({
       id: row.id,
       name: row.name,
       input: row.input,
+      output: row.output === null ? void 0 : row.output,
       properties: row.properties
     });
   });
@@ -19040,7 +18535,7 @@ async function publishEvalSuite(client, definition, options = {}) {
   });
   return defineEvalSuite({
     ...suite,
-    dataset: slug,
+    dataset: dataset.dataset.slug,
     datasetVersionId: dataset.version.id,
     evaluators
   });
@@ -19093,6 +18588,7 @@ init_portable();
 // src/evals/pull.ts
 var import_zod11 = require("zod");
 init_definition();
+init_trace();
 init_portable();
 var SNAPSHOT_VERSION = 1;
 var PulledEvaluatorSchema = import_zod11.z.object({
@@ -19113,7 +18609,7 @@ var EvalSnapshotSchema = import_zod11.z.object({
         id: import_zod11.z.string(),
         name: import_zod11.z.string(),
         input: import_zod11.z.string().nullable(),
-        output: import_zod11.z.string().nullable(),
+        output: JsonValueSchema,
         properties: import_zod11.z.record(import_zod11.z.string())
       })
     )
@@ -19122,9 +18618,7 @@ var EvalSnapshotSchema = import_zod11.z.object({
 });
 
 // src/evals/run.ts
-var import_api8 = require("@opentelemetry/api");
 init_definition();
-init_trace();
 
 // src/evals/policy.ts
 init_definition();
@@ -19169,132 +18663,6 @@ function outputMismatch(name, output) {
 }
 
 // src/evals/run.ts
-init_portable();
-
-// src/evals/workshop.ts
-var import_zod12 = require("zod");
-var WorkshopRunSchema = import_zod12.z.object({
-  id: import_zod12.z.string(),
-  event_id: import_zod12.z.string().nullable(),
-  name: import_zod12.z.string().nullable(),
-  event_name: import_zod12.z.string().nullable(),
-  user_id: import_zod12.z.string().nullable(),
-  convo_id: import_zod12.z.string().nullable(),
-  started_at: import_zod12.z.number().nullable(),
-  last_updated_at: import_zod12.z.number().nullable(),
-  metadata: import_zod12.z.string().nullable(),
-  model: import_zod12.z.string().nullable(),
-  finished: import_zod12.z.number().nullable()
-});
-var WorkshopSpanSchema = import_zod12.z.object({
-  id: import_zod12.z.string(),
-  run_id: import_zod12.z.string(),
-  parent_span_id: import_zod12.z.string().nullable(),
-  name: import_zod12.z.string(),
-  span_type: import_zod12.z.string().nullable(),
-  status: import_zod12.z.string().nullable(),
-  input_payload: import_zod12.z.string().nullable(),
-  output_payload: import_zod12.z.string().nullable(),
-  start_time_ms: import_zod12.z.number().nullable(),
-  end_time_ms: import_zod12.z.number().nullable(),
-  duration_ms: import_zod12.z.number().nullable(),
-  model: import_zod12.z.string().nullable(),
-  provider: import_zod12.z.string().nullable(),
-  input_tokens: import_zod12.z.number().nullable(),
-  output_tokens: import_zod12.z.number().nullable(),
-  attributes: import_zod12.z.string().nullable()
-});
-var WorkshopTraceEnvelopeSchema = import_zod12.z.object({
-  status: import_zod12.z.literal("complete"),
-  correlationId: import_zod12.z.string(),
-  traceId: import_zod12.z.string(),
-  rootSpanId: import_zod12.z.string(),
-  spanCount: import_zod12.z.number().int().positive(),
-  run: WorkshopRunSchema,
-  spans: import_zod12.z.array(WorkshopSpanSchema).min(1)
-});
-async function readWorkshopTrace(input) {
-  if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
-    throw new Error("Workshop waitMs must be finite and non-negative");
-  const deadlineAt = Date.now() + input.waitMs;
-  if (input.expectedSpanIds.length === 0) {
-    throw new Error(`Workshop trace ${input.correlationId} exported no spans`);
-  }
-  const url = new URL(
-    `${credentialEndpoint(input.workshopUrl).href.replace(/\/+$/, "")}/v1/evals/traces/${encodeURIComponent(input.correlationId)}`
-  );
-  for (const spanId of input.expectedSpanIds) url.searchParams.append("expectedSpanId", spanId);
-  while (true) {
-    const response = await runWithTracingSuppressed(
-      () => fetch(url, {
-        redirect: "error",
-        signal: AbortSignal.timeout(
-          Math.min(3e4, Math.max(1, Math.ceil(deadlineAt - Date.now())))
-        )
-      })
-    );
-    if (response.ok) {
-      const parsed = WorkshopTraceEnvelopeSchema.safeParse(await response.json());
-      if (!parsed.success) {
-        throw new Error(`Workshop returned an invalid completed trace: ${parsed.error.message}`);
-      }
-      return parsed.data;
-    }
-    if (response.status !== 404 && response.status !== 409) {
-      throw new Error(
-        `Workshop trace read failed with ${response.status}: ${await response.text()}`
-      );
-    }
-    if (Date.now() >= deadlineAt) {
-      const state2 = response.status === 404 ? "missing" : "incomplete";
-      throw new Error(`Workshop trace ${input.correlationId} remained ${state2}`);
-    }
-    await new Promise(
-      (resolve3) => setTimeout(resolve3, Math.min(50, Math.max(0, deadlineAt - Date.now())))
-    );
-  }
-}
-async function writeWorkshopEvalRun(workshopUrl, packet) {
-  const normalized = {
-    ...packet,
-    cases: packet.cases.map((entry) => ({
-      ...entry,
-      task: entry.task.state === "completed" ? { ...entry.task, output: serializableOutput(entry.task.output) } : entry.task
-    }))
-  };
-  const response = await runWithTracingSuppressed(
-    () => fetch(`${credentialEndpoint(workshopUrl).href.replace(/\/+$/, "")}/v1/evals/runs`, {
-      method: "POST",
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify(normalized),
-      redirect: "error",
-      signal: AbortSignal.timeout(3e4)
-    })
-  );
-  if (!response.ok) {
-    throw new Error(
-      `Workshop eval result write failed with ${response.status}: ${await response.text()}`
-    );
-  }
-  const body = await response.json();
-  if (typeof body !== "object" || body === null || !("id" in body) || typeof body.id !== "string") {
-    throw new Error("Workshop eval result write returned an invalid response");
-  }
-  return { id: body.id };
-}
-function serializableOutput(value) {
-  try {
-    const serialized = JSON.stringify(boundedClone(value, 64e3));
-    if (serialized === void 0) return null;
-    if (serialized.length > 64e3) return "[Output exceeds 64000 characters]";
-    return JSON.parse(serialized);
-  } catch (e) {
-    return "[Output could not be serialized]";
-  }
-}
-
-// src/evals/run.ts
-var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
 function bindSuite(definition, parameters) {
   if (definition.agent === void 0) {
     const { run } = definition;
@@ -19306,66 +18674,41 @@ function bindSuite(definition, parameters) {
   return definition.agent.start(parameters);
 }
 function createSuiteSession(client, validated, run, options) {
-  if (validated.datasetVersionId && "selection" in options && options.selection.dataset.version.id !== validated.datasetVersionId) {
+  if (validated.datasetVersionId && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
-  if (options.destination.kind === "raindrop") {
-    if (!("selection" in options)) {
-      throw new Error("Raindrop remote eval sessions require selected rows");
-    }
-    if (typeof validated.dataset !== "string") {
-      throw new Error("Raindrop remote eval sessions require a dataset slug");
-    }
-    if (options.selection.dataset.dataset.slug !== validated.dataset) {
-      throw new Error(
-        `Raindrop eval ${validated.name} pinned dataset ${options.selection.dataset.dataset.slug} does not match ${validated.dataset}`
-      );
-    }
-    const localEvaluators = validated.evaluators.flatMap((entry) => {
-      const evaluator = replayEvaluator(entry);
-      return typeof evaluator === "string" ? [] : [evaluator];
-    });
-    if (localEvaluators.length !== validated.evaluators.length) {
-      throw new Error("Raindrop row-scoped eval sessions support local evaluators only");
-    }
-    const selection = validateRemoteSelection(validated.dataset, options.selection);
-    if (!selection) throw new Error("Raindrop remote eval sessions require selected rows");
-    return new RaindropRowEvalRun(
-      client,
-      validated,
-      run,
-      options.destination,
-      options.selection.dataset,
-      selection.rowIds,
-      localEvaluators
-    );
-  }
-  if (!isEvalDataset(validated.dataset)) {
-    throw new Error("Workshop eval execution requires a pulled or local dataset snapshot");
-  }
-  credentialEndpoint(options.destination.url);
-  const hosted = validated.evaluators.find((entry) => typeof entry.evaluator === "string");
-  if (hosted && typeof hosted.evaluator === "string") {
-    throw new Error(`Workshop evaluator ${hosted.evaluator} must be pulled before local execution`);
+  if (typeof validated.dataset !== "string") {
+    throw new Error("Raindrop eval sessions require a dataset slug");
   }
-  if (validated.evaluators.some(
-    (entry) => typeof entry.evaluator !== "string" && "kind" in entry.evaluator && entry.evaluator.kind === "program"
-  )) {
+  if (options.selection.dataset.dataset.id !== validated.dataset && options.selection.dataset.dataset.slug !== validated.dataset) {
     throw new Error(
-      "Workshop authored evaluator programs must be pulled as portable evaluators before local execution"
+      `Raindrop eval ${validated.name} pinned dataset ${options.selection.dataset.dataset.slug} does not match ${validated.dataset}`
     );
   }
-  return new WorkshopEvalRun(
+  const localEvaluators = validated.evaluators.flatMap((entry) => {
+    const evaluator = replayEvaluator(entry);
+    return typeof evaluator === "string" ? [] : [evaluator];
+  });
+  if (localEvaluators.length !== validated.evaluators.length) {
+    throw new Error("Raindrop row-scoped eval sessions support local evaluators only");
+  }
+  const selection = validateRemoteSelection(validated.dataset, options.selection);
+  if (!selection) throw new Error("Raindrop eval sessions require selected rows");
+  return new RaindropRowEvalRun(
     client,
-    { ...validated, dataset: validated.dataset },
+    validated,
     run,
-    options.destination
+    options.destination,
+    options.selection.dataset,
+    selection.rowIds,
+    localEvaluators
   );
 }
 async function runEvalSuite(client, definition, options = {}) {
-  var _a;
+  var _a, _b;
+  assertRaindropDestination((_a = options.destination) != null ? _a : { kind: "raindrop" });
   const validated = validateDefinition(definition);
-  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  const execution = bindSuite(validated, (_b = options.parameters) != null ? _b : {});
   return finishWith(execution, () => runSuite(client, validated, execution.run, options));
 }
 async function runSuite(client, definition, run, options) {
@@ -19375,12 +18718,6 @@ async function runSuite(client, definition, run, options) {
   if (validated.datasetVersionId && ((_b = options.selection) == null ? void 0 : _b.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
-  if (destination.kind === "workshop") {
-    if (options.runId !== void 0) throw new Error("Queued runs are only supported in Raindrop");
-    const session = createSuiteSession(client, validated, run, { destination });
-    await session.run({ selection: options.selection, attempt: options.attempt });
-    return session.finish();
-  }
   const alreadyPublished = isPublishedEvalSuite(validated);
   if (typeof validated.dataset === "string" && ((_c = options.selection) == null ? void 0 : _c.dataset)) {
     validateRemoteSelection(validated.dataset, options.selection);
@@ -19610,328 +18947,6 @@ function replayEvaluatorsFromVerdicts(verdicts) {
   }
   return [...evaluators.values()];
 }
-var WorkshopEvalRun = class {
-  constructor(client, definition, runRow, destination) {
-    this.client = client;
-    this.definition = definition;
-    this.runRow = runRow;
-    this.destination = destination;
-    this.id = randomId();
-    this.startedAt = /* @__PURE__ */ new Date();
-    this.completed = [];
-    this.active = /* @__PURE__ */ new Set();
-    this.state = "open";
-    this.runningRows = 0;
-    this.rowWaiters = [];
-  }
-  async run(options = {}) {
-    if (this.state !== "open") throw new Error(`Raindrop eval run ${this.id} is already finishing`);
-    const operation = this.runOnce(options);
-    this.active.add(operation);
-    try {
-      return await operation;
-    } finally {
-      this.active.delete(operation);
-    }
-  }
-  async runOnce(options) {
-    var _a;
-    const rows = selectLocalRows(this.definition, options.selection);
-    const completed = await Promise.all(
-      rows.map((row) => this.withRowPermit(() => {
-        var _a2;
-        return this.executeRow(row, (_a2 = options.attempt) != null ? _a2 : 0);
-      }))
-    );
-    const graded = await this.applyPortableEvaluators(completed, (_a = options.attempt) != null ? _a : 0);
-    this.completed.push(...graded);
-    return workshopResult(
-      this.id,
-      this.definition.name,
-      graded.map((entry) => entry.result)
-    );
-  }
-  finish() {
-    if (this.finishPromise) return this.finishPromise;
-    this.state = "finishing";
-    this.finishPromise = this.finishOnce();
-    return this.finishPromise;
-  }
-  async finishOnce() {
-    await Promise.allSettled([...this.active]);
-    const result = workshopResult(
-      this.id,
-      this.definition.name,
-      this.completed.map((entry) => entry.result)
-    );
-    if (this.completed.length > 0) {
-      const packet = {
-        schemaVersion: 1,
-        run: {
-          id: this.id,
-          name: this.definition.name,
-          destination: "workshop",
-          startedAt: this.startedAt.toISOString(),
-          finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
-          dataset: {
-            id: this.definition.dataset.id,
-            name: this.definition.dataset.name,
-            version: this.definition.dataset.version
-          },
-          evaluators: this.definition.evaluators.map((entry) => {
-            if (typeof entry.evaluator === "string") {
-              throw new Error(
-                `Workshop evaluator ${entry.evaluator} must be pulled before local execution`
-              );
-            }
-            if (isPortableEvaluator(entry.evaluator)) {
-              return {
-                slug: entry.evaluator.slug,
-                name: entry.evaluator.name,
-                output: entry.evaluator.output,
-                execution: "portable",
-                version: entry.evaluator.artifact.evaluator.programVersion,
-                artifactSha256: entry.evaluator.artifact.sha256,
-                scope: entry.evaluator.scope,
-                threshold: entry.threshold
-              };
-            }
-            return {
-              slug: entry.evaluator.slug,
-              name: entry.evaluator.name,
-              output: entry.evaluator.output,
-              execution: "local",
-              scope: entry.evaluator.scope === "row" ? "case" : entry.evaluator.scope,
-              threshold: entry.threshold
-            };
-          })
-        },
-        cases: this.completed.map((entry) => entry.packet)
-      };
-      await writeWorkshopEvalRun(this.destination.url, packet);
-    }
-    this.state = "finished";
-    return result;
-  }
-  async executeRow(row, attempt) {
-    var _a, _b, _c;
-    const correlationId = randomId();
-    const exportedSpanIds = /* @__PURE__ */ new Set();
-    const startedAt = Date.now();
-    let completedAgent;
-    let taskError;
-    try {
-      const destination = {
-        kind: "workshop",
-        url: this.destination.url,
-        exportedSpanIds
-      };
-      const value = await withReplayTraceDestination(
-        destination,
-        () => withEvalScope(
-          { correlationId },
-          () => this.client.tracer().withSpan(
-            {
-              name: "raindrop.eval.case",
-              properties: {
-                "raindrop.eval.run_id": this.id,
-                "raindrop.eval.case_id": row.id,
-                "raindrop.eval.attempt": String(attempt)
-              },
-              inputParameters: [row],
-              traceContent: true
-            },
-            () => {
-              var _a2;
-              const parent = (_a2 = import_api8.trace.getSpan(import_api8.context.active())) == null ? void 0 : _a2.spanContext();
-              if (!parent) return this.runRow(row);
-              return withReplayTraceDestination(
-                {
-                  ...destination,
-                  parentSpanContext: {
-                    traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
-                    spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
-                    eventId: parent.traceId
-                  }
-                },
-                () => this.runRow(row)
-              );
-            }
-          )
-        )
-      );
-      completedAgent = { value };
-    } catch (cause) {
-      taskError = describeError2(cause);
-    }
-    const durationMs = Date.now() - startedAt;
-    const taskCompleted = completedAgent !== void 0;
-    let trace8;
-    let traceId = null;
-    let traceError;
-    try {
-      await this.client.forceFlush();
-      const envelope = await readWorkshopTrace({
-        workshopUrl: this.destination.url,
-        correlationId,
-        waitMs: (_a = this.definition.traceWaitMs) != null ? _a : DEFAULT_WORKSHOP_TRACE_WAIT_MS,
-        expectedSpanIds: [...exportedSpanIds]
-      });
-      traceId = envelope.traceId;
-      trace8 = canonicalWorkshopTrace(envelope);
-    } catch (cause) {
-      traceError = describeError2(cause);
-    }
-    const verdicts = [];
-    if (completedAgent && !traceError && trace8) {
-      const agentResult = completedAgent.value;
-      await Promise.all(
-        this.definition.evaluators.map(async (entry) => {
-          if (typeof entry.evaluator === "string") return;
-          if (isPortableEvaluator(entry.evaluator)) return;
-          if ("kind" in entry.evaluator && entry.evaluator.kind === "program") return;
-          try {
-            const verdict = parseLocalVerdict2(
-              entry.evaluator.output,
-              await entry.evaluator.judge({ trace: trace8, row, result: agentResult })
-            );
-            verdicts.push(
-              withVerdictIdentity(entry.evaluator.slug, this.id, this.destination.url, verdict)
-            );
-          } catch (cause) {
-            verdicts.push({
-              evaluator: entry.evaluator.slug,
-              runId: this.id,
-              url: workshopEvalUrl(this.destination.url, this.id),
-              state: "errored",
-              message: describeError2(cause)
-            });
-          }
-        })
-      );
-    }
-    if (taskCompleted && traceError) {
-      for (const entry of this.definition.evaluators) {
-        if (typeof entry.evaluator === "string") continue;
-        verdicts.push({
-          evaluator: entry.evaluator.slug,
-          runId: this.id,
-          url: workshopEvalUrl(this.destination.url, this.id),
-          state: "ungraded",
-          reason: "trace unavailable"
-        });
-      }
-    }
-    const assessments = verdicts.map((verdict) => {
-      const evaluator = this.definition.evaluators.find(
-        (entry) => evaluatorName(entry) === verdict.evaluator
-      );
-      if (!evaluator)
-        throw new Error(`Raindrop eval result included unknown evaluator ${verdict.evaluator}`);
-      return assessEvalVerdict(evaluator, verdict, evaluatorOutput(evaluator));
-    });
-    const error = (_b = taskError != null ? taskError : traceError) != null ? _b : null;
-    const result = {
-      ...row,
-      attempt,
-      status: error ? "missing" : "done",
-      traceId,
-      error,
-      verdicts: assessments
-    };
-    const packet = {
-      caseId: row.id,
-      caseName: row.name,
-      input: row.input,
-      ...row.output === null ? {} : { expected: row.output },
-      attempt,
-      traceId,
-      ...taskCompleted && traceError ? { traceError } : {},
-      task: taskError ? { state: "failed", error: taskError, durationMs } : { state: "completed", output: (_c = completedAgent == null ? void 0 : completedAgent.value) != null ? _c : null, durationMs },
-      verdicts: verdicts.map(workshopVerdict),
-      passed: error === null && assessments.every(
-        (assessment) => assessment.state === "passed" || assessment.state === "measurement"
-      )
-    };
-    return { result, packet, trace: trace8 };
-  }
-  async withRowPermit(run) {
-    var _a;
-    const limit = (_a = this.definition.concurrency) != null ? _a : 4;
-    if (this.runningRows >= limit) {
-      await new Promise((resolve3) => this.rowWaiters.push(resolve3));
-    } else {
-      this.runningRows += 1;
-    }
-    try {
-      return await run();
-    } finally {
-      const next = this.rowWaiters.shift();
-      if (next) next();
-      else this.runningRows -= 1;
-    }
-  }
-  async applyPortableEvaluators(completed, attempt) {
-    let current = completed;
-    for (const entry of this.definition.evaluators) {
-      if (typeof entry.evaluator === "string" || !isPortableEvaluator(entry.evaluator)) continue;
-      const portableEvaluator = entry.evaluator;
-      const traces = current.flatMap(
-        (item) => item.packet.task.state === "completed" && item.trace ? [item.trace] : []
-      );
-      if (traces.length === 0) continue;
-      let outcomes;
-      try {
-        const portable = await evaluatePortableEvaluator(portableEvaluator, {
-          traces,
-          seed: `${this.id}:${attempt}:${portableEvaluator.slug}`
-        });
-        const programFailure = portable.failures.join("; ");
-        outcomes = new Map(
-          portable.outcomes.map((outcome) => [
-            outcome.traceId,
-            programFailure ? {
-              state: "errored",
-              traceId: outcome.traceId,
-              message: programFailure
-            } : outcome
-          ])
-        );
-      } catch (cause) {
-        const message = describeError2(cause);
-        outcomes = new Map(
-          traces.map((trace8) => [
-            trace8.event.id,
-            { state: "errored", traceId: trace8.event.id, message }
-          ])
-        );
-      }
-      current = current.map((item) => {
-        if (item.packet.task.state !== "completed" || !item.trace) return item;
-        const outcome = outcomes.get(item.trace.event.id);
-        if (!outcome) return item;
-        const verdict = portableVerdict(
-          portableEvaluator.slug,
-          this.id,
-          this.destination.url,
-          outcome
-        );
-        const assessment = assessEvalVerdict(entry, verdict, portableEvaluator.output);
-        const verdicts = [...item.result.verdicts, assessment];
-        return {
-          ...item,
-          result: { ...item.result, verdicts },
-          packet: {
-            ...item.packet,
-            verdicts: [...item.packet.verdicts, workshopVerdict(verdict)],
-            passed: item.packet.passed && (assessment.state === "passed" || assessment.state === "measurement")
-          }
-        };
-      });
-    }
-    return current;
-  }
-};
 async function runRaindropEval(client, definition, run, destination, options) {
   var _a, _b, _c, _d;
   if (typeof definition.dataset !== "string") {
@@ -19944,7 +18959,7 @@ async function runRaindropEval(client, definition, run, destination, options) {
     queryUrl: destination.queryUrl,
     versionId: definition.datasetVersionId
   }) : void 0;
-  if (pinnedDataset && pinnedDataset.dataset.slug !== datasetReference) {
+  if (pinnedDataset && pinnedDataset.dataset.id !== datasetReference && pinnedDataset.dataset.slug !== datasetReference) {
     throw new Error(
       `Raindrop eval ${definition.name} pinned dataset ${pinnedDataset.dataset.slug} does not match ${datasetReference}`
     );
@@ -20073,20 +19088,6 @@ function validateDefinition(definition) {
     throw new Error("Raindrop runEvalSuite requires a definition created by defineEvalSuite");
   return definition;
 }
-function selectLocalRows(definition, selection) {
-  if (!selection) return definition.dataset.rows;
-  if (selection.rowIds.length === 0)
-    throw new Error(`Raindrop eval ${definition.name} selection cannot be empty`);
-  if (new Set(selection.rowIds).size !== selection.rowIds.length)
-    throw new Error(`Raindrop eval ${definition.name} selection row ids must be unique`);
-  const byId = new Map(definition.dataset.rows.map((row) => [row.id, row]));
-  return selection.rowIds.map((id) => {
-    const row = byId.get(id);
-    if (!row)
-      throw new Error(`Raindrop eval ${definition.name} selection contains unknown row ${id}`);
-    return row;
-  });
-}
 function validateRemoteSelection(dataset, selection) {
   var _a;
   if (!selection) return void 0;
@@ -20102,157 +19103,20 @@ function validateRemoteSelection(dataset, selection) {
     throw new Error(`Raindrop eval ${dataset} selection contains unknown row ${unknown}`);
   return { rowIds: selection.rowIds };
 }
-function canonicalWorkshopTrace(envelope) {
-  var _a, _b, _c, _d, _e, _f;
-  const spans = envelope.spans.map(rawTraceSpan);
-  const timestamp = new Date((_a = envelope.run.started_at) != null ? _a : Date.now()).toISOString();
-  const event = projectCapturedEvent({
-    eventId: (_b = envelope.run.event_id) != null ? _b : envelope.traceId,
-    internalEventId: envelope.run.id,
-    name: (_d = (_c = envelope.run.event_name) != null ? _c : envelope.run.name) != null ? _d : "ai_generation",
-    timestamp,
-    receivedAt: new Date((_e = envelope.run.last_updated_at) != null ? _e : Date.now()).toISOString(),
-    userId: (_f = envelope.run.user_id) != null ? _f : "local-eval",
-    convoId: envelope.run.convo_id,
-    properties: parseAttributes(envelope.run.metadata),
-    spans
-  });
-  return projectWorkshopTrace({ origin: "replay", event, spans });
-}
-function rawTraceSpan(span) {
-  var _a, _b, _c, _d, _e, _f;
-  const attributes = parseAttributes(span.attributes);
-  return {
-    span_id: span.id,
-    parent_span_id: (_a = span.parent_span_id) != null ? _a : void 0,
-    span_name: span.name,
-    span_type: (_b = span.span_type) != null ? _b : void 0,
-    status: (_c = span.status) != null ? _c : void 0,
-    duration_ns: span.duration_ms === null ? void 0 : span.duration_ms * 1e6,
-    start_unix_ns: span.start_time_ms === null ? void 0 : span.start_time_ms * 1e6,
-    model: (_d = span.model) != null ? _d : void 0,
-    input_tokens: (_e = span.input_tokens) != null ? _e : void 0,
-    output_tokens: (_f = span.output_tokens) != null ? _f : void 0,
-    input_payload: span.input_payload,
-    output_payload: span.output_payload,
-    attributes_string: Object.fromEntries(
-      Object.entries(attributes).filter(
-        (entry) => typeof entry[1] === "string"
-      )
-    ),
-    attributes_num: Object.fromEntries(
-      Object.entries(attributes).filter(
-        (entry) => typeof entry[1] === "number"
-      )
-    )
-  };
-}
-function parseAttributes(raw) {
-  if (!raw) return {};
-  try {
-    const value = JSON.parse(raw);
-    return isAttributeRecord(value) ? value : {};
-  } catch (e) {
-    return {};
-  }
-}
-function isAttributeRecord(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function parseLocalVerdict2(output, value) {
-  if (output === "boolean") return { state: "graded", ...LocalBooleanVerdictSchema.parse(value) };
-  if (output === "score") return { state: "graded", ...LocalScoreVerdictSchema.parse(value) };
-  return { state: "graded", ...LocalNumberVerdictSchema.parse(value) };
-}
-function withVerdictIdentity(evaluator, runId, workshopUrl, verdict) {
-  return {
-    evaluator,
-    runId,
-    url: workshopEvalUrl(workshopUrl, runId),
-    ...verdict
-  };
-}
-function workshopVerdict(assessment) {
-  const { evaluator, state: state2 } = assessment;
-  if (state2 === "errored") return { evaluator, state: state2, message: assessment.message };
-  if (state2 === "ungraded") return { evaluator, state: state2, reason: assessment.reason };
-  if ("pass" in assessment)
-    return {
-      evaluator,
-      state: state2,
-      pass: assessment.pass,
-      ...assessment.note ? { note: assessment.note } : {}
-    };
-  if ("score" in assessment)
-    return {
-      evaluator,
-      state: state2,
-      score: assessment.score,
-      ...assessment.note ? { note: assessment.note } : {}
-    };
-  return {
-    evaluator,
-    state: state2,
-    value: assessment.value,
-    ...assessment.note ? { note: assessment.note } : {}
-  };
-}
-function portableVerdict(evaluator, runId, workshopUrl, outcome) {
-  const identity = { evaluator, runId, url: workshopEvalUrl(workshopUrl, runId) };
-  if (outcome.state === "errored") {
-    return { ...identity, state: outcome.state, message: outcome.message };
-  }
-  if (outcome.state === "ungraded") {
-    return { ...identity, state: outcome.state, reason: outcome.reason };
-  }
-  if ("pass" in outcome)
-    return {
-      ...identity,
-      state: outcome.state,
-      pass: outcome.pass,
-      ...outcome.note ? { note: outcome.note } : {}
-    };
-  if ("score" in outcome)
-    return {
-      ...identity,
-      state: outcome.state,
-      score: outcome.score,
-      ...outcome.note ? { note: outcome.note } : {}
-    };
-  return {
-    ...identity,
-    state: outcome.state,
-    value: outcome.value,
-    ...outcome.note ? { note: outcome.note } : {}
-  };
-}
-function workshopResult(id, name, rows) {
-  const done = rows.filter((entry) => entry.status === "done").length;
-  const missing = rows.filter((entry) => entry.status === "missing").length;
-  return {
-    runId: id,
-    name,
-    status: missing > 0 ? "failed" : "complete",
-    counts: { total: rows.length, pending: 0, done, missing },
-    tracesExpireAt: null,
-    tracesExpired: false,
-    evaluators: [],
-    rows
-  };
-}
-function workshopEvalUrl(baseUrl, id) {
-  return `${baseUrl.replace(/\/+$/, "")}/evals/${encodeURIComponent(id)}`;
-}
 function randomId() {
   var _a, _b, _c;
   return (_c = (_b = (_a = globalThis.crypto) == null ? void 0 : _a.randomUUID) == null ? void 0 : _b.call(_a)) != null ? _c : `eval-${Date.now()}-${Math.random().toString(16).slice(2)}`;
 }
-function describeError2(cause) {
-  return cause instanceof Error ? cause.message : String(cause);
-}
 function isPortableEvaluator(value) {
   return "kind" in value && value.kind === "portable";
 }
+function assertRaindropDestination(destination) {
+  if (destination.kind !== "raindrop") {
+    throw new Error(
+      "Eval runs store results in Raindrop; Workshop is no longer an eval destination"
+    );
+  }
+}
 
 // src/index.ts
 function rawAppGitProperties(properties) {
@@ -20720,14 +19584,14 @@ var Raindrop = class {
     const description = buildSelfDiagnosticsDescription(signals, options.guidance);
     const inputSchema = buildSelfDiagnosticsInputSchema(signalKeys);
     const onMissingEventId = (_b = options.onMissingEventId) != null ? _b : "warn";
-    const execute = async (rawInput, context9) => {
+    const execute = async (rawInput, context8) => {
       var _a2, _b2, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
       const input = isRecord(rawInput) ? rawInput : void 0;
       const fallbackCategory = (_a2 = signalKeys[0]) != null ? _a2 : "unknown";
       const categoryCandidate = normalizeEventId(input == null ? void 0 : input["category"]);
       const category = categoryCandidate && signalKeySet.has(categoryCandidate) ? categoryCandidate : fallbackCategory;
       const detail = (_b2 = normalizeEventId(input == null ? void 0 : input["detail"])) != null ? _b2 : "";
-      const eventId = (_l = (_k = (_i = (_g = (_f = (_e = (_d = normalizeEventId(context9 == null ? void 0 : context9.eventId)) != null ? _d : (_c = context9 == null ? void 0 : context9.interaction) == null ? void 0 : _c.getEventId()) != null ? _e : resolveEventIdFromMetadata(context9 == null ? void 0 : context9.metadata)) != null ? _f : normalizeEventId(input == null ? void 0 : input["event_id"])) != null ? _g : normalizeEventId(input == null ? void 0 : input["eventId"])) != null ? _i : (_h = options.interaction) == null ? void 0 : _h.getEventId()) != null ? _k : (_j = options.getEventId) == null ? void 0 : _j.call(options)) != null ? _l : normalizeEventId(options.eventId);
+      const eventId = (_l = (_k = (_i = (_g = (_f = (_e = (_d = normalizeEventId(context8 == null ? void 0 : context8.eventId)) != null ? _d : (_c = context8 == null ? void 0 : context8.interaction) == null ? void 0 : _c.getEventId()) != null ? _e : resolveEventIdFromMetadata(context8 == null ? void 0 : context8.metadata)) != null ? _f : normalizeEventId(input == null ? void 0 : input["event_id"])) != null ? _g : normalizeEventId(input == null ? void 0 : input["eventId"])) != null ? _i : (_h = options.interaction) == null ? void 0 : _h.getEventId()) != null ? _k : (_j = options.getEventId) == null ? void 0 : _j.call(options)) != null ? _l : normalizeEventId(options.eventId);
       if (!eventId) {
         const message = "self diagnostics signal skipped: missing eventId. Pass eventId, interaction, or getEventId when creating or executing the tool.";
         if (onMissingEventId === "throw") {
@@ -20745,11 +19609,11 @@ var Raindrop = class {
         type: "agent",
         sentiment: signalDefinition == null ? void 0 : signalDefinition.sentiment,
         properties: {
-          source: (context9 == null ? void 0 : context9.source) || options.source || SELF_DIAGNOSTICS_SOURCE_DEFAULT,
+          source: (context8 == null ? void 0 : context8.source) || options.source || SELF_DIAGNOSTICS_SOURCE_DEFAULT,
           category,
           signal_description: signalDefinition == null ? void 0 : signalDefinition.description,
           ...detail ? { detail } : {},
-          ...(_m = context9 == null ? void 0 : context9.properties) != null ? _m : {}
+          ...(_m = context8 == null ? void 0 : context8.properties) != null ? _m : {}
         }
       });
       return { acknowledged: true, category, eventId };
@@ -21155,7 +20019,7 @@ var Raindrop = class {
   }
   getContext() {
     const runtime = detectRuntime();
-    const context9 = {
+    const context8 = {
       library: {
         name: package_default.name,
         version: package_default.version
@@ -21165,9 +20029,9 @@ var Raindrop = class {
       }
     };
     if (runtime === "node") {
-      context9.metadata.nodeVersion = process.version;
+      context8.metadata.nodeVersion = process.version;
     }
-    return context9;
+    return context8;
   }
   formatZodError(error) {
     return error.issues.map((issue) => `(path: ${issue.path.join(".")}): ${issue.message}`).join(", ");
@@ -21609,8 +20473,8 @@ var index_default = Raindrop;
 
 // src/evals/cli.ts
 init_definition();
-var SuiteModuleSchema = import_zod13.z.object({
-  default: import_zod13.z.custom(isEvalSuiteDefinition)
+var SuiteModuleSchema = import_zod12.z.object({
+  default: import_zod12.z.custom(isEvalSuiteDefinition)
 });
 async function main() {
   var _a, _b;