raindrop-ai 0.8.0 → 0.9.0

package/dist/{chunk-EFATLUNL.mjs → chunk-UY6537WW.mjs}

@@ -1,77 +1,7 @@
-// src/evals/definition.ts
-import { createHash } from "crypto";
+// src/evals/generated/manifest.ts
 import { z } from "zod";
-var EvalScoreSchema = z.union([
-  z.literal(1),
-  z.literal(2),
-  z.literal(3),
-  z.literal(4),
-  z.literal(5)
-]);
-var EVAL_DATASET = /* @__PURE__ */ Symbol.for("raindrop.evalDataset");
-var DatasetRowInputSchema = z.object({
-  id: z.string().trim().min(1),
-  name: z.string().optional(),
-  input: z.string().nullable(),
-  output: z.string().nullable().default(null),
-  properties: z.record(z.string()).default({})
-});
-function defineDataset(dataset) {
-  var _a, _b;
-  const id = dataset.id.trim();
-  const name = dataset.name.trim();
-  const rows = dataset.rows.map((row) => {
-    var _a2;
-    const parsed = DatasetRowInputSchema.parse(row);
-    return {
-      id: parsed.id,
-      name: (_a2 = parsed.name) != null ? _a2 : parsed.id,
-      input: parsed.input,
-      output: parsed.output,
-      properties: parsed.properties
-    };
-  });
-  const version = (_b = (_a = dataset.version) == null ? void 0 : _a.trim()) != null ? _b : createHash("sha256").update(
-    JSON.stringify(
-      rows.map((row) => ({
-        ...row,
-        properties: Object.fromEntries(
-          Object.keys(row.properties).sort().map((key) => [key, row.properties[key]])
-        )
-      }))
-    )
-  ).digest("hex");
-  if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
-  if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
-  if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
-  const ids = rows.map((entry) => entry.id);
-  if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
-  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
-  if (dataset.remote) {
-    if (!dataset.remote.reference.trim())
-      throw new Error("Raindrop remote dataset reference cannot be empty");
-    if (!/^[a-f0-9]{64}$/.test(dataset.remote.fingerprint)) {
-      throw new Error("Raindrop remote dataset fingerprint must be a SHA-256 digest");
-    }
-  }
-  const defined = {
-    ...dataset,
-    id,
-    name,
-    version,
-    rows,
-    [EVAL_DATASET]: true
-  };
-  Object.defineProperty(defined, EVAL_DATASET, { enumerable: false });
-  return defined;
-}
-function isEvalDataset(value) {
-  return typeof value === "object" && value !== null && EVAL_DATASET in value && value[EVAL_DATASET] === true;
-}
 var BooleanThresholdSchema = z.object({ equals: z.boolean() }).strict();
 var FiniteThresholdSchema = z.number().finite();
-var EvalOutputSchema = z.enum(["boolean", "score", "number"]);
 var NumericThresholdSchema = z.union([
   z.object({ gte: FiniteThresholdSchema, lte: FiniteThresholdSchema.optional() }).strict(),
   z.object({ lte: FiniteThresholdSchema }).strict()
@@ -79,194 +9,32 @@ var NumericThresholdSchema = z.union([
   (threshold) => !("gte" in threshold) || threshold.lte === void 0 || threshold.gte <= threshold.lte,
   { message: "Lower threshold cannot exceed upper threshold" }
 );
-function defineEvaluatorProgram(program) {
-  var _a, _b, _c, _d;
-  const slug = program.slug.trim();
-  const name = program.name.trim();
-  const source = program.source.trim();
-  const intent = program.intent.trim();
-  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug) || slug.length > 64) {
-    throw new Error(
-      "Raindrop evaluator program slugs must use lowercase letters, numbers, and single hyphens"
-    );
-  }
-  if (!name || name.length > 120) {
-    throw new Error("Raindrop evaluator program names must be between 1 and 120 characters");
-  }
-  if (!source) throw new Error(`Raindrop evaluator program ${slug} source cannot be empty`);
-  if (!intent || intent.length > 1e4) {
-    throw new Error(
-      `Raindrop evaluator program ${slug} intent must be between 1 and 10000 characters`
-    );
-  }
-  if (program.description != null && program.description.trim().length > 2e3) {
-    throw new Error(`Raindrop evaluator program ${slug} description exceeds 2000 characters`);
-  }
-  const rules = (_b = (_a = program.rules) == null ? void 0 : _a.map((rule) => rule.trim())) != null ? _b : [];
-  if (rules.length > 12 || rules.some((rule) => !rule || rule.length > 2e3)) {
-    throw new Error(`Raindrop evaluator program ${slug} rules are invalid`);
-  }
-  if (program.expected && (!z.string().uuid().safeParse(program.expected.evalId).success || !Number.isInteger(program.expected.programVersion) || program.expected.programVersion <= 0)) {
-    throw new Error(`Raindrop evaluator program ${slug} expected identity is invalid`);
-  }
-  return {
-    ...program,
-    kind: "program",
-    scope: z.literal("batch").parse((_c = program.scope) != null ? _c : "batch"),
-    slug,
-    name,
-    source,
-    intent,
-    description: ((_d = program.description) == null ? void 0 : _d.trim()) || null,
-    rules
-  };
-}
-function defineLocalEvaluator(evaluator) {
-  var _a;
-  return { ...evaluator, scope: z.literal("row").parse((_a = evaluator.scope) != null ? _a : "row") };
-}
-var EVAL_DEFINITION = /* @__PURE__ */ Symbol.for("raindrop.evalDefinition");
-function defineEvalSuite(definition) {
-  const name = definition.name.trim();
-  if (name.length === 0) throw new Error("Raindrop eval suite names cannot be empty");
-  const dataset = typeof definition.dataset === "string" ? definition.dataset.trim() : defineDataset(definition.dataset);
-  if (typeof dataset === "string" && dataset.length === 0) {
-    throw new Error("Raindrop eval suite datasets cannot be empty");
-  }
-  if (definition.datasetVersionId !== void 0) {
-    z.string().uuid().parse(definition.datasetVersionId);
-    if (typeof dataset !== "string")
-      throw new Error("A dataset version pin requires a published dataset slug");
-  }
-  if (definition.agent !== void 0 && definition.run !== void 0) {
-    throw new Error("An eval suite accepts either agent or run, not both");
-  }
-  if (definition.agent === void 0 && typeof definition.run !== "function") {
-    throw new Error("Raindrop eval suite run must be a function");
-  }
-  if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
-    throw new Error("Raindrop eval suites require at least one evaluator");
-  }
-  validateOptionalNumber("concurrency", definition.concurrency, true, false);
-  validateOptionalNumber("traceWaitMs", definition.traceWaitMs, false, true);
-  validateOptionalNumber("evalWaitMs", definition.evalWaitMs, false, true);
-  validateOptionalNumber("evalPollIntervalMs", definition.evalPollIntervalMs, false, false);
-  const evaluators = definition.evaluators.map((evaluator) => validateEvaluator(evaluator));
-  const evaluatorNames = evaluators.map((entry) => evaluatorName(entry));
-  if (new Set(evaluatorNames).size !== evaluatorNames.length) {
-    throw new Error("Raindrop eval suite evaluators must be unique");
-  }
-  const branded = {
-    ...definition,
-    [EVAL_DEFINITION]: true,
-    name,
-    dataset,
-    evaluators
-  };
-  Object.defineProperty(branded, EVAL_DEFINITION, { enumerable: false });
-  return branded;
-}
-function isEvalSuiteDefinition(value) {
-  return typeof value === "object" && value !== null && EVAL_DEFINITION in value && value[EVAL_DEFINITION] === true;
-}
-function isPublishedEvalSuite(definition) {
-  return typeof definition.dataset === "string" && definition.datasetVersionId !== void 0 && definition.evaluators.every(
-    ({ evaluator }) => typeof evaluator !== "string" && (!("kind" in evaluator) || evaluator.kind !== "program" || evaluator.expected !== void 0)
-  );
-}
-function evaluatorName(evaluator) {
-  return typeof evaluator.evaluator === "string" ? evaluator.evaluator.trim() : evaluator.evaluator.slug.trim();
-}
-function evaluatorOutput(evaluator) {
-  return typeof evaluator.evaluator === "string" ? void 0 : evaluator.evaluator.output;
-}
-function validateEvaluatorOutput(evaluator, rawOutput) {
-  const name = evaluatorName(evaluator);
-  const output = EvalOutputSchema.parse(rawOutput);
-  const localOutput = evaluatorOutput(evaluator);
-  if (localOutput !== void 0 && localOutput !== output) {
-    throw new Error(`Raindrop local evaluator ${name} expected ${localOutput}, received ${output}`);
-  }
-  switch (output) {
-    case "boolean":
-      parseBooleanThreshold(evaluator.threshold);
-      break;
-    case "score":
-      parseNumericThreshold(name, "score", evaluator.threshold);
-      break;
-    case "number":
-      parseNumericThreshold(name, "number", evaluator.threshold);
-      break;
-  }
-}
-function validateEvaluator(evaluator) {
-  if (typeof evaluator !== "object" || evaluator === null || !("evaluator" in evaluator)) {
-    throw new Error("Raindrop eval evaluator entries must name an evaluator");
-  }
-  if ("output" in evaluator) {
-    throw new Error(
-      "Raindrop eval evaluator entries do not accept output; the evaluator definition owns it"
-    );
-  }
-  if (typeof evaluator.evaluator !== "string" && (typeof evaluator.evaluator !== "object" || evaluator.evaluator === null || typeof evaluator.evaluator.slug !== "string" || typeof evaluator.evaluator.name !== "string")) {
-    throw new Error("Raindrop local eval evaluators require a slug, name, output, and judge");
-  }
-  const name = evaluatorName(evaluator);
-  if (name.length === 0) throw new Error("Raindrop eval evaluator names cannot be empty");
-  if (typeof evaluator.evaluator === "string") {
-    const threshold = evaluator.threshold;
-    if (threshold !== void 0) {
-      const boolean = BooleanThresholdSchema.safeParse(threshold);
-      const numeric = NumericThresholdSchema.safeParse(threshold);
-      if (!boolean.success && !numeric.success) {
-        throw new Error(`Raindrop hosted evaluator ${name} has an invalid threshold`);
-      }
-    }
-    return { ...evaluator, evaluator: name };
-  }
-  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "portable") {
-    validateEvaluatorOutput(evaluator, evaluator.evaluator.output);
-    return evaluator;
-  }
-  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "program") {
-    const program = defineEvaluatorProgram(evaluator.evaluator);
-    validateEvaluatorOutput({ ...evaluator, evaluator: program }, program.output);
-    return { ...evaluator, evaluator: program };
-  }
-  if (typeof evaluator.evaluator.judge !== "function") {
-    throw new Error(`Raindrop local evaluator ${name} judge must be a function`);
-  }
-  if (evaluator.evaluator.scope !== "row") {
-    throw new Error(`Raindrop local evaluator ${name} scope must be row`);
-  }
-  const output = EvalOutputSchema.parse(evaluator.evaluator.output);
-  validateEvaluatorOutput(evaluator, output);
-  return evaluator;
-}
-function validateOptionalNumber(name, value, integer, allowZero) {
-  if (value === void 0) return;
-  const valid = Number.isFinite(value) && (!integer || Number.isInteger(value)) && (allowZero ? value >= 0 : value > 0);
-  if (!valid) {
-    const kind = integer ? "a positive integer" : allowZero ? "non-negative" : "positive";
-    throw new Error(`Raindrop eval ${name} must be ${kind}`);
-  }
-}
-function parseBooleanThreshold(threshold) {
-  return threshold === void 0 ? void 0 : BooleanThresholdSchema.parse(threshold);
-}
-function parseNumericThreshold(name, output, threshold) {
-  const parsed = threshold === void 0 ? void 0 : NumericThresholdSchema.parse(threshold);
-  if (output === "score" && parsed) {
-    const lower = "gte" in parsed ? parsed.gte : void 0;
-    const inRange = (lower === void 0 || lower >= 1 && lower <= 5) && (parsed.lte === void 0 || parsed.lte >= 1 && parsed.lte <= 5);
-    if (!inRange)
-      throw new Error(`Raindrop score evaluator ${name} thresholds must be from 1 to 5`);
-  }
-  return parsed;
-}
+var EvalSuiteManifestSchema = z.object({
+  name: z.string().trim().min(1),
+  dataset: z.string().trim().min(1),
+  datasetVersionId: z.string().uuid().optional(),
+  evaluators: z.array(
+    z.object({
+      evaluator: z.string().trim().min(1),
+      threshold: z.union([BooleanThresholdSchema, NumericThresholdSchema]).optional()
+    }).strict()
+  ).min(1),
+  concurrency: z.number().int().positive().optional(),
+  traceWaitMs: z.number().nonnegative().optional(),
+  evalWaitMs: z.number().nonnegative().optional(),
+  evalPollIntervalMs: z.number().positive().optional()
+}).strict();
 
 // src/evals/generated/trace.ts
 import z2 from "zod";
+var JsonValueSchema = z2.lazy(() => z2.union([
+  z2.string(),
+  z2.number(),
+  z2.boolean(),
+  z2.null(),
+  z2.array(JsonValueSchema),
+  z2.record(JsonValueSchema)
+]));
 var stringToJSONSchema = z2.string().transform((str, ctx) => {
   try {
     return JSON.parse(str);
@@ -635,14 +403,20 @@ var ReplayTraceSnapshotSpanSchema = TracesListPipe.output.extend({
   duration_ns: NanosecondsStringSchema,
   _inserted_at: z2.string().min(1)
 }).strict();
+var EvalDatasetCaseIdSchema = z2.string().min(1).max(512).refine((value) => value === value.trim(), "Case id cannot have surrounding whitespace");
+var EvalCaseExpectationSchema = z2.object({
+  description: z2.string().trim().min(1).max(1e4),
+  basis: z2.enum(["human", "policy", "code", "inferred"]).optional(),
+  reference: z2.string().trim().min(1).max(2e3).optional()
+}).strict();
 var TraceSchema = z2.object({
   origin: z2.enum(["production", "dataset", "simulation", "replay"]),
   event: AIAnalyticsEventSchema,
-  entries: z2.array(RichInteractionEntrySchema)
+  entries: z2.array(RichInteractionEntrySchema),
+  caseContext: z2.object({ caseId: EvalDatasetCaseIdSchema, expectation: EvalCaseExpectationSchema.optional() }).optional()
 });
 var CapturedTraceSpanSchema = ReplayTraceSnapshotSpanSchema;
 var EVAL_DATASET_MAX_SPANS_PER_TRACE = 500;
-var EvalDatasetCaseIdSchema = z2.string().min(1).max(512).refine((value) => value === value.trim(), "Case id cannot have surrounding whitespace");
 var CapturedTraceSnapshotSchema = z2.object({
   event: AIAnalyticsEventSchema,
   spans: z2.array(CapturedTraceSpanSchema).min(1).max(EVAL_DATASET_MAX_SPANS_PER_TRACE)
@@ -712,15 +486,6 @@ var CapturedTraceSnapshotSchema = z2.object({
       path: ["spans"]
     });
   }
-  for (const [index, span] of spans.entries()) {
-    if (span.parent_span_id && !spanIds.has(span.parent_span_id)) {
-      context.addIssue({
-        code: z2.ZodIssueCode.custom,
-        message: `Missing parent span ${span.parent_span_id}`,
-        path: ["spans", index, "parent_span_id"]
-      });
-    }
-  }
   const parentBySpanId = new Map(spans.map((span) => [span.span_id, span.parent_span_id]));
   for (const [index, span] of spans.entries()) {
     const path = /* @__PURE__ */ new Set();
@@ -745,8 +510,9 @@ var EvalCaseTraceSchema = z2.object({
 });
 var EvalCaseTracePairSchema = z2.object({
   caseId: EvalDatasetCaseIdSchema,
+  output: JsonValueSchema.optional(),
   candidate: EvalCaseTraceSchema.extend({
-    trace: TraceSchema.extend({ origin: z2.literal("replay") })
+    trace: TraceSchema.extend({ origin: z2.enum(["replay", "dataset"]) })
   }),
   reference: EvalCaseTraceSchema.extend({
     trace: TraceSchema.extend({ origin: z2.literal("dataset") })
@@ -1012,37 +778,6 @@ function traceStepsToEntries(steps) {
 function projectWorkshopTrace(input) {
   return createTrace(input.origin, input.event, traceStepsToEntries(buildTraceSteps([...input.spans], { includePayloads: true })).map(boundEvalEntry));
 }
-function projectCapturedEvent(input) {
-  var _a, _b, _c, _d, _e, _f;
-  const ordered = [...input.spans].sort((left, right) => compareNanoseconds(left.start_unix_ns, right.start_unix_ns));
-  const eventSpan = ordered.find((span) => span.span_id === `evt_${input.eventId}`);
-  const llmSpans = ordered.filter((span) => span.span_type === "LLM_GENERATION" || span.span_type === "LLM_GENERATION_STREAM");
-  const inputSpan = ((_a = eventSpan == null ? void 0 : eventSpan.input_payload) == null ? void 0 : _a.trim()) ? eventSpan : llmSpans.find((span) => {
-    var _a2;
-    return Boolean((_a2 = span.input_payload) == null ? void 0 : _a2.trim());
-  });
-  const reverseLlmSpans = [...llmSpans].reverse();
-  const outputSpan = (_c = ((_b = eventSpan == null ? void 0 : eventSpan.output_payload) == null ? void 0 : _b.trim()) ? eventSpan : void 0) != null ? _c : reverseLlmSpans.find((span) => {
-    var _a2;
-    return Boolean((_a2 = span.output_payload) == null ? void 0 : _a2.trim()) && extractFinishReason(span) !== "tool-calls";
-  });
-  const modelSpan = (eventSpan == null ? void 0 : eventSpan.model) ? eventSpan : reverseLlmSpans.find((span) => Boolean(span.model));
-  return AIAnalyticsEventSchema.parse({
-    id: input.eventId,
-    _raindropDatabaseId: input.internalEventId,
-    name: input.name,
-    timestamp: input.timestamp,
-    receivedAt: input.receivedAt,
-    userId: input.userId,
-    aiData: {
-      input: (_d = inputSpan == null ? void 0 : inputSpan.input_payload) != null ? _d : null,
-      output: (_e = outputSpan == null ? void 0 : outputSpan.output_payload) != null ? _e : null,
-      model: (_f = modelSpan == null ? void 0 : modelSpan.model) != null ? _f : null,
-      convoId: input.convoId
-    },
-    properties: input.properties
-  });
-}
 function traceOutput(trace) {
   var _a;
   return (_a = trace.event.aiData.output) != null ? _a : null;
@@ -1050,64 +785,329 @@ function traceOutput(trace) {
 function traceToolCalls(trace, name) {
   return trace.entries.filter((entry) => entry.type === "tool_call" && (name === void 0 || entry.name === name));
 }
-var TRACE_PROJECTION_SHA256 = "7d12eb03150b8f203e4404622f047676f50b3aa1e5442f0ed42d77d6002a33a6";
+var TRACE_PROJECTION_SHA256 = "22c666c6ace47d74d3f7dc477e68b27547257b319ba4014a4a32fbf977baac72";
 
-// src/evals/portable.ts
+// src/evals/definition.ts
+import { createHash } from "crypto";
 import { z as z3 } from "zod";
-var PortableEvalArtifactSchema = z3.object({
-  format: z3.literal("raindrop.eval-program"),
-  formatVersion: z3.literal(1),
-  runtimeAbi: z3.literal("raindrop-eval-v2"),
-  evaluator: z3.object({
-    id: z3.string().uuid(),
-    slug: z3.string().min(1),
-    programVersion: z3.number().int().positive(),
-    executionMode: z3.enum(["deterministic", "judge"]),
-    outputType: z3.enum(["boolean", "score", "number"]),
-    scope: z3.literal("batch")
+var EvalScoreSchema = z3.union([
+  z3.literal(1),
+  z3.literal(2),
+  z3.literal(3),
+  z3.literal(4),
+  z3.literal(5)
+]);
+var EVAL_DATASET = /* @__PURE__ */ Symbol.for("raindrop.evalDataset");
+var DatasetRowInputSchema = z3.object({
+  id: z3.string().trim().min(1),
+  name: z3.string().optional(),
+  input: z3.string().nullable(),
+  output: JsonValueSchema.default(null),
+  properties: z3.record(z3.string()).default({})
+});
+function defineDataset(dataset) {
+  var _a, _b;
+  const id = dataset.id.trim();
+  const name = dataset.name.trim();
+  const rows = dataset.rows.map((row) => {
+    var _a2;
+    const parsed = DatasetRowInputSchema.parse(row);
+    return {
+      id: parsed.id,
+      name: (_a2 = parsed.name) != null ? _a2 : parsed.id,
+      input: parsed.input,
+      output: parsed.output,
+      properties: parsed.properties
+    };
+  });
+  const version = (_b = (_a = dataset.version) == null ? void 0 : _a.trim()) != null ? _b : createHash("sha256").update(
+    JSON.stringify(
+      rows.map((row) => ({
+        ...row,
+        properties: Object.fromEntries(
+          Object.keys(row.properties).sort().map((key) => [key, row.properties[key]])
+        )
+      }))
+    )
+  ).digest("hex");
+  if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
+  if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
+  if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
+  if (dataset.rows.length === 0) throw new Error("Raindrop eval datasets require at least one row");
+  const ids = rows.map((entry) => entry.id);
+  if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
+  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
+  if (dataset.remote) {
+    if (!dataset.remote.reference.trim())
+      throw new Error("Raindrop remote dataset reference cannot be empty");
+    if (!/^[a-f0-9]{64}$/.test(dataset.remote.fingerprint)) {
+      throw new Error("Raindrop remote dataset fingerprint must be a SHA-256 digest");
+    }
+  }
+  const defined = {
+    ...dataset,
+    id,
+    name,
+    version,
+    rows,
+    [EVAL_DATASET]: true
+  };
+  Object.defineProperty(defined, EVAL_DATASET, { enumerable: false });
+  return defined;
+}
+function isEvalDataset(value) {
+  return typeof value === "object" && value !== null && EVAL_DATASET in value && value[EVAL_DATASET] === true;
+}
+var EvalOutputSchema = z3.enum(["boolean", "score", "number"]);
+function defineEvaluatorProgram(program) {
+  var _a, _b, _c, _d;
+  const slug = program.slug.trim();
+  const name = program.name.trim();
+  const source = program.source.trim();
+  const intent = program.intent.trim();
+  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug) || slug.length > 64) {
+    throw new Error(
+      "Raindrop evaluator program slugs must use lowercase letters, numbers, and single hyphens"
+    );
+  }
+  if (!name || name.length > 120) {
+    throw new Error("Raindrop evaluator program names must be between 1 and 120 characters");
+  }
+  if (!source) throw new Error(`Raindrop evaluator program ${slug} source cannot be empty`);
+  if (!intent || intent.length > 1e4) {
+    throw new Error(
+      `Raindrop evaluator program ${slug} intent must be between 1 and 10000 characters`
+    );
+  }
+  if (program.description != null && program.description.trim().length > 2e3) {
+    throw new Error(`Raindrop evaluator program ${slug} description exceeds 2000 characters`);
+  }
+  const rules = (_b = (_a = program.rules) == null ? void 0 : _a.map((rule) => rule.trim())) != null ? _b : [];
+  if (rules.length > 12 || rules.some((rule) => !rule || rule.length > 2e3)) {
+    throw new Error(`Raindrop evaluator program ${slug} rules are invalid`);
+  }
+  if (program.expected && (!z3.string().uuid().safeParse(program.expected.evalId).success || !Number.isInteger(program.expected.programVersion) || program.expected.programVersion <= 0)) {
+    throw new Error(`Raindrop evaluator program ${slug} expected identity is invalid`);
+  }
+  return {
+    ...program,
+    kind: "program",
+    scope: z3.literal("batch").parse((_c = program.scope) != null ? _c : "batch"),
+    slug,
+    name,
+    source,
+    intent,
+    description: ((_d = program.description) == null ? void 0 : _d.trim()) || null,
+    rules
+  };
+}
+function defineLocalEvaluator(evaluator) {
+  var _a;
+  return { ...evaluator, scope: z3.literal("row").parse((_a = evaluator.scope) != null ? _a : "row") };
+}
+var EVAL_DEFINITION = /* @__PURE__ */ Symbol.for("raindrop.evalDefinition");
+function defineEvalSuite(definition, binding) {
+  if (binding) return createEvalSuite({ ...EvalSuiteManifestSchema.parse(definition), ...binding });
+  if (!("run" in definition) && !("agent" in definition)) {
+    throw new Error("Bind the eval manifest to your local run function or agent");
+  }
+  return createEvalSuite(definition);
+}
+function createEvalSuite(definition) {
+  const name = definition.name.trim();
+  if (name.length === 0) throw new Error("Raindrop eval suite names cannot be empty");
+  const dataset = typeof definition.dataset === "string" ? definition.dataset.trim() : defineDataset(definition.dataset);
+  if (typeof dataset === "string" && dataset.length === 0) {
+    throw new Error("Raindrop eval suite datasets cannot be empty");
+  }
+  if (definition.datasetVersionId !== void 0) {
+    z3.string().uuid().parse(definition.datasetVersionId);
+    if (typeof dataset !== "string")
+      throw new Error("A dataset version pin requires a published dataset slug");
+  }
+  if (definition.agent !== void 0 && definition.run !== void 0) {
+    throw new Error("An eval suite accepts either agent or run, not both");
+  }
+  if (definition.agent === void 0 && typeof definition.run !== "function") {
+    throw new Error("Raindrop eval suite run must be a function");
+  }
+  if (!Array.isArray(definition.evaluators) || definition.evaluators.length === 0) {
+    throw new Error("Raindrop eval suites require at least one evaluator");
+  }
+  validateOptionalNumber("concurrency", definition.concurrency, true, false);
+  validateOptionalNumber("traceWaitMs", definition.traceWaitMs, false, true);
+  validateOptionalNumber("evalWaitMs", definition.evalWaitMs, false, true);
+  validateOptionalNumber("evalPollIntervalMs", definition.evalPollIntervalMs, false, false);
+  const evaluators = definition.evaluators.map((evaluator) => validateEvaluator(evaluator));
+  const evaluatorNames = evaluators.map((entry) => evaluatorName(entry));
+  if (new Set(evaluatorNames).size !== evaluatorNames.length) {
+    throw new Error("Raindrop eval suite evaluators must be unique");
+  }
+  const branded = {
+    ...definition,
+    [EVAL_DEFINITION]: true,
+    name,
+    dataset,
+    evaluators
+  };
+  Object.defineProperty(branded, EVAL_DEFINITION, { enumerable: false });
+  return branded;
+}
+function isEvalSuiteDefinition(value) {
+  return typeof value === "object" && value !== null && EVAL_DEFINITION in value && value[EVAL_DEFINITION] === true;
+}
+function isPublishedEvalSuite(definition) {
+  return typeof definition.dataset === "string" && definition.datasetVersionId !== void 0 && definition.evaluators.every(
+    ({ evaluator }) => typeof evaluator !== "string" && (!("kind" in evaluator) || evaluator.kind !== "program" || evaluator.expected !== void 0)
+  );
+}
+function evaluatorName(evaluator) {
+  return typeof evaluator.evaluator === "string" ? evaluator.evaluator.trim() : evaluator.evaluator.slug.trim();
+}
+function evaluatorOutput(evaluator) {
+  return typeof evaluator.evaluator === "string" ? void 0 : evaluator.evaluator.output;
+}
+function validateEvaluatorOutput(evaluator, rawOutput) {
+  const name = evaluatorName(evaluator);
+  const output = EvalOutputSchema.parse(rawOutput);
+  const localOutput = evaluatorOutput(evaluator);
+  if (localOutput !== void 0 && localOutput !== output) {
+    throw new Error(`Raindrop local evaluator ${name} expected ${localOutput}, received ${output}`);
+  }
+  switch (output) {
+    case "boolean":
+      parseBooleanThreshold(evaluator.threshold);
+      break;
+    case "score":
+      parseNumericThreshold(name, "score", evaluator.threshold);
+      break;
+    case "number":
+      parseNumericThreshold(name, "number", evaluator.threshold);
+      break;
+  }
+}
+function validateEvaluator(evaluator) {
+  if (typeof evaluator !== "object" || evaluator === null || !("evaluator" in evaluator)) {
+    throw new Error("Raindrop eval evaluator entries must name an evaluator");
+  }
+  if ("output" in evaluator) {
+    throw new Error(
+      "Raindrop eval evaluator entries do not accept output; the evaluator definition owns it"
+    );
+  }
+  if (typeof evaluator.evaluator !== "string" && (typeof evaluator.evaluator !== "object" || evaluator.evaluator === null || typeof evaluator.evaluator.slug !== "string" || typeof evaluator.evaluator.name !== "string")) {
+    throw new Error("Raindrop local eval evaluators require a slug, name, output, and judge");
+  }
+  const name = evaluatorName(evaluator);
+  if (name.length === 0) throw new Error("Raindrop eval evaluator names cannot be empty");
+  if (typeof evaluator.evaluator === "string") {
+    const threshold = evaluator.threshold;
+    if (threshold !== void 0) {
+      const boolean = BooleanThresholdSchema.safeParse(threshold);
+      const numeric = NumericThresholdSchema.safeParse(threshold);
+      if (!boolean.success && !numeric.success) {
+        throw new Error(`Raindrop hosted evaluator ${name} has an invalid threshold`);
+      }
+    }
+    return { ...evaluator, evaluator: name };
+  }
+  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "portable") {
+    validateEvaluatorOutput(evaluator, evaluator.evaluator.output);
+    return evaluator;
+  }
+  if ("kind" in evaluator.evaluator && evaluator.evaluator.kind === "program") {
+    const program = defineEvaluatorProgram(evaluator.evaluator);
+    validateEvaluatorOutput({ ...evaluator, evaluator: program }, program.output);
+    return { ...evaluator, evaluator: program };
+  }
+  if (typeof evaluator.evaluator.judge !== "function") {
+    throw new Error(`Raindrop local evaluator ${name} judge must be a function`);
+  }
+  if (evaluator.evaluator.scope !== "row") {
+    throw new Error(`Raindrop local evaluator ${name} scope must be row`);
+  }
+  const output = EvalOutputSchema.parse(evaluator.evaluator.output);
+  validateEvaluatorOutput(evaluator, output);
+  return evaluator;
+}
+function validateOptionalNumber(name, value, integer, allowZero) {
+  if (value === void 0) return;
+  const valid = Number.isFinite(value) && (!integer || Number.isInteger(value)) && (allowZero ? value >= 0 : value > 0);
+  if (!valid) {
+    const kind = integer ? "a positive integer" : allowZero ? "non-negative" : "positive";
+    throw new Error(`Raindrop eval ${name} must be ${kind}`);
+  }
+}
+function parseBooleanThreshold(threshold) {
+  return threshold === void 0 ? void 0 : BooleanThresholdSchema.parse(threshold);
+}
+function parseNumericThreshold(name, output, threshold) {
+  const parsed = threshold === void 0 ? void 0 : NumericThresholdSchema.parse(threshold);
+  if (output === "score" && parsed) {
+    const lower = "gte" in parsed ? parsed.gte : void 0;
+    const inRange = (lower === void 0 || lower >= 1 && lower <= 5) && (parsed.lte === void 0 || parsed.lte >= 1 && parsed.lte <= 5);
+    if (!inRange)
+      throw new Error(`Raindrop score evaluator ${name} thresholds must be from 1 to 5`);
+  }
+  return parsed;
+}
+
+// src/evals/portable.ts
+import { z as z4 } from "zod";
+var PortableEvalArtifactSchema = z4.object({
+  format: z4.literal("raindrop.eval-program"),
+  formatVersion: z4.literal(1),
+  runtimeAbi: z4.literal("raindrop-eval-v2"),
+  evaluator: z4.object({
+    id: z4.string().uuid(),
+    slug: z4.string().min(1),
+    programVersion: z4.number().int().positive(),
+    executionMode: z4.enum(["deterministic", "judge"]),
+    outputType: z4.enum(["boolean", "score", "number"]),
+    scope: z4.literal("batch")
   }).strict(),
-  traceProjectionVersion: z3.literal(1),
-  traceProjectionSha256: z3.string().regex(/^[a-f0-9]{64}$/),
-  source: z3.string().min(1),
-  sha256: z3.string().regex(/^[a-f0-9]{64}$/)
+  traceProjectionVersion: z4.literal(1),
+  traceProjectionSha256: z4.string().regex(/^[a-f0-9]{64}$/),
+  source: z4.string().min(1),
+  sha256: z4.string().regex(/^[a-f0-9]{64}$/)
 }).strict();
-var PortableEvalResultSchema = z3.object({
-  outcomes: z3.array(
-    z3.union([
-      z3.object({
-        state: z3.literal("graded"),
-        traceId: z3.string(),
-        pass: z3.boolean(),
-        note: z3.string().optional()
+var PortableEvalResultSchema = z4.object({
+  outcomes: z4.array(
+    z4.union([
+      z4.object({
+        state: z4.literal("graded"),
+        traceId: z4.string(),
+        pass: z4.boolean(),
+        note: z4.string().optional()
       }),
-      z3.object({
-        state: z3.literal("graded"),
-        traceId: z3.string(),
+      z4.object({
+        state: z4.literal("graded"),
+        traceId: z4.string(),
         score: EvalScoreSchema,
-        note: z3.string().optional()
+        note: z4.string().optional()
       }),
-      z3.object({
-        state: z3.literal("graded"),
-        traceId: z3.string(),
-        value: z3.number().finite(),
-        note: z3.string().optional()
+      z4.object({
+        state: z4.literal("graded"),
+        traceId: z4.string(),
+        value: z4.number().finite(),
+        note: z4.string().optional()
       }),
-      z3.object({ state: z3.literal("errored"), traceId: z3.string(), message: z3.string() }),
-      z3.object({
-        state: z3.literal("ungraded"),
-        traceId: z3.string(),
-        reason: z3.enum(["not sampled", "trace unavailable", "content unreadable"])
+      z4.object({ state: z4.literal("errored"), traceId: z4.string(), message: z4.string() }),
+      z4.object({
+        state: z4.literal("ungraded"),
+        traceId: z4.string(),
+        reason: z4.enum(["not sampled", "trace unavailable", "content unreadable"])
       })
     ])
   ),
-  failures: z3.array(z3.string()),
-  stats: z3.object({
-    traceCount: z3.number().int().nonnegative(),
-    hydratedCount: z3.number().int().nonnegative(),
-    judgeCalls: z3.number().int().nonnegative(),
-    durationMs: z3.number().nonnegative(),
-    skippedRows: z3.number().int().nonnegative(),
-    skippedReason: z3.literal("no trace").nullable()
+  failures: z4.array(z4.string()),
+  stats: z4.object({
+    traceCount: z4.number().int().nonnegative(),
+    hydratedCount: z4.number().int().nonnegative(),
+    judgeCalls: z4.number().int().nonnegative(),
+    durationMs: z4.number().nonnegative(),
+    skippedRows: z4.number().int().nonnegative(),
+    skippedReason: z4.literal("no trace").nullable()
   })
 });
 function parsePortableEvalResult(value) {
@@ -1138,7 +1138,7 @@ async function evaluatePortableEvaluator(evaluator, input) {
   const artifact = await verifyPortableEvalArtifact(evaluator.artifact, {
     expectedSha256: evaluator.expectedSha256
   });
-  const { runPortableEvaluator } = await import("./portable-runtime-6X2HHB4Y.mjs");
+  const { runPortableEvaluator } = await import("./portable-runtime-F7W4STGH.mjs");
   const cases = ((_a = input.rows) != null ? _a : []).map(({ rowId, ...pair }) => ({ ...pair, caseId: rowId }));
   if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
     (entry, index) => {
@@ -1218,11 +1218,21 @@ function canonicalJson(value) {
 }
 
 export {
+  BooleanThresholdSchema,
+  NumericThresholdSchema,
+  EvalSuiteManifestSchema,
+  JsonValueSchema,
+  EvalDatasetCaseIdSchema,
+  EvalCaseExpectationSchema,
+  TraceSchema,
+  CapturedTraceSnapshotSchema,
+  EvalCaseTracePairSchema,
+  projectWorkshopTrace,
+  traceOutput,
+  traceToolCalls,
   EvalScoreSchema,
   defineDataset,
   isEvalDataset,
-  BooleanThresholdSchema,
-  NumericThresholdSchema,
   defineEvaluatorProgram,
   defineLocalEvaluator,
   defineEvalSuite,
@@ -1231,14 +1241,6 @@ export {
   evaluatorName,
   evaluatorOutput,
   validateEvaluatorOutput,
-  TraceSchema,
-  EvalDatasetCaseIdSchema,
-  CapturedTraceSnapshotSchema,
-  EvalCaseTracePairSchema,
-  projectWorkshopTrace,
-  projectCapturedEvent,
-  traceOutput,
-  traceToolCalls,
   parsePortableEvalResult,
   importPortableEvaluator,
   evaluatePortableEvaluator,