raindrop-ai 0.6.0 → 0.7.0

package/dist/{chunk-ZVHQBBST.mjs → chunk-OX6X4ZWQ.mjs}

@@ -1238,10 +1238,13 @@ installTracingSuppressionHook();
 // package.json
 var package_default = {
   name: "raindrop-ai",
-  version: "0.6.0",
+  version: "0.7.0",
   main: "dist/index.js",
   module: "dist/index.mjs",
   types: "dist/index.d.ts",
+  bin: {
+    "raindrop-evals": "dist/evals/cli.js"
+  },
   license: "MIT",
   homepage: "https://www.raindrop.ai/docs/sdk/typescript/",
   bugs: {
@@ -1338,6 +1341,7 @@ var package_default = {
   tsup: {
     entry: [
       "src/index.ts",
+      "src/evals/cli.ts",
       "src/tracing/index.ts",
       "src/otel/index.ts"
     ],

package/dist/{chunk-CFGEE5ID.mjs → chunk-RMP6BZSY.mjs}

@@ -1,4 +1,5 @@
 // src/evals/definition.ts
+import { createHash } from "crypto";
 import { z } from "zod";
 var EvalScoreSchema = z.union([
   z.literal(1),
@@ -8,18 +9,46 @@ var EvalScoreSchema = z.union([
   z.literal(5)
 ]);
 var EVAL_DATASET = /* @__PURE__ */ Symbol.for("raindrop.evalDataset");
+var DatasetRowInputSchema = z.object({
+  id: z.string().trim().min(1),
+  name: z.string().optional(),
+  input: z.string().nullable(),
+  output: z.string().nullable().default(null),
+  properties: z.record(z.string()).default({})
+});
 function defineDataset(dataset) {
+  var _a, _b;
   const id = dataset.id.trim();
   const name = dataset.name.trim();
-  const version = dataset.version.trim();
+  const rows = dataset.rows.map((row) => {
+    var _a2;
+    const parsed = DatasetRowInputSchema.parse(row);
+    return {
+      id: parsed.id,
+      name: (_a2 = parsed.name) != null ? _a2 : parsed.id,
+      input: parsed.input,
+      output: parsed.output,
+      properties: parsed.properties
+    };
+  });
+  const version = (_b = (_a = dataset.version) == null ? void 0 : _a.trim()) != null ? _b : createHash("sha256").update(
+    JSON.stringify(
+      rows.map((row) => ({
+        ...row,
+        properties: Object.fromEntries(
+          Object.keys(row.properties).sort().map((key) => [key, row.properties[key]])
+        )
+      }))
+    )
+  ).digest("hex");
   if (!id) throw new Error("Raindrop eval dataset ids cannot be empty");
   if (!name) throw new Error("Raindrop eval dataset names cannot be empty");
   if (!version) throw new Error("Raindrop eval dataset versions cannot be empty");
-  if (dataset.cases.length === 0)
-    throw new Error("Raindrop eval datasets require at least one case");
-  const ids = dataset.cases.map((entry) => entry.id.trim());
-  if (ids.some((caseId) => !caseId)) throw new Error("Raindrop eval case ids cannot be empty");
-  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval case ids must be unique");
+  if (dataset.rows.length === 0)
+    throw new Error("Raindrop eval datasets require at least one row");
+  const ids = rows.map((entry) => entry.id);
+  if (ids.some((rowId) => !rowId)) throw new Error("Raindrop eval row ids cannot be empty");
+  if (new Set(ids).size !== ids.length) throw new Error("Raindrop eval row ids must be unique");
   if (dataset.remote) {
     if (!dataset.remote.reference.trim())
       throw new Error("Raindrop remote dataset reference cannot be empty");
@@ -32,7 +61,7 @@ function defineDataset(dataset) {
     id,
     name,
     version,
-    cases: dataset.cases.map((entry) => ({ ...entry, id: entry.id.trim() })),
+    rows,
     [EVAL_DATASET]: true
   };
   Object.defineProperty(defined, EVAL_DATASET, { enumerable: false });
@@ -52,7 +81,7 @@ var NumericThresholdSchema = z.union([
   { message: "Lower threshold cannot exceed upper threshold" }
 );
 function defineEvaluatorProgram(program) {
-  var _a, _b, _c;
+  var _a, _b, _c, _d;
   const slug = program.slug.trim();
   const name = program.name.trim();
   const source = program.source.trim();
@@ -84,16 +113,18 @@ function defineEvaluatorProgram(program) {
   return {
     ...program,
     kind: "program",
+    scope: z.literal("batch").parse((_c = program.scope) != null ? _c : "batch"),
     slug,
     name,
     source,
     intent,
-    description: ((_c = program.description) == null ? void 0 : _c.trim()) || null,
+    description: ((_d = program.description) == null ? void 0 : _d.trim()) || null,
     rules
   };
 }
 function defineLocalEvaluator(evaluator) {
-  return evaluator;
+  var _a;
+  return { ...evaluator, scope: z.literal("row").parse((_a = evaluator.scope) != null ? _a : "row") };
 }
 var EVAL_DEFINITION = /* @__PURE__ */ Symbol.for("raindrop.evalDefinition");
 function defineEvalSuite(definition) {
@@ -103,6 +134,11 @@ function defineEvalSuite(definition) {
   if (typeof dataset === "string" && dataset.length === 0) {
     throw new Error("Raindrop eval suite datasets cannot be empty");
   }
+  if (definition.datasetVersionId !== void 0) {
+    z.string().uuid().parse(definition.datasetVersionId);
+    if (typeof dataset !== "string")
+      throw new Error("A dataset version pin requires a published dataset slug");
+  }
   if (typeof definition.run !== "function") {
     throw new Error("Raindrop eval suite run must be a function");
   }
@@ -131,6 +167,11 @@ function defineEvalSuite(definition) {
 function isEvalSuiteDefinition(value) {
   return typeof value === "object" && value !== null && EVAL_DEFINITION in value && value[EVAL_DEFINITION] === true;
 }
+function isPublishedEvalSuite(definition) {
+  return typeof definition.dataset === "string" && definition.datasetVersionId !== void 0 && definition.evaluators.every(
+    ({ evaluator }) => typeof evaluator !== "string" && (!("kind" in evaluator) || evaluator.kind !== "program" || evaluator.expected !== void 0)
+  );
+}
 function evaluatorName(evaluator) {
   return typeof evaluator.evaluator === "string" ? evaluator.evaluator.trim() : evaluator.evaluator.slug.trim();
 }
@@ -193,8 +234,8 @@ function validateEvaluator(evaluator) {
   if (typeof evaluator.evaluator.judge !== "function") {
     throw new Error(`Raindrop local evaluator ${name} judge must be a function`);
   }
-  if (evaluator.evaluator.scope !== "case") {
-    throw new Error(`Raindrop local evaluator ${name} scope must be case`);
+  if (evaluator.evaluator.scope !== "row") {
+    throw new Error(`Raindrop local evaluator ${name} scope must be row`);
   }
   const output = EvalOutputSchema.parse(evaluator.evaluator.output);
   validateEvaluatorOutput(evaluator, output);
@@ -1095,17 +1136,18 @@ async function evaluatePortableEvaluator(evaluator, input) {
   const artifact = await verifyPortableEvalArtifact(evaluator.artifact, {
     expectedSha256: evaluator.expectedSha256
   });
-  const { runPortableEvaluator } = await import("./portable-runtime-QZKU2VXS.mjs");
-  const cases = (_a = input.cases) != null ? _a : [];
+  const { runPortableEvaluator } = await import("./portable-runtime-MFVOVFPB.mjs");
+  const cases = ((_a = input.rows) != null ? _a : []).map(({ rowId, ...pair }) => ({ ...pair, caseId: rowId }));
   if (cases.length > 0 && (cases.length !== input.traces.length || cases.some(
     (entry, index) => {
       var _a2;
       return entry.candidate.trace.event.id !== ((_a2 = input.traces[index]) == null ? void 0 : _a2.event.id);
     }
   ))) {
-    throw new Error("Raindrop portable evaluator cases must match traces exactly, in slice order");
+    throw new Error("Raindrop portable evaluator rows must match traces exactly, in slice order");
   }
-  const result = await runPortableEvaluator(evaluator, { ...input, cases });
+  const { rows: _rows, ...runtimeInput } = input;
+  const result = await runPortableEvaluator(evaluator, { ...runtimeInput, cases });
   validatePortableResult(artifact, input.traces, result);
   return result;
 }
@@ -1183,6 +1225,7 @@ export {
   defineLocalEvaluator,
   defineEvalSuite,
   isEvalSuiteDefinition,
+  isPublishedEvalSuite,
   evaluatorName,
   evaluatorOutput,
   validateEvaluatorOutput,

package/dist/evals/cli.d.mts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/evals/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node