raindrop-ai 0.6.0 → 0.7.0

package/dist/evals/cli.mjs

@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+import {
+  compareEvalRuns,
+  index_default,
+  publishEvalSuite,
+  readEvaluator,
+  runEvalSuite
+} from "../chunk-4Z7UHC7X.mjs";
+import "../chunk-OX6X4ZWQ.mjs";
+import {
+  isEvalSuiteDefinition
+} from "../chunk-RMP6BZSY.mjs";
+import "../chunk-UJCSKKID.mjs";
+
+// src/evals/cli.ts
+import { resolve } from "path";
+import { pathToFileURL } from "url";
+import { parseArgs } from "util";
+import { z } from "zod";
+var SuiteModuleSchema = z.object({
+  default: z.custom(isEvalSuiteDefinition)
+});
+async function main() {
+  var _a, _b;
+  const { positionals, values } = parseArgs({
+    allowPositionals: true,
+    options: {
+      project: { type: "string" },
+      "query-url": { type: "string" },
+      "replay-ingest-url": { type: "string" },
+      "expected-dataset-version": { type: "string" },
+      "run-id": { type: "string" },
+      evaluator: { type: "string" },
+      json: { type: "boolean" },
+      help: { type: "boolean" }
+    }
+  });
+  const [command, file, afterRunId] = positionals;
+  if (values.help || !command) {
+    process.stdout.write(
+      [
+        "raindrop-evals publish <suite.mjs> [--project slug] [--json]",
+        "raindrop-evals run <suite.mjs> [--run-id queued-run-id] [--project slug] [--json]",
+        "raindrop-evals read <evaluator-slug> [--json]",
+        "raindrop-evals compare <before-run-id> <after-run-id> [--evaluator slug] [--json]",
+        "",
+        "Publish uploads dataset rows and pins existing evaluators without running them.",
+        "The module must default-export defineEvalSuite(...). Run uploads dataset rows first.",
+        "Create hosted evaluators in Raindrop and reference their slugs. Evaluator source is not uploaded.",
+        "Hosted judges run on Raindrop; agent callbacks and local evaluators run locally.",
+        "Requires RAINDROP_QUERY_API_KEY. Project defaults to RAINDROP_PROJECT_ID or default.",
+        "Use --query-url and --replay-ingest-url for a full-local Raindrop stack.",
+        "TypeScript modules need your runtime's TypeScript loader, such as node --import tsx.",
+        "Output is JSON. Run exits 1 for failed/incomplete evals; setup errors exit 2."
+      ].join("\n") + "\n"
+    );
+    return;
+  }
+  if (!file || !["publish", "run", "read", "compare"].includes(command))
+    throw new Error("Invalid command. Use --help.");
+  const apiKey = process.env.RAINDROP_QUERY_API_KEY;
+  if (!apiKey) throw new Error("Set RAINDROP_QUERY_API_KEY to a Query SDK organization key");
+  const client = new index_default({
+    apiKey,
+    projectId: (_b = (_a = values.project) != null ? _a : process.env.RAINDROP_PROJECT_ID) != null ? _b : "default",
+    localWorkshopUrl: false
+  });
+  const options = { queryUrl: values["query-url"] };
+  try {
+    if (command === "compare") {
+      if (!afterRunId) throw new Error("Compare requires before and after run IDs");
+      print(await compareEvalRuns(client, { ...options, beforeRunId: file, afterRunId, evaluator: values.evaluator }));
+    } else if (command === "read") {
+      print(await readEvaluator(client, file, options));
+    } else {
+      const module = SuiteModuleSchema.parse(await import(pathToFileURL(resolve(file)).href));
+      if (command === "publish") {
+        const suite = await publishEvalSuite(client, module.default, {
+          ...options,
+          expectedCurrentDatasetVersionId: values["expected-dataset-version"]
+        });
+        print({
+          name: suite.name,
+          dataset: suite.dataset,
+          datasetVersionId: suite.datasetVersionId,
+          evaluators: suite.evaluators.map(
+            ({ evaluator }) => typeof evaluator === "string" ? { slug: evaluator } : {
+              slug: evaluator.slug,
+              output: evaluator.output,
+              expected: "kind" in evaluator && evaluator.kind === "program" ? evaluator.expected : void 0
+            }
+          )
+        });
+      } else {
+        const result = await runEvalSuite(client, module.default, {
+          runId: values["run-id"],
+          expectedCurrentDatasetVersionId: values["expected-dataset-version"],
+          destination: {
+            kind: "raindrop",
+            ...options,
+            replayIngestUrl: values["replay-ingest-url"]
+          }
+        });
+        print(result);
+        if (result.status !== "complete" || result.evaluators.some((evaluator) => evaluator.status !== "completed") || result.rows.some(
+          (entry) => entry.status !== "done" || entry.verdicts.some(
+            (verdict) => verdict.state !== "passed" && verdict.state !== "measurement"
+          )
+        ))
+          process.exitCode = 1;
+      }
+    }
+  } finally {
+    await client.close();
+  }
+}
+function print(value) {
+  process.stdout.write(JSON.stringify(value, null, 2) + "\n");
+}
+void main().catch((error) => {
+  process.stderr.write(
+    JSON.stringify({ error: error instanceof Error ? error.message : String(error) }) + "\n"
+  );
+  process.exitCode = 2;
+});