raindrop-ai 0.8.0 → 0.9.0

package/dist/{chunk-22CNYHSA.mjs → chunk-VBZXQNJY.mjs}

@@ -8,7 +8,6 @@ import {
   POST_SHUTDOWN_TIMEOUT_MS,
   SHUTDOWN_DEADLINE_MS,
   authHintForKey,
-  boundedClone,
   capText,
   createAppGitContext,
   currentEvalScope,
@@ -37,29 +36,28 @@ import {
   withReplayTraceDestination,
   withRoutingContext,
   withTask
-} from "./chunk-EWHJ2YBW.mjs";
+} from "./chunk-RIL2CV7Q.mjs";
 import {
   CapturedTraceSnapshotSchema,
+  EvalCaseExpectationSchema,
   EvalCaseTracePairSchema,
   EvalDatasetCaseIdSchema,
   EvalScoreSchema,
+  JsonValueSchema,
   defineDataset,
   defineEvalSuite,
   defineEvaluatorProgram,
-  evaluatePortableEvaluator,
   evaluatorName,
   evaluatorOutput,
   importPortableEvaluator,
   isEvalDataset,
   isEvalSuiteDefinition,
   isPublishedEvalSuite,
-  projectCapturedEvent,
-  projectWorkshopTrace,
   traceOutput,
   traceToolCalls,
   validateEvaluatorOutput,
   verifyPortableEvalArtifact
-} from "./chunk-EFATLUNL.mjs";
+} from "./chunk-UY6537WW.mjs";
 import {
   __commonJS,
   __toESM
@@ -755,11 +753,11 @@ var require_lodash = __commonJS({
       function unicodeWords(string) {
         return string.match(reUnicodeWord) || [];
       }
-      var runInContext = (function runInContext2(context3) {
-        context3 = context3 == null ? root : _.defaults(root.Object(), context3, _.pick(root, contextProps));
-        var Array2 = context3.Array, Date2 = context3.Date, Error2 = context3.Error, Function2 = context3.Function, Math2 = context3.Math, Object2 = context3.Object, RegExp2 = context3.RegExp, String2 = context3.String, TypeError = context3.TypeError;
+      var runInContext = (function runInContext2(context2) {
+        context2 = context2 == null ? root : _.defaults(root.Object(), context2, _.pick(root, contextProps));
+        var Array2 = context2.Array, Date2 = context2.Date, Error2 = context2.Error, Function2 = context2.Function, Math2 = context2.Math, Object2 = context2.Object, RegExp2 = context2.RegExp, String2 = context2.String, TypeError = context2.TypeError;
         var arrayProto = Array2.prototype, funcProto = Function2.prototype, objectProto = Object2.prototype;
-        var coreJsData = context3["__core-js_shared__"];
+        var coreJsData = context2["__core-js_shared__"];
         var funcToString = funcProto.toString;
         var hasOwnProperty = objectProto.hasOwnProperty;
         var idCounter = 0;
@@ -773,7 +771,7 @@ var require_lodash = __commonJS({
         var reIsNative = RegExp2(
           "^" + funcToString.call(hasOwnProperty).replace(reRegExpChar, "\\$&").replace(/hasOwnProperty|(function).*?(?=\\\()| for .+?(?=\\\])/g, "$1.*?") + "$"
         );
-        var Buffer2 = moduleExports ? context3.Buffer : undefined2, Symbol2 = context3.Symbol, Uint8Array2 = context3.Uint8Array, allocUnsafe = Buffer2 ? Buffer2.allocUnsafe : undefined2, getPrototype = overArg(Object2.getPrototypeOf, Object2), objectCreate = Object2.create, propertyIsEnumerable = objectProto.propertyIsEnumerable, splice = arrayProto.splice, spreadableSymbol = Symbol2 ? Symbol2.isConcatSpreadable : undefined2, symIterator = Symbol2 ? Symbol2.iterator : undefined2, symToStringTag = Symbol2 ? Symbol2.toStringTag : undefined2;
+        var Buffer2 = moduleExports ? context2.Buffer : undefined2, Symbol2 = context2.Symbol, Uint8Array2 = context2.Uint8Array, allocUnsafe = Buffer2 ? Buffer2.allocUnsafe : undefined2, getPrototype = overArg(Object2.getPrototypeOf, Object2), objectCreate = Object2.create, propertyIsEnumerable = objectProto.propertyIsEnumerable, splice = arrayProto.splice, spreadableSymbol = Symbol2 ? Symbol2.isConcatSpreadable : undefined2, symIterator = Symbol2 ? Symbol2.iterator : undefined2, symToStringTag = Symbol2 ? Symbol2.toStringTag : undefined2;
         var defineProperty = (function() {
           try {
             var func = getNative(Object2, "defineProperty");
@@ -782,9 +780,9 @@ var require_lodash = __commonJS({
           } catch (e) {
           }
         })();
-        var ctxClearTimeout = context3.clearTimeout !== root.clearTimeout && context3.clearTimeout, ctxNow = Date2 && Date2.now !== root.Date.now && Date2.now, ctxSetTimeout = context3.setTimeout !== root.setTimeout && context3.setTimeout;
-        var nativeCeil = Math2.ceil, nativeFloor = Math2.floor, nativeGetSymbols = Object2.getOwnPropertySymbols, nativeIsBuffer = Buffer2 ? Buffer2.isBuffer : undefined2, nativeIsFinite = context3.isFinite, nativeJoin = arrayProto.join, nativeKeys = overArg(Object2.keys, Object2), nativeMax = Math2.max, nativeMin = Math2.min, nativeNow = Date2.now, nativeParseInt = context3.parseInt, nativeRandom = Math2.random, nativeReverse = arrayProto.reverse;
-        var DataView = getNative(context3, "DataView"), Map2 = getNative(context3, "Map"), Promise2 = getNative(context3, "Promise"), Set2 = getNative(context3, "Set"), WeakMap2 = getNative(context3, "WeakMap"), nativeCreate = getNative(Object2, "create");
+        var ctxClearTimeout = context2.clearTimeout !== root.clearTimeout && context2.clearTimeout, ctxNow = Date2 && Date2.now !== root.Date.now && Date2.now, ctxSetTimeout = context2.setTimeout !== root.setTimeout && context2.setTimeout;
+        var nativeCeil = Math2.ceil, nativeFloor = Math2.floor, nativeGetSymbols = Object2.getOwnPropertySymbols, nativeIsBuffer = Buffer2 ? Buffer2.isBuffer : undefined2, nativeIsFinite = context2.isFinite, nativeJoin = arrayProto.join, nativeKeys = overArg(Object2.keys, Object2), nativeMax = Math2.max, nativeMin = Math2.min, nativeNow = Date2.now, nativeParseInt = context2.parseInt, nativeRandom = Math2.random, nativeReverse = arrayProto.reverse;
+        var DataView = getNative(context2, "DataView"), Map2 = getNative(context2, "Map"), Promise2 = getNative(context2, "Promise"), Set2 = getNative(context2, "Set"), WeakMap2 = getNative(context2, "WeakMap"), nativeCreate = getNative(Object2, "create");
         var metaMap = WeakMap2 && new WeakMap2();
         var realNames = {};
         var dataViewCtorString = toSource(DataView), mapCtorString = toSource(Map2), promiseCtorString = toSource(Promise2), setCtorString = toSource(Set2), weakMapCtorString = toSource(WeakMap2);
@@ -10041,21 +10039,37 @@ var EVAL_DATASET_MAX_REQUEST_BYTES = 4 * 1024 * 1024;
 var EVAL_DATASET_MAX_REFERENCE_BYTES = 1024 * 1024;
 var EvalDatasetSlugSchema = z2.string().min(1).max(64).regex(/^[a-z0-9]+(-[a-z0-9]+)*$/);
 var EvalDatasetFingerprintSchema = z2.string().regex(/^[0-9a-f]{64}$/);
+var EvalDatasetEventSourceSchema = z2.object({
+  sourceEventId: z2.string().trim().min(1).max(512),
+  sourceEventTimestamp: z2.string().datetime({ offset: true }).optional()
+}).strict();
+var EvalDatasetCaptureRefSchema = EvalDatasetEventSourceSchema.extend({
+  id: EvalDatasetFingerprintSchema
+}).strict();
+var EvalReferenceVerdictSchema = z2.union([
+  z2.object({ pass: z2.boolean() }).strict(),
+  z2.object({ score: z2.number().int().min(1).max(5) }).strict(),
+  z2.object({ value: z2.number().finite() }).strict()
+]);
 var EvalDatasetCaseInputSchema = z2.object({
   id: EvalDatasetCaseIdSchema,
   name: z2.string().trim().min(1).max(200),
   input: z2.string().nullable(),
+  output: JsonValueSchema.optional(),
   properties: z2.record(z2.string(), z2.string()),
-  referenceTrace: CapturedTraceSnapshotSchema.optional()
+  referenceTrace: CapturedTraceSnapshotSchema.optional(),
+  referenceCapture: EvalDatasetCaptureRefSchema.optional(),
+  expectation: EvalCaseExpectationSchema.optional(),
+  expectedVerdict: EvalReferenceVerdictSchema.optional()
 }).strict();
 var EvalDatasetVersionCaseSchema = EvalDatasetCaseInputSchema.extend({
   referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
 });
-var EvalDatasetCaseInputsSchema = z2.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context3) => {
+var EvalDatasetCaseInputsSchema = z2.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context2) => {
   const seen = /* @__PURE__ */ new Set();
   for (const [index, entry] of cases.entries()) {
     if (seen.has(entry.id)) {
-      context3.addIssue({
+      context2.addIssue({
         code: z2.ZodIssueCode.custom,
         message: `Duplicate row id ${entry.id}`,
         path: [index, "id"]
@@ -10071,6 +10085,21 @@ var PublishEvalDatasetVersionInputSchema = z2.object({
   fingerprint: EvalDatasetFingerprintSchema,
   cases: EvalDatasetCaseInputsSchema
 }).strict();
+var CaptureEvalDatasetCaseSchema = EvalDatasetCaseInputSchema.pick({
+  id: true,
+  name: true,
+  properties: true,
+  expectation: true,
+  expectedVerdict: true
+}).merge(EvalDatasetEventSourceSchema).strict();
+var CaptureEvalDatasetVersionInputSchema = PublishEvalDatasetVersionInputSchema.omit({
+  cases: true,
+  fingerprint: true
+}).extend({
+  requestKey: z2.string().uuid().optional(),
+  expectedCurrentVersionId: z2.string().uuid().nullable().optional(),
+  cases: z2.array(CaptureEvalDatasetCaseSchema).min(1).max(EVAL_DATASET_MAX_CASES).refine((cases) => new Set(cases.map((entry) => entry.id)).size === cases.length, "Case IDs must be unique")
+}).strict();
 var EvalDatasetIdentitySchema = z2.object({
   id: z2.string().uuid(),
   slug: EvalDatasetSlugSchema,
@@ -10092,9 +10121,13 @@ var EvalDatasetManifestSchema = z2.object({
     id: true,
     name: true,
     input: true,
+    output: true,
     properties: true
   }).extend({
-    referenceTraceSha256: EvalDatasetFingerprintSchema.optional()
+    referenceTraceSha256: EvalDatasetFingerprintSchema.optional(),
+    referenceCapture: EvalDatasetCaptureRefSchema.optional(),
+    expectation: EvalCaseExpectationSchema.optional(),
+    expectedVerdict: EvalReferenceVerdictSchema.optional()
   }))
 });
 function compareNanoseconds(left, right) {
@@ -10183,11 +10216,12 @@ var ReplayRowSchema = z3.object({
   id: z3.string(),
   name: z3.string(),
   input: z3.string().nullable(),
-  output: z3.string().nullable(),
+  output: JsonValueSchema,
   properties: z3.record(z3.string())
 });
 var WireEvalManifestSchema = z3.object({
   dataset_id: z3.string().uuid(),
+  dataset_version_id: z3.string().uuid().optional(),
   fingerprint: z3.string().regex(/^[a-f0-9]{64}$/),
   rows: z3.array(ReplayRowSchema.extend({ id: z3.string().min(1) }))
 });
@@ -10273,20 +10307,20 @@ var ReplayTraceEvidenceEntrySchema = z3.object({
   rowId: z3.string().min(1).max(512),
   traceId: z3.string().regex(/^[a-f0-9]{32}$/),
   spanIds: z3.array(z3.string().regex(/^[a-f0-9]{16}$/)).min(1).max(500)
-}).superRefine((entry, context3) => {
+}).superRefine((entry, context2) => {
   if (new Set(entry.spanIds).size !== entry.spanIds.length) {
-    context3.addIssue({
+    context2.addIssue({
       code: z3.ZodIssueCode.custom,
       message: "Replay trace evidence span ids must be unique",
       path: ["spanIds"]
     });
   }
 });
-var ReplayTraceEvidenceSchema = z3.array(ReplayTraceEvidenceEntrySchema).superRefine((entries, context3) => {
+var ReplayTraceEvidenceSchema = z3.array(ReplayTraceEvidenceEntrySchema).superRefine((entries, context2) => {
   const seen = /* @__PURE__ */ new Set();
   for (const [index, entry] of entries.entries()) {
     if (seen.has(entry.rowId)) {
-      context3.addIssue({
+      context2.addIssue({
         code: z3.ZodIssueCode.custom,
         message: "Replay trace evidence row ids must be unique",
         path: [index, "rowId"]
@@ -10730,6 +10764,7 @@ async function readEvalManifest(client, dataset, options) {
   }
   return {
     datasetId: manifest.dataset_id,
+    datasetVersionId: manifest.dataset_version_id,
     fingerprint: manifest.fingerprint,
     cases: manifest.rows
   };
@@ -10751,7 +10786,7 @@ async function readEvalDataset(client, dataset, options) {
       schema: EvalDatasetManifestSchema
     })
   );
-  if (manifest.dataset.slug !== slug) {
+  if (manifest.dataset.slug !== slug && manifest.dataset.id !== slug) {
     throw new Error(`Raindrop eval dataset ${slug} returned mismatched dataset identity`);
   }
   if (versionId !== void 0 && manifest.version.id !== versionId) {
@@ -12407,15 +12442,11 @@ async function publishEvalSuite(client, definition, options = {}) {
   const localDataset = isEvalDataset(suite.dataset) ? suite.dataset : void 0;
   const slug = EvalDatasetSlugSchema.parse(localDataset ? localDataset.id : suite.dataset);
   const rows = localDataset == null ? void 0 : localDataset.rows.map((row) => {
-    if (row.output !== null) {
-      throw new Error(
-        "Publish reference evidence with publishEvalDataset first; a local row output is not a reference trace"
-      );
-    }
     return EvalDatasetCaseInputSchema.parse({
       id: row.id,
       name: row.name,
       input: row.input,
+      output: row.output === null ? void 0 : row.output,
       properties: row.properties
     });
   });
@@ -12462,7 +12493,7 @@ async function publishEvalSuite(client, definition, options = {}) {
   });
   return defineEvalSuite({
     ...suite,
-    dataset: slug,
+    dataset: dataset.dataset.slug,
     datasetVersionId: dataset.version.id,
     evaluators
   });
@@ -12528,7 +12559,7 @@ var EvalSnapshotSchema = z7.object({
         id: z7.string(),
         name: z7.string(),
         input: z7.string().nullable(),
-        output: z7.string().nullable(),
+        output: JsonValueSchema,
         properties: z7.record(z7.string())
       })
     )
@@ -12688,9 +12719,6 @@ async function writeSnapshot(path, value) {
   }
 }
 
-// src/evals/run.ts
-import { context as context2, trace as otelTrace } from "@opentelemetry/api";
-
 // src/evals/policy.ts
 function assessEvalVerdict(evaluator, verdict, resolvedOutput) {
   var _a;
@@ -12732,132 +12760,10 @@ function outputMismatch(name, output) {
   return new Error(`Raindrop evaluator ${name} expected a ${output} verdict`);
 }
 
-// src/evals/workshop.ts
-import { z as z8 } from "zod";
-var WorkshopRunSchema = z8.object({
-  id: z8.string(),
-  event_id: z8.string().nullable(),
-  name: z8.string().nullable(),
-  event_name: z8.string().nullable(),
-  user_id: z8.string().nullable(),
-  convo_id: z8.string().nullable(),
-  started_at: z8.number().nullable(),
-  last_updated_at: z8.number().nullable(),
-  metadata: z8.string().nullable(),
-  model: z8.string().nullable(),
-  finished: z8.number().nullable()
-});
-var WorkshopSpanSchema = z8.object({
-  id: z8.string(),
-  run_id: z8.string(),
-  parent_span_id: z8.string().nullable(),
-  name: z8.string(),
-  span_type: z8.string().nullable(),
-  status: z8.string().nullable(),
-  input_payload: z8.string().nullable(),
-  output_payload: z8.string().nullable(),
-  start_time_ms: z8.number().nullable(),
-  end_time_ms: z8.number().nullable(),
-  duration_ms: z8.number().nullable(),
-  model: z8.string().nullable(),
-  provider: z8.string().nullable(),
-  input_tokens: z8.number().nullable(),
-  output_tokens: z8.number().nullable(),
-  attributes: z8.string().nullable()
-});
-var WorkshopTraceEnvelopeSchema = z8.object({
-  status: z8.literal("complete"),
-  correlationId: z8.string(),
-  traceId: z8.string(),
-  rootSpanId: z8.string(),
-  spanCount: z8.number().int().positive(),
-  run: WorkshopRunSchema,
-  spans: z8.array(WorkshopSpanSchema).min(1)
-});
-async function readWorkshopTrace(input) {
-  if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
-    throw new Error("Workshop waitMs must be finite and non-negative");
-  const deadlineAt = Date.now() + input.waitMs;
-  if (input.expectedSpanIds.length === 0) {
-    throw new Error(`Workshop trace ${input.correlationId} exported no spans`);
-  }
-  const url = new URL(
-    `${credentialEndpoint(input.workshopUrl).href.replace(/\/+$/, "")}/v1/evals/traces/${encodeURIComponent(input.correlationId)}`
-  );
-  for (const spanId of input.expectedSpanIds) url.searchParams.append("expectedSpanId", spanId);
-  while (true) {
-    const response = await runWithTracingSuppressed(
-      () => fetch(url, {
-        redirect: "error",
-        signal: AbortSignal.timeout(
-          Math.min(3e4, Math.max(1, Math.ceil(deadlineAt - Date.now())))
-        )
-      })
-    );
-    if (response.ok) {
-      const parsed = WorkshopTraceEnvelopeSchema.safeParse(await response.json());
-      if (!parsed.success) {
-        throw new Error(`Workshop returned an invalid completed trace: ${parsed.error.message}`);
-      }
-      return parsed.data;
-    }
-    if (response.status !== 404 && response.status !== 409) {
-      throw new Error(
-        `Workshop trace read failed with ${response.status}: ${await response.text()}`
-      );
-    }
-    if (Date.now() >= deadlineAt) {
-      const state = response.status === 404 ? "missing" : "incomplete";
-      throw new Error(`Workshop trace ${input.correlationId} remained ${state}`);
-    }
-    await new Promise(
-      (resolve) => setTimeout(resolve, Math.min(50, Math.max(0, deadlineAt - Date.now())))
-    );
-  }
-}
-async function writeWorkshopEvalRun(workshopUrl, packet) {
-  const normalized = {
-    ...packet,
-    cases: packet.cases.map((entry) => ({
-      ...entry,
-      task: entry.task.state === "completed" ? { ...entry.task, output: serializableOutput(entry.task.output) } : entry.task
-    }))
-  };
-  const response = await runWithTracingSuppressed(
-    () => fetch(`${credentialEndpoint(workshopUrl).href.replace(/\/+$/, "")}/v1/evals/runs`, {
-      method: "POST",
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify(normalized),
-      redirect: "error",
-      signal: AbortSignal.timeout(3e4)
-    })
-  );
-  if (!response.ok) {
-    throw new Error(
-      `Workshop eval result write failed with ${response.status}: ${await response.text()}`
-    );
-  }
-  const body = await response.json();
-  if (typeof body !== "object" || body === null || !("id" in body) || typeof body.id !== "string") {
-    throw new Error("Workshop eval result write returned an invalid response");
-  }
-  return { id: body.id };
-}
-function serializableOutput(value) {
-  try {
-    const serialized = JSON.stringify(boundedClone(value, 64e3));
-    if (serialized === void 0) return null;
-    if (serialized.length > 64e3) return "[Output exceeds 64000 characters]";
-    return JSON.parse(serialized);
-  } catch (e) {
-    return "[Output could not be serialized]";
-  }
-}
-
 // src/evals/run.ts
-var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
 function createEvalSuiteRun(client, definition, options) {
   var _a;
+  assertRaindropDestination(options.destination);
   const validated = validateDefinition(definition);
   const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
   const session = createSuiteSession(client, validated, execution.run, options);
@@ -12879,66 +12785,41 @@ function bindSuite(definition, parameters) {
   return definition.agent.start(parameters);
 }
 function createSuiteSession(client, validated, run, options) {
-  if (validated.datasetVersionId && "selection" in options && options.selection.dataset.version.id !== validated.datasetVersionId) {
+  if (validated.datasetVersionId && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
-  if (options.destination.kind === "raindrop") {
-    if (!("selection" in options)) {
-      throw new Error("Raindrop remote eval sessions require selected rows");
-    }
-    if (typeof validated.dataset !== "string") {
-      throw new Error("Raindrop remote eval sessions require a dataset slug");
-    }
-    if (options.selection.dataset.dataset.slug !== validated.dataset) {
-      throw new Error(
-        `Raindrop eval ${validated.name} pinned dataset ${options.selection.dataset.dataset.slug} does not match ${validated.dataset}`
-      );
-    }
-    const localEvaluators = validated.evaluators.flatMap((entry) => {
-      const evaluator = replayEvaluator(entry);
-      return typeof evaluator === "string" ? [] : [evaluator];
-    });
-    if (localEvaluators.length !== validated.evaluators.length) {
-      throw new Error("Raindrop row-scoped eval sessions support local evaluators only");
-    }
-    const selection = validateRemoteSelection(validated.dataset, options.selection);
-    if (!selection) throw new Error("Raindrop remote eval sessions require selected rows");
-    return new RaindropRowEvalRun(
-      client,
-      validated,
-      run,
-      options.destination,
-      options.selection.dataset,
-      selection.rowIds,
-      localEvaluators
-    );
-  }
-  if (!isEvalDataset(validated.dataset)) {
-    throw new Error("Workshop eval execution requires a pulled or local dataset snapshot");
-  }
-  credentialEndpoint(options.destination.url);
-  const hosted = validated.evaluators.find((entry) => typeof entry.evaluator === "string");
-  if (hosted && typeof hosted.evaluator === "string") {
-    throw new Error(`Workshop evaluator ${hosted.evaluator} must be pulled before local execution`);
+  if (typeof validated.dataset !== "string") {
+    throw new Error("Raindrop eval sessions require a dataset slug");
   }
-  if (validated.evaluators.some(
-    (entry) => typeof entry.evaluator !== "string" && "kind" in entry.evaluator && entry.evaluator.kind === "program"
-  )) {
+  if (options.selection.dataset.dataset.id !== validated.dataset && options.selection.dataset.dataset.slug !== validated.dataset) {
     throw new Error(
-      "Workshop authored evaluator programs must be pulled as portable evaluators before local execution"
+      `Raindrop eval ${validated.name} pinned dataset ${options.selection.dataset.dataset.slug} does not match ${validated.dataset}`
     );
   }
-  return new WorkshopEvalRun(
+  const localEvaluators = validated.evaluators.flatMap((entry) => {
+    const evaluator = replayEvaluator(entry);
+    return typeof evaluator === "string" ? [] : [evaluator];
+  });
+  if (localEvaluators.length !== validated.evaluators.length) {
+    throw new Error("Raindrop row-scoped eval sessions support local evaluators only");
+  }
+  const selection = validateRemoteSelection(validated.dataset, options.selection);
+  if (!selection) throw new Error("Raindrop eval sessions require selected rows");
+  return new RaindropRowEvalRun(
     client,
-    { ...validated, dataset: validated.dataset },
+    validated,
     run,
-    options.destination
+    options.destination,
+    options.selection.dataset,
+    selection.rowIds,
+    localEvaluators
   );
 }
 async function runEvalSuite(client, definition, options = {}) {
-  var _a;
+  var _a, _b;
+  assertRaindropDestination((_a = options.destination) != null ? _a : { kind: "raindrop" });
   const validated = validateDefinition(definition);
-  const execution = bindSuite(validated, (_a = options.parameters) != null ? _a : {});
+  const execution = bindSuite(validated, (_b = options.parameters) != null ? _b : {});
   return finishWith(execution, () => runSuite(client, validated, execution.run, options));
 }
 async function runSuite(client, definition, run, options) {
@@ -12948,12 +12829,6 @@ async function runSuite(client, definition, run, options) {
   if (validated.datasetVersionId && ((_b = options.selection) == null ? void 0 : _b.dataset) && options.selection.dataset.version.id !== validated.datasetVersionId) {
     throw new Error("Selected dataset version does not match the published suite");
   }
-  if (destination.kind === "workshop") {
-    if (options.runId !== void 0) throw new Error("Queued runs are only supported in Raindrop");
-    const session = createSuiteSession(client, validated, run, { destination });
-    await session.run({ selection: options.selection, attempt: options.attempt });
-    return session.finish();
-  }
   const alreadyPublished = isPublishedEvalSuite(validated);
   if (typeof validated.dataset === "string" && ((_c = options.selection) == null ? void 0 : _c.dataset)) {
     validateRemoteSelection(validated.dataset, options.selection);
@@ -13183,328 +13058,6 @@ function replayEvaluatorsFromVerdicts(verdicts) {
   }
   return [...evaluators.values()];
 }
-var WorkshopEvalRun = class {
-  constructor(client, definition, runRow, destination) {
-    this.client = client;
-    this.definition = definition;
-    this.runRow = runRow;
-    this.destination = destination;
-    this.id = randomId();
-    this.startedAt = /* @__PURE__ */ new Date();
-    this.completed = [];
-    this.active = /* @__PURE__ */ new Set();
-    this.state = "open";
-    this.runningRows = 0;
-    this.rowWaiters = [];
-  }
-  async run(options = {}) {
-    if (this.state !== "open") throw new Error(`Raindrop eval run ${this.id} is already finishing`);
-    const operation = this.runOnce(options);
-    this.active.add(operation);
-    try {
-      return await operation;
-    } finally {
-      this.active.delete(operation);
-    }
-  }
-  async runOnce(options) {
-    var _a;
-    const rows = selectLocalRows(this.definition, options.selection);
-    const completed = await Promise.all(
-      rows.map((row) => this.withRowPermit(() => {
-        var _a2;
-        return this.executeRow(row, (_a2 = options.attempt) != null ? _a2 : 0);
-      }))
-    );
-    const graded = await this.applyPortableEvaluators(completed, (_a = options.attempt) != null ? _a : 0);
-    this.completed.push(...graded);
-    return workshopResult(
-      this.id,
-      this.definition.name,
-      graded.map((entry) => entry.result)
-    );
-  }
-  finish() {
-    if (this.finishPromise) return this.finishPromise;
-    this.state = "finishing";
-    this.finishPromise = this.finishOnce();
-    return this.finishPromise;
-  }
-  async finishOnce() {
-    await Promise.allSettled([...this.active]);
-    const result = workshopResult(
-      this.id,
-      this.definition.name,
-      this.completed.map((entry) => entry.result)
-    );
-    if (this.completed.length > 0) {
-      const packet = {
-        schemaVersion: 1,
-        run: {
-          id: this.id,
-          name: this.definition.name,
-          destination: "workshop",
-          startedAt: this.startedAt.toISOString(),
-          finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
-          dataset: {
-            id: this.definition.dataset.id,
-            name: this.definition.dataset.name,
-            version: this.definition.dataset.version
-          },
-          evaluators: this.definition.evaluators.map((entry) => {
-            if (typeof entry.evaluator === "string") {
-              throw new Error(
-                `Workshop evaluator ${entry.evaluator} must be pulled before local execution`
-              );
-            }
-            if (isPortableEvaluator(entry.evaluator)) {
-              return {
-                slug: entry.evaluator.slug,
-                name: entry.evaluator.name,
-                output: entry.evaluator.output,
-                execution: "portable",
-                version: entry.evaluator.artifact.evaluator.programVersion,
-                artifactSha256: entry.evaluator.artifact.sha256,
-                scope: entry.evaluator.scope,
-                threshold: entry.threshold
-              };
-            }
-            return {
-              slug: entry.evaluator.slug,
-              name: entry.evaluator.name,
-              output: entry.evaluator.output,
-              execution: "local",
-              scope: entry.evaluator.scope === "row" ? "case" : entry.evaluator.scope,
-              threshold: entry.threshold
-            };
-          })
-        },
-        cases: this.completed.map((entry) => entry.packet)
-      };
-      await writeWorkshopEvalRun(this.destination.url, packet);
-    }
-    this.state = "finished";
-    return result;
-  }
-  async executeRow(row, attempt) {
-    var _a, _b, _c;
-    const correlationId = randomId();
-    const exportedSpanIds = /* @__PURE__ */ new Set();
-    const startedAt = Date.now();
-    let completedAgent;
-    let taskError;
-    try {
-      const destination = {
-        kind: "workshop",
-        url: this.destination.url,
-        exportedSpanIds
-      };
-      const value = await withReplayTraceDestination(
-        destination,
-        () => withEvalScope(
-          { correlationId },
-          () => this.client.tracer().withSpan(
-            {
-              name: "raindrop.eval.case",
-              properties: {
-                "raindrop.eval.run_id": this.id,
-                "raindrop.eval.case_id": row.id,
-                "raindrop.eval.attempt": String(attempt)
-              },
-              inputParameters: [row],
-              traceContent: true
-            },
-            () => {
-              var _a2;
-              const parent = (_a2 = otelTrace.getSpan(context2.active())) == null ? void 0 : _a2.spanContext();
-              if (!parent) return this.runRow(row);
-              return withReplayTraceDestination(
-                {
-                  ...destination,
-                  parentSpanContext: {
-                    traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
-                    spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
-                    eventId: parent.traceId
-                  }
-                },
-                () => this.runRow(row)
-              );
-            }
-          )
-        )
-      );
-      completedAgent = { value };
-    } catch (cause) {
-      taskError = describeError2(cause);
-    }
-    const durationMs = Date.now() - startedAt;
-    const taskCompleted = completedAgent !== void 0;
-    let trace2;
-    let traceId = null;
-    let traceError;
-    try {
-      await this.client.forceFlush();
-      const envelope = await readWorkshopTrace({
-        workshopUrl: this.destination.url,
-        correlationId,
-        waitMs: (_a = this.definition.traceWaitMs) != null ? _a : DEFAULT_WORKSHOP_TRACE_WAIT_MS,
-        expectedSpanIds: [...exportedSpanIds]
-      });
-      traceId = envelope.traceId;
-      trace2 = canonicalWorkshopTrace(envelope);
-    } catch (cause) {
-      traceError = describeError2(cause);
-    }
-    const verdicts = [];
-    if (completedAgent && !traceError && trace2) {
-      const agentResult = completedAgent.value;
-      await Promise.all(
-        this.definition.evaluators.map(async (entry) => {
-          if (typeof entry.evaluator === "string") return;
-          if (isPortableEvaluator(entry.evaluator)) return;
-          if ("kind" in entry.evaluator && entry.evaluator.kind === "program") return;
-          try {
-            const verdict = parseLocalVerdict2(
-              entry.evaluator.output,
-              await entry.evaluator.judge({ trace: trace2, row, result: agentResult })
-            );
-            verdicts.push(
-              withVerdictIdentity(entry.evaluator.slug, this.id, this.destination.url, verdict)
-            );
-          } catch (cause) {
-            verdicts.push({
-              evaluator: entry.evaluator.slug,
-              runId: this.id,
-              url: workshopEvalUrl(this.destination.url, this.id),
-              state: "errored",
-              message: describeError2(cause)
-            });
-          }
-        })
-      );
-    }
-    if (taskCompleted && traceError) {
-      for (const entry of this.definition.evaluators) {
-        if (typeof entry.evaluator === "string") continue;
-        verdicts.push({
-          evaluator: entry.evaluator.slug,
-          runId: this.id,
-          url: workshopEvalUrl(this.destination.url, this.id),
-          state: "ungraded",
-          reason: "trace unavailable"
-        });
-      }
-    }
-    const assessments = verdicts.map((verdict) => {
-      const evaluator = this.definition.evaluators.find(
-        (entry) => evaluatorName(entry) === verdict.evaluator
-      );
-      if (!evaluator)
-        throw new Error(`Raindrop eval result included unknown evaluator ${verdict.evaluator}`);
-      return assessEvalVerdict(evaluator, verdict, evaluatorOutput(evaluator));
-    });
-    const error = (_b = taskError != null ? taskError : traceError) != null ? _b : null;
-    const result = {
-      ...row,
-      attempt,
-      status: error ? "missing" : "done",
-      traceId,
-      error,
-      verdicts: assessments
-    };
-    const packet = {
-      caseId: row.id,
-      caseName: row.name,
-      input: row.input,
-      ...row.output === null ? {} : { expected: row.output },
-      attempt,
-      traceId,
-      ...taskCompleted && traceError ? { traceError } : {},
-      task: taskError ? { state: "failed", error: taskError, durationMs } : { state: "completed", output: (_c = completedAgent == null ? void 0 : completedAgent.value) != null ? _c : null, durationMs },
-      verdicts: verdicts.map(workshopVerdict),
-      passed: error === null && assessments.every(
-        (assessment) => assessment.state === "passed" || assessment.state === "measurement"
-      )
-    };
-    return { result, packet, trace: trace2 };
-  }
-  async withRowPermit(run) {
-    var _a;
-    const limit = (_a = this.definition.concurrency) != null ? _a : 4;
-    if (this.runningRows >= limit) {
-      await new Promise((resolve) => this.rowWaiters.push(resolve));
-    } else {
-      this.runningRows += 1;
-    }
-    try {
-      return await run();
-    } finally {
-      const next = this.rowWaiters.shift();
-      if (next) next();
-      else this.runningRows -= 1;
-    }
-  }
-  async applyPortableEvaluators(completed, attempt) {
-    let current = completed;
-    for (const entry of this.definition.evaluators) {
-      if (typeof entry.evaluator === "string" || !isPortableEvaluator(entry.evaluator)) continue;
-      const portableEvaluator = entry.evaluator;
-      const traces = current.flatMap(
-        (item) => item.packet.task.state === "completed" && item.trace ? [item.trace] : []
-      );
-      if (traces.length === 0) continue;
-      let outcomes;
-      try {
-        const portable = await evaluatePortableEvaluator(portableEvaluator, {
-          traces,
-          seed: `${this.id}:${attempt}:${portableEvaluator.slug}`
-        });
-        const programFailure = portable.failures.join("; ");
-        outcomes = new Map(
-          portable.outcomes.map((outcome) => [
-            outcome.traceId,
-            programFailure ? {
-              state: "errored",
-              traceId: outcome.traceId,
-              message: programFailure
-            } : outcome
-          ])
-        );
-      } catch (cause) {
-        const message = describeError2(cause);
-        outcomes = new Map(
-          traces.map((trace2) => [
-            trace2.event.id,
-            { state: "errored", traceId: trace2.event.id, message }
-          ])
-        );
-      }
-      current = current.map((item) => {
-        if (item.packet.task.state !== "completed" || !item.trace) return item;
-        const outcome = outcomes.get(item.trace.event.id);
-        if (!outcome) return item;
-        const verdict = portableVerdict(
-          portableEvaluator.slug,
-          this.id,
-          this.destination.url,
-          outcome
-        );
-        const assessment = assessEvalVerdict(entry, verdict, portableEvaluator.output);
-        const verdicts = [...item.result.verdicts, assessment];
-        return {
-          ...item,
-          result: { ...item.result, verdicts },
-          packet: {
-            ...item.packet,
-            verdicts: [...item.packet.verdicts, workshopVerdict(verdict)],
-            passed: item.packet.passed && (assessment.state === "passed" || assessment.state === "measurement")
-          }
-        };
-      });
-    }
-    return current;
-  }
-};
 async function runRaindropEval(client, definition, run, destination, options) {
   var _a, _b, _c, _d;
   if (typeof definition.dataset !== "string") {
@@ -13517,7 +13070,7 @@ async function runRaindropEval(client, definition, run, destination, options) {
     queryUrl: destination.queryUrl,
     versionId: definition.datasetVersionId
   }) : void 0;
-  if (pinnedDataset && pinnedDataset.dataset.slug !== datasetReference) {
+  if (pinnedDataset && pinnedDataset.dataset.id !== datasetReference && pinnedDataset.dataset.slug !== datasetReference) {
     throw new Error(
       `Raindrop eval ${definition.name} pinned dataset ${pinnedDataset.dataset.slug} does not match ${datasetReference}`
     );
@@ -13646,20 +13199,6 @@ function validateDefinition(definition) {
     throw new Error("Raindrop runEvalSuite requires a definition created by defineEvalSuite");
   return definition;
 }
-function selectLocalRows(definition, selection) {
-  if (!selection) return definition.dataset.rows;
-  if (selection.rowIds.length === 0)
-    throw new Error(`Raindrop eval ${definition.name} selection cannot be empty`);
-  if (new Set(selection.rowIds).size !== selection.rowIds.length)
-    throw new Error(`Raindrop eval ${definition.name} selection row ids must be unique`);
-  const byId = new Map(definition.dataset.rows.map((row) => [row.id, row]));
-  return selection.rowIds.map((id) => {
-    const row = byId.get(id);
-    if (!row)
-      throw new Error(`Raindrop eval ${definition.name} selection contains unknown row ${id}`);
-    return row;
-  });
-}
 function validateRemoteSelection(dataset, selection) {
   var _a;
   if (!selection) return void 0;
@@ -13675,157 +13214,20 @@ function validateRemoteSelection(dataset, selection) {
     throw new Error(`Raindrop eval ${dataset} selection contains unknown row ${unknown}`);
   return { rowIds: selection.rowIds };
 }
-function canonicalWorkshopTrace(envelope) {
-  var _a, _b, _c, _d, _e, _f;
-  const spans = envelope.spans.map(rawTraceSpan);
-  const timestamp = new Date((_a = envelope.run.started_at) != null ? _a : Date.now()).toISOString();
-  const event = projectCapturedEvent({
-    eventId: (_b = envelope.run.event_id) != null ? _b : envelope.traceId,
-    internalEventId: envelope.run.id,
-    name: (_d = (_c = envelope.run.event_name) != null ? _c : envelope.run.name) != null ? _d : "ai_generation",
-    timestamp,
-    receivedAt: new Date((_e = envelope.run.last_updated_at) != null ? _e : Date.now()).toISOString(),
-    userId: (_f = envelope.run.user_id) != null ? _f : "local-eval",
-    convoId: envelope.run.convo_id,
-    properties: parseAttributes(envelope.run.metadata),
-    spans
-  });
-  return projectWorkshopTrace({ origin: "replay", event, spans });
-}
-function rawTraceSpan(span) {
-  var _a, _b, _c, _d, _e, _f;
-  const attributes = parseAttributes(span.attributes);
-  return {
-    span_id: span.id,
-    parent_span_id: (_a = span.parent_span_id) != null ? _a : void 0,
-    span_name: span.name,
-    span_type: (_b = span.span_type) != null ? _b : void 0,
-    status: (_c = span.status) != null ? _c : void 0,
-    duration_ns: span.duration_ms === null ? void 0 : span.duration_ms * 1e6,
-    start_unix_ns: span.start_time_ms === null ? void 0 : span.start_time_ms * 1e6,
-    model: (_d = span.model) != null ? _d : void 0,
-    input_tokens: (_e = span.input_tokens) != null ? _e : void 0,
-    output_tokens: (_f = span.output_tokens) != null ? _f : void 0,
-    input_payload: span.input_payload,
-    output_payload: span.output_payload,
-    attributes_string: Object.fromEntries(
-      Object.entries(attributes).filter(
-        (entry) => typeof entry[1] === "string"
-      )
-    ),
-    attributes_num: Object.fromEntries(
-      Object.entries(attributes).filter(
-        (entry) => typeof entry[1] === "number"
-      )
-    )
-  };
-}
-function parseAttributes(raw) {
-  if (!raw) return {};
-  try {
-    const value = JSON.parse(raw);
-    return isAttributeRecord(value) ? value : {};
-  } catch (e) {
-    return {};
-  }
-}
-function isAttributeRecord(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function parseLocalVerdict2(output, value) {
-  if (output === "boolean") return { state: "graded", ...LocalBooleanVerdictSchema.parse(value) };
-  if (output === "score") return { state: "graded", ...LocalScoreVerdictSchema.parse(value) };
-  return { state: "graded", ...LocalNumberVerdictSchema.parse(value) };
-}
-function withVerdictIdentity(evaluator, runId, workshopUrl, verdict) {
-  return {
-    evaluator,
-    runId,
-    url: workshopEvalUrl(workshopUrl, runId),
-    ...verdict
-  };
-}
-function workshopVerdict(assessment) {
-  const { evaluator, state } = assessment;
-  if (state === "errored") return { evaluator, state, message: assessment.message };
-  if (state === "ungraded") return { evaluator, state, reason: assessment.reason };
-  if ("pass" in assessment)
-    return {
-      evaluator,
-      state,
-      pass: assessment.pass,
-      ...assessment.note ? { note: assessment.note } : {}
-    };
-  if ("score" in assessment)
-    return {
-      evaluator,
-      state,
-      score: assessment.score,
-      ...assessment.note ? { note: assessment.note } : {}
-    };
-  return {
-    evaluator,
-    state,
-    value: assessment.value,
-    ...assessment.note ? { note: assessment.note } : {}
-  };
-}
-function portableVerdict(evaluator, runId, workshopUrl, outcome) {
-  const identity = { evaluator, runId, url: workshopEvalUrl(workshopUrl, runId) };
-  if (outcome.state === "errored") {
-    return { ...identity, state: outcome.state, message: outcome.message };
-  }
-  if (outcome.state === "ungraded") {
-    return { ...identity, state: outcome.state, reason: outcome.reason };
-  }
-  if ("pass" in outcome)
-    return {
-      ...identity,
-      state: outcome.state,
-      pass: outcome.pass,
-      ...outcome.note ? { note: outcome.note } : {}
-    };
-  if ("score" in outcome)
-    return {
-      ...identity,
-      state: outcome.state,
-      score: outcome.score,
-      ...outcome.note ? { note: outcome.note } : {}
-    };
-  return {
-    ...identity,
-    state: outcome.state,
-    value: outcome.value,
-    ...outcome.note ? { note: outcome.note } : {}
-  };
-}
-function workshopResult(id, name, rows) {
-  const done = rows.filter((entry) => entry.status === "done").length;
-  const missing = rows.filter((entry) => entry.status === "missing").length;
-  return {
-    runId: id,
-    name,
-    status: missing > 0 ? "failed" : "complete",
-    counts: { total: rows.length, pending: 0, done, missing },
-    tracesExpireAt: null,
-    tracesExpired: false,
-    evaluators: [],
-    rows
-  };
-}
-function workshopEvalUrl(baseUrl, id) {
-  return `${baseUrl.replace(/\/+$/, "")}/evals/${encodeURIComponent(id)}`;
-}
 function randomId() {
   var _a, _b, _c;
   return (_c = (_b = (_a = globalThis.crypto) == null ? void 0 : _a.randomUUID) == null ? void 0 : _b.call(_a)) != null ? _c : `eval-${Date.now()}-${Math.random().toString(16).slice(2)}`;
 }
-function describeError2(cause) {
-  return cause instanceof Error ? cause.message : String(cause);
-}
 function isPortableEvaluator(value) {
   return "kind" in value && value.kind === "portable";
 }
+function assertRaindropDestination(destination) {
+  if (destination.kind !== "raindrop") {
+    throw new Error(
+      "Eval runs store results in Raindrop; Workshop is no longer an eval destination"
+    );
+  }
+}
 
 // src/index.ts
 function rawAppGitProperties(properties) {
@@ -14293,14 +13695,14 @@ var Raindrop = class {
     const description = buildSelfDiagnosticsDescription(signals, options.guidance);
     const inputSchema = buildSelfDiagnosticsInputSchema(signalKeys);
     const onMissingEventId = (_b = options.onMissingEventId) != null ? _b : "warn";
-    const execute = async (rawInput, context3) => {
+    const execute = async (rawInput, context2) => {
       var _a2, _b2, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
       const input = isRecord(rawInput) ? rawInput : void 0;
       const fallbackCategory = (_a2 = signalKeys[0]) != null ? _a2 : "unknown";
       const categoryCandidate = normalizeEventId(input == null ? void 0 : input["category"]);
       const category = categoryCandidate && signalKeySet.has(categoryCandidate) ? categoryCandidate : fallbackCategory;
       const detail = (_b2 = normalizeEventId(input == null ? void 0 : input["detail"])) != null ? _b2 : "";
-      const eventId = (_l = (_k = (_i = (_g = (_f = (_e = (_d = normalizeEventId(context3 == null ? void 0 : context3.eventId)) != null ? _d : (_c = context3 == null ? void 0 : context3.interaction) == null ? void 0 : _c.getEventId()) != null ? _e : resolveEventIdFromMetadata(context3 == null ? void 0 : context3.metadata)) != null ? _f : normalizeEventId(input == null ? void 0 : input["event_id"])) != null ? _g : normalizeEventId(input == null ? void 0 : input["eventId"])) != null ? _i : (_h = options.interaction) == null ? void 0 : _h.getEventId()) != null ? _k : (_j = options.getEventId) == null ? void 0 : _j.call(options)) != null ? _l : normalizeEventId(options.eventId);
+      const eventId = (_l = (_k = (_i = (_g = (_f = (_e = (_d = normalizeEventId(context2 == null ? void 0 : context2.eventId)) != null ? _d : (_c = context2 == null ? void 0 : context2.interaction) == null ? void 0 : _c.getEventId()) != null ? _e : resolveEventIdFromMetadata(context2 == null ? void 0 : context2.metadata)) != null ? _f : normalizeEventId(input == null ? void 0 : input["event_id"])) != null ? _g : normalizeEventId(input == null ? void 0 : input["eventId"])) != null ? _i : (_h = options.interaction) == null ? void 0 : _h.getEventId()) != null ? _k : (_j = options.getEventId) == null ? void 0 : _j.call(options)) != null ? _l : normalizeEventId(options.eventId);
       if (!eventId) {
         const message = "self diagnostics signal skipped: missing eventId. Pass eventId, interaction, or getEventId when creating or executing the tool.";
         if (onMissingEventId === "throw") {
@@ -14318,11 +13720,11 @@ var Raindrop = class {
         type: "agent",
         sentiment: signalDefinition == null ? void 0 : signalDefinition.sentiment,
         properties: {
-          source: (context3 == null ? void 0 : context3.source) || options.source || SELF_DIAGNOSTICS_SOURCE_DEFAULT,
+          source: (context2 == null ? void 0 : context2.source) || options.source || SELF_DIAGNOSTICS_SOURCE_DEFAULT,
           category,
           signal_description: signalDefinition == null ? void 0 : signalDefinition.description,
           ...detail ? { detail } : {},
-          ...(_m = context3 == null ? void 0 : context3.properties) != null ? _m : {}
+          ...(_m = context2 == null ? void 0 : context2.properties) != null ? _m : {}
         }
       });
       return { acknowledged: true, category, eventId };
@@ -14728,7 +14130,7 @@ var Raindrop = class {
   }
   getContext() {
     const runtime = detectRuntime();
-    const context3 = {
+    const context2 = {
       library: {
         name: package_default.name,
         version: package_default.version
@@ -14738,9 +14140,9 @@ var Raindrop = class {
       }
     };
     if (runtime === "node") {
-      context3.metadata.nodeVersion = process.version;
+      context2.metadata.nodeVersion = process.version;
     }
-    return context3;
+    return context2;
   }
   formatZodError(error) {
     return error.issues.map((issue) => `(path: ${issue.path.join(".")}): ${issue.message}`).join(", ");