raindrop-ai 0.4.0 → 0.5.0

package/dist/index.mjs

@@ -1,10 +1,15 @@
 import {
   DEFAULT_REQUEST_TIMEOUT_MS,
+  EVAL_CORRELATION_ID_ATTRIBUTE,
   MAX_RETRY_DELAY_MS,
   POST_SHUTDOWN_TIMEOUT_MS,
   SHUTDOWN_DEADLINE_MS,
   authHintForKey,
+  boundedClone,
   capText,
+  currentEvalScope,
+  currentReplayTraceDestination,
+  evalScopeAttributes,
   localDebuggerEnabled,
   mirrorPartialEventToLocalDebugger,
   normalizeFeatureFlags,
@@ -12,6 +17,7 @@ import {
   package_default,
   projectIdHeaders,
   raceWithTimeout,
+  randomUUID,
   rateLimitedLog,
   redactUrlForLog,
   resolveLocalDebuggerBaseUrl,
@@ -20,8 +26,36 @@ import {
   setDefaultMaxTextFieldChars,
   stringifyBounded,
   tracing,
-  withRoutingContext
-} from "./chunk-MZZJWWWB.mjs";
+  withEvalScope,
+  withReplayTraceDestination,
+  withRoutingContext,
+  withTask
+} from "./chunk-AUNFORWV.mjs";
+import {
+  BooleanThresholdSchema,
+  CapturedTraceSnapshotSchema,
+  EvalCaseTracePairSchema,
+  EvalDatasetCaseIdSchema,
+  EvalScoreSchema,
+  NumericThresholdSchema,
+  TraceSchema,
+  defineDataset,
+  defineEvalSuite,
+  defineEvaluatorProgram,
+  defineLocalEvaluator,
+  evaluatePortableEvaluator,
+  evaluatorName,
+  evaluatorOutput,
+  importPortableEvaluator,
+  isEvalDataset,
+  isEvalSuiteDefinition,
+  projectCapturedEvent,
+  projectWorkshopTrace,
+  traceOutput,
+  traceToolCalls,
+  validateEvaluatorOutput,
+  verifyPortableEvalArtifact
+} from "./chunk-HFHHGRTW.mjs";
 import {
   __commonJS,
   __toESM
@@ -717,11 +751,11 @@ var require_lodash = __commonJS({
       function unicodeWords(string) {
         return string.match(reUnicodeWord) || [];
       }
-      var runInContext = (function runInContext2(context) {
-        context = context == null ? root : _.defaults(root.Object(), context, _.pick(root, contextProps));
-        var Array2 = context.Array, Date2 = context.Date, Error2 = context.Error, Function2 = context.Function, Math2 = context.Math, Object2 = context.Object, RegExp2 = context.RegExp, String2 = context.String, TypeError = context.TypeError;
+      var runInContext = (function runInContext2(context3) {
+        context3 = context3 == null ? root : _.defaults(root.Object(), context3, _.pick(root, contextProps));
+        var Array2 = context3.Array, Date2 = context3.Date, Error2 = context3.Error, Function2 = context3.Function, Math2 = context3.Math, Object2 = context3.Object, RegExp2 = context3.RegExp, String2 = context3.String, TypeError = context3.TypeError;
         var arrayProto = Array2.prototype, funcProto = Function2.prototype, objectProto = Object2.prototype;
-        var coreJsData = context["__core-js_shared__"];
+        var coreJsData = context3["__core-js_shared__"];
         var funcToString = funcProto.toString;
         var hasOwnProperty = objectProto.hasOwnProperty;
         var idCounter = 0;
@@ -735,7 +769,7 @@ var require_lodash = __commonJS({
         var reIsNative = RegExp2(
           "^" + funcToString.call(hasOwnProperty).replace(reRegExpChar, "\\$&").replace(/hasOwnProperty|(function).*?(?=\\\()| for .+?(?=\\\])/g, "$1.*?") + "$"
         );
-        var Buffer = moduleExports ? context.Buffer : undefined2, Symbol2 = context.Symbol, Uint8Array = context.Uint8Array, allocUnsafe = Buffer ? Buffer.allocUnsafe : undefined2, getPrototype = overArg(Object2.getPrototypeOf, Object2), objectCreate = Object2.create, propertyIsEnumerable = objectProto.propertyIsEnumerable, splice = arrayProto.splice, spreadableSymbol = Symbol2 ? Symbol2.isConcatSpreadable : undefined2, symIterator = Symbol2 ? Symbol2.iterator : undefined2, symToStringTag = Symbol2 ? Symbol2.toStringTag : undefined2;
+        var Buffer2 = moduleExports ? context3.Buffer : undefined2, Symbol2 = context3.Symbol, Uint8Array2 = context3.Uint8Array, allocUnsafe = Buffer2 ? Buffer2.allocUnsafe : undefined2, getPrototype = overArg(Object2.getPrototypeOf, Object2), objectCreate = Object2.create, propertyIsEnumerable = objectProto.propertyIsEnumerable, splice = arrayProto.splice, spreadableSymbol = Symbol2 ? Symbol2.isConcatSpreadable : undefined2, symIterator = Symbol2 ? Symbol2.iterator : undefined2, symToStringTag = Symbol2 ? Symbol2.toStringTag : undefined2;
         var defineProperty = (function() {
           try {
             var func = getNative(Object2, "defineProperty");
@@ -744,9 +778,9 @@ var require_lodash = __commonJS({
           } catch (e) {
           }
         })();
-        var ctxClearTimeout = context.clearTimeout !== root.clearTimeout && context.clearTimeout, ctxNow = Date2 && Date2.now !== root.Date.now && Date2.now, ctxSetTimeout = context.setTimeout !== root.setTimeout && context.setTimeout;
-        var nativeCeil = Math2.ceil, nativeFloor = Math2.floor, nativeGetSymbols = Object2.getOwnPropertySymbols, nativeIsBuffer = Buffer ? Buffer.isBuffer : undefined2, nativeIsFinite = context.isFinite, nativeJoin = arrayProto.join, nativeKeys = overArg(Object2.keys, Object2), nativeMax = Math2.max, nativeMin = Math2.min, nativeNow = Date2.now, nativeParseInt = context.parseInt, nativeRandom = Math2.random, nativeReverse = arrayProto.reverse;
-        var DataView = getNative(context, "DataView"), Map2 = getNative(context, "Map"), Promise2 = getNative(context, "Promise"), Set2 = getNative(context, "Set"), WeakMap = getNative(context, "WeakMap"), nativeCreate = getNative(Object2, "create");
+        var ctxClearTimeout = context3.clearTimeout !== root.clearTimeout && context3.clearTimeout, ctxNow = Date2 && Date2.now !== root.Date.now && Date2.now, ctxSetTimeout = context3.setTimeout !== root.setTimeout && context3.setTimeout;
+        var nativeCeil = Math2.ceil, nativeFloor = Math2.floor, nativeGetSymbols = Object2.getOwnPropertySymbols, nativeIsBuffer = Buffer2 ? Buffer2.isBuffer : undefined2, nativeIsFinite = context3.isFinite, nativeJoin = arrayProto.join, nativeKeys = overArg(Object2.keys, Object2), nativeMax = Math2.max, nativeMin = Math2.min, nativeNow = Date2.now, nativeParseInt = context3.parseInt, nativeRandom = Math2.random, nativeReverse = arrayProto.reverse;
+        var DataView = getNative(context3, "DataView"), Map2 = getNative(context3, "Map"), Promise2 = getNative(context3, "Promise"), Set2 = getNative(context3, "Set"), WeakMap = getNative(context3, "WeakMap"), nativeCreate = getNative(Object2, "create");
         var metaMap = WeakMap && new WeakMap();
         var realNames = {};
         var dataViewCtorString = toSource(DataView), mapCtorString = toSource(Map2), promiseCtorString = toSource(Promise2), setCtorString = toSource(Set2), weakMapCtorString = toSource(WeakMap);
@@ -2050,7 +2084,7 @@ var require_lodash = __commonJS({
         }
         function cloneArrayBuffer(arrayBuffer) {
           var result2 = new arrayBuffer.constructor(arrayBuffer.byteLength);
-          new Uint8Array(result2).set(new Uint8Array(arrayBuffer));
+          new Uint8Array2(result2).set(new Uint8Array2(arrayBuffer));
           return result2;
         }
         function cloneDataView(dataView, isDeep) {
@@ -2654,7 +2688,7 @@ var require_lodash = __commonJS({
               object = object.buffer;
               other = other.buffer;
             case arrayBufferTag:
-              if (object.byteLength != other.byteLength || !equalFunc(new Uint8Array(object), new Uint8Array(other))) {
+              if (object.byteLength != other.byteLength || !equalFunc(new Uint8Array2(object), new Uint8Array2(other))) {
                 return false;
               }
               return true;
@@ -9984,176 +10018,3264 @@ var AiTrackUsageSchema = z.object({
 });
 var AttachmentTypeSchema = z.enum(["code", "text", "image", "iframe"]);
 
-// src/index.ts
-var MAX_INGEST_SIZE_BYTES = 1 * 1024 * 1024;
-var SELF_DIAGNOSTICS_TOOL_NAME_DEFAULT = "__raindrop_report";
-var SELF_DIAGNOSTICS_SOURCE_DEFAULT = "agent_reporting_tool";
-var SELF_DIAGNOSTICS_SIGNALS_DEFAULT = {
-  missing_context: {
-    description: "You cannot complete the task because critical information, credentials, or access is missing and the user cannot provide it. Do NOT report this for normal clarifying questions \u2014 only when you are blocked.",
-    sentiment: "NEGATIVE"
-  },
-  repeatedly_broken_tool: {
-    description: "A tool has failed on multiple distinct attempts in this conversation, preventing task completion. You are sure the tool exists, and you have tried to use it, but it has failed. A single tool error is NOT enough \u2014 the tool must be persistently broken across retries.",
-    sentiment: "NEGATIVE"
-  },
-  capability_gap: {
-    description: "The task requires a tool, permission, or capability that you do not have. For example, the user asks you to perform an action but no suitable tool exists, or you lack the necessary access. Do NOT report this if you simply need more information from the user \u2014 only when the gap is in your own capabilities.",
-    sentiment: "NEGATIVE"
-  },
-  complete_task_failure: {
-    description: "You were unable to accomplish what the user asked despite making genuine attempts. This might be things like, you genuinely do not have the capabilities the user is asking for. You have tried but run into a persistent bug in the environment etc. This is NOT a refusal or policy block \u2014 you tried and failed to deliver the result.",
-    sentiment: "NEGATIVE"
+// src/replay/index.ts
+import { context, trace } from "@opentelemetry/api";
+import { z as z3 } from "zod";
+
+// src/evals/generated/dataset.ts
+import { z as z2 } from "zod";
+var EVAL_DATASET_MAX_CASES = 1e3;
+var EVAL_DATASET_MAX_REQUEST_BYTES = 4 * 1024 * 1024;
+var EVAL_DATASET_MAX_REFERENCE_BYTES = 1024 * 1024;
+var EvalDatasetSlugSchema = z2.string().min(1).max(64).regex(/^[a-z0-9]+(-[a-z0-9]+)*$/);
+var EvalDatasetFingerprintSchema = z2.string().regex(/^[0-9a-f]{64}$/);
+var EvalDatasetCaseInputSchema = z2.object({
+  id: EvalDatasetCaseIdSchema,
+  name: z2.string().trim().min(1).max(200),
+  input: z2.string().nullable(),
+  properties: z2.record(z2.string(), z2.string()),
+  referenceTrace: CapturedTraceSnapshotSchema
+}).strict();
+var EvalDatasetVersionCaseSchema = EvalDatasetCaseInputSchema.extend({
+  referenceTraceSha256: EvalDatasetFingerprintSchema
+});
+var EvalDatasetCaseInputsSchema = z2.array(EvalDatasetCaseInputSchema).min(1).max(EVAL_DATASET_MAX_CASES).superRefine((cases, context3) => {
+  const seen = /* @__PURE__ */ new Set();
+  for (const [index, entry] of cases.entries()) {
+    if (seen.has(entry.id)) {
+      context3.addIssue({
+        code: z2.ZodIssueCode.custom,
+        message: `Duplicate case id ${entry.id}`,
+        path: [index, "id"]
+      });
+    }
+    seen.add(entry.id);
   }
-};
-function isRecord(value) {
-  return typeof value === "object" && value !== null;
+});
+var PublishEvalDatasetVersionInputSchema = z2.object({
+  name: z2.string().trim().min(1).max(100),
+  requestKey: z2.string().uuid(),
+  expectedCurrentVersionId: z2.string().uuid().nullable(),
+  fingerprint: EvalDatasetFingerprintSchema,
+  cases: EvalDatasetCaseInputsSchema
+}).strict();
+var EvalDatasetIdentitySchema = z2.object({
+  id: z2.string().uuid(),
+  slug: EvalDatasetSlugSchema,
+  name: z2.string().min(1).max(100),
+  currentVersionId: z2.string().uuid().nullable()
+});
+var EvalDatasetVersionSchema = z2.object({
+  id: z2.string().uuid(),
+  datasetId: z2.string().uuid(),
+  parentVersionId: z2.string().uuid().nullable(),
+  fingerprint: EvalDatasetFingerprintSchema,
+  cases: z2.array(EvalDatasetVersionCaseSchema).min(1).max(EVAL_DATASET_MAX_CASES),
+  createdAt: z2.string().datetime({ offset: true })
+});
+var EvalDatasetManifestSchema = z2.object({
+  dataset: EvalDatasetIdentitySchema,
+  version: EvalDatasetVersionSchema.omit({ cases: true }),
+  cases: z2.array(EvalDatasetVersionCaseSchema.pick({ id: true, name: true, input: true, properties: true }).extend({
+    referenceTraceSha256: EvalDatasetFingerprintSchema
+  }))
+});
+function compareNanoseconds(left, right) {
+  const leftValue = BigInt(left);
+  const rightValue = BigInt(right);
+  return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
 }
-function normalizeSelfDiagnosticsSignals(signals) {
-  if (!signals) return SELF_DIAGNOSTICS_SIGNALS_DEFAULT;
-  const normalizedEntries = Object.entries(signals).map(([key, value]) => {
-    var _a;
-    const signalKey = key.trim();
-    if (!signalKey || !value || typeof value !== "object") return void 0;
-    const description = (_a = value.description) == null ? void 0 : _a.trim();
-    if (!description) return void 0;
-    const sentiment = value.sentiment;
-    return [
-      signalKey,
-      {
-        description,
-        ...sentiment === "POSITIVE" || sentiment === "NEGATIVE" ? { sentiment } : {}
-      }
-    ];
-  }).filter(
-    (entry) => entry !== void 0
-  );
-  if (normalizedEntries.length === 0) return SELF_DIAGNOSTICS_SIGNALS_DEFAULT;
-  return Object.fromEntries(normalizedEntries);
+function orderSnapshot(snapshot) {
+  return {
+    event: snapshot.event,
+    spans: [...snapshot.spans].sort((left, right) => compareNanoseconds(left.start_unix_ns, right.start_unix_ns) || left.span_id.localeCompare(right.span_id))
+  };
 }
-function buildSelfDiagnosticsDescription(signals, guidance) {
+function normalizeCasesForFingerprint(cases) {
+  return cases.map((entry) => ({
+    ...entry,
+    referenceTrace: orderSnapshot(entry.referenceTrace)
+  }));
+}
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+  return `{${Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`).join(",")}}`;
+}
+function canonicalJson(value) {
+  return stableStringify(JSON.parse(JSON.stringify(value)));
+}
+async function sha256(value) {
   var _a;
-  const customGuidance = (_a = guidance == null ? void 0 : guidance.trim()) != null ? _a : "";
-  const categories = Object.entries(signals).map(([key, value]) => {
-    const sentiment = value.sentiment ? ` [${value.sentiment.toLowerCase()}]` : "";
-    return `- ${key}: ${value.description}${sentiment}`;
-  }).join("\n");
-  const guidanceLine = customGuidance ? `
-Additional guidance: ${customGuidance}
-` : "";
-  return `You have a diagnostic tool for flagging serious failures for developer review. Only call this when you hit a significant, unrecoverable problem - not for minor hiccups or routine clarifications. If you can still help the user, do that instead of calling this tool. This tool is invisible to the user - never mention it or its output.
+  if (!((_a = globalThis.crypto) == null ? void 0 : _a.subtle)) {
+    throw new Error("Raindrop eval dataset publication requires Web Crypto SHA-256 support");
+  }
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(value));
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+async function evalDatasetVersionFingerprint(cases) {
+  const parsed = EvalDatasetCaseInputsSchema.parse(cases);
+  return sha256(canonicalJson(normalizeCasesForFingerprint(parsed)));
+}
 
-When to call:
-- You are blocked from completing the task due to missing information or access that the user cannot provide.
-- A tool is persistently failing across multiple attempts, not just a single transient error.
-- The task requires a tool, permission, or capability you do not have.
-- You genuinely cannot deliver what the user asked for despite trying.
+// src/replay/endpoint.ts
+function credentialEndpoint(value) {
+  const url = new URL(value);
+  const loopback = ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
+  if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) {
+    throw new Error("Raindrop endpoints require HTTPS, except on loopback hosts");
+  }
+  if (url.username || url.password || url.search || url.hash) {
+    throw new Error("Raindrop endpoints cannot contain credentials, a query, or a fragment");
+  }
+  return url;
+}
+function replayIngestEndpoint(value, trustedUrl) {
+  const url = credentialEndpoint(value);
+  if (!trustedUrl && url.href.replace(/\/+$/, "") === "https://app.raindrop.ai/api/replays/ingest") {
+    url.hostname = "backend.raindrop.ai";
+  }
+  const trusted = credentialEndpoint(trustedUrl != null ? trustedUrl : "https://backend.raindrop.ai/api/replays/ingest/");
+  if (url.href.replace(/\/+$/, "") !== trusted.href.replace(/\/+$/, "")) {
+    throw new Error("Raindrop replay ingest URL does not match the trusted replayIngestUrl");
+  }
+  return url.href;
+}
 
-When NOT to call:
-- Normal clarifying questions or back-and-forth with the user.
-- A single tool error that you can recover from or retry.
-- You successfully completed the task, even if it was difficult.
-- Policy refusals or content filtering - those are working as intended.
+// src/replay/git.ts
+import { execFileSync } from "child_process";
+function readGitCommit() {
+  const sha = git(["rev-parse", "HEAD"]);
+  if (sha === null) return null;
+  const branch = git(["rev-parse", "--abbrev-ref", "HEAD"]);
+  const status = git(["status", "--porcelain"]);
+  if (status === null) return null;
+  return {
+    sha,
+    branch: branch === null || branch === "HEAD" ? null : branch,
+    dirty: status !== ""
+  };
+}
+function git(args) {
+  try {
+    return execFileSync("git", [...args], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+      timeout: 2e3
+    }).trim();
+  } catch (e) {
+    return null;
+  }
+}
 
-Rules:
-1. Pick the single best category.
-2. Do not fabricate issues. Only report what is evident from the conversation.
-3. Err on the side of NOT calling this tool. When in doubt, help the user instead.
-${guidanceLine}
-Categories:
-` + categories;
+// src/replay/index.ts
+var DEFAULT_QUERY_URL = "https://query.raindrop.ai";
+var DEFAULT_TRACE_WAIT_MS = 3e4;
+var DEFAULT_EVAL_WAIT_MS = 10 * 6e4;
+var DEFAULT_EVAL_POLL_INTERVAL_MS = 2e3;
+var MAX_EVAL_STATUS_RETRIES = 2;
+var TRACE_POLL_INTERVAL_MS = 500;
+var DEFAULT_CONCURRENCY = 4;
+var EVAL_RETRY_BASE_DELAY_MS = 50;
+var MAX_REPLAY_REPORT_RETRIES = 2;
+var QUERY_REQUEST_TIMEOUT_MS = 3e4;
+var LOCAL_EVALUATOR_SLUG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+function retryDelayMs(response, retry) {
+  const retryAfter = response.headers.get("retry-after");
+  if (retryAfter !== null) {
+    const seconds = Number(retryAfter);
+    if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1e3;
+  }
+  return EVAL_RETRY_BASE_DELAY_MS * 2 ** retry;
 }
-function buildSelfDiagnosticsInputSchema(signalKeys) {
+var ReplayRowSchema = z3.object({
+  id: z3.string(),
+  name: z3.string(),
+  input: z3.string().nullable(),
+  output: z3.string().nullable(),
+  properties: z3.record(z3.string())
+});
+var WireEvalManifestSchema = z3.object({
+  dataset_id: z3.string().uuid(),
+  fingerprint: z3.string().regex(/^[a-f0-9]{64}$/),
+  rows: z3.array(ReplayRowSchema.extend({ id: z3.string().min(1) }))
+});
+var ReplayCountsSchema = z3.object({
+  total: z3.number(),
+  pending: z3.number(),
+  done: z3.number(),
+  missing: z3.number()
+});
+var ReplayEvaluatorCountsSchema = z3.object({
+  traces: z3.number(),
+  graded: z3.number(),
+  errored: z3.number(),
+  ungraded: z3.number(),
+  pending: z3.number()
+});
+var ReplayEvaluatorStatsSchema = z3.object({
+  traceCount: z3.number(),
+  hydratedCount: z3.number(),
+  judgeCalls: z3.number(),
+  durationMs: z3.number(),
+  skippedRows: z3.number(),
+  skippedReason: z3.literal("no trace").nullable()
+});
+var ReplayEvaluatorSummarySchema = z3.discriminatedUnion("outputType", [
+  z3.object({
+    outputType: z3.literal("boolean"),
+    counts: ReplayEvaluatorCountsSchema,
+    stats: ReplayEvaluatorStatsSchema,
+    passed: z3.number(),
+    failed: z3.number(),
+    passRate: z3.number().nullable()
+  }),
+  z3.object({
+    outputType: z3.literal("score"),
+    counts: ReplayEvaluatorCountsSchema,
+    stats: ReplayEvaluatorStatsSchema,
+    mean: z3.number().nullable(),
+    median: z3.number().nullable(),
+    p10: z3.number().nullable(),
+    p90: z3.number().nullable(),
+    atLeast4Share: z3.number().nullable(),
+    histogram: z3.tuple([z3.number(), z3.number(), z3.number(), z3.number(), z3.number()])
+  }),
+  z3.object({
+    outputType: z3.literal("number"),
+    counts: ReplayEvaluatorCountsSchema,
+    stats: ReplayEvaluatorStatsSchema,
+    median: z3.number().nullable(),
+    p10: z3.number().nullable(),
+    p90: z3.number().nullable()
+  })
+]);
+var ReplayVerdictIdentitySchema = z3.object({
+  evaluator: z3.string(),
+  runId: z3.string(),
+  url: z3.string()
+});
+var ReplayVerdictGradedSchema = ReplayVerdictIdentitySchema.extend({
+  state: z3.literal("graded"),
+  note: z3.string().optional()
+});
+var ReplayVerdictSchema = z3.union([
+  ReplayVerdictGradedSchema.extend({ pass: z3.boolean() }),
+  ReplayVerdictGradedSchema.extend({ score: EvalScoreSchema }),
+  ReplayVerdictGradedSchema.extend({ value: z3.number().finite() }),
+  ReplayVerdictIdentitySchema.extend({
+    state: z3.literal("errored"),
+    message: z3.string()
+  }),
+  ReplayVerdictIdentitySchema.extend({
+    state: z3.literal("ungraded"),
+    reason: z3.enum(["not sampled", "trace unavailable", "content unreadable"])
+  })
+]);
+var ReplayResultRowSchema = ReplayRowSchema.extend({
+  status: z3.enum(["pending", "done", "missing"]),
+  traceId: z3.string().nullable(),
+  error: z3.string().nullable(),
+  verdicts: z3.array(ReplayVerdictSchema)
+});
+var ReplayTraceEvidenceEntrySchema = z3.object({
+  rowId: z3.string().min(1).max(512),
+  traceId: z3.string().regex(/^[a-f0-9]{32}$/),
+  spanIds: z3.array(z3.string().regex(/^[a-f0-9]{16}$/)).min(1).max(500)
+}).superRefine((entry, context3) => {
+  if (new Set(entry.spanIds).size !== entry.spanIds.length) {
+    context3.addIssue({
+      code: z3.ZodIssueCode.custom,
+      message: "Replay trace evidence span ids must be unique",
+      path: ["spanIds"]
+    });
+  }
+});
+var ReplayTraceEvidenceSchema = z3.array(ReplayTraceEvidenceEntrySchema).superRefine((entries, context3) => {
+  const seen = /* @__PURE__ */ new Set();
+  for (const [index, entry] of entries.entries()) {
+    if (seen.has(entry.rowId)) {
+      context3.addIssue({
+        code: z3.ZodIssueCode.custom,
+        message: "Replay trace evidence row ids must be unique",
+        path: [index, "rowId"]
+      });
+    }
+    seen.add(entry.rowId);
+  }
+});
+var LocalBooleanVerdictSchema = z3.object({ pass: z3.boolean(), note: z3.string().trim().min(1).max(600).optional() }).strict();
+var LocalScoreVerdictSchema = z3.object({
+  score: EvalScoreSchema,
+  note: z3.string().trim().min(1).max(600).optional()
+}).strict();
+var LocalNumberVerdictSchema = z3.object({ value: z3.number().finite(), note: z3.string().trim().min(1).max(600).optional() }).strict();
+var LocalVerdictSchema = z3.union([
+  LocalBooleanVerdictSchema,
+  LocalScoreVerdictSchema,
+  LocalNumberVerdictSchema
+]);
+async function queryApi(request) {
+  var _a, _b;
+  if (!((_a = request.client.connection.apiKey) == null ? void 0 : _a.trim())) {
+    throw new Error("Raindrop evals and datasets require a Query SDK API key (apiKey)");
+  }
+  const headers = {
+    Authorization: `Bearer ${request.client.connection.apiKey}`,
+    "Content-Type": "application/json"
+  };
+  const projectId = request.client.connection.projectId;
+  if (projectId !== void 0) headers["X-Raindrop-Project-Id"] = projectId;
+  const url = `${credentialEndpoint(request.queryUrl).href.replace(/\/+$/, "")}${request.path}`;
+  const maxRetries = (_b = request.maxRetries) != null ? _b : 0;
+  const body = request.body === void 0 ? void 0 : JSON.stringify(request.body);
+  for (let retry = 0; ; retry += 1) {
+    let response;
+    let text;
+    const controller = new AbortController();
+    const remainingMs = () => request.deadlineAt === void 0 ? QUERY_REQUEST_TIMEOUT_MS : Math.max(0, request.deadlineAt - Date.now());
+    const timeout = setTimeout(
+      () => controller.abort(),
+      Math.max(1, Math.min(QUERY_REQUEST_TIMEOUT_MS, remainingMs()))
+    );
+    try {
+      response = await runWithTracingSuppressed(
+        () => fetch(url, {
+          method: request.method,
+          headers,
+          body,
+          signal: controller.signal,
+          redirect: "error"
+        })
+      );
+      text = await response.text();
+    } catch (cause) {
+      if (retry >= maxRetries || remainingMs() <= 0) throw cause;
+      const retryMs = EVAL_RETRY_BASE_DELAY_MS * 2 ** retry;
+      const budgetMs = remainingMs();
+      await sleep(Math.min(retryMs, budgetMs));
+      if (retryMs >= budgetMs || remainingMs() <= 0) throw cause;
+      continue;
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!response.ok) {
+      const error = new Error(
+        `Raindrop API ${request.method} ${request.path}: ${response.status} ${text}`
+      );
+      if (retry >= maxRetries || !isRetryableStatus(response.status) || remainingMs() <= 0)
+        throw error;
+      const retryMs = retryDelayMs(response, retry);
+      const budgetMs = remainingMs();
+      await sleep(Math.min(retryMs, budgetMs));
+      if (retryMs >= budgetMs || remainingMs() <= 0) throw error;
+      continue;
+    }
+    let json;
+    try {
+      json = JSON.parse(text);
+    } catch (e) {
+      throw new Error(`Raindrop API ${request.method} ${request.path} returned invalid JSON`);
+    }
+    const parsed = request.schema.safeParse(json);
+    if (!parsed.success) {
+      throw new Error(
+        `Raindrop API ${request.method} ${request.path} returned an invalid response: ${parsed.error.message}`
+      );
+    }
+    return parsed.data;
+  }
+}
+var ReplayStatusSchema = z3.enum(["queued", "running", "complete", "failed"]);
+var ReplayRowStatusSchema = z3.enum(["pending", "done", "missing"]);
+var WireReplaySchema = z3.object({
+  id: z3.string(),
+  status: ReplayStatusSchema,
+  counts: ReplayCountsSchema,
+  traces_expire_at: z3.string(),
+  traces_expired: z3.boolean()
+});
+var WireRowSchema = z3.object({
+  row_id: z3.string(),
+  attempt: z3.number().int().nonnegative(),
+  status: ReplayRowStatusSchema,
+  trace_id: z3.string().nullable(),
+  error: z3.string().nullable()
+});
+var WirePlanSchema = z3.object({
+  replay: WireReplaySchema,
+  ingest_url: z3.string().min(1).nullable(),
+  rows: z3.array(
+    ReplayRowSchema.omit({ id: true }).extend({
+      row_id: z3.string(),
+      attempt: z3.literal(0),
+      correlation_id: z3.string()
+    })
+  )
+});
+var WireReplayDetailSchema = z3.object({
+  replay: WireReplaySchema,
+  rows: z3.array(WireRowSchema)
+});
+var WireReportSchema = z3.object({ status: z3.string() });
+var WireAttemptStartedSchema = z3.object({
+  status: z3.literal("pending"),
+  row: z3.object({
+    row_id: z3.string().min(1),
+    attempt: z3.number().int().positive(),
+    correlation_id: z3.string().min(1)
+  })
+});
+var WireLocalEvaluatorSchema = z3.object({
+  data: z3.object({
+    id: z3.string().uuid(),
+    slug: z3.string(),
+    name: z3.string(),
+    output: z3.enum(["boolean", "score", "number"]),
+    kind: z3.literal("local"),
+    programVersion: z3.number().int().positive()
+  })
+});
+var WireStartedEvalRunSchema = z3.object({
+  data: z3.object({
+    runId: z3.string(),
+    status: z3.literal("running"),
+    url: z3.string(),
+    evalId: z3.string().uuid().optional(),
+    programVersion: z3.number().int().positive().optional()
+  })
+});
+var WireEvalRunSchema = z3.object({
+  data: z3.object({
+    id: z3.string(),
+    evalSlug: z3.string(),
+    status: z3.enum(["running", "completed", "failed"]),
+    summary: ReplayEvaluatorSummarySchema.nullable(),
+    error: z3.string().nullable(),
+    traces: z3.array(
+      z3.union([
+        z3.object({
+          state: z3.literal("graded"),
+          traceId: z3.string(),
+          pass: z3.boolean(),
+          note: z3.string().optional()
+        }),
+        z3.object({
+          state: z3.literal("graded"),
+          traceId: z3.string(),
+          score: EvalScoreSchema,
+          note: z3.string().optional()
+        }),
+        z3.object({
+          state: z3.literal("graded"),
+          traceId: z3.string(),
+          value: z3.number().finite(),
+          note: z3.string().optional()
+        }),
+        z3.object({ state: z3.literal("errored"), traceId: z3.string(), message: z3.string() }),
+        z3.object({
+          state: z3.literal("ungraded"),
+          traceId: z3.string(),
+          reason: z3.enum(["not sampled", "trace unavailable", "content unreadable"])
+        })
+      ])
+    )
+  })
+});
+var WireLocalEvalRunSchema = z3.object({
+  data: WireEvalRunSchema.shape.data.extend({ status: z3.literal("completed"), url: z3.string() })
+});
+var WireLocalEvalRunStartedSchema = z3.object({
+  data: WireEvalRunSchema.shape.data.omit({ traces: true }).extend({ status: z3.literal("running"), url: z3.string() })
+});
+var WireLocalEvalAttemptSchema = z3.object({
+  data: z3.object({
+    runId: z3.string().uuid(),
+    requestId: z3.string().uuid(),
+    rowId: z3.string().min(1),
+    attempt: z3.number().int().nonnegative(),
+    traceId: z3.string().regex(/^[a-f0-9]{32}$/),
+    outcome: WireEvalRunSchema.shape.data.shape.traces.element
+  })
+});
+var WireEvalCaseTracePairReadSchema = z3.discriminatedUnion("state", [
+  EvalCaseTracePairSchema.extend({
+    state: z3.literal("ready"),
+    attempt: z3.number().int().nonnegative(),
+    datasetVersion: z3.object({
+      id: z3.string().uuid(),
+      fingerprint: EvalDatasetFingerprintSchema
+    })
+  }),
+  z3.object({
+    state: z3.literal("pending"),
+    caseId: z3.string().min(1),
+    attempt: z3.number().int().nonnegative()
+  }),
+  z3.object({
+    state: z3.literal("missing"),
+    caseId: z3.string().min(1),
+    attempt: z3.number().int().nonnegative(),
+    reason: z3.string()
+  }),
+  z3.object({
+    state: z3.literal("expired"),
+    caseId: z3.string().min(1),
+    attempt: z3.number().int().nonnegative(),
+    expiredAt: z3.string().datetime({ offset: true })
+  }),
+  z3.object({
+    state: z3.literal("errored"),
+    caseId: z3.string().min(1),
+    attempt: z3.number().int().nonnegative(),
+    reason: z3.string()
+  })
+]);
+function toReading(wire) {
   return {
-    type: "object",
-    additionalProperties: false,
-    properties: {
-      category: {
-        type: "string",
-        enum: signalKeys,
-        description: "The single best-matching category from the list above."
-      },
-      detail: {
-        type: "string",
-        description: "One sentence of factual context: what happened and why it matters. Do not include PII or secrets."
-      }
-    },
-    required: ["category", "detail"]
+    replayId: wire.replay.id,
+    status: wire.replay.status,
+    counts: wire.replay.counts,
+    rows: wire.rows.map((row) => ({
+      rowId: row.row_id,
+      attempt: row.attempt,
+      status: row.status,
+      traceId: row.trace_id,
+      error: row.error
+    })),
+    tracesExpireAt: wire.replay.traces_expire_at,
+    tracesExpired: wire.replay.traces_expired
   };
 }
-function normalizeEventId(value) {
-  if (typeof value !== "string") return void 0;
-  const trimmed = value.trim();
-  return trimmed ? trimmed : void 0;
+async function createReplay(client, options) {
+  var _a, _b, _c, _d;
+  const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  const manifest = typeof options.dataset === "string" ? await readEvalDataset(client, options.dataset, { queryUrl }) : validateEvalDatasetManifest(options.dataset);
+  const rowIds = selectEvalDatasetRows(manifest, options.rowIds);
+  const commit = readGitCommit();
+  const created = await queryApi({
+    client,
+    queryUrl,
+    path: "/v1/replays",
+    method: "POST",
+    schema: WirePlanSchema,
+    maxRetries: MAX_REPLAY_REPORT_RETRIES,
+    body: {
+      dataset_slug: manifest.dataset.slug,
+      dataset_version_id: manifest.version.id,
+      dataset_fingerprint: manifest.version.fingerprint,
+      request_key: randomUUID(),
+      commit_sha: (_b = commit == null ? void 0 : commit.sha) != null ? _b : null,
+      commit_branch: (_c = commit == null ? void 0 : commit.branch) != null ? _c : null,
+      commit_dirty: (_d = commit == null ? void 0 : commit.dirty) != null ? _d : null,
+      row_ids: rowIds
+    }
+  });
+  const handle = {
+    ...toHandle(created),
+    datasetVersion: {
+      id: manifest.version.id,
+      fingerprint: manifest.version.fingerprint
+    }
+  };
+  assertSelectedRows(handle, rowIds);
+  return handle;
 }
-function resolveEventIdFromMetadata(metadata) {
-  var _a, _b, _c;
-  if (!metadata) return void 0;
-  return (_c = (_b = (_a = normalizeEventId(metadata["raindrop.eventId"])) != null ? _a : normalizeEventId(metadata["event_id"])) != null ? _b : normalizeEventId(metadata["eventId"])) != null ? _c : normalizeEventId(metadata["traceloop.association.properties.event_id"]);
+function selectEvalDatasetRows(manifest, requestedRowIds) {
+  const rowIds = requestedRowIds != null ? requestedRowIds : manifest.cases.map((entry) => entry.id);
+  if (rowIds.length === 0) throw new Error("Raindrop replay rowIds cannot be empty");
+  if (rowIds.some((rowId) => rowId.length === 0)) {
+    throw new Error("Raindrop replay rowIds cannot contain an empty id");
+  }
+  if (new Set(rowIds).size !== rowIds.length) {
+    throw new Error("Raindrop replay rowIds must be unique");
+  }
+  const selected = new Set(rowIds);
+  const ordered = manifest.cases.flatMap((entry) => selected.has(entry.id) ? [entry.id] : []);
+  if (ordered.length !== selected.size) {
+    const available = new Set(manifest.cases.map((entry) => entry.id));
+    const unknown = rowIds.find((rowId) => !available.has(rowId));
+    throw new Error(`Raindrop replay rowIds contains unknown case ${unknown != null ? unknown : "unknown"}`);
+  }
+  return ordered;
 }
-async function wait(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+function assertSelectedRows(handle, requestedRowIds) {
+  const returnedRowIds = handle.rows.map((row) => row.id);
+  const requested = new Set(requestedRowIds);
+  const returned = new Set(returnedRowIds);
+  const matches = returned.size === returnedRowIds.length && requested.size === returned.size && requestedRowIds.every((rowId, index) => returnedRowIds[index] === rowId);
+  if (!matches) {
+    throw new Error(
+      `Raindrop replay ${handle.replayId} returned rows that do not match the manifest selection`
+    );
+  }
 }
-function parseRetryAfter(headers) {
+function toHandle(wire) {
+  return {
+    replayId: wire.replay.id,
+    ingestUrl: wire.ingest_url,
+    rows: wire.rows.map((row) => ({
+      id: row.row_id,
+      attempt: row.attempt,
+      correlationId: row.correlation_id,
+      name: row.name,
+      input: row.input,
+      output: row.output,
+      properties: row.properties
+    }))
+  };
+}
+async function claimReplay(client, replayId, options) {
   var _a;
-  const value = (_a = headers.get("Retry-After")) != null ? _a : headers.get("retry-after");
-  if (!value) return void 0;
-  const asNumber = Number(value);
-  if (!Number.isNaN(asNumber)) return asNumber * 1e3;
-  const asDate = new Date(value).getTime();
-  if (!Number.isNaN(asDate)) {
-    const delta = asDate - Date.now();
-    return delta > 0 ? delta : 0;
-  }
-  return void 0;
+  return toHandle(
+    await queryApi({
+      client,
+      queryUrl: (_a = options == null ? void 0 : options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+      path: `/v1/replays/${encodeURIComponent(replayId)}/claim`,
+      method: "POST",
+      schema: WirePlanSchema
+    })
+  );
 }
-function getRetryDelayMs(attemptNumber, previousError) {
-  if (previousError && typeof previousError.retryAfterMs === "number") {
-    return Math.min(Math.max(0, previousError.retryAfterMs), MAX_RETRY_DELAY_MS);
+async function readReplay(client, replayId, options) {
+  var _a;
+  return toReading(
+    await queryApi({
+      client,
+      queryUrl: (_a = options == null ? void 0 : options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+      path: `/v1/replays/${encodeURIComponent(replayId)}`,
+      deadlineAt: options == null ? void 0 : options.deadlineAt,
+      method: "GET",
+      schema: WireReplayDetailSchema
+    })
+  );
+}
+async function readEvalManifest(client, dataset, options) {
+  var _a;
+  const normalized = dataset.trim();
+  if (normalized.length === 0) throw new Error("Raindrop eval manifest dataset cannot be empty");
+  const manifest = await queryApi({
+    client,
+    queryUrl: (_a = options == null ? void 0 : options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+    path: `/v1/replays/manifest?dataset=${encodeURIComponent(normalized)}`,
+    method: "GET",
+    schema: WireEvalManifestSchema
+  });
+  const caseIds = manifest.rows.map((row) => row.id);
+  if (new Set(caseIds).size !== caseIds.length) {
+    throw new Error(`Raindrop eval manifest for ${normalized} contains duplicate case ids`);
   }
-  if (attemptNumber <= 1) return 0;
-  const base = 500;
-  const factor = Math.pow(2, attemptNumber - 2);
-  return Math.min(base * factor, MAX_RETRY_DELAY_MS);
+  return {
+    datasetId: manifest.dataset_id,
+    fingerprint: manifest.fingerprint,
+    cases: manifest.rows
+  };
 }
-async function withRetry(operation, opName, maxAttempts, debugLogs = false) {
+async function readEvalDataset(client, dataset, options) {
   var _a;
-  let lastError = void 0;
-  for (let attemptNumber = 1; attemptNumber <= maxAttempts; attemptNumber++) {
-    if (attemptNumber > 1) {
-      const delay = getRetryDelayMs(attemptNumber, lastError);
-      if (debugLogs) {
-        console.warn(
-          `[raindrop] ${opName} retrying attempt ${attemptNumber}/${maxAttempts} in ${delay}ms...`
-        );
-      }
-      if (delay > 0) await wait(delay);
-    } else if (debugLogs) {
-      console.log(`[raindrop] ${opName} attempt ${attemptNumber}/${maxAttempts}`);
-    }
-    try {
-      return await operation();
-    } catch (err) {
-      lastError = err;
-      const isLast = attemptNumber === maxAttempts;
-      if (debugLogs) {
-        console.warn(
-          `[raindrop] ${opName} attempt ${attemptNumber} failed: ${(_a = err == null ? void 0 : err.message) != null ? _a : err}. ${isLast ? "No more retries." : "Will retry."}`
-        );
-      }
-      if (isLast) break;
-    }
+  const slug = EvalDatasetSlugSchema.parse(dataset.trim());
+  const versionId = options == null ? void 0 : options.versionId;
+  if (versionId !== void 0 && !z3.string().uuid().safeParse(versionId).success) {
+    throw new Error("Raindrop eval dataset versionId must be a UUID");
+  }
+  const path = versionId ? `/v1/eval-datasets/${encodeURIComponent(slug)}/versions/${encodeURIComponent(versionId)}` : `/v1/eval-datasets/${encodeURIComponent(slug)}`;
+  const manifest = validateEvalDatasetManifest(
+    await queryApi({
+      client,
+      queryUrl: (_a = options == null ? void 0 : options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+      path,
+      method: "GET",
+      schema: EvalDatasetManifestSchema
+    })
+  );
+  if (manifest.dataset.slug !== slug) {
+    throw new Error(`Raindrop eval dataset ${slug} returned mismatched dataset identity`);
   }
-  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+  if (versionId !== void 0 && manifest.version.id !== versionId) {
+    throw new Error(`Raindrop eval dataset ${slug} returned mismatched version identity`);
+  }
+  return manifest;
 }
-function resolveDisableBatching(explicit, signals) {
-  return explicit != null ? explicit : !signals.isProduction || signals.isLocalDebugger;
+function validateEvalDatasetManifest(value) {
+  const manifest = EvalDatasetManifestSchema.parse(value);
+  if (manifest.dataset.id !== manifest.version.datasetId) {
+    throw new Error("Raindrop eval dataset and version identities do not match");
+  }
+  const caseIds = manifest.cases.map((entry) => entry.id);
+  if (new Set(caseIds).size !== caseIds.length) {
+    throw new Error(`Raindrop eval dataset ${manifest.dataset.slug} contains duplicate case ids`);
+  }
+  return manifest;
 }
-var Raindrop = class {
-  constructor(config) {
-    this.flushTimer = null;
-    // New members for partial event handling
-    this.partialEventBuffer = /* @__PURE__ */ new Map();
+async function runReplay(client, handle, options) {
+  return (await runReplayRows(client, handle, options)).reading;
+}
+async function runReplayRows(client, handle, options) {
+  var _a, _b, _c, _d, _e, _f, _g;
+  if (!Number.isSafeInteger((_a = options.concurrency) != null ? _a : DEFAULT_CONCURRENCY) || ((_b = options.concurrency) != null ? _b : DEFAULT_CONCURRENCY) < 1) {
+    throw new Error("Raindrop replay concurrency must be a positive safe integer");
+  }
+  validateWait("traceWaitMs", (_c = options.traceWaitMs) != null ? _c : DEFAULT_TRACE_WAIT_MS, true);
+  const ingestUrl = handle.ingestUrl && replayIngestEndpoint(handle.ingestUrl, options.replayIngestUrl);
+  if (!ingestUrl) {
+    throw new Error(
+      `Raindrop replay ${handle.replayId} has no ingest URL; create or claim it with this runner before running it`
+    );
+  }
+  const queryUrl = (_d = options.queryUrl) != null ? _d : DEFAULT_QUERY_URL;
+  const report = (body) => queryApi({
+    client,
+    queryUrl,
+    path: `/v1/replays/${encodeURIComponent(handle.replayId)}/report`,
+    method: "POST",
+    schema: WireReportSchema,
+    body,
+    maxRetries: MAX_REPLAY_REPORT_RETRIES
+  });
+  await report({ type: "started" });
+  const rows = [...handle.rows];
+  const results = /* @__PURE__ */ new Map();
+  const captures = /* @__PURE__ */ new Map();
+  let next = 0;
+  const worker = async () => {
+    var _a2, _b2;
+    while (next < rows.length) {
+      const row = rows[next++];
+      if (!row) return;
+      const spanIds = /* @__PURE__ */ new Set();
+      const traceIds = /* @__PURE__ */ new Set();
+      captures.set(row.id, { spanIds, traceIds });
+      try {
+        const { correlationId: _attribution, attempt: _attempt, ...visible } = row;
+        const replayDestination = {
+          kind: "raindrop",
+          url: ingestUrl,
+          replayId: handle.replayId,
+          apiKey: (_a2 = client.connection.apiKey) != null ? _a2 : "",
+          projectId: client.connection.projectId,
+          exportedSpanIds: spanIds,
+          exportedTraceIds: traceIds
+        };
+        const result = await context.with(
+          trace.deleteSpan(context.active()),
+          () => withReplayTraceDestination(
+            replayDestination,
+            () => withEvalScope(
+              { correlationId: row.correlationId },
+              () => withTask(
+                { name: "raindrop.replay.row", inputParameters: [visible], traceContent: true },
+                async () => {
+                  var _a3;
+                  const parent = (_a3 = trace.getSpan(context.active())) == null ? void 0 : _a3.spanContext();
+                  if (!parent) return options.run(visible);
+                  return withReplayTraceDestination({
+                    ...replayDestination,
+                    parentSpanContext: {
+                      traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
+                      spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
+                      eventId: parent.traceId
+                    }
+                  }, () => options.run(visible));
+                }
+              )
+            )
+          )
+        );
+        results.set(row.id, { value: result });
+      } catch (cause) {
+        const error = describeError(cause);
+        await report({
+          type: "row_missing",
+          row_id: row.id,
+          attempt: row.attempt,
+          error
+        });
+        await client.forceFlush();
+        try {
+          await waitForFailedTraceAssociation({
+            client,
+            queryUrl,
+            replayId: handle.replayId,
+            rowId: row.id,
+            attempt: row.attempt,
+            traceIds,
+            spanIds,
+            waitMs: (_b2 = options.traceWaitMs) != null ? _b2 : DEFAULT_TRACE_WAIT_MS
+          });
+        } catch (retentionCause) {
+          throw failedTraceRetentionError(cause, retentionCause, row.id);
+        }
+        captures.delete(row.id);
+      }
+    }
+  };
+  await Promise.all(
+    Array.from(
+      { length: Math.min((_e = options.concurrency) != null ? _e : DEFAULT_CONCURRENCY, rows.length) },
+      () => worker()
+    )
+  );
+  await client.forceFlush();
+  const traceEvidence = [];
+  for (const row of rows) {
+    const capture = captures.get(row.id);
+    if (!capture) continue;
+    const traceIds = [...capture.traceIds];
+    const spanIds = [...capture.spanIds];
+    if (traceIds.length !== 1 || spanIds.length === 0 || spanIds.length > 500) {
+      const detail = traceIds.length !== 1 ? `expected one trace, captured ${traceIds.length}` : spanIds.length === 0 ? "captured no ended spans" : `captured ${spanIds.length} spans, exceeding the 500 span limit`;
+      await report({ type: "row_missing", row_id: row.id, attempt: row.attempt, error: detail });
+      continue;
+    }
+    const traceId = traceIds[0];
+    if (traceId === void 0) {
+      throw new Error(`Raindrop replay ${handle.replayId} lost trace evidence for ${row.id}`);
+    }
+    traceEvidence.push({ rowId: row.id, traceId, spanIds });
+  }
+  const settled = await waitForTraces({
+    client,
+    queryUrl,
+    replayId: handle.replayId,
+    deadlineAt: Date.now() + ((_f = options.traceWaitMs) != null ? _f : DEFAULT_TRACE_WAIT_MS)
+  });
+  for (const row of settled.rows) {
+    if (row.status !== "pending") continue;
+    await report({
+      type: "row_missing",
+      row_id: row.rowId,
+      attempt: row.attempt,
+      error: `no trace arrived for this row within ${Math.round(
+        ((_g = options.traceWaitMs) != null ? _g : DEFAULT_TRACE_WAIT_MS) / 1e3
+      )}s`
+    });
+  }
+  await report({ type: "finished" });
+  return {
+    reading: await readReplay(client, handle.replayId, { queryUrl }),
+    results,
+    traceEvidence: ReplayTraceEvidenceSchema.parse(traceEvidence)
+  };
+}
+async function readEvalCaseTracePairs(input) {
+  const deadlineAt = Date.now() + input.waitMs;
+  const entries = await Promise.all(
+    input.evidence.map(async (evidence) => {
+      var _a, _b, _c;
+      let reading = await readEvalCaseTracePair(
+        { ...input, deadlineAt },
+        evidence,
+        (_a = input.attempt) != null ? _a : 0
+      );
+      while (reading.state === "pending" && Date.now() < deadlineAt) {
+        await sleep(Math.min(TRACE_POLL_INTERVAL_MS, Math.max(0, deadlineAt - Date.now())));
+        if (Date.now() >= deadlineAt) break;
+        reading = await readEvalCaseTracePair({ ...input, deadlineAt }, evidence, (_b = input.attempt) != null ? _b : 0);
+      }
+      if (reading.state !== "ready") {
+        const detail = reading.state === "pending" ? `did not become complete within ${input.waitMs}ms` : reading.state === "expired" ? `expired at ${reading.expiredAt}` : reading.reason;
+        throw new Error(
+          `Raindrop replay ${input.replayId} trace pair for ${evidence.rowId} ${detail}`
+        );
+      }
+      if (reading.caseId !== evidence.rowId) {
+        throw new Error(
+          `Raindrop replay ${input.replayId} returned a trace pair for the wrong case`
+        );
+      }
+      if (reading.attempt !== ((_c = input.attempt) != null ? _c : 0)) {
+        throw new Error(
+          `Raindrop replay ${input.replayId} returned a trace pair for the wrong attempt`
+        );
+      }
+      if (input.datasetVersion && (reading.datasetVersion.id !== input.datasetVersion.id || reading.datasetVersion.fingerprint !== input.datasetVersion.fingerprint)) {
+        throw new Error(
+          `Raindrop replay ${input.replayId} returned a trace pair from the wrong dataset version`
+        );
+      }
+      return [evidence.rowId, reading];
+    })
+  );
+  return new Map(entries);
+}
+async function readEvalCaseTracePair(input, evidence, attempt) {
+  const query = new URLSearchParams({
+    row_id: evidence.rowId,
+    attempt: String(attempt),
+    expected_trace_id: evidence.traceId
+  });
+  for (const spanId of evidence.spanIds) query.append("expected_span_id", spanId);
+  return queryApi({
+    client: input.client,
+    queryUrl: input.queryUrl,
+    path: `/v1/replays/${encodeURIComponent(input.replayId)}/trace?${query.toString()}`,
+    method: "GET",
+    schema: WireEvalCaseTracePairReadSchema,
+    deadlineAt: input.deadlineAt,
+    maxRetries: MAX_EVAL_STATUS_RETRIES
+  });
+}
+async function createLocalEvalReplaySession(client, options) {
+  var _a;
+  if (options.evaluators.length === 0) {
+    throw new Error("Raindrop local eval replay requires at least one evaluator");
+  }
+  const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  const handle = await createReplay(client, {
+    dataset: options.dataset,
+    rowIds: options.rowIds,
+    queryUrl
+  });
+  const validatedHandle = {
+    ...handle,
+    ingestUrl: replayIngestEndpoint(requireReplayIngestUrl(handle), options.replayIngestUrl)
+  };
+  await reportReplay(client, queryUrl, handle.replayId, { type: "started" });
+  const parents = await Promise.all(
+    options.evaluators.map(async (evaluator) => {
+      const registered = await registerLocalEvaluator(client, evaluator, queryUrl);
+      const runId = randomUUID();
+      const started = await queryApi({
+        client,
+        queryUrl,
+        path: `/v1/evals/${encodeURIComponent(evaluator.slug)}/runs/local`,
+        method: "POST",
+        schema: WireLocalEvalRunStartedSchema,
+        maxRetries: MAX_REPLAY_REPORT_RETRIES,
+        body: {
+          type: "started",
+          runId,
+          expectedEvalId: registered.data.id,
+          expectedProgramVersion: registered.data.programVersion,
+          source: { replay: handle.replayId }
+        }
+      });
+      if (started.data.id !== runId) {
+        throw new Error(`Raindrop local evaluator ${evaluator.slug} returned the wrong run id`);
+      }
+      return {
+        evaluator,
+        runId,
+        url: started.data.url
+      };
+    })
+  );
+  return new LocalEvalReplaySessionImpl(client, validatedHandle, parents, queryUrl);
+}
+var LocalEvalReplaySessionImpl = class {
+  constructor(client, handle, parents, queryUrl) {
+    this.client = client;
+    this.handle = handle;
+    this.parents = parents;
+    this.queryUrl = queryUrl;
+    this.state = "open";
+    this.replayId = handle.replayId;
+    this.rows = handle.rows.map(
+      ({ correlationId: _correlationId, attempt: _attempt, ...row }) => row
+    );
+  }
+  async run(input) {
+    var _a, _b, _c, _d;
+    validateWait("traceWaitMs", (_a = input.traceWaitMs) != null ? _a : DEFAULT_TRACE_WAIT_MS, true);
+    if (this.state !== "open") {
+      throw new Error(`Raindrop replay ${this.replayId} is already finishing`);
+    }
+    if (!Number.isInteger(input.attempt) || input.attempt < 0) {
+      throw new Error("Raindrop replay attempt must be a non-negative integer");
+    }
+    const planned = this.handle.rows.find((row2) => row2.id === input.rowId);
+    if (!planned) throw new Error(`Raindrop replay ${this.replayId} has no row ${input.rowId}`);
+    const correlationId = input.attempt === 0 ? planned.correlationId : await this.allocateRetry(input.rowId, input.attempt);
+    const { correlationId: _correlationId, attempt: _attempt, ...row } = planned;
+    const spanIds = /* @__PURE__ */ new Set();
+    const traceIds = /* @__PURE__ */ new Set();
+    let result;
+    const startedAt = Date.now();
+    const replayDestination = {
+      kind: "raindrop",
+      url: requireReplayIngestUrl(this.handle),
+      replayId: this.replayId,
+      apiKey: (_b = this.client.connection.apiKey) != null ? _b : "",
+      projectId: this.client.connection.projectId,
+      exportedSpanIds: spanIds,
+      exportedTraceIds: traceIds
+    };
+    try {
+      result = await context.with(
+        trace.deleteSpan(context.active()),
+        () => withReplayTraceDestination(
+          replayDestination,
+          () => withEvalScope(
+            { correlationId },
+            () => withTask(
+              { name: "raindrop.replay.row", inputParameters: [row], traceContent: true },
+              () => {
+                var _a2;
+                const parent = (_a2 = trace.getSpan(context.active())) == null ? void 0 : _a2.spanContext();
+                if (!parent) return input.run(row);
+                return withReplayTraceDestination({
+                  ...replayDestination,
+                  parentSpanContext: {
+                    traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
+                    spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
+                    eventId: parent.traceId
+                  }
+                }, () => input.run(row));
+              }
+            )
+          )
+        )
+      );
+    } catch (cause) {
+      const error = describeError(cause);
+      await reportReplay(this.client, this.queryUrl, this.replayId, {
+        type: "row_missing",
+        row_id: row.id,
+        attempt: input.attempt,
+        error
+      });
+      await this.client.forceFlush();
+      let retainedTraceId;
+      try {
+        retainedTraceId = await waitForFailedTraceAssociation({
+          client: this.client,
+          queryUrl: this.queryUrl,
+          replayId: this.replayId,
+          rowId: row.id,
+          attempt: input.attempt,
+          traceIds,
+          spanIds,
+          waitMs: (_c = input.traceWaitMs) != null ? _c : DEFAULT_TRACE_WAIT_MS
+        });
+      } catch (retentionCause) {
+        throw failedTraceRetentionError(cause, retentionCause, row.id);
+      }
+      return {
+        row,
+        attempt: input.attempt,
+        status: "missing",
+        traceId: retainedTraceId,
+        error,
+        verdicts: []
+      };
+    }
+    await this.client.forceFlush();
+    let pair;
+    let evidence;
+    try {
+      evidence = capturedEvidence(row.id, traceIds, spanIds);
+      const pairs = await readEvalCaseTracePairs({
+        client: this.client,
+        queryUrl: this.queryUrl,
+        replayId: this.replayId,
+        datasetVersion: this.handle.datasetVersion,
+        evidence: [evidence],
+        waitMs: (_d = input.traceWaitMs) != null ? _d : DEFAULT_TRACE_WAIT_MS,
+        attempt: input.attempt
+      });
+      const captured = pairs.get(row.id);
+      if (!captured)
+        throw new Error(`Raindrop replay ${this.replayId} omitted trace pair ${row.id}`);
+      pair = captured;
+    } catch (cause) {
+      const error = describeError(cause);
+      await reportReplay(this.client, this.queryUrl, this.replayId, {
+        type: "row_missing",
+        row_id: row.id,
+        attempt: input.attempt,
+        error
+      });
+      return { row, attempt: input.attempt, status: "missing", traceId: null, error, verdicts: [] };
+    }
+    const verdicts = await Promise.all(
+      this.parents.map(async (parent) => {
+        const judgeStartedAt = Date.now();
+        let verdict;
+        try {
+          verdict = parseLocalVerdict(
+            parent.evaluator,
+            await parent.evaluator.judge({
+              trace: pair.candidate.trace,
+              case: pair,
+              row,
+              result
+            })
+          );
+        } catch (cause) {
+          verdict = { error: localEvaluatorError(cause) };
+        }
+        const requestId = randomUUID();
+        let receipt;
+        try {
+          receipt = await queryApi({
+            client: this.client,
+            queryUrl: this.queryUrl,
+            path: `/v1/evals/${encodeURIComponent(parent.evaluator.slug)}/runs/local`,
+            method: "POST",
+            schema: WireLocalEvalAttemptSchema,
+            maxRetries: MAX_REPLAY_REPORT_RETRIES,
+            body: {
+              type: "attempt",
+              runId: parent.runId,
+              requestId,
+              source: {
+                replay: this.replayId,
+                rowId: row.id,
+                attempt: input.attempt,
+                traceEvidence: [evidence]
+              },
+              verdict,
+              durationMs: Date.now() - judgeStartedAt
+            }
+          });
+        } catch (cause) {
+          throw new Error(
+            `Raindrop local evaluator ${parent.evaluator.slug} could not record attempt at ${parent.url}: ${describeError(cause)}`
+          );
+        }
+        const data = receipt.data;
+        if (data.runId !== parent.runId || data.requestId !== requestId || data.rowId !== row.id || data.attempt !== input.attempt || data.traceId !== evidence.traceId) {
+          throw new Error(
+            `Raindrop local evaluator ${parent.evaluator.slug} returned a mismatched attempt receipt`
+          );
+        }
+        return replayVerdict(
+          {
+            evaluator: parent.evaluator.slug,
+            runId: parent.runId,
+            status: "completed",
+            url: parent.url,
+            summary: null,
+            error: null
+          },
+          data.outcome
+        );
+      })
+    );
+    return {
+      row,
+      attempt: input.attempt,
+      status: "done",
+      traceId: evidence.traceId,
+      error: null,
+      result,
+      verdicts
+    };
+  }
+  finish() {
+    if (this.finishPromise) return this.finishPromise;
+    this.state = "finishing";
+    this.finishPromise = this.finishOnce();
+    return this.finishPromise;
+  }
+  async finishOnce() {
+    await reportReplay(this.client, this.queryUrl, this.replayId, { type: "finished" });
+    const completed = await Promise.all(
+      this.parents.map(async (parent) => {
+        const response = await queryApi({
+          client: this.client,
+          queryUrl: this.queryUrl,
+          path: `/v1/evals/${encodeURIComponent(parent.evaluator.slug)}/runs/local`,
+          method: "POST",
+          schema: WireLocalEvalRunSchema,
+          maxRetries: MAX_REPLAY_REPORT_RETRIES,
+          body: { type: "finished", runId: parent.runId, source: { replay: this.replayId } }
+        });
+        if (response.data.id !== parent.runId) {
+          throw new Error(
+            `Raindrop local evaluator ${parent.evaluator.slug} returned the wrong run id`
+          );
+        }
+        return {
+          evaluator: parent.evaluator.slug,
+          runId: response.data.id,
+          status: response.data.status,
+          url: response.data.url,
+          summary: response.data.summary,
+          error: response.data.error
+        };
+      })
+    );
+    const reading = await readReplay(this.client, this.replayId, { queryUrl: this.queryUrl });
+    this.state = "finished";
+    return { reading, evaluators: completed };
+  }
+  async allocateRetry(rowId, attempt) {
+    const response = await queryApi({
+      client: this.client,
+      queryUrl: this.queryUrl,
+      path: `/v1/replays/${encodeURIComponent(this.replayId)}/report`,
+      method: "POST",
+      schema: WireAttemptStartedSchema,
+      body: { type: "attempt_started", row_id: rowId, attempt, request_key: randomUUID() },
+      maxRetries: MAX_REPLAY_REPORT_RETRIES
+    });
+    if (response.row.row_id !== rowId || response.row.attempt !== attempt) {
+      throw new Error(`Raindrop replay ${this.replayId} returned the wrong retry attempt`);
+    }
+    return response.row.correlation_id;
+  }
+};
+function requireReplayIngestUrl(handle) {
+  if (!handle.ingestUrl) {
+    throw new Error(`Raindrop replay ${handle.replayId} has no ingest URL`);
+  }
+  return handle.ingestUrl;
+}
+function capturedEvidence(rowId, traceIds, spanIds) {
+  const traces = [...traceIds];
+  const spans = [...spanIds];
+  if (traces.length !== 1 || spans.length === 0 || spans.length > 500) {
+    const detail = traces.length !== 1 ? `expected one trace, captured ${traces.length}` : spans.length === 0 ? "captured no ended spans" : `captured ${spans.length} spans, exceeding the 500 span limit`;
+    throw new Error(`Raindrop replay row ${rowId} ${detail}`);
+  }
+  const traceId = traces[0];
+  if (traceId === void 0) throw new Error(`Raindrop replay row ${rowId} lost its trace id`);
+  return ReplayTraceEvidenceEntrySchema.parse({ rowId, traceId, spanIds: spans });
+}
+function reportReplay(client, queryUrl, replayId, body) {
+  return queryApi({
+    client,
+    queryUrl,
+    path: `/v1/replays/${encodeURIComponent(replayId)}/report`,
+    method: "POST",
+    schema: WireReportSchema,
+    body,
+    maxRetries: MAX_REPLAY_REPORT_RETRIES
+  });
+}
+async function replay(client, options) {
+  var _a, _b;
+  const evaluators = normalizeEvaluators(options.evaluators);
+  const local = evaluators.flatMap(
+    (evaluator) => typeof evaluator === "string" ? [] : [{ evaluator, verdicts: [], durationMs: 0 }]
+  );
+  await Promise.all(
+    local.map(
+      (state) => {
+        var _a2;
+        return registerLocalEvaluator(client, state.evaluator, (_a2 = options.queryUrl) != null ? _a2 : DEFAULT_QUERY_URL);
+      }
+    )
+  );
+  const handle = await createReplay(client, options);
+  const execution = await runReplayRows(client, handle, options);
+  const { reading } = execution;
+  const doneRowIds = new Set(
+    reading.rows.flatMap((row) => row.status === "done" ? [row.rowId] : [])
+  );
+  const traceEvidence = execution.traceEvidence.filter((entry) => doneRowIds.has(entry.rowId));
+  if (traceEvidence.length !== doneRowIds.size) {
+    throw new Error(`Raindrop replay ${reading.replayId} completed without full trace evidence`);
+  }
+  if (local.length > 0) {
+    const pairs = await readEvalCaseTracePairs({
+      client,
+      queryUrl: (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL,
+      replayId: reading.replayId,
+      datasetVersion: handle.datasetVersion,
+      evidence: traceEvidence,
+      waitMs: (_b = options.traceWaitMs) != null ? _b : DEFAULT_TRACE_WAIT_MS
+    });
+    const rowsById = new Map(
+      handle.rows.map(({ correlationId: _correlationId, ...row }) => [row.id, row])
+    );
+    for (const [rowId, pair] of pairs) {
+      const row = rowsById.get(rowId);
+      const result = execution.results.get(rowId);
+      if (!row || !result) {
+        throw new Error(`Raindrop replay ${reading.replayId} omitted result for ${rowId}`);
+      }
+      await runLocalEvaluators(local, pair, row, result.value);
+    }
+  }
+  const hosted = evaluators.flatMap(
+    (evaluator) => typeof evaluator === "string" ? [evaluator] : []
+  );
+  const [hostedResults, localResults] = await Promise.all([
+    hosted.length === 0 ? Promise.resolve([]) : runReplayEvaluators(client, reading.replayId, {
+      evaluators: hosted,
+      evaluatorPins: options.evaluatorPins,
+      queryUrl: options.queryUrl,
+      waitMs: options.evalWaitMs,
+      pollIntervalMs: options.evalPollIntervalMs,
+      traceEvidence
+    }),
+    Promise.all(
+      local.map(
+        (state) => {
+          var _a2;
+          return submitLocalEvaluator(
+            client,
+            reading.replayId,
+            state,
+            (_a2 = options.queryUrl) != null ? _a2 : DEFAULT_QUERY_URL,
+            traceEvidence
+          );
+        }
+      )
+    )
+  ]);
+  const resultsByEvaluator = new Map(
+    [...hostedResults, ...localResults].map((result) => [result.evaluator, result])
+  );
+  const evaluatorResults = evaluators.map((evaluator) => {
+    const name = typeof evaluator === "string" ? evaluator : evaluator.slug;
+    const result = resultsByEvaluator.get(name);
+    if (!result) throw new Error(`Raindrop replay omitted evaluator ${name}`);
+    return result;
+  });
+  return {
+    ...reading,
+    rows: replayResultRows(handle, reading, evaluatorResults),
+    evaluators: evaluatorResults.map(({ outcomes: _outcomes, ...evaluator }) => evaluator)
+  };
+}
+async function runLocalEvaluators(evaluators, evalCase, row, result) {
+  await Promise.all(
+    evaluators.map(async (state) => {
+      const startedAt = Date.now();
+      try {
+        const verdict = parseLocalVerdict(
+          state.evaluator,
+          await state.evaluator.judge({
+            trace: evalCase.candidate.trace,
+            row,
+            result,
+            case: evalCase
+          })
+        );
+        state.verdicts.push({ rowId: row.id, ...verdict });
+      } catch (cause) {
+        state.verdicts.push({ rowId: row.id, error: localEvaluatorError(cause) });
+      } finally {
+        state.durationMs += Date.now() - startedAt;
+      }
+    })
+  );
+}
+async function registerLocalEvaluator(client, evaluator, queryUrl) {
+  return queryApi({
+    client,
+    queryUrl,
+    path: `/v1/evals/${encodeURIComponent(evaluator.slug)}/local`,
+    method: "PUT",
+    schema: WireLocalEvaluatorSchema,
+    body: { name: evaluator.name, output: evaluator.output }
+  });
+}
+function parseLocalVerdict(evaluator, verdict) {
+  switch (evaluator.output) {
+    case "boolean":
+      return LocalBooleanVerdictSchema.parse(verdict);
+    case "score":
+      return LocalScoreVerdictSchema.parse(verdict);
+    case "number":
+      return LocalNumberVerdictSchema.parse(verdict);
+  }
+}
+async function submitLocalEvaluator(client, replayId, state, queryUrl, traceEvidence) {
+  const response = await queryApi({
+    client,
+    queryUrl,
+    path: `/v1/evals/${encodeURIComponent(state.evaluator.slug)}/runs/local`,
+    method: "POST",
+    schema: WireLocalEvalRunSchema,
+    body: {
+      source: { replay: replayId, traceEvidence },
+      verdicts: state.verdicts,
+      durationMs: state.durationMs
+    }
+  });
+  return {
+    evaluator: state.evaluator.slug,
+    runId: response.data.id,
+    status: response.data.status,
+    url: response.data.url,
+    summary: response.data.summary,
+    error: response.data.error,
+    outcomes: response.data.traces
+  };
+}
+async function evaluateReplay(client, replayId, options) {
+  const results = await runReplayEvaluators(client, replayId, options);
+  return results.map(({ outcomes: _outcomes, ...evaluator }) => evaluator);
+}
+async function runReplayEvaluators(client, replayId, options) {
+  var _a, _b, _c;
+  const evaluators = validateEvaluatorNames(options.evaluators);
+  const evaluatorPins = validateEvaluatorPins(evaluators, options.evaluatorPins);
+  const queryUrl = (_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL;
+  const waitMs = (_b = options.waitMs) != null ? _b : DEFAULT_EVAL_WAIT_MS;
+  const pollIntervalMs = (_c = options.pollIntervalMs) != null ? _c : DEFAULT_EVAL_POLL_INTERVAL_MS;
+  validateWait("waitMs", waitMs, true);
+  validateWait("pollIntervalMs", pollIntervalMs, false);
+  const starts = await Promise.allSettled(
+    evaluators.map(async (evaluator) => {
+      const pin = evaluatorPins.get(evaluator);
+      const started = await queryApi({
+        client,
+        queryUrl,
+        path: `/v1/evals/${encodeURIComponent(evaluator)}/runs`,
+        method: "POST",
+        schema: WireStartedEvalRunSchema,
+        body: {
+          source: {
+            replay: replayId,
+            ...options.traceEvidence ? { traceEvidence: options.traceEvidence } : {}
+          },
+          ...pin != null ? pin : {}
+        }
+      });
+      if (pin && (started.data.evalId !== pin.expectedEvalId || started.data.programVersion !== pin.expectedProgramVersion)) {
+        throw new Error(`Raindrop evaluator ${evaluator} did not honor its pulled identity pin`);
+      }
+      return { evaluator, ...started.data };
+    })
+  );
+  const startedRuns = starts.flatMap(
+    (entry) => entry.status === "fulfilled" ? [entry.value] : []
+  );
+  const failures = starts.flatMap((entry) => entry.status === "rejected" ? [entry.reason] : []);
+  if (failures.length > 0) {
+    throw Object.assign(
+      new Error(
+        "Raindrop could not start every evaluator; startedRuns contains the runs that did start"
+      ),
+      { startedRuns, causes: failures }
+    );
+  }
+  const deadlineAt = Date.now() + waitMs;
+  return Promise.all(
+    startedRuns.map(async (started) => {
+      let run = await readEvalRun(client, queryUrl, started.runId, deadlineAt);
+      while (run.status === "running" && Date.now() < deadlineAt) {
+        await sleep(Math.min(pollIntervalMs, Math.max(0, deadlineAt - Date.now())));
+        if (Date.now() >= deadlineAt) break;
+        run = await readEvalRun(client, queryUrl, started.runId, deadlineAt);
+      }
+      if (run.status === "running") {
+        throw new Error(
+          `Raindrop eval ${started.evaluator} (${started.runId}) did not finish within ${waitMs}ms`
+        );
+      }
+      return {
+        evaluator: started.evaluator,
+        runId: started.runId,
+        status: run.status,
+        url: started.url,
+        summary: run.summary,
+        error: run.error,
+        outcomes: run.traces
+      };
+    })
+  );
+}
+function replayVerdict(evaluator, outcome) {
+  const identity = {
+    evaluator: evaluator.evaluator,
+    runId: evaluator.runId,
+    url: evaluator.url
+  };
+  if (outcome.state === "errored") {
+    return { ...identity, state: outcome.state, message: outcome.message };
+  }
+  if (outcome.state === "ungraded") {
+    return { ...identity, state: outcome.state, reason: outcome.reason };
+  }
+  if ("pass" in outcome) {
+    return { ...identity, state: outcome.state, pass: outcome.pass, note: outcome.note };
+  }
+  if ("score" in outcome) {
+    return { ...identity, state: outcome.state, score: outcome.score, note: outcome.note };
+  }
+  return { ...identity, state: outcome.state, value: outcome.value, note: outcome.note };
+}
+function replayResultRows(handle, reading, evaluators) {
+  var _a;
+  const resultsByRow = new Map(reading.rows.map((row) => [row.rowId, row]));
+  const verdictsByTrace = /* @__PURE__ */ new Map();
+  for (const evaluator of evaluators) {
+    for (const outcome of evaluator.outcomes) {
+      const verdicts = (_a = verdictsByTrace.get(outcome.traceId)) != null ? _a : [];
+      verdicts.push(replayVerdict(evaluator, outcome));
+      verdictsByTrace.set(outcome.traceId, verdicts);
+    }
+  }
+  return handle.rows.map(({ correlationId: _correlationId, ...row }) => {
+    var _a2, _b;
+    const result = resultsByRow.get(row.id);
+    if (!result) throw new Error(`Raindrop replay result omitted row ${row.id}`);
+    const replayRow = ReplayResultRowSchema.parse({
+      ...row,
+      status: result.status,
+      traceId: result.traceId,
+      error: result.error,
+      verdicts: evaluators.map((evaluator) => {
+        var _a3, _b2, _c;
+        const matches = result.traceId === null ? [] : (_b2 = (_a3 = verdictsByTrace.get(result.traceId)) == null ? void 0 : _a3.filter((entry) => entry.evaluator === evaluator.evaluator)) != null ? _b2 : [];
+        if (matches.length > 1)
+          throw new Error(
+            `Raindrop replay row ${row.id} received duplicate verdicts from ${evaluator.evaluator}`
+          );
+        const verdict = matches[0];
+        const identity = {
+          evaluator: evaluator.evaluator,
+          runId: evaluator.runId,
+          url: evaluator.url
+        };
+        if (evaluator.status === "failed")
+          return { ...identity, state: "errored", message: (_c = evaluator.error) != null ? _c : "Evaluator failed" };
+        if (verdict) return verdict;
+        if (result.traceId === null)
+          return { ...identity, state: "ungraded", reason: "trace unavailable" };
+        throw new Error(
+          `Raindrop replay ${reading.replayId} row ${row.id} omitted verdict from ${evaluator.evaluator}`
+        );
+      })
+    });
+    if (replayRow.status === "done") {
+      const counts = /* @__PURE__ */ new Map();
+      for (const verdict of replayRow.verdicts) {
+        counts.set(verdict.evaluator, ((_a2 = counts.get(verdict.evaluator)) != null ? _a2 : 0) + 1);
+      }
+      for (const evaluator of evaluators) {
+        const count = (_b = counts.get(evaluator.evaluator)) != null ? _b : 0;
+        if (count !== 1) {
+          const failure = evaluator.status === "failed" ? ` Evaluator failed${evaluator.error ? `: ${evaluator.error}` : "."}` : "";
+          throw new Error(
+            `Raindrop replay ${reading.replayId} row ${row.id} expected one verdict from ${evaluator.evaluator}, received ${count}.${failure} Run: ${evaluator.url}`
+          );
+        }
+      }
+    }
+    return replayRow;
+  });
+}
+async function readEvalRun(client, queryUrl, runId, deadlineAt) {
+  const response = await queryApi({
+    client,
+    queryUrl,
+    path: `/v1/eval-runs/${encodeURIComponent(runId)}?include=traces`,
+    method: "GET",
+    schema: WireEvalRunSchema,
+    deadlineAt,
+    maxRetries: MAX_EVAL_STATUS_RETRIES
+  });
+  return response.data;
+}
+async function waitForTraces(input) {
+  let reading = await readReplay(input.client, input.replayId, {
+    queryUrl: input.queryUrl,
+    deadlineAt: input.deadlineAt
+  });
+  while (reading.counts.pending > 0 && Date.now() < input.deadlineAt) {
+    await sleep(Math.min(TRACE_POLL_INTERVAL_MS, Math.max(0, input.deadlineAt - Date.now())));
+    if (Date.now() >= input.deadlineAt) break;
+    try {
+      reading = await readReplay(input.client, input.replayId, {
+        queryUrl: input.queryUrl,
+        deadlineAt: input.deadlineAt
+      });
+    } catch (error) {
+      if (Date.now() < input.deadlineAt) throw error;
+      break;
+    }
+  }
+  return reading;
+}
+async function waitForFailedTraceAssociation(input) {
+  if (input.spanIds.size === 0 && input.traceIds.size === 0) return null;
+  if (input.spanIds.size === 0) {
+    throw new Error(
+      `Raindrop replay ${input.replayId} failed row ${input.rowId} captured a trace without ended spans`
+    );
+  }
+  if (input.traceIds.size !== 1) {
+    throw new Error(
+      `Raindrop replay ${input.replayId} failed row ${input.rowId} expected one trace, captured ${input.traceIds.size}`
+    );
+  }
+  validateWait("traceWaitMs", input.waitMs, true);
+  const expectedTraceId = [...input.traceIds][0];
+  if (expectedTraceId === void 0) {
+    throw new Error(
+      `Raindrop replay ${input.replayId} lost failed trace evidence for ${input.rowId}`
+    );
+  }
+  const deadlineAt = Date.now() + input.waitMs;
+  while (true) {
+    const reading = await readReplay(input.client, input.replayId, {
+      queryUrl: input.queryUrl,
+      deadlineAt
+    });
+    const row = reading.rows.find(
+      (candidate) => candidate.rowId === input.rowId && candidate.attempt === input.attempt
+    );
+    if (!row) {
+      throw new Error(
+        `Raindrop replay ${input.replayId} omitted failed row ${input.rowId} attempt ${input.attempt}`
+      );
+    }
+    if (row.status !== "missing") {
+      throw new Error(
+        `Raindrop replay ${input.replayId} changed failed row ${input.rowId} attempt ${input.attempt} to ${row.status}`
+      );
+    }
+    if (row.traceId === expectedTraceId) return expectedTraceId;
+    if (row.traceId !== null) {
+      throw new Error(
+        `Raindrop replay ${input.replayId} linked failed row ${input.rowId} attempt ${input.attempt} to the wrong trace`
+      );
+    }
+    if (Date.now() >= deadlineAt) {
+      throw new Error(
+        `Raindrop replay ${input.replayId} did not retain failed trace ${expectedTraceId} for ${input.rowId} attempt ${input.attempt} within ${input.waitMs}ms`
+      );
+    }
+    await sleep(Math.min(TRACE_POLL_INTERVAL_MS, Math.max(0, deadlineAt - Date.now())));
+  }
+}
+function failedTraceRetentionError(taskCause, retentionCause, rowId) {
+  const taskError = taskCause instanceof Error ? taskCause : new Error(describeError(taskCause));
+  const retentionError = retentionCause instanceof Error ? retentionCause : new Error(describeError(retentionCause));
+  return new AggregateError(
+    [taskError, retentionError],
+    `Raindrop replay task ${rowId} failed and its trace could not be retained`,
+    { cause: taskError }
+  );
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isRetryableStatus(status) {
+  return status === 408 || status === 429 || status >= 500;
+}
+function validateEvaluatorNames(evaluators) {
+  if (evaluators.length === 0) {
+    throw new Error("Raindrop replay requires at least one evaluator");
+  }
+  const normalized = evaluators.map((evaluator) => evaluator.trim());
+  if (normalized.some((evaluator) => evaluator.length === 0)) {
+    throw new Error("Raindrop replay evaluators cannot be empty");
+  }
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("Raindrop replay evaluators must be unique");
+  }
+  return normalized;
+}
+function validateEvaluatorPins(evaluators, raw) {
+  if (!raw) return /* @__PURE__ */ new Map();
+  const allowed = new Set(evaluators);
+  const pins = /* @__PURE__ */ new Map();
+  for (const [slug, pin] of Object.entries(raw)) {
+    if (!allowed.has(slug)) {
+      throw new Error(`Raindrop evaluator pin names unknown evaluator ${slug}`);
+    }
+    if (!z3.string().uuid().safeParse(pin.expectedEvalId).success) {
+      throw new Error(`Raindrop evaluator ${slug} pin has an invalid evaluator id`);
+    }
+    if (!Number.isInteger(pin.expectedProgramVersion) || pin.expectedProgramVersion <= 0) {
+      throw new Error(`Raindrop evaluator ${slug} pin has an invalid program version`);
+    }
+    pins.set(slug, pin);
+  }
+  return pins;
+}
+function normalizeEvaluators(evaluators) {
+  if (evaluators.length === 0) {
+    throw new Error("Raindrop replay requires at least one evaluator");
+  }
+  const normalized = evaluators.map(
+    (evaluator) => typeof evaluator === "string" ? evaluator.trim() : {
+      ...evaluator,
+      slug: evaluator.slug.trim(),
+      name: evaluator.name.trim()
+    }
+  );
+  const names = normalized.map(
+    (evaluator) => typeof evaluator === "string" ? evaluator : evaluator.slug
+  );
+  validateEvaluatorNames(names);
+  for (const evaluator of normalized) {
+    if (typeof evaluator === "string") continue;
+    if (!LOCAL_EVALUATOR_SLUG_PATTERN.test(evaluator.slug) || evaluator.slug.length > 64) {
+      throw new Error(
+        "Raindrop local evaluator slugs must use lowercase letters, numbers, and single hyphens"
+      );
+    }
+    if (evaluator.name.length === 0 || evaluator.name.length > 120) {
+      throw new Error("Raindrop local evaluator names must be between 1 and 120 characters");
+    }
+  }
+  return normalized;
+}
+function localEvaluatorError(cause) {
+  const message = describeError(cause).trim();
+  return (message === "" ? "Local evaluator failed" : message).slice(0, 600);
+}
+function validateWait(name, value, allowZero) {
+  const valid = Number.isFinite(value) && (allowZero ? value >= 0 : value > 0);
+  if (!valid) {
+    throw new Error(`Raindrop replay ${name} must be ${allowZero ? "non-negative" : "positive"}`);
+  }
+}
+function describeError(cause) {
+  if (cause instanceof Error) {
+    return cause.message.trim() === "" ? cause.name : `${cause.name}: ${cause.message}`;
+  }
+  return String(cause);
+}
+
+// src/evals/dataset.ts
+var MAX_PUBLISH_RETRIES = 2;
+var RETRY_BASE_DELAY_MS = 100;
+var MAX_RETRY_DELAY_MS2 = 3e4;
+var PUBLISH_REQUEST_TIMEOUT_MS = 3e4;
+var EvalDatasetPublishConflictError = class extends Error {
+  constructor(slug, message) {
+    super(message);
+    this.name = "EvalDatasetPublishConflictError";
+    this.slug = slug;
+  }
+};
+async function publishEvalDataset(client, options) {
+  var _a, _b, _c;
+  if (!((_a = client.connection.apiKey) == null ? void 0 : _a.trim())) {
+    throw new Error("Raindrop eval dataset publication requires a Query SDK API key (apiKey)");
+  }
+  const queryUrl = credentialEndpoint((_b = options.queryUrl) != null ? _b : DEFAULT_QUERY_URL).href;
+  const slug = EvalDatasetSlugSchema.parse(options.slug.trim());
+  const fingerprint = await evalDatasetVersionFingerprint(options.cases);
+  const body = PublishEvalDatasetVersionInputSchema.parse({
+    name: options.name,
+    requestKey: randomUUID(),
+    expectedCurrentVersionId: (_c = options.expectedCurrentVersionId) != null ? _c : null,
+    fingerprint,
+    cases: options.cases
+  });
+  validateUploadSizes(body);
+  const path = `/v1/eval-datasets/${encodeURIComponent(slug)}`;
+  const response = await publishRequest({
+    client,
+    url: `${queryUrl.replace(/\/+$/, "")}${path}`,
+    path,
+    slug,
+    body: JSON.stringify(body)
+  });
+  const manifest = EvalDatasetManifestSchema.parse(response);
+  if (manifest.dataset.slug !== slug || manifest.dataset.id !== manifest.version.datasetId || manifest.version.fingerprint !== fingerprint) {
+    throw new Error(`Raindrop eval dataset ${slug} returned mismatched publication identity`);
+  }
+  return manifest;
+}
+function validateUploadSizes(body) {
+  for (const [index, entry] of body.cases.entries()) {
+    const bytes2 = encodedBytes(entry.referenceTrace);
+    if (bytes2 > EVAL_DATASET_MAX_REFERENCE_BYTES) {
+      throw new Error(
+        `Raindrop eval dataset case ${entry.id} at index ${index} exceeds ${EVAL_DATASET_MAX_REFERENCE_BYTES} reference bytes`
+      );
+    }
+  }
+  const bytes = encodedBytes(body);
+  if (bytes > EVAL_DATASET_MAX_REQUEST_BYTES) {
+    throw new Error(`Raindrop eval dataset upload exceeds ${EVAL_DATASET_MAX_REQUEST_BYTES} bytes`);
+  }
+}
+function encodedBytes(value) {
+  try {
+    return new TextEncoder().encode(JSON.stringify(value)).byteLength;
+  } catch (cause) {
+    throw new Error("Raindrop eval dataset upload must be serializable", { cause });
+  }
+}
+async function publishRequest(input) {
+  const headers = {
+    Authorization: `Bearer ${input.client.connection.apiKey}`,
+    "Content-Type": "application/json"
+  };
+  const projectId = input.client.connection.projectId;
+  if (projectId !== void 0) headers["X-Raindrop-Project-Id"] = projectId;
+  for (let retry = 0; ; retry += 1) {
+    let response;
+    let text;
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), PUBLISH_REQUEST_TIMEOUT_MS);
+    try {
+      response = await runWithTracingSuppressed(
+        () => fetch(input.url, {
+          method: "PUT",
+          headers,
+          body: input.body,
+          signal: controller.signal,
+          redirect: "error"
+        })
+      );
+      text = await response.text();
+    } catch (cause) {
+      if (retry >= MAX_PUBLISH_RETRIES) throw cause;
+      await sleep2(RETRY_BASE_DELAY_MS * 2 ** retry);
+      continue;
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (response.ok) {
+      try {
+        return JSON.parse(text);
+      } catch (cause) {
+        throw new Error(`Raindrop API PUT ${input.path} returned invalid JSON`, { cause });
+      }
+    }
+    if (response.status === 409) {
+      throw new EvalDatasetPublishConflictError(
+        input.slug,
+        `Raindrop eval dataset ${input.slug} changed before publication: ${text}`
+      );
+    }
+    if (retry < MAX_PUBLISH_RETRIES && isRetryableStatus2(response.status)) {
+      await sleep2(retryDelayMs2(response, retry));
+      continue;
+    }
+    throw new Error(`Raindrop API PUT ${input.path}: ${response.status} ${text}`);
+  }
+}
+function retryDelayMs2(response, retry) {
+  const retryAfter = response.headers.get("retry-after");
+  if (retryAfter !== null) {
+    const seconds = Number(retryAfter);
+    if (Number.isFinite(seconds) && seconds >= 0) {
+      return Math.min(seconds * 1e3, MAX_RETRY_DELAY_MS2);
+    }
+  }
+  return RETRY_BASE_DELAY_MS * 2 ** retry;
+}
+function isRetryableStatus2(status) {
+  return status === 408 || status === 429 || status >= 500;
+}
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/evals/pull.ts
+import { z as z4 } from "zod";
+var SNAPSHOT_VERSION = 1;
+var PulledEvaluatorSchema = z4.object({
+  slug: z4.string().min(1),
+  name: z4.string().min(1),
+  expectedSha256: z4.string().regex(/^[a-f0-9]{64}$/),
+  artifact: z4.unknown()
+});
+var EvalSnapshotSchema = z4.object({
+  schemaVersion: z4.literal(SNAPSHOT_VERSION),
+  pulledAt: z4.string().datetime(),
+  dataset: z4.object({
+    reference: z4.string().min(1),
+    datasetId: z4.string().uuid(),
+    fingerprint: z4.string().regex(/^[a-f0-9]{64}$/),
+    rows: z4.array(
+      z4.object({
+        id: z4.string(),
+        name: z4.string(),
+        input: z4.string().nullable(),
+        output: z4.string().nullable(),
+        properties: z4.record(z4.string())
+      })
+    )
+  }),
+  evaluators: z4.array(PulledEvaluatorSchema)
+});
+async function pullEval(client, options) {
+  var _a;
+  const datasetReference = options.dataset.trim();
+  if (!datasetReference) throw new Error("Raindrop pullEval dataset cannot be empty");
+  const evaluatorSlugs = options.evaluators.map((slug) => slug.trim());
+  if (evaluatorSlugs.length === 0) throw new Error("Raindrop pullEval requires evaluators");
+  if (evaluatorSlugs.some((slug) => !slug)) {
+    throw new Error("Raindrop pullEval evaluator slugs cannot be empty");
+  }
+  if (new Set(evaluatorSlugs).size !== evaluatorSlugs.length) {
+    throw new Error("Raindrop pullEval evaluator slugs must be unique");
+  }
+  if (!options.snapshotPath.trim())
+    throw new Error("Raindrop pullEval snapshotPath cannot be empty");
+  const queryUrl = ((_a = options.queryUrl) != null ? _a : DEFAULT_QUERY_URL).replace(/\/+$/, "");
+  const [manifest, evaluators] = await Promise.all([
+    readEvalManifest(client, datasetReference, { queryUrl }),
+    Promise.all(evaluatorSlugs.map((slug) => pullEvaluator(client, queryUrl, slug)))
+  ]);
+  const snapshot = {
+    schemaVersion: SNAPSHOT_VERSION,
+    pulledAt: (/* @__PURE__ */ new Date()).toISOString(),
+    dataset: {
+      reference: datasetReference,
+      datasetId: manifest.datasetId,
+      fingerprint: manifest.fingerprint,
+      rows: [...manifest.cases]
+    },
+    evaluators
+  };
+  const materialized = await materializeSnapshot(options.snapshotPath, snapshot, options.judges);
+  await writeSnapshot(options.snapshotPath, snapshot);
+  return materialized;
+}
+async function loadEvalSnapshot(snapshotPath, options = {}) {
+  const fs = await import("fs/promises");
+  const parsed = EvalSnapshotSchema.parse(JSON.parse(await fs.readFile(snapshotPath, "utf8")));
+  return materializeSnapshot(snapshotPath, parsed, options.judges);
+}
+async function pullEvaluator(client, queryUrl, rawSlug) {
+  var _a;
+  if (!rawSlug) throw new Error("Raindrop pullEval evaluator slugs cannot be empty");
+  if (!((_a = client.connection.apiKey) == null ? void 0 : _a.trim())) {
+    throw new Error("Raindrop pullEval requires an org API key for evaluator artifacts");
+  }
+  const response = await runWithTracingSuppressed(
+    () => fetch(
+      `${credentialEndpoint(queryUrl).href.replace(/\/+$/, "")}/v1/evals/${encodeURIComponent(rawSlug)}`,
+      {
+        redirect: "error",
+        signal: AbortSignal.timeout(3e4),
+        headers: {
+          Authorization: `Bearer ${client.connection.apiKey}`,
+          ...client.connection.projectId ? { "X-Raindrop-Project-Id": client.connection.projectId } : {}
+        }
+      }
+    )
+  );
+  if (!response.ok) {
+    throw new Error(
+      `Raindrop evaluator ${rawSlug} pull failed with ${response.status}: ${await response.text()}`
+    );
+  }
+  const body = z4.object({
+    data: z4.object({
+      slug: z4.string(),
+      name: z4.string(),
+      artifact: z4.unknown().nullable()
+    })
+  }).parse(await response.json());
+  if (!body.data.artifact)
+    throw new Error(`Raindrop evaluator ${rawSlug} has no portable artifact`);
+  const artifact = await verifyPortableEvalArtifact(body.data.artifact);
+  if (body.data.slug !== rawSlug || artifact.evaluator.slug !== rawSlug) {
+    throw new Error(`Raindrop evaluator ${rawSlug} returned mismatched artifact identity`);
+  }
+  const evaluator = importPortableEvaluator(artifact, {
+    name: body.data.name
+  });
+  return {
+    slug: body.data.slug,
+    name: body.data.name,
+    expectedSha256: evaluator.artifact.sha256,
+    artifact: evaluator.artifact
+  };
+}
+async function materializeSnapshot(snapshotPath, raw, judges) {
+  const snapshot = EvalSnapshotSchema.parse(raw);
+  const fingerprint = await datasetFingerprint(snapshot.dataset.datasetId, snapshot.dataset.rows);
+  if (fingerprint !== snapshot.dataset.fingerprint) {
+    throw new Error("Raindrop eval snapshot dataset fingerprint is invalid");
+  }
+  const dataset = defineDataset({
+    id: snapshot.dataset.datasetId,
+    name: snapshot.dataset.reference,
+    version: snapshot.dataset.fingerprint,
+    cases: snapshot.dataset.rows,
+    remote: {
+      reference: snapshot.dataset.reference,
+      datasetId: snapshot.dataset.datasetId,
+      fingerprint: snapshot.dataset.fingerprint
+    }
+  });
+  const evaluators = await Promise.all(
+    snapshot.evaluators.map(async (entry) => {
+      const artifact = await verifyPortableEvalArtifact(entry.artifact, {
+        expectedSha256: entry.expectedSha256
+      });
+      if (artifact.evaluator.slug !== entry.slug) {
+        throw new Error(`Raindrop eval snapshot evaluator ${entry.slug} has mismatched identity`);
+      }
+      return importPortableEvaluator(artifact, {
+        name: entry.name,
+        expectedSha256: entry.expectedSha256,
+        judge: judges == null ? void 0 : judges[entry.slug]
+      });
+    })
+  );
+  return { dataset, evaluators, snapshotPath };
+}
+async function datasetFingerprint(datasetId, rows) {
+  const canonicalRows = rows.map((row) => ({
+    id: row.id,
+    name: row.name,
+    input: row.input,
+    output: row.output,
+    properties: Object.fromEntries(
+      Object.entries(row.properties).sort(([left], [right]) => compareCodeUnits(left, right))
+    )
+  })).sort((left, right) => compareCodeUnits(left.id, right.id));
+  const value = JSON.stringify({ dataset_id: datasetId, rows: canonicalRows });
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(value));
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+function compareCodeUnits(left, right) {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+async function writeSnapshot(path, value) {
+  const [fs, nodePath] = await Promise.all([import("fs/promises"), import("path")]);
+  const absolute = nodePath.resolve(path);
+  await fs.mkdir(nodePath.dirname(absolute), { recursive: true });
+  const temporary = `${absolute}.${Date.now()}-${Math.random().toString(16).slice(2)}.tmp`;
+  try {
+    await fs.writeFile(temporary, `${JSON.stringify(value, null, 2)}
+`, { flag: "wx" });
+    await fs.rename(temporary, absolute);
+  } catch (cause) {
+    await fs.rm(temporary, { force: true }).catch(() => {
+    });
+    throw cause;
+  }
+}
+
+// src/evals/run.ts
+import { context as context2, trace as otelTrace } from "@opentelemetry/api";
+
+// src/evals/policy.ts
+function assessEvalVerdict(evaluator, verdict, resolvedOutput) {
+  var _a;
+  const name = evaluatorName(evaluator);
+  if (verdict.evaluator !== name) {
+    throw new Error(`Raindrop evaluator ${name} received a verdict for ${verdict.evaluator}`);
+  }
+  if (verdict.state === "errored") return { state: "errored", evaluator: name, verdict };
+  if (verdict.state === "ungraded") return { state: "ungraded", evaluator: name, verdict };
+  const output = resolvedOutput != null ? resolvedOutput : evaluatorOutput(evaluator);
+  if (output === void 0) {
+    throw new Error(`Raindrop hosted evaluator ${name} requires its canonical output type`);
+  }
+  if (output === "boolean") {
+    if (!("pass" in verdict)) throw outputMismatch(name, output);
+    const threshold2 = evaluator.threshold;
+    if (threshold2 && !("equals" in threshold2)) throw outputMismatch(name, output);
+    const expected = (_a = threshold2 == null ? void 0 : threshold2.equals) != null ? _a : true;
+    return {
+      state: verdict.pass === expected ? "passed" : "failed",
+      evaluator: name,
+      verdict
+    };
+  }
+  const value = output === "score" && "score" in verdict ? verdict.score : output === "number" && "value" in verdict ? verdict.value : null;
+  if (value === null) throw outputMismatch(name, output);
+  const threshold = evaluator.threshold;
+  if (!threshold) return { state: "measurement", evaluator: name, verdict };
+  if ("equals" in threshold) throw outputMismatch(name, output);
+  const passesLower = !("gte" in threshold) || value >= threshold.gte;
+  const passesUpper = threshold.lte === void 0 || value <= threshold.lte;
+  return {
+    state: passesLower && passesUpper ? "passed" : "failed",
+    evaluator: name,
+    verdict
+  };
+}
+function outputMismatch(name, output) {
+  return new Error(`Raindrop evaluator ${name} expected a ${output} verdict`);
+}
+
+// src/evals/workshop.ts
+import { z as z5 } from "zod";
+var WorkshopRunSchema = z5.object({
+  id: z5.string(),
+  event_id: z5.string().nullable(),
+  name: z5.string().nullable(),
+  event_name: z5.string().nullable(),
+  user_id: z5.string().nullable(),
+  convo_id: z5.string().nullable(),
+  started_at: z5.number().nullable(),
+  last_updated_at: z5.number().nullable(),
+  metadata: z5.string().nullable(),
+  model: z5.string().nullable(),
+  finished: z5.number().nullable()
+});
+var WorkshopSpanSchema = z5.object({
+  id: z5.string(),
+  run_id: z5.string(),
+  parent_span_id: z5.string().nullable(),
+  name: z5.string(),
+  span_type: z5.string().nullable(),
+  status: z5.string().nullable(),
+  input_payload: z5.string().nullable(),
+  output_payload: z5.string().nullable(),
+  start_time_ms: z5.number().nullable(),
+  end_time_ms: z5.number().nullable(),
+  duration_ms: z5.number().nullable(),
+  model: z5.string().nullable(),
+  provider: z5.string().nullable(),
+  input_tokens: z5.number().nullable(),
+  output_tokens: z5.number().nullable(),
+  attributes: z5.string().nullable()
+});
+var WorkshopTraceEnvelopeSchema = z5.object({
+  status: z5.literal("complete"),
+  correlationId: z5.string(),
+  traceId: z5.string(),
+  rootSpanId: z5.string(),
+  spanCount: z5.number().int().positive(),
+  run: WorkshopRunSchema,
+  spans: z5.array(WorkshopSpanSchema).min(1)
+});
+async function readWorkshopTrace(input) {
+  if (!Number.isFinite(input.waitMs) || input.waitMs < 0)
+    throw new Error("Workshop waitMs must be finite and non-negative");
+  const deadlineAt = Date.now() + input.waitMs;
+  if (input.expectedSpanIds.length === 0) {
+    throw new Error(`Workshop trace ${input.correlationId} exported no spans`);
+  }
+  const url = new URL(
+    `${credentialEndpoint(input.workshopUrl).href.replace(/\/+$/, "")}/v1/evals/traces/${encodeURIComponent(input.correlationId)}`
+  );
+  for (const spanId of input.expectedSpanIds) url.searchParams.append("expectedSpanId", spanId);
+  while (true) {
+    const response = await runWithTracingSuppressed(
+      () => fetch(url, {
+        redirect: "error",
+        signal: AbortSignal.timeout(
+          Math.min(3e4, Math.max(1, Math.ceil(deadlineAt - Date.now())))
+        )
+      })
+    );
+    if (response.ok) {
+      const parsed = WorkshopTraceEnvelopeSchema.safeParse(await response.json());
+      if (!parsed.success) {
+        throw new Error(`Workshop returned an invalid completed trace: ${parsed.error.message}`);
+      }
+      return parsed.data;
+    }
+    if (response.status !== 404 && response.status !== 409) {
+      throw new Error(
+        `Workshop trace read failed with ${response.status}: ${await response.text()}`
+      );
+    }
+    if (Date.now() >= deadlineAt) {
+      const state = response.status === 404 ? "missing" : "incomplete";
+      throw new Error(`Workshop trace ${input.correlationId} remained ${state}`);
+    }
+    await new Promise(
+      (resolve) => setTimeout(resolve, Math.min(50, Math.max(0, deadlineAt - Date.now())))
+    );
+  }
+}
+async function writeWorkshopEvalRun(workshopUrl, packet) {
+  const normalized = {
+    ...packet,
+    cases: packet.cases.map((entry) => ({
+      ...entry,
+      task: entry.task.state === "completed" ? { ...entry.task, output: serializableOutput(entry.task.output) } : entry.task
+    }))
+  };
+  const response = await runWithTracingSuppressed(
+    () => fetch(`${credentialEndpoint(workshopUrl).href.replace(/\/+$/, "")}/v1/evals/runs`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(normalized),
+      redirect: "error",
+      signal: AbortSignal.timeout(3e4)
+    })
+  );
+  if (!response.ok) {
+    throw new Error(
+      `Workshop eval result write failed with ${response.status}: ${await response.text()}`
+    );
+  }
+  const body = await response.json();
+  if (typeof body !== "object" || body === null || !("id" in body) || typeof body.id !== "string") {
+    throw new Error("Workshop eval result write returned an invalid response");
+  }
+  return { id: body.id };
+}
+function serializableOutput(value) {
+  try {
+    const serialized = JSON.stringify(boundedClone(value, 64e3));
+    if (serialized === void 0) return null;
+    if (serialized.length > 64e3) return "[Output exceeds 64000 characters]";
+    return JSON.parse(serialized);
+  } catch (e) {
+    return "[Output could not be serialized]";
+  }
+}
+
+// src/evals/run.ts
+var DEFAULT_WORKSHOP_TRACE_WAIT_MS = 3e4;
+function createEvalSuiteRun(client, definition, options) {
+  const validated = validateDefinition(definition);
+  if (options.destination.kind === "raindrop") {
+    if (!("selection" in options)) {
+      throw new Error("Raindrop remote eval sessions require selected cases");
+    }
+    if (typeof validated.dataset !== "string") {
+      throw new Error("Raindrop remote eval sessions require a dataset slug");
+    }
+    if (options.selection.dataset.dataset.slug !== validated.dataset) {
+      throw new Error(
+        `Raindrop eval ${validated.name} pinned dataset ${options.selection.dataset.dataset.slug} does not match ${validated.dataset}`
+      );
+    }
+    const localEvaluators = validated.evaluators.flatMap((entry) => {
+      const evaluator = replayEvaluator(entry);
+      return typeof evaluator === "string" ? [] : [evaluator];
+    });
+    if (localEvaluators.length !== validated.evaluators.length) {
+      throw new Error("Raindrop case-scoped eval sessions support local evaluators only");
+    }
+    const selection = validateRemoteSelection(validated.dataset, options.selection);
+    if (!selection) throw new Error("Raindrop remote eval sessions require selected cases");
+    return new RaindropCaseEvalRun(
+      client,
+      validated,
+      options.destination,
+      options.selection.dataset,
+      selection.caseIds,
+      localEvaluators
+    );
+  }
+  if (!isEvalDataset(validated.dataset)) {
+    throw new Error("Workshop eval execution requires a pulled or local dataset snapshot");
+  }
+  credentialEndpoint(options.destination.url);
+  const hosted = validated.evaluators.find((entry) => typeof entry.evaluator === "string");
+  if (hosted && typeof hosted.evaluator === "string") {
+    throw new Error(`Workshop evaluator ${hosted.evaluator} must be pulled before local execution`);
+  }
+  if (validated.evaluators.some(
+    (entry) => typeof entry.evaluator !== "string" && "kind" in entry.evaluator && entry.evaluator.kind === "program"
+  )) {
+    throw new Error(
+      "Workshop authored evaluator programs must be pulled as portable evaluators before local execution"
+    );
+  }
+  return new WorkshopEvalRun(
+    client,
+    { ...validated, dataset: validated.dataset },
+    options.destination
+  );
+}
+async function runEvalSuite(client, definition, options) {
+  var _a, _b, _c, _d;
+  const validated = validateDefinition(definition);
+  if (options === void 0) throw new Error("Raindrop runEvalSuite requires an explicit destination");
+  if (options.destination.kind === "workshop") {
+    const session = createEvalSuiteRun(client, validated, { destination: options.destination });
+    await session.run({ selection: options.selection, attempt: options.attempt });
+    return session.finish();
+  }
+  if (isCaseScopedLocalEval(validated)) {
+    if (typeof validated.dataset !== "string") {
+      throw new Error("Raindrop remote execution requires an eval dataset slug");
+    }
+    const dataset = (_b = (_a = options.selection) == null ? void 0 : _a.dataset) != null ? _b : await readEvalDataset(client, validated.dataset, {
+      queryUrl: options.destination.queryUrl
+    });
+    const selection = validateRemoteSelection(validated.dataset, {
+      dataset,
+      caseIds: (_d = (_c = options.selection) == null ? void 0 : _c.caseIds) != null ? _d : dataset.cases.map((entry) => entry.id)
+    });
+    if (!selection) throw new Error("Raindrop remote eval requires selected cases");
+    const session = createEvalSuiteRun(client, validated, {
+      destination: options.destination,
+      selection: { dataset, caseIds: selection.caseIds }
+    });
+    const attempts = await Promise.allSettled(
+      selection.caseIds.map(
+        (caseId) => session.run({ selection: { dataset, caseIds: [caseId] }, attempt: 0 })
+      )
+    );
+    const failures = attempts.filter(
+      (attempt) => attempt.status === "rejected"
+    );
+    let finished;
+    try {
+      finished = await session.finish();
+    } catch (cause) {
+      if (failures.length > 0) {
+        throw new AggregateError(
+          [...failures.map((failure) => failure.reason), cause],
+          "Raindrop eval case execution and run finalization failed"
+        );
+      }
+      throw cause;
+    }
+    const onlyFailure = failures.length === 1 ? failures[0] : void 0;
+    if (onlyFailure) throw onlyFailure.reason;
+    if (failures.length > 1) {
+      throw new AggregateError(
+        failures.map((failure) => failure.reason),
+        "Multiple Raindrop eval cases failed"
+      );
+    }
+    return finished;
+  }
+  return runRaindropEval(client, validated, options.destination, options);
+}
+function isCaseScopedLocalEval(definition) {
+  return definition.evaluators.every(
+    (entry) => typeof entry.evaluator !== "string" && entry.evaluator.scope === "case" && !("kind" in entry.evaluator)
+  );
+}
+var RaindropCaseEvalRun = class {
+  constructor(client, definition, destination, dataset, caseIds, evaluators) {
+    this.client = client;
+    this.definition = definition;
+    this.destination = destination;
+    this.dataset = dataset;
+    this.caseIds = caseIds;
+    this.evaluators = evaluators;
+    this.id = randomId();
+    this.active = /* @__PURE__ */ new Set();
+    this.completed = [];
+    this.attempts = /* @__PURE__ */ new Set();
+    this.state = "open";
+    this.runningCases = 0;
+    this.caseWaiters = [];
+  }
+  async run(options = {}) {
+    if (this.state !== "open") throw new Error(`Raindrop eval run ${this.id} is already finishing`);
+    const operation = this.runOnce(options);
+    this.active.add(operation);
+    try {
+      return await operation;
+    } finally {
+      this.active.delete(operation);
+    }
+  }
+  finish() {
+    if (this.finishPromise) return this.finishPromise;
+    this.state = "finishing";
+    this.finishPromise = this.finishOnce();
+    return this.finishPromise;
+  }
+  async runOnce(options) {
+    var _a, _b;
+    const selected = (_a = options.selection) == null ? void 0 : _a.caseIds;
+    if (!selected || selected.length !== 1) {
+      throw new Error("Raindrop case-scoped eval session run requires exactly one case");
+    }
+    const caseId = selected[0];
+    if (caseId === void 0 || !this.caseIds.includes(caseId)) {
+      throw new Error(`Raindrop eval ${this.definition.name} did not select a planned case`);
+    }
+    const attempt = (_b = options.attempt) != null ? _b : 0;
+    const attemptKey = `${caseId}:${attempt}`;
+    if (this.attempts.has(attemptKey)) {
+      throw new Error(`Raindrop eval case ${caseId} attempt ${attempt} already ran`);
+    }
+    this.attempts.add(attemptKey);
+    return this.withCasePermit(async () => {
+      const replay2 = await this.getReplay();
+      const attemptResult = await replay2.run({
+        rowId: caseId,
+        attempt,
+        run: this.definition.run,
+        traceWaitMs: this.definition.traceWaitMs
+      });
+      const configured = new Map(
+        this.definition.evaluators.map((entry) => [evaluatorName(entry), entry])
+      );
+      const result = {
+        ...attemptResult.row,
+        attempt,
+        status: attemptResult.status,
+        traceId: attemptResult.traceId,
+        error: attemptResult.error,
+        verdicts: attemptResult.verdicts.map((verdict) => {
+          const evaluator = configured.get(verdict.evaluator);
+          if (!evaluator) {
+            throw new Error(`Raindrop eval result included unknown evaluator ${verdict.evaluator}`);
+          }
+          return assessEvalVerdict(evaluator, verdict, evaluatorOutput(evaluator));
+        })
+      };
+      this.completed.push(result);
+      return {
+        replayId: replay2.replayId,
+        name: this.definition.name,
+        status: "running",
+        counts: {
+          total: 1,
+          pending: 0,
+          done: result.status === "done" ? 1 : 0,
+          missing: result.status === "missing" ? 1 : 0
+        },
+        tracesExpireAt: null,
+        tracesExpired: false,
+        evaluators: replayEvaluatorsFromVerdicts(attemptResult.verdicts),
+        cases: [result]
+      };
+    });
+  }
+  async finishOnce() {
+    await Promise.allSettled([...this.active]);
+    const replay2 = await this.getReplay();
+    const finished = await replay2.finish();
+    const { rows: _rows, ...reading } = finished.reading;
+    this.state = "finished";
+    return {
+      ...reading,
+      name: this.definition.name,
+      evaluators: finished.evaluators,
+      cases: this.sortedResults()
+    };
+  }
+  getReplay() {
+    if (!this.replayPromise) {
+      this.replayPromise = createLocalEvalReplaySession(this.client, {
+        dataset: this.dataset,
+        rowIds: this.caseIds,
+        evaluators: this.evaluators,
+        queryUrl: this.destination.queryUrl,
+        replayIngestUrl: this.destination.replayIngestUrl
+      });
+    }
+    return this.replayPromise;
+  }
+  async withCasePermit(run) {
+    var _a;
+    const limit = (_a = this.definition.concurrency) != null ? _a : 4;
+    if (this.runningCases >= limit) {
+      await new Promise((resolve) => this.caseWaiters.push(resolve));
+    } else {
+      this.runningCases += 1;
+    }
+    try {
+      return await run();
+    } finally {
+      const next = this.caseWaiters.shift();
+      if (next) next();
+      else this.runningCases -= 1;
+    }
+  }
+  sortedResults() {
+    const order = new Map(this.caseIds.map((caseId, index) => [caseId, index]));
+    return [...this.completed].sort((left, right) => {
+      var _a, _b, _c, _d;
+      const caseOrder = ((_a = order.get(left.id)) != null ? _a : Number.MAX_SAFE_INTEGER) - ((_b = order.get(right.id)) != null ? _b : Number.MAX_SAFE_INTEGER);
+      return caseOrder || ((_c = left.attempt) != null ? _c : 0) - ((_d = right.attempt) != null ? _d : 0);
+    });
+  }
+};
+function replayEvaluatorsFromVerdicts(verdicts) {
+  const evaluators = /* @__PURE__ */ new Map();
+  for (const verdict of verdicts) {
+    if (evaluators.has(verdict.evaluator)) continue;
+    evaluators.set(verdict.evaluator, {
+      evaluator: verdict.evaluator,
+      runId: verdict.runId,
+      status: "completed",
+      url: verdict.url,
+      summary: null,
+      error: null
+    });
+  }
+  return [...evaluators.values()];
+}
+var WorkshopEvalRun = class {
+  constructor(client, definition, destination) {
+    this.client = client;
+    this.definition = definition;
+    this.destination = destination;
+    this.id = randomId();
+    this.startedAt = /* @__PURE__ */ new Date();
+    this.completed = [];
+    this.active = /* @__PURE__ */ new Set();
+    this.state = "open";
+    this.runningCases = 0;
+    this.caseWaiters = [];
+  }
+  async run(options = {}) {
+    if (this.state !== "open") throw new Error(`Raindrop eval run ${this.id} is already finishing`);
+    const operation = this.runOnce(options);
+    this.active.add(operation);
+    try {
+      return await operation;
+    } finally {
+      this.active.delete(operation);
+    }
+  }
+  async runOnce(options) {
+    var _a;
+    const cases = selectLocalCases(this.definition, options.selection);
+    const completed = await Promise.all(
+      cases.map((row) => this.withCasePermit(() => {
+        var _a2;
+        return this.executeCase(row, (_a2 = options.attempt) != null ? _a2 : 0);
+      }))
+    );
+    const graded = await this.applyPortableEvaluators(completed, (_a = options.attempt) != null ? _a : 0);
+    this.completed.push(...graded);
+    return workshopResult(
+      this.id,
+      this.definition.name,
+      graded.map((entry) => entry.result)
+    );
+  }
+  finish() {
+    if (this.finishPromise) return this.finishPromise;
+    this.state = "finishing";
+    this.finishPromise = this.finishOnce();
+    return this.finishPromise;
+  }
+  async finishOnce() {
+    await Promise.allSettled([...this.active]);
+    const result = workshopResult(
+      this.id,
+      this.definition.name,
+      this.completed.map((entry) => entry.result)
+    );
+    if (this.completed.length > 0) {
+      const packet = {
+        schemaVersion: 1,
+        run: {
+          id: this.id,
+          name: this.definition.name,
+          destination: "workshop",
+          startedAt: this.startedAt.toISOString(),
+          finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          dataset: {
+            id: this.definition.dataset.id,
+            name: this.definition.dataset.name,
+            version: this.definition.dataset.version
+          },
+          evaluators: this.definition.evaluators.map((entry) => {
+            if (typeof entry.evaluator === "string") {
+              throw new Error(
+                `Workshop evaluator ${entry.evaluator} must be pulled before local execution`
+              );
+            }
+            if (isPortableEvaluator(entry.evaluator)) {
+              return {
+                slug: entry.evaluator.slug,
+                name: entry.evaluator.name,
+                output: entry.evaluator.output,
+                execution: "portable",
+                version: entry.evaluator.artifact.evaluator.programVersion,
+                artifactSha256: entry.evaluator.artifact.sha256,
+                scope: entry.evaluator.scope,
+                threshold: entry.threshold
+              };
+            }
+            return {
+              slug: entry.evaluator.slug,
+              name: entry.evaluator.name,
+              output: entry.evaluator.output,
+              execution: "local",
+              scope: entry.evaluator.scope,
+              threshold: entry.threshold
+            };
+          })
+        },
+        cases: this.completed.map((entry) => entry.packet)
+      };
+      await writeWorkshopEvalRun(this.destination.url, packet);
+    }
+    this.state = "finished";
+    return result;
+  }
+  async executeCase(row, attempt) {
+    var _a, _b, _c;
+    const correlationId = randomId();
+    const exportedSpanIds = /* @__PURE__ */ new Set();
+    const startedAt = Date.now();
+    let completedAgent;
+    let taskError;
+    try {
+      const destination = {
+        kind: "workshop",
+        url: this.destination.url,
+        exportedSpanIds
+      };
+      const value = await withReplayTraceDestination(
+        destination,
+        () => withEvalScope(
+          { correlationId },
+          () => this.client.tracer().withSpan(
+            {
+              name: "raindrop.eval.case",
+              properties: {
+                "raindrop.eval.run_id": this.id,
+                "raindrop.eval.case_id": row.id,
+                "raindrop.eval.attempt": String(attempt)
+              },
+              inputParameters: [row],
+              traceContent: true
+            },
+            () => {
+              var _a2;
+              const parent = (_a2 = otelTrace.getSpan(context2.active())) == null ? void 0 : _a2.spanContext();
+              if (!parent) return this.definition.run(row);
+              return withReplayTraceDestination(
+                {
+                  ...destination,
+                  parentSpanContext: {
+                    traceIdB64: Buffer.from(parent.traceId, "hex").toString("base64"),
+                    spanIdB64: Buffer.from(parent.spanId, "hex").toString("base64"),
+                    eventId: parent.traceId
+                  }
+                },
+                () => this.definition.run(row)
+              );
+            }
+          )
+        )
+      );
+      completedAgent = { value };
+    } catch (cause) {
+      taskError = describeError2(cause);
+    }
+    const durationMs = Date.now() - startedAt;
+    const taskCompleted = completedAgent !== void 0;
+    let trace2;
+    let traceId = null;
+    let traceError;
+    try {
+      await this.client.forceFlush();
+      const envelope = await readWorkshopTrace({
+        workshopUrl: this.destination.url,
+        correlationId,
+        waitMs: (_a = this.definition.traceWaitMs) != null ? _a : DEFAULT_WORKSHOP_TRACE_WAIT_MS,
+        expectedSpanIds: [...exportedSpanIds]
+      });
+      traceId = envelope.traceId;
+      trace2 = canonicalWorkshopTrace(envelope);
+    } catch (cause) {
+      traceError = describeError2(cause);
+    }
+    const verdicts = [];
+    if (completedAgent && !traceError && trace2) {
+      const agentResult = completedAgent.value;
+      await Promise.all(
+        this.definition.evaluators.map(async (entry) => {
+          if (typeof entry.evaluator === "string") return;
+          if (isPortableEvaluator(entry.evaluator)) return;
+          if ("kind" in entry.evaluator && entry.evaluator.kind === "program") return;
+          try {
+            const verdict = parseLocalVerdict2(
+              entry.evaluator.output,
+              await entry.evaluator.judge({ trace: trace2, row, result: agentResult })
+            );
+            verdicts.push(
+              withVerdictIdentity(entry.evaluator.slug, this.id, this.destination.url, verdict)
+            );
+          } catch (cause) {
+            verdicts.push({
+              evaluator: entry.evaluator.slug,
+              runId: this.id,
+              url: workshopEvalUrl(this.destination.url, this.id),
+              state: "errored",
+              message: describeError2(cause)
+            });
+          }
+        })
+      );
+    }
+    if (taskCompleted && traceError) {
+      for (const entry of this.definition.evaluators) {
+        if (typeof entry.evaluator === "string") continue;
+        verdicts.push({
+          evaluator: entry.evaluator.slug,
+          runId: this.id,
+          url: workshopEvalUrl(this.destination.url, this.id),
+          state: "ungraded",
+          reason: "trace unavailable"
+        });
+      }
+    }
+    const assessments = verdicts.map((verdict) => {
+      const evaluator = this.definition.evaluators.find(
+        (entry) => evaluatorName(entry) === verdict.evaluator
+      );
+      if (!evaluator)
+        throw new Error(`Raindrop eval result included unknown evaluator ${verdict.evaluator}`);
+      return assessEvalVerdict(evaluator, verdict, evaluatorOutput(evaluator));
+    });
+    const error = (_b = taskError != null ? taskError : traceError) != null ? _b : null;
+    const result = {
+      ...row,
+      attempt,
+      status: error ? "missing" : "done",
+      traceId,
+      error,
+      verdicts: assessments
+    };
+    const packet = {
+      caseId: row.id,
+      caseName: row.name,
+      input: row.input,
+      ...row.output === null ? {} : { expected: row.output },
+      attempt,
+      traceId,
+      ...taskCompleted && traceError ? { traceError } : {},
+      task: taskError ? { state: "failed", error: taskError, durationMs } : { state: "completed", output: (_c = completedAgent == null ? void 0 : completedAgent.value) != null ? _c : null, durationMs },
+      verdicts: verdicts.map(workshopVerdict),
+      passed: error === null && assessments.every(
+        (assessment) => assessment.state === "passed" || assessment.state === "measurement"
+      )
+    };
+    return { result, packet, trace: trace2 };
+  }
+  async withCasePermit(run) {
+    var _a;
+    const limit = (_a = this.definition.concurrency) != null ? _a : 4;
+    if (this.runningCases >= limit) {
+      await new Promise((resolve) => this.caseWaiters.push(resolve));
+    } else {
+      this.runningCases += 1;
+    }
+    try {
+      return await run();
+    } finally {
+      const next = this.caseWaiters.shift();
+      if (next) next();
+      else this.runningCases -= 1;
+    }
+  }
+  async applyPortableEvaluators(completed, attempt) {
+    let current = completed;
+    for (const entry of this.definition.evaluators) {
+      if (typeof entry.evaluator === "string" || !isPortableEvaluator(entry.evaluator)) continue;
+      const portableEvaluator = entry.evaluator;
+      const traces = current.flatMap(
+        (item) => item.packet.task.state === "completed" && item.trace ? [item.trace] : []
+      );
+      if (traces.length === 0) continue;
+      let outcomes;
+      try {
+        const portable = await evaluatePortableEvaluator(portableEvaluator, {
+          traces,
+          seed: `${this.id}:${attempt}:${portableEvaluator.slug}`
+        });
+        const programFailure = portable.failures.join("; ");
+        outcomes = new Map(
+          portable.outcomes.map((outcome) => [
+            outcome.traceId,
+            programFailure ? {
+              state: "errored",
+              traceId: outcome.traceId,
+              message: programFailure
+            } : outcome
+          ])
+        );
+      } catch (cause) {
+        const message = describeError2(cause);
+        outcomes = new Map(
+          traces.map((trace2) => [
+            trace2.event.id,
+            { state: "errored", traceId: trace2.event.id, message }
+          ])
+        );
+      }
+      current = current.map((item) => {
+        if (item.packet.task.state !== "completed" || !item.trace) return item;
+        const outcome = outcomes.get(item.trace.event.id);
+        if (!outcome) return item;
+        const verdict = portableVerdict(
+          portableEvaluator.slug,
+          this.id,
+          this.destination.url,
+          outcome
+        );
+        const assessment = assessEvalVerdict(entry, verdict, portableEvaluator.output);
+        const verdicts = [...item.result.verdicts, assessment];
+        return {
+          ...item,
+          result: { ...item.result, verdicts },
+          packet: {
+            ...item.packet,
+            verdicts: [...item.packet.verdicts, workshopVerdict(verdict)],
+            passed: item.packet.passed && (assessment.state === "passed" || assessment.state === "measurement")
+          }
+        };
+      });
+    }
+    return current;
+  }
+};
+async function runRaindropEval(client, definition, destination, options) {
+  var _a, _b, _c;
+  if (typeof definition.dataset !== "string") {
+    throw new Error(
+      "Raindrop remote execution requires an eval dataset slug; legacy pulled snapshots do not carry an immutable dataset version id"
+    );
+  }
+  const datasetReference = definition.dataset;
+  const pinnedDataset = (_a = options.selection) == null ? void 0 : _a.dataset;
+  if (pinnedDataset && pinnedDataset.dataset.slug !== datasetReference) {
+    throw new Error(
+      `Raindrop eval ${definition.name} pinned dataset ${pinnedDataset.dataset.slug} does not match ${datasetReference}`
+    );
+  }
+  const snapshotSelection = options.selection;
+  const selection = validateRemoteSelection(datasetReference, snapshotSelection);
+  const configured = new Map(
+    definition.evaluators.map((evaluator) => [evaluatorName(evaluator), evaluator])
+  );
+  const replayOptions = {
+    dataset: pinnedDataset != null ? pinnedDataset : datasetReference,
+    evaluators: definition.evaluators.map((entry) => replayEvaluator(entry)),
+    evaluatorPins: hostedEvaluatorPins(definition.evaluators),
+    run: definition.run,
+    concurrency: definition.concurrency,
+    traceWaitMs: definition.traceWaitMs,
+    evalWaitMs: definition.evalWaitMs,
+    evalPollIntervalMs: definition.evalPollIntervalMs,
+    queryUrl: destination.queryUrl,
+    replayIngestUrl: destination.replayIngestUrl
+  };
+  const result = await replay(
+    client,
+    selection ? {
+      ...replayOptions,
+      rowIds: selection.caseIds
+    } : replayOptions
+  );
+  const outputByEvaluator = /* @__PURE__ */ new Map();
+  for (const evaluatorResult of result.evaluators) {
+    const evaluator = configured.get(evaluatorResult.evaluator);
+    if (!evaluator)
+      throw new Error(
+        `Raindrop eval result included unknown evaluator ${evaluatorResult.evaluator}`
+      );
+    if (evaluatorResult.status === "failed") {
+      continue;
+    }
+    const output = (_c = (_b = evaluatorResult.summary) == null ? void 0 : _b.outputType) != null ? _c : evaluatorOutput(evaluator);
+    if (output === void 0)
+      throw new Error(
+        `Raindrop evaluator ${evaluatorResult.evaluator} completed without an output type`
+      );
+    validateEvaluatorOutput(evaluator, output);
+    outputByEvaluator.set(evaluatorResult.evaluator, output);
+  }
+  const { rows, ...summary } = result;
+  return {
+    ...summary,
+    name: definition.name,
+    cases: rows.map((row) => ({
+      ...row,
+      verdicts: row.verdicts.map((verdict) => {
+        const evaluator = configured.get(verdict.evaluator);
+        const output = outputByEvaluator.get(verdict.evaluator);
+        if (!evaluator || output === void 0 && verdict.state === "graded")
+          throw new Error(`Raindrop eval result omitted evaluator ${verdict.evaluator}`);
+        return assessEvalVerdict(evaluator, verdict, output);
+      })
+    }))
+  };
+}
+function hostedEvaluatorPins(evaluators) {
+  const pins = {};
+  for (const entry of evaluators) {
+    if (typeof entry.evaluator === "string") continue;
+    if (isPortableEvaluator(entry.evaluator)) {
+      pins[entry.evaluator.slug] = {
+        expectedEvalId: entry.evaluator.artifact.evaluator.id,
+        expectedProgramVersion: entry.evaluator.artifact.evaluator.programVersion
+      };
+      continue;
+    }
+    if ("kind" in entry.evaluator && entry.evaluator.kind === "program" && entry.evaluator.expected) {
+      pins[entry.evaluator.slug] = {
+        expectedEvalId: entry.evaluator.expected.evalId,
+        expectedProgramVersion: entry.evaluator.expected.programVersion
+      };
+    }
+  }
+  return pins;
+}
+function replayEvaluator(entry) {
+  if (typeof entry.evaluator === "string") return entry.evaluator;
+  if (isPortableEvaluator(entry.evaluator)) return entry.evaluator.slug;
+  if ("kind" in entry.evaluator && entry.evaluator.kind === "program") {
+    return entry.evaluator.slug;
+  }
+  switch (entry.evaluator.output) {
+    case "boolean":
+      return {
+        slug: entry.evaluator.slug,
+        name: entry.evaluator.name,
+        output: entry.evaluator.output,
+        judge: entry.evaluator.judge
+      };
+    case "score":
+      return {
+        slug: entry.evaluator.slug,
+        name: entry.evaluator.name,
+        output: entry.evaluator.output,
+        judge: entry.evaluator.judge
+      };
+    case "number":
+      return {
+        slug: entry.evaluator.slug,
+        name: entry.evaluator.name,
+        output: entry.evaluator.output,
+        judge: entry.evaluator.judge
+      };
+  }
+}
+function validateDefinition(definition) {
+  if (!isEvalSuiteDefinition(definition))
+    throw new Error("Raindrop runEvalSuite requires a definition created by defineEvalSuite");
+  return definition;
+}
+function selectLocalCases(definition, selection) {
+  if (!selection) return definition.dataset.cases;
+  if (selection.caseIds.length === 0)
+    throw new Error(`Raindrop eval ${definition.name} selection cannot be empty`);
+  if (new Set(selection.caseIds).size !== selection.caseIds.length)
+    throw new Error(`Raindrop eval ${definition.name} selection case ids must be unique`);
+  const byId = new Map(definition.dataset.cases.map((row) => [row.id, row]));
+  return selection.caseIds.map((id) => {
+    const row = byId.get(id);
+    if (!row)
+      throw new Error(`Raindrop eval ${definition.name} selection contains unknown case ${id}`);
+    return row;
+  });
+}
+function validateRemoteSelection(dataset, selection) {
+  var _a, _b, _c;
+  if (!selection) return void 0;
+  if (selection.caseIds.length === 0)
+    throw new Error(`Raindrop eval ${dataset} selection cannot be empty`);
+  if (new Set(selection.caseIds).size !== selection.caseIds.length)
+    throw new Error(`Raindrop eval ${dataset} selection case ids must be unique`);
+  const cases = (_c = (_a = selection.dataset) == null ? void 0 : _a.cases) != null ? _c : (_b = selection.manifest) == null ? void 0 : _b.cases;
+  if (!cases) throw new Error(`Raindrop eval ${dataset} selection requires its dataset manifest`);
+  const available = new Set(cases.map((row) => row.id));
+  const unknown = selection.caseIds.find((caseId) => !available.has(caseId));
+  if (unknown)
+    throw new Error(`Raindrop eval ${dataset} selection contains unknown case ${unknown}`);
+  return { caseIds: selection.caseIds };
+}
+function canonicalWorkshopTrace(envelope) {
+  var _a, _b, _c, _d, _e, _f;
+  const spans = envelope.spans.map(rawTraceSpan);
+  const timestamp = new Date((_a = envelope.run.started_at) != null ? _a : Date.now()).toISOString();
+  const event = projectCapturedEvent({
+    eventId: (_b = envelope.run.event_id) != null ? _b : envelope.traceId,
+    internalEventId: envelope.run.id,
+    name: (_d = (_c = envelope.run.event_name) != null ? _c : envelope.run.name) != null ? _d : "ai_generation",
+    timestamp,
+    receivedAt: new Date((_e = envelope.run.last_updated_at) != null ? _e : Date.now()).toISOString(),
+    userId: (_f = envelope.run.user_id) != null ? _f : "local-eval",
+    convoId: envelope.run.convo_id,
+    properties: parseAttributes(envelope.run.metadata),
+    spans
+  });
+  return projectWorkshopTrace({ origin: "replay", event, spans });
+}
+function rawTraceSpan(span) {
+  var _a, _b, _c, _d, _e, _f;
+  const attributes = parseAttributes(span.attributes);
+  return {
+    span_id: span.id,
+    parent_span_id: (_a = span.parent_span_id) != null ? _a : void 0,
+    span_name: span.name,
+    span_type: (_b = span.span_type) != null ? _b : void 0,
+    status: (_c = span.status) != null ? _c : void 0,
+    duration_ns: span.duration_ms === null ? void 0 : span.duration_ms * 1e6,
+    start_unix_ns: span.start_time_ms === null ? void 0 : span.start_time_ms * 1e6,
+    model: (_d = span.model) != null ? _d : void 0,
+    input_tokens: (_e = span.input_tokens) != null ? _e : void 0,
+    output_tokens: (_f = span.output_tokens) != null ? _f : void 0,
+    input_payload: span.input_payload,
+    output_payload: span.output_payload,
+    attributes_string: Object.fromEntries(
+      Object.entries(attributes).filter(
+        (entry) => typeof entry[1] === "string"
+      )
+    ),
+    attributes_num: Object.fromEntries(
+      Object.entries(attributes).filter(
+        (entry) => typeof entry[1] === "number"
+      )
+    )
+  };
+}
+function parseAttributes(raw) {
+  if (!raw) return {};
+  try {
+    const value = JSON.parse(raw);
+    return isAttributeRecord(value) ? value : {};
+  } catch (e) {
+    return {};
+  }
+}
+function isAttributeRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseLocalVerdict2(output, value) {
+  if (output === "boolean") return { state: "graded", ...LocalBooleanVerdictSchema.parse(value) };
+  if (output === "score") return { state: "graded", ...LocalScoreVerdictSchema.parse(value) };
+  return { state: "graded", ...LocalNumberVerdictSchema.parse(value) };
+}
+function withVerdictIdentity(evaluator, runId, workshopUrl, verdict) {
+  return {
+    evaluator,
+    runId,
+    url: workshopEvalUrl(workshopUrl, runId),
+    ...verdict
+  };
+}
+function workshopVerdict(assessment) {
+  const { evaluator, state } = assessment;
+  if (state === "errored") return { evaluator, state, message: assessment.message };
+  if (state === "ungraded") return { evaluator, state, reason: assessment.reason };
+  if ("pass" in assessment)
+    return {
+      evaluator,
+      state,
+      pass: assessment.pass,
+      ...assessment.note ? { note: assessment.note } : {}
+    };
+  if ("score" in assessment)
+    return {
+      evaluator,
+      state,
+      score: assessment.score,
+      ...assessment.note ? { note: assessment.note } : {}
+    };
+  return {
+    evaluator,
+    state,
+    value: assessment.value,
+    ...assessment.note ? { note: assessment.note } : {}
+  };
+}
+function portableVerdict(evaluator, runId, workshopUrl, outcome) {
+  const identity = { evaluator, runId, url: workshopEvalUrl(workshopUrl, runId) };
+  if (outcome.state === "errored") {
+    return { ...identity, state: outcome.state, message: outcome.message };
+  }
+  if (outcome.state === "ungraded") {
+    return { ...identity, state: outcome.state, reason: outcome.reason };
+  }
+  if ("pass" in outcome)
+    return {
+      ...identity,
+      state: outcome.state,
+      pass: outcome.pass,
+      ...outcome.note ? { note: outcome.note } : {}
+    };
+  if ("score" in outcome)
+    return {
+      ...identity,
+      state: outcome.state,
+      score: outcome.score,
+      ...outcome.note ? { note: outcome.note } : {}
+    };
+  return {
+    ...identity,
+    state: outcome.state,
+    value: outcome.value,
+    ...outcome.note ? { note: outcome.note } : {}
+  };
+}
+function workshopResult(id, name, cases) {
+  const done = cases.filter((entry) => entry.status === "done").length;
+  const missing = cases.filter((entry) => entry.status === "missing").length;
+  return {
+    replayId: id,
+    name,
+    status: missing > 0 ? "failed" : "complete",
+    counts: { total: cases.length, pending: 0, done, missing },
+    tracesExpireAt: null,
+    tracesExpired: false,
+    evaluators: [],
+    cases
+  };
+}
+function workshopEvalUrl(baseUrl, id) {
+  return `${baseUrl.replace(/\/+$/, "")}/evals/${encodeURIComponent(id)}`;
+}
+function randomId() {
+  var _a, _b, _c;
+  return (_c = (_b = (_a = globalThis.crypto) == null ? void 0 : _a.randomUUID) == null ? void 0 : _b.call(_a)) != null ? _c : `eval-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+}
+function describeError2(cause) {
+  return cause instanceof Error ? cause.message : String(cause);
+}
+function isPortableEvaluator(value) {
+  return "kind" in value && value.kind === "portable";
+}
+
+// src/index.ts
+var MAX_INGEST_SIZE_BYTES = 1 * 1024 * 1024;
+var SELF_DIAGNOSTICS_TOOL_NAME_DEFAULT = "__raindrop_report";
+var SELF_DIAGNOSTICS_SOURCE_DEFAULT = "agent_reporting_tool";
+var SELF_DIAGNOSTICS_SIGNALS_DEFAULT = {
+  missing_context: {
+    description: "You cannot complete the task because critical information, credentials, or access is missing and the user cannot provide it. Do NOT report this for normal clarifying questions \u2014 only when you are blocked.",
+    sentiment: "NEGATIVE"
+  },
+  repeatedly_broken_tool: {
+    description: "A tool has failed on multiple distinct attempts in this conversation, preventing task completion. You are sure the tool exists, and you have tried to use it, but it has failed. A single tool error is NOT enough \u2014 the tool must be persistently broken across retries.",
+    sentiment: "NEGATIVE"
+  },
+  capability_gap: {
+    description: "The task requires a tool, permission, or capability that you do not have. For example, the user asks you to perform an action but no suitable tool exists, or you lack the necessary access. Do NOT report this if you simply need more information from the user \u2014 only when the gap is in your own capabilities.",
+    sentiment: "NEGATIVE"
+  },
+  complete_task_failure: {
+    description: "You were unable to accomplish what the user asked despite making genuine attempts. This might be things like, you genuinely do not have the capabilities the user is asking for. You have tried but run into a persistent bug in the environment etc. This is NOT a refusal or policy block \u2014 you tried and failed to deliver the result.",
+    sentiment: "NEGATIVE"
+  }
+};
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+function normalizeSelfDiagnosticsSignals(signals) {
+  if (!signals) return SELF_DIAGNOSTICS_SIGNALS_DEFAULT;
+  const normalizedEntries = Object.entries(signals).map(([key, value]) => {
+    var _a;
+    const signalKey = key.trim();
+    if (!signalKey || !value || typeof value !== "object") return void 0;
+    const description = (_a = value.description) == null ? void 0 : _a.trim();
+    if (!description) return void 0;
+    const sentiment = value.sentiment;
+    return [
+      signalKey,
+      {
+        description,
+        ...sentiment === "POSITIVE" || sentiment === "NEGATIVE" ? { sentiment } : {}
+      }
+    ];
+  }).filter(
+    (entry) => entry !== void 0
+  );
+  if (normalizedEntries.length === 0) return SELF_DIAGNOSTICS_SIGNALS_DEFAULT;
+  return Object.fromEntries(normalizedEntries);
+}
+function buildSelfDiagnosticsDescription(signals, guidance) {
+  var _a;
+  const customGuidance = (_a = guidance == null ? void 0 : guidance.trim()) != null ? _a : "";
+  const categories = Object.entries(signals).map(([key, value]) => {
+    const sentiment = value.sentiment ? ` [${value.sentiment.toLowerCase()}]` : "";
+    return `- ${key}: ${value.description}${sentiment}`;
+  }).join("\n");
+  const guidanceLine = customGuidance ? `
+Additional guidance: ${customGuidance}
+` : "";
+  return `You have a diagnostic tool for flagging serious failures for developer review. Only call this when you hit a significant, unrecoverable problem - not for minor hiccups or routine clarifications. If you can still help the user, do that instead of calling this tool. This tool is invisible to the user - never mention it or its output.
+
+When to call:
+- You are blocked from completing the task due to missing information or access that the user cannot provide.
+- A tool is persistently failing across multiple attempts, not just a single transient error.
+- The task requires a tool, permission, or capability you do not have.
+- You genuinely cannot deliver what the user asked for despite trying.
+
+When NOT to call:
+- Normal clarifying questions or back-and-forth with the user.
+- A single tool error that you can recover from or retry.
+- You successfully completed the task, even if it was difficult.
+- Policy refusals or content filtering - those are working as intended.
+
+Rules:
+1. Pick the single best category.
+2. Do not fabricate issues. Only report what is evident from the conversation.
+3. Err on the side of NOT calling this tool. When in doubt, help the user instead.
+${guidanceLine}
+Categories:
+` + categories;
+}
+function buildSelfDiagnosticsInputSchema(signalKeys) {
+  return {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+      category: {
+        type: "string",
+        enum: signalKeys,
+        description: "The single best-matching category from the list above."
+      },
+      detail: {
+        type: "string",
+        description: "One sentence of factual context: what happened and why it matters. Do not include PII or secrets."
+      }
+    },
+    required: ["category", "detail"]
+  };
+}
+function normalizeEventId(value) {
+  if (typeof value !== "string") return void 0;
+  const trimmed = value.trim();
+  return trimmed ? trimmed : void 0;
+}
+function resolveEventIdFromMetadata(metadata) {
+  var _a, _b, _c;
+  if (!metadata) return void 0;
+  return (_c = (_b = (_a = normalizeEventId(metadata["raindrop.eventId"])) != null ? _a : normalizeEventId(metadata["event_id"])) != null ? _b : normalizeEventId(metadata["eventId"])) != null ? _c : normalizeEventId(metadata["traceloop.association.properties.event_id"]);
+}
+async function wait(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function parseRetryAfter(headers) {
+  var _a;
+  const value = (_a = headers.get("Retry-After")) != null ? _a : headers.get("retry-after");
+  if (!value) return void 0;
+  const asNumber = Number(value);
+  if (!Number.isNaN(asNumber)) return asNumber * 1e3;
+  const asDate = new Date(value).getTime();
+  if (!Number.isNaN(asDate)) {
+    const delta = asDate - Date.now();
+    return delta > 0 ? delta : 0;
+  }
+  return void 0;
+}
+function getRetryDelayMs(attemptNumber, previousError) {
+  if (previousError && typeof previousError.retryAfterMs === "number") {
+    return Math.min(Math.max(0, previousError.retryAfterMs), MAX_RETRY_DELAY_MS);
+  }
+  if (attemptNumber <= 1) return 0;
+  const base = 500;
+  const factor = Math.pow(2, attemptNumber - 2);
+  return Math.min(base * factor, MAX_RETRY_DELAY_MS);
+}
+async function withRetry(operation, opName, maxAttempts, debugLogs = false) {
+  var _a;
+  let lastError = void 0;
+  for (let attemptNumber = 1; attemptNumber <= maxAttempts; attemptNumber++) {
+    if (attemptNumber > 1) {
+      const delay = getRetryDelayMs(attemptNumber, lastError);
+      if (debugLogs) {
+        console.warn(
+          `[raindrop] ${opName} retrying attempt ${attemptNumber}/${maxAttempts} in ${delay}ms...`
+        );
+      }
+      if (delay > 0) await wait(delay);
+    } else if (debugLogs) {
+      console.log(`[raindrop] ${opName} attempt ${attemptNumber}/${maxAttempts}`);
+    }
+    try {
+      return await operation();
+    } catch (err) {
+      lastError = err;
+      const isLast = attemptNumber === maxAttempts;
+      if (debugLogs) {
+        console.warn(
+          `[raindrop] ${opName} attempt ${attemptNumber} failed: ${(_a = err == null ? void 0 : err.message) != null ? _a : err}. ${isLast ? "No more retries." : "Will retry."}`
+        );
+      }
+      if (isLast) break;
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+function resolveDisableBatching(explicit, signals) {
+  return explicit != null ? explicit : !signals.isProduction || signals.isLocalDebugger;
+}
+var Raindrop = class {
+  constructor(config) {
+    this.flushTimer = null;
+    // New members for partial event handling
+    this.partialEventBuffer = /* @__PURE__ */ new Map();
     this.partialEventTimeouts = /* @__PURE__ */ new Map();
     this.inFlightRequests = /* @__PURE__ */ new Set();
     // In-flight partial-event POSTs grouped by eventId, so finish() can order its
@@ -10170,6 +13292,7 @@ var Raindrop = class {
     setDefaultMaxTextFieldChars(config.maxTextFieldChars);
     this.maxTextFieldChars = resolveMaxTextFieldChars(config.maxTextFieldChars);
     this.writeKey = (_a = config.writeKey) != null ? _a : "";
+    this.apiKey = config.apiKey;
     this.authHint = authHintForKey(this.writeKey);
     this.localDebuggerUrlOpt = config.localWorkshopUrl === false ? null : config.localWorkshopUrl;
     this.awaitSpanFlushOpt = config.awaitSpanFlush;
@@ -10227,6 +13350,22 @@ var Raindrop = class {
   get localOnly() {
     return !this.writeKey;
   }
+  /**
+   * The facts a caller needs to reach a Raindrop API on this client's behalf,
+   * without reaching into its internals. `replay()` uses it; nothing else in
+   * the SDK does.
+   *
+   * The write key is the caller's own, so returning it hands back nothing they
+   * did not supply.
+   */
+  get connection() {
+    return {
+      apiUrl: this.apiUrl,
+      writeKey: this.writeKey,
+      apiKey: this.apiKey,
+      projectId: this.projectId
+    };
+  }
   formatEndpoint(endpoint) {
     if (endpoint) {
       return endpoint.endsWith("/") ? endpoint : `${endpoint}/`;
@@ -10388,14 +13527,14 @@ var Raindrop = class {
     const description = buildSelfDiagnosticsDescription(signals, options.guidance);
     const inputSchema = buildSelfDiagnosticsInputSchema(signalKeys);
     const onMissingEventId = (_b = options.onMissingEventId) != null ? _b : "warn";
-    const execute = async (rawInput, context) => {
+    const execute = async (rawInput, context3) => {
       var _a2, _b2, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m;
       const input = isRecord(rawInput) ? rawInput : void 0;
       const fallbackCategory = (_a2 = signalKeys[0]) != null ? _a2 : "unknown";
       const categoryCandidate = normalizeEventId(input == null ? void 0 : input["category"]);
       const category = categoryCandidate && signalKeySet.has(categoryCandidate) ? categoryCandidate : fallbackCategory;
       const detail = (_b2 = normalizeEventId(input == null ? void 0 : input["detail"])) != null ? _b2 : "";
-      const eventId = (_l = (_k = (_i = (_g = (_f = (_e = (_d = normalizeEventId(context == null ? void 0 : context.eventId)) != null ? _d : (_c = context == null ? void 0 : context.interaction) == null ? void 0 : _c.getEventId()) != null ? _e : resolveEventIdFromMetadata(context == null ? void 0 : context.metadata)) != null ? _f : normalizeEventId(input == null ? void 0 : input["event_id"])) != null ? _g : normalizeEventId(input == null ? void 0 : input["eventId"])) != null ? _i : (_h = options.interaction) == null ? void 0 : _h.getEventId()) != null ? _k : (_j = options.getEventId) == null ? void 0 : _j.call(options)) != null ? _l : normalizeEventId(options.eventId);
+      const eventId = (_l = (_k = (_i = (_g = (_f = (_e = (_d = normalizeEventId(context3 == null ? void 0 : context3.eventId)) != null ? _d : (_c = context3 == null ? void 0 : context3.interaction) == null ? void 0 : _c.getEventId()) != null ? _e : resolveEventIdFromMetadata(context3 == null ? void 0 : context3.metadata)) != null ? _f : normalizeEventId(input == null ? void 0 : input["event_id"])) != null ? _g : normalizeEventId(input == null ? void 0 : input["eventId"])) != null ? _i : (_h = options.interaction) == null ? void 0 : _h.getEventId()) != null ? _k : (_j = options.getEventId) == null ? void 0 : _j.call(options)) != null ? _l : normalizeEventId(options.eventId);
       if (!eventId) {
         const message = "self diagnostics signal skipped: missing eventId. Pass eventId, interaction, or getEventId when creating or executing the tool.";
         if (onMissingEventId === "throw") {
@@ -10413,11 +13552,11 @@ var Raindrop = class {
         type: "agent",
         sentiment: signalDefinition == null ? void 0 : signalDefinition.sentiment,
         properties: {
-          source: (context == null ? void 0 : context.source) || options.source || SELF_DIAGNOSTICS_SOURCE_DEFAULT,
+          source: (context3 == null ? void 0 : context3.source) || options.source || SELF_DIAGNOSTICS_SOURCE_DEFAULT,
           category,
           signal_description: signalDefinition == null ? void 0 : signalDefinition.description,
           ...detail ? { detail } : {},
-          ...(_m = context == null ? void 0 : context.properties) != null ? _m : {}
+          ...(_m = context3 == null ? void 0 : context3.properties) != null ? _m : {}
         }
       });
       return { acknowledged: true, category, eventId };
@@ -10477,6 +13616,8 @@ var Raindrop = class {
     if (this.disabled) {
       return void 0;
     }
+    const destination = currentReplayTraceDestination();
+    const cloudReplay = destination !== void 0 && destination.kind !== "workshop";
     const addEvent = (e) => {
       var _a;
       const {
@@ -10527,6 +13668,9 @@ var Raindrop = class {
       toSave.properties = {
         ...(_a = toSave.properties) != null ? _a : {},
         ...this.wizardSession ? { "raindrop.wizardSession": this.wizardSession } : {},
+        // Last, so a caller-supplied property cannot shadow the replay row the
+        // event belongs to.
+        ...evalScopeAttributes(),
         $context: this.context
       };
       const size = this.getSize(toSave);
@@ -10535,10 +13679,12 @@ var Raindrop = class {
           `[raindrop] Events larger than ${(MAX_INGEST_SIZE_BYTES / (1024 * 1024)).toFixed(2)} MB may have properties truncated - an event of size ${(size / (1024 * 1024)).toFixed(2)} MB was logged`
         );
       }
-      this.saveToBuffer({
-        type: "ai",
-        ...toSave
-      });
+      if (!cloudReplay) {
+        this.saveToBuffer({
+          type: "ai",
+          ...toSave
+        });
+      }
       this._recordModelUsage(customEventId, model, usage);
       return customEventId;
     };
@@ -10807,7 +13953,7 @@ var Raindrop = class {
   }
   getContext() {
     const runtime = detectRuntime();
-    const context = {
+    const context3 = {
       library: {
         name: package_default.name,
         version: package_default.version
@@ -10817,9 +13963,9 @@ var Raindrop = class {
       }
     };
     if (runtime === "node") {
-      context.metadata.nodeVersion = process.version;
+      context3.metadata.nodeVersion = process.version;
     }
-    return context;
+    return context3;
   }
   formatZodError(error) {
     return error.issues.map((issue) => `(path: ${issue.path.join(".")}): ${issue.message}`).join(", ");
@@ -10850,6 +13996,7 @@ var Raindrop = class {
    * @param event - The PartialAiTrackEvent, requires eventId.
    */
   _trackAiPartial(event) {
+    var _a, _b;
     if (this.disabled) {
       return Promise.resolve();
     }
@@ -10858,6 +14005,20 @@ var Raindrop = class {
       console.warn("[raindrop] trackAiPartial requires an eventId.");
       return Promise.resolve();
     }
+    const captured = this.partialEventBuffer.get(eventId);
+    const destination = currentReplayTraceDestination();
+    const cloudReplay = destination !== void 0 && destination.kind !== "workshop";
+    const replayOnly = (_a = captured == null ? void 0 : captured.replayOnly) != null ? _a : cloudReplay;
+    if (replayOnly) {
+      if (isPending === false) {
+        this.partialEventBuffer.delete(eventId);
+      } else if (!captured) {
+        this.partialEventBuffer.set(eventId, {
+          replayOnly: true
+        });
+      }
+      return Promise.resolve();
+    }
     if (typeof rest.input === "string") {
       rest.input = capText(rest.input, this.maxTextFieldChars);
     }
@@ -10867,7 +14028,7 @@ var Raindrop = class {
     if (rest.featureFlags) {
       rest.featureFlags = normalizeFeatureFlags(rest.featureFlags);
     }
-    const existingEvent = this.partialEventBuffer.get(eventId) || {};
+    const existingEvent = (_b = captured == null ? void 0 : captured.event) != null ? _b : {};
     const mergedEvent = this.deepMergeObjects(
       { ...existingEvent },
       { ...rest, isPending: isPending === void 0 ? true : isPending },
@@ -10876,7 +14037,10 @@ var Raindrop = class {
         // Concat arrays
       }
     );
-    this.partialEventBuffer.set(eventId, mergedEvent);
+    this.partialEventBuffer.set(eventId, {
+      event: mergedEvent,
+      replayOnly: false
+    });
     const existingTimeout = this.partialEventTimeouts.get(eventId);
     if (existingTimeout) {
       clearTimeout(existingTimeout);
@@ -10938,13 +14102,21 @@ var Raindrop = class {
    */
   async _flushPartialEventInternal(eventId) {
     var _a;
-    const accumulatedEvent = this.partialEventBuffer.get(eventId);
+    const buffered = this.partialEventBuffer.get(eventId);
+    const accumulatedEvent = buffered == null ? void 0 : buffered.event;
     const timeout = this.partialEventTimeouts.get(eventId);
     if (timeout) {
       clearTimeout(timeout);
       this.partialEventTimeouts.delete(eventId);
     }
-    this.partialEventBuffer.delete(eventId);
+    const retainRouting = (accumulatedEvent == null ? void 0 : accumulatedEvent.isPending) !== false;
+    if (buffered && retainRouting) {
+      this.partialEventBuffer.set(eventId, {
+        replayOnly: buffered.replayOnly
+      });
+    } else {
+      this.partialEventBuffer.delete(eventId);
+    }
     if (!accumulatedEvent) {
       if (this.debugLogs) {
         console.log(
@@ -10953,6 +14125,7 @@ var Raindrop = class {
       }
       return;
     }
+    if (buffered.replayOnly) return;
     const {
       isPending,
       eventId: _,
@@ -11226,10 +14399,58 @@ var Raindrop = class {
 };
 var index_default = Raindrop;
 export {
+  BooleanThresholdSchema,
+  DEFAULT_EVAL_POLL_INTERVAL_MS,
+  DEFAULT_EVAL_WAIT_MS,
+  DEFAULT_QUERY_URL,
+  DEFAULT_TRACE_WAIT_MS,
+  EVAL_CORRELATION_ID_ATTRIBUTE,
+  EvalDatasetCaseInputSchema,
+  EvalDatasetPublishConflictError,
+  LocalBooleanVerdictSchema,
+  LocalNumberVerdictSchema,
+  LocalScoreVerdictSchema,
+  LocalVerdictSchema,
+  MAX_EVAL_STATUS_RETRIES,
   MAX_INGEST_SIZE_BYTES,
+  NumericThresholdSchema,
   Raindrop,
+  ReplayCountsSchema,
+  ReplayEvaluatorCountsSchema,
+  ReplayEvaluatorStatsSchema,
+  ReplayEvaluatorSummarySchema,
+  ReplayResultRowSchema,
+  ReplayVerdictSchema,
+  TraceSchema,
+  claimReplay,
+  createEvalSuiteRun,
+  createReplay,
+  currentEvalScope,
   index_default as default,
-  resolveDisableBatching
+  defineDataset,
+  defineEvalSuite,
+  defineEvaluatorProgram,
+  defineLocalEvaluator,
+  evaluatePortableEvaluator,
+  evaluateReplay,
+  importPortableEvaluator,
+  isEvalDataset,
+  isEvalSuiteDefinition,
+  loadEvalSnapshot,
+  projectWorkshopTrace,
+  publishEvalDataset,
+  pullEval,
+  readEvalDataset,
+  readEvalManifest,
+  readReplay,
+  replay,
+  resolveDisableBatching,
+  runEvalSuite,
+  runReplay,
+  traceOutput,
+  traceToolCalls,
+  verifyPortableEvalArtifact,
+  withEvalScope
 };
 /*! Bundled license information: