raindrop-ai 0.4.0 → 0.5.0

package/dist/portable-runtime-GBE3SHBZ.mjs

@@ -0,0 +1,193 @@
+import {
+  parsePortableEvalResult
+} from "./chunk-HFHHGRTW.mjs";
+import "./chunk-UJCSKKID.mjs";
+
+// src/evals/portable-runtime.ts
+import variant from "@jitl/quickjs-singlefile-cjs-release-sync";
+import {
+  newQuickJSWASMModuleFromVariant,
+  shouldInterruptAfterDeadline
+} from "quickjs-emscripten-core";
+import { z } from "zod";
+var DEFAULT_MEMORY_BYTES = 32 * 1024 * 1024;
+var DEFAULT_STACK_BYTES = 64 * 1024;
+var DEFAULT_TIMEOUT_MS = 1e3;
+var DEFAULT_JUDGE_TIMEOUT_MS = 6e4;
+var DEFAULT_JUDGE_REQUEST_TIMEOUT_MS = 3e4;
+var DEFAULT_JUDGE_CONCURRENCY = 4;
+var DEFAULT_MAX_INPUT_BYTES = 4 * 1024 * 1024;
+var DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+var JudgeRequestSchema = z.object({
+  type: z.literal("judge_request"),
+  requestId: z.string(),
+  traceId: z.string(),
+  rubric: z.string()
+});
+async function runPortableEvaluator(evaluator, input) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+  const inputJson = JSON.stringify(input);
+  const maxInputBytes = (_b = (_a = evaluator.limits) == null ? void 0 : _a.maxInputBytes) != null ? _b : DEFAULT_MAX_INPUT_BYTES;
+  if (byteLength(inputJson) > maxInputBytes) {
+    throw new Error(`Raindrop portable evaluator input exceeds ${maxInputBytes} bytes`);
+  }
+  const QuickJS = await newQuickJSWASMModuleFromVariant(variant);
+  const runtime = QuickJS.newRuntime();
+  runtime.setMemoryLimit((_d = (_c = evaluator.limits) == null ? void 0 : _c.memoryBytes) != null ? _d : DEFAULT_MEMORY_BYTES);
+  runtime.setMaxStackSize((_f = (_e = evaluator.limits) == null ? void 0 : _e.stackBytes) != null ? _f : DEFAULT_STACK_BYTES);
+  const timeoutMs = (_h = (_g = evaluator.limits) == null ? void 0 : _g.timeoutMs) != null ? _h : evaluator.artifact.evaluator.executionMode === "judge" ? DEFAULT_JUDGE_TIMEOUT_MS : DEFAULT_TIMEOUT_MS;
+  const deadline = Date.now() + timeoutMs;
+  runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadline));
+  const context = runtime.newContext();
+  const deferreds = [];
+  const hostCalls = /* @__PURE__ */ new Set();
+  const tracesById = new Map(input.traces.map((trace) => [trace.event.id, trace]));
+  const casesByTraceId = new Map(
+    input.cases.map((entry) => [entry.candidate.trace.event.id, entry])
+  );
+  let activeJudgeCalls = 0;
+  try {
+    const bridge = context.newFunction("__raindrop_eval_judge", (requestHandle) => {
+      var _a2, _b2;
+      const deferred = context.newPromise();
+      deferreds.push(deferred);
+      const requestJson = context.getString(requestHandle);
+      const judgeConcurrency = (_b2 = (_a2 = evaluator.limits) == null ? void 0 : _a2.judgeConcurrency) != null ? _b2 : DEFAULT_JUDGE_CONCURRENCY;
+      let acquired = false;
+      const hostCall = resolveJudge(
+        evaluator,
+        requestJson,
+        Math.max(1, deadline - Date.now()),
+        tracesById,
+        casesByTraceId,
+        () => {
+          if (activeJudgeCalls >= judgeConcurrency) return false;
+          activeJudgeCalls += 1;
+          acquired = true;
+          return true;
+        }
+      ).then((responseJson) => {
+        const response = context.newString(responseJson);
+        try {
+          deferred.resolve(response);
+        } finally {
+          response.dispose();
+        }
+      }).finally(() => {
+        if (acquired) activeJudgeCalls -= 1;
+        hostCalls.delete(hostCall);
+      });
+      hostCalls.add(hostCall);
+      return deferred.handle;
+    });
+    context.setProp(context.global, "__raindrop_eval_judge", bridge);
+    bridge.dispose();
+    const loaded = context.evalCode(evaluator.artifact.source, "raindrop-evaluator.js");
+    context.unwrapResult(loaded).dispose();
+    const entrypoint = context.getProp(context.global, "__raindrop_eval_run");
+    const inputHandle = context.newString(inputJson);
+    let promiseHandle;
+    try {
+      promiseHandle = context.unwrapResult(
+        context.callFunction(entrypoint, context.global, inputHandle)
+      );
+    } finally {
+      inputHandle.dispose();
+      entrypoint.dispose();
+    }
+    try {
+      const result = await drivePromise(runtime, context, promiseHandle, deadline);
+      const outputHandle = context.unwrapResult(result);
+      try {
+        const outputJson = context.getString(outputHandle);
+        const maxOutputBytes = (_j = (_i = evaluator.limits) == null ? void 0 : _i.maxOutputBytes) != null ? _j : DEFAULT_MAX_OUTPUT_BYTES;
+        if (byteLength(outputJson) > maxOutputBytes) {
+          throw new Error(`Raindrop portable evaluator output exceeds ${maxOutputBytes} bytes`);
+        }
+        const envelope = JSON.parse(outputJson);
+        if (typeof envelope !== "object" || envelope === null || !("type" in envelope) || envelope.type !== "result" || !("result" in envelope)) {
+          throw new Error("Raindrop portable evaluator returned an invalid result envelope");
+        }
+        return parsePortableEvalResult(envelope.result);
+      } finally {
+        outputHandle.dispose();
+      }
+    } finally {
+      promiseHandle.dispose();
+    }
+  } catch (cause) {
+    const message = cause instanceof Error ? cause.message : String(cause);
+    if (Date.now() >= deadline || message.includes("interrupted")) {
+      throw new Error(`Raindrop portable evaluator exceeded ${timeoutMs}ms`);
+    }
+    throw new Error(`Raindrop portable evaluator failed: ${message}`);
+  } finally {
+    await Promise.allSettled([...hostCalls]);
+    for (const deferred of deferreds) deferred.dispose();
+    context.dispose();
+    runtime.dispose();
+  }
+}
+async function drivePromise(runtime, context, promise, deadline) {
+  while (true) {
+    runtime.executePendingJobs(100).unwrap();
+    const state = context.getPromiseState(promise);
+    if (state.type === "fulfilled") return { value: state.value };
+    if (state.type === "rejected") return { error: state.error };
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    if (Date.now() >= deadline) throw new Error("interrupted");
+  }
+}
+async function resolveJudge(evaluator, rawRequest, remainingMs, tracesById, casesByTraceId, acquire) {
+  var _a, _b;
+  let requestId = "unknown";
+  try {
+    const request = JudgeRequestSchema.parse(JSON.parse(rawRequest));
+    requestId = request.requestId;
+    const trace = tracesById.get(request.traceId);
+    if (!trace) throw new Error("Judge trace is outside this run");
+    if (!evaluator.judge) throw new Error("Judge unavailable");
+    if (!acquire()) throw new Error("Judge concurrency exceeded");
+    const controller = new AbortController();
+    const timeoutMs = Math.min(
+      (_b = (_a = evaluator.limits) == null ? void 0 : _a.judgeTimeoutMs) != null ? _b : DEFAULT_JUDGE_REQUEST_TIMEOUT_MS,
+      remainingMs
+    );
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    const result = await Promise.race([
+      Promise.resolve().then(
+        () => {
+          var _a2;
+          return (_a2 = evaluator.judge) == null ? void 0 : _a2.call(evaluator, {
+            trace,
+            tracePair: casesByTraceId.get(request.traceId),
+            rubric: request.rubric.slice(0, 4e4),
+            signal: controller.signal
+          });
+        }
+      ),
+      new Promise((_resolve, reject) => {
+        controller.signal.addEventListener(
+          "abort",
+          () => reject(new Error(`Judge exceeded ${timeoutMs}ms`)),
+          { once: true }
+        );
+      })
+    ]).finally(() => clearTimeout(timeout));
+    if (typeof result !== "object" || result === null) throw new Error("invalid judge response");
+    return JSON.stringify({ type: "judge_response", requestId, status: "completed", result });
+  } catch (e) {
+    return JSON.stringify({
+      type: "judge_response",
+      requestId,
+      status: "unavailable",
+      message: "Judge unavailable"
+    });
+  }
+}
+function byteLength(value) {
+  return new TextEncoder().encode(value).byteLength;
+}
+export {
+  runPortableEvaluator
+};

package/dist/tracing/index.d.mts

@@ -1,8 +1,9 @@
 import { Instrumentation } from '@opentelemetry/instrumentation';
-import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-CDmWtd83.mjs';
+import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-CQj4bn2v.mjs';
 import '@opentelemetry/api';
 import 'zod';
 import '@opentelemetry/sdk-trace-base';
+import './eval-scope-BYkhP4uI.js';
 
 interface TracingOptions extends InitializeOptions {
     /**

package/dist/tracing/index.d.ts

@@ -1,8 +1,9 @@
 import { Instrumentation } from '@opentelemetry/instrumentation';
-import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-CDmWtd83.js';
+import { R as Raindrop, I as InitializeOptions, S as SpanProcessorOptions, a as RaindropSpanProcessor, P as PartialAiTrackEvent, b as Interaction, A as AiTrackUsage, C as CarrierHeaders, T as TraceCarrier, c as SubagentResumeOptions, d as SubagentRun, e as Tracer } from '../index-CQj4bn2v.js';
 import '@opentelemetry/api';
 import 'zod';
 import '@opentelemetry/sdk-trace-base';
+import './eval-scope-BYkhP4uI.js';
 
 interface TracingOptions extends InitializeOptions {
     /**