rightmodeler 0.2.1 → 0.4.0

package/dist-bundle/provenance.js

@@ -0,0 +1,89 @@
+const DATED_SNAPSHOT = /^\d{2,4}(?:-?\d{2}){1,2}$/;
+const PORTKEY_CACHE_HITS = new Set(["hit", "semantic hit"]);
+function objectOf(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+function nonEmptyString(value) {
+    return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+export function servedModel(body) {
+    return nonEmptyString(objectOf(body)?.model);
+}
+function modelParts(id) {
+    const segments = id.toLowerCase().split("/");
+    return {
+        name: segments[segments.length - 1],
+        ...(segments.length > 1 ? { vendor: segments[segments.length - 2] } : {}),
+    };
+}
+export function sameModel(requested, served) {
+    const want = modelParts(requested);
+    const got = modelParts(served);
+    if (want.vendor !== undefined &&
+        got.vendor !== undefined &&
+        want.vendor !== got.vendor) {
+        return false;
+    }
+    if (got.name === want.name)
+        return true;
+    const prefix = `${want.name}-`;
+    return (got.name.startsWith(prefix) &&
+        DATED_SNAPSHOT.test(got.name.slice(prefix.length)));
+}
+function header(headers, name) {
+    if (headers instanceof Headers)
+        return headers.get(name) ?? undefined;
+    const value = headers[name];
+    return typeof value === "string" ? value : value?.join(", ");
+}
+function substitution(kind, evidence) {
+    return { kind, evidence: evidence.slice(0, 200) };
+}
+export function responseSubstitution(input) {
+    const body = objectOf(input.body);
+    const extra = objectOf(body?.extra_fields);
+    const served = servedModel(body);
+    if (served !== undefined && !sameModel(input.requestedModel, served)) {
+        return substitution("model", `served ${served} for requested ${input.requestedModel}`);
+    }
+    const fallback = nonEmptyString(objectOf(extra?.routing_info)?.server_side_fallback_model);
+    if (fallback !== undefined) {
+        return substitution("model", `bifrost server-side fallback served ${fallback}`);
+    }
+    const cacheStatus = header(input.headers, "x-portkey-cache-status");
+    if (cacheStatus !== undefined &&
+        PORTKEY_CACHE_HITS.has(cacheStatus.toLowerCase())) {
+        return substitution("cache", `x-portkey-cache-status: ${cacheStatus}`);
+    }
+    const cacheDebug = objectOf(extra?.cache_debug);
+    if (cacheDebug?.cache_hit === true) {
+        const hitType = cacheDebug.hit_type;
+        return substitution("cache", `bifrost cache hit${typeof hitType === "string" ? ` (${hitType})` : ""}`);
+    }
+    const hookResults = objectOf(body?.hook_results);
+    const transformed = [
+        hookResults?.before_request_hooks,
+        hookResults?.after_request_hooks,
+    ]
+        .flatMap((hooks) => (Array.isArray(hooks) ? hooks : []))
+        .map(objectOf)
+        .find((hook) => hook?.transformed === true);
+    if (transformed !== undefined) {
+        const id = nonEmptyString(transformed.id) ?? "unnamed";
+        return substitution("request", `portkey hook ${id} transformed the call`);
+    }
+    const dropped = [
+        extra?.dropped_compat_plugin_params,
+        extra?.dropped_unsupported_tools,
+    ].flatMap((items) => (Array.isArray(items) ? items : []));
+    if (dropped.length > 0) {
+        return substitution("request", `bifrost dropped ${dropped.join(", ")}`);
+    }
+    const converted = nonEmptyString(extra?.converted_request_type);
+    if (converted !== undefined) {
+        return substitution("request", `bifrost converted the request to ${converted}`);
+    }
+    return undefined;
+}

package/dist-bundle/proxy/container-supervisor.mjs

@@ -30,6 +30,11 @@ const heartbeat = setInterval(
   500,
 );
 heartbeat.unref();
+const deadlineMs = Number(process.env.RM_DEADLINE_MS ?? 0);
+let active = null;
+if (deadlineMs > 0) {
+  setTimeout(() => active?.kill("SIGKILL"), deadlineMs).unref();
+}
 
 function start(command, stdoutPath, stderrPath, flags = "w") {
   const stdout = createWriteStream(stdoutPath, { flags });
@@ -38,6 +43,7 @@ function start(command, stdoutPath, stderrPath, flags = "w") {
     cwd: APP_ROOT,
     stdio: ["ignore", "pipe", "pipe"],
   });
+  active = child;
   child.stdout.pipe(stdout);
   child.stderr.pipe(stderr);
   const closed = new Promise((resolve, reject) => {

package/dist-bundle/proxy/proxy-runtime.mjs

@@ -12,6 +12,7 @@ import { join } from "node:path";
 import { PassThrough } from "node:stream";
 
 import { hopByHopHeaders } from "./headers.js";
+import { responseSubstitution, servedModel } from "../provenance.js";
 import { classifyStream } from "../transport/stream.js";
 
 const maxRequestBytes = 10 * 1024 * 1024;
@@ -39,6 +40,12 @@ function isObject(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
+// The same 1e-12 snap as roundUsd in ../budget.ts, which this runtime ships without:
+// lease sums then do not depend on the order requests settle in.
+function roundUsd(value) {
+  return Number(value.toFixed(12));
+}
+
 function parseConfig() {
   const rawSwapPolicy = jsonEnv("RM_SWAP_POLICY");
   if (
@@ -69,7 +76,32 @@ function parseConfig() {
         "RM_PRICING_TABLE values must contain non-negative input and output prices",
       );
     }
-    pricingTable[model] = { input: pricing.input, output: pricing.output };
+    if (
+      pricing.maxOutputTokens !== undefined &&
+      (!Number.isSafeInteger(pricing.maxOutputTokens) ||
+        pricing.maxOutputTokens <= 0)
+    ) {
+      throw new Error(
+        "RM_PRICING_TABLE maxOutputTokens must be a positive integer",
+      );
+    }
+    pricingTable[model] = {
+      input: pricing.input,
+      output: pricing.output,
+      ...(pricing.maxOutputTokens === undefined
+        ? {}
+        : { maxOutputTokens: pricing.maxOutputTokens }),
+    };
+  }
+
+  const defaultMaxOutputTokens = Number(
+    requiredEnv("RM_DEFAULT_MAX_OUTPUT_TOKENS"),
+  );
+  if (
+    !Number.isSafeInteger(defaultMaxOutputTokens) ||
+    defaultMaxOutputTokens <= 0
+  ) {
+    throw new Error("RM_DEFAULT_MAX_OUTPUT_TOKENS must be a positive integer");
   }
 
   const lease = jsonEnv("RM_BUDGET_LEASE");
@@ -88,6 +120,21 @@ function parseConfig() {
     throw new Error("RM_EGRESS_URL must use http or https");
   }
 
+  const requestHeaders =
+    process.env.RM_REQUEST_HEADERS === undefined
+      ? {}
+      : jsonEnv("RM_REQUEST_HEADERS");
+  if (
+    !isObject(requestHeaders) ||
+    Object.entries(requestHeaders).some(
+      ([name, value]) => name.length === 0 || typeof value !== "string",
+    )
+  ) {
+    throw new Error(
+      "RM_REQUEST_HEADERS must map header names to string values",
+    );
+  }
+
   return {
     runId: requiredEnv("RM_RUN_ID"),
     caseId: requiredEnv("RM_CASE_ID"),
@@ -98,7 +145,9 @@ function parseConfig() {
     egressUrl,
     swapPolicy,
     pricingTable,
+    defaultMaxOutputTokens,
     lease,
+    requestHeaders,
   };
 }
 
@@ -184,11 +233,11 @@ function loadState(spoolPath, checkpointPath) {
     state.lastAttemptGroup = Math.max(state.lastAttemptGroup, row.attemptGroup);
     state.groups.set(row.logicalCallId, row.attemptGroup);
     state.attemptIds.add(row.attemptId);
-    state.spentUsd += row.costUsd;
+    state.spentUsd = roundUsd(state.spentUsd + row.costUsd);
   }
 
   for (const reservation of state.reservations.values()) {
-    state.spentUsd += reservation.reservedUsd;
+    state.spentUsd = roundUsd(state.spentUsd + reservation.reservedUsd);
   }
 
   return state;
@@ -222,12 +271,13 @@ function responseHeaders(headers) {
   return forwarded;
 }
 
-function requestHeaders(headers, bodyLength) {
+function requestHeaders(headers, bodyLength, configured) {
   const forwarded = {};
   for (const [name, value] of Object.entries(headers)) {
     if (
       value !== undefined &&
       !hopByHopHeaders.has(name) &&
+      name !== "accept-encoding" &&
       name !== "authorization" &&
       name !== "host" &&
       name !== "content-length"
@@ -235,6 +285,8 @@ function requestHeaders(headers, bodyLength) {
       forwarded[name] = value;
     }
   }
+  Object.assign(forwarded, configured);
+  forwarded["accept-encoding"] = "identity";
   forwarded["content-length"] = String(bodyLength);
   return forwarded;
 }
@@ -288,7 +340,11 @@ function requestUpstream(
   return new Promise((resolve, reject) => {
     const request = send(
       url,
-      { method, path: requestTarget, headers },
+      {
+        method,
+        path: `${url.pathname.replace(/\/$/, "")}${requestTarget}`,
+        headers,
+      },
       (response) => {
         clearTimeout(deadline);
         resolve(response);
@@ -437,6 +493,7 @@ async function forwardStreaming(upstream, outgoing, status, spoolSink) {
     spoolPath: result.spoolPath ?? null,
     finishedWithoutSentinel: result.finishedWithoutSentinel === true,
     upstreamFailed,
+    model: result.model ?? null,
   };
 }
 
@@ -489,6 +546,7 @@ async function forwardNonStreaming(upstream, outgoing, status) {
       streamOutcome: "completed",
       usage: isObject(body) ? (body.usage ?? null) : null,
       upstreamFailed: false,
+      body,
     };
   } catch {
     return { streamOutcome: "truncated", usage: null, upstreamFailed: false };
@@ -624,13 +682,14 @@ async function main() {
       return;
     }
 
-    const maxTokens = parsed?.max_tokens;
+    const requestedLimit =
+      parsed?.max_completion_tokens ?? parsed?.max_tokens ?? undefined;
     if (
       !isObject(parsed) ||
       typeof parsed.model !== "string" ||
       parsed.model.length === 0 ||
-      !Number.isSafeInteger(maxTokens) ||
-      maxTokens < 0
+      (requestedLimit !== undefined &&
+        (!Number.isSafeInteger(requestedLimit) || requestedLimit < 0))
     ) {
       recordLost({
         attemptGroup,
@@ -640,13 +699,26 @@ async function main() {
         startedAt,
       });
       sendJson(outgoing, 400, {
-        error: "Request body requires model and a non-negative max_tokens.",
+        error:
+          "Request body requires model; max_completion_tokens or max_tokens must be a non-negative integer when present.",
       });
       return;
     }
 
+    const swapped = stepId in config.swapPolicy;
     const model = config.swapPolicy[stepId] ?? parsed.model;
-    const rewritten = { ...parsed, model };
+    const rewritten = {
+      ...parsed,
+      model,
+      ...(parsed.stream === true
+        ? {
+            stream_options: {
+              ...(isObject(parsed.stream_options) ? parsed.stream_options : {}),
+              include_usage: true,
+            },
+          }
+        : {}),
+    };
     const forwardedBody = Buffer.from(JSON.stringify(rewritten));
     const pricing = config.pricingTable[model];
     if (pricing === undefined) {
@@ -662,13 +734,18 @@ async function main() {
       });
       return;
     }
+    const maxTokens =
+      requestedLimit ??
+      pricing.maxOutputTokens ??
+      config.defaultMaxOutputTokens;
 
     const estimatedInputTokens = forwardedBody.length;
     const estimatedWorstCaseUsd =
       estimatedInputTokens * pricing.input + maxTokens * pricing.output;
-    const requiredLeaseUsd =
-      state.spentUsd + reservedUsd + estimatedWorstCaseUsd;
-    if (requiredLeaseUsd > config.lease.maxUsd) {
+    const requiredLeaseUsd = roundUsd(
+      state.spentUsd + reservedUsd + estimatedWorstCaseUsd,
+    );
+    if (requiredLeaseUsd > roundUsd(config.lease.maxUsd)) {
       appendRow(spoolPath, {
         kind: "blocked",
         runId: config.runId,
@@ -693,7 +770,7 @@ async function main() {
       return;
     }
 
-    reservedUsd += estimatedWorstCaseUsd;
+    reservedUsd = roundUsd(reservedUsd + estimatedWorstCaseUsd);
     const attemptId = nextAttemptId(state);
     const responseSpoolPath = join(streamDirectory, `${attemptId}.txt`);
     appendRow(spoolPath, {
@@ -726,21 +803,25 @@ async function main() {
         upstreamSource: null,
         upstreamFailed: false,
       };
+      let provenance = {};
       try {
         const upstream = await requestUpstream(
           config.egressUrl,
           incoming.url ?? "/",
           incoming.method ?? "POST",
-          requestHeaders(incoming.headers, forwardedBody.length),
+          requestHeaders(
+            incoming.headers,
+            forwardedBody.length,
+            config.requestHeaders,
+          ),
           forwardedBody,
           streamHardDeadlineMs,
         );
         const status = upstream.statusCode ?? 502;
-        const declaredSource = upstream.headers[egressSourceHeader];
         const upstreamSource =
-          declaredSource === "provider" || declaredSource === "egress"
-            ? declaredSource
-            : null;
+          upstream.headers[egressSourceHeader] === "egress"
+            ? "egress"
+            : "provider";
         const forwarded =
           rewritten.stream === true && status < 400
             ? await forwardStreaming(upstream, outgoing, status, {
@@ -755,6 +836,23 @@ async function main() {
           upstreamStatus: status,
           upstreamSource: forwarded.upstreamFailed ? "egress" : upstreamSource,
         };
+        if (swapped && status < 400) {
+          const answered =
+            forwarded.body ??
+            (typeof forwarded.model === "string"
+              ? { model: forwarded.model }
+              : undefined);
+          const served = servedModel(answered);
+          const substitution = responseSubstitution({
+            requestedModel: model,
+            headers: upstream.headers,
+            body: answered,
+          });
+          provenance = {
+            ...(served === undefined ? {} : { servedModel: served }),
+            ...(substitution === undefined ? {} : { substitution }),
+          };
+        }
       } catch {
         if (!outgoing.headersSent) {
           sendJson(outgoing, 502, { error: "Egress request failed." });
@@ -790,6 +888,7 @@ async function main() {
         ...(result.finishedWithoutSentinel
           ? { finishedWithoutSentinel: true }
           : {}),
+        ...provenance,
         usage,
         responseSpoolPath: result.spoolPath,
         costUsd: leaseChargeUsd,
@@ -800,10 +899,10 @@ async function main() {
         endedAt: new Date().toISOString(),
       });
       state.reservations.delete(attemptId);
-      state.spentUsd += leaseChargeUsd;
+      state.spentUsd = roundUsd(state.spentUsd + leaseChargeUsd);
       if (!outgoing.destroyed && !outgoing.writableEnded) outgoing.end();
     } finally {
-      reservedUsd -= estimatedWorstCaseUsd;
+      reservedUsd = roundUsd(reservedUsd - estimatedWorstCaseUsd);
     }
   }
 

package/dist-bundle/transport/stream.js

@@ -65,13 +65,13 @@ function parseUsage(value) {
     if (!isRecord(value) ||
         typeof value.prompt_tokens !== "number" ||
         typeof value.completion_tokens !== "number" ||
-        typeof value.total_tokens !== "number") {
+        (value.total_tokens !== undefined && typeof value.total_tokens !== "number")) {
         return null;
     }
     return {
         inputTokens: value.prompt_tokens,
         outputTokens: value.completion_tokens,
-        totalTokens: value.total_tokens,
+        totalTokens: value.total_tokens ?? value.prompt_tokens + value.completion_tokens,
     };
 }
 function parseEvent(data) {
@@ -121,6 +121,9 @@ function parseEvent(data) {
         usage: value.usage === undefined || value.usage === null
             ? null
             : parseUsage(value.usage),
+        ...(typeof value.model === "string" && value.model.length > 0
+            ? { model: value.model }
+            : {}),
     };
 }
 function eventData(event) {
@@ -144,6 +147,7 @@ export async function classifyStream(byteStream, options) {
     const collector = new ContentCollector(options.spoolSink, processingController.signal);
     let chunks = 0;
     let usage = null;
+    let model;
     let sawFinish = false;
     let selectedResult;
     const result = (outcome, reason, finishedWithoutSentinel = false) => {
@@ -154,6 +158,7 @@ export async function classifyStream(byteStream, options) {
             usage,
             chunks,
             ...(finishedWithoutSentinel ? { finishedWithoutSentinel: true } : {}),
+            ...(model === undefined ? {} : { model }),
         };
         return selectedResult;
     };
@@ -283,6 +288,7 @@ export async function classifyStream(byteStream, options) {
                     throw error;
                 }
                 usage = event.usage ?? usage;
+                model ??= event.model;
                 sawFinish ||= event.finished;
             }
         }