responses-proxy 0.1.0

package/dist/error-response.js

@@ -0,0 +1,174 @@
+import { ProviderUsageLimitError } from "./provider-usage.js";
+const MAX_UPSTREAM_ERROR_BODY_CHARS = 4_096;
+export function resolveProxyError(args) {
+    const parsedUpstream = parseUpstreamErrorBody(args.upstreamBody);
+    const mapped = applyProviderErrorPolicy({
+        statusCode: args.statusCode,
+        message: args.message,
+        parsedUpstream,
+        upstreamBody: args.upstreamBody,
+        defaultCode: args.defaultCode ?? defaultProxyErrorCode(args.statusCode),
+        retryable: isRetryableStatusCode(args.statusCode),
+    }, args.providerErrorPolicy);
+    return {
+        errorCode: mapped.code,
+        retryable: mapped.retryable,
+        envelope: {
+            error: {
+                type: args.errorType ??
+                    (args.statusCode >= 400 && args.statusCode < 500 ? "request_error" : "internal_error"),
+                code: mapped.code,
+                message: resolveProxyErrorMessage(mapped.message, parsedUpstream),
+                request_id: args.requestId,
+                upstream_status: args.statusCode,
+                upstream_body: truncateUpstreamErrorBody(args.upstreamBody),
+                upstream_error: parsedUpstream,
+                retryable: mapped.retryable,
+                usage: args.usage instanceof ProviderUsageLimitError
+                    ? args.usage.usage
+                    : args.usage
+                        ? summarizeUsageLike(args.usage)
+                        : undefined,
+            },
+        },
+    };
+}
+export function buildProxyErrorEnvelope(args) {
+    return resolveProxyError(args).envelope;
+}
+export function defaultProxyErrorCode(statusCode) {
+    if (statusCode === 413) {
+        return "REQUEST_BODY_TOO_LARGE";
+    }
+    return statusCode >= 500 ? "UPSTREAM_REQUEST_FAILED" : "UPSTREAM_BAD_REQUEST";
+}
+export function isRetryableStatusCode(statusCode) {
+    return statusCode === 408 || statusCode === 409 || statusCode === 429 || statusCode >= 500;
+}
+export function parseUpstreamErrorBody(body) {
+    if (typeof body !== "string" || !body.trim()) {
+        return undefined;
+    }
+    const parsed = parseJsonLike(body);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+        return undefined;
+    }
+    const root = parsed;
+    const errorRecord = isRecord(root.error) ? root.error : root;
+    const message = readString(errorRecord.message) ?? readString(root.message);
+    const type = readString(errorRecord.type) ?? readString(root.type);
+    const code = readString(errorRecord.code) ?? readString(root.code);
+    const param = readString(errorRecord.param) ?? readString(root.param);
+    if (!message && !type && !code && !param) {
+        return undefined;
+    }
+    return {
+        message,
+        type,
+        code,
+        param,
+    };
+}
+export function truncateUpstreamErrorBody(body) {
+    if (typeof body !== "string") {
+        return undefined;
+    }
+    const trimmed = body.trim();
+    if (!trimmed) {
+        return undefined;
+    }
+    if (trimmed.length <= MAX_UPSTREAM_ERROR_BODY_CHARS) {
+        return trimmed;
+    }
+    const headBudget = Math.max(256, Math.floor(MAX_UPSTREAM_ERROR_BODY_CHARS * 0.75));
+    const tailBudget = Math.max(128, MAX_UPSTREAM_ERROR_BODY_CHARS - headBudget - 32);
+    return `${trimmed.slice(0, headBudget)}\n... upstream error body truncated ...\n${trimmed.slice(Math.max(0, trimmed.length - tailBudget))}`.trim();
+}
+function resolveProxyErrorMessage(message, parsedUpstream) {
+    if (!parsedUpstream?.message) {
+        return message;
+    }
+    if (message.includes("upstream rejected request") ||
+        message.includes("upstream request failed") ||
+        message.includes("Unknown proxy error")) {
+        return message.includes(parsedUpstream.message)
+            ? message
+            : `${message}: ${parsedUpstream.message}`;
+    }
+    return message;
+}
+function parseJsonLike(value) {
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readString(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function summarizeUsageLike(usage) {
+    return {
+        allowed: usage.allowed,
+        remaining: usage.remaining,
+        limit: usage.limit,
+        used: usage.used,
+        raw: usage.raw,
+    };
+}
+function applyProviderErrorPolicy(base, policy) {
+    if (!policy?.rules?.length) {
+        return {
+            code: base.defaultCode,
+            message: base.message,
+            retryable: base.retryable,
+        };
+    }
+    const upstreamCode = base.parsedUpstream?.code?.toLowerCase();
+    const upstreamType = base.parsedUpstream?.type?.toLowerCase();
+    const message = base.message.toLowerCase();
+    const body = base.upstreamBody?.toLowerCase() ?? "";
+    for (const rule of policy.rules) {
+        if (rule.statusCodes?.length && !rule.statusCodes.includes(base.statusCode)) {
+            continue;
+        }
+        if (rule.upstreamCodes?.length) {
+            const allowed = rule.upstreamCodes.map((item) => item.toLowerCase());
+            if (!upstreamCode || !allowed.includes(upstreamCode)) {
+                continue;
+            }
+        }
+        if (rule.upstreamTypes?.length) {
+            const allowed = rule.upstreamTypes.map((item) => item.toLowerCase());
+            if (!upstreamType || !allowed.includes(upstreamType)) {
+                continue;
+            }
+        }
+        if (rule.messageIncludes?.length) {
+            const matched = rule.messageIncludes.some((item) => message.includes(item.toLowerCase()));
+            if (!matched) {
+                continue;
+            }
+        }
+        if (rule.bodyIncludes?.length) {
+            const matched = rule.bodyIncludes.some((item) => body.includes(item.toLowerCase()));
+            if (!matched) {
+                continue;
+            }
+        }
+        return {
+            code: rule.code?.trim() || base.defaultCode,
+            message: rule.message?.trim() || base.message,
+            retryable: typeof rule.retryable === "boolean" ? rule.retryable : base.retryable,
+        };
+    }
+    return {
+        code: base.defaultCode,
+        message: base.message,
+        retryable: base.retryable,
+    };
+}

package/dist/error-response.test.js

@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { buildProxyErrorEnvelope, resolveProxyError, isRetryableStatusCode, parseUpstreamErrorBody, truncateUpstreamErrorBody, } from "./error-response.js";
+test("parses structured upstream JSON error payloads", () => {
+    assert.deepEqual(parseUpstreamErrorBody(JSON.stringify({
+        error: {
+            message: "Model quota exceeded",
+            type: "rate_limit_error",
+            code: "quota_exceeded",
+            param: "model",
+        },
+    })), {
+        message: "Model quota exceeded",
+        type: "rate_limit_error",
+        code: "quota_exceeded",
+        param: "model",
+    });
+});
+test("builds backward-compatible proxy error envelopes with upstream details", () => {
+    const envelope = buildProxyErrorEnvelope({
+        statusCode: 429,
+        requestId: "req_123",
+        message: "[req_123] upstream rejected request (429 Too Many Requests)",
+        upstreamBody: JSON.stringify({
+            error: {
+                message: "Rate limit reached",
+                type: "rate_limit_error",
+                code: "rate_limit",
+            },
+        }),
+        defaultCode: "UPSTREAM_BAD_REQUEST",
+        errorType: "proxy_error",
+    });
+    assert.equal(envelope.error.type, "proxy_error");
+    assert.equal(envelope.error.code, "UPSTREAM_BAD_REQUEST");
+    assert.equal(envelope.error.request_id, "req_123");
+    assert.equal(envelope.error.upstream_status, 429);
+    assert.equal(envelope.error.retryable, true);
+    assert.match(String(envelope.error.message), /Rate limit reached/);
+    assert.deepEqual(envelope.error.upstream_error, {
+        message: "Rate limit reached",
+        type: "rate_limit_error",
+        code: "rate_limit",
+        param: undefined,
+    });
+    assert.match(String(envelope.error.upstream_body), /rate_limit_error/);
+});
+test("truncates oversized upstream error bodies without dropping the tail", () => {
+    const raw = `${"A".repeat(5000)}TAIL`;
+    const truncated = truncateUpstreamErrorBody(raw);
+    assert.ok(truncated);
+    assert.match(String(truncated), /upstream error body truncated/);
+    assert.match(String(truncated), /TAIL$/);
+});
+test("marks only retryable upstream statuses as retryable", () => {
+    assert.equal(isRetryableStatusCode(400), false);
+    assert.equal(isRetryableStatusCode(413), false);
+    assert.equal(isRetryableStatusCode(429), true);
+    assert.equal(isRetryableStatusCode(500), true);
+});
+test("provider error policy can remap verbose upstream failures to cleaner codes and messages", () => {
+    const resolved = resolveProxyError({
+        statusCode: 500,
+        message: "[req_123] upstream rejected request (500 Internal Server Error)",
+        upstreamBody: JSON.stringify({
+            error: {
+                message: "Request body is too large",
+                type: "server_error",
+                code: "internal_error",
+            },
+        }),
+        defaultCode: "UPSTREAM_REQUEST_FAILED",
+        errorType: "proxy_error",
+        providerErrorPolicy: {
+            rules: [
+                {
+                    bodyIncludes: ["request body is too large"],
+                    code: "UPSTREAM_REQUEST_TOO_LARGE",
+                    message: "Upstream rejected the request because the serialized prompt body is too large",
+                    retryable: false,
+                },
+            ],
+        },
+    });
+    assert.equal(resolved.errorCode, "UPSTREAM_REQUEST_TOO_LARGE");
+    assert.equal(resolved.retryable, false);
+    assert.equal(resolved.envelope.error.message, "Upstream rejected the request because the serialized prompt body is too large");
+});

package/dist/forward.js

@@ -0,0 +1,357 @@
+import { setTimeout as delay } from "node:timers/promises";
+export async function forwardJson({ requestId, url, body, apiKey, headers, timeoutMs, logger, onEvent, }) {
+    const startedAt = Date.now();
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const response = await fetch(url, {
+            method: "POST",
+            headers: buildHeaders(apiKey, headers),
+            body: JSON.stringify(body),
+            signal: controller.signal,
+        });
+        logger.info({
+            requestId,
+            upstreamStatus: response.status,
+            connectMs: Date.now() - startedAt,
+        }, "upstream JSON response received");
+        await onEvent?.({
+            event: "upstream_json_response",
+            requestId,
+            upstreamStatus: response.status,
+            connectMs: Date.now() - startedAt,
+        });
+        return response;
+    }
+    catch (error) {
+        throw wrapFetchError(requestId, error);
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+export async function forwardSse({ requestId, url, body, apiKey, headers, timeoutMs, idleTimeoutMs, responseRaw, logger, onEvent, }) {
+    const startedAt = Date.now();
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    let upstream;
+    try {
+        upstream = await fetch(url, {
+            method: "POST",
+            headers: buildHeaders(apiKey, {
+                ...headers,
+                Accept: "text/event-stream",
+            }),
+            body: JSON.stringify(body),
+            signal: controller.signal,
+        });
+    }
+    catch (error) {
+        clearTimeout(timeout);
+        throw wrapFetchError(requestId, error);
+    }
+    if (!upstream.ok || !upstream.body) {
+        clearTimeout(timeout);
+        throw await buildUpstreamError(requestId, upstream);
+    }
+    logger.info({
+        requestId,
+        upstreamStatus: upstream.status,
+        connectMs: Date.now() - startedAt,
+    }, "upstream SSE stream opened");
+    await onEvent?.({
+        event: "upstream_sse_opened",
+        requestId,
+        upstreamStatus: upstream.status,
+        connectMs: Date.now() - startedAt,
+    });
+    responseRaw.setHeader("Content-Type", "text/event-stream");
+    responseRaw.setHeader("Cache-Control", "no-cache, no-transform");
+    responseRaw.setHeader("Connection", "keep-alive");
+    responseRaw.setHeader("X-Accel-Buffering", "no");
+    responseRaw.flushHeaders?.();
+    const reader = upstream.body.getReader();
+    let sseBuffer = "";
+    let firstForwardedAt;
+    let firstEventType;
+    let forwardedFrames = 0;
+    let forwardedBytes = 0;
+    let filteredNullFrames = 0;
+    let usageCaptured = false;
+    let idleTimer;
+    const resetIdleTimer = () => {
+        if (idleTimer)
+            clearTimeout(idleTimer);
+        idleTimer = setTimeout(() => {
+            logger.warn({ requestId }, "upstream SSE idle timeout reached");
+            controller.abort();
+            responseRaw.destroy(new Error("upstream stream idle timeout"));
+        }, idleTimeoutMs);
+    };
+    resetIdleTimer();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            resetIdleTimer();
+            if (value) {
+                const chunk = Buffer.from(value).toString("utf8");
+                sseBuffer += chunk;
+                const frames = extractCompleteSseFrames(sseBuffer);
+                sseBuffer = frames.remaining;
+                for (const frame of frames.complete) {
+                    const filtered = filterMalformedSseFrame(frame);
+                    if (filtered) {
+                        if (!usageCaptured) {
+                            const usageEvent = extractSseUsageEvent(filtered, requestId);
+                            if (usageEvent) {
+                                usageCaptured = true;
+                                await onEvent?.(usageEvent);
+                            }
+                        }
+                        if (firstForwardedAt === undefined) {
+                            firstForwardedAt = Date.now();
+                            firstEventType = extractSseEventType(filtered);
+                            logger.info({
+                                requestId,
+                                firstChunkMs: firstForwardedAt - startedAt,
+                                firstEventType,
+                            }, "upstream SSE first event forwarded");
+                            await onEvent?.({
+                                event: "upstream_sse_first_event",
+                                requestId,
+                                firstChunkMs: firstForwardedAt - startedAt,
+                                firstEventType,
+                            });
+                        }
+                        forwardedFrames += 1;
+                        forwardedBytes += Buffer.byteLength(filtered);
+                        responseRaw.write(filtered);
+                    }
+                    else {
+                        filteredNullFrames += 1;
+                    }
+                }
+            }
+            else {
+                await delay(0);
+            }
+        }
+        if (sseBuffer.length > 0) {
+            const filtered = filterMalformedSseFrame(sseBuffer);
+            if (filtered) {
+                if (!usageCaptured) {
+                    const usageEvent = extractSseUsageEvent(filtered, requestId);
+                    if (usageEvent) {
+                        usageCaptured = true;
+                        await onEvent?.(usageEvent);
+                    }
+                }
+                if (firstForwardedAt === undefined) {
+                    firstForwardedAt = Date.now();
+                    firstEventType = extractSseEventType(filtered);
+                    logger.info({
+                        requestId,
+                        firstChunkMs: firstForwardedAt - startedAt,
+                        firstEventType,
+                    }, "upstream SSE first event forwarded");
+                    await onEvent?.({
+                        event: "upstream_sse_first_event",
+                        requestId,
+                        firstChunkMs: firstForwardedAt - startedAt,
+                        firstEventType,
+                    });
+                }
+                forwardedFrames += 1;
+                forwardedBytes += Buffer.byteLength(filtered);
+                responseRaw.write(filtered);
+            }
+            else {
+                filteredNullFrames += 1;
+            }
+        }
+        logger.info({
+            requestId,
+            totalMs: Date.now() - startedAt,
+            forwardedFrames,
+            forwardedBytes,
+            filteredNullFrames,
+            firstEventType,
+        }, "upstream SSE stream completed");
+        await onEvent?.({
+            event: "upstream_sse_completed",
+            requestId,
+            totalMs: Date.now() - startedAt,
+            forwardedFrames,
+            forwardedBytes,
+            filteredNullFrames,
+            firstEventType,
+        });
+        responseRaw.end();
+    }
+    catch (error) {
+        throw wrapFetchError(requestId, error);
+    }
+    finally {
+        if (idleTimer)
+            clearTimeout(idleTimer);
+        clearTimeout(timeout);
+    }
+}
+function extractCompleteSseFrames(buffer) {
+    const complete = [];
+    let cursor = 0;
+    while (cursor < buffer.length) {
+        const lfLfIndex = buffer.indexOf("\n\n", cursor);
+        const crlfCrlfIndex = buffer.indexOf("\r\n\r\n", cursor);
+        let endIndex = -1;
+        let delimiterLength = 0;
+        if (lfLfIndex !== -1 && (crlfCrlfIndex === -1 || lfLfIndex < crlfCrlfIndex)) {
+            endIndex = lfLfIndex;
+            delimiterLength = 2;
+        }
+        else if (crlfCrlfIndex !== -1) {
+            endIndex = crlfCrlfIndex;
+            delimiterLength = 4;
+        }
+        if (endIndex === -1) {
+            break;
+        }
+        complete.push(buffer.slice(cursor, endIndex + delimiterLength));
+        cursor = endIndex + delimiterLength;
+    }
+    return {
+        complete,
+        remaining: buffer.slice(cursor),
+    };
+}
+function filterMalformedSseFrame(frame) {
+    const normalized = frame.replace(/\r\n/g, "\n");
+    const dataLines = normalized
+        .split("\n")
+        .filter((line) => line.startsWith("data:"))
+        .map((line) => line.slice(5).trim());
+    if (dataLines.length === 1 && dataLines[0] === "null") {
+        return "";
+    }
+    return frame;
+}
+function extractSseEventType(frame) {
+    const normalized = frame.replace(/\r\n/g, "\n");
+    const dataLine = normalized
+        .split("\n")
+        .find((line) => line.startsWith("data:") && line.slice(5).trim().startsWith("{"));
+    if (!dataLine) {
+        return normalized.includes("[DONE]") ? "done" : undefined;
+    }
+    try {
+        const payload = JSON.parse(dataLine.slice(5).trim());
+        return typeof payload.type === "string" ? payload.type : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function extractSseUsageEvent(frame, requestId) {
+    const payload = extractSsePayload(frame);
+    if (!payload || typeof payload.type !== "string") {
+        return undefined;
+    }
+    const response = isRecord(payload.response) ? payload.response : undefined;
+    const usage = response && isRecord(response.usage) ? response.usage : undefined;
+    if (!usage) {
+        return undefined;
+    }
+    const totalTokens = readNumber(usage.total_tokens);
+    const outputTokens = readNumber(usage.output_tokens);
+    const inputTokens = readNumber(usage.input_tokens);
+    const inputTokensDetails = isRecord(usage.input_tokens_details)
+        ? usage.input_tokens_details
+        : undefined;
+    const cachedTokens = inputTokensDetails && readNumber(inputTokensDetails.cached_tokens) !== undefined
+        ? readNumber(inputTokensDetails.cached_tokens)
+        : undefined;
+    const cacheSavedPercent = computeCacheSavedPercent(inputTokens, cachedTokens);
+    if (payload.type !== "response.completed" &&
+        payload.type !== "response.incomplete" &&
+        totalTokens === undefined &&
+        outputTokens === undefined &&
+        inputTokens === undefined &&
+        cachedTokens === undefined) {
+        return undefined;
+    }
+    return {
+        event: "upstream_response_usage",
+        requestId,
+        responseEventType: payload.type,
+        responseId: typeof response?.id === "string" ? response.id : undefined,
+        usage,
+        inputTokens,
+        outputTokens,
+        totalTokens,
+        inputTokensDetails,
+        cachedTokens,
+        cacheSavedPercent,
+    };
+}
+function extractSsePayload(frame) {
+    const normalized = frame.replace(/\r\n/g, "\n");
+    const dataLine = normalized
+        .split("\n")
+        .find((line) => line.startsWith("data:") && line.slice(5).trim().startsWith("{"));
+    if (!dataLine) {
+        return undefined;
+    }
+    try {
+        const payload = JSON.parse(dataLine.slice(5).trim());
+        return isRecord(payload) ? payload : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function readNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+function computeCacheSavedPercent(inputTokens, cachedTokens) {
+    if (inputTokens === undefined ||
+        cachedTokens === undefined ||
+        inputTokens <= 0 ||
+        cachedTokens < 0) {
+        return undefined;
+    }
+    return Math.round((cachedTokens / inputTokens) * 1000) / 10;
+}
+export async function buildUpstreamError(requestId, upstream) {
+    const body = await upstream.text();
+    const error = new Error(`upstream rejected request (${upstream.status} ${upstream.statusText})`);
+    error.statusCode = upstream.status;
+    error.body = body;
+    error.message = `[${requestId}] ${error.message}`;
+    return error;
+}
+function buildHeaders(apiKey, extraHeaders) {
+    const headers = {
+        "Content-Type": "application/json",
+        ...extraHeaders,
+    };
+    if (apiKey) {
+        headers.Authorization = `Bearer ${apiKey}`;
+    }
+    return headers;
+}
+function wrapFetchError(requestId, error) {
+    const wrapped = new Error(error instanceof Error ? `[${requestId}] ${error.message}` : `[${requestId}] upstream request failed`);
+    if (error?.name === "AbortError") {
+        wrapped.statusCode = 504;
+    }
+    else {
+        wrapped.statusCode = 502;
+    }
+    return wrapped;
+}