saros-proxy 0.10.2 → 1.0.0

package/dist/proxy.js

@@ -9,16 +9,18 @@ import { cors } from 'hono/cors';
 import { rateLimiter } from 'hono-rate-limiter';
 import crypto from 'node:crypto';
 import { mkdirSync, writeFileSync } from 'node:fs';
-import { homedir } from 'node:os';
 import { join } from 'node:path';
-import { createProxyState, selectKeyForRequest, failoverRequest, completeRequest, markKeyFailed, markKeySucceeded, classifyHttpError, } from './proxy-logic.js';
+import { createProxyState, selectKeyForRequest, failoverRequest, completeRequest, markKeyFailed, markKeySucceeded, classifyHttpError, describeKeys, } from './proxy-logic.js';
+import { getSarosDir } from './config.js';
 import { logger, maskKey } from './logger.js';
+import { redactSecrets } from './log-redact.js';
 import { MAX_BODY_SIZE, MAX_RETRIES, RATE_LIMIT_WINDOW_MS, RATE_LIMIT_MAX, } from './constants.js';
 import { getModelsList } from './models-fetcher.js';
 import { maybeRefreshUsage } from './usage-client.js';
 import { recordModelRequest } from './model-stats.js';
 import { recordKeyTokens } from './token-usage.js';
-import { getProvider, inferProvider } from './providers/index.js';
+import { getProvider } from './providers/index.js';
+import { createRecentRequestBuffer, isDebugBufferEnabled, } from './debug-buffer.js';
 /** Canonical client-facing chat route prefix (the opencode-go shape). */
 const CANONICAL_CHAT_BASE = '/zen/go/v1';
 /**
@@ -58,11 +60,18 @@ export function stripProviderSuffix(modelId) {
 function generateRequestId() {
     return crypto.randomUUID();
 }
+/**
+ * Relative folder name for `SAROS_CAPTURE_4XX` diagnostics. Literal: no
+ * request, response or header value can reach it.
+ */
+const CAPTURE_DIR_NAME = 'captures';
+/** Disambiguates captures that land in the same millisecond. */
+let captureSequence = 0;
 /**
  * Opt-in diagnostics: when SAROS_CAPTURE_4XX=1, write the full request
- * context of every upstream 4xx to <config dir>/captures/ so provider-side
- * errors (role validation, plan gating) can be replayed and debugged.
- * Never enabled by default — bodies may contain user content.
+ * context of every upstream 4xx to <saros config dir>/captures/ so
+ * provider-side errors (role validation, plan gating) can be replayed and
+ * debugged. Never enabled by default — bodies may contain user content.
  */
 function captureFailedRequest(ctx, status, errorBody) {
     if (process.env.SAROS_CAPTURE_4XX !== '1')
@@ -70,10 +79,16 @@ function captureFailedRequest(ctx, status, errorBody) {
     if (status < 400 || status >= 500)
         return;
     try {
-        const dir = join(homedir(), '.config', 'saros', 'captures');
+        // Path = saros config dir + literal folder name. Nothing network-derived
+        // reaches this expression, so a capture stays under the user's config dir.
+        const dir = join(getSarosDir(), CAPTURE_DIR_NAME);
         mkdirSync(dir, { recursive: true });
-        const file = join(dir, `${Date.now()}-${ctx.requestId.slice(0, 8)}.json`);
-        writeFileSync(file, JSON.stringify({
+        // Filename = timestamp + monotonic sequence. The request id is kept in
+        // the JSON payload instead, so no request-derived byte ever selects the
+        // path, and `wx` refuses a pre-created file.
+        captureSequence += 1;
+        const file = join(dir, `${Date.now()}-${captureSequence}.json`);
+        const payload = JSON.stringify({
             ts: new Date().toISOString(),
             requestId: ctx.requestId,
             status,
@@ -83,7 +98,13 @@ function captureFailedRequest(ctx, status, errorBody) {
             kind: ctx.kind,
             upstreamError: errorBody.slice(0, 2000),
             requestBody: ctx.bodyText.slice(0, 100_000),
-        }, null, 2), 'utf-8');
+        }, null, 2);
+        // Opt-in diagnostic (SAROS_CAPTURE_4XX=1), not a general sink: the target
+        // is always <saros config dir>/captures/<timestamp>-<counter>.json, so no
+        // network-derived value selects the path. The file is created with mode
+        // 0600 and flag 'wx', which fails on a pre-existing file instead of
+        // following or overwriting it.
+        writeFileSync(file, payload, { encoding: 'utf-8', mode: 0o600, flag: 'wx' });
         logger.warn({ requestId: ctx.requestId, file }, 'captured failing request (SAROS_CAPTURE_4XX)');
     }
     catch {
@@ -166,18 +187,21 @@ export function prepareOutboundRequest(bodyText, modelId, provider, options) {
 }
 /**
  * Provider for a key snapshot. Suffix-routed requests (​@provider model ids)
- * pin the provider; otherwise structural inference from the key itself.
+ * pin the provider; otherwise the key's configured provider decides.
+ *
+ * The credential prefix is deliberately not re-inferred here: `key.provider`
+ * is normalized from config (an explicit `provider:` wins over the prefix) and
+ * it is what selection filters on. Routing on a different signal sent a key
+ * declared `provider: commandcode` through the OpenCode-Go URL.
  */
-function providerForKey(key, config, modelId) {
+function providerForKey(key, modelId) {
     const suffixProvider = parseProviderSuffix(modelId);
     if (suffixProvider) {
         const pinned = getProvider(suffixProvider);
         if (pinned)
             return pinned;
     }
-    const inferred = inferProvider({ label: key.label, key: key.key });
-    void config;
-    return getProvider(inferred) ?? getProvider('opencode-go');
+    return getProvider(key.provider) ?? getProvider('opencode-go');
 }
 // ---------------------------------------------------------------------------
 // Helpers
@@ -335,6 +359,9 @@ async function executeWithRetry(opts) {
         kind: opts.kind,
         modelId: opts.modelId,
         onSuccess: opts.onSuccess,
+        startedAt: Date.now(),
+        attempt: 0,
+        recentRequests: opts.recentRequests,
     };
     let lastError = null;
     for (let attempt = 0; attempt < opts.maxRetries; attempt++) {
@@ -346,15 +373,25 @@ async function executeWithRetry(opts) {
     return handleAllRetriesFailed(ctx, opts.maxRetries, lastError);
 }
 async function executeSingleAttempt(attempt, ctx) {
+    ctx.attempt = attempt;
     const key = pickKey(ctx, attempt);
     if (!key) {
-        return { kind: 'done', response: buildNoKeysResponse(ctx.state, ctx.requestId, ctx.kind, attempt) };
+        return { kind: 'done', response: buildNoKeysResponse(ctx, attempt) };
     }
     logKeyAttempt(ctx.requestId, key, attempt, ctx.kind);
     // Per-attempt upstream: keys may be mixed-provider within one failover chain.
     // Suffix-routed models (@provider) pin the provider and lose the suffix
     // in the forwarded body; the upstream never sees saros's routing shim.
-    const provider = providerForKey(key, ctx.config, ctx.modelId);
+    const provider = providerForKey(key, ctx.modelId);
+    ctx.lastProvider = provider.id;
+    logger.debug({
+        evt: 'key.selected',
+        requestId: ctx.requestId,
+        keyLabel: key.label,
+        provider: provider.id,
+        attempt,
+        pinned: Boolean(parseProviderSuffix(ctx.modelId)),
+    }, 'key selected');
     const upstreamUrl = resolveUpstreamUrl(ctx.config, provider, ctx.path);
     const sessionHeader = provider.sessionHeader;
     const prepared = ctx.bodyText
@@ -376,17 +413,30 @@ async function executeSingleAttempt(attempt, ctx) {
         provider,
         sessionId: prepared.sessionId,
     });
+    const attemptStarted = Date.now();
     try {
         const response = await fetchWithTimeout(upstreamUrl, fetchOptions, ctx.config.requestTimeoutMs);
         if (response.ok) {
-            markKeySucceeded(ctx.state, key.label);
+            logger.info({
+                evt: 'upstream.response',
+                requestId: ctx.requestId,
+                keyLabel: key.label,
+                provider: provider.id,
+                status: response.status,
+                durationMs: Date.now() - attemptStarted,
+                stream: ctx.kind === 'streaming',
+            }, 'upstream response');
+            recordKeySuccess(ctx.state, key.label, ctx.requestId);
             completeRequest(ctx.state, ctx.requestId, true);
-            return {
-                kind: 'done',
-                response: await ctx.onSuccess(response, ctx.state, ctx.requestId, key.label),
-            };
+            const downstream = await ctx.onSuccess(response, ctx.state, ctx.requestId, key.label, provider.id);
+            // Logged after the success handler: if it throws, the request retries and
+            // this is not a completed request (the retry path logs its own outcome).
+            // The downstream status is what the client saw — the streaming handler
+            // synthesizes a 502 when the upstream 2xx carried no body.
+            logRequestComplete(ctx, downstream.status, key.label);
+            return { kind: 'done', response: downstream };
         }
-        return await handleUpstreamErrorResponse(response, ctx, key);
+        return await handleUpstreamErrorResponse(response, ctx, key, attemptStarted);
     }
     catch (err) {
         return handleNetworkError(err, ctx, key);
@@ -422,20 +472,24 @@ function buildFetchOptions(opts) {
     options.duplex = 'half';
     return options;
 }
-async function handleUpstreamErrorResponse(response, ctx, key) {
+async function handleUpstreamErrorResponse(response, ctx, key, attemptStarted) {
     const errorBody = await response.text();
     const errorType = classifyHttpError(response.status, errorBody);
-    logUpstreamError(ctx.requestId, key, response.status, errorType, errorBody, ctx.kind);
+    logUpstreamError(ctx.requestId, key, response.status, errorType, errorBody, ctx.kind, Date.now() - attemptStarted);
     captureFailedRequest(ctx, response.status, errorBody);
     if (errorType === 'RequestFault') {
         // Client error — don't retry, don't penalise the key
         completeRequest(ctx.state, ctx.requestId, false);
+        logRequestComplete(ctx, response.status, key.label);
         return { kind: 'done', response: buildClientErrorResponse(response.status, errorBody, ctx.requestId) };
     }
     // KeyFault or ServerFault — penalise the key and retry
-    markKeyFailed(ctx.state, key.label, errorType);
+    recordKeyFailure(ctx.state, key.label, errorType, ctx.requestId, {
+        status: response.status,
+        body: errorBody,
+    });
     const lastError = ctx.kind === 'standard'
-        ? new Error(`Upstream error ${response.status}: ${errorBody.slice(0, 200)}`)
+        ? new Error(`Upstream error ${response.status}: ${redactSecrets(errorBody).slice(0, 200)}`)
         : null;
     return { kind: 'retry', lastError };
 }
@@ -453,12 +507,117 @@ function handleNetworkError(err, ctx, key) {
     markKeyFailed(ctx.state, key.label, 'NetworkFault');
     return { kind: 'retry', lastError };
 }
-function buildNoKeysResponse(state, requestId, kind, attempt) {
-    logger.warn({ requestId, attempt, kind }, kind === 'standard'
-        ? 'No API keys available on attempt %d'
-        : 'No API keys available for streaming on attempt %d', attempt);
+/** Human explanation for each reason a key can be out of rotation. */
+const FAILURE_CAUSES = {
+    auth: 'a key was rejected upstream (401): revoked or rotated, or it does not serve this model',
+    quota: 'the account is out of credit or quota for this provider',
+    rate_limit: 'the upstream rate-limited the key (429) — it retries after the cooldown',
+    server: 'the upstream returned 5xx or broke the stream',
+};
+/** Why a key cannot serve a request, in the 503/502 vocabulary. */
+function unavailableState(report) {
+    if (report.disabledByConfig)
+        return 'disabled_by_config';
+    if (!report.entitled)
+        return 'not_entitled';
+    return 'disabled';
+}
+/**
+ * Keys that cannot serve a request right now, with the reason and the upstream
+ * message. Labels are already public; the credential is never included.
+ * `enabled=false` and `consecutiveFailures >= threshold` are the same state —
+ * the circuit breaker flips `enabled` when the counter reaches it.
+ */
+function unavailableKeyReports(state, onlyLabels) {
+    const now = Date.now();
+    return describeKeys(state, now)
+        .filter((report) => !report.enabled || !report.entitled)
+        .filter((report) => !onlyLabels || onlyLabels.includes(report.label))
+        .map((report) => {
+        const lastError = state.keys.find((k) => k.label === report.label)?.lastError ?? null;
+        return {
+            label: report.label,
+            provider: report.provider,
+            state: unavailableState(report),
+            reason: report.lastError?.reason ?? null,
+            lastStatus: report.lastError?.status ?? null,
+            disabledAt: report.disabledAt,
+            retryAfterMs: report.retryAfterMs,
+            detail: lastError?.detail ?? null,
+        };
+    });
+}
+/**
+ * Shared diagnosis for "no key can serve this": which keys are out, why, when
+ * one may return. Pure so it can be tested without an app instance.
+ */
+export function buildKeyDiagnosis(keys, modelId, triedLabels = []) {
+    const retryTimes = keys
+        .map((k) => k.retryAfterMs)
+        .filter((ms) => typeof ms === 'number' && ms > 0);
+    const retryAfterMs = retryTimes.length > 0 ? Math.min(...retryTimes) : null;
+    const causes = new Set();
+    for (const key of keys) {
+        if (key.reason)
+            causes.add(FAILURE_CAUSES[key.reason]);
+    }
+    if (keys.some((k) => k.state === 'not_entitled')) {
+        causes.add('the account has no plan for this provider (403 entitlement)');
+    }
+    if (keys.some((k) => k.state === 'disabled_by_config')) {
+        causes.add('a key is disabled in config (enabled: false) — re-enable it to put it back in rotation');
+    }
+    const pinned = modelId?.includes('@') ? modelId.slice(modelId.lastIndexOf('@') + 1) : null;
+    if (pinned) {
+        causes.add(`the request is pinned to @${pinned} and none of its keys are usable`);
+    }
+    if (triedLabels.length > 0) {
+        // No key was selectable, so every key this request touched failed — say
+        // which ones. Gating this on an empty keys[] hid the case where an
+        // unrelated retired key was listed while the tried keys were not.
+        causes.add(`keys tried for this request: ${triedLabels.join(', ')} — the upstream kept answering with a retryable error`);
+    }
+    return { retryAfterMs, keys, causes: [...causes] };
+}
+/** Build the 503 body sent when no key can be selected at all. */
+export function buildNoKeysBody(keys, requestId, modelId, triedLabels = []) {
+    const diagnosis = buildKeyDiagnosis(keys, modelId, triedLabels);
+    const pinned = modelId?.includes('@') ? modelId.slice(modelId.lastIndexOf('@') + 1) : null;
+    const retryText = diagnosis.retryAfterMs === null ? '' : `; retry in ${Math.ceil(diagnosis.retryAfterMs / 1000)}s`;
+    return {
+        // Kept for clients that match on the legacy message.
+        error: 'All API keys are temporarily unavailable',
+        message: pinned
+            ? `No usable key for provider ${pinned}${retryText}.`
+            : `No usable API key${retryText}.`,
+        requestId,
+        ...diagnosis,
+    };
+}
+function buildNoKeysResponse(ctx, attempt) {
+    const { state, requestId, kind, modelId } = ctx;
+    const triedLabels = [...(state.activeRequests.get(requestId)?.triedKeys ?? [])];
+    // A 503 raised between attempts still follows keys that failed: the body
+    // names them, so the access entry must too. Null stays reserved for the
+    // attempt-0 case, when no key was ever selected.
+    const lastTried = triedLabels.at(-1) ?? null;
+    const keys = unavailableKeyReports(state);
+    const body = buildNoKeysBody(keys, requestId, modelId, triedLabels);
+    logger.warn({
+        evt: 'request.no_keys',
+        requestId,
+        attempt,
+        kind,
+        modelId: modelId ?? null,
+        retryAfterMs: body.retryAfterMs,
+        unavailable: keys.map((k) => ({ label: k.label, reason: k.reason, status: k.lastStatus })),
+    }, 'No API key available for this request');
     completeRequest(state, requestId, false);
-    return new Response(JSON.stringify({ error: 'All API keys are temporarily unavailable', requestId }), { status: 503, headers: { 'content-type': 'application/json' } });
+    logRequestComplete(ctx, 503, lastTried);
+    return new Response(JSON.stringify(body), {
+        status: 503,
+        headers: { 'content-type': 'application/json' },
+    });
 }
 function buildClientErrorResponse(status, errorBody, requestId) {
     // Sanitize upstream error - don't leak internal details
@@ -477,43 +636,140 @@ function buildClientErrorResponse(status, errorBody, requestId) {
     });
 }
 function handleAllRetriesFailed(ctx, maxRetries, lastError) {
+    // Read before completeRequest deletes the context. The last attempt never
+    // goes through failoverRequest (the loop ends), so its key is only in
+    // currentKey — omitting it hides the key that failed last.
+    const active = ctx.state.activeRequests.get(ctx.requestId);
+    const lastKeyLabel = active?.currentKey?.label ?? null;
+    const triedLabels = [
+        ...new Set([...(active?.triedKeys ?? []), ...(active?.currentKey ? [active.currentKey.label] : [])]),
+    ];
     completeRequest(ctx.state, ctx.requestId, false);
-    logger.error({ requestId: ctx.requestId, err: lastError, kind: ctx.kind }, ctx.kind === 'standard'
+    logger.error({ evt: 'request.retries_exhausted', requestId: ctx.requestId, err: lastError, kind: ctx.kind }, ctx.kind === 'standard'
         ? 'All %d retries exhausted, returning 502'
         : 'Streaming failed after %d attempts', maxRetries);
+    logRequestComplete(ctx, 502, lastKeyLabel);
     const message = ctx.kind === 'standard'
         ? 'Upstream service unavailable after multiple attempts'
         : 'Streaming service unavailable after multiple attempts';
-    return new Response(JSON.stringify({ error: 'Bad Gateway', message, requestId: ctx.requestId }), { status: 502, headers: { 'content-type': 'application/json' } });
+    // Status stays 502 (the upstream did answer with errors), but when every key
+    // is now out of rotation the body says which and why — otherwise the caller
+    // sees "unavailable" and has no way to tell a revoked key from a 429.
+    // Scoped to the keys this request tried: an unrelated key retired in config
+    // must not be blamed for the failure.
+    const unavailable = unavailableKeyReports(ctx.state, triedLabels);
+    // Always present: an empty keys[] with a cause is the honest answer when the
+    // pool is healthy but every key was tried.
+    const diagnosis = buildKeyDiagnosis(unavailable, ctx.modelId, triedLabels);
+    return new Response(JSON.stringify({
+        error: 'Bad Gateway',
+        message,
+        requestId: ctx.requestId,
+        ...diagnosis,
+    }), { status: 502, headers: { 'content-type': 'application/json' } });
 }
 // --- Logging helpers (preserve original log message format) ---
+/**
+ * One line per finished request — the access log. Exactly one per request:
+ * success, upstream client error, no usable key (503) and retries exhausted
+ * (502) are mutually exclusive paths.
+ *
+ * The same entry is pushed to the in-memory buffer served by `GET /debug/recent`
+ * (when enabled), so the two views cannot drift. `keyLabel` is the key that
+ * served the request, or the last key tried when the proxy answers 502 or a 503
+ * raised after keys were tried; null only when no key was ever selected.
+ */
+function logRequestComplete(ctx, status, keyLabel) {
+    const entry = {
+        requestId: ctx.requestId,
+        at: new Date().toISOString(),
+        method: ctx.method,
+        path: ctx.path,
+        model: ctx.modelId ?? null,
+        provider: ctx.lastProvider ?? null,
+        keyLabel,
+        status,
+        durationMs: Date.now() - ctx.startedAt,
+        stream: ctx.kind === 'streaming',
+        attempt: ctx.attempt + 1,
+    };
+    logger.info({ evt: 'request.complete', ...entry }, 'request complete');
+    ctx.recentRequests?.push(entry);
+}
 function logKeyAttempt(requestId, key, attempt, kind) {
     logger.info({ requestId, keyLabel: key.label, attempt, kind }, kind === 'standard'
         ? 'Forwarding request to upstream using key %s'
         : 'Streaming request with key %s', maskKey(key.key));
 }
-function logUpstreamError(requestId, key, status, errorType, errorBody, kind) {
+/**
+ * Record a key failure and log the transition. The line is the `key.state`
+ * event: reason + upstream status are what make a flapping key visible without
+ * reading the source.
+ */
+function recordKeyFailure(state, keyLabel, errorType, requestId, info) {
+    markKeyFailed(state, keyLabel, errorType, info);
+    const key = state.keys.find((k) => k.label === keyLabel);
+    if (!key)
+        return;
+    // Below the threshold the key keeps serving — nothing changed, no line.
+    if (key.enabled && key.consecutiveFailures < state.circuitBreakerThreshold)
+        return;
+    logger.warn({
+        evt: 'key.state',
+        requestId,
+        keyLabel,
+        provider: key.provider,
+        enabled: key.enabled,
+        reason: key.lastError?.reason,
+        status: key.lastError?.status,
+        consecutiveFailures: key.consecutiveFailures,
+        cooldownMs: state.circuitBreakerCooldownMs,
+    }, 'Key left rotation');
+}
+/** Mark a key as succeeded and log when it returns from a disable. */
+function recordKeySuccess(state, keyLabel, requestId) {
+    const before = state.keys.find((k) => k.label === keyLabel);
+    if (!before)
+        return;
+    const wasOut = !before.enabled || before.disabledAt !== null;
+    markKeySucceeded(state, keyLabel);
+    if (!wasOut)
+        return;
+    logger.info({ evt: 'key.state', requestId, keyLabel, provider: before.provider, enabled: true }, 'Key back in rotation');
+}
+function logUpstreamError(requestId, key, status, errorType, errorBody, kind, durationMs) {
     if (kind === 'standard') {
-        logger.warn({ requestId, keyLabel: key.label, status, errorType, kind }, 'Upstream returned error: %s', errorBody.slice(0, 200));
+        logger.warn({
+            evt: 'upstream.error',
+            requestId,
+            keyLabel: key.label,
+            status,
+            errorType,
+            kind,
+            durationMs,
+            // Redact before truncating: a 401 that echoes the bearer must not write
+            // a working key into the access log or daemon.log.
+            bodyPrefix: redactSecrets(errorBody).slice(0, 200),
+        }, 'Upstream returned error');
     }
     else {
-        logger.warn({ requestId, keyLabel: key.label, status, kind }, 'Upstream streaming error');
+        logger.warn({ evt: 'upstream.error', requestId, keyLabel: key.label, status, kind, durationMs }, 'Upstream streaming error');
     }
 }
 function logTimeout(requestId, key, timeoutMs, kind) {
-    logger.warn({ requestId, keyLabel: key.label, kind }, kind === 'standard'
-        ? 'Request timed out after %dms, retrying with next key'
-        : 'Streaming request timed out after %dms, retrying with next key', timeoutMs);
+    logger.warn({ evt: 'upstream.timeout', requestId, keyLabel: key.label, kind, timeoutMs }, kind === 'standard'
+        ? 'Request timed out, retrying with next key'
+        : 'Streaming request timed out, retrying with next key');
 }
 function logNetworkError(requestId, key, err, kind) {
-    logger.error({ requestId, keyLabel: key.label, err, kind }, kind === 'standard'
+    logger.error({ evt: 'upstream.network_error', requestId, keyLabel: key.label, err, kind }, kind === 'standard'
         ? 'Network error forwarding to upstream'
         : 'Network error during streaming');
 }
 // ---------------------------------------------------------------------------
 // Success handlers
 // ---------------------------------------------------------------------------
-const standardSuccessHandler = async (response, state, requestId, keyLabel) => {
+const standardSuccessHandler = async (response, state, requestId, keyLabel, provider) => {
     logger.info({ requestId, keyLabel, status: response.status }, 'Upstream request succeeded');
     const responseBody = await response.text();
     // Non-streaming bodies report the request's token usage at the top level
@@ -523,12 +779,13 @@ const standardSuccessHandler = async (response, state, requestId, keyLabel) => {
     const downstreamHeaders = buildDownstreamHeaders(response.headers);
     downstreamHeaders.set('X-Proxy-Key-Label', keyLabel);
     downstreamHeaders.set('X-Proxy-Request-Id', requestId);
+    downstreamHeaders.set('X-Proxy-Provider', provider);
     return new Response(responseBody, {
         status: response.status,
         headers: downstreamHeaders,
     });
 };
-const streamingSuccessHandler = async (response, state, requestId, keyLabel) => {
+const streamingSuccessHandler = async (response, state, requestId, keyLabel, provider) => {
     if (!response.body) {
         logger.error({ requestId, keyLabel }, 'Streaming response missing body');
         return new Response(JSON.stringify({ error: 'Bad Gateway', message: 'Upstream response missing body', requestId }), { status: 502, headers: { 'content-type': 'application/json' } });
@@ -540,6 +797,7 @@ const streamingSuccessHandler = async (response, state, requestId, keyLabel) =>
     responseHeaders.set('connection', 'keep-alive');
     responseHeaders.set('X-Proxy-Key-Label', keyLabel);
     responseHeaders.set('X-Proxy-Request-Id', requestId);
+    responseHeaders.set('X-Proxy-Provider', provider);
     return new Response(wrappedStream, { status: 200, headers: responseHeaders });
 };
 /**
@@ -559,6 +817,7 @@ async function handleWithFailover(opts) {
         modelId: opts.modelId,
         onSuccess: standardSuccessHandler,
         maxRetries: opts.maxRetries ?? MAX_RETRIES,
+        recentRequests: opts.recentRequests,
     });
 }
 /**
@@ -579,24 +838,50 @@ async function handleStreamingRequest(opts) {
         modelId: opts.modelId,
         onSuccess: streamingSuccessHandler,
         maxRetries: opts.maxRetries ?? MAX_RETRIES,
+        recentRequests: opts.recentRequests,
     });
 }
 /**
- * Pull token counts out of one SSE event. Chat completions carry `usage` at the
- * top level; the OpenAI Responses schema nests it under `response.usage` on
- * `response.completed`, so a top-level-only reader books zero for those streams.
+ * Pull token counts out of one SSE event, across every streaming surface saros
+ * relays: Chat Completions (top-level `usage`), OpenAI Responses
+ * (`response.usage` on `response.completed`) and Anthropic Messages
+ * (`message.usage` on `message_start`). A reader that only knew the first
+ * shape booked zero for the other two. `message` is only consulted on
+ * `message_start`: other event types may carry an unrelated business object
+ * under that key (e.g. streamed decision payloads).
  */
 function extractEventUsage(parsed) {
     if (parsed.usage)
         return parsed.usage;
-    const response = parsed.response;
-    if (response && typeof response === 'object' && !Array.isArray(response)) {
-        const nested = response.usage;
+    const containers = [parsed.response];
+    if (parsed.type === 'message_start')
+        containers.push(parsed.message);
+    for (const container of containers) {
+        if (!container || typeof container !== 'object' || Array.isArray(container))
+            continue;
+        const nested = container.usage;
         if (nested)
             return nested;
     }
     return null;
 }
+/**
+ * Merge one event's usage into the stream total. Anthropic splits the counts
+ * across events — `message_start` carries the prompt tokens, `message_delta`
+ * the cumulative output tokens — so last-one-wins would book zero prompt
+ * tokens for every Messages stream. Every documented surface reports
+ * cumulative per-field counts, so the largest value seen is the final one;
+ * identical repeats (CommandCode sends `usage` twice) are no-ops.
+ */
+function mergeStreamUsage(target, next) {
+    if (!next || typeof next !== 'object' || Array.isArray(next))
+        return;
+    for (const [field, value] of Object.entries(next)) {
+        if (typeof value !== 'number' || !Number.isFinite(value))
+            continue;
+        target[field] = Math.max(target[field] ?? 0, value);
+    }
+}
 /**
  * Wrap an upstream SSE stream with error detection.
  * Parses SSE events properly to avoid false positives on
@@ -609,20 +894,17 @@ function wrapStreamWithErrorDetection(upstreamStream, state, requestId, keyLabel
     const decoder = new TextDecoder();
     const encoder = new TextEncoder();
     let buffer = '';
-    // Upstreams may report the same `usage` on more than one chunk: CommandCode
-    // sends it on the finish chunk (`choices[0].finish_reason`) and again on the
-    // trailing usage-only chunk (`choices: []`), so booking per chunk would
-    // count every streamed request twice. Keep the canonical usage-only chunk
-    // (last one wins, OpenAI's `include_usage` contract) and fall back to the
-    // finish chunk's copy; book exactly once, when the stream ends.
-    let lastUsage = null;
-    let usageOnlyChunk = null;
+    // Usage is booked exactly once, when the stream ends. Upstreams may repeat
+    // the same counts on more than one chunk (CommandCode sends them on the
+    // finish chunk and on the trailing usage-only chunk) and Anthropic splits
+    // them across events, so counts are merged per field instead of replaced.
+    let streamUsage = null;
     let usageBooked = false;
     const bookUsage = () => {
         if (usageBooked)
             return;
         usageBooked = true;
-        recordKeyTokens(keyLabel, usageOnlyChunk ?? lastUsage);
+        recordKeyTokens(keyLabel, streamUsage);
     };
     return new ReadableStream({
         async pull(controller) {
@@ -648,7 +930,9 @@ function wrapStreamWithErrorDetection(upstreamStream, state, requestId, keyLabel
                         const parsed = JSON.parse(dataLine.slice(5).trim());
                         if (parsed && (parsed.error || parsed.type === 'error')) {
                             logger.warn({ requestId, keyLabel }, 'Error detected in stream');
-                            markKeyFailed(state, keyLabel, 'ServerFault');
+                            recordKeyFailure(state, keyLabel, 'ServerFault', requestId, {
+                                body: JSON.stringify(parsed),
+                            });
                             // Whatever the upstream reported before failing was consumed.
                             bookUsage();
                             controller.enqueue(encoder.encode('data: {"type":"proxy_error","message":"Upstream stream error"}\n\n'));
@@ -656,15 +940,12 @@ function wrapStreamWithErrorDetection(upstreamStream, state, requestId, keyLabel
                             return;
                         }
                         // Intermediate chunks send `usage: null`; the counts arrive on the
-                        // finish chunk and/or the usage-only chunk (see bookUsage). The
-                        // OpenAI Responses schema nests them under `response.usage` on the
-                        // `response.completed` event instead of at the top level.
+                        // finish chunk and/or a usage-only chunk (see bookUsage), or split
+                        // across `response.completed` / `message_start` / `message_delta`.
                         const eventUsage = extractEventUsage(parsed);
                         if (eventUsage) {
-                            lastUsage = eventUsage;
-                            if (Array.isArray(parsed.choices) && parsed.choices.length === 0) {
-                                usageOnlyChunk = eventUsage;
-                            }
+                            streamUsage ??= {};
+                            mergeStreamUsage(streamUsage, eventUsage);
                         }
                     }
                     catch {
@@ -676,7 +957,9 @@ function wrapStreamWithErrorDetection(upstreamStream, state, requestId, keyLabel
             }
             catch (err) {
                 logger.error({ requestId, keyLabel, err }, 'Stream read error');
-                markKeyFailed(state, keyLabel, 'ServerFault');
+                recordKeyFailure(state, keyLabel, 'ServerFault', requestId, {
+                    body: err instanceof Error ? err.message : String(err),
+                });
                 bookUsage();
                 controller.enqueue(encoder.encode('data: {"type":"proxy_error","message":"Stream interrupted"}\n\n'));
                 controller.close();
@@ -697,6 +980,9 @@ export function createProxyApp(config) {
         circuitBreakerThreshold: config.circuitBreakerThreshold,
         circuitBreakerCooldownMs: config.circuitBreakerCooldownMs,
     });
+    // Last proxied requests, in memory only. `SAROS_DEBUG_BUFFER=0` removes the
+    // instance entirely: `/debug/recent` answers 404 and nothing is stored.
+    const recentRequests = isDebugBufferEnabled() ? createRecentRequestBuffer() : undefined;
     // Model→provider affinity: requests naming a provider-specific model
     // (e.g. commandcode's vendor-prefixed ids) are routed to a matching key
     // first; 'maybe' providers remain as fallback. Bare-name matching lets
@@ -736,6 +1022,9 @@ export function createProxyApp(config) {
     }));
     // --- CORS (configurable origins) ---
     app.use('*', cors({
+        // Browser JS cannot read a response header unless it is exposed: without
+        // this the X-Proxy-* annotations are visible to curl but not to fetch().
+        exposeHeaders: ['X-Proxy-Key-Label', 'X-Proxy-Request-Id', 'X-Proxy-Provider'],
         origin: (origin) => {
             if (!origin)
                 return null; // Non-browser requests (no Origin header) — skip CORS
@@ -769,6 +1058,9 @@ export function createProxyApp(config) {
             activeRequests: activeCount,
             circuitBreakerThreshold: state.circuitBreakerThreshold,
             circuitBreakerCooldownMs: state.circuitBreakerCooldownMs,
+            // Per-key state: what is out of rotation, why, and when it may return.
+            // Never contains the credential.
+            keys: describeKeys(state),
             // Per-provider key counts (multi-provider pools)
             providers: keys.reduce((acc, k) => {
                 const entry = (acc[k.provider] ??= { total: 0, enabled: 0 });
@@ -779,6 +1071,14 @@ export function createProxyApp(config) {
             }, {}),
         });
     });
+    // --- Recent proxied requests (in-memory, opt-out) ---
+    // Same entries as the `request.complete` access lines: requestId, provider,
+    // key label and outcome — never credentials, headers or bodies.
+    app.get('/debug/recent', (c) => {
+        if (!recentRequests)
+            return c.json({ error: 'Not Found' }, 404);
+        return c.json(recentRequests.snapshot());
+    });
     // --- Models discovery endpoint (OpenAI-compatible) ---
     // Returns the full model list so the OpenCode client can discover all
     // available models. Without this, OpenCode falls back to a hardcoded
@@ -823,6 +1123,7 @@ export function createProxyApp(config) {
                 incomingHeaders,
                 bodyText,
                 modelId: typeof parsedBody.model === 'string' ? parsedBody.model : undefined,
+                recentRequests,
             });
         }
         // Non-streaming request with failover
@@ -835,6 +1136,7 @@ export function createProxyApp(config) {
             incomingHeaders,
             bodyText,
             modelId: typeof parsedBody?.model === 'string' ? parsedBody.model : undefined,
+            recentRequests,
         });
     }
     // Unified catch-all: rewrites /v1/* → /zen/go/v1/* for upstream forwarding