zevairouter 1.0.154 → 1.0.155

package/src/sse/handlers/tts.js

@@ -1,114 +0,0 @@
-import {
-  extractApiKey, isValidApiKey,
-  getProviderCredentials, markAccountUnavailable,
-} from "../services/auth.js";
-import { getSettings } from "@/lib/localDb";
-import { getModelInfo, getComboModels } from "../services/model.js";
-import { handleTtsCore } from "open-sse/handlers/ttsCore.js";
-import { errorResponse, unavailableResponse } from "open-sse/utils/error.js";
-import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
-import { AI_PROVIDERS } from "@/shared/constants/providers";
-import { handleComboChat } from "open-sse/services/combo.js";
-import * as log from "../utils/logger.js";
-
-// Derived from providers.js: any TTS provider not noAuth requires stored credentials
-const CREDENTIALED_PROVIDERS = new Set(
-  Object.entries(AI_PROVIDERS)
-    .filter(([, p]) => p.serviceKinds?.includes("tts") && !p.noAuth && p.ttsConfig?.authType !== "none")
-    .map(([id]) => id)
-);
-
-export async function handleTts(request) {
-  let body;
-  try {
-    body = await request.json();
-  } catch {
-    return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
-  }
-
-  const url = new URL(request.url);
-  const modelStr = body.model;
-  const responseFormat = url.searchParams.get("response_format") || "mp3"; // mp3 (default) | json
-  const language = body.language || ""; // Optional language hint (currently used by Gemini)
-  log.request("POST", `${url.pathname} | ${modelStr} | format=${responseFormat}${language ? ` | lang=${language}` : ""}`);
-
-  const settings = await getSettings();
-  if (settings.requireApiKey) {
-    const apiKey = extractApiKey(request);
-    if (!apiKey) return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key");
-    const valid = await isValidApiKey(apiKey);
-    if (!valid) return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
-  }
-
-  if (!modelStr) return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model");
-  if (!body.input) return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing required field: input");
-
-  // Combo expansion: model may be a combo name → run fallback/round-robin across models
-  const comboModels = await getComboModels(modelStr);
-  if (comboModels) {
-    const comboStrategies = settings.comboStrategies || {};
-    const comboStrategy = comboStrategies[modelStr]?.fallbackStrategy || settings.comboStrategy || "fallback";
-    const comboStickyLimit = settings.comboStickyRoundRobinLimit;
-    log.info("TTS", `Combo "${modelStr}" with ${comboModels.length} models (strategy: ${comboStrategy}, sticky: ${comboStickyLimit})`);
-    return handleComboChat({
-      body,
-      models: comboModels,
-      handleSingleModel: (b, m) => handleSingleModelTts(b, m, responseFormat, language),
-      log,
-      comboName: modelStr,
-      comboStrategy,
-      comboStickyLimit,
-    });
-  }
-
-  return handleSingleModelTts(body, modelStr, responseFormat, language);
-}
-
-async function handleSingleModelTts(body, modelStr, responseFormat, language) {
-  const modelInfo = await getModelInfo(modelStr);
-  if (!modelInfo.provider) return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid model format");
-
-  const { provider, model } = modelInfo;
-  log.info("ROUTING", `Provider: ${provider}, Voice: ${model}`);
-
-  // noAuth providers — no credential needed
-  if (!CREDENTIALED_PROVIDERS.has(provider)) {
-    const result = await handleTtsCore({ provider, model, input: body.input, responseFormat, language });
-    if (result.success) return result.response;
-    return errorResponse(result.status || HTTP_STATUS.BAD_GATEWAY, result.error || "TTS failed");
-  }
-
-  // Credentialed providers — fallback loop (same pattern as embeddings)
-  const excludeConnectionIds = new Set();
-  let lastError = null;
-  let lastStatus = null;
-
-  while (true) {
-    const credentials = await getProviderCredentials(provider, excludeConnectionIds, model);
-
-    if (!credentials || credentials.allRateLimited) {
-      if (credentials?.allRateLimited) {
-        const msg = lastError || credentials.lastError || "Unavailable";
-        const status = lastStatus || Number(credentials.lastErrorCode) || HTTP_STATUS.SERVICE_UNAVAILABLE;
-        return unavailableResponse(status, `[${provider}/${model}] ${msg}`, credentials.retryAfter, credentials.retryAfterHuman);
-      }
-      if (excludeConnectionIds.size === 0) return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`);
-      return errorResponse(lastStatus || HTTP_STATUS.SERVICE_UNAVAILABLE, lastError || "All accounts unavailable");
-    }
-
-    log.info("AUTH", `\x1b[32mUsing ${provider} account: ${credentials.connectionName}\x1b[0m`);
-
-    const result = await handleTtsCore({ provider, model, input: body.input, credentials, responseFormat, language });
-
-    if (result.success) return result.response;
-
-    const { shouldFallback } = await markAccountUnavailable(credentials.connectionId, result.status, result.error, provider, model);
-    if (shouldFallback) {
-      excludeConnectionIds.add(credentials.connectionId);
-      lastError = result.error;
-      lastStatus = result.status;
-      continue;
-    }
-    return result.response || errorResponse(result.status, result.error);
-  }
-}

package/src/sse/services/auth.js

@@ -1,576 +0,0 @@
-import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings, getQuotaSnapshot, getAllQuotaSnapshots, getProxyPools } from "@/lib/localDb";
-import { resolveConnectionProxyConfig, pickProxyPoolId } from "@/lib/network/connectionProxy";
-import { formatRetryAfter, checkFallbackError, isModelLockActive, isModelLockRecoveredByQuota, isModelRecoveryProbeActive, buildModelLockUpdate, buildModelRecoveryProbeUpdate, getModelRecoveryProbeKey, getModelLockIdentity, getModelLockUntil, getEarliestModelLockUntil, getQuotaRemainingScore, isNonAccountError, isSharedEgressError, extractProviderErrorCode, getQuotaResetAt, getQuotaExhaustedUntil } from "open-sse/services/accountFallback.js";
-import { MAX_RATE_LIMIT_COOLDOWN_MS } from "open-sse/config/errorConfig.js";
-import { resolveProviderId, FREE_PROVIDERS } from "@/shared/constants/providers.js";
-import * as log from "../utils/logger.js";
-
-// Mutex to prevent race conditions during account selection
-let selectionMutex = Promise.resolve();
-
-/**
- * Get provider credentials from localDb
- * Filters out unavailable accounts and returns the selected account based on strategy
- * @param {string} provider - Provider name
- * @param {Set<string>|string|null} excludeConnectionIds - Connection ID(s) to exclude (for retry with next account)
- * @param {string|null} model - Model name for per-model rate limit filtering
- */
-export async function getProviderCredentials(provider, excludeConnectionIds = null, model = null, options = {}) {
-  // Normalize to Set for consistent handling
-  const excludeSet = excludeConnectionIds instanceof Set
-    ? excludeConnectionIds
-    : (excludeConnectionIds ? new Set([excludeConnectionIds]) : new Set());
-  const preferredConnectionId = options?.preferredConnectionId || null;
-  const strictPreferred = !!options?.strictPreferred;
-  // Acquire mutex to prevent race conditions
-  const currentMutex = selectionMutex;
-  let resolveMutex;
-  selectionMutex = new Promise(resolve => { resolveMutex = resolve; });
-
-  try {
-    await currentMutex;
-
-    // Resolve alias to provider ID (e.g., "kc" -> "kilocode")
-    const providerId = resolveProviderId(provider);
-
-    // Inject a virtual connection for no-auth free providers (with optional proxy pool / rotation)
-    if (FREE_PROVIDERS[providerId]?.noAuth) {
-      const settings = await getSettings();
-      const override = (settings.providerStrategies || {})[providerId] || {};
-      const strategy = override.rotateStrategy || "none";
-      let pickedId = override.proxyPoolId || null;
-      if (strategy !== "none") {
-        const allPools = await getProxyPools({ isActive: true });
-        const poolIds = allPools.filter((p) => p.proxyUrl).map((p) => p.id);
-        pickedId = pickProxyPoolId(poolIds, strategy, providerId);
-      }
-      const resolvedProxy = await resolveConnectionProxyConfig({ proxyPoolId: pickedId || "" });
-      return {
-        id: "noauth",
-        connectionName: "Public",
-        isActive: true,
-        accessToken: "public",
-        providerSpecificData: {
-          connectionProxyEnabled: resolvedProxy.connectionProxyEnabled,
-          connectionProxyUrl: resolvedProxy.connectionProxyUrl,
-          connectionNoProxy: resolvedProxy.connectionNoProxy,
-          connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
-          vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
-        },
-      };
-    }
-
-    const connections = await getProviderConnections({ provider: providerId, isActive: true });
-    log.debug("AUTH", `${provider} | total connections: ${connections.length}, excludeIds: ${excludeSet.size > 0 ? [...excludeSet].join(",") : "none"}, model: ${model || "any"}`);
-
-    if (connections.length === 0) {
-      log.warn("AUTH", `No credentials for ${provider}`);
-      return null;
-    }
-
-    const quotaExhausted = new Map();
-    if (providerId === "codebuddy-cn") {
-      const snapshots = await getAllQuotaSnapshots(providerId);
-      for (const snapshot of snapshots) {
-        const until = getQuotaExhaustedUntil(snapshot.usage);
-        if (until) quotaExhausted.set(snapshot.connectionId, until);
-      }
-    }
-
-    // Codex can return a precise usage-limit reset which creates a long-lived
-    // model lock. If a newer quota poll explicitly reports capacity again, let
-    // that account be probed once instead of serving a stale all-rate-limited
-    // response until the old reset timestamp. A new failure is newer than the
-    // snapshot, so it immediately restores the lock without a retry loop.
-    const recoveredModelLocks = new Set();
-    const recoveryProbesInFlight = new Set();
-    if (providerId === "codex" && model) {
-      const snapshots = await getAllQuotaSnapshots(providerId);
-      const byConnection = new Map(snapshots.map((snapshot) => [snapshot.connectionId, snapshot]));
-      for (const connection of connections) {
-        if (isModelRecoveryProbeActive(connection, model, providerId)) {
-          recoveryProbesInFlight.add(connection.id);
-        }
-        if (isModelLockRecoveredByQuota(
-          connection,
-          model,
-          byConnection.get(connection.id),
-          providerId,
-          { ignoreProbe: true },
-        )) {
-          if (!recoveryProbesInFlight.has(connection.id)) {
-            recoveredModelLocks.add(connection.id);
-          }
-        }
-      }
-    }
-
-    const hasBlockingModelLock = (connection) => (
-      isModelLockActive(connection, model, providerId) && !recoveredModelLocks.has(connection.id)
-    );
-    const getBlockingModelLockUntil = (connection) => {
-      if (
-        recoveryProbesInFlight.has(connection.id)
-      ) {
-        // Another request owns the bounded recovery probe. Poll briefly instead
-        // of advertising the stale multi-day lock which the probe may clear.
-        return new Date(Date.now() + 1000).toISOString();
-      }
-      return getModelLockUntil(connection, model, providerId);
-    };
-
-    const baseAvailableConnections = connections.filter(c => {
-      if (excludeSet.has(c.id)) return false;
-      if (hasBlockingModelLock(c)) return false;
-      if (quotaExhausted.has(c.id)) return false;
-      return true;
-    });
-
-    // Filter out model-locked and excluded connections. Strict pinning is used
-    // by Codex gateway account aliases, where silently switching identity would
-    // break the user's mental model of "this session/account".
-    let availableConnections = baseAvailableConnections;
-    if (strictPreferred && preferredConnectionId) {
-      const preferred = connections.find((c) => c.id === preferredConnectionId);
-      if (!preferred) {
-        log.warn("AUTH", `${provider} | pinned account not found: ${preferredConnectionId}`);
-        return null;
-      }
-      if (excludeSet.has(preferred.id)) {
-        log.warn("AUTH", `${provider} | pinned account excluded after failure: ${preferred.id?.slice(0, 8)}`);
-        return null;
-      }
-      if (hasBlockingModelLock(preferred) || quotaExhausted.has(preferred.id)) {
-        const retryAfter = [
-          getBlockingModelLockUntil(preferred),
-          quotaExhausted.has(preferred.id)
-            ? new Date(quotaExhausted.get(preferred.id)).toISOString()
-            : null,
-        ].filter(Boolean).sort().at(-1);
-        return {
-          allRateLimited: true,
-          retryAfter,
-          retryAfterHuman: formatRetryAfter(retryAfter),
-          lastError: preferred.lastError || "Pinned account unavailable",
-          lastErrorCode: preferred.errorCode || null,
-        };
-      }
-      availableConnections = [preferred];
-    }
-
-    log.debug("AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}`);
-    connections.forEach(c => {
-      const excluded = excludeSet.has(c.id);
-      const locked = hasBlockingModelLock(c);
-      const recovered = recoveredModelLocks.has(c.id);
-      const exhaustedUntil = quotaExhausted.get(c.id);
-      if (excluded || locked || exhaustedUntil || recovered) {
-        const lockUntil = getEarliestModelLockUntil(c);
-        const probing = recoveryProbesInFlight.has(c.id);
-        log.debug("AUTH", `  \u2192 ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${locked ? `modelLocked(${model}) until ${lockUntil}` : ""} ${recovered ? `staleLockRecovered(${model})` : ""} ${probing ? `recoveryProbeInFlight(${model})` : ""} ${exhaustedUntil ? `quotaExhausted until ${new Date(exhaustedUntil).toISOString()}` : ""}`);
-      }
-    });
-
-    if (availableConnections.length === 0) {
-      // Find earliest lock expiry across all connections for retry timing
-      const lockedConns = connections.filter(c => hasBlockingModelLock(c));
-      const expiries = connections.map((connection) => [
-        hasBlockingModelLock(connection)
-          ? getBlockingModelLockUntil(connection)
-          : null,
-        quotaExhausted.has(connection.id)
-          ? new Date(quotaExhausted.get(connection.id)).toISOString()
-          : null,
-      ].filter(Boolean).sort().at(-1)).filter(Boolean);
-      const earliest = expiries.sort()[0] || null;
-      if (earliest) {
-        const earliestConn = lockedConns[0];
-        log.warn("AUTH", `${provider} | all ${connections.length} accounts locked for ${model || "all"} (${formatRetryAfter(earliest)}) | lastError=${earliestConn?.lastError?.slice(0, 50)}`);
-        return {
-          allRateLimited: true,
-          retryAfter: earliest,
-          retryAfterHuman: formatRetryAfter(earliest),
-          lastError: earliestConn?.lastError || null,
-          lastErrorCode: earliestConn?.errorCode || null
-        };
-      }
-      log.warn("AUTH", `${provider} | all ${connections.length} accounts unavailable`);
-      return null;
-    }
-
-    const settings = await getSettings();
-    // Per-provider strategy overrides global setting
-    const providerOverride = (settings.providerStrategies || {})[providerId] || {};
-    const strategy = providerOverride.fallbackStrategy || settings.fallbackStrategy || "fill-first";
-
-    let connection;
-    // Pin to preferred connection if specified and available
-    if (preferredConnectionId) {
-      connection = availableConnections.find((c) => c.id === preferredConnectionId);
-      if (connection) {
-        log.info("AUTH", `${provider} | pinned to ${connection.id?.slice(0, 8)} (${connection.name || connection.email || "unnamed"})`);
-      }
-    }
-    if (connection) {
-      // skip strategy
-    } else if (strictPreferred && preferredConnectionId) {
-      log.warn("AUTH", `${provider} | pinned account unavailable: ${preferredConnectionId}`);
-      return null;
-    } else if (strategy === "round-robin") {
-      const stickyLimit = providerOverride.stickyRoundRobinLimit || settings.stickyRoundRobinLimit || 3;
-
-      // Sort by lastUsed (most recent first) to find current candidate
-      const byRecency = [...availableConnections].sort((a, b) => {
-        if (!a.lastUsedAt && !b.lastUsedAt) return (a.priority || 999) - (b.priority || 999);
-        if (!a.lastUsedAt) return 1;
-        if (!b.lastUsedAt) return -1;
-        return new Date(b.lastUsedAt) - new Date(a.lastUsedAt);
-      });
-
-      const current = byRecency[0];
-      const currentCount = current?.consecutiveUseCount || 0;
-
-      if (current && current.lastUsedAt && currentCount < stickyLimit) {
-        // Stay with current account
-        connection = current;
-        // Update lastUsedAt and increment count (await to ensure persistence)
-        await updateProviderConnection(connection.id, {
-          lastUsedAt: new Date().toISOString(),
-          consecutiveUseCount: (connection.consecutiveUseCount || 0) + 1
-        });
-      } else {
-        // Pick the least recently used (excluding current if possible)
-        const sortedByOldest = [...availableConnections].sort((a, b) => {
-          if (!a.lastUsedAt && !b.lastUsedAt) return (a.priority || 999) - (b.priority || 999);
-          if (!a.lastUsedAt) return -1;
-          if (!b.lastUsedAt) return 1;
-          return new Date(a.lastUsedAt) - new Date(b.lastUsedAt);
-        });
-
-        connection = sortedByOldest[0];
-
-        // Update lastUsedAt and reset count to 1 (await to ensure persistence)
-        await updateProviderConnection(connection.id, {
-          lastUsedAt: new Date().toISOString(),
-          consecutiveUseCount: 1
-        });
-      }
-    } else if (strategy === "most-quota") {
-      // Pick the connection with the most remaining quota for this model, read
-      // from the server-side quota cache (refreshed by the background loop).
-      // Connections with no cached snapshot are treated as unknown and ranked
-      // after any that have a known positive remaining. Falls back to priority
-      // order when nothing has usable cache data.
-      const scored = await Promise.all(
-        availableConnections.map(async (c) => {
-          let remaining = -1; // unknown
-          try {
-            const snap = await getQuotaSnapshot(c.id);
-            remaining = getQuotaRemainingScore(snap?.usage, providerId, model);
-          } catch { /* treat as unknown */ }
-          return { conn: c, remaining };
-        })
-      );
-
-      // Prefer known-positive remaining (highest first); unknown (-1) ranked by
-      // priority as a tiebreaker after known ones.
-      scored.sort((a, b) => {
-        if (a.remaining !== b.remaining) return b.remaining - a.remaining;
-        return (a.conn.priority || 999) - (b.conn.priority || 999);
-      });
-
-      // Skip accounts known to be fully exhausted (remaining === 0) unless every
-      // account is exhausted, in which case fall through to the best available.
-      const withQuota = scored.filter((s) => s.remaining !== 0);
-      connection = (withQuota[0] || scored[0]).conn;
-      await updateProviderConnection(connection.id, {
-        lastUsedAt: new Date().toISOString(),
-        consecutiveUseCount: 1,
-      });
-    } else if (strategy === "lowest-latency") {
-      // Pick the account with the lowest avg response latency over a recent
-      // window. Unknown-latency accounts are ranked after known ones (treated
-      // as "unproven", not "slow") so new accounts still get traffic to build
-      // up their stats. Tiebreak: priority.
-      const { getLatencyStats } = await import("@/lib/db/index.js");
-      const latencyStats = await getLatencyStats();
-      const scored = availableConnections.map((c) => {
-        const stat = latencyStats.get(`${c.provider}|${c.id}`) || latencyStats.get(`${providerId}|${c.id}`);
-        return { conn: c, avgMs: stat ? stat.avgMs : null, samples: stat ? stat.samples : 0 };
-      });
-      scored.sort((a, b) => {
-        if (a.avgMs === null && b.avgMs === null) return (a.conn.priority || 999) - (b.conn.priority || 999);
-        if (a.avgMs === null) return 1;
-        if (b.avgMs === null) return -1;
-        if (a.avgMs !== b.avgMs) return a.avgMs - b.avgMs;
-        return (a.conn.priority || 999) - (b.conn.priority || 999);
-      });
-      connection = scored[0].conn;
-      log.info("AUTH", `${provider} | lowest-latency → ${connection.id?.slice(0, 8)} (${scored[0].avgMs ?? "?"}ms avg, ${scored[0].samples} samples)`);
-      await updateProviderConnection(connection.id, {
-        lastUsedAt: new Date().toISOString(),
-        consecutiveUseCount: 1,
-      });
-    } else {
-      // Default: fill-first (already sorted by priority in getProviderConnections)
-      connection = availableConnections[0];
-    }
-
-    if (recoveredModelLocks.has(connection.id)) {
-      const probeUpdate = buildModelRecoveryProbeUpdate(model, providerId);
-      await updateProviderConnection(connection.id, probeUpdate);
-      connection = { ...connection, ...probeUpdate };
-    }
-
-    const resolvedProxy = await resolveConnectionProxyConfig(connection.providerSpecificData || {});
-
-    return {
-      authType: connection.authType,
-      apiKey: connection.apiKey,
-      accessToken: connection.accessToken,
-      refreshToken: connection.refreshToken,
-      idToken: connection.idToken,
-      expiresAt: connection.expiresAt,
-      expiresIn: connection.expiresIn,
-      lastRefreshAt: connection.lastRefreshAt,
-      projectId: connection.projectId || connection.providerSpecificData?.projectId || null,
-      connectionName: connection.displayName || connection.name || connection.email || connection.id,
-      copilotToken: connection.providerSpecificData?.copilotToken,
-      providerSpecificData: {
-        ...(connection.providerSpecificData || {}),
-        connectionProxyEnabled: resolvedProxy.connectionProxyEnabled,
-        connectionProxyUrl: resolvedProxy.connectionProxyUrl,
-        connectionNoProxy: resolvedProxy.connectionNoProxy,
-        connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
-        vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
-      },
-      connectionId: connection.id,
-      // Include current status for optimization check
-      testStatus: connection.testStatus,
-      lastError: connection.lastError,
-      // Pass full connection for clearAccountError to read modelLock_* keys
-      _connection: connection
-    };
-  } finally {
-    if (resolveMutex) resolveMutex();
-  }
-}
-
-/**
- * Mark account+model as unavailable \u2014 locks modelLock_${model} in DB.
- * All errors (429, 401, 5xx, etc.) lock per model, not per account.
- * @param {string} connectionId
- * @param {number} status - HTTP status code from upstream
- * @param {string} errorText
- * @param {string|null} provider
- * @param {string|null} model - The specific model that triggered the error
- * @returns shouldFallback: boolean, cooldownMs: number
- */
-export async function markAccountUnavailable(connectionId, status, errorText, provider = null, model = null, resetsAtMs = null) {
-  if (!connectionId || connectionId === "noauth") return { shouldFallback: false, cooldownMs: 0 };
-  const providerId = resolveProviderId(provider);
-  const probeKey = getModelRecoveryProbeKey(model, providerId);
-  if (isSharedEgressError(status, errorText)) {
-    await updateProviderConnection(connectionId, { [probeKey]: null });
-    log.warn("AUTH", `shared egress/WAF error; fallback without locking account [${status}]`);
-    return { shouldFallback: true, cooldownMs: 0 };
-  }
-  if (isNonAccountError(status, errorText)) {
-    await updateProviderConnection(connectionId, { [probeKey]: null });
-    log.warn("AUTH", `non-account error; no fallback [${status}] ${typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error"}`);
-    return { shouldFallback: false, cooldownMs: 0 };
-  }
-  const connections = await getProviderConnections({ provider: providerId });
-  const conn = connections.find(c => c.id === connectionId);
-  const backoffLevel = conn?.backoffLevel || 0;
-  const providerErrorCode = extractProviderErrorCode(errorText);
-
-  // CodeBuddy quota/eligibility errors are model-specific. Re-probe after the
-  // real quota-cycle reset instead of retrying every generic 403 cooldown.
-  const isCodeBuddyModelError =
-    providerId === "codebuddy-cn" &&
-    (providerErrorCode === 14018 || providerErrorCode === 11140);
-
-  if (isCodeBuddyModelError) {
-    const snapshot = await getQuotaSnapshot(connectionId);
-    resetsAtMs = getQuotaResetAt(snapshot?.usage) || resetsAtMs;
-  }
-
-  // Provider-specific precise cooldown (e.g. codex usage_limit_reached resets_at) overrides backoff
-  let shouldFallback, cooldownMs, newBackoffLevel;
-  if (resetsAtMs && resetsAtMs > Date.now()) {
-    shouldFallback = true;
-    cooldownMs = Math.min(resetsAtMs - Date.now(), MAX_RATE_LIMIT_COOLDOWN_MS);
-    newBackoffLevel = 0;
-  } else {
-    ({ shouldFallback, cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel));
-  }
-  if (!shouldFallback) {
-    await updateProviderConnection(connectionId, { [probeKey]: null });
-    return { shouldFallback: false, cooldownMs: 0 };
-  }
-
-  const reason = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error";
-  const lockModel = getModelLockIdentity(providerId, model);
-  const lockUpdate = buildModelLockUpdate(lockModel, cooldownMs, providerId);
-  const lockedAt = new Date().toISOString();
-
-  await updateProviderConnection(connectionId, {
-    ...lockUpdate,
-    [probeKey]: null,
-    testStatus: "unavailable",
-    lastError: reason,
-    errorCode: status,
-    providerErrorCode,
-    modelLockReason: lockModel ? { model: lockModel, providerErrorCode, reason, lockedAt, resetsAt: new Date(Date.now() + cooldownMs).toISOString() } : null,
-    lastErrorAt: lockedAt,
-    backoffLevel: newBackoffLevel ?? backoffLevel
-  });
-
-  const lockKey = Object.keys(lockUpdate)[0];
-  const connName = conn?.displayName || conn?.name || conn?.email || connectionId.slice(0, 8);
-  log.warn("AUTH", `${connName} locked ${lockKey} for ${Math.round(cooldownMs / 1000)}s [${status}]`);
-  if (provider) {
-    import("@/lib/alerter.js")
-      .then((alerts) => {
-        alerts.alertProviderDown(provider, `${connName}: ${reason}`);
-        if (status === 429 || reason.toLowerCase().includes("quota")) {
-          alerts.alertQuotaExhausted(provider, connName);
-        }
-      })
-      .catch(() => {});
-  }
-
-  if (provider && status && reason) {
-    console.error(`\u274c ${provider} [${status}]: ${reason}`);
-  }
-
-  // --- Auto-disable on terminal auth errors ---
-  // Terminal errors: token expired/invalid/revoked (401), banned/suspended (403)
-  // Only disable after consecutive failures to avoid false positives from transient issues
-  const TERMINAL_AUTH_STATUSES = new Set([401, 403]);
-  const TERMINAL_ERROR_MARKERS = [
-    "token expired", "token invalid", "invalid token", "revoked",
-    "unauthorized", "invalid api key", "invalid_api_key",
-    "banned", "suspended", "restricted", "account disabled",
-  ];
-
-  if (TERMINAL_AUTH_STATUSES.has(status) && !isCodeBuddyModelError) {
-    const lowerError = (typeof errorText === "string" ? errorText : "").toLowerCase();
-    const isTerminalError = TERMINAL_ERROR_MARKERS.some(marker => lowerError.includes(marker));
-
-    if (isTerminalError) {
-      const prevFailures = conn?.consecutiveAuthFailures || 0;
-      const newFailures = prevFailures + 1;
-
-      if (newFailures >= 3) {
-        // Auto-disable after 3 consecutive terminal auth errors
-        await updateProviderConnection(connectionId, {
-          isActive: false,
-          autoDisabledAt: new Date().toISOString(),
-          autoDisabledReason: lowerError.includes("banned") || lowerError.includes("suspended") || lowerError.includes("restricted")
-            ? "banned"
-            : lowerError.includes("quota") || lowerError.includes("payment")
-              ? "quota_exhausted"
-              : "token_expired",
-          consecutiveAuthFailures: newFailures,
-        });
-        log.warn("AUTH", `\u26d4 Auto-disabled ${connName} after ${newFailures} consecutive auth failures [${status}]: ${reason}`);
-      } else {
-        // Increment failure counter
-        await updateProviderConnection(connectionId, {
-          consecutiveAuthFailures: newFailures,
-        });
-        log.warn("AUTH", `${connName} auth failure ${newFailures}/3 [${status}]: ${reason}`);
-      }
-    }
-  }
-
-  return { shouldFallback: true, cooldownMs };
-}
-
-/**
- * Clear account error status on successful request.
- * - Clears modelLock_${model} (the model that just succeeded)
- * - Lazy-cleans any other expired modelLock_* keys
- * - Resets error state only if no active locks remain
- * @param {string} connectionId
- * @param {object} currentConnection - credentials object (has _connection) or raw connection
- * @param {string|null} model - model that succeeded
- */
-export async function clearAccountError(connectionId, currentConnection, model = null) {
-  if (!connectionId || connectionId === "noauth") return;
-  const conn = currentConnection._connection || currentConnection;
-  const providerId = resolveProviderId(conn.provider);
-  const lockIdentity = getModelLockIdentity(providerId, model);
-  const now = Date.now();
-  const allLockKeys = Object.keys(conn).filter(k => k.startsWith("modelLock_"));
-
-  if (!conn.testStatus && !conn.lastError && allLockKeys.length === 0) return;
-
-  // Keys to clear: current model's lock + all expired locks
-  const keysToClear = allLockKeys.filter(k => {
-    const storedModel = k.slice("modelLock_".length);
-    if (model && getModelLockIdentity(providerId, storedModel) === lockIdentity) return true;
-    if (model && k === "modelLock___all") return true;    // account-level lock
-    const expiry = conn[k];
-    return expiry && new Date(expiry).getTime() <= now;   // expired
-  });
-
-  if (keysToClear.length === 0 && conn.testStatus !== "unavailable" && !conn.lastError) return;
-
-  // Check if any active locks remain after clearing
-  const remainingActiveLocks = allLockKeys.filter(k => {
-    if (keysToClear.includes(k)) return false;
-    const expiry = conn[k];
-    return expiry && new Date(expiry).getTime() > now;
-  });
-
-  const clearObj = {
-    ...Object.fromEntries(keysToClear.map(k => [k, null])),
-    [getModelRecoveryProbeKey(model, providerId)]: null,
-  };
-
-  // Only reset error state if no active locks remain
-  if (remainingActiveLocks.length === 0) {
-    Object.assign(clearObj, {
-      testStatus: "active",
-      lastError: null,
-      errorCode: null,
-      providerErrorCode: null,
-      modelLockReason: null,
-      lastErrorAt: null,
-      backoffLevel: 0,
-      consecutiveAuthFailures: 0
-    });
-  }
-
-  await updateProviderConnection(connectionId, clearObj);
-}
-
-/**
- * Extract API key from request headers
- */
-export function extractApiKey(request) {
-  // Check Authorization header first
-  const authHeader = request.headers.get("Authorization");
-  if (authHeader?.startsWith("Bearer ")) {
-    return authHeader.slice(7);
-  }
-
-  // Check Anthropic x-api-key header
-  const xApiKey = request.headers.get("x-api-key");
-  if (xApiKey) {
-    return xApiKey;
-  }
-
-  return null;
-}
-
-/**
- * Validate API key (optional - for local use can skip)
- */
-export async function isValidApiKey(apiKey) {
-  if (!apiKey) return false;
-  return await validateApiKey(apiKey);
-}