zevairouter 1.0.151 → 1.0.153

package/cli/app/CHANGELOG.md

@@ -1,3 +1,15 @@
+# v1.0.153 (2026-07-27)
+
+## Fixes
+- **Legacy Codex lock recovery**: accounts whose health checks already cleared `lastErrorAt` can now use a newer positive quota snapshot to bypass a stale model lock once, restoring accounts that still have usable quota.
+- **Durable lock evidence**: new model locks retain their own `lockedAt` timestamp, while successful authentication tests preserve active-lock error metadata instead of silently breaking future recovery decisions.
+
+# v1.0.152 (2026-07-27)
+
+## Fixes
+- **Accurate Codex quota recovery**: a newer quota snapshot that explicitly reports capacity can now supersede a stale per-model lock instead of returning a false long-lived 429 while quota remains available.
+- **Bounded recovery probes**: recovered accounts are retried once only when the quota snapshot is newer than the locking error; a fresh upstream failure immediately restores the lock and prevents retry loops.
+
 # v1.0.151 (2026-07-27)
 
 ## Fixes

package/cli/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zevairouter",
-  "version": "1.0.151",
+  "version": "1.0.153",
   "description": "Self-hosted AI router: many providers, one OpenAI-compatible /v1 endpoint and dashboard",
   "bin": {
     "zevai": "./cli/cli.js"

package/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zevairouter-cli",
-  "version": "1.0.151",
+  "version": "1.0.153",
   "description": "ZevaiRouter CLI - Start and manage ZevaiRouter server",
   "bin": {
     "zevai": "./cli.js"

package/open-sse/services/accountFallback.js

@@ -53,6 +53,53 @@ export function getQuotaExhaustedUntil(usage) {
   return getQuotaResetAt(usage);
 }
 
+/**
+ * Return true only when a quota snapshot explicitly says every known window
+ * still has capacity. Requiring all windows avoids treating an account with an
+ * exhausted weekly window as recovered just because its short window is open.
+ */
+export function hasAvailableQuota(usage) {
+  if (!usage || usage.limitReached === true) return false;
+  const quotas = Object.values(usage.quotas || {});
+  const remaining = quotas.map((quota) => {
+    if (Number.isFinite(quota?.remaining)) return quota.remaining;
+    if (Number.isFinite(quota?.remainingPercentage)) return quota.remainingPercentage;
+    return null;
+  }).filter((value) => value !== null);
+
+  return remaining.length > 0
+    && remaining.length === quotas.length
+    && remaining.every((value) => value > 0);
+}
+
+/**
+ * A successful quota poll newer than the error which created a model lock is
+ * authoritative evidence that a Codex account recovered. New locks carry an
+ * explicit lockedAt timestamp. Legacy locks fall back to lastErrorAt, then the
+ * connection update timestamp when older health checks already cleared the
+ * error metadata.
+ */
+export function isModelLockRecoveredByQuota(connection, model, snapshot) {
+  if (!isModelLockActive(connection, model) || !hasAvailableQuota(snapshot?.usage)) {
+    return false;
+  }
+
+  const fetchedAt = Date.parse(snapshot?.fetchedAt);
+  const reason = connection?.modelLockReason;
+  const reasonAt = reason?.model === model ? Date.parse(reason?.lockedAt) : Number.NaN;
+  const lastErrorAt = Date.parse(connection?.lastErrorAt);
+  const legacyUpdatedAt = Date.parse(connection?.updatedAt);
+  const errorAt = Number.isFinite(reasonAt)
+    ? reasonAt
+    : Number.isFinite(lastErrorAt)
+      ? lastErrorAt
+      : legacyUpdatedAt;
+
+  return Number.isFinite(fetchedAt)
+    && Number.isFinite(errorAt)
+    && fetchedAt > errorAt;
+}
+
 /**
  * Calculate exponential backoff cooldown for rate limits (429)
  * Level 1: 1s, Level 2: 2s, Level 3: 4s... → max 4 min

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zevairouter",
-  "version": "1.0.151",
+  "version": "1.0.153",
   "description": "Self-hosted AI router: many providers, one OpenAI-compatible /v1 endpoint and dashboard",
   "bin": {
     "zevai": "./cli/cli.js"

package/src/app/api/providers/[id]/test/testUtils.js

@@ -5,6 +5,7 @@ import { isOpenAICompatibleProvider, isAnthropicCompatibleProvider } from "@/sha
 import { PROVIDER_ENDPOINTS } from "@/shared/constants/config";
 import { getDefaultModel } from "open-sse/config/providerModels.js";
 import { resolveOllamaLocalHost } from "open-sse/config/providers.js";
+import { getEarliestModelLockUntil } from "open-sse/services/accountFallback.js";
 import {
   refreshProviderCredentials,
   shouldRefreshCredentials,
@@ -769,12 +770,20 @@ export async function testSingleConnection(id) {
 
   const latencyMs = Date.now() - start;
 
+  // A successful auth/health test does not prove that a model-specific quota
+  // lock recovered. Preserve its error timestamp so a newer quota snapshot can
+  // safely authorize one bounded recovery probe.
+  const preserveModelLockError = result.valid && !!getEarliestModelLockUntil(connection);
   const updateData = result.inconclusive
     ? {}
     : {
         testStatus: result.valid ? "active" : "error",
-        lastError: result.valid ? null : result.error,
-        lastErrorAt: result.valid ? null : new Date().toISOString(),
+        lastError: result.valid
+          ? (preserveModelLockError ? connection.lastError : null)
+          : result.error,
+        lastErrorAt: result.valid
+          ? (preserveModelLockError ? connection.lastErrorAt : null)
+          : new Date().toISOString(),
       };
 
   if (result.refreshed && result.newTokens) {

package/src/sse/services/auth.js

@@ -1,6 +1,6 @@
 import { getProviderConnections, validateApiKey, updateProviderConnection, getSettings, getQuotaSnapshot, getAllQuotaSnapshots, getProxyPools } from "@/lib/localDb";
 import { resolveConnectionProxyConfig, pickProxyPoolId } from "@/lib/network/connectionProxy";
-import { formatRetryAfter, checkFallbackError, isModelLockActive, buildModelLockUpdate, getEarliestModelLockUntil, isNonAccountError, extractProviderErrorCode, getQuotaResetAt, getQuotaExhaustedUntil } from "open-sse/services/accountFallback.js";
+import { formatRetryAfter, checkFallbackError, isModelLockActive, isModelLockRecoveredByQuota, buildModelLockUpdate, getEarliestModelLockUntil, isNonAccountError, extractProviderErrorCode, getQuotaResetAt, getQuotaExhaustedUntil } from "open-sse/services/accountFallback.js";
 import { MAX_RATE_LIMIT_COOLDOWN_MS } from "open-sse/config/errorConfig.js";
 import { resolveProviderId, FREE_PROVIDERS } from "@/shared/constants/providers.js";
 import * as log from "../utils/logger.js";
@@ -77,9 +77,29 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
       }
     }
 
+    // Codex can return a precise usage-limit reset which creates a long-lived
+    // model lock. If a newer quota poll explicitly reports capacity again, let
+    // that account be probed once instead of serving a stale all-rate-limited
+    // response until the old reset timestamp. A new failure is newer than the
+    // snapshot, so it immediately restores the lock without a retry loop.
+    const recoveredModelLocks = new Set();
+    if (providerId === "codex" && model) {
+      const snapshots = await getAllQuotaSnapshots(providerId);
+      const byConnection = new Map(snapshots.map((snapshot) => [snapshot.connectionId, snapshot]));
+      for (const connection of connections) {
+        if (isModelLockRecoveredByQuota(connection, model, byConnection.get(connection.id))) {
+          recoveredModelLocks.add(connection.id);
+        }
+      }
+    }
+
+    const hasBlockingModelLock = (connection) => (
+      isModelLockActive(connection, model) && !recoveredModelLocks.has(connection.id)
+    );
+
     const baseAvailableConnections = connections.filter(c => {
       if (excludeSet.has(c.id)) return false;
-      if (isModelLockActive(c, model)) return false;
+      if (hasBlockingModelLock(c)) return false;
       if (quotaExhausted.has(c.id)) return false;
       return true;
     });
@@ -98,7 +118,7 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
         log.warn("AUTH", `${provider} | pinned account excluded after failure: ${preferred.id?.slice(0, 8)}`);
         return null;
       }
-      if (isModelLockActive(preferred, model) || quotaExhausted.has(preferred.id)) {
+      if (hasBlockingModelLock(preferred) || quotaExhausted.has(preferred.id)) {
         const retryAfter = [
           getEarliestModelLockUntil(preferred),
           quotaExhausted.has(preferred.id)
@@ -119,17 +139,18 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
     log.debug("AUTH", `${provider} | available: ${availableConnections.length}/${connections.length}`);
     connections.forEach(c => {
       const excluded = excludeSet.has(c.id);
-      const locked = isModelLockActive(c, model);
+      const locked = hasBlockingModelLock(c);
+      const recovered = recoveredModelLocks.has(c.id);
       const exhaustedUntil = quotaExhausted.get(c.id);
-      if (excluded || locked || exhaustedUntil) {
+      if (excluded || locked || exhaustedUntil || recovered) {
         const lockUntil = getEarliestModelLockUntil(c);
-        log.debug("AUTH", `  \u2192 ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${locked ? `modelLocked(${model}) until ${lockUntil}` : ""} ${exhaustedUntil ? `quotaExhausted until ${new Date(exhaustedUntil).toISOString()}` : ""}`);
+        log.debug("AUTH", `  \u2192 ${c.id?.slice(0, 8)} | ${excluded ? "excluded" : ""} ${locked ? `modelLocked(${model}) until ${lockUntil}` : ""} ${recovered ? `staleLockRecovered(${model})` : ""} ${exhaustedUntil ? `quotaExhausted until ${new Date(exhaustedUntil).toISOString()}` : ""}`);
       }
     });
 
     if (availableConnections.length === 0) {
       // Find earliest lock expiry across all connections for retry timing
-      const lockedConns = connections.filter(c => isModelLockActive(c, model));
+      const lockedConns = connections.filter(c => hasBlockingModelLock(c));
       const expiries = [
         ...lockedConns.map(c => getEarliestModelLockUntil(c)).filter(Boolean),
         ...quotaExhausted.values().map(timestamp => new Date(timestamp).toISOString()),
@@ -359,6 +380,7 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
 
   const reason = typeof errorText === "string" ? errorText.slice(0, 100) : "Provider error";
   const lockUpdate = buildModelLockUpdate(model, cooldownMs);
+  const lockedAt = new Date().toISOString();
 
   await updateProviderConnection(connectionId, {
     ...lockUpdate,
@@ -366,8 +388,8 @@ export async function markAccountUnavailable(connectionId, status, errorText, pr
     lastError: reason,
     errorCode: status,
     providerErrorCode,
-    modelLockReason: model ? { model, providerErrorCode, reason, resetsAt: new Date(Date.now() + cooldownMs).toISOString() } : null,
-    lastErrorAt: new Date().toISOString(),
+    modelLockReason: model ? { model, providerErrorCode, reason, lockedAt, resetsAt: new Date(Date.now() + cooldownMs).toISOString() } : null,
+    lastErrorAt: lockedAt,
     backoffLevel: newBackoffLevel ?? backoffLevel
   });