teamclaude-cloud 1.8.1 → 1.9.0

package/src/server.js

@@ -484,7 +484,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
         // tried429/tried5xx/authRetried hold account OBJECTS (not indexes), and
         // `held` is the acquired account OBJECT — both stable across a concurrent
         // removeAccount() re-index, so a release/exclude can't target the wrong account.
-        const ctx = { account: null, status: null, authRetried: new Set(), tried429: new Set(), tried5xx: new Set(), overloadRetries: 0, held: null, queueTimeoutMs, abortSignal: null, affinityKey: sessionAffinity ? req.socket : null, sawModelWeekly: false };
+        const ctx = { account: null, status: null, authRetried: new Set(), tried429: new Set(), tried5xx: new Set(), triedError: new Set(), overloadRetries: 0, held: null, queueTimeoutMs, abortSignal: null, affinityKey: sessionAffinity ? req.socket : null, sawModelWeekly: false };
         try {
           // Buffer request body (needed for retry on 429), bounded by maxBodyBytes.
           const bodyChunks = [];
@@ -776,8 +776,8 @@ async function forwardRequest(req, res, body, accountManager, upstream, retryCou
   // Select account. On a failover retry (a prior account 429'd / 5xx'd for this
   // request) ctx.tried* is non-empty → pick a different account, skipping the
   // ones already tried.
-  const excludeForSelect = (ctx.tried429.size || ctx.tried5xx.size)
-    ? new Set([...ctx.tried429, ...ctx.tried5xx])
+  const excludeForSelect = (ctx.tried429.size || ctx.tried5xx.size || ctx.triedError.size)
+    ? new Set([...ctx.tried429, ...ctx.tried5xx, ...ctx.triedError])
     : null;
 
   // Reserve a per-account concurrency slot. On a 401 same-account refresh-retry
@@ -805,19 +805,28 @@ async function forwardRequest(req, res, body, accountManager, upstream, retryCou
 
   if (!account) {
     ctx.account = '(none available)';
-    // If every account is in auth-error state, this is an authentication
-    // problem (revoked/expired tokens needing re-login), not a rate limit —
-    // return 401 so the client surfaces it instead of pointlessly backing off.
+    // Every account is in error state. Classify by the FAILURE CLASS recorded when each errored:
+    //  • ALL transient (cloud refresh 5xx/network — the recovery timer heals them) → retryable 503,
+    //    not a 401 (a 401 "re-login required" would trigger needless client re-auth/teardown for a
+    //    temporary failure the client can't fix — the proxy's accounts).
+    //  • ANY terminal failure (a 4xx invalid_grant/revoked key, or a non-cloud local auth failure
+    //    the recovery timer never touches) → 401 so it isn't silently retried forever.
     const accts = accountManager.accounts;
     if (accts.length > 0 && accts.every(a => a.status === 'error')) {
+      if (accts.every(a => a.errorTransient === true)) {
+        ctx.status = 503;
+        res.writeHead(503, { 'Content-Type': 'application/json', 'retry-after': '30' });
+        res.end(JSON.stringify({
+          type: 'error',
+          error: { type: 'overloaded_error', message: `All ${accts.length} cloud-managed accounts are temporarily unavailable (token refresh failing); recovering — retry shortly.` },
+        }));
+        return;
+      }
       ctx.status = 401;
       res.writeHead(401, { 'Content-Type': 'application/json' });
       res.end(JSON.stringify({
         type: 'error',
-        error: {
-          type: 'authentication_error',
-          message: `All ${accts.length} accounts failed authentication. Re-login required.`,
-        },
+        error: { type: 'authentication_error', message: `All ${accts.length} accounts failed authentication. Re-login required.` },
       }));
       return;
     }
@@ -871,9 +880,71 @@ async function forwardRequest(req, res, body, accountManager, upstream, retryCou
     return;
   }
 
-  if (account.status === 'error' && retryCount < maxRetries) {
-    releaseHeld(); // failing over to a different account
-    return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir);
+  if (account.status === 'error') {
+    releaseHeld(); // this account can't serve — fail over, or give up locally
+    if (res.destroyed) return; // client gone — outer finally frees the slot
+    // Fail over while ANY untried account is still usable — do NOT cap on the shared
+    // `retryCount` budget here: 401 same-account refresh-retries and 429/5xx failovers may
+    // have already elevated it, which would wrongly 429 the client while a HEALTHY account
+    // (e.g. the user's 4th of 11) sits untried. Record this errored account so `anyUsable`
+    // shrinks each pass (error accounts are already excluded from selection by status too),
+    // guaranteeing termination once no usable account remains.
+    ctx.triedError.add(account);
+    // Termination is guaranteed by the EXCLUSION SET, not anyUsable's internal availability
+    // logic: triedError only grows and holds distinct account OBJECTS, so once every account
+    // is in excludeTried, `!excludeTried.has(a)` is false for all → anyUsable returns false
+    // regardless of _isAvailable/_hasCapacity. So a future change to anyUsable can't create an
+    // unbounded loop (bounded by accounts.length distinct objects).
+    const excludeTried = new Set([...ctx.tried429, ...ctx.tried5xx, ...ctx.triedError]);
+    // Accept anyCapped too (like the 429/5xx failover paths): a healthy untried account at
+    // its concurrency cap will free shortly, so fail over and let acquireAccount's bounded
+    // overflow queue wait for it rather than immediately 429'ing under load. Still bounded —
+    // a capped account either frees and serves, errors (→ triedError), or the queue times out.
+    if (accountManager.anyUsable(excludeTried) || accountManager.anyCapped(excludeTried)) {
+      return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir);
+    }
+    // No untried usable account remains — the selected account's token is dead (e.g. a
+    // cloud-managed account whose central refresh failed; single-authority forbids a local
+    // rotation, so ensureTokenFresh left it status='error'). Do NOT forward its EXPIRED
+    // bearer token (that returns a confusing upstream 401 and defeats the immediate-
+    // unavailability contract). Classify the terminal EXACTLY like the no-account path above so
+    // identical "all credentials failed" states behave identically to the client: if EVERY
+    // account is in auth-error state it's an authentication problem (re-login) → 401; otherwise
+    // it's transient unavailability → local 429 so the client backs off while the recovery timer
+    // re-probes the cloud and heals the account.
+    if (!res.headersSent) {
+      const accts = accountManager.accounts;
+      if (accts.length > 0 && accts.every(a => a.status === 'error')) {
+        // Same classification as the no-account terminal above (consistent across both
+        // "all credentials failed" paths): all-transient → healable → retryable 503; any
+        // terminal failure (4xx invalid_grant / non-cloud auth) → 401.
+        if (accts.every(a => a.errorTransient === true)) {
+          ctx.status = 503;
+          res.writeHead(503, { 'Content-Type': 'application/json', 'retry-after': '30' });
+          res.end(JSON.stringify({
+            type: 'error',
+            error: { type: 'overloaded_error', message: `All ${accts.length} cloud-managed accounts are temporarily unavailable (token refresh failing); recovering — retry shortly.` },
+          }));
+          return;
+        }
+        ctx.status = 401;
+        res.writeHead(401, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({
+          type: 'error',
+          error: { type: 'authentication_error', message: `All ${accts.length} accounts failed authentication. Re-login required.` },
+        }));
+        return;
+      }
+      ctx.status = 429;
+      res.writeHead(429, { 'Content-Type': 'application/json', 'retry-after': '10' });
+      res.end(JSON.stringify({
+        type: 'error',
+        error: { type: 'rate_limit_error', message: 'No account with a valid token is available right now; please retry shortly.' },
+      }));
+    } else {
+      ctx.status = ctx.status || 429;
+    }
+    return;
   }
 
   // Build upstream request headers

package/src/tui.js

@@ -479,6 +479,13 @@ export class TUI {
         const prev = this.config.accounts[idx];
         if (prev.enabled !== undefined) entry.enabled = prev.enabled;
         if (prev.priority !== undefined) entry.priority = prev.priority;
+        // CRITICAL: preserve cloud provenance/display too — a re-import replaces the TOKEN, not
+        // the account's cloud-sharing status. Dropping cloudSources/cloudPushed would make a
+        // cloud-managed (shared) account look LOCAL → it would locally rotate its now-shared
+        // refresh token and poison the chain for every other PC (single-authority invariant).
+        if (Array.isArray(prev.cloudSources) && prev.cloudSources.length) entry.cloudSources = prev.cloudSources;
+        if (prev.cloudPushed) entry.cloudPushed = prev.cloudPushed;
+        if (prev.cloudPrivate) entry.cloudPrivate = prev.cloudPrivate;
         this.config.accounts[idx] = entry;
         // Update the running account manager entry, matched by IDENTITY (the
         // previous entry's UUID first, then name) — NOT the config index `idx`,