qwenproxy-cli 1.0.0 โ†’ 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/LICENSE +14 -14
  2. package/README.md +906 -906
  3. package/bin/qwenproxy.js +5 -1
  4. package/package.json +78 -78
  5. package/src/api/error-classifier.ts +159 -159
  6. package/src/api/error-helpers.ts +118 -118
  7. package/src/api/models.ts +261 -261
  8. package/src/api/server.ts +860 -859
  9. package/src/cache/memory-cache.ts +385 -385
  10. package/src/clean-cache.ts +204 -204
  11. package/src/core/account-concurrency.ts +671 -671
  12. package/src/core/account-manager.ts +301 -297
  13. package/src/core/account-priority.ts +163 -163
  14. package/src/core/accounts.ts +186 -186
  15. package/src/core/config.ts +383 -383
  16. package/src/core/crypto-utils.ts +79 -79
  17. package/src/core/database.ts +276 -276
  18. package/src/core/errors.ts +118 -118
  19. package/src/core/logger.ts +269 -269
  20. package/src/core/memory-usage.ts +84 -84
  21. package/src/core/metrics.ts +291 -291
  22. package/src/core/model-alias.ts +77 -77
  23. package/src/core/model-registry.ts +544 -544
  24. package/src/core/mutex.ts +119 -119
  25. package/src/core/paths.ts +199 -199
  26. package/src/core/prompt-limits.ts +214 -214
  27. package/src/core/reasoning-effort.ts +102 -102
  28. package/src/core/stream-registry.ts +96 -96
  29. package/src/core/waf-isolation.ts +117 -117
  30. package/src/core/watchdog.ts +195 -195
  31. package/src/delete-chats.ts +23 -23
  32. package/src/index.ts +65 -64
  33. package/src/login.ts +147 -147
  34. package/src/reset-cooldowns.ts +11 -11
  35. package/src/routes/anthropic/index.ts +355 -355
  36. package/src/routes/anthropic/translate.ts +522 -522
  37. package/src/routes/anthropic/types.ts +154 -154
  38. package/src/routes/anthropic/validation.ts +144 -144
  39. package/src/routes/chat/account.ts +1817 -1817
  40. package/src/routes/chat/context.ts +241 -241
  41. package/src/routes/chat/errors.ts +85 -85
  42. package/src/routes/chat/helpers.ts +268 -268
  43. package/src/routes/chat/index.ts +618 -618
  44. package/src/routes/chat/media.ts +285 -285
  45. package/src/routes/chat/retry-policy.ts +754 -754
  46. package/src/routes/chat/stop.ts +98 -98
  47. package/src/routes/chat/streaming.ts +2710 -2710
  48. package/src/routes/chat/validation.ts +526 -526
  49. package/src/routes/chat.ts +2 -2
  50. package/src/routes/completions.ts +290 -290
  51. package/src/routes/images.ts +139 -139
  52. package/src/routes/responses/adapter.ts +503 -503
  53. package/src/routes/responses/index.ts +405 -405
  54. package/src/routes/responses/state.ts +230 -230
  55. package/src/routes/responses/streaming.ts +528 -528
  56. package/src/routes/responses/types.ts +285 -285
  57. package/src/routes/responses/validation.ts +202 -202
  58. package/src/routes/upload.ts +731 -731
  59. package/src/routes/videos.ts +214 -214
  60. package/src/services/auth-playwright.ts +173 -173
  61. package/src/services/captcha-coordinator.ts +161 -161
  62. package/src/services/captcha-solver.ts +553 -553
  63. package/src/services/chat-cleanup.ts +80 -80
  64. package/src/services/context-meter.ts +317 -317
  65. package/src/services/fingerprint.ts +242 -242
  66. package/src/services/human-behavior.ts +173 -173
  67. package/src/services/media-generation.ts +1748 -1748
  68. package/src/services/playwright.ts +2800 -2800
  69. package/src/services/qwen-chat-pool.ts +345 -345
  70. package/src/services/qwen-errors.ts +133 -133
  71. package/src/services/qwen-headers.ts +79 -79
  72. package/src/services/qwen-thread-state.ts +393 -393
  73. package/src/services/qwen-url.ts +19 -19
  74. package/src/services/qwen.ts +3126 -3126
  75. package/src/services/session-keeper.ts +88 -88
  76. package/src/services/token-estimation-metrics.ts +118 -118
  77. package/src/sync/claude-code.ts +75 -75
  78. package/src/sync/codex.ts +123 -123
  79. package/src/sync/index.ts +362 -362
  80. package/src/sync/omp.ts +105 -105
  81. package/src/sync/opencode.ts +214 -214
  82. package/src/sync/types.ts +53 -53
  83. package/src/sync/utils.ts +27 -27
  84. package/src/sync-clients.ts +189 -189
  85. package/src/tools/instructions.ts +137 -137
  86. package/src/tools/manifest.ts +81 -81
  87. package/src/tools/parser.ts +2989 -2989
  88. package/src/tools/toolcall-tags.ts +142 -142
  89. package/src/tui/app.ts +264 -264
  90. package/src/tui/index.ts +61 -61
  91. package/src/tui/markdown.ts +258 -258
  92. package/src/tui/proxy-client.ts +331 -326
  93. package/src/tui/screen.ts +294 -278
  94. package/src/tui/server-manager.ts +270 -270
  95. package/src/tui/theme.ts +432 -432
  96. package/src/tui/types.ts +33 -33
  97. package/src/tui/views/accounts-view.ts +656 -656
  98. package/src/tui/views/chat-view.ts +1018 -823
  99. package/src/tui/views/logs-view.ts +479 -413
  100. package/src/tui/views/status-view.ts +204 -204
  101. package/src/tui/views/storage-view.ts +304 -291
  102. package/src/tui/views/sync-view.ts +409 -409
  103. package/src/types/ali-oss.d.ts +32 -32
  104. package/src/utils/context-truncation.ts +84 -84
  105. package/src/utils/json.ts +380 -380
  106. package/src/utils/session-id.ts +37 -37
  107. package/src/utils/tool-call-guard.ts +84 -84
  108. package/src/utils/types.ts +109 -109
@@ -1,1817 +1,1817 @@
1
- import { v4 as uuidv4 } from "uuid";
2
- import {
3
- getAccountCooldownInfo,
4
- getNextAccount,
5
- getNextAvailableAccount,
6
- markAccountRateLimited,
7
- syncCooldownsFromDb,
8
- } from "../../core/account-manager.ts";
9
- import { markAccountSuccessful, markAccountFailed, getAccountsByPriority } from "../../core/account-priority.ts";
10
- import { recordWafHardBlock, noteWafRecovery } from "../../core/waf-isolation.ts";
11
- import { loadAccounts, type QwenAccount } from "../../core/accounts.ts";
12
- import { config, type ChatMode } from "../../core/config.ts";
13
- import { ClientAbortedError, UpstreamRateLimit, ValidationError } from "../../core/errors.ts";
14
- import {
15
- assertPromptWithinLimits,
16
- truncatePromptToIntelligentLimit,
17
- } from "../../core/prompt-limits.ts";
18
- import {
19
- formatCooldownUntil,
20
- isToolcallDebugEnabled,
21
- logger,
22
- maskEmail,
23
- } from "../../core/logger.ts";
24
- import { Mutex } from "../../core/mutex.ts";
25
- import { registerStream, removeStream } from "../../core/stream-registry.ts";
26
- import {
27
- abortLeaseByLabel,
28
- acquireAccountLease,
29
- isAccountBusy,
30
- isAccountSlotHeldByOtherSession,
31
- isAccountTemporarilyBusy,
32
- markAccountTemporarilyBusy,
33
- markLeaseCompletion,
34
- tryAcquireAccountLease,
35
- type AccountLease,
36
- } from "../../core/account-concurrency.ts";
37
- import { isAuthMockEnabled } from "../../services/auth-playwright.ts";
38
- import { refreshHeaders } from "../../services/playwright.ts";
39
- import {
40
- clearAllSessionsForAccount,
41
- createQwenStream,
42
- fetchQwenModels,
43
- getQwenErrorCode,
44
- getLogicalThreadState,
45
- invalidateLogicalThreadParent,
46
- type LogicalThreadEntry,
47
- PersonalizationSyncError,
48
- QwenSessionExpiredError,
49
- RetryableQwenStreamError,
50
- syncQwenRequestPersonalization,
51
- updateLogicalThreadState,
52
- } from "../../services/qwen.ts";
53
- import type { TokenEstimationContext } from "../../services/token-estimation-metrics.ts";
54
- import {
55
- buildContextMeterSnapshot,
56
- contextMeterLogData,
57
- type ContextMeterMode,
58
- } from "../../services/context-meter.ts";
59
- import type { QwenFileEntry } from "../upload.ts";
60
- import type { Message } from "../../utils/types.ts";
61
- import { buildRepeatedToolCallReminder } from "../../utils/tool-call-guard.ts";
62
- import {
63
- classifyRetryAction,
64
- isAntiBotError as isAntiBotPolicyError,
65
- isAccountInitializationError,
66
- isChatInProgressError,
67
- isQuotaLikeError,
68
- isTerminalLocalError,
69
- shouldRetryInvalidInputOnSameAccount,
70
- } from "./retry-policy.ts";
71
-
72
- /** How many alternate accounts a single request may try after a WAF challenge. */
73
- const MAX_ANTI_BOT_ROTATIONS = 1;
74
-
75
- /**
76
- * Hard deadline for the whole personalization sync. A normal sync takes ~2s;
77
- * a stuck account page (closed context / WAF) can otherwise hold each browser
78
- * op for 60s and keep the personalization mutex blocked for minutes.
79
- */
80
- const PERSONALIZATION_SYNC_DEADLINE_MS = 30_000;
81
-
82
- /**
83
- * Hard deadline for a single stream-acquire attempt (models sync + truncation
84
- * + personalization + header capture + completion fetch metadata + internal
85
- * retries). A silent hang past this (observed: 180s with zero logs) fails the
86
- * attempt with a visible retryable error so the outer loop switches account.
87
- * Configurable via ACQUIRE_DEADLINE_MS (default 120000).
88
- */
89
-
90
- // Per-chat lock: serializes requests to the same Qwen chat session
91
- const chatLocks = new Map<string, Mutex>();
92
- // Account-level personalization is global mutable Qwen state; keep update+stream
93
- // creation serialized per account when the experimental request-sync mode is used.
94
- const personalizationLocks = new Map<string, Mutex>();
95
-
96
- export async function acquireChatLock(chatId: string): Promise<() => void> {
97
- const acquireStartedAt = Date.now();
98
- const timeoutMs = config.concurrency.chatLockTimeoutMs;
99
- let mutex = chatLocks.get(chatId);
100
- if (!mutex) {
101
- mutex = new Mutex(
102
- `chat:${chatId.substring(0, 8)}`,
103
- // The chat lock is held for the whole stream lifetime. A long
104
- // generation (reasoning + huge context) can legitimately exceed the
105
- // global 120s hold limit; use the same budget as the acquire timeout
106
- // so the force-release never kills a healthy mid-stream turn.
107
- timeoutMs,
108
- );
109
- chatLocks.set(chatId, mutex);
110
- }
111
- const release = await mutex.acquire(timeoutMs, `chat:${chatId.substring(0, 12)}`);
112
- // Held time must exclude the wait: capture right after the acquire settles,
113
- // not at function entry (the wait is already visible as `waited Xms` above).
114
- const heldStartedAt = Date.now();
115
- if (logger.isLevelEnabled("info")) {
116
- console.log(
117
- `๐Ÿ” [Chat] Chat lock acquired | chat=${chatId.substring(0, 12)} | waited ${heldStartedAt - acquireStartedAt}ms`,
118
- );
119
- }
120
- return () => {
121
- release();
122
- if (logger.isLevelEnabled("info")) {
123
- console.log(
124
- `๐Ÿ”“ [Chat] Chat lock released | chat=${chatId.substring(0, 12)} | held ${Date.now() - heldStartedAt}ms`,
125
- );
126
- }
127
- if (mutex!.isIdle()) {
128
- chatLocks.delete(chatId);
129
- }
130
- };
131
- }
132
-
133
- async function acquirePersonalizationLock(
134
- accountId: string,
135
- ): Promise<() => void> {
136
- let mutex = personalizationLocks.get(accountId);
137
- if (!mutex) {
138
- mutex = new Mutex(`personalization:${accountId.substring(0, 8)}`);
139
- personalizationLocks.set(accountId, mutex);
140
- }
141
- const release = await mutex.acquire(60_000, `personalization:${accountId.substring(0, 8)}`);
142
- return () => {
143
- release();
144
- if (mutex!.isIdle()) {
145
- personalizationLocks.delete(accountId);
146
- }
147
- };
148
- }
149
-
150
- export interface SelectedAccount {
151
- id: string;
152
- email: string;
153
- password: string;
154
- }
155
-
156
- export interface StreamCreationResult {
157
- stream: ReadableStream;
158
- uiSessionId: string;
159
- activeAccountId: string;
160
- activeAccountLabel: string;
161
- /** True when the request resent the FULL prompt on a new upstream chat
162
- * (account switch / missing thread parent). The ๐Ÿ“ค log line uses this to
163
- * show the real payload instead of the thread-native delta. */
164
- replayedFullContext: boolean;
165
- completionId: string;
166
- logicalSessionId: string | null;
167
- createdNewChat: boolean;
168
- tokenEstimationContext: TokenEstimationContext;
169
- releaseAccountLease: () => void;
170
- }
171
-
172
- export interface StreamCreationFailure {
173
- error: any;
174
- completionId: string;
175
- allOnCooldown: boolean;
176
- retryAfterMs?: number;
177
- }
178
-
179
- export interface AcquireParams {
180
- finalPrompt: string;
181
- fullPrompt: string;
182
- isThinkingModel: boolean;
183
- model: string;
184
- reasoningMode?: "auto" | "thinking" | "fast";
185
- shouldResetUpstreamThread: boolean;
186
- allFiles: QwenFileEntry[];
187
- isNewSession: boolean;
188
- sessionId: string | null;
189
- useThreadNative: boolean;
190
- updateLogicalThread: boolean;
191
- allowThreadReuse: boolean;
192
- /** "thread" (reuse upstream chat) or "temp" (new ephemeral chat per request). */
193
- chatMode: ChatMode;
194
- /** Full message history for intelligent context truncation after model sync. */
195
- messages?: Message[];
196
- forceNewChat?: boolean;
197
- /**
198
- * Prefer this account when available.
199
- * - undefined/omit: use sticky thread account when present, else round-robin
200
- * - string: pin to that account if configured
201
- * - null: explicitly rotate away from sticky/current account (error failover)
202
- */
203
- preferredAccountId?: string | null;
204
- /** When rotating, exclude these account ids from the first pick. */
205
- excludeAccountIds?: string[];
206
- messageCount?: number;
207
- fullMessageCount?: number;
208
- toolsCount?: number;
209
- requestPersonalizationInstruction?: string | null;
210
- /** Mapped Qwen model id used for local prompt-budget validation. */
211
- contextModelId?: string;
212
- requestSignal?: AbortSignal;
213
- /** Context accounting mode for this concrete upstream attempt. */
214
- contextMode?: ContextMeterMode;
215
- /** Allow this request to retry the account it just marked temporarily busy. */
216
- allowTemporarilyBusyAccountId?: string;
217
- /**
218
- * True when this request races a same-session stream that has NOT emitted
219
- * yet: run on its OWN chat and hop accounts fast instead of waiting.
220
- */
221
- parallelEscape?: boolean;
222
- }
223
-
224
- /** Exported for unit tests โ€” selects the first account for a request. */
225
- export function resolveInitialAccount(
226
- preferredAccountId?: string | null,
227
- excludeAccountIds?: Iterable<string>,
228
- ): {
229
- account: SelectedAccount;
230
- configuredAccounts: SelectedAccount[];
231
- } {
232
- if (isAuthMockEnabled()) {
233
- return {
234
- account: { id: "mock-account", email: "mock@test.com", password: "" },
235
- configuredAccounts: [],
236
- };
237
- }
238
-
239
- const configuredAccounts = loadAccounts();
240
- if (configuredAccounts.length > 0) {
241
- syncCooldownsFromDb(configuredAccounts);
242
- const excluded = new Set(excludeAccountIds ?? []);
243
-
244
- // Explicit preferred account (sticky / same-account retry)
245
- if (typeof preferredAccountId === "string" && preferredAccountId) {
246
- const preferred = configuredAccounts.find(
247
- (candidate) => candidate.id === preferredAccountId,
248
- );
249
- if (preferred && !getAccountCooldownInfo(preferred.id)) {
250
- return { account: preferred, configuredAccounts };
251
- }
252
- // Preferred is missing/on cooldown: fall through to next available.
253
- if (preferred) excluded.add(preferred.id);
254
- }
255
-
256
- // Error failover: rotate away from sticky/current account when requested.
257
- if (preferredAccountId === null || excluded.size > 0) {
258
- const next = getNextAvailableAccount(excluded);
259
- if (next) return { account: next, configuredAccounts };
260
- }
261
-
262
- const account = getNextAccount();
263
- if (!account) {
264
- // All accounts on cooldown; caller will handle this.
265
- return { account: configuredAccounts[0], configuredAccounts };
266
- }
267
- return { account, configuredAccounts };
268
- }
269
-
270
- throw new ValidationError(
271
- "Nenhuma conta Qwen configurada no servidor. Adicione uma conta na aba [5] Contas da TUI.",
272
- );
273
- }
274
-
275
- function isAccountUnavailableError(err: any): boolean {
276
- // Quota/rate-limit style failures that should cool the account and rotate.
277
- if (isQuotaLikeError(err)) return true;
278
- return (
279
- (err instanceof UpstreamRateLimit &&
280
- !(err instanceof RetryableQwenStreamError)) ||
281
- err?.upstreamCode === "RateLimited" ||
282
- err?.upstreamStatus === 429
283
- );
284
- }
285
-
286
- function isAntiBotError(err: any): boolean {
287
- return isAntiBotPolicyError(err);
288
- }
289
-
290
- function hasFreeAlternateAccount(
291
- accounts: SelectedAccount[],
292
- currentAccountId: string,
293
- triedAccountIds: Set<string>,
294
- ): boolean {
295
- return accounts.some(
296
- (candidate) =>
297
- candidate.id !== currentAccountId &&
298
- !triedAccountIds.has(candidate.id) &&
299
- !getAccountCooldownInfo(candidate.id) &&
300
- !isAccountTemporarilyBusy(candidate.id) &&
301
- !isAccountBusy(candidate.id),
302
- );
303
- }
304
-
305
- /**
306
- * Pick the next account for a PARALLEL escape, preferring one with a FREE slot:
307
- * not busy, not temporarily busy, not on cooldown, not already tried. A normal
308
- * request keeps getNextAvailableAccount (cooldown-only picker) so saturated or
309
- * single-account pools stay lossless โ€” but an auxiliary parallel request must
310
- * land on an available slot fast and never queue behind a second occupied
311
- * account (the 2026-08-20 stall rotated ldyjl->cgnx3, both busy, ~14s wait).
312
- * Falls back to the cooldown-only picker when no FREE account is left, so a
313
- * fully-busy pool still rotates instead of dead-ending.
314
- */
315
- function getNextFreeAccountForParallel(
316
- accounts: QwenAccount[],
317
- triedAccountIds: Set<string>,
318
- currentAccountId: string,
319
- ): QwenAccount | null {
320
- const ordered = getAccountsByPriority(accounts);
321
- const free = ordered.find(
322
- (c) =>
323
- c.id !== currentAccountId &&
324
- !triedAccountIds.has(c.id) &&
325
- !getAccountCooldownInfo(c.id) &&
326
- !isAccountTemporarilyBusy(c.id) &&
327
- !isAccountBusy(c.id),
328
- );
329
- if (free) return free;
330
- // No free slot anywhere: fall back to the normal picker so we still rotate
331
- // (the tryAcquireAccountLease fail-fast will report account_busy and the
332
- // loop gives up rather than blocking on a busy pool).
333
- return getNextAvailableAccount(triedAccountIds);
334
- }
335
-
336
-
337
- async function attemptRelogin(
338
- accountId: string,
339
- accountEmail: string,
340
- ): Promise<boolean> {
341
- try {
342
- await refreshHeaders(accountId);
343
- console.log(
344
- `โœ… [Chat] Playwright headers refreshed for ${maskEmail(accountEmail)}. Retrying...`,
345
- );
346
- return true;
347
- } catch (refreshErr: unknown) {
348
- logger.error("[Chat] Playwright header refresh failed", {
349
- accountEmail: maskEmail(accountEmail),
350
- error:
351
- refreshErr instanceof Error ? refreshErr.message : String(refreshErr),
352
- cause:
353
- refreshErr instanceof Error
354
- ? refreshErr.constructor.name
355
- : typeof refreshErr,
356
- });
357
- }
358
- return false;
359
- }
360
-
361
- export async function acquireUpstreamStream(
362
- params: AcquireParams,
363
- ): Promise<StreamCreationResult | StreamCreationFailure> {
364
- const {
365
- finalPrompt,
366
- isThinkingModel,
367
- model,
368
- reasoningMode,
369
- shouldResetUpstreamThread,
370
- allFiles,
371
- isNewSession,
372
- sessionId,
373
- useThreadNative,
374
- updateLogicalThread,
375
- allowThreadReuse,
376
- chatMode,
377
- forceNewChat = false,
378
- preferredAccountId,
379
- excludeAccountIds,
380
- } = params;
381
-
382
- const completionId = "chatcmpl-" + uuidv4();
383
- // Sticky thread binding is independent of forceNewChat. forceNewChat only
384
- // means "open a fresh upstream chat", not "forget which account owned the
385
- // logical conversation".
386
- const threadState =
387
- allowThreadReuse && sessionId ? getLogicalThreadState(sessionId) : null;
388
- const stickyThreadAccountId = threadState?.accountId ?? null;
389
- const canReuseUpstreamChat =
390
- !!threadState &&
391
- !forceNewChat &&
392
- !!threadState.chatSessionId &&
393
- threadState.chatSessionId.length > 0 &&
394
- !!threadState.parentId;
395
- // A thread with an upstream chat but no committed parent is dirty (failed
396
- // first turn, interrupted generation, corrupted history). It must not be
397
- // appended to; rebuild a fresh chat with the full prompt instead.
398
- const threadMissingParent =
399
- !!threadState &&
400
- !!threadState.chatSessionId &&
401
- threadState.chatSessionId.length > 0 &&
402
- !threadState.parentId;
403
- const existingThread = canReuseUpstreamChat ? threadState : null;
404
-
405
- // preferredAccountId:
406
- // - string: pin to account
407
- // - null: explicit failover away from sticky (error path)
408
- // - undefined: keep sticky when available
409
- // A PARALLEL escape must NOT pin to the sticky thread owner: it races the
410
- // main generation that is likely using that very account, so targeting the
411
- // sticky would just fail-fast account_busy and waste a rotation hop (the
412
- // 2026-08-20 02:43:41 stall: parallel req chose the sticky busy account, then
413
- // a second busy one, ~18s until the client aborted). Rotate to any account
414
- // so the first hop has a real chance of landing on a free slot.
415
- const effectivePreferred = params.parallelEscape ? null : preferredAccountId;
416
- const resolvedPreferred =
417
- effectivePreferred === null
418
- ? null
419
- : (effectivePreferred ?? stickyThreadAccountId ?? undefined);
420
- const excludeSet = new Set(excludeAccountIds ?? []);
421
- // When rotating away (resolvedPreferred === null) โ€” either an explicit
422
- // failover OR a parallel escape โ€” exclude the sticky owner so the rotation
423
- // can never land back on the account the main generation is using.
424
- if (resolvedPreferred === null && stickyThreadAccountId) {
425
- excludeSet.add(stickyThreadAccountId);
426
- }
427
-
428
- const resolved = resolveInitialAccount(resolvedPreferred, excludeSet);
429
-
430
- if (logger.isLevelEnabled("info")) {
431
- // Why THIS account? The operator needs the decision, not just the
432
- // result โ€” the previous rounds' "stale label" / "switching" confusion
433
- // came from logs that showed only the outcome.
434
- const poolSize = resolved.configuredAccounts.length;
435
- const cooldownCount = resolved.configuredAccounts.filter((a) =>
436
- getAccountCooldownInfo(a.id),
437
- ).length;
438
- const why =
439
- resolved.account.id === stickyThreadAccountId
440
- ? "sticky"
441
- : typeof resolvedPreferred === "string" &&
442
- resolved.account.id === resolvedPreferred
443
- ? "preferred"
444
- : resolvedPreferred === null
445
- ? "failover-rotate"
446
- : "round-robin";
447
- console.log(
448
- `๐ŸŽฏ [Chat] Account selected | ${maskEmail(resolved.account.email)} (${resolved.account.id}) | reason=${why} | pool=${poolSize}${cooldownCount ? ` | cooldown=${cooldownCount}` : ""}${stickyThreadAccountId ? ` | sticky=${stickyThreadAccountId === resolved.account.id}` : ""}`,
449
- );
450
- }
451
-
452
- let account: SelectedAccount | null = resolved.account;
453
- const configuredAccounts = resolved.configuredAccounts;
454
- const triedAccountIds = new Set<string>();
455
- let lastError: any = null;
456
- let antiBotRotations = 0;
457
-
458
- while (account) {
459
- const accountId = account.id;
460
- const accountEmail = maskEmail(account.email);
461
-
462
- if (triedAccountIds.has(accountId)) {
463
- account = getNextAvailableAccount(triedAccountIds);
464
- continue;
465
- }
466
- triedAccountIds.add(accountId);
467
-
468
- // Skip accounts that recently returned chat_in_progress (temporary busy) โ€”
469
- // except the sticky thread owner: hopping the owner splinters the
470
- // conversation and replays the full context on a cold account (~12s
471
- // reopen + captcha) when the upstream chat is merely settling (2-4s,
472
- // covered by the same-chat settle retries). Mirrors the saturated-account
473
- // exception below.
474
- if (
475
- isAccountTemporarilyBusy(accountId) &&
476
- params.allowTemporarilyBusyAccountId !== accountId &&
477
- accountId !== stickyThreadAccountId
478
- ) {
479
- console.log(
480
- `โญ๏ธ [Chat] Skipping account ${accountEmail} (${accountId}) temporarily busy (chat in progress)`,
481
- );
482
- account = getNextAvailableAccount(triedAccountIds);
483
- continue;
484
- }
485
-
486
- // Do not wait 30 seconds on a saturated account when another account is
487
- // already free. Keep the queue behavior only when this is the last usable
488
- // account, so single-account deployments remain lossless.
489
- // The thread owner is excluded: rotating the sticky account during a
490
- // tool/think pause splinters the conversation across upstream chats, so
491
- // it must queue on its own slot instead of being skipped.
492
- if (
493
- isAccountBusy(accountId) &&
494
- accountId !== stickyThreadAccountId &&
495
- hasFreeAlternateAccount(configuredAccounts, accountId, triedAccountIds)
496
- ) {
497
- console.log(
498
- `โญ๏ธ [Chat] Skipping account ${accountEmail} (${accountId}) busy; rotating to a free account`,
499
- );
500
- const nextCandidate = getNextAvailableAccount(triedAccountIds);
501
- if (nextCandidate && !getAccountCooldownInfo(nextCandidate.id)) {
502
- account = nextCandidate;
503
- continue;
504
- }
505
- // No usable alternate account exists (e.g. all other accounts are on cooldown):
506
- // do NOT skip this busy account to death. Fall through and queue on its slot.
507
- }
508
-
509
- const cooldownInfo = getAccountCooldownInfo(accountId);
510
- if (cooldownInfo) {
511
- console.log(
512
- `โญ๏ธ [Chat] Skipping account ${accountEmail} (${accountId}) on cooldown for ${Math.round(cooldownInfo.remainingMs / 1000)}s (${cooldownInfo.reason})`,
513
- );
514
- if (stickyThreadAccountId === accountId) {
515
- console.warn(
516
- `โš ๏ธ [Chat] Sticky account is on cooldown; recreating upstream chat on another account with full context.`,
517
- );
518
- }
519
- account = getNextAvailableAccount(triedAccountIds);
520
- continue;
521
- }
522
-
523
- if (isToolcallDebugEnabled()) {
524
- logger.debug("[chat] account selected", {
525
- accountId,
526
- accountEmail,
527
- isNewSession,
528
- isThinkingModel,
529
- promptLength: finalPrompt.length,
530
- });
531
- }
532
-
533
- if (useThreadNative && logger && process.env.CHAT_REQUEST_LOG === "true") {
534
- logger.info("[chat] thread-native routing", {
535
- sessionId,
536
- accountId,
537
- stickyAccountId: stickyThreadAccountId,
538
- hasExistingThread: !!existingThread,
539
- existingChatSessionId: existingThread?.chatSessionId || null,
540
- existingParentId: existingThread?.parentId || null,
541
- instructionsSent: existingThread?.instructionsSent || false,
542
- allowThreadReuse,
543
- forceNewChat,
544
- hasExplicitConversationKey: params.allowThreadReuse,
545
- });
546
- }
547
-
548
- try {
549
- // Any account change vs the sticky owner must resend full history into a
550
- // brand-new upstream chat โ€” the previous account's parent chain is unusable.
551
- // Same-account forceNewChat keeps the caller's finalPrompt (may already be
552
- // a rollover summary or a full-history rebuild from the retry layer).
553
- const recreatingOnNewAccount =
554
- !!stickyThreadAccountId && accountId !== stickyThreadAccountId;
555
- const mustReplayFullContext =
556
- recreatingOnNewAccount || threadMissingParent;
557
- const attemptForceNewChat = forceNewChat || mustReplayFullContext;
558
- const attemptFinalPrompt = mustReplayFullContext
559
- ? params.fullPrompt
560
- : finalPrompt;
561
- // The thread owner (or a deployment where no alternate account is
562
- // free) must queue on its own slot until generation finishes. A hard
563
- // 30s busy timeout here would needlessly 500 the same conversation
564
- // while the model is paused mid-tool/think.
565
- const waitForSlot =
566
- (!!stickyThreadAccountId && accountId === stickyThreadAccountId) ||
567
- !hasFreeAlternateAccount(
568
- configuredAccounts,
569
- accountId,
570
- triedAccountIds,
571
- );
572
- const result = await tryCreateStreamWithRetry(
573
- {
574
- finalPrompt: attemptFinalPrompt,
575
- isThinkingModel,
576
- model,
577
- reasoningMode,
578
- shouldResetUpstreamThread,
579
- allFiles,
580
- sessionId,
581
- useThreadNative,
582
- updateLogicalThread,
583
- forceNewChat: attemptForceNewChat,
584
- existingThread:
585
- !mustReplayFullContext &&
586
- existingThread &&
587
- existingThread.accountId === accountId
588
- ? existingThread
589
- : null,
590
- messageCount: mustReplayFullContext
591
- ? (params.fullMessageCount ?? params.messageCount)
592
- : params.messageCount,
593
- fullMessageCount: params.fullMessageCount,
594
- toolsCount: params.toolsCount,
595
- requestPersonalizationInstruction:
596
- params.requestPersonalizationInstruction,
597
- contextModelId: params.contextModelId,
598
- fullPrompt: params.fullPrompt,
599
- contextMode: mustReplayFullContext
600
- ? "replay"
601
- : params.contextMode,
602
- requestSignal: params.requestSignal,
603
- queueSlotUntilFree: waitForSlot,
604
- messages: params.messages,
605
- completionId,
606
- parallelEscape: params.parallelEscape,
607
- chatMode,
608
- },
609
- accountId,
610
- accountEmail,
611
- );
612
-
613
- if (result.success) {
614
- registerStream(completionId, {
615
- abortController: result.controller,
616
- accountId: result.accountId,
617
- uiSessionId: result.uiSessionId,
618
- targetResponseId: "",
619
- headers: result.headers,
620
- });
621
-
622
- return {
623
- stream: result.stream,
624
- uiSessionId: result.uiSessionId,
625
- activeAccountId: result.accountId,
626
- activeAccountLabel: result.accountEmail,
627
- replayedFullContext: mustReplayFullContext,
628
- completionId,
629
- logicalSessionId:
630
- useThreadNative && updateLogicalThread ? sessionId : null,
631
- createdNewChat: result.createdNewChat,
632
- tokenEstimationContext: {
633
- ...result.tokenEstimationContext,
634
- requestDeclaredToolCount: params.toolsCount ?? 0,
635
- },
636
- releaseAccountLease: result.releaseAccountLease,
637
- };
638
- }
639
-
640
- lastError = result.error;
641
- } catch (err: any) {
642
- lastError = err;
643
- }
644
-
645
- // The request signal is shared by every account attempt. Once the client
646
- // disconnects, stop the outer rotation loop as well as inner retries.
647
- if (params.requestSignal?.aborted) {
648
- break;
649
- }
650
-
651
- // Client/proxy validation errors must not be retried on other accounts.
652
- // In particular, an oversized prompt is independent of the selected
653
- // account; rotating accounts only repeats the same 400 response and can
654
- // also rebuild the full history several times.
655
- if (isTerminalLocalError(lastError)) {
656
- break;
657
- }
658
-
659
- const quotaInfo = (lastError as any)?.quotaInfo as
660
- | {
661
- email: string;
662
- cooldownSeconds: number;
663
- untilStr: string;
664
- message: string;
665
- }
666
- | undefined;
667
- if (quotaInfo) {
668
- const stickyRotation =
669
- stickyThreadAccountId === accountId &&
670
- (isAccountUnavailableError(lastError) ||
671
- isAccountInitializationError(lastError) ||
672
- isChatInProgressError(lastError));
673
- console.warn(
674
- `โš ๏ธ [Chat] Quota exceeded | ${quotaInfo.email} | cooldown=${quotaInfo.cooldownSeconds}s${quotaInfo.untilStr} | ${quotaInfo.message}${stickyRotation ? " | switching sticky account with full context" : ""}`,
675
- );
676
- }
677
-
678
- // chat_in_progress exhaustion is TERMINAL: tryCreateStreamWithRetry already
679
- // spent the full same-chat settle budget AND its single bounded escalation
680
- // (fresh chat + full replay). Reaching this point means even the escalated
681
- // fresh chat failed โ€” the inner loop already cleared the origin binding, so
682
- // the client's next turn starts fresh. Rotating from here would only add a
683
- // SECOND full-context replay on a cold account for no gain.
684
- if (isChatInProgressError(lastError)) {
685
- if (logger.isLevelEnabled("info")) {
686
- console.log(
687
- `๐Ÿ›‘ [Chat] chat_in_progress budget exhausted (post-escalation) | ${maskEmail(accountEmail)} | failing without account rotation`,
688
- );
689
- }
690
- break;
691
- }
692
-
693
- if (stickyThreadAccountId === accountId) {
694
- // A challenged sticky account must be allowed to fall through to the
695
- // anti-bot handling below; otherwise the whole conversation dies on the
696
- // account the WAF happened to pick.
697
- const stickyAccountMustRotate =
698
- isAccountUnavailableError(lastError) ||
699
- isAccountInitializationError(lastError) ||
700
- isAntiBotError(lastError);
701
- if (stickyAccountMustRotate) {
702
- if (!quotaInfo) {
703
- console.warn(
704
- `โš ๏ธ [Chat] Sticky account unavailable (${isAntiBotError(lastError) ? "waf_challenge" : "upstream failure"}); trying another account with full context.`,
705
- );
706
- }
707
- } else {
708
- break;
709
- }
710
- }
711
-
712
- // The inner retry loop already replayed this account and tried to clear the
713
- // challenge. Hand the request to one other account rather than failing it
714
- // outright, then stop: walking the whole pool would only get every account
715
- // challenged in turn and multiply the solver budget by the pool size.
716
- if (isAntiBotError(lastError)) {
717
- // Hard block: the challenge was NOT solved here. Quarantine with an
718
- // escalating window AND rotate the device fingerprint + reset the
719
- // browser context, so the account does not return from cooldown on the
720
- // same identity the WAF already flagged (upstream account-isolation).
721
- recordWafHardBlock(accountId);
722
-
723
- if (antiBotRotations >= MAX_ANTI_BOT_ROTATIONS) {
724
- console.warn(
725
- `โš ๏ธ [Chat] WAF challenge retries exhausted | ${accountEmail} | no further rotation`,
726
- );
727
- break;
728
- }
729
-
730
- const nextAfterChallenge = getNextAvailableAccount(triedAccountIds);
731
- if (!nextAfterChallenge) {
732
- console.warn(
733
- `โš ๏ธ [Chat] WAF challenge retries exhausted | ${accountEmail} | no other account available`,
734
- );
735
- break;
736
- }
737
-
738
- antiBotRotations++;
739
- console.warn(
740
- `๐Ÿ”„ [Chat] WAF challenge on ${accountEmail}; retrying on ${maskEmail(nextAfterChallenge.email)}`,
741
- );
742
- account = nextAfterChallenge;
743
- continue;
744
- }
745
-
746
- if (isToolcallDebugEnabled()) {
747
- logger.debug("[chat] account failed, rotating", {
748
- accountId,
749
- accountEmail: maskEmail(accountEmail),
750
- triedAccounts: Array.from(triedAccountIds),
751
- });
752
- }
753
-
754
- account = getNextAvailableAccount(triedAccountIds);
755
- }
756
-
757
- // All accounts exhausted.
758
- removeStream(completionId);
759
-
760
- if (!lastError && configuredAccounts.length > 0) {
761
- const cooldownInfos = configuredAccounts
762
- .map((acc) => getAccountCooldownInfo(acc.id))
763
- .filter(
764
- (
765
- info,
766
- ): info is NonNullable<ReturnType<typeof getAccountCooldownInfo>> =>
767
- info !== null,
768
- );
769
-
770
- if (cooldownInfos.length === configuredAccounts.length) {
771
- const retryAfterMs = Math.min(
772
- ...cooldownInfos.map((info) => info.remainingMs),
773
- );
774
- const cooldownError: any = new Error(
775
- `All configured accounts are on cooldown. Retry in about ${Math.max(1, Math.ceil(retryAfterMs / 1000))}s.`,
776
- );
777
- cooldownError.upstreamStatus = 429;
778
- cooldownError.retryAfterMs = retryAfterMs;
779
- return {
780
- error: cooldownError,
781
- completionId,
782
- allOnCooldown: true,
783
- retryAfterMs,
784
- };
785
- }
786
- }
787
-
788
- if (!lastError) {
789
- const busyOrCooldownError: any = new Error(
790
- "No accounts available: all accounts are either in use or on cooldown. Retry shortly.",
791
- );
792
- busyOrCooldownError.upstreamStatus = 429;
793
- return {
794
- error: busyOrCooldownError,
795
- completionId,
796
- allOnCooldown: false,
797
- };
798
- }
799
-
800
- return {
801
- error: lastError,
802
- completionId,
803
- allOnCooldown: false,
804
- };
805
- }
806
-
807
- interface CreateStreamSuccess {
808
- success: true;
809
- stream: ReadableStream;
810
- uiSessionId: string;
811
- accountId: string;
812
- /** Account email that actually served the request (inner rotation may
813
- * switch accounts โ€” parallel escape / other retry policies). */
814
- accountEmail: string;
815
- controller: AbortController;
816
- headers: Record<string, string>;
817
- createdNewChat: boolean;
818
- tokenEstimationContext: TokenEstimationContext;
819
- releaseAccountLease: () => void;
820
- }
821
-
822
- interface CreateStreamFailure {
823
- success: false;
824
- error: any;
825
- }
826
-
827
- /**
828
- * Pure: jittered same-chat settle wait for the nth chat_in_progress retry.
829
- * The base grows with the failure count (busyMs โ†’ 2ร—busyMs from the 4th
830
- * retry) and context size (promptChars > 500KB/1MB/2MB), randomized within ยฑ25%
831
- * so concurrent sessions sharing an account never retry in lock-step.
832
- */
833
- export function jitterChatInProgressDelay(
834
- retryCount: number,
835
- busyMs: number,
836
- rand: () => number = Math.random,
837
- promptChars: number = 0,
838
- ): number {
839
- let base = retryCount >= 4 ? busyMs * 2 : busyMs;
840
- if (promptChars > 2_000_000) {
841
- base = Math.round(base * 1.75);
842
- } else if (promptChars > 1_000_000) {
843
- base = Math.round(base * 1.5);
844
- } else if (promptChars > 500_000) {
845
- base = Math.round(base * 1.25);
846
- }
847
- const raw = base * (0.75 + 0.5 * Math.min(1, Math.max(0, rand())));
848
- return Math.min(20_000, Math.max(1, Math.round(raw)));
849
- }
850
-
851
- /**
852
- * Decision helper for the per-account lease queue deadline.
853
- *
854
- * `true` means the request may wait up to `queueWaitForeverCapMs` for the
855
- * slot (thread owner waiting on its OWN session, or the last usable account).
856
- * `false` means it waits at most `busyWaitMs` and then fails with
857
- * `account_busy` so the attempt loop can rotate accounts.
858
- *
859
- * The thread-owner preference is NOT enough on its own: when another session
860
- * holds the slot, waiting long is pure latency (the other session may keep
861
- * generating for minutes). Only the same-session holder justifies the long
862
- * wait (same-session latest-wins / tool loop).
863
- */
864
- export function shouldWaitQueueForever(
865
- isThreadOwnerWaiting: boolean,
866
- heldByOtherSession: boolean,
867
- hasFreeAlternate: boolean,
868
- ): boolean {
869
- return (isThreadOwnerWaiting && !heldByOtherSession) || !hasFreeAlternate;
870
- }
871
-
872
- async function tryCreateStreamWithRetry(
873
- params: {
874
- finalPrompt: string;
875
- fullPrompt: string;
876
- isThinkingModel: boolean;
877
- model: string;
878
- reasoningMode?: "auto" | "thinking" | "fast";
879
- shouldResetUpstreamThread: boolean;
880
- allFiles: QwenFileEntry[];
881
- sessionId: string | null;
882
- useThreadNative: boolean;
883
- updateLogicalThread: boolean;
884
- forceNewChat: boolean;
885
- existingThread: LogicalThreadEntry | null;
886
- messageCount?: number;
887
- fullMessageCount?: number;
888
- toolsCount?: number;
889
- requestPersonalizationInstruction?: string | null;
890
- contextModelId?: string;
891
- contextMode?: ContextMeterMode;
892
- requestSignal?: AbortSignal;
893
- queueSlotUntilFree?: boolean;
894
- messages?: Message[];
895
- /** Stream registry key; the emit-aware supersede links the lease to it. */
896
- completionId: string;
897
- /**
898
- * True when this request races a same-session stream that has NOT
899
- * emitted yet (title/parallel request): run on its OWN chat instead of
900
- * waiting on the main chat's lock, and hop accounts fast (tryAcquire).
901
- */
902
- parallelEscape?: boolean;
903
- /** "thread" (reuse upstream chat) or "temp" (new ephemeral chat per request). */
904
- chatMode: ChatMode;
905
- },
906
- accountId: string,
907
- accountEmail: string,
908
- ): Promise<CreateStreamSuccess | CreateStreamFailure> {
909
- const maxAttempts = Math.max(1, config.retry.maxAttempts);
910
- const maxAccountSwitches = Math.max(0, config.retry.maxAccountSwitches);
911
- let attemptsLeft = maxAttempts;
912
- let retryDelay = config.retry.baseDelayMs;
913
- let attempt = 0;
914
- let quotaRetried = false;
915
- let accountSwitches = 0;
916
- let chatInProgressCount = 0;
917
- let chatInProgressEscalated = false;
918
- // Account that accumulated the chat_in_progress failures (the one whose
919
- // upstream chat is actually stuck "in progress"). The post-budget
920
- // escalation switches to a FRESH chat, and the loop-exit session clear
921
- // must drop the binding to the stuck chat โ€” NOT clear the sessions of an
922
- // account that never served this session (cross-session damage).
923
- let chatInProgressOriginAccountId: string | null = null;
924
- let chatInProgressOriginAccountEmail: string | null = null;
925
- let lastAttemptError: any = null;
926
- let invalidInputSameAccountRetried = false;
927
- const accounts = loadAccounts();
928
- const isSingleAccount = accounts.length <= 1;
929
- let currentAccountId = accountId;
930
- let currentAccountEmail = accountEmail;
931
- const triedAccounts = new Set<string>([accountId]);
932
-
933
- while (attemptsLeft > 0) {
934
- attempt++;
935
- if (attempt > 1) {
936
- console.log(
937
- `๐Ÿ”„ [Chat] Retrying request | ${currentAccountEmail} | ${params.model} | ${params.messageCount ?? "?"} msg(s) | ${params.finalPrompt.length} chars${params.toolsCount ? ` | ${params.toolsCount} tool(s)` : ""} | attempt ${attempt}`,
938
- );
939
- }
940
- let attemptError: any = null;
941
- let accountLease: AccountLease | null = null;
942
- const acquireStartedAt = Date.now();
943
-
944
- try {
945
- // The client may have cancelled between account selection and this
946
- // attempt (or while a previous attempt was running). Bail before
947
- // spending time on model sync, truncation or personalization.
948
- if (params.requestSignal?.aborted) {
949
- return {
950
- success: false,
951
- error: new ClientAbortedError(
952
- "client aborted before stream creation",
953
- ),
954
- };
955
- }
956
-
957
- // Always sync the model catalog so the truncation and prompt-limit
958
- // checks use the real context window published by Qwen, not the
959
- // conservative registry fallback. The call is cached per account.
960
- try {
961
- await fetchQwenModels(currentAccountId);
962
- } catch (metadataError) {
963
- logger.warn("[chat] model metadata sync unavailable; using registry fallback", {
964
- model: params.contextModelId ?? params.model,
965
- error:
966
- metadataError instanceof Error
967
- ? metadataError.message
968
- : String(metadataError),
969
- });
970
- }
971
-
972
- // Truncate after the real context window is known so long conversations
973
- // are not cut down to the conservative 128K fallback.
974
- const contextModelId = params.contextModelId ?? params.model;
975
- const truncation = truncatePromptToIntelligentLimit(
976
- params.finalPrompt,
977
- contextModelId,
978
- currentAccountId,
979
- params.messages,
980
- );
981
- if (truncation.wasTruncated) {
982
- logger.warn(
983
- "[chat] prompt exceeded model context limit; intelligent truncation applied",
984
- {
985
- originalTokens: truncation.originalTokens,
986
- truncatedTokens: truncation.truncatedTokens,
987
- messagesKept: truncation.messagesKept,
988
- messagesDropped: truncation.messagesDropped,
989
- },
990
- );
991
- }
992
- const loopReminder = buildRepeatedToolCallReminder(
993
- params.messages,
994
- config.retry.repeatedToolCallWarnThreshold,
995
- );
996
- const effectivePrompt = loopReminder
997
- ? `${truncation.prompt}\n\n${loopReminder}`
998
- : truncation.prompt;
999
-
1000
- assertPromptWithinLimits(
1001
- effectivePrompt,
1002
- contextModelId,
1003
- { accountId: currentAccountId },
1004
- );
1005
-
1006
- const threadParentId = params.useThreadNative
1007
- ? params.forceNewChat || params.parallelEscape
1008
- ? null
1009
- : (params.existingThread?.parentId ?? null)
1010
- : params.shouldResetUpstreamThread
1011
- ? null
1012
- : undefined;
1013
- // Acquire account concurrency lease before personalization + stream creation.
1014
- // The lease is held for the entire stream lifetime and released by the caller
1015
- // via the returned releaseAccountLease function.
1016
- // The thread owner or the last usable account waits without a hard
1017
- // deadline (bounded by the client's abort signal): a fixed 30s timeout
1018
- // here rejects a single conversation while the model is paused
1019
- // mid-tool/thinking, which burns the request instead of serving it.
1020
- const hasFreeAlt =
1021
- !isSingleAccount &&
1022
- hasFreeAlternateAccount(accounts, currentAccountId, triedAccounts);
1023
- const sessionLabel = params.sessionId ?? currentAccountEmail;
1024
- // The thread owner (or the last usable account) may wait for the slot
1025
- // without a short deadline โ€” but ONLY when the slot is busy with OUR
1026
- // session (same-session latest-wins / tool loop). If ANOTHER session is
1027
- // generating on this account, waiting 120s is pure latency: the other
1028
- // session may hold the slot for minutes. Fail fast with account_busy so
1029
- // the attempt loop rotates to a different account instead (or retries
1030
- // quickly when no alternative exists).
1031
- const heldByOtherSession = isAccountSlotHeldByOtherSession(
1032
- currentAccountId,
1033
- sessionLabel,
1034
- );
1035
- const waitQueueForever = shouldWaitQueueForever(
1036
- params.queueSlotUntilFree === true,
1037
- heldByOtherSession,
1038
- hasFreeAlt,
1039
- );
1040
-
1041
- // Latest-wins: if the client retried the same session, abort the old
1042
- // generation and free the slot immediately instead of queueing behind it.
1043
- // onlyIfEmitted: a stream that has NOT reached the client yet is
1044
- // protected โ€” killing it would waste a generation the client has not
1045
- // consumed. A PARALLEL request (parallelEscape) never kills at all: it
1046
- // runs on its own chat and must not abort the main generation even
1047
- // after the main emits its first chunk.
1048
- if (params.sessionId && !params.parallelEscape) {
1049
- abortLeaseByLabel(currentAccountId, sessionLabel, {
1050
- onlyIfEmitted: true,
1051
- });
1052
- }
1053
-
1054
- // Create an AbortController for this lease so a future same-session
1055
- // retry can abort it via abortLeaseByLabel().
1056
- const leaseAbort = new AbortController();
1057
- // Second per-attempt controller: the acquire deadline aborts it so a
1058
- // race-lost createQwenStream (still queued on the account stream
1059
- // lock) dies instead of winning the lock later and burning an
1060
- // upstream request while unobserved.
1061
- const acquireAbort = new AbortController();
1062
- const combinedSignal = params.requestSignal
1063
- ? AbortSignal.any([
1064
- params.requestSignal,
1065
- leaseAbort.signal,
1066
- acquireAbort.signal,
1067
- ])
1068
- : AbortSignal.any([leaseAbort.signal, acquireAbort.signal]);
1069
-
1070
- if (params.parallelEscape) {
1071
- // Parallel request racing an unemitted stream: do NOT queue on this
1072
- // account's slot (the main may hold it for minutes while thinking).
1073
- // Fail fast with account_busy so the attempt loop hops to a free
1074
- // account; on a free account the request proceeds on its own chat.
1075
- const quick = tryAcquireAccountLease(
1076
- currentAccountId,
1077
- sessionLabel,
1078
- leaseAbort,
1079
- true,
1080
- );
1081
- if (!quick) {
1082
- const busyError = new Error(
1083
- `Account ${currentAccountId} busy: parallel request (session stream unemitted)`,
1084
- ) as Error & { code?: string; parallelEscape?: boolean };
1085
- busyError.code = "account_busy";
1086
- // Expected hop, not an error: suppress the "Request failed" warn.
1087
- busyError.parallelEscape = true;
1088
- throw busyError;
1089
- }
1090
- accountLease = quick;
1091
- } else {
1092
- accountLease = await acquireAccountLease(currentAccountId, {
1093
- timeoutMs: waitQueueForever
1094
- ? config.concurrency.queueWaitForeverCapMs
1095
- : config.concurrency.busyWaitMs,
1096
- signal: combinedSignal,
1097
- label: sessionLabel,
1098
- leaseAbortController: leaseAbort,
1099
- });
1100
- }
1101
- // Client may have disconnected (or a same-session retry superseded us)
1102
- // while waiting for the lease. Bail before spending time on
1103
- // personalization sync / captcha solve.
1104
- if (combinedSignal.aborted) {
1105
- accountLease.release();
1106
- return {
1107
- success: false,
1108
- error: new ClientAbortedError(
1109
- "client aborted before stream creation",
1110
- ),
1111
- };
1112
- }
1113
- if (logger.isLevelEnabled("info")) {
1114
- console.log(
1115
- `โฑ๏ธ [Chat] Acquire: lease | account=${currentAccountEmail} | +${Date.now() - acquireStartedAt}ms`,
1116
- );
1117
- }
1118
- const hasRequestPersonalization =
1119
- params.requestPersonalizationInstruction !== null &&
1120
- params.requestPersonalizationInstruction !== undefined;
1121
- const releasePersonalization = hasRequestPersonalization
1122
- ? await acquirePersonalizationLock(currentAccountId)
1123
- : null;
1124
- // A same-session retry (or client disconnect) can abort this request
1125
- // while the personalization sync is still stuck on a hung page op
1126
- // (closed Playwright context / WAF). The sync never resolves, so the
1127
- // finally below would not run and the mutex would stay held for
1128
- // minutes, blocking the retry until its 60s acquire timeout fires.
1129
- // Release the lock immediately on abort instead.
1130
- const onPersonalizationAbort = () => releasePersonalization?.();
1131
- if (combinedSignal.aborted) {
1132
- onPersonalizationAbort();
1133
- } else {
1134
- combinedSignal.addEventListener("abort", onPersonalizationAbort, {
1135
- once: true,
1136
- });
1137
- }
1138
- let result: Awaited<ReturnType<typeof createQwenStream>>;
1139
- try {
1140
- let promptForUpstream = effectivePrompt;
1141
- if (hasRequestPersonalization) {
1142
- // Let the hash-based cache in syncQwenRequestPersonalization decide
1143
- // whether to actually POST. A new chat does not imply the account's
1144
- // global settings were reset โ€” only session refresh or profile reset
1145
- // should bypass the cache.
1146
- const instruction =
1147
- params.requestPersonalizationInstruction ?? "";
1148
- let personalizationApplied = false;
1149
- let syncFailure: string | null = null;
1150
- try {
1151
- // Hard deadline for the whole sync (browser ops each have
1152
- // their own 60s timeout; several sequential stuck ops can
1153
- // hold the personalization mutex for minutes). A normal
1154
- // sync takes ~2s; beyond 30s the account page is stuck โ€”
1155
- // fail fast so the retry loop switches accounts.
1156
- let syncSettled = false;
1157
- let personalizationDeadlineTimer: NodeJS.Timeout | undefined;
1158
- const syncPromise = syncQwenRequestPersonalization(
1159
- instruction,
1160
- currentAccountId === "global"
1161
- ? undefined
1162
- : currentAccountId,
1163
- {
1164
- model: params.model,
1165
- toolsCount: params.toolsCount ?? 0,
1166
- sessionId: params.sessionId,
1167
- promptChars: effectivePrompt.length,
1168
- forceSync: false,
1169
- },
1170
- ).then(
1171
- (value) => {
1172
- syncSettled = true;
1173
- return value;
1174
- },
1175
- (error) => {
1176
- syncSettled = true;
1177
- syncFailure =
1178
- error instanceof Error ? error.message : String(error);
1179
- return false;
1180
- },
1181
- );
1182
- personalizationApplied = await Promise.race([
1183
- syncPromise,
1184
- new Promise<boolean>((resolve) => {
1185
- personalizationDeadlineTimer = setTimeout(() => {
1186
- if (!syncSettled) {
1187
- syncFailure = `sync timed out after ${PERSONALIZATION_SYNC_DEADLINE_MS}ms`;
1188
- }
1189
- resolve(false);
1190
- }, PERSONALIZATION_SYNC_DEADLINE_MS);
1191
- }),
1192
- ]);
1193
- // The sync won the race: stop the deadline so it cannot keep the
1194
- // event loop alive for the full 30s window (it used to leak one
1195
- // 30s timer per request โ†’ ~30s of test-suite drain per file).
1196
- if (personalizationDeadlineTimer) {
1197
- clearTimeout(personalizationDeadlineTimer);
1198
- }
1199
- } catch (error) {
1200
- syncFailure =
1201
- error instanceof Error ? error.message : String(error);
1202
- }
1203
-
1204
- // Agent instructions ride ONLY the account-level personalization โ€”
1205
- // the prompt never carries them. An unconfirmed sync must fail the
1206
- // attempt (retryable โ†’ rotates accounts, each re-syncs on its own
1207
- // account) instead of degrading to inline. An empty instruction has
1208
- // nothing to guarantee (plain chat), so it stays best-effort.
1209
- if (instruction && !personalizationApplied) {
1210
- throw new PersonalizationSyncError(
1211
- `personalization sync not confirmed for ${currentAccountEmail}: ${syncFailure ?? "settings response did not confirm the instruction"}`,
1212
- );
1213
- }
1214
- }
1215
- if (logger.isLevelEnabled("info")) {
1216
- console.log(
1217
- `โฑ๏ธ [Chat] Acquire: sync | account=${currentAccountEmail} | +${Date.now() - acquireStartedAt}ms`,
1218
- );
1219
- }
1220
-
1221
- assertPromptWithinLimits(
1222
- promptForUpstream,
1223
- params.contextModelId ?? params.model,
1224
- { accountId: currentAccountId },
1225
- );
1226
- // Bound the whole acquire with a hard deadline: a silent hang in any
1227
- // phase (mutex wait, header capture, fetch metadata, internal retries)
1228
- // fails fast and retryable instead of blocking the request for minutes
1229
- // with zero log output.
1230
- const acquireDeadlineMs = config.concurrency.acquireDeadlineMs;
1231
- let acquireDeadlineTimer: NodeJS.Timeout | undefined;
1232
- const acquireDeadline = new Promise<never>((_, reject) => {
1233
- acquireDeadlineTimer = setTimeout(() => {
1234
- // Abort the losing createQwenStream (it is still queued on the
1235
- // stream lock or mid-create); the post-lock signal re-check in
1236
- // createQwenStream then throws instead of letting the orphan
1237
- // win the lock later and waste an upstream request.
1238
- acquireAbort.abort();
1239
- const err = new Error(
1240
- `Acquire deadline (${acquireDeadlineMs}ms) exceeded creating stream on ${currentAccountEmail}`,
1241
- ) as Error & { code?: string };
1242
- err.code = "acquire_deadline";
1243
- reject(err);
1244
- }, acquireDeadlineMs);
1245
- acquireDeadlineTimer.unref?.();
1246
- });
1247
- result = await Promise.race([
1248
- createQwenStream(
1249
- promptForUpstream,
1250
- params.isThinkingModel,
1251
- params.model,
1252
- threadParentId,
1253
- currentAccountId === "global" ? undefined : currentAccountId,
1254
- params.allFiles.length > 0 ? params.allFiles : undefined,
1255
- params.forceNewChat || params.useThreadNative || params.parallelEscape
1256
- ? {
1257
- chatSessionId:
1258
- params.forceNewChat || params.parallelEscape
1259
- ? null
1260
- : (params.existingThread?.chatSessionId ?? null),
1261
- forceNewChat: false,
1262
- reasoningMode: params.reasoningMode,
1263
- parallelEscape: params.parallelEscape,
1264
- chatMode: params.chatMode,
1265
- }
1266
- : params.reasoningMode ? { reasoningMode: params.reasoningMode } : undefined,
1267
- combinedSignal,
1268
- ),
1269
- acquireDeadline,
1270
- ]);
1271
- // The acquire won: stop the deadline so it cannot fire later and
1272
- // abort a signal nobody observes anymore.
1273
- if (acquireDeadlineTimer) clearTimeout(acquireDeadlineTimer);
1274
-
1275
- if (logger.isLevelEnabled("info")) {
1276
- console.log(
1277
- `โฑ๏ธ [Chat] Acquire done | completion=${params.completionId.substring(0, 8)} | account=${currentAccountEmail} | +${Date.now() - acquireStartedAt}ms`,
1278
- );
1279
- }
1280
-
1281
- const contextMeter = buildContextMeterSnapshot({
1282
- modelId: params.contextModelId ?? params.model,
1283
- accountId: currentAccountId,
1284
- requestPrompt: promptForUpstream,
1285
- fullPrompt: params.fullPrompt,
1286
- mode:
1287
- params.contextMode ??
1288
- (params.forceNewChat
1289
- ? "replay"
1290
- : params.existingThread
1291
- ? "delta"
1292
- : "full"),
1293
- qwenPayloadBytes: result.tokenEstimationContext.qwenPayloadBytes,
1294
- qwenPayloadPromptChars:
1295
- result.tokenEstimationContext.qwenPayloadPromptChars,
1296
- qwenPayloadMessageCount:
1297
- result.tokenEstimationContext.qwenPayloadMessageCount,
1298
- messageCount: params.messageCount,
1299
- fullMessageCount: params.fullMessageCount,
1300
- toolsCount: params.toolsCount,
1301
- filesCount: params.allFiles.length,
1302
- activePersonalization:
1303
- result.tokenEstimationContext.activePersonalization,
1304
- });
1305
-
1306
- if (contextMeter) {
1307
- logger.debug("[context_meter] request", {
1308
- ...contextMeterLogData(contextMeter),
1309
- account: currentAccountEmail,
1310
- attempt,
1311
- });
1312
- result = {
1313
- ...result,
1314
- tokenEstimationContext: {
1315
- ...result.tokenEstimationContext,
1316
- contextMeter,
1317
- },
1318
- };
1319
- }
1320
- } finally {
1321
- combinedSignal.removeEventListener(
1322
- "abort",
1323
- onPersonalizationAbort,
1324
- );
1325
- releasePersonalization?.();
1326
- }
1327
-
1328
- // Client cancelled (or a same-session retry superseded us) during the
1329
- // (potentially slow) personalization sync. Bail before createQwenStream
1330
- // spends time on header capture / captcha.
1331
- if (combinedSignal.aborted) {
1332
- // Never drop a created stream without cancelling it: the wrapped
1333
- // stream's cancel() releases the per-account stream lock. Dropping it
1334
- // silently LEAKS that lock and the next acquire on this account blocks
1335
- // until the acquire deadline (observed symptom: 150s phantom wait).
1336
- void result.stream
1337
- .cancel("client aborted after stream creation")
1338
- .catch(() => {});
1339
- accountLease?.release();
1340
- return {
1341
- success: false,
1342
- error: new ClientAbortedError(
1343
- "client aborted during stream creation",
1344
- ),
1345
- };
1346
- }
1347
-
1348
- if (
1349
- params.useThreadNative &&
1350
- params.updateLogicalThread &&
1351
- !params.parallelEscape &&
1352
- params.sessionId &&
1353
- result.uiSessionId
1354
- ) {
1355
- // Bind chat/account immediately. Do NOT write the request parent as the
1356
- // sticky parent โ€” that is the *previous* assistant id we attached to.
1357
- // Streaming will rememberParent(response_id) with the new assistant id
1358
- // so the next turn appends (user_action=chat + parent_id=last response).
1359
- // Preserve any existing sticky parent until the stream updates it.
1360
- const priorParent =
1361
- params.existingThread?.parentId ??
1362
- getLogicalThreadState(params.sessionId)?.parentId ??
1363
- null;
1364
- updateLogicalThreadState(params.sessionId, {
1365
- accountId: result.accountId,
1366
- chatSessionId: result.uiSessionId,
1367
- parentId: params.forceNewChat ? null : priorParent,
1368
- instructionsSent: true,
1369
- });
1370
-
1371
- if (process.env.CHAT_REQUEST_LOG === "true") {
1372
- logger.info("[chat] thread-native upstream session", {
1373
- sessionId: params.sessionId,
1374
- accountId: result.accountId,
1375
- chatSessionId: result.uiSessionId,
1376
- requestParentId: threadParentId ?? null,
1377
- stickyParentId: params.forceNewChat ? null : priorParent,
1378
- createdNewChat: !params.existingThread,
1379
- });
1380
- }
1381
- }
1382
-
1383
- if (isToolcallDebugEnabled()) {
1384
- logger.debug("[chat] stream created successfully", {
1385
- accountId: currentAccountId,
1386
- accountEmail: currentAccountEmail,
1387
- uiSessionId: result.uiSessionId,
1388
- });
1389
- }
1390
-
1391
- // A served stream means the WAF accepted this account's identity:
1392
- // clear the hard-block escalation streak (keeps the next block at the
1393
- // base window instead of compounding forever).
1394
- noteWafRecovery(currentAccountId);
1395
- markAccountSuccessful(currentAccountId);
1396
- if (accountLease) {
1397
- markLeaseCompletion(
1398
- currentAccountId,
1399
- accountLease.leaseId,
1400
- params.completionId,
1401
- );
1402
- }
1403
- return {
1404
- success: true,
1405
- ...result,
1406
- accountEmail: currentAccountEmail,
1407
- releaseAccountLease: accountLease.release,
1408
- };
1409
- } catch (err: any) {
1410
- attemptError = err;
1411
- lastAttemptError = err;
1412
- // Release the lease on failure โ€” the stream was never created or
1413
- // will not be consumed by the caller.
1414
- accountLease?.release();
1415
- }
1416
-
1417
- attemptsLeft--;
1418
- const err = attemptError;
1419
- // The account that actually failed THIS attempt โ€” captured before any
1420
- // branch below can switch currentAccountId/Email (chat_in_progress
1421
- // escalation moves to a fresh account). The generic retry log must name
1422
- // the account that failed, not the newly-selected one that was never
1423
- // attempted (observed: "Qwen request failed for 280wu" when 280wu had
1424
- // never been tried and ldyjl had failed 4x with chat_in_progress).
1425
- const failedAccountEmail = currentAccountEmail;
1426
-
1427
- // Once the client request is aborted, do not rotate accounts or retry. The
1428
- // old request can otherwise keep acquiring leases after the client is gone.
1429
- if (params.requestSignal?.aborted) {
1430
- return { success: false, error: err };
1431
- }
1432
-
1433
- // Log the error details for debugging (skip quota errors โ€” logged separately below,
1434
- // client aborts โ€” they are silent by design, and chat_in_progress โ€” handled
1435
- // by the dedicated settling log below to avoid alarming false-positive error spam).
1436
- const errMsg = err instanceof Error ? err.message : String(err || "");
1437
- if (
1438
- err &&
1439
- !(err instanceof ClientAbortedError) &&
1440
- !isAccountUnavailableError(err) &&
1441
- !(err as any)?.parallelEscape &&
1442
- !isChatInProgressError(err)
1443
- ) {
1444
- const errCode = getQwenErrorCode(err) || "unknown";
1445
- console.warn(
1446
- `โŒ [Chat] Request failed | ${currentAccountEmail} | ${errCode} | ${errMsg.substring(0, 200)}`,
1447
- );
1448
- }
1449
-
1450
-
1451
-
1452
- if (!err) {
1453
- return {
1454
- success: false,
1455
- error: new Error("Failed to create Qwen stream"),
1456
- };
1457
- }
1458
-
1459
- if (
1460
- err instanceof QwenSessionExpiredError ||
1461
- err.name === "QwenSessionExpiredError"
1462
- ) {
1463
- console.warn(
1464
- `๐Ÿ”„ [Chat] Session expired for ${currentAccountEmail} (${currentAccountId}). Attempting re-login...`,
1465
- );
1466
- const reLoginOk = await attemptRelogin(
1467
- currentAccountId,
1468
- currentAccountEmail,
1469
- );
1470
- if (reLoginOk) continue;
1471
- return { success: false, error: err };
1472
- }
1473
-
1474
-
1475
-
1476
-
1477
-
1478
- // Account-scoped quota/rate-limit: cool this account and stop local retries
1479
- // so outer account rotation can pick another one immediately.
1480
- if (isAccountUnavailableError(err)) {
1481
- const quotaMsg = err.message || "Unknown quota error";
1482
- const policy = classifyRetryAction(err, {
1483
- requestAborted: params.requestSignal?.aborted === true,
1484
- });
1485
- const isTemporary = policy.accountCooldownReason === "RateLimitTemporary";
1486
-
1487
- // Temporary load shedding or single account: retry same account
1488
- // after a short delay before giving up / rotating.
1489
- if ((isTemporary || isSingleAccount) && !quotaRetried && attemptsLeft > 0) {
1490
- quotaRetried = true;
1491
- const delayMs = isTemporary ? 3_000 : config.retry.baseDelayMs;
1492
- console.warn(
1493
- `โš ๏ธ [Chat] Quota exceeded | ${currentAccountEmail} | ${isTemporary ? "temporary, " : ""}retrying in ${delayMs}ms...`,
1494
- );
1495
- await new Promise((resolve) =>
1496
- setTimeout(resolve, delayMs),
1497
- );
1498
- continue;
1499
- }
1500
-
1501
- // Consolidate quota details into a single log emitted by the outer
1502
- // rotation loop. The cooldown itself is set silently to avoid duplicates.
1503
- const cooldownSeconds = policy.accountCooldownMs
1504
- ? Math.round(policy.accountCooldownMs / 1000)
1505
- : 0;
1506
- const cooldownUntil = policy.accountCooldownMs
1507
- ? new Date(Date.now() + policy.accountCooldownMs)
1508
- : null;
1509
- const untilStr = cooldownUntil
1510
- ? ` | until=${formatCooldownUntil(cooldownUntil)}`
1511
- : "";
1512
-
1513
- try {
1514
- (err as any).quotaInfo = {
1515
- email: currentAccountEmail,
1516
- cooldownSeconds,
1517
- untilStr,
1518
- message: quotaMsg.substring(0, 150),
1519
- };
1520
- } catch {
1521
- // Best-effort metadata for logging.
1522
- }
1523
-
1524
- markAccountFailed(currentAccountId);
1525
- markAccountRateLimited(
1526
- currentAccountId,
1527
- policy.accountCooldownMs,
1528
- policy.accountCooldownReason || "QuotaExceeded",
1529
- { silent: true },
1530
- );
1531
- return { success: false, error: err };
1532
- }
1533
-
1534
- const policy = classifyRetryAction(err, {
1535
- requestAborted: params.requestSignal?.aborted === true,
1536
- });
1537
-
1538
- // The full retry decision โ€” the `โŒ Request failed` line shows the error
1539
- // but not WHY this action was chosen. Surface every field so the next
1540
- // escalation/switch/cooldown is explainable from the log alone.
1541
- if (logger.isLevelEnabled("info")) {
1542
- console.log(
1543
- `๐Ÿงญ [Chat] Retry policy | account=${currentAccountEmail} | reason=${policy.reason} | retryable=${policy.retryable} | switch=${policy.switchAccount} | newChat=${policy.forceNewChat} | fullPrompt=${policy.retryWithFullPrompt}${policy.dropFiles ? ` | dropFiles` : ""} | retryAfter=${policy.retryAfterMs}ms${policy.accountCooldownMs ? ` | cooldown=${Math.round(policy.accountCooldownMs / 1000)}s (${policy.accountCooldownReason ?? ""})` : ""}`,
1544
- );
1545
- }
1546
-
1547
- // Corrupted history means the stored parent chain is unusable. Purge the
1548
- // parent immediately so a failed recovery cannot leave the tainted thread
1549
- // bound for the next turn.
1550
- if (policy.reason === "corrupted_chat_history") {
1551
- invalidateLogicalThreadParent(params.sessionId);
1552
- }
1553
-
1554
- // A generic invalid_input is often a stale/corrupted upstream chat rather
1555
- // than an account failure. Rebuild it once on the same account first. If the
1556
- // fresh chat fails again, the normal policy is allowed to rotate.
1557
- const retryInvalidInputOnSameAccount =
1558
- shouldRetryInvalidInputOnSameAccount(
1559
- policy.reason,
1560
- invalidInputSameAccountRetried,
1561
- );
1562
- if (retryInvalidInputOnSameAccount) {
1563
- invalidInputSameAccountRetried = true;
1564
- }
1565
- const shouldSwitchAccount =
1566
- policy.switchAccount && !retryInvalidInputOnSameAccount;
1567
-
1568
- // chat_in_progress means the previous Qwen generation has not stopped
1569
- // yet (the tool loop fires the next turn the instant the previous one
1570
- // completes; the upstream chat stays "in progress" for a few seconds
1571
- // after the terminal event โ€” usually 2-4s, measured >6s after a 491KB
1572
- // turn). Policy design (settle-aware, upstream-aligned):
1573
- // 1. Retry the SAME chat with JITTERED busyMs-based waits โ€” never a
1574
- // fixed ladder (concurrent sessions would retry in lock-step).
1575
- // 2. After the settle budget (CHAT_IN_PROGRESS_MAX_RETRIES, ~35s):
1576
- // ONE bounded escalation. A chat can be "in progress" for MINUTES
1577
- // when a superseded generation keeps running server-side โ€” retrying
1578
- // the same chat then fails every request until it frees (observed:
1579
- // 2.1MB turn held a chat busy ~9min). The single escalation opens a
1580
- // FRESH chat with the full context so the turn makes progress; it
1581
- // fires at most once per request (a second replay would only repeat
1582
- // the ~1MB re-upload cost the settle design removes).
1583
- // 3. If the escalated attempt ALSO fails with chat_in_progress, the
1584
- // request FAILS and the origin binding is cleared (next client turn
1585
- // starts fresh with a replay instead of wedging on the stuck chat).
1586
- if (policy.reason === "chat_in_progress") {
1587
- if (chatInProgressOriginAccountId === null) {
1588
- // First chat_in_progress of this request: remember the account
1589
- // whose chat is stuck (used by the loop-exit session handling).
1590
- chatInProgressOriginAccountId = currentAccountId;
1591
- chatInProgressOriginAccountEmail = currentAccountEmail;
1592
- }
1593
- chatInProgressCount++;
1594
- markAccountTemporarilyBusy(
1595
- currentAccountId,
1596
- config.retry.chatInProgressBusyMs,
1597
- );
1598
-
1599
- if (chatInProgressCount > config.retry.chatInProgressMaxAttempts) {
1600
- if (!chatInProgressEscalated) {
1601
- // Same-chat settle budget exhausted: the chat is genuinely
1602
- // busy, not settling. One bounded escalation โ€” a fresh chat
1603
- // with the full context (the only way to progress while the
1604
- // old chat runs on server-side). Bounded: this fires at most
1605
- // once per request.
1606
- chatInProgressEscalated = true;
1607
- console.warn(
1608
- `๐Ÿ”„ [Chat] chat_in_progress escalation (${chatInProgressCount}) | forcing a new chat with full context on ${currentAccountEmail}`,
1609
- );
1610
- if (params.useThreadNative) {
1611
- params.existingThread = null;
1612
- params.finalPrompt = params.fullPrompt;
1613
- params.messageCount =
1614
- params.fullMessageCount ?? params.messageCount;
1615
- params.forceNewChat = true;
1616
- }
1617
- // The escalation targets a FRESH chat (not the busy one), so
1618
- // no settle wait โ€” it gets its own attempt budget.
1619
- attemptsLeft = Math.max(attemptsLeft, 1);
1620
- policy.retryAfterMs = 0;
1621
- } else {
1622
- // The escalated fresh chat ALSO failed with chat_in_progress
1623
- // (bizarre, but possible on a wedged account). Give up โ€” the
1624
- // outer rotation treats this as terminal.
1625
- attemptsLeft = 0;
1626
- policy.retryable = false;
1627
- }
1628
- } else {
1629
- // The settle window has its own budget, independent of the
1630
- // global RETRY_MAX_ATTEMPTS: with maxAttempts=3 the counter
1631
- // above would hit 0 on the 3rd failure and skip the longer
1632
- // waits that absorb the >6s settles of huge turns (2026-08-11).
1633
- attemptsLeft = Math.max(attemptsLeft, 1);
1634
-
1635
- // The 1st failure keeps the upstream-suggested wait (~1.2s);
1636
- // later retries wait a jittered context-scaled busyMs-based window.
1637
- if (chatInProgressCount >= 2) {
1638
- const promptChars =
1639
- params.fullPrompt?.length ??
1640
- params.finalPrompt?.length ??
1641
- 0;
1642
- policy.retryAfterMs = jitterChatInProgressDelay(
1643
- chatInProgressCount,
1644
- config.retry.chatInProgressBusyMs,
1645
- Math.random,
1646
- promptChars,
1647
- );
1648
- }
1649
- }
1650
- }
1651
-
1652
- if (policy.reason === "account_initialization_failed") {
1653
- console.warn(
1654
- `โš ๏ธ [Chat] Account initialization failed | ${currentAccountEmail} | cooldown=${Math.round((policy.accountCooldownMs ?? 0) / 1000)}s`,
1655
- );
1656
- markAccountFailed(currentAccountId);
1657
- markAccountRateLimited(
1658
- currentAccountId,
1659
- policy.accountCooldownMs,
1660
- policy.accountCooldownReason,
1661
- );
1662
- return { success: false, error: err };
1663
- }
1664
-
1665
- // Prefer switching account for any retryable upstream error when possible.
1666
- // A PARALLEL escape hops to a FREE account (skip busy/temporarily-busy):
1667
- // the auxiliary request must land on an available slot fast, never on a
1668
- // second occupied account (the 2026-08-20 stall rotated ldyjlโ†’cgnx3, both
1669
- // busy, ~14s lease wait). Normal requests keep the cooldown-only picker so
1670
- // single-account/saturated pools stay lossless.
1671
- if (
1672
- policy.retryable &&
1673
- shouldSwitchAccount &&
1674
- !isSingleAccount &&
1675
- accountSwitches < maxAccountSwitches
1676
- ) {
1677
- const nextAccount = params.parallelEscape
1678
- ? getNextFreeAccountForParallel(accounts, triedAccounts, currentAccountId)
1679
- : getNextAvailableAccount(triedAccounts);
1680
- if (nextAccount && nextAccount.id !== currentAccountId) {
1681
- console.warn(
1682
- `๐Ÿ”„ [Chat] Switching account after ${policy.reason} | ${currentAccountEmail} -> ${maskEmail(nextAccount.email)}`,
1683
- );
1684
- if (policy.accountCooldownMs || policy.accountCooldownReason) {
1685
- markAccountRateLimited(
1686
- currentAccountId,
1687
- policy.accountCooldownMs,
1688
- policy.accountCooldownReason || "RetrySwitch",
1689
- );
1690
- }
1691
- triedAccounts.add(currentAccountId);
1692
- currentAccountId = nextAccount.id;
1693
- currentAccountEmail = maskEmail(nextAccount.email);
1694
- accountSwitches++;
1695
-
1696
- // Account switch always rebuilds a fresh upstream chat with full history.
1697
- // Do NOT persist sticky binding until create succeeds โ€” premature empty
1698
- // chatSessionId writes make subsequent turns rotate/lose context.
1699
- if (params.useThreadNative) {
1700
- params.existingThread = null;
1701
- params.finalPrompt = params.fullPrompt;
1702
- params.messageCount = params.fullMessageCount ?? params.messageCount;
1703
- params.forceNewChat = true;
1704
- }
1705
-
1706
- await new Promise((resolve) =>
1707
- setTimeout(
1708
- resolve,
1709
- Math.min(policy.retryAfterMs ?? config.retry.baseDelayMs, 1000),
1710
- ),
1711
- );
1712
- continue;
1713
- }
1714
-
1715
- console.warn(
1716
- `โš ๏ธ [Chat] No other account available after ${policy.reason} | Retrying on same account`,
1717
- );
1718
- }
1719
-
1720
- // Force new chat / full context when policy requests it (invalid_input, chat gone, etc.)
1721
- if (
1722
- policy.retryable &&
1723
- (policy.forceNewChat || policy.retryWithFullPrompt) &&
1724
- params.useThreadNative
1725
- ) {
1726
- console.warn(
1727
- `๐Ÿ”„ [Chat] Forcing new chat/full context | reason=${policy.reason}`,
1728
- );
1729
- params.existingThread = null;
1730
- params.finalPrompt = params.fullPrompt;
1731
- params.messageCount = params.fullMessageCount ?? params.messageCount;
1732
- params.forceNewChat = true;
1733
- }
1734
-
1735
- // Drop files on retry for invalid_input to isolate file-related errors
1736
- if (policy.dropFiles && params.allFiles.length > 0) {
1737
- console.warn(
1738
- `๐Ÿ—‚๏ธ [Chat] Dropping ${params.allFiles.length} file(s) on retry to isolate invalid_input error:`,
1739
- params.allFiles.map((f) => ({
1740
- name: f.name,
1741
- type: f.type,
1742
- size: f.size ?? "unknown",
1743
- })),
1744
- );
1745
- params.allFiles = [];
1746
- }
1747
-
1748
- if (!policy.retryable || attemptsLeft <= 0) {
1749
- if (policy.accountCooldownMs || policy.accountCooldownReason) {
1750
- markAccountRateLimited(
1751
- currentAccountId,
1752
- policy.accountCooldownMs,
1753
- policy.accountCooldownReason || "RetryExhausted",
1754
- );
1755
- }
1756
-
1757
- if (
1758
- err instanceof RetryableQwenStreamError ||
1759
- isChatInProgressError(err)
1760
- ) {
1761
- // Chat_in_progress give-up only happens AFTER the bounded escalation
1762
- // (the settle window alone cannot exhaust the budget โ€” the first
1763
- // over-budget failure escalates). The escalated chat was freshly
1764
- // created; the STORED binding still points at the stuck chat, so
1765
- // clearing the origin account's sessions frees the next turn to
1766
- // start fresh instead of re-wedging on the stuck chat. For other
1767
- // retryable upstream errors (network/quota) the same clear drops a
1768
- // binding that may point at a genuinely stuck chat. Clear the ORIGIN
1769
- // account โ€” never the current one (other policies may have switched).
1770
- const clearTargetId = chatInProgressOriginAccountId ?? currentAccountId;
1771
- const clearTargetEmail =
1772
- chatInProgressOriginAccountEmail ?? currentAccountEmail;
1773
- console.warn(
1774
- `๐Ÿงน [Chat] Clearing session state for ${clearTargetEmail} (${clearTargetId}) after exhausted retries`,
1775
- );
1776
- clearAllSessionsForAccount(clearTargetId);
1777
- }
1778
-
1779
- return { success: false, error: err };
1780
- }
1781
-
1782
- const useDelay = Math.max(
1783
- 0,
1784
- policy.retryAfterMs ?? retryDelay ?? config.retry.baseDelayMs,
1785
- );
1786
-
1787
- if (policy.reason === "chat_in_progress") {
1788
- const promptChars =
1789
- params.fullPrompt?.length ??
1790
- params.finalPrompt?.length ??
1791
- 0;
1792
- const contextLabel =
1793
- promptChars > 1_000_000
1794
- ? `${(promptChars / (1024 * 1024)).toFixed(1)}MB context`
1795
- : promptChars > 100_000
1796
- ? `${Math.round(promptChars / 1024)}KB context`
1797
- : "";
1798
- const contextSuffix = contextLabel ? ` | ${contextLabel}` : "";
1799
- console.warn(
1800
- `โณ [Chat] Chat settling | ${failedAccountEmail}${contextSuffix} | waiting ${(useDelay / 1000).toFixed(1)}s (attempt ${chatInProgressCount}/${config.retry.chatInProgressMaxAttempts})...`,
1801
- );
1802
- } else {
1803
- console.warn(
1804
- `๐Ÿ”„ [Chat] Qwen request failed for ${failedAccountEmail}, retrying in ${useDelay}ms... (${attemptsLeft} left). reason=${policy.reason} error=${errMsg.slice(0, 200)}`,
1805
- );
1806
- }
1807
- await new Promise((r) => setTimeout(r, useDelay));
1808
- retryDelay = Math.min(retryDelay * 2, config.retry.maxDelayMs);
1809
- }
1810
-
1811
- return {
1812
- success: false,
1813
- error:
1814
- lastAttemptError ??
1815
- new Error("Qwen stream retry attempts were exhausted"),
1816
- };
1817
- }
1
+ import { v4 as uuidv4 } from "uuid";
2
+ import {
3
+ getAccountCooldownInfo,
4
+ getNextAccount,
5
+ getNextAvailableAccount,
6
+ markAccountRateLimited,
7
+ syncCooldownsFromDb,
8
+ } from "../../core/account-manager.ts";
9
+ import { markAccountSuccessful, markAccountFailed, getAccountsByPriority } from "../../core/account-priority.ts";
10
+ import { recordWafHardBlock, noteWafRecovery } from "../../core/waf-isolation.ts";
11
+ import { loadAccounts, type QwenAccount } from "../../core/accounts.ts";
12
+ import { config, type ChatMode } from "../../core/config.ts";
13
+ import { ClientAbortedError, UpstreamRateLimit, ValidationError } from "../../core/errors.ts";
14
+ import {
15
+ assertPromptWithinLimits,
16
+ truncatePromptToIntelligentLimit,
17
+ } from "../../core/prompt-limits.ts";
18
+ import {
19
+ formatCooldownUntil,
20
+ isToolcallDebugEnabled,
21
+ logger,
22
+ maskEmail,
23
+ } from "../../core/logger.ts";
24
+ import { Mutex } from "../../core/mutex.ts";
25
+ import { registerStream, removeStream } from "../../core/stream-registry.ts";
26
+ import {
27
+ abortLeaseByLabel,
28
+ acquireAccountLease,
29
+ isAccountBusy,
30
+ isAccountSlotHeldByOtherSession,
31
+ isAccountTemporarilyBusy,
32
+ markAccountTemporarilyBusy,
33
+ markLeaseCompletion,
34
+ tryAcquireAccountLease,
35
+ type AccountLease,
36
+ } from "../../core/account-concurrency.ts";
37
+ import { isAuthMockEnabled } from "../../services/auth-playwright.ts";
38
+ import { refreshHeaders } from "../../services/playwright.ts";
39
+ import {
40
+ clearAllSessionsForAccount,
41
+ createQwenStream,
42
+ fetchQwenModels,
43
+ getQwenErrorCode,
44
+ getLogicalThreadState,
45
+ invalidateLogicalThreadParent,
46
+ type LogicalThreadEntry,
47
+ PersonalizationSyncError,
48
+ QwenSessionExpiredError,
49
+ RetryableQwenStreamError,
50
+ syncQwenRequestPersonalization,
51
+ updateLogicalThreadState,
52
+ } from "../../services/qwen.ts";
53
+ import type { TokenEstimationContext } from "../../services/token-estimation-metrics.ts";
54
+ import {
55
+ buildContextMeterSnapshot,
56
+ contextMeterLogData,
57
+ type ContextMeterMode,
58
+ } from "../../services/context-meter.ts";
59
+ import type { QwenFileEntry } from "../upload.ts";
60
+ import type { Message } from "../../utils/types.ts";
61
+ import { buildRepeatedToolCallReminder } from "../../utils/tool-call-guard.ts";
62
+ import {
63
+ classifyRetryAction,
64
+ isAntiBotError as isAntiBotPolicyError,
65
+ isAccountInitializationError,
66
+ isChatInProgressError,
67
+ isQuotaLikeError,
68
+ isTerminalLocalError,
69
+ shouldRetryInvalidInputOnSameAccount,
70
+ } from "./retry-policy.ts";
71
+
72
+ /** How many alternate accounts a single request may try after a WAF challenge. */
73
+ const MAX_ANTI_BOT_ROTATIONS = 1;
74
+
75
+ /**
76
+ * Hard deadline for the whole personalization sync. A normal sync takes ~2s;
77
+ * a stuck account page (closed context / WAF) can otherwise hold each browser
78
+ * op for 60s and keep the personalization mutex blocked for minutes.
79
+ */
80
+ const PERSONALIZATION_SYNC_DEADLINE_MS = 30_000;
81
+
82
+ /**
83
+ * Hard deadline for a single stream-acquire attempt (models sync + truncation
84
+ * + personalization + header capture + completion fetch metadata + internal
85
+ * retries). A silent hang past this (observed: 180s with zero logs) fails the
86
+ * attempt with a visible retryable error so the outer loop switches account.
87
+ * Configurable via ACQUIRE_DEADLINE_MS (default 120000).
88
+ */
89
+
90
+ // Per-chat lock: serializes requests to the same Qwen chat session
91
+ const chatLocks = new Map<string, Mutex>();
92
+ // Account-level personalization is global mutable Qwen state; keep update+stream
93
+ // creation serialized per account when the experimental request-sync mode is used.
94
+ const personalizationLocks = new Map<string, Mutex>();
95
+
96
+ export async function acquireChatLock(chatId: string): Promise<() => void> {
97
+ const acquireStartedAt = Date.now();
98
+ const timeoutMs = config.concurrency.chatLockTimeoutMs;
99
+ let mutex = chatLocks.get(chatId);
100
+ if (!mutex) {
101
+ mutex = new Mutex(
102
+ `chat:${chatId.substring(0, 8)}`,
103
+ // The chat lock is held for the whole stream lifetime. A long
104
+ // generation (reasoning + huge context) can legitimately exceed the
105
+ // global 120s hold limit; use the same budget as the acquire timeout
106
+ // so the force-release never kills a healthy mid-stream turn.
107
+ timeoutMs,
108
+ );
109
+ chatLocks.set(chatId, mutex);
110
+ }
111
+ const release = await mutex.acquire(timeoutMs, `chat:${chatId.substring(0, 12)}`);
112
+ // Held time must exclude the wait: capture right after the acquire settles,
113
+ // not at function entry (the wait is already visible as `waited Xms` above).
114
+ const heldStartedAt = Date.now();
115
+ if (logger.isLevelEnabled("info")) {
116
+ console.log(
117
+ `๐Ÿ” [Chat] Chat lock acquired | chat=${chatId.substring(0, 12)} | waited ${heldStartedAt - acquireStartedAt}ms`,
118
+ );
119
+ }
120
+ return () => {
121
+ release();
122
+ if (logger.isLevelEnabled("info")) {
123
+ console.log(
124
+ `๐Ÿ”“ [Chat] Chat lock released | chat=${chatId.substring(0, 12)} | held ${Date.now() - heldStartedAt}ms`,
125
+ );
126
+ }
127
+ if (mutex!.isIdle()) {
128
+ chatLocks.delete(chatId);
129
+ }
130
+ };
131
+ }
132
+
133
+ async function acquirePersonalizationLock(
134
+ accountId: string,
135
+ ): Promise<() => void> {
136
+ let mutex = personalizationLocks.get(accountId);
137
+ if (!mutex) {
138
+ mutex = new Mutex(`personalization:${accountId.substring(0, 8)}`);
139
+ personalizationLocks.set(accountId, mutex);
140
+ }
141
+ const release = await mutex.acquire(60_000, `personalization:${accountId.substring(0, 8)}`);
142
+ return () => {
143
+ release();
144
+ if (mutex!.isIdle()) {
145
+ personalizationLocks.delete(accountId);
146
+ }
147
+ };
148
+ }
149
+
150
+ export interface SelectedAccount {
151
+ id: string;
152
+ email: string;
153
+ password: string;
154
+ }
155
+
156
+ export interface StreamCreationResult {
157
+ stream: ReadableStream;
158
+ uiSessionId: string;
159
+ activeAccountId: string;
160
+ activeAccountLabel: string;
161
+ /** True when the request resent the FULL prompt on a new upstream chat
162
+ * (account switch / missing thread parent). The ๐Ÿ“ค log line uses this to
163
+ * show the real payload instead of the thread-native delta. */
164
+ replayedFullContext: boolean;
165
+ completionId: string;
166
+ logicalSessionId: string | null;
167
+ createdNewChat: boolean;
168
+ tokenEstimationContext: TokenEstimationContext;
169
+ releaseAccountLease: () => void;
170
+ }
171
+
172
+ export interface StreamCreationFailure {
173
+ error: any;
174
+ completionId: string;
175
+ allOnCooldown: boolean;
176
+ retryAfterMs?: number;
177
+ }
178
+
179
+ export interface AcquireParams {
180
+ finalPrompt: string;
181
+ fullPrompt: string;
182
+ isThinkingModel: boolean;
183
+ model: string;
184
+ reasoningMode?: "auto" | "thinking" | "fast";
185
+ shouldResetUpstreamThread: boolean;
186
+ allFiles: QwenFileEntry[];
187
+ isNewSession: boolean;
188
+ sessionId: string | null;
189
+ useThreadNative: boolean;
190
+ updateLogicalThread: boolean;
191
+ allowThreadReuse: boolean;
192
+ /** "thread" (reuse upstream chat) or "temp" (new ephemeral chat per request). */
193
+ chatMode: ChatMode;
194
+ /** Full message history for intelligent context truncation after model sync. */
195
+ messages?: Message[];
196
+ forceNewChat?: boolean;
197
+ /**
198
+ * Prefer this account when available.
199
+ * - undefined/omit: use sticky thread account when present, else round-robin
200
+ * - string: pin to that account if configured
201
+ * - null: explicitly rotate away from sticky/current account (error failover)
202
+ */
203
+ preferredAccountId?: string | null;
204
+ /** When rotating, exclude these account ids from the first pick. */
205
+ excludeAccountIds?: string[];
206
+ messageCount?: number;
207
+ fullMessageCount?: number;
208
+ toolsCount?: number;
209
+ requestPersonalizationInstruction?: string | null;
210
+ /** Mapped Qwen model id used for local prompt-budget validation. */
211
+ contextModelId?: string;
212
+ requestSignal?: AbortSignal;
213
+ /** Context accounting mode for this concrete upstream attempt. */
214
+ contextMode?: ContextMeterMode;
215
+ /** Allow this request to retry the account it just marked temporarily busy. */
216
+ allowTemporarilyBusyAccountId?: string;
217
+ /**
218
+ * True when this request races a same-session stream that has NOT emitted
219
+ * yet: run on its OWN chat and hop accounts fast instead of waiting.
220
+ */
221
+ parallelEscape?: boolean;
222
+ }
223
+
224
+ /** Exported for unit tests โ€” selects the first account for a request. */
225
+ export function resolveInitialAccount(
226
+ preferredAccountId?: string | null,
227
+ excludeAccountIds?: Iterable<string>,
228
+ ): {
229
+ account: SelectedAccount;
230
+ configuredAccounts: SelectedAccount[];
231
+ } {
232
+ if (isAuthMockEnabled()) {
233
+ return {
234
+ account: { id: "mock-account", email: "mock@test.com", password: "" },
235
+ configuredAccounts: [],
236
+ };
237
+ }
238
+
239
+ const configuredAccounts = loadAccounts();
240
+ if (configuredAccounts.length > 0) {
241
+ syncCooldownsFromDb(configuredAccounts);
242
+ const excluded = new Set(excludeAccountIds ?? []);
243
+
244
+ // Explicit preferred account (sticky / same-account retry)
245
+ if (typeof preferredAccountId === "string" && preferredAccountId) {
246
+ const preferred = configuredAccounts.find(
247
+ (candidate) => candidate.id === preferredAccountId,
248
+ );
249
+ if (preferred && !getAccountCooldownInfo(preferred.id)) {
250
+ return { account: preferred, configuredAccounts };
251
+ }
252
+ // Preferred is missing/on cooldown: fall through to next available.
253
+ if (preferred) excluded.add(preferred.id);
254
+ }
255
+
256
+ // Error failover: rotate away from sticky/current account when requested.
257
+ if (preferredAccountId === null || excluded.size > 0) {
258
+ const next = getNextAvailableAccount(excluded);
259
+ if (next) return { account: next, configuredAccounts };
260
+ }
261
+
262
+ const account = getNextAccount();
263
+ if (!account) {
264
+ // All accounts on cooldown; caller will handle this.
265
+ return { account: configuredAccounts[0], configuredAccounts };
266
+ }
267
+ return { account, configuredAccounts };
268
+ }
269
+
270
+ throw new ValidationError(
271
+ "Nenhuma conta Qwen configurada no servidor. Adicione uma conta na aba [5] Contas da TUI.",
272
+ );
273
+ }
274
+
275
+ function isAccountUnavailableError(err: any): boolean {
276
+ // Quota/rate-limit style failures that should cool the account and rotate.
277
+ if (isQuotaLikeError(err)) return true;
278
+ return (
279
+ (err instanceof UpstreamRateLimit &&
280
+ !(err instanceof RetryableQwenStreamError)) ||
281
+ err?.upstreamCode === "RateLimited" ||
282
+ err?.upstreamStatus === 429
283
+ );
284
+ }
285
+
286
+ function isAntiBotError(err: any): boolean {
287
+ return isAntiBotPolicyError(err);
288
+ }
289
+
290
+ function hasFreeAlternateAccount(
291
+ accounts: SelectedAccount[],
292
+ currentAccountId: string,
293
+ triedAccountIds: Set<string>,
294
+ ): boolean {
295
+ return accounts.some(
296
+ (candidate) =>
297
+ candidate.id !== currentAccountId &&
298
+ !triedAccountIds.has(candidate.id) &&
299
+ !getAccountCooldownInfo(candidate.id) &&
300
+ !isAccountTemporarilyBusy(candidate.id) &&
301
+ !isAccountBusy(candidate.id),
302
+ );
303
+ }
304
+
305
+ /**
306
+ * Pick the next account for a PARALLEL escape, preferring one with a FREE slot:
307
+ * not busy, not temporarily busy, not on cooldown, not already tried. A normal
308
+ * request keeps getNextAvailableAccount (cooldown-only picker) so saturated or
309
+ * single-account pools stay lossless โ€” but an auxiliary parallel request must
310
+ * land on an available slot fast and never queue behind a second occupied
311
+ * account (the 2026-08-20 stall rotated ldyjl->cgnx3, both busy, ~14s wait).
312
+ * Falls back to the cooldown-only picker when no FREE account is left, so a
313
+ * fully-busy pool still rotates instead of dead-ending.
314
+ */
315
+ function getNextFreeAccountForParallel(
316
+ accounts: QwenAccount[],
317
+ triedAccountIds: Set<string>,
318
+ currentAccountId: string,
319
+ ): QwenAccount | null {
320
+ const ordered = getAccountsByPriority(accounts);
321
+ const free = ordered.find(
322
+ (c) =>
323
+ c.id !== currentAccountId &&
324
+ !triedAccountIds.has(c.id) &&
325
+ !getAccountCooldownInfo(c.id) &&
326
+ !isAccountTemporarilyBusy(c.id) &&
327
+ !isAccountBusy(c.id),
328
+ );
329
+ if (free) return free;
330
+ // No free slot anywhere: fall back to the normal picker so we still rotate
331
+ // (the tryAcquireAccountLease fail-fast will report account_busy and the
332
+ // loop gives up rather than blocking on a busy pool).
333
+ return getNextAvailableAccount(triedAccountIds);
334
+ }
335
+
336
+
337
+ async function attemptRelogin(
338
+ accountId: string,
339
+ accountEmail: string,
340
+ ): Promise<boolean> {
341
+ try {
342
+ await refreshHeaders(accountId);
343
+ console.log(
344
+ `โœ… [Chat] Playwright headers refreshed for ${maskEmail(accountEmail)}. Retrying...`,
345
+ );
346
+ return true;
347
+ } catch (refreshErr: unknown) {
348
+ logger.error("[Chat] Playwright header refresh failed", {
349
+ accountEmail: maskEmail(accountEmail),
350
+ error:
351
+ refreshErr instanceof Error ? refreshErr.message : String(refreshErr),
352
+ cause:
353
+ refreshErr instanceof Error
354
+ ? refreshErr.constructor.name
355
+ : typeof refreshErr,
356
+ });
357
+ }
358
+ return false;
359
+ }
360
+
361
+ export async function acquireUpstreamStream(
362
+ params: AcquireParams,
363
+ ): Promise<StreamCreationResult | StreamCreationFailure> {
364
+ const {
365
+ finalPrompt,
366
+ isThinkingModel,
367
+ model,
368
+ reasoningMode,
369
+ shouldResetUpstreamThread,
370
+ allFiles,
371
+ isNewSession,
372
+ sessionId,
373
+ useThreadNative,
374
+ updateLogicalThread,
375
+ allowThreadReuse,
376
+ chatMode,
377
+ forceNewChat = false,
378
+ preferredAccountId,
379
+ excludeAccountIds,
380
+ } = params;
381
+
382
+ const completionId = "chatcmpl-" + uuidv4();
383
+ // Sticky thread binding is independent of forceNewChat. forceNewChat only
384
+ // means "open a fresh upstream chat", not "forget which account owned the
385
+ // logical conversation".
386
+ const threadState =
387
+ allowThreadReuse && sessionId ? getLogicalThreadState(sessionId) : null;
388
+ const stickyThreadAccountId = threadState?.accountId ?? null;
389
+ const canReuseUpstreamChat =
390
+ !!threadState &&
391
+ !forceNewChat &&
392
+ !!threadState.chatSessionId &&
393
+ threadState.chatSessionId.length > 0 &&
394
+ !!threadState.parentId;
395
+ // A thread with an upstream chat but no committed parent is dirty (failed
396
+ // first turn, interrupted generation, corrupted history). It must not be
397
+ // appended to; rebuild a fresh chat with the full prompt instead.
398
+ const threadMissingParent =
399
+ !!threadState &&
400
+ !!threadState.chatSessionId &&
401
+ threadState.chatSessionId.length > 0 &&
402
+ !threadState.parentId;
403
+ const existingThread = canReuseUpstreamChat ? threadState : null;
404
+
405
+ // preferredAccountId:
406
+ // - string: pin to account
407
+ // - null: explicit failover away from sticky (error path)
408
+ // - undefined: keep sticky when available
409
+ // A PARALLEL escape must NOT pin to the sticky thread owner: it races the
410
+ // main generation that is likely using that very account, so targeting the
411
+ // sticky would just fail-fast account_busy and waste a rotation hop (the
412
+ // 2026-08-20 02:43:41 stall: parallel req chose the sticky busy account, then
413
+ // a second busy one, ~18s until the client aborted). Rotate to any account
414
+ // so the first hop has a real chance of landing on a free slot.
415
+ const effectivePreferred = params.parallelEscape ? null : preferredAccountId;
416
+ const resolvedPreferred =
417
+ effectivePreferred === null
418
+ ? null
419
+ : (effectivePreferred ?? stickyThreadAccountId ?? undefined);
420
+ const excludeSet = new Set(excludeAccountIds ?? []);
421
+ // When rotating away (resolvedPreferred === null) โ€” either an explicit
422
+ // failover OR a parallel escape โ€” exclude the sticky owner so the rotation
423
+ // can never land back on the account the main generation is using.
424
+ if (resolvedPreferred === null && stickyThreadAccountId) {
425
+ excludeSet.add(stickyThreadAccountId);
426
+ }
427
+
428
+ const resolved = resolveInitialAccount(resolvedPreferred, excludeSet);
429
+
430
+ if (logger.isLevelEnabled("info")) {
431
+ // Why THIS account? The operator needs the decision, not just the
432
+ // result โ€” the previous rounds' "stale label" / "switching" confusion
433
+ // came from logs that showed only the outcome.
434
+ const poolSize = resolved.configuredAccounts.length;
435
+ const cooldownCount = resolved.configuredAccounts.filter((a) =>
436
+ getAccountCooldownInfo(a.id),
437
+ ).length;
438
+ const why =
439
+ resolved.account.id === stickyThreadAccountId
440
+ ? "sticky"
441
+ : typeof resolvedPreferred === "string" &&
442
+ resolved.account.id === resolvedPreferred
443
+ ? "preferred"
444
+ : resolvedPreferred === null
445
+ ? "failover-rotate"
446
+ : "round-robin";
447
+ console.log(
448
+ `๐ŸŽฏ [Chat] Account selected | ${maskEmail(resolved.account.email)} (${resolved.account.id}) | reason=${why} | pool=${poolSize}${cooldownCount ? ` | cooldown=${cooldownCount}` : ""}${stickyThreadAccountId ? ` | sticky=${stickyThreadAccountId === resolved.account.id}` : ""}`,
449
+ );
450
+ }
451
+
452
+ let account: SelectedAccount | null = resolved.account;
453
+ const configuredAccounts = resolved.configuredAccounts;
454
+ const triedAccountIds = new Set<string>();
455
+ let lastError: any = null;
456
+ let antiBotRotations = 0;
457
+
458
+ while (account) {
459
+ const accountId = account.id;
460
+ const accountEmail = maskEmail(account.email);
461
+
462
+ if (triedAccountIds.has(accountId)) {
463
+ account = getNextAvailableAccount(triedAccountIds);
464
+ continue;
465
+ }
466
+ triedAccountIds.add(accountId);
467
+
468
+ // Skip accounts that recently returned chat_in_progress (temporary busy) โ€”
469
+ // except the sticky thread owner: hopping the owner splinters the
470
+ // conversation and replays the full context on a cold account (~12s
471
+ // reopen + captcha) when the upstream chat is merely settling (2-4s,
472
+ // covered by the same-chat settle retries). Mirrors the saturated-account
473
+ // exception below.
474
+ if (
475
+ isAccountTemporarilyBusy(accountId) &&
476
+ params.allowTemporarilyBusyAccountId !== accountId &&
477
+ accountId !== stickyThreadAccountId
478
+ ) {
479
+ console.log(
480
+ `โญ๏ธ [Chat] Skipping account ${accountEmail} (${accountId}) temporarily busy (chat in progress)`,
481
+ );
482
+ account = getNextAvailableAccount(triedAccountIds);
483
+ continue;
484
+ }
485
+
486
+ // Do not wait 30 seconds on a saturated account when another account is
487
+ // already free. Keep the queue behavior only when this is the last usable
488
+ // account, so single-account deployments remain lossless.
489
+ // The thread owner is excluded: rotating the sticky account during a
490
+ // tool/think pause splinters the conversation across upstream chats, so
491
+ // it must queue on its own slot instead of being skipped.
492
+ if (
493
+ isAccountBusy(accountId) &&
494
+ accountId !== stickyThreadAccountId &&
495
+ hasFreeAlternateAccount(configuredAccounts, accountId, triedAccountIds)
496
+ ) {
497
+ console.log(
498
+ `โญ๏ธ [Chat] Skipping account ${accountEmail} (${accountId}) busy; rotating to a free account`,
499
+ );
500
+ const nextCandidate = getNextAvailableAccount(triedAccountIds);
501
+ if (nextCandidate && !getAccountCooldownInfo(nextCandidate.id)) {
502
+ account = nextCandidate;
503
+ continue;
504
+ }
505
+ // No usable alternate account exists (e.g. all other accounts are on cooldown):
506
+ // do NOT skip this busy account to death. Fall through and queue on its slot.
507
+ }
508
+
509
+ const cooldownInfo = getAccountCooldownInfo(accountId);
510
+ if (cooldownInfo) {
511
+ console.log(
512
+ `โญ๏ธ [Chat] Skipping account ${accountEmail} (${accountId}) on cooldown for ${Math.round(cooldownInfo.remainingMs / 1000)}s (${cooldownInfo.reason})`,
513
+ );
514
+ if (stickyThreadAccountId === accountId) {
515
+ console.warn(
516
+ `โš ๏ธ [Chat] Sticky account is on cooldown; recreating upstream chat on another account with full context.`,
517
+ );
518
+ }
519
+ account = getNextAvailableAccount(triedAccountIds);
520
+ continue;
521
+ }
522
+
523
+ if (isToolcallDebugEnabled()) {
524
+ logger.debug("[chat] account selected", {
525
+ accountId,
526
+ accountEmail,
527
+ isNewSession,
528
+ isThinkingModel,
529
+ promptLength: finalPrompt.length,
530
+ });
531
+ }
532
+
533
+ if (useThreadNative && logger && process.env.CHAT_REQUEST_LOG === "true") {
534
+ logger.info("[chat] thread-native routing", {
535
+ sessionId,
536
+ accountId,
537
+ stickyAccountId: stickyThreadAccountId,
538
+ hasExistingThread: !!existingThread,
539
+ existingChatSessionId: existingThread?.chatSessionId || null,
540
+ existingParentId: existingThread?.parentId || null,
541
+ instructionsSent: existingThread?.instructionsSent || false,
542
+ allowThreadReuse,
543
+ forceNewChat,
544
+ hasExplicitConversationKey: params.allowThreadReuse,
545
+ });
546
+ }
547
+
548
+ try {
549
+ // Any account change vs the sticky owner must resend full history into a
550
+ // brand-new upstream chat โ€” the previous account's parent chain is unusable.
551
+ // Same-account forceNewChat keeps the caller's finalPrompt (may already be
552
+ // a rollover summary or a full-history rebuild from the retry layer).
553
+ const recreatingOnNewAccount =
554
+ !!stickyThreadAccountId && accountId !== stickyThreadAccountId;
555
+ const mustReplayFullContext =
556
+ recreatingOnNewAccount || threadMissingParent;
557
+ const attemptForceNewChat = forceNewChat || mustReplayFullContext;
558
+ const attemptFinalPrompt = mustReplayFullContext
559
+ ? params.fullPrompt
560
+ : finalPrompt;
561
+ // The thread owner (or a deployment where no alternate account is
562
+ // free) must queue on its own slot until generation finishes. A hard
563
+ // 30s busy timeout here would needlessly 500 the same conversation
564
+ // while the model is paused mid-tool/think.
565
+ const waitForSlot =
566
+ (!!stickyThreadAccountId && accountId === stickyThreadAccountId) ||
567
+ !hasFreeAlternateAccount(
568
+ configuredAccounts,
569
+ accountId,
570
+ triedAccountIds,
571
+ );
572
+ const result = await tryCreateStreamWithRetry(
573
+ {
574
+ finalPrompt: attemptFinalPrompt,
575
+ isThinkingModel,
576
+ model,
577
+ reasoningMode,
578
+ shouldResetUpstreamThread,
579
+ allFiles,
580
+ sessionId,
581
+ useThreadNative,
582
+ updateLogicalThread,
583
+ forceNewChat: attemptForceNewChat,
584
+ existingThread:
585
+ !mustReplayFullContext &&
586
+ existingThread &&
587
+ existingThread.accountId === accountId
588
+ ? existingThread
589
+ : null,
590
+ messageCount: mustReplayFullContext
591
+ ? (params.fullMessageCount ?? params.messageCount)
592
+ : params.messageCount,
593
+ fullMessageCount: params.fullMessageCount,
594
+ toolsCount: params.toolsCount,
595
+ requestPersonalizationInstruction:
596
+ params.requestPersonalizationInstruction,
597
+ contextModelId: params.contextModelId,
598
+ fullPrompt: params.fullPrompt,
599
+ contextMode: mustReplayFullContext
600
+ ? "replay"
601
+ : params.contextMode,
602
+ requestSignal: params.requestSignal,
603
+ queueSlotUntilFree: waitForSlot,
604
+ messages: params.messages,
605
+ completionId,
606
+ parallelEscape: params.parallelEscape,
607
+ chatMode,
608
+ },
609
+ accountId,
610
+ accountEmail,
611
+ );
612
+
613
+ if (result.success) {
614
+ registerStream(completionId, {
615
+ abortController: result.controller,
616
+ accountId: result.accountId,
617
+ uiSessionId: result.uiSessionId,
618
+ targetResponseId: "",
619
+ headers: result.headers,
620
+ });
621
+
622
+ return {
623
+ stream: result.stream,
624
+ uiSessionId: result.uiSessionId,
625
+ activeAccountId: result.accountId,
626
+ activeAccountLabel: result.accountEmail,
627
+ replayedFullContext: mustReplayFullContext,
628
+ completionId,
629
+ logicalSessionId:
630
+ useThreadNative && updateLogicalThread ? sessionId : null,
631
+ createdNewChat: result.createdNewChat,
632
+ tokenEstimationContext: {
633
+ ...result.tokenEstimationContext,
634
+ requestDeclaredToolCount: params.toolsCount ?? 0,
635
+ },
636
+ releaseAccountLease: result.releaseAccountLease,
637
+ };
638
+ }
639
+
640
+ lastError = result.error;
641
+ } catch (err: any) {
642
+ lastError = err;
643
+ }
644
+
645
+ // The request signal is shared by every account attempt. Once the client
646
+ // disconnects, stop the outer rotation loop as well as inner retries.
647
+ if (params.requestSignal?.aborted) {
648
+ break;
649
+ }
650
+
651
+ // Client/proxy validation errors must not be retried on other accounts.
652
+ // In particular, an oversized prompt is independent of the selected
653
+ // account; rotating accounts only repeats the same 400 response and can
654
+ // also rebuild the full history several times.
655
+ if (isTerminalLocalError(lastError)) {
656
+ break;
657
+ }
658
+
659
+ const quotaInfo = (lastError as any)?.quotaInfo as
660
+ | {
661
+ email: string;
662
+ cooldownSeconds: number;
663
+ untilStr: string;
664
+ message: string;
665
+ }
666
+ | undefined;
667
+ if (quotaInfo) {
668
+ const stickyRotation =
669
+ stickyThreadAccountId === accountId &&
670
+ (isAccountUnavailableError(lastError) ||
671
+ isAccountInitializationError(lastError) ||
672
+ isChatInProgressError(lastError));
673
+ console.warn(
674
+ `โš ๏ธ [Chat] Quota exceeded | ${quotaInfo.email} | cooldown=${quotaInfo.cooldownSeconds}s${quotaInfo.untilStr} | ${quotaInfo.message}${stickyRotation ? " | switching sticky account with full context" : ""}`,
675
+ );
676
+ }
677
+
678
+ // chat_in_progress exhaustion is TERMINAL: tryCreateStreamWithRetry already
679
+ // spent the full same-chat settle budget AND its single bounded escalation
680
+ // (fresh chat + full replay). Reaching this point means even the escalated
681
+ // fresh chat failed โ€” the inner loop already cleared the origin binding, so
682
+ // the client's next turn starts fresh. Rotating from here would only add a
683
+ // SECOND full-context replay on a cold account for no gain.
684
+ if (isChatInProgressError(lastError)) {
685
+ if (logger.isLevelEnabled("info")) {
686
+ console.log(
687
+ `๐Ÿ›‘ [Chat] chat_in_progress budget exhausted (post-escalation) | ${maskEmail(accountEmail)} | failing without account rotation`,
688
+ );
689
+ }
690
+ break;
691
+ }
692
+
693
+ if (stickyThreadAccountId === accountId) {
694
+ // A challenged sticky account must be allowed to fall through to the
695
+ // anti-bot handling below; otherwise the whole conversation dies on the
696
+ // account the WAF happened to pick.
697
+ const stickyAccountMustRotate =
698
+ isAccountUnavailableError(lastError) ||
699
+ isAccountInitializationError(lastError) ||
700
+ isAntiBotError(lastError);
701
+ if (stickyAccountMustRotate) {
702
+ if (!quotaInfo) {
703
+ console.warn(
704
+ `โš ๏ธ [Chat] Sticky account unavailable (${isAntiBotError(lastError) ? "waf_challenge" : "upstream failure"}); trying another account with full context.`,
705
+ );
706
+ }
707
+ } else {
708
+ break;
709
+ }
710
+ }
711
+
712
+ // The inner retry loop already replayed this account and tried to clear the
713
+ // challenge. Hand the request to one other account rather than failing it
714
+ // outright, then stop: walking the whole pool would only get every account
715
+ // challenged in turn and multiply the solver budget by the pool size.
716
+ if (isAntiBotError(lastError)) {
717
+ // Hard block: the challenge was NOT solved here. Quarantine with an
718
+ // escalating window AND rotate the device fingerprint + reset the
719
+ // browser context, so the account does not return from cooldown on the
720
+ // same identity the WAF already flagged (upstream account-isolation).
721
+ recordWafHardBlock(accountId);
722
+
723
+ if (antiBotRotations >= MAX_ANTI_BOT_ROTATIONS) {
724
+ console.warn(
725
+ `โš ๏ธ [Chat] WAF challenge retries exhausted | ${accountEmail} | no further rotation`,
726
+ );
727
+ break;
728
+ }
729
+
730
+ const nextAfterChallenge = getNextAvailableAccount(triedAccountIds);
731
+ if (!nextAfterChallenge) {
732
+ console.warn(
733
+ `โš ๏ธ [Chat] WAF challenge retries exhausted | ${accountEmail} | no other account available`,
734
+ );
735
+ break;
736
+ }
737
+
738
+ antiBotRotations++;
739
+ console.warn(
740
+ `๐Ÿ”„ [Chat] WAF challenge on ${accountEmail}; retrying on ${maskEmail(nextAfterChallenge.email)}`,
741
+ );
742
+ account = nextAfterChallenge;
743
+ continue;
744
+ }
745
+
746
+ if (isToolcallDebugEnabled()) {
747
+ logger.debug("[chat] account failed, rotating", {
748
+ accountId,
749
+ accountEmail: maskEmail(accountEmail),
750
+ triedAccounts: Array.from(triedAccountIds),
751
+ });
752
+ }
753
+
754
+ account = getNextAvailableAccount(triedAccountIds);
755
+ }
756
+
757
+ // All accounts exhausted.
758
+ removeStream(completionId);
759
+
760
+ if (!lastError && configuredAccounts.length > 0) {
761
+ const cooldownInfos = configuredAccounts
762
+ .map((acc) => getAccountCooldownInfo(acc.id))
763
+ .filter(
764
+ (
765
+ info,
766
+ ): info is NonNullable<ReturnType<typeof getAccountCooldownInfo>> =>
767
+ info !== null,
768
+ );
769
+
770
+ if (cooldownInfos.length === configuredAccounts.length) {
771
+ const retryAfterMs = Math.min(
772
+ ...cooldownInfos.map((info) => info.remainingMs),
773
+ );
774
+ const cooldownError: any = new Error(
775
+ `All configured accounts are on cooldown. Retry in about ${Math.max(1, Math.ceil(retryAfterMs / 1000))}s.`,
776
+ );
777
+ cooldownError.upstreamStatus = 429;
778
+ cooldownError.retryAfterMs = retryAfterMs;
779
+ return {
780
+ error: cooldownError,
781
+ completionId,
782
+ allOnCooldown: true,
783
+ retryAfterMs,
784
+ };
785
+ }
786
+ }
787
+
788
+ if (!lastError) {
789
+ const busyOrCooldownError: any = new Error(
790
+ "No accounts available: all accounts are either in use or on cooldown. Retry shortly.",
791
+ );
792
+ busyOrCooldownError.upstreamStatus = 429;
793
+ return {
794
+ error: busyOrCooldownError,
795
+ completionId,
796
+ allOnCooldown: false,
797
+ };
798
+ }
799
+
800
+ return {
801
+ error: lastError,
802
+ completionId,
803
+ allOnCooldown: false,
804
+ };
805
+ }
806
+
807
+ interface CreateStreamSuccess {
808
+ success: true;
809
+ stream: ReadableStream;
810
+ uiSessionId: string;
811
+ accountId: string;
812
+ /** Account email that actually served the request (inner rotation may
813
+ * switch accounts โ€” parallel escape / other retry policies). */
814
+ accountEmail: string;
815
+ controller: AbortController;
816
+ headers: Record<string, string>;
817
+ createdNewChat: boolean;
818
+ tokenEstimationContext: TokenEstimationContext;
819
+ releaseAccountLease: () => void;
820
+ }
821
+
822
+ interface CreateStreamFailure {
823
+ success: false;
824
+ error: any;
825
+ }
826
+
827
+ /**
828
+ * Pure: jittered same-chat settle wait for the nth chat_in_progress retry.
829
+ * The base grows with the failure count (busyMs โ†’ 2ร—busyMs from the 4th
830
+ * retry) and context size (promptChars > 500KB/1MB/2MB), randomized within ยฑ25%
831
+ * so concurrent sessions sharing an account never retry in lock-step.
832
+ */
833
+ export function jitterChatInProgressDelay(
834
+ retryCount: number,
835
+ busyMs: number,
836
+ rand: () => number = Math.random,
837
+ promptChars: number = 0,
838
+ ): number {
839
+ let base = retryCount >= 4 ? busyMs * 2 : busyMs;
840
+ if (promptChars > 2_000_000) {
841
+ base = Math.round(base * 1.75);
842
+ } else if (promptChars > 1_000_000) {
843
+ base = Math.round(base * 1.5);
844
+ } else if (promptChars > 500_000) {
845
+ base = Math.round(base * 1.25);
846
+ }
847
+ const raw = base * (0.75 + 0.5 * Math.min(1, Math.max(0, rand())));
848
+ return Math.min(20_000, Math.max(1, Math.round(raw)));
849
+ }
850
+
851
+ /**
852
+ * Decision helper for the per-account lease queue deadline.
853
+ *
854
+ * `true` means the request may wait up to `queueWaitForeverCapMs` for the
855
+ * slot (thread owner waiting on its OWN session, or the last usable account).
856
+ * `false` means it waits at most `busyWaitMs` and then fails with
857
+ * `account_busy` so the attempt loop can rotate accounts.
858
+ *
859
+ * The thread-owner preference is NOT enough on its own: when another session
860
+ * holds the slot, waiting long is pure latency (the other session may keep
861
+ * generating for minutes). Only the same-session holder justifies the long
862
+ * wait (same-session latest-wins / tool loop).
863
+ */
864
+ export function shouldWaitQueueForever(
865
+ isThreadOwnerWaiting: boolean,
866
+ heldByOtherSession: boolean,
867
+ hasFreeAlternate: boolean,
868
+ ): boolean {
869
+ return (isThreadOwnerWaiting && !heldByOtherSession) || !hasFreeAlternate;
870
+ }
871
+
872
+ async function tryCreateStreamWithRetry(
873
+ params: {
874
+ finalPrompt: string;
875
+ fullPrompt: string;
876
+ isThinkingModel: boolean;
877
+ model: string;
878
+ reasoningMode?: "auto" | "thinking" | "fast";
879
+ shouldResetUpstreamThread: boolean;
880
+ allFiles: QwenFileEntry[];
881
+ sessionId: string | null;
882
+ useThreadNative: boolean;
883
+ updateLogicalThread: boolean;
884
+ forceNewChat: boolean;
885
+ existingThread: LogicalThreadEntry | null;
886
+ messageCount?: number;
887
+ fullMessageCount?: number;
888
+ toolsCount?: number;
889
+ requestPersonalizationInstruction?: string | null;
890
+ contextModelId?: string;
891
+ contextMode?: ContextMeterMode;
892
+ requestSignal?: AbortSignal;
893
+ queueSlotUntilFree?: boolean;
894
+ messages?: Message[];
895
+ /** Stream registry key; the emit-aware supersede links the lease to it. */
896
+ completionId: string;
897
+ /**
898
+ * True when this request races a same-session stream that has NOT
899
+ * emitted yet (title/parallel request): run on its OWN chat instead of
900
+ * waiting on the main chat's lock, and hop accounts fast (tryAcquire).
901
+ */
902
+ parallelEscape?: boolean;
903
+ /** "thread" (reuse upstream chat) or "temp" (new ephemeral chat per request). */
904
+ chatMode: ChatMode;
905
+ },
906
+ accountId: string,
907
+ accountEmail: string,
908
+ ): Promise<CreateStreamSuccess | CreateStreamFailure> {
909
+ const maxAttempts = Math.max(1, config.retry.maxAttempts);
910
+ const maxAccountSwitches = Math.max(0, config.retry.maxAccountSwitches);
911
+ let attemptsLeft = maxAttempts;
912
+ let retryDelay = config.retry.baseDelayMs;
913
+ let attempt = 0;
914
+ let quotaRetried = false;
915
+ let accountSwitches = 0;
916
+ let chatInProgressCount = 0;
917
+ let chatInProgressEscalated = false;
918
+ // Account that accumulated the chat_in_progress failures (the one whose
919
+ // upstream chat is actually stuck "in progress"). The post-budget
920
+ // escalation switches to a FRESH chat, and the loop-exit session clear
921
+ // must drop the binding to the stuck chat โ€” NOT clear the sessions of an
922
+ // account that never served this session (cross-session damage).
923
+ let chatInProgressOriginAccountId: string | null = null;
924
+ let chatInProgressOriginAccountEmail: string | null = null;
925
+ let lastAttemptError: any = null;
926
+ let invalidInputSameAccountRetried = false;
927
+ const accounts = loadAccounts();
928
+ const isSingleAccount = accounts.length <= 1;
929
+ let currentAccountId = accountId;
930
+ let currentAccountEmail = accountEmail;
931
+ const triedAccounts = new Set<string>([accountId]);
932
+
933
+ while (attemptsLeft > 0) {
934
+ attempt++;
935
+ if (attempt > 1) {
936
+ console.log(
937
+ `๐Ÿ”„ [Chat] Retrying request | ${currentAccountEmail} | ${params.model} | ${params.messageCount ?? "?"} msg(s) | ${params.finalPrompt.length} chars${params.toolsCount ? ` | ${params.toolsCount} tool(s)` : ""} | attempt ${attempt}`,
938
+ );
939
+ }
940
+ let attemptError: any = null;
941
+ let accountLease: AccountLease | null = null;
942
+ const acquireStartedAt = Date.now();
943
+
944
+ try {
945
+ // The client may have cancelled between account selection and this
946
+ // attempt (or while a previous attempt was running). Bail before
947
+ // spending time on model sync, truncation or personalization.
948
+ if (params.requestSignal?.aborted) {
949
+ return {
950
+ success: false,
951
+ error: new ClientAbortedError(
952
+ "client aborted before stream creation",
953
+ ),
954
+ };
955
+ }
956
+
957
+ // Always sync the model catalog so the truncation and prompt-limit
958
+ // checks use the real context window published by Qwen, not the
959
+ // conservative registry fallback. The call is cached per account.
960
+ try {
961
+ await fetchQwenModels(currentAccountId);
962
+ } catch (metadataError) {
963
+ logger.warn("[chat] model metadata sync unavailable; using registry fallback", {
964
+ model: params.contextModelId ?? params.model,
965
+ error:
966
+ metadataError instanceof Error
967
+ ? metadataError.message
968
+ : String(metadataError),
969
+ });
970
+ }
971
+
972
+ // Truncate after the real context window is known so long conversations
973
+ // are not cut down to the conservative 128K fallback.
974
+ const contextModelId = params.contextModelId ?? params.model;
975
+ const truncation = truncatePromptToIntelligentLimit(
976
+ params.finalPrompt,
977
+ contextModelId,
978
+ currentAccountId,
979
+ params.messages,
980
+ );
981
+ if (truncation.wasTruncated) {
982
+ logger.warn(
983
+ "[chat] prompt exceeded model context limit; intelligent truncation applied",
984
+ {
985
+ originalTokens: truncation.originalTokens,
986
+ truncatedTokens: truncation.truncatedTokens,
987
+ messagesKept: truncation.messagesKept,
988
+ messagesDropped: truncation.messagesDropped,
989
+ },
990
+ );
991
+ }
992
+ const loopReminder = buildRepeatedToolCallReminder(
993
+ params.messages,
994
+ config.retry.repeatedToolCallWarnThreshold,
995
+ );
996
+ const effectivePrompt = loopReminder
997
+ ? `${truncation.prompt}\n\n${loopReminder}`
998
+ : truncation.prompt;
999
+
1000
+ assertPromptWithinLimits(
1001
+ effectivePrompt,
1002
+ contextModelId,
1003
+ { accountId: currentAccountId },
1004
+ );
1005
+
1006
+ const threadParentId = params.useThreadNative
1007
+ ? params.forceNewChat || params.parallelEscape
1008
+ ? null
1009
+ : (params.existingThread?.parentId ?? null)
1010
+ : params.shouldResetUpstreamThread
1011
+ ? null
1012
+ : undefined;
1013
+ // Acquire account concurrency lease before personalization + stream creation.
1014
+ // The lease is held for the entire stream lifetime and released by the caller
1015
+ // via the returned releaseAccountLease function.
1016
+ // The thread owner or the last usable account waits without a hard
1017
+ // deadline (bounded by the client's abort signal): a fixed 30s timeout
1018
+ // here rejects a single conversation while the model is paused
1019
+ // mid-tool/thinking, which burns the request instead of serving it.
1020
+ const hasFreeAlt =
1021
+ !isSingleAccount &&
1022
+ hasFreeAlternateAccount(accounts, currentAccountId, triedAccounts);
1023
+ const sessionLabel = params.sessionId ?? currentAccountEmail;
1024
+ // The thread owner (or the last usable account) may wait for the slot
1025
+ // without a short deadline โ€” but ONLY when the slot is busy with OUR
1026
+ // session (same-session latest-wins / tool loop). If ANOTHER session is
1027
+ // generating on this account, waiting 120s is pure latency: the other
1028
+ // session may hold the slot for minutes. Fail fast with account_busy so
1029
+ // the attempt loop rotates to a different account instead (or retries
1030
+ // quickly when no alternative exists).
1031
+ const heldByOtherSession = isAccountSlotHeldByOtherSession(
1032
+ currentAccountId,
1033
+ sessionLabel,
1034
+ );
1035
+ const waitQueueForever = shouldWaitQueueForever(
1036
+ params.queueSlotUntilFree === true,
1037
+ heldByOtherSession,
1038
+ hasFreeAlt,
1039
+ );
1040
+
1041
+ // Latest-wins: if the client retried the same session, abort the old
1042
+ // generation and free the slot immediately instead of queueing behind it.
1043
+ // onlyIfEmitted: a stream that has NOT reached the client yet is
1044
+ // protected โ€” killing it would waste a generation the client has not
1045
+ // consumed. A PARALLEL request (parallelEscape) never kills at all: it
1046
+ // runs on its own chat and must not abort the main generation even
1047
+ // after the main emits its first chunk.
1048
+ if (params.sessionId && !params.parallelEscape) {
1049
+ abortLeaseByLabel(currentAccountId, sessionLabel, {
1050
+ onlyIfEmitted: true,
1051
+ });
1052
+ }
1053
+
1054
+ // Create an AbortController for this lease so a future same-session
1055
+ // retry can abort it via abortLeaseByLabel().
1056
+ const leaseAbort = new AbortController();
1057
+ // Second per-attempt controller: the acquire deadline aborts it so a
1058
+ // race-lost createQwenStream (still queued on the account stream
1059
+ // lock) dies instead of winning the lock later and burning an
1060
+ // upstream request while unobserved.
1061
+ const acquireAbort = new AbortController();
1062
+ const combinedSignal = params.requestSignal
1063
+ ? AbortSignal.any([
1064
+ params.requestSignal,
1065
+ leaseAbort.signal,
1066
+ acquireAbort.signal,
1067
+ ])
1068
+ : AbortSignal.any([leaseAbort.signal, acquireAbort.signal]);
1069
+
1070
+ if (params.parallelEscape) {
1071
+ // Parallel request racing an unemitted stream: do NOT queue on this
1072
+ // account's slot (the main may hold it for minutes while thinking).
1073
+ // Fail fast with account_busy so the attempt loop hops to a free
1074
+ // account; on a free account the request proceeds on its own chat.
1075
+ const quick = tryAcquireAccountLease(
1076
+ currentAccountId,
1077
+ sessionLabel,
1078
+ leaseAbort,
1079
+ true,
1080
+ );
1081
+ if (!quick) {
1082
+ const busyError = new Error(
1083
+ `Account ${currentAccountId} busy: parallel request (session stream unemitted)`,
1084
+ ) as Error & { code?: string; parallelEscape?: boolean };
1085
+ busyError.code = "account_busy";
1086
+ // Expected hop, not an error: suppress the "Request failed" warn.
1087
+ busyError.parallelEscape = true;
1088
+ throw busyError;
1089
+ }
1090
+ accountLease = quick;
1091
+ } else {
1092
+ accountLease = await acquireAccountLease(currentAccountId, {
1093
+ timeoutMs: waitQueueForever
1094
+ ? config.concurrency.queueWaitForeverCapMs
1095
+ : config.concurrency.busyWaitMs,
1096
+ signal: combinedSignal,
1097
+ label: sessionLabel,
1098
+ leaseAbortController: leaseAbort,
1099
+ });
1100
+ }
1101
+ // Client may have disconnected (or a same-session retry superseded us)
1102
+ // while waiting for the lease. Bail before spending time on
1103
+ // personalization sync / captcha solve.
1104
+ if (combinedSignal.aborted) {
1105
+ accountLease.release();
1106
+ return {
1107
+ success: false,
1108
+ error: new ClientAbortedError(
1109
+ "client aborted before stream creation",
1110
+ ),
1111
+ };
1112
+ }
1113
+ if (logger.isLevelEnabled("info")) {
1114
+ console.log(
1115
+ `โฑ๏ธ [Chat] Acquire: lease | account=${currentAccountEmail} | +${Date.now() - acquireStartedAt}ms`,
1116
+ );
1117
+ }
1118
+ const hasRequestPersonalization =
1119
+ params.requestPersonalizationInstruction !== null &&
1120
+ params.requestPersonalizationInstruction !== undefined;
1121
+ const releasePersonalization = hasRequestPersonalization
1122
+ ? await acquirePersonalizationLock(currentAccountId)
1123
+ : null;
1124
+ // A same-session retry (or client disconnect) can abort this request
1125
+ // while the personalization sync is still stuck on a hung page op
1126
+ // (closed Playwright context / WAF). The sync never resolves, so the
1127
+ // finally below would not run and the mutex would stay held for
1128
+ // minutes, blocking the retry until its 60s acquire timeout fires.
1129
+ // Release the lock immediately on abort instead.
1130
+ const onPersonalizationAbort = () => releasePersonalization?.();
1131
+ if (combinedSignal.aborted) {
1132
+ onPersonalizationAbort();
1133
+ } else {
1134
+ combinedSignal.addEventListener("abort", onPersonalizationAbort, {
1135
+ once: true,
1136
+ });
1137
+ }
1138
+ let result: Awaited<ReturnType<typeof createQwenStream>>;
1139
+ try {
1140
+ let promptForUpstream = effectivePrompt;
1141
+ if (hasRequestPersonalization) {
1142
+ // Let the hash-based cache in syncQwenRequestPersonalization decide
1143
+ // whether to actually POST. A new chat does not imply the account's
1144
+ // global settings were reset โ€” only session refresh or profile reset
1145
+ // should bypass the cache.
1146
+ const instruction =
1147
+ params.requestPersonalizationInstruction ?? "";
1148
+ let personalizationApplied = false;
1149
+ let syncFailure: string | null = null;
1150
+ try {
1151
+ // Hard deadline for the whole sync (browser ops each have
1152
+ // their own 60s timeout; several sequential stuck ops can
1153
+ // hold the personalization mutex for minutes). A normal
1154
+ // sync takes ~2s; beyond 30s the account page is stuck โ€”
1155
+ // fail fast so the retry loop switches accounts.
1156
+ let syncSettled = false;
1157
+ let personalizationDeadlineTimer: NodeJS.Timeout | undefined;
1158
+ const syncPromise = syncQwenRequestPersonalization(
1159
+ instruction,
1160
+ currentAccountId === "global"
1161
+ ? undefined
1162
+ : currentAccountId,
1163
+ {
1164
+ model: params.model,
1165
+ toolsCount: params.toolsCount ?? 0,
1166
+ sessionId: params.sessionId,
1167
+ promptChars: effectivePrompt.length,
1168
+ forceSync: false,
1169
+ },
1170
+ ).then(
1171
+ (value) => {
1172
+ syncSettled = true;
1173
+ return value;
1174
+ },
1175
+ (error) => {
1176
+ syncSettled = true;
1177
+ syncFailure =
1178
+ error instanceof Error ? error.message : String(error);
1179
+ return false;
1180
+ },
1181
+ );
1182
+ personalizationApplied = await Promise.race([
1183
+ syncPromise,
1184
+ new Promise<boolean>((resolve) => {
1185
+ personalizationDeadlineTimer = setTimeout(() => {
1186
+ if (!syncSettled) {
1187
+ syncFailure = `sync timed out after ${PERSONALIZATION_SYNC_DEADLINE_MS}ms`;
1188
+ }
1189
+ resolve(false);
1190
+ }, PERSONALIZATION_SYNC_DEADLINE_MS);
1191
+ }),
1192
+ ]);
1193
+ // The sync won the race: stop the deadline so it cannot keep the
1194
+ // event loop alive for the full 30s window (it used to leak one
1195
+ // 30s timer per request โ†’ ~30s of test-suite drain per file).
1196
+ if (personalizationDeadlineTimer) {
1197
+ clearTimeout(personalizationDeadlineTimer);
1198
+ }
1199
+ } catch (error) {
1200
+ syncFailure =
1201
+ error instanceof Error ? error.message : String(error);
1202
+ }
1203
+
1204
+ // Agent instructions ride ONLY the account-level personalization โ€”
1205
+ // the prompt never carries them. An unconfirmed sync must fail the
1206
+ // attempt (retryable โ†’ rotates accounts, each re-syncs on its own
1207
+ // account) instead of degrading to inline. An empty instruction has
1208
+ // nothing to guarantee (plain chat), so it stays best-effort.
1209
+ if (instruction && !personalizationApplied) {
1210
+ throw new PersonalizationSyncError(
1211
+ `personalization sync not confirmed for ${currentAccountEmail}: ${syncFailure ?? "settings response did not confirm the instruction"}`,
1212
+ );
1213
+ }
1214
+ }
1215
+ if (logger.isLevelEnabled("info")) {
1216
+ console.log(
1217
+ `โฑ๏ธ [Chat] Acquire: sync | account=${currentAccountEmail} | +${Date.now() - acquireStartedAt}ms`,
1218
+ );
1219
+ }
1220
+
1221
+ assertPromptWithinLimits(
1222
+ promptForUpstream,
1223
+ params.contextModelId ?? params.model,
1224
+ { accountId: currentAccountId },
1225
+ );
1226
+ // Bound the whole acquire with a hard deadline: a silent hang in any
1227
+ // phase (mutex wait, header capture, fetch metadata, internal retries)
1228
+ // fails fast and retryable instead of blocking the request for minutes
1229
+ // with zero log output.
1230
+ const acquireDeadlineMs = config.concurrency.acquireDeadlineMs;
1231
+ let acquireDeadlineTimer: NodeJS.Timeout | undefined;
1232
+ const acquireDeadline = new Promise<never>((_, reject) => {
1233
+ acquireDeadlineTimer = setTimeout(() => {
1234
+ // Abort the losing createQwenStream (it is still queued on the
1235
+ // stream lock or mid-create); the post-lock signal re-check in
1236
+ // createQwenStream then throws instead of letting the orphan
1237
+ // win the lock later and waste an upstream request.
1238
+ acquireAbort.abort();
1239
+ const err = new Error(
1240
+ `Acquire deadline (${acquireDeadlineMs}ms) exceeded creating stream on ${currentAccountEmail}`,
1241
+ ) as Error & { code?: string };
1242
+ err.code = "acquire_deadline";
1243
+ reject(err);
1244
+ }, acquireDeadlineMs);
1245
+ acquireDeadlineTimer.unref?.();
1246
+ });
1247
+ result = await Promise.race([
1248
+ createQwenStream(
1249
+ promptForUpstream,
1250
+ params.isThinkingModel,
1251
+ params.model,
1252
+ threadParentId,
1253
+ currentAccountId === "global" ? undefined : currentAccountId,
1254
+ params.allFiles.length > 0 ? params.allFiles : undefined,
1255
+ params.forceNewChat || params.useThreadNative || params.parallelEscape
1256
+ ? {
1257
+ chatSessionId:
1258
+ params.forceNewChat || params.parallelEscape
1259
+ ? null
1260
+ : (params.existingThread?.chatSessionId ?? null),
1261
+ forceNewChat: false,
1262
+ reasoningMode: params.reasoningMode,
1263
+ parallelEscape: params.parallelEscape,
1264
+ chatMode: params.chatMode,
1265
+ }
1266
+ : params.reasoningMode ? { reasoningMode: params.reasoningMode } : undefined,
1267
+ combinedSignal,
1268
+ ),
1269
+ acquireDeadline,
1270
+ ]);
1271
+ // The acquire won: stop the deadline so it cannot fire later and
1272
+ // abort a signal nobody observes anymore.
1273
+ if (acquireDeadlineTimer) clearTimeout(acquireDeadlineTimer);
1274
+
1275
+ if (logger.isLevelEnabled("info")) {
1276
+ console.log(
1277
+ `โฑ๏ธ [Chat] Acquire done | completion=${params.completionId.substring(0, 8)} | account=${currentAccountEmail} | +${Date.now() - acquireStartedAt}ms`,
1278
+ );
1279
+ }
1280
+
1281
+ const contextMeter = buildContextMeterSnapshot({
1282
+ modelId: params.contextModelId ?? params.model,
1283
+ accountId: currentAccountId,
1284
+ requestPrompt: promptForUpstream,
1285
+ fullPrompt: params.fullPrompt,
1286
+ mode:
1287
+ params.contextMode ??
1288
+ (params.forceNewChat
1289
+ ? "replay"
1290
+ : params.existingThread
1291
+ ? "delta"
1292
+ : "full"),
1293
+ qwenPayloadBytes: result.tokenEstimationContext.qwenPayloadBytes,
1294
+ qwenPayloadPromptChars:
1295
+ result.tokenEstimationContext.qwenPayloadPromptChars,
1296
+ qwenPayloadMessageCount:
1297
+ result.tokenEstimationContext.qwenPayloadMessageCount,
1298
+ messageCount: params.messageCount,
1299
+ fullMessageCount: params.fullMessageCount,
1300
+ toolsCount: params.toolsCount,
1301
+ filesCount: params.allFiles.length,
1302
+ activePersonalization:
1303
+ result.tokenEstimationContext.activePersonalization,
1304
+ });
1305
+
1306
+ if (contextMeter) {
1307
+ logger.debug("[context_meter] request", {
1308
+ ...contextMeterLogData(contextMeter),
1309
+ account: currentAccountEmail,
1310
+ attempt,
1311
+ });
1312
+ result = {
1313
+ ...result,
1314
+ tokenEstimationContext: {
1315
+ ...result.tokenEstimationContext,
1316
+ contextMeter,
1317
+ },
1318
+ };
1319
+ }
1320
+ } finally {
1321
+ combinedSignal.removeEventListener(
1322
+ "abort",
1323
+ onPersonalizationAbort,
1324
+ );
1325
+ releasePersonalization?.();
1326
+ }
1327
+
1328
+ // Client cancelled (or a same-session retry superseded us) during the
1329
+ // (potentially slow) personalization sync. Bail before createQwenStream
1330
+ // spends time on header capture / captcha.
1331
+ if (combinedSignal.aborted) {
1332
+ // Never drop a created stream without cancelling it: the wrapped
1333
+ // stream's cancel() releases the per-account stream lock. Dropping it
1334
+ // silently LEAKS that lock and the next acquire on this account blocks
1335
+ // until the acquire deadline (observed symptom: 150s phantom wait).
1336
+ void result.stream
1337
+ .cancel("client aborted after stream creation")
1338
+ .catch(() => {});
1339
+ accountLease?.release();
1340
+ return {
1341
+ success: false,
1342
+ error: new ClientAbortedError(
1343
+ "client aborted during stream creation",
1344
+ ),
1345
+ };
1346
+ }
1347
+
1348
+ if (
1349
+ params.useThreadNative &&
1350
+ params.updateLogicalThread &&
1351
+ !params.parallelEscape &&
1352
+ params.sessionId &&
1353
+ result.uiSessionId
1354
+ ) {
1355
+ // Bind chat/account immediately. Do NOT write the request parent as the
1356
+ // sticky parent โ€” that is the *previous* assistant id we attached to.
1357
+ // Streaming will rememberParent(response_id) with the new assistant id
1358
+ // so the next turn appends (user_action=chat + parent_id=last response).
1359
+ // Preserve any existing sticky parent until the stream updates it.
1360
+ const priorParent =
1361
+ params.existingThread?.parentId ??
1362
+ getLogicalThreadState(params.sessionId)?.parentId ??
1363
+ null;
1364
+ updateLogicalThreadState(params.sessionId, {
1365
+ accountId: result.accountId,
1366
+ chatSessionId: result.uiSessionId,
1367
+ parentId: params.forceNewChat ? null : priorParent,
1368
+ instructionsSent: true,
1369
+ });
1370
+
1371
+ if (process.env.CHAT_REQUEST_LOG === "true") {
1372
+ logger.info("[chat] thread-native upstream session", {
1373
+ sessionId: params.sessionId,
1374
+ accountId: result.accountId,
1375
+ chatSessionId: result.uiSessionId,
1376
+ requestParentId: threadParentId ?? null,
1377
+ stickyParentId: params.forceNewChat ? null : priorParent,
1378
+ createdNewChat: !params.existingThread,
1379
+ });
1380
+ }
1381
+ }
1382
+
1383
+ if (isToolcallDebugEnabled()) {
1384
+ logger.debug("[chat] stream created successfully", {
1385
+ accountId: currentAccountId,
1386
+ accountEmail: currentAccountEmail,
1387
+ uiSessionId: result.uiSessionId,
1388
+ });
1389
+ }
1390
+
1391
+ // A served stream means the WAF accepted this account's identity:
1392
+ // clear the hard-block escalation streak (keeps the next block at the
1393
+ // base window instead of compounding forever).
1394
+ noteWafRecovery(currentAccountId);
1395
+ markAccountSuccessful(currentAccountId);
1396
+ if (accountLease) {
1397
+ markLeaseCompletion(
1398
+ currentAccountId,
1399
+ accountLease.leaseId,
1400
+ params.completionId,
1401
+ );
1402
+ }
1403
+ return {
1404
+ success: true,
1405
+ ...result,
1406
+ accountEmail: currentAccountEmail,
1407
+ releaseAccountLease: accountLease.release,
1408
+ };
1409
+ } catch (err: any) {
1410
+ attemptError = err;
1411
+ lastAttemptError = err;
1412
+ // Release the lease on failure โ€” the stream was never created or
1413
+ // will not be consumed by the caller.
1414
+ accountLease?.release();
1415
+ }
1416
+
1417
+ attemptsLeft--;
1418
+ const err = attemptError;
1419
+ // The account that actually failed THIS attempt โ€” captured before any
1420
+ // branch below can switch currentAccountId/Email (chat_in_progress
1421
+ // escalation moves to a fresh account). The generic retry log must name
1422
+ // the account that failed, not the newly-selected one that was never
1423
+ // attempted (observed: "Qwen request failed for 280wu" when 280wu had
1424
+ // never been tried and ldyjl had failed 4x with chat_in_progress).
1425
+ const failedAccountEmail = currentAccountEmail;
1426
+
1427
+ // Once the client request is aborted, do not rotate accounts or retry. The
1428
+ // old request can otherwise keep acquiring leases after the client is gone.
1429
+ if (params.requestSignal?.aborted) {
1430
+ return { success: false, error: err };
1431
+ }
1432
+
1433
+ // Log the error details for debugging (skip quota errors โ€” logged separately below,
1434
+ // client aborts โ€” they are silent by design, and chat_in_progress โ€” handled
1435
+ // by the dedicated settling log below to avoid alarming false-positive error spam).
1436
+ const errMsg = err instanceof Error ? err.message : String(err || "");
1437
+ if (
1438
+ err &&
1439
+ !(err instanceof ClientAbortedError) &&
1440
+ !isAccountUnavailableError(err) &&
1441
+ !(err as any)?.parallelEscape &&
1442
+ !isChatInProgressError(err)
1443
+ ) {
1444
+ const errCode = getQwenErrorCode(err) || "unknown";
1445
+ console.warn(
1446
+ `โŒ [Chat] Request failed | ${currentAccountEmail} | ${errCode} | ${errMsg.substring(0, 200)}`,
1447
+ );
1448
+ }
1449
+
1450
+
1451
+
1452
+ if (!err) {
1453
+ return {
1454
+ success: false,
1455
+ error: new Error("Failed to create Qwen stream"),
1456
+ };
1457
+ }
1458
+
1459
+ if (
1460
+ err instanceof QwenSessionExpiredError ||
1461
+ err.name === "QwenSessionExpiredError"
1462
+ ) {
1463
+ console.warn(
1464
+ `๐Ÿ”„ [Chat] Session expired for ${currentAccountEmail} (${currentAccountId}). Attempting re-login...`,
1465
+ );
1466
+ const reLoginOk = await attemptRelogin(
1467
+ currentAccountId,
1468
+ currentAccountEmail,
1469
+ );
1470
+ if (reLoginOk) continue;
1471
+ return { success: false, error: err };
1472
+ }
1473
+
1474
+
1475
+
1476
+
1477
+
1478
+ // Account-scoped quota/rate-limit: cool this account and stop local retries
1479
+ // so outer account rotation can pick another one immediately.
1480
+ if (isAccountUnavailableError(err)) {
1481
+ const quotaMsg = err.message || "Unknown quota error";
1482
+ const policy = classifyRetryAction(err, {
1483
+ requestAborted: params.requestSignal?.aborted === true,
1484
+ });
1485
+ const isTemporary = policy.accountCooldownReason === "RateLimitTemporary";
1486
+
1487
+ // Temporary load shedding or single account: retry same account
1488
+ // after a short delay before giving up / rotating.
1489
+ if ((isTemporary || isSingleAccount) && !quotaRetried && attemptsLeft > 0) {
1490
+ quotaRetried = true;
1491
+ const delayMs = isTemporary ? 3_000 : config.retry.baseDelayMs;
1492
+ console.warn(
1493
+ `โš ๏ธ [Chat] Quota exceeded | ${currentAccountEmail} | ${isTemporary ? "temporary, " : ""}retrying in ${delayMs}ms...`,
1494
+ );
1495
+ await new Promise((resolve) =>
1496
+ setTimeout(resolve, delayMs),
1497
+ );
1498
+ continue;
1499
+ }
1500
+
1501
+ // Consolidate quota details into a single log emitted by the outer
1502
+ // rotation loop. The cooldown itself is set silently to avoid duplicates.
1503
+ const cooldownSeconds = policy.accountCooldownMs
1504
+ ? Math.round(policy.accountCooldownMs / 1000)
1505
+ : 0;
1506
+ const cooldownUntil = policy.accountCooldownMs
1507
+ ? new Date(Date.now() + policy.accountCooldownMs)
1508
+ : null;
1509
+ const untilStr = cooldownUntil
1510
+ ? ` | until=${formatCooldownUntil(cooldownUntil)}`
1511
+ : "";
1512
+
1513
+ try {
1514
+ (err as any).quotaInfo = {
1515
+ email: currentAccountEmail,
1516
+ cooldownSeconds,
1517
+ untilStr,
1518
+ message: quotaMsg.substring(0, 150),
1519
+ };
1520
+ } catch {
1521
+ // Best-effort metadata for logging.
1522
+ }
1523
+
1524
+ markAccountFailed(currentAccountId);
1525
+ markAccountRateLimited(
1526
+ currentAccountId,
1527
+ policy.accountCooldownMs,
1528
+ policy.accountCooldownReason || "QuotaExceeded",
1529
+ { silent: true },
1530
+ );
1531
+ return { success: false, error: err };
1532
+ }
1533
+
1534
+ const policy = classifyRetryAction(err, {
1535
+ requestAborted: params.requestSignal?.aborted === true,
1536
+ });
1537
+
1538
+ // The full retry decision โ€” the `โŒ Request failed` line shows the error
1539
+ // but not WHY this action was chosen. Surface every field so the next
1540
+ // escalation/switch/cooldown is explainable from the log alone.
1541
+ if (logger.isLevelEnabled("info")) {
1542
+ console.log(
1543
+ `๐Ÿงญ [Chat] Retry policy | account=${currentAccountEmail} | reason=${policy.reason} | retryable=${policy.retryable} | switch=${policy.switchAccount} | newChat=${policy.forceNewChat} | fullPrompt=${policy.retryWithFullPrompt}${policy.dropFiles ? ` | dropFiles` : ""} | retryAfter=${policy.retryAfterMs}ms${policy.accountCooldownMs ? ` | cooldown=${Math.round(policy.accountCooldownMs / 1000)}s (${policy.accountCooldownReason ?? ""})` : ""}`,
1544
+ );
1545
+ }
1546
+
1547
+ // Corrupted history means the stored parent chain is unusable. Purge the
1548
+ // parent immediately so a failed recovery cannot leave the tainted thread
1549
+ // bound for the next turn.
1550
+ if (policy.reason === "corrupted_chat_history") {
1551
+ invalidateLogicalThreadParent(params.sessionId);
1552
+ }
1553
+
1554
+ // A generic invalid_input is often a stale/corrupted upstream chat rather
1555
+ // than an account failure. Rebuild it once on the same account first. If the
1556
+ // fresh chat fails again, the normal policy is allowed to rotate.
1557
+ const retryInvalidInputOnSameAccount =
1558
+ shouldRetryInvalidInputOnSameAccount(
1559
+ policy.reason,
1560
+ invalidInputSameAccountRetried,
1561
+ );
1562
+ if (retryInvalidInputOnSameAccount) {
1563
+ invalidInputSameAccountRetried = true;
1564
+ }
1565
+ const shouldSwitchAccount =
1566
+ policy.switchAccount && !retryInvalidInputOnSameAccount;
1567
+
1568
+ // chat_in_progress means the previous Qwen generation has not stopped
1569
+ // yet (the tool loop fires the next turn the instant the previous one
1570
+ // completes; the upstream chat stays "in progress" for a few seconds
1571
+ // after the terminal event โ€” usually 2-4s, measured >6s after a 491KB
1572
+ // turn). Policy design (settle-aware, upstream-aligned):
1573
+ // 1. Retry the SAME chat with JITTERED busyMs-based waits โ€” never a
1574
+ // fixed ladder (concurrent sessions would retry in lock-step).
1575
+ // 2. After the settle budget (CHAT_IN_PROGRESS_MAX_RETRIES, ~35s):
1576
+ // ONE bounded escalation. A chat can be "in progress" for MINUTES
1577
+ // when a superseded generation keeps running server-side โ€” retrying
1578
+ // the same chat then fails every request until it frees (observed:
1579
+ // 2.1MB turn held a chat busy ~9min). The single escalation opens a
1580
+ // FRESH chat with the full context so the turn makes progress; it
1581
+ // fires at most once per request (a second replay would only repeat
1582
+ // the ~1MB re-upload cost the settle design removes).
1583
+ // 3. If the escalated attempt ALSO fails with chat_in_progress, the
1584
+ // request FAILS and the origin binding is cleared (next client turn
1585
+ // starts fresh with a replay instead of wedging on the stuck chat).
1586
+ if (policy.reason === "chat_in_progress") {
1587
+ if (chatInProgressOriginAccountId === null) {
1588
+ // First chat_in_progress of this request: remember the account
1589
+ // whose chat is stuck (used by the loop-exit session handling).
1590
+ chatInProgressOriginAccountId = currentAccountId;
1591
+ chatInProgressOriginAccountEmail = currentAccountEmail;
1592
+ }
1593
+ chatInProgressCount++;
1594
+ markAccountTemporarilyBusy(
1595
+ currentAccountId,
1596
+ config.retry.chatInProgressBusyMs,
1597
+ );
1598
+
1599
+ if (chatInProgressCount > config.retry.chatInProgressMaxAttempts) {
1600
+ if (!chatInProgressEscalated) {
1601
+ // Same-chat settle budget exhausted: the chat is genuinely
1602
+ // busy, not settling. One bounded escalation โ€” a fresh chat
1603
+ // with the full context (the only way to progress while the
1604
+ // old chat runs on server-side). Bounded: this fires at most
1605
+ // once per request.
1606
+ chatInProgressEscalated = true;
1607
+ console.warn(
1608
+ `๐Ÿ”„ [Chat] chat_in_progress escalation (${chatInProgressCount}) | forcing a new chat with full context on ${currentAccountEmail}`,
1609
+ );
1610
+ if (params.useThreadNative) {
1611
+ params.existingThread = null;
1612
+ params.finalPrompt = params.fullPrompt;
1613
+ params.messageCount =
1614
+ params.fullMessageCount ?? params.messageCount;
1615
+ params.forceNewChat = true;
1616
+ }
1617
+ // The escalation targets a FRESH chat (not the busy one), so
1618
+ // no settle wait โ€” it gets its own attempt budget.
1619
+ attemptsLeft = Math.max(attemptsLeft, 1);
1620
+ policy.retryAfterMs = 0;
1621
+ } else {
1622
+ // The escalated fresh chat ALSO failed with chat_in_progress
1623
+ // (bizarre, but possible on a wedged account). Give up โ€” the
1624
+ // outer rotation treats this as terminal.
1625
+ attemptsLeft = 0;
1626
+ policy.retryable = false;
1627
+ }
1628
+ } else {
1629
+ // The settle window has its own budget, independent of the
1630
+ // global RETRY_MAX_ATTEMPTS: with maxAttempts=3 the counter
1631
+ // above would hit 0 on the 3rd failure and skip the longer
1632
+ // waits that absorb the >6s settles of huge turns (2026-08-11).
1633
+ attemptsLeft = Math.max(attemptsLeft, 1);
1634
+
1635
+ // The 1st failure keeps the upstream-suggested wait (~1.2s);
1636
+ // later retries wait a jittered context-scaled busyMs-based window.
1637
+ if (chatInProgressCount >= 2) {
1638
+ const promptChars =
1639
+ params.fullPrompt?.length ??
1640
+ params.finalPrompt?.length ??
1641
+ 0;
1642
+ policy.retryAfterMs = jitterChatInProgressDelay(
1643
+ chatInProgressCount,
1644
+ config.retry.chatInProgressBusyMs,
1645
+ Math.random,
1646
+ promptChars,
1647
+ );
1648
+ }
1649
+ }
1650
+ }
1651
+
1652
+ if (policy.reason === "account_initialization_failed") {
1653
+ console.warn(
1654
+ `โš ๏ธ [Chat] Account initialization failed | ${currentAccountEmail} | cooldown=${Math.round((policy.accountCooldownMs ?? 0) / 1000)}s`,
1655
+ );
1656
+ markAccountFailed(currentAccountId);
1657
+ markAccountRateLimited(
1658
+ currentAccountId,
1659
+ policy.accountCooldownMs,
1660
+ policy.accountCooldownReason,
1661
+ );
1662
+ return { success: false, error: err };
1663
+ }
1664
+
1665
+ // Prefer switching account for any retryable upstream error when possible.
1666
+ // A PARALLEL escape hops to a FREE account (skip busy/temporarily-busy):
1667
+ // the auxiliary request must land on an available slot fast, never on a
1668
+ // second occupied account (the 2026-08-20 stall rotated ldyjlโ†’cgnx3, both
1669
+ // busy, ~14s lease wait). Normal requests keep the cooldown-only picker so
1670
+ // single-account/saturated pools stay lossless.
1671
+ if (
1672
+ policy.retryable &&
1673
+ shouldSwitchAccount &&
1674
+ !isSingleAccount &&
1675
+ accountSwitches < maxAccountSwitches
1676
+ ) {
1677
+ const nextAccount = params.parallelEscape
1678
+ ? getNextFreeAccountForParallel(accounts, triedAccounts, currentAccountId)
1679
+ : getNextAvailableAccount(triedAccounts);
1680
+ if (nextAccount && nextAccount.id !== currentAccountId) {
1681
+ console.warn(
1682
+ `๐Ÿ”„ [Chat] Switching account after ${policy.reason} | ${currentAccountEmail} -> ${maskEmail(nextAccount.email)}`,
1683
+ );
1684
+ if (policy.accountCooldownMs || policy.accountCooldownReason) {
1685
+ markAccountRateLimited(
1686
+ currentAccountId,
1687
+ policy.accountCooldownMs,
1688
+ policy.accountCooldownReason || "RetrySwitch",
1689
+ );
1690
+ }
1691
+ triedAccounts.add(currentAccountId);
1692
+ currentAccountId = nextAccount.id;
1693
+ currentAccountEmail = maskEmail(nextAccount.email);
1694
+ accountSwitches++;
1695
+
1696
+ // Account switch always rebuilds a fresh upstream chat with full history.
1697
+ // Do NOT persist sticky binding until create succeeds โ€” premature empty
1698
+ // chatSessionId writes make subsequent turns rotate/lose context.
1699
+ if (params.useThreadNative) {
1700
+ params.existingThread = null;
1701
+ params.finalPrompt = params.fullPrompt;
1702
+ params.messageCount = params.fullMessageCount ?? params.messageCount;
1703
+ params.forceNewChat = true;
1704
+ }
1705
+
1706
+ await new Promise((resolve) =>
1707
+ setTimeout(
1708
+ resolve,
1709
+ Math.min(policy.retryAfterMs ?? config.retry.baseDelayMs, 1000),
1710
+ ),
1711
+ );
1712
+ continue;
1713
+ }
1714
+
1715
+ console.warn(
1716
+ `โš ๏ธ [Chat] No other account available after ${policy.reason} | Retrying on same account`,
1717
+ );
1718
+ }
1719
+
1720
+ // Force new chat / full context when policy requests it (invalid_input, chat gone, etc.)
1721
+ if (
1722
+ policy.retryable &&
1723
+ (policy.forceNewChat || policy.retryWithFullPrompt) &&
1724
+ params.useThreadNative
1725
+ ) {
1726
+ console.warn(
1727
+ `๐Ÿ”„ [Chat] Forcing new chat/full context | reason=${policy.reason}`,
1728
+ );
1729
+ params.existingThread = null;
1730
+ params.finalPrompt = params.fullPrompt;
1731
+ params.messageCount = params.fullMessageCount ?? params.messageCount;
1732
+ params.forceNewChat = true;
1733
+ }
1734
+
1735
+ // Drop files on retry for invalid_input to isolate file-related errors
1736
+ if (policy.dropFiles && params.allFiles.length > 0) {
1737
+ console.warn(
1738
+ `๐Ÿ—‚๏ธ [Chat] Dropping ${params.allFiles.length} file(s) on retry to isolate invalid_input error:`,
1739
+ params.allFiles.map((f) => ({
1740
+ name: f.name,
1741
+ type: f.type,
1742
+ size: f.size ?? "unknown",
1743
+ })),
1744
+ );
1745
+ params.allFiles = [];
1746
+ }
1747
+
1748
+ if (!policy.retryable || attemptsLeft <= 0) {
1749
+ if (policy.accountCooldownMs || policy.accountCooldownReason) {
1750
+ markAccountRateLimited(
1751
+ currentAccountId,
1752
+ policy.accountCooldownMs,
1753
+ policy.accountCooldownReason || "RetryExhausted",
1754
+ );
1755
+ }
1756
+
1757
+ if (
1758
+ err instanceof RetryableQwenStreamError ||
1759
+ isChatInProgressError(err)
1760
+ ) {
1761
+ // Chat_in_progress give-up only happens AFTER the bounded escalation
1762
+ // (the settle window alone cannot exhaust the budget โ€” the first
1763
+ // over-budget failure escalates). The escalated chat was freshly
1764
+ // created; the STORED binding still points at the stuck chat, so
1765
+ // clearing the origin account's sessions frees the next turn to
1766
+ // start fresh instead of re-wedging on the stuck chat. For other
1767
+ // retryable upstream errors (network/quota) the same clear drops a
1768
+ // binding that may point at a genuinely stuck chat. Clear the ORIGIN
1769
+ // account โ€” never the current one (other policies may have switched).
1770
+ const clearTargetId = chatInProgressOriginAccountId ?? currentAccountId;
1771
+ const clearTargetEmail =
1772
+ chatInProgressOriginAccountEmail ?? currentAccountEmail;
1773
+ console.warn(
1774
+ `๐Ÿงน [Chat] Clearing session state for ${clearTargetEmail} (${clearTargetId}) after exhausted retries`,
1775
+ );
1776
+ clearAllSessionsForAccount(clearTargetId);
1777
+ }
1778
+
1779
+ return { success: false, error: err };
1780
+ }
1781
+
1782
+ const useDelay = Math.max(
1783
+ 0,
1784
+ policy.retryAfterMs ?? retryDelay ?? config.retry.baseDelayMs,
1785
+ );
1786
+
1787
+ if (policy.reason === "chat_in_progress") {
1788
+ const promptChars =
1789
+ params.fullPrompt?.length ??
1790
+ params.finalPrompt?.length ??
1791
+ 0;
1792
+ const contextLabel =
1793
+ promptChars > 1_000_000
1794
+ ? `${(promptChars / (1024 * 1024)).toFixed(1)}MB context`
1795
+ : promptChars > 100_000
1796
+ ? `${Math.round(promptChars / 1024)}KB context`
1797
+ : "";
1798
+ const contextSuffix = contextLabel ? ` | ${contextLabel}` : "";
1799
+ console.warn(
1800
+ `โณ [Chat] Chat settling | ${failedAccountEmail}${contextSuffix} | waiting ${(useDelay / 1000).toFixed(1)}s (attempt ${chatInProgressCount}/${config.retry.chatInProgressMaxAttempts})...`,
1801
+ );
1802
+ } else {
1803
+ console.warn(
1804
+ `๐Ÿ”„ [Chat] Qwen request failed for ${failedAccountEmail}, retrying in ${useDelay}ms... (${attemptsLeft} left). reason=${policy.reason} error=${errMsg.slice(0, 200)}`,
1805
+ );
1806
+ }
1807
+ await new Promise((r) => setTimeout(r, useDelay));
1808
+ retryDelay = Math.min(retryDelay * 2, config.retry.maxDelayMs);
1809
+ }
1810
+
1811
+ return {
1812
+ success: false,
1813
+ error:
1814
+ lastAttemptError ??
1815
+ new Error("Qwen stream retry attempts were exhausted"),
1816
+ };
1817
+ }