qwenproxy-cli 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/LICENSE +14 -14
  2. package/README.md +906 -906
  3. package/bin/qwenproxy.js +5 -1
  4. package/package.json +77 -78
  5. package/src/api/error-classifier.ts +159 -159
  6. package/src/api/error-helpers.ts +118 -118
  7. package/src/api/models.ts +261 -261
  8. package/src/api/server.ts +860 -859
  9. package/src/cache/memory-cache.ts +385 -385
  10. package/src/clean-cache.ts +204 -204
  11. package/src/core/account-concurrency.ts +671 -671
  12. package/src/core/account-manager.ts +301 -297
  13. package/src/core/account-priority.ts +163 -163
  14. package/src/core/accounts.ts +186 -186
  15. package/src/core/config.ts +383 -383
  16. package/src/core/crypto-utils.ts +79 -79
  17. package/src/core/database.ts +276 -276
  18. package/src/core/errors.ts +118 -118
  19. package/src/core/logger.ts +269 -269
  20. package/src/core/memory-usage.ts +84 -84
  21. package/src/core/metrics.ts +291 -291
  22. package/src/core/model-alias.ts +77 -77
  23. package/src/core/model-registry.ts +544 -544
  24. package/src/core/mutex.ts +119 -119
  25. package/src/core/paths.ts +199 -199
  26. package/src/core/prompt-limits.ts +214 -214
  27. package/src/core/reasoning-effort.ts +102 -102
  28. package/src/core/stream-registry.ts +96 -96
  29. package/src/core/waf-isolation.ts +117 -117
  30. package/src/core/watchdog.ts +195 -195
  31. package/src/delete-chats.ts +23 -23
  32. package/src/index.ts +65 -64
  33. package/src/login.ts +147 -147
  34. package/src/reset-cooldowns.ts +11 -11
  35. package/src/routes/anthropic/index.ts +355 -355
  36. package/src/routes/anthropic/translate.ts +522 -522
  37. package/src/routes/anthropic/types.ts +154 -154
  38. package/src/routes/anthropic/validation.ts +144 -144
  39. package/src/routes/chat/account.ts +1817 -1817
  40. package/src/routes/chat/context.ts +241 -241
  41. package/src/routes/chat/errors.ts +85 -85
  42. package/src/routes/chat/helpers.ts +268 -268
  43. package/src/routes/chat/index.ts +618 -618
  44. package/src/routes/chat/media.ts +285 -285
  45. package/src/routes/chat/retry-policy.ts +754 -754
  46. package/src/routes/chat/stop.ts +98 -98
  47. package/src/routes/chat/streaming.ts +2710 -2710
  48. package/src/routes/chat/validation.ts +526 -526
  49. package/src/routes/chat.ts +2 -2
  50. package/src/routes/completions.ts +290 -290
  51. package/src/routes/images.ts +139 -139
  52. package/src/routes/responses/adapter.ts +503 -503
  53. package/src/routes/responses/index.ts +405 -405
  54. package/src/routes/responses/state.ts +230 -230
  55. package/src/routes/responses/streaming.ts +528 -528
  56. package/src/routes/responses/types.ts +285 -285
  57. package/src/routes/responses/validation.ts +202 -202
  58. package/src/routes/upload.ts +731 -731
  59. package/src/routes/videos.ts +214 -214
  60. package/src/services/auth-playwright.ts +173 -173
  61. package/src/services/captcha-coordinator.ts +161 -161
  62. package/src/services/captcha-solver.ts +553 -553
  63. package/src/services/chat-cleanup.ts +80 -80
  64. package/src/services/context-meter.ts +317 -317
  65. package/src/services/fingerprint.ts +242 -242
  66. package/src/services/human-behavior.ts +173 -173
  67. package/src/services/media-generation.ts +1748 -1748
  68. package/src/services/playwright.ts +2878 -2800
  69. package/src/services/qwen-chat-pool.ts +345 -345
  70. package/src/services/qwen-errors.ts +133 -133
  71. package/src/services/qwen-headers.ts +79 -79
  72. package/src/services/qwen-thread-state.ts +393 -393
  73. package/src/services/qwen-url.ts +19 -19
  74. package/src/services/qwen.ts +3126 -3126
  75. package/src/services/session-keeper.ts +88 -88
  76. package/src/services/token-estimation-metrics.ts +118 -118
  77. package/src/sync/claude-code.ts +75 -75
  78. package/src/sync/codex.ts +123 -123
  79. package/src/sync/index.ts +362 -362
  80. package/src/sync/omp.ts +105 -105
  81. package/src/sync/opencode.ts +214 -214
  82. package/src/sync/types.ts +53 -53
  83. package/src/sync/utils.ts +27 -27
  84. package/src/sync-clients.ts +189 -189
  85. package/src/tools/instructions.ts +137 -137
  86. package/src/tools/manifest.ts +81 -81
  87. package/src/tools/parser.ts +2989 -2989
  88. package/src/tools/toolcall-tags.ts +142 -142
  89. package/src/tui/app.ts +259 -264
  90. package/src/tui/index.ts +61 -61
  91. package/src/tui/markdown.ts +258 -258
  92. package/src/tui/proxy-client.ts +331 -326
  93. package/src/tui/screen.ts +294 -278
  94. package/src/tui/server-manager.ts +270 -270
  95. package/src/tui/theme.ts +432 -432
  96. package/src/tui/types.ts +33 -33
  97. package/src/tui/views/accounts-view.ts +656 -656
  98. package/src/tui/views/chat-view.ts +1018 -823
  99. package/src/tui/views/logs-view.ts +479 -413
  100. package/src/tui/views/status-view.ts +204 -204
  101. package/src/tui/views/storage-view.ts +304 -291
  102. package/src/tui/views/sync-view.ts +409 -409
  103. package/src/types/ali-oss.d.ts +32 -32
  104. package/src/update-cli.ts +121 -0
  105. package/src/utils/context-truncation.ts +84 -84
  106. package/src/utils/json.ts +380 -380
  107. package/src/utils/session-id.ts +37 -37
  108. package/src/utils/tool-call-guard.ts +84 -84
  109. package/src/utils/types.ts +109 -109
@@ -1,754 +1,754 @@
1
- /*
2
- * Generic upstream retry / account-switch policy.
3
- *
4
- * Default: retry + prefer another account for unknown/upstream failures.
5
- * Stop only for a small denylist of terminal local errors.
6
- */
7
-
8
- import { config } from "../../core/config.ts";
9
- import { computeQuotaCooldownMs } from "../../core/account-manager.ts";
10
- import { logger } from "../../core/logger.ts";
11
- import {
12
- PersonalizationSyncError,
13
- QwenNetworkError,
14
- QwenUpstreamError,
15
- QwenUpstreamUnavailableError,
16
- RetryableQwenStreamError,
17
- } from "../../services/qwen.ts";
18
- import {
19
- AuthError,
20
- ClientAbortedError,
21
- NotFoundError,
22
- ValidationError,
23
- } from "../../core/errors.ts";
24
- import { isAbortError } from "./helpers.ts";
25
-
26
- export type RetryAction = {
27
- /** Outer/create-stream layer should retry this failure */
28
- retryable: boolean;
29
- /** Prefer switching to another account when available */
30
- switchAccount: boolean;
31
- /** Force a new Qwen chat on retry */
32
- forceNewChat: boolean;
33
- /** Resend full conversation context (not just delta) */
34
- retryWithFullPrompt: boolean;
35
- /** Drop attached files on retry (for invalid_input caused by bad attachments) */
36
- dropFiles?: boolean;
37
- /** Suggested delay before next attempt */
38
- retryAfterMs: number;
39
- /** Optional short cooldown for the failing account */
40
- accountCooldownMs?: number;
41
- /** Cooldown reason label */
42
- accountCooldownReason?: string;
43
- /** Why this action was chosen (logging/debug) */
44
- reason: string;
45
- };
46
-
47
- export type RetryableStreamError = RetryableQwenStreamError & {
48
- upstreamCode?: string;
49
- forceNewChat?: boolean;
50
- retryWithFullPrompt?: boolean;
51
- switchAccount?: boolean;
52
- dropFiles?: boolean;
53
- };
54
-
55
- function errMessage(err: unknown): string {
56
- if (err instanceof Error) return err.message || "";
57
- return String(err ?? "");
58
- }
59
-
60
- function errCode(err: unknown): string {
61
- const anyErr = err as { upstreamCode?: unknown; code?: unknown };
62
- if (typeof anyErr?.upstreamCode === "string" && anyErr.upstreamCode) {
63
- return anyErr.upstreamCode;
64
- }
65
- if (typeof anyErr?.code === "string" && anyErr.code) {
66
- return anyErr.code;
67
- }
68
- return "";
69
- }
70
-
71
- function statusOf(err: unknown): number | undefined {
72
- const anyErr = err as { upstreamStatus?: unknown; statusCode?: unknown };
73
- if (typeof anyErr?.upstreamStatus === "number") return anyErr.upstreamStatus;
74
- if (typeof anyErr?.statusCode === "number") return anyErr.statusCode;
75
- return undefined;
76
- }
77
-
78
- /** Errors that belong to the proxy/client request itself — retrying is useless. */
79
- export function isTerminalLocalError(err: unknown): boolean {
80
- if (!err) return false;
81
-
82
- if (
83
- err instanceof ValidationError ||
84
- err instanceof AuthError ||
85
- err instanceof NotFoundError
86
- ) {
87
- return true;
88
- }
89
-
90
- const status = statusOf(err);
91
- const code = errCode(err).toLowerCase();
92
- const message = errMessage(err).toLowerCase();
93
-
94
- // Local proxy auth / validation / not found
95
- if (status === 400 || status === 401 || status === 404) {
96
- // Exception: Qwen upstream can also return 404 for missing chat — that is retryable.
97
- if (
98
- message.includes("qwen") ||
99
- message.includes("upstream") ||
100
- code.includes("not_found") ||
101
- message.includes("is not exist") ||
102
- message.includes("does not exist")
103
- ) {
104
- return false;
105
- }
106
- return true;
107
- }
108
-
109
- if (
110
- code === "invalid_api_key" ||
111
- code === "authentication_error" ||
112
- message.includes("missing or invalid authorization") ||
113
- message.includes("invalid api key") ||
114
- message.includes("messages is required") ||
115
- message.includes("at least one user message") ||
116
- message.includes("no qwen accounts configured")
117
- ) {
118
- return true;
119
- }
120
-
121
- // bad_request from Qwen upstream is NOT terminal — it's a corrupted chat
122
- // or invalid payload that can be recovered with a new chat + full prompt.
123
- // Only treat as terminal when it's clearly a local proxy validation error.
124
- if (code === "bad_request") {
125
- const isQwenUpstream =
126
- message.includes("qwen") ||
127
- message.includes("upstream") ||
128
- message.includes("invalid input") ||
129
- message.includes("first message must not") ||
130
- message.includes("entrada ou anexo");
131
- if (!isQwenUpstream) {
132
- return true;
133
- }
134
- }
135
-
136
- return false;
137
- }
138
-
139
- export function isClientAbortError(
140
- err: unknown,
141
- clientDisconnected = false,
142
- requestAborted = false,
143
- ): boolean {
144
- if (clientDisconnected || requestAborted) return true;
145
- // Our own client-abort markers: the client disconnected OR a same-session
146
- // retry superseded this request's lease during stream creation. A superseded
147
- // request must die silently — the newer request owns the session, and
148
- // retrying the old one resends full context on another account for nothing
149
- // (and can queue indefinitely behind the new stream's lease).
150
- if (err instanceof ClientAbortedError) return true;
151
- if (err instanceof Error && err.message.includes("client aborted")) return true;
152
- // Bare AbortError mid-stream is usually idle/upstream timeout (retryable).
153
- return false;
154
- }
155
-
156
- export function isInvalidInputError(err: unknown): boolean {
157
- // "Invalid input the chat X is not exist" is a chat-missing error, not attachment invalid.
158
- if (isChatNotExistError(err)) return false;
159
-
160
- const code = errCode(err).toLowerCase();
161
- const message = errMessage(err).toLowerCase();
162
- return (
163
- code === "invalid_input" ||
164
- message.includes("invalid_input") ||
165
- message.includes("entrada ou anexo inválido") ||
166
- message.includes("invalid input") ||
167
- message.includes("invalid attachment")
168
- );
169
- }
170
-
171
- /**
172
- * Qwen content-safety moderation rejections (data_inspection_failed).
173
- * These are deterministic: the same content will be rejected on any account,
174
- * so retrying or switching accounts only wastes resources and time.
175
- */
176
- export function isContentModerationError(err: unknown): boolean {
177
- const code = errCode(err).toLowerCase();
178
- const message = errMessage(err).toLowerCase();
179
- return (
180
- code === "data_inspection_failed" ||
181
- message.includes("data_inspection_failed") ||
182
- message.includes("conteúdo inadequado") ||
183
- message.includes("inappropriate content") ||
184
- message.includes("aviso de segurança do conteúdo") ||
185
- message.includes("content safety")
186
- );
187
- }
188
-
189
- /** Prefer a clean chat on the current account before paying the cost of replaying
190
- * the full context on another account. Callers keep their own per-request count. */
191
- export function shouldRetryInvalidInputOnSameAccount(
192
- reason: string,
193
- alreadyRetried: boolean,
194
- ): boolean {
195
- return (
196
- (reason === "invalid_input" || reason === "corrupted_chat_history") &&
197
- !alreadyRetried
198
- );
199
- }
200
-
201
- /** Keep TWO retries on the current account while an upstream generation
202
- * settles. The tool loop fires the next turn the instant the previous one
203
- * completes, and the upstream chat stays "in progress" for 2-4s after the
204
- * terminal event — a single ~1.2s retry often loses that settle race, and
205
- * escalating replays the FULL context on a cold account (~12s context reopen
206
- * + captcha). Rotate only after the second failure. */
207
- export function shouldRetryChatInProgressOnSameAccount(
208
- reason: string,
209
- alreadyRetriedCount: number,
210
- ): boolean {
211
- // Three same-chat retries: settle is usually 2-4s but was measured >6s after
212
- // huge turns, and the escalation alternative (full-context replay on a cold
213
- // account) is far more expensive than one more bounded wait.
214
- return reason === "chat_in_progress" && alreadyRetriedCount < 3;
215
- }
216
-
217
- export function isAccountInitializationError(err: unknown): boolean {
218
- const message = errMessage(err).toLowerCase();
219
- return (
220
- message.includes("header capture returned incomplete anti-fraud headers") ||
221
- message.includes("required qwen anti-fraud headers are unavailable") ||
222
- message.includes("playwright not initialized for account") ||
223
- message.includes("playwright page unavailable") ||
224
- message.includes("playwright page operation timed out") ||
225
- message.includes("playwright re-initialization timed out")
226
- );
227
- }
228
-
229
- export function isQuotaLikeError(err: unknown): boolean {
230
- // Chat-not-exist / invalid attachment must never look like quota.
231
- if (isChatNotExistError(err) || isInvalidInputError(err)) return false;
232
-
233
- const code = errCode(err).toLowerCase();
234
- const message = errMessage(err).toLowerCase();
235
-
236
- // Note: RetryableQwenStreamError inherits OpenAI-style code "rate_limit_exceeded".
237
- // Never treat that local code alone as quota — require message/upstream evidence.
238
- return (
239
- code === "quota_limit" ||
240
- code === "ratelimited" ||
241
- message.includes("quota_limit") ||
242
- message.includes("quota exceeded") ||
243
- message.includes("allocated quota") ||
244
- message.includes("token-limit") ||
245
- message.includes("insufficient quota") ||
246
- message.includes("alta demanda") ||
247
- message.includes("high demand") ||
248
- message.includes("request rate increased too quickly") ||
249
- message.includes("rate increased too quickly") ||
250
- message.includes("upper limit for today's usage") ||
251
- message.includes("you've reached the upper limit") ||
252
- // Accept local rate_limit code only when message also looks like quota/rate
253
- (code === "rate_limit_exceeded" &&
254
- (message.includes("quota") ||
255
- message.includes("rate") ||
256
- message.includes("limit") ||
257
- message.includes("demanda") ||
258
- message.includes("demand")))
259
- );
260
- }
261
-
262
- export function isAntiBotError(err: unknown): boolean {
263
- const code = errCode(err);
264
- const codeLower = code.toLowerCase();
265
- const message = errMessage(err).toLowerCase();
266
- if (err instanceof RetryableQwenStreamError) {
267
- return codeLower === "waf_challenge" || message.includes("anti-bot");
268
- }
269
- return (
270
- code === "FAIL_SYS_USER_VALIDATE" ||
271
- code === "RGV587_ERROR" ||
272
- codeLower === "waf_challenge" ||
273
- message.includes("fail_sys_user_validate") ||
274
- message.includes("rgv587_error") ||
275
- message.includes("_____tmd_____") ||
276
- message.includes("tmd anti-bot") ||
277
- message.includes("captcha") ||
278
- message.includes("security verification") ||
279
- message.includes("verify you are human") ||
280
- message.includes("human verification") ||
281
- message.includes("denyfromx5")
282
- );
283
- }
284
-
285
- function classifyQuotaCooldown(message: string): {
286
- accountCooldownMs?: number;
287
- accountCooldownReason: string;
288
- } {
289
- const lower = message.toLowerCase();
290
- const temporary =
291
- lower.includes("rate increased too quickly") ||
292
- lower.includes("request rate increased too quickly") ||
293
- lower.includes("alta demanda") ||
294
- lower.includes("high demand") ||
295
- lower.includes("tente novamente mais tarde") ||
296
- lower.includes("try again later");
297
-
298
- if (temporary) {
299
- return {
300
- accountCooldownMs: 2 * 60 * 1000,
301
- accountCooldownReason: "RateLimitTemporary",
302
- };
303
- }
304
-
305
- // REAL daily quota: the Qwen resets the account at the next UTC midnight
306
- // (verified against 2026-08-21 production log: every proxy `until` matched
307
- // the next 00:00 UTC exactly). The upstream "Wait about N hour(s)" hint is
308
- // ONLY accurate when the error lands mid-day; near midnight it rounds UP to
309
- // N≈24 while the real reset is minutes away (mzgns errored 23:37, hint
310
- // "23h", but the account was usable 23 minutes later). Trust the daily
311
- // reset, never the literal hint, never a blind 24h.
312
- return {
313
- accountCooldownMs: computeQuotaCooldownMs(Date.now()),
314
- accountCooldownReason: "RateLimited",
315
- };
316
- }
317
-
318
- /**
319
- * Milliseconds until the next UTC midnight plus a small safety margin. The
320
- * Qwen daily quota resets at 00:00 UTC, so this is the correct "when is this
321
- * account usable again" for a quota exhaust — regardless of what the upstream
322
- * "Wait about N hour(s)" hint guessed.
323
- */
324
- // Implemented in core/account-manager.ts (shared with the @[] fallback); kept
325
- // re-exporting here for callers that already import from retry-policy.
326
- export { computeQuotaCooldownMs } from "../../core/account-manager.ts";
327
-
328
- export function isChatNotExistError(err: unknown): boolean {
329
- const message = errMessage(err).toLowerCase();
330
- return (
331
- message.includes("is not exist") ||
332
- message.includes("not exist") ||
333
- message.includes("does not exist")
334
- );
335
- }
336
-
337
- export function isChatInProgressError(err: unknown): boolean {
338
- return errMessage(err).toLowerCase().includes("in progress");
339
- }
340
-
341
- /**
342
- * Qwen rejects a model the account cannot serve with Not_Found: Model not found.
343
- * This is deterministic per request — retrying on the same (or any) account
344
- * with the same model can never succeed, so it must terminate instead of
345
- * burning retry attempts / account cooldowns and ending in a misleading 502.
346
- */
347
- export function isModelNotFoundError(err: unknown): boolean {
348
- const code = errCode(err).toLowerCase();
349
- const message = errMessage(err).toLowerCase();
350
- return (
351
- (code === "not_found" &&
352
- message.includes("model") &&
353
- message.includes("not found")) ||
354
- message.includes("model not found")
355
- );
356
- }
357
-
358
- /**
359
- * Browser fetch and ReadableStream failures often arrive as plain Error
360
- * instances, especially when the stream is consumed outside Playwright.
361
- * Keep this matcher narrow so local programming errors are not retried as
362
- * account/network failures.
363
- */
364
- export function isNetworkLikeError(err: unknown): boolean {
365
- if (err instanceof QwenNetworkError) return true;
366
- const message = errMessage(err).toLowerCase();
367
- return (
368
- message === "network error" ||
369
- message.includes("failed to fetch") ||
370
- message.includes("fetch failed") ||
371
- message.includes("network connection was lost") ||
372
- message.includes("connection reset") ||
373
- message.includes("connection closed") ||
374
- message.includes("socket hang up") ||
375
- message.includes("econnreset") ||
376
- message.includes("econnrefused") ||
377
- message.includes("etimedout")
378
- );
379
- }
380
-
381
- /**
382
- * Corrupted chat history: Qwen rejects because the first message in the
383
- * upstream chat thread is an assistant message (broken parent_id chain).
384
- * Recovery: force new chat + resend full prompt + switch account.
385
- */
386
- export function isCorruptedChatHistoryError(err: unknown): boolean {
387
- const message = errMessage(err).toLowerCase();
388
- return (
389
- message.includes("first message must not") ||
390
- message.includes("first message must be") ||
391
- message.includes("must not assistant message") ||
392
- message.includes("must not be assistant")
393
- );
394
- }
395
-
396
- /**
397
- * Build a RetryAction with sane defaults so each classification branch only
398
- * spells out the fields it actually changes. Defaults: retryable, no account
399
- * switch, same chat, delta replay, no delay. Branch ordering below is
400
- * load-bearing (specific recoveries must win over broad substring matches).
401
- */
402
- function makeRetryAction(
403
- reason: string,
404
- overrides: Partial<Omit<RetryAction, "reason">> = {},
405
- ): RetryAction {
406
- return {
407
- retryable: true,
408
- switchAccount: false,
409
- forceNewChat: false,
410
- retryWithFullPrompt: false,
411
- retryAfterMs: 0,
412
- ...overrides,
413
- reason,
414
- };
415
- }
416
-
417
- /**
418
- * Generic recovery policy for create-stream + mid-stream failures.
419
- * Unknown upstream errors are retryable by default when enabled in config.
420
- */
421
- export function classifyRetryAction(
422
- err: unknown,
423
- options?: {
424
- clientDisconnected?: boolean;
425
- requestAborted?: boolean;
426
- baseDelayMs?: number;
427
- },
428
- ): RetryAction {
429
- const baseDelayMs = options?.baseDelayMs ?? config.retry.baseDelayMs;
430
- const unknownEnabled = config.retry.onUnknownUpstream !== false;
431
-
432
- if (
433
- isClientAbortError(
434
- err,
435
- options?.clientDisconnected === true,
436
- options?.requestAborted === true,
437
- )
438
- ) {
439
- return makeRetryAction("client_abort", { retryable: false });
440
- }
441
-
442
- if (isTerminalLocalError(err)) {
443
- return makeRetryAction("terminal_local", { retryable: false });
444
- }
445
-
446
- const message = errMessage(err).toLowerCase();
447
- const code = errCode(err).toLowerCase();
448
- if (isAccountInitializationError(err)) {
449
- return makeRetryAction("account_initialization_failed", {
450
- switchAccount: true,
451
- retryAfterMs: Math.min(baseDelayMs, 1_000),
452
- accountCooldownMs: config.concurrency.initFailureCooldownMs,
453
- accountCooldownReason: "AuthInitFailed",
454
- });
455
- }
456
-
457
- if (
458
- code === "account_busy" ||
459
- message.includes("waiting for a free slot") ||
460
- message.includes("busy: timed out")
461
- ) {
462
- return makeRetryAction("account_busy", {
463
- switchAccount: true,
464
- retryAfterMs: Math.min(baseDelayMs, 1_000),
465
- });
466
- }
467
-
468
- // Agent instructions ride ONLY the account-level personalization. An
469
- // unconfirmed sync means this account cannot serve the request as-is —
470
- // rotate to another account (each attempt re-syncs on its own account).
471
- if (err instanceof PersonalizationSyncError) {
472
- return makeRetryAction("personalization_sync_failed", {
473
- switchAccount: true,
474
- forceNewChat: true,
475
- retryAfterMs: baseDelayMs,
476
- });
477
- }
478
-
479
- // Specialized recoveries first (even if wrapped as RetryableQwenStreamError)
480
- // Corrupted chat history must win over broad "invalid input" matches.
481
- // Try a fresh chat on the SAME account first — the corruption is in the
482
- // upstream parent chain, not the account. Only rotate if the rebuild fails.
483
- if (isCorruptedChatHistoryError(err)) {
484
- return makeRetryAction("corrupted_chat_history", {
485
- forceNewChat: true,
486
- retryWithFullPrompt: true,
487
- });
488
- }
489
-
490
- // Chat missing must win over broad "invalid input" substring matches.
491
- if (isChatNotExistError(err) || isChatInProgressError(err)) {
492
- const typed = err as RetryableStreamError;
493
- const inProgress = isChatInProgressError(err);
494
- // chat_in_progress: do NOT switch immediately — the account is just
495
- // temporarily busy (escalation happens in tryCreateStreamWithRetry after
496
- // repeated failures). chat_not_exist needs a new chat + full replay;
497
- // in_progress retries the same chat first.
498
- return makeRetryAction(inProgress ? "chat_in_progress" : "chat_not_exist", {
499
- forceNewChat: !inProgress,
500
- retryWithFullPrompt: !inProgress,
501
- retryAfterMs: inProgress
502
- ? (typed.retryAfterMs ?? config.retry.chatInProgressDelayMs)
503
- : (typed.retryAfterMs ?? 0),
504
- });
505
- }
506
-
507
- if (isInvalidInputError(err)) {
508
- const typed = err as RetryableStreamError;
509
- return makeRetryAction("invalid_input", {
510
- switchAccount: typed.switchAccount !== false,
511
- forceNewChat: true,
512
- retryWithFullPrompt: true,
513
- retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
514
- dropFiles: typed.dropFiles,
515
- });
516
- }
517
-
518
- // Content moderation rejections are deterministic — retrying on any
519
- // account with the same content produces the same rejection. Fail fast
520
- // instead of burning through accounts, personalization syncs and captchas.
521
- if (isContentModerationError(err)) {
522
- return makeRetryAction("content_moderation", { retryable: false });
523
- }
524
-
525
- // Model not found is equally deterministic (the account cannot serve the
526
- // requested model). Fail fast with a clear error instead of retrying the
527
- // same doomed request and cooldown-marking accounts for ~5 hours.
528
- if (isModelNotFoundError(err)) {
529
- return makeRetryAction("model_not_found", { retryable: false });
530
- }
531
-
532
- if (isAntiBotError(err)) {
533
- // WAF/captcha is only identified here. Retry the same request on the
534
- // same account immediately; recovery, cooldown and account rotation are
535
- // intentionally left out so the failure path stays observable.
536
- return makeRetryAction("anti_bot");
537
- }
538
-
539
- if (isQuotaLikeError(err)) {
540
- const typed = err as RetryableStreamError;
541
- const quota = classifyQuotaCooldown(errMessage(err));
542
- const isTemporary = quota.accountCooldownReason === "RateLimitTemporary";
543
- // Temporary load shedding: retry same account first, only switch on
544
- // repeated failure. Real quota exhaustion: switch immediately.
545
- return makeRetryAction("quota_or_rate_limit", {
546
- switchAccount: isTemporary ? false : typed.switchAccount !== false,
547
- forceNewChat: typed.forceNewChat === true,
548
- retryWithFullPrompt: typed.retryWithFullPrompt === true,
549
- retryAfterMs: typed.retryAfterMs ?? (isTemporary ? 3_000 : baseDelayMs),
550
- accountCooldownMs: quota.accountCooldownMs,
551
- accountCooldownReason: quota.accountCooldownReason,
552
- });
553
- }
554
-
555
- if (
556
- isNetworkLikeError(err) ||
557
- err instanceof QwenUpstreamUnavailableError ||
558
- err instanceof QwenUpstreamError ||
559
- isAbortError(err)
560
- ) {
561
- const typed = err as RetryableStreamError;
562
- return makeRetryAction(
563
- isNetworkLikeError(err)
564
- ? "network"
565
- : err instanceof QwenUpstreamUnavailableError
566
- ? "upstream_unavailable"
567
- : isAbortError(err)
568
- ? "stream_aborted"
569
- : "upstream_error",
570
- {
571
- switchAccount: typed.switchAccount !== false,
572
- forceNewChat: true,
573
- retryWithFullPrompt: typed.retryWithFullPrompt === true,
574
- retryAfterMs:
575
- typed.retryAfterMs ??
576
- (isNetworkLikeError(err)
577
- ? 3000
578
- : err instanceof QwenUpstreamUnavailableError
579
- ? 2000
580
- : Math.min(baseDelayMs * 2, 3000)),
581
- },
582
- );
583
- }
584
-
585
- // Preserve explicit RetryableQwenStreamError flags for remaining cases
586
- if (err instanceof RetryableQwenStreamError) {
587
- const typed = err as RetryableStreamError;
588
- // Default switch unless caller explicitly set switchAccount=false
589
- return makeRetryAction("explicit_retryable", {
590
- switchAccount: typed.switchAccount !== false,
591
- forceNewChat: typed.forceNewChat === true,
592
- retryWithFullPrompt: typed.retryWithFullPrompt === true,
593
- retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
594
- });
595
- }
596
-
597
- // Default for unknown failures: retry when policy enabled
598
- if (unknownEnabled) {
599
- return makeRetryAction("unknown_upstream_default_retry", {
600
- switchAccount: true,
601
- forceNewChat: true,
602
- retryAfterMs: baseDelayMs,
603
- });
604
- }
605
-
606
- return makeRetryAction("unknown_not_retryable", { retryable: false });
607
- }
608
-
609
- /** Build a RetryableQwenStreamError for SSE/mid-stream failures with policy flags. */
610
- export function toRetryableStreamError(
611
- errCode: string,
612
- errDetails: string,
613
- options?: Partial<RetryAction>,
614
- ): RetryableStreamError {
615
- const policy = classifyRetryAction(
616
- Object.assign(new Error(`${errCode}: ${errDetails}`), {
617
- upstreamCode: errCode,
618
- }),
619
- );
620
- const merged: RetryAction = {
621
- ...policy,
622
- ...options,
623
- retryable: true,
624
- reason: options?.reason || policy.reason,
625
- };
626
-
627
- const error = new RetryableQwenStreamError(
628
- `Qwen retryable upstream error: ${errCode}: ${errDetails.substring(0, 200)}`,
629
- merged.retryAfterMs || config.retry.baseDelayMs,
630
- ) as RetryableStreamError;
631
-
632
- error.upstreamCode = errCode;
633
- error.forceNewChat = merged.forceNewChat;
634
- error.retryWithFullPrompt = merged.retryWithFullPrompt;
635
- error.switchAccount = merged.switchAccount;
636
- error.dropFiles = merged.dropFiles;
637
- return error;
638
- }
639
-
640
- /** For SSE error chunks: map any upstream SSE error to throw path. */
641
- export function throwFromSseUpstreamError(
642
- errCode: string,
643
- errDetails: string,
644
- ): never {
645
- const detailsLower = errDetails.toLowerCase();
646
- // Qwen sometimes labels the chat-state error as RateLimited. Normalize it
647
- // before retry/logging so it cannot be mistaken for account quota exhaustion.
648
- const normalizedErrCode =
649
- detailsLower.includes("chat is in progress") ||
650
- detailsLower.includes("the chat is in progress")
651
- ? "chat_in_progress"
652
- : errCode;
653
-
654
- // Log upstream errors. Expected retryable codes (quota, rate limit, chat
655
- // state) use warn level to avoid noisy stderr stack traces in production.
656
- const expectedCodes = new Set([
657
- "quota_limit",
658
- "rate_limit",
659
- "rate_limit_exceeded",
660
- "chat_in_progress",
661
- "invalid_input",
662
- "data_inspection_failed",
663
- ]);
664
- if (expectedCodes.has(normalizedErrCode.toLowerCase())) {
665
- logger.warn(
666
- `[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
667
- );
668
- } else {
669
- console.error(
670
- `[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
671
- );
672
- }
673
-
674
- // invalid_input keeps dedicated wording for logs/tests (not "chat is not exist")
675
- const isChatMissing =
676
- detailsLower.includes("is not exist") ||
677
- detailsLower.includes("does not exist") ||
678
- /\bnot exist\b/.test(detailsLower);
679
- if (
680
- !isChatMissing &&
681
- (errCode.toLowerCase() === "invalid_input" ||
682
- detailsLower.includes("entrada ou anexo inválido") ||
683
- detailsLower.includes("invalid input") ||
684
- detailsLower.includes("invalid attachment"))
685
- ) {
686
- logger.warn("[Upstream] invalid_input mid-stream detected", {
687
- code: errCode,
688
- detailsLength: errDetails.length,
689
- messageMentionsAttachment:
690
- detailsLower.includes("anexo") || detailsLower.includes("attachment"),
691
- messageMentionsFile:
692
- detailsLower.includes("file") || detailsLower.includes("arquivo"),
693
- });
694
-
695
- const error = new RetryableQwenStreamError(
696
- `Qwen retryable invalid input: ${errCode}: ${errDetails.substring(0, 200)}`,
697
- config.retry.baseDelayMs,
698
- ) as RetryableStreamError;
699
- error.upstreamCode = errCode;
700
- error.forceNewChat = true;
701
- error.retryWithFullPrompt = true;
702
- error.switchAccount = true;
703
- error.dropFiles = true; // Drop files on retry to isolate file-related errors
704
- throw error;
705
- }
706
-
707
- // Content moderation rejections are deterministic — the same content will
708
- // be rejected on every account. Throw as RetryableQwenStreamError so it
709
- // propagates through the streaming catch blocks, but classifyRetryAction
710
- // will mark it non-retryable.
711
- if (isContentModerationError({ upstreamCode: normalizedErrCode, message: errDetails })) {
712
- logger.warn(
713
- `[Upstream] Content moderation rejection (not retrying): ${normalizedErrCode}`,
714
- );
715
- const error = new RetryableQwenStreamError(
716
- `Qwen content moderation: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
717
- 0,
718
- ) as RetryableStreamError;
719
- error.upstreamCode = normalizedErrCode;
720
- error.switchAccount = false;
721
- throw error;
722
- }
723
-
724
- // A model the account cannot serve is a deterministic rejection too — never
725
- // transparently retrofit this doomed model request on the same/other account.
726
- if (isModelNotFoundError({ upstreamCode: normalizedErrCode, message: errDetails })) {
727
- logger.warn(
728
- `[Upstream] Model not available (not retrying): ${normalizedErrCode}`,
729
- );
730
- const error = new RetryableQwenStreamError(
731
- `Qwen model not found: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
732
- 0,
733
- ) as RetryableStreamError;
734
- error.upstreamCode = normalizedErrCode;
735
- error.switchAccount = false;
736
- throw error;
737
- }
738
-
739
- if (
740
- errDetails.includes("FAIL_SYS_USER_VALIDATE") ||
741
- errDetails.includes("RGV587_ERROR") ||
742
- errDetails.includes("user validate")
743
- ) {
744
- const error = new RetryableQwenStreamError(
745
- `Qwen anti-bot: ${errCode}: ${errDetails}`,
746
- 0,
747
- ) as RetryableStreamError;
748
- error.upstreamCode = errCode;
749
- error.switchAccount = true;
750
- throw error;
751
- }
752
-
753
- throw toRetryableStreamError(normalizedErrCode, errDetails);
754
- }
1
+ /*
2
+ * Generic upstream retry / account-switch policy.
3
+ *
4
+ * Default: retry + prefer another account for unknown/upstream failures.
5
+ * Stop only for a small denylist of terminal local errors.
6
+ */
7
+
8
+ import { config } from "../../core/config.ts";
9
+ import { computeQuotaCooldownMs } from "../../core/account-manager.ts";
10
+ import { logger } from "../../core/logger.ts";
11
+ import {
12
+ PersonalizationSyncError,
13
+ QwenNetworkError,
14
+ QwenUpstreamError,
15
+ QwenUpstreamUnavailableError,
16
+ RetryableQwenStreamError,
17
+ } from "../../services/qwen.ts";
18
+ import {
19
+ AuthError,
20
+ ClientAbortedError,
21
+ NotFoundError,
22
+ ValidationError,
23
+ } from "../../core/errors.ts";
24
+ import { isAbortError } from "./helpers.ts";
25
+
26
+ export type RetryAction = {
27
+ /** Outer/create-stream layer should retry this failure */
28
+ retryable: boolean;
29
+ /** Prefer switching to another account when available */
30
+ switchAccount: boolean;
31
+ /** Force a new Qwen chat on retry */
32
+ forceNewChat: boolean;
33
+ /** Resend full conversation context (not just delta) */
34
+ retryWithFullPrompt: boolean;
35
+ /** Drop attached files on retry (for invalid_input caused by bad attachments) */
36
+ dropFiles?: boolean;
37
+ /** Suggested delay before next attempt */
38
+ retryAfterMs: number;
39
+ /** Optional short cooldown for the failing account */
40
+ accountCooldownMs?: number;
41
+ /** Cooldown reason label */
42
+ accountCooldownReason?: string;
43
+ /** Why this action was chosen (logging/debug) */
44
+ reason: string;
45
+ };
46
+
47
+ export type RetryableStreamError = RetryableQwenStreamError & {
48
+ upstreamCode?: string;
49
+ forceNewChat?: boolean;
50
+ retryWithFullPrompt?: boolean;
51
+ switchAccount?: boolean;
52
+ dropFiles?: boolean;
53
+ };
54
+
55
+ function errMessage(err: unknown): string {
56
+ if (err instanceof Error) return err.message || "";
57
+ return String(err ?? "");
58
+ }
59
+
60
+ function errCode(err: unknown): string {
61
+ const anyErr = err as { upstreamCode?: unknown; code?: unknown };
62
+ if (typeof anyErr?.upstreamCode === "string" && anyErr.upstreamCode) {
63
+ return anyErr.upstreamCode;
64
+ }
65
+ if (typeof anyErr?.code === "string" && anyErr.code) {
66
+ return anyErr.code;
67
+ }
68
+ return "";
69
+ }
70
+
71
+ function statusOf(err: unknown): number | undefined {
72
+ const anyErr = err as { upstreamStatus?: unknown; statusCode?: unknown };
73
+ if (typeof anyErr?.upstreamStatus === "number") return anyErr.upstreamStatus;
74
+ if (typeof anyErr?.statusCode === "number") return anyErr.statusCode;
75
+ return undefined;
76
+ }
77
+
78
+ /** Errors that belong to the proxy/client request itself — retrying is useless. */
79
+ export function isTerminalLocalError(err: unknown): boolean {
80
+ if (!err) return false;
81
+
82
+ if (
83
+ err instanceof ValidationError ||
84
+ err instanceof AuthError ||
85
+ err instanceof NotFoundError
86
+ ) {
87
+ return true;
88
+ }
89
+
90
+ const status = statusOf(err);
91
+ const code = errCode(err).toLowerCase();
92
+ const message = errMessage(err).toLowerCase();
93
+
94
+ // Local proxy auth / validation / not found
95
+ if (status === 400 || status === 401 || status === 404) {
96
+ // Exception: Qwen upstream can also return 404 for missing chat — that is retryable.
97
+ if (
98
+ message.includes("qwen") ||
99
+ message.includes("upstream") ||
100
+ code.includes("not_found") ||
101
+ message.includes("is not exist") ||
102
+ message.includes("does not exist")
103
+ ) {
104
+ return false;
105
+ }
106
+ return true;
107
+ }
108
+
109
+ if (
110
+ code === "invalid_api_key" ||
111
+ code === "authentication_error" ||
112
+ message.includes("missing or invalid authorization") ||
113
+ message.includes("invalid api key") ||
114
+ message.includes("messages is required") ||
115
+ message.includes("at least one user message") ||
116
+ message.includes("no qwen accounts configured")
117
+ ) {
118
+ return true;
119
+ }
120
+
121
+ // bad_request from Qwen upstream is NOT terminal — it's a corrupted chat
122
+ // or invalid payload that can be recovered with a new chat + full prompt.
123
+ // Only treat as terminal when it's clearly a local proxy validation error.
124
+ if (code === "bad_request") {
125
+ const isQwenUpstream =
126
+ message.includes("qwen") ||
127
+ message.includes("upstream") ||
128
+ message.includes("invalid input") ||
129
+ message.includes("first message must not") ||
130
+ message.includes("entrada ou anexo");
131
+ if (!isQwenUpstream) {
132
+ return true;
133
+ }
134
+ }
135
+
136
+ return false;
137
+ }
138
+
139
+ export function isClientAbortError(
140
+ err: unknown,
141
+ clientDisconnected = false,
142
+ requestAborted = false,
143
+ ): boolean {
144
+ if (clientDisconnected || requestAborted) return true;
145
+ // Our own client-abort markers: the client disconnected OR a same-session
146
+ // retry superseded this request's lease during stream creation. A superseded
147
+ // request must die silently — the newer request owns the session, and
148
+ // retrying the old one resends full context on another account for nothing
149
+ // (and can queue indefinitely behind the new stream's lease).
150
+ if (err instanceof ClientAbortedError) return true;
151
+ if (err instanceof Error && err.message.includes("client aborted")) return true;
152
+ // Bare AbortError mid-stream is usually idle/upstream timeout (retryable).
153
+ return false;
154
+ }
155
+
156
+ export function isInvalidInputError(err: unknown): boolean {
157
+ // "Invalid input the chat X is not exist" is a chat-missing error, not attachment invalid.
158
+ if (isChatNotExistError(err)) return false;
159
+
160
+ const code = errCode(err).toLowerCase();
161
+ const message = errMessage(err).toLowerCase();
162
+ return (
163
+ code === "invalid_input" ||
164
+ message.includes("invalid_input") ||
165
+ message.includes("entrada ou anexo inválido") ||
166
+ message.includes("invalid input") ||
167
+ message.includes("invalid attachment")
168
+ );
169
+ }
170
+
171
+ /**
172
+ * Qwen content-safety moderation rejections (data_inspection_failed).
173
+ * These are deterministic: the same content will be rejected on any account,
174
+ * so retrying or switching accounts only wastes resources and time.
175
+ */
176
+ export function isContentModerationError(err: unknown): boolean {
177
+ const code = errCode(err).toLowerCase();
178
+ const message = errMessage(err).toLowerCase();
179
+ return (
180
+ code === "data_inspection_failed" ||
181
+ message.includes("data_inspection_failed") ||
182
+ message.includes("conteúdo inadequado") ||
183
+ message.includes("inappropriate content") ||
184
+ message.includes("aviso de segurança do conteúdo") ||
185
+ message.includes("content safety")
186
+ );
187
+ }
188
+
189
+ /** Prefer a clean chat on the current account before paying the cost of replaying
190
+ * the full context on another account. Callers keep their own per-request count. */
191
+ export function shouldRetryInvalidInputOnSameAccount(
192
+ reason: string,
193
+ alreadyRetried: boolean,
194
+ ): boolean {
195
+ return (
196
+ (reason === "invalid_input" || reason === "corrupted_chat_history") &&
197
+ !alreadyRetried
198
+ );
199
+ }
200
+
201
+ /** Keep TWO retries on the current account while an upstream generation
202
+ * settles. The tool loop fires the next turn the instant the previous one
203
+ * completes, and the upstream chat stays "in progress" for 2-4s after the
204
+ * terminal event — a single ~1.2s retry often loses that settle race, and
205
+ * escalating replays the FULL context on a cold account (~12s context reopen
206
+ * + captcha). Rotate only after the second failure. */
207
+ export function shouldRetryChatInProgressOnSameAccount(
208
+ reason: string,
209
+ alreadyRetriedCount: number,
210
+ ): boolean {
211
+ // Three same-chat retries: settle is usually 2-4s but was measured >6s after
212
+ // huge turns, and the escalation alternative (full-context replay on a cold
213
+ // account) is far more expensive than one more bounded wait.
214
+ return reason === "chat_in_progress" && alreadyRetriedCount < 3;
215
+ }
216
+
217
+ export function isAccountInitializationError(err: unknown): boolean {
218
+ const message = errMessage(err).toLowerCase();
219
+ return (
220
+ message.includes("header capture returned incomplete anti-fraud headers") ||
221
+ message.includes("required qwen anti-fraud headers are unavailable") ||
222
+ message.includes("playwright not initialized for account") ||
223
+ message.includes("playwright page unavailable") ||
224
+ message.includes("playwright page operation timed out") ||
225
+ message.includes("playwright re-initialization timed out")
226
+ );
227
+ }
228
+
229
+ export function isQuotaLikeError(err: unknown): boolean {
230
+ // Chat-not-exist / invalid attachment must never look like quota.
231
+ if (isChatNotExistError(err) || isInvalidInputError(err)) return false;
232
+
233
+ const code = errCode(err).toLowerCase();
234
+ const message = errMessage(err).toLowerCase();
235
+
236
+ // Note: RetryableQwenStreamError inherits OpenAI-style code "rate_limit_exceeded".
237
+ // Never treat that local code alone as quota — require message/upstream evidence.
238
+ return (
239
+ code === "quota_limit" ||
240
+ code === "ratelimited" ||
241
+ message.includes("quota_limit") ||
242
+ message.includes("quota exceeded") ||
243
+ message.includes("allocated quota") ||
244
+ message.includes("token-limit") ||
245
+ message.includes("insufficient quota") ||
246
+ message.includes("alta demanda") ||
247
+ message.includes("high demand") ||
248
+ message.includes("request rate increased too quickly") ||
249
+ message.includes("rate increased too quickly") ||
250
+ message.includes("upper limit for today's usage") ||
251
+ message.includes("you've reached the upper limit") ||
252
+ // Accept local rate_limit code only when message also looks like quota/rate
253
+ (code === "rate_limit_exceeded" &&
254
+ (message.includes("quota") ||
255
+ message.includes("rate") ||
256
+ message.includes("limit") ||
257
+ message.includes("demanda") ||
258
+ message.includes("demand")))
259
+ );
260
+ }
261
+
262
+ export function isAntiBotError(err: unknown): boolean {
263
+ const code = errCode(err);
264
+ const codeLower = code.toLowerCase();
265
+ const message = errMessage(err).toLowerCase();
266
+ if (err instanceof RetryableQwenStreamError) {
267
+ return codeLower === "waf_challenge" || message.includes("anti-bot");
268
+ }
269
+ return (
270
+ code === "FAIL_SYS_USER_VALIDATE" ||
271
+ code === "RGV587_ERROR" ||
272
+ codeLower === "waf_challenge" ||
273
+ message.includes("fail_sys_user_validate") ||
274
+ message.includes("rgv587_error") ||
275
+ message.includes("_____tmd_____") ||
276
+ message.includes("tmd anti-bot") ||
277
+ message.includes("captcha") ||
278
+ message.includes("security verification") ||
279
+ message.includes("verify you are human") ||
280
+ message.includes("human verification") ||
281
+ message.includes("denyfromx5")
282
+ );
283
+ }
284
+
285
+ function classifyQuotaCooldown(message: string): {
286
+ accountCooldownMs?: number;
287
+ accountCooldownReason: string;
288
+ } {
289
+ const lower = message.toLowerCase();
290
+ const temporary =
291
+ lower.includes("rate increased too quickly") ||
292
+ lower.includes("request rate increased too quickly") ||
293
+ lower.includes("alta demanda") ||
294
+ lower.includes("high demand") ||
295
+ lower.includes("tente novamente mais tarde") ||
296
+ lower.includes("try again later");
297
+
298
+ if (temporary) {
299
+ return {
300
+ accountCooldownMs: 2 * 60 * 1000,
301
+ accountCooldownReason: "RateLimitTemporary",
302
+ };
303
+ }
304
+
305
+ // REAL daily quota: the Qwen resets the account at the next UTC midnight
306
+ // (verified against 2026-08-21 production log: every proxy `until` matched
307
+ // the next 00:00 UTC exactly). The upstream "Wait about N hour(s)" hint is
308
+ // ONLY accurate when the error lands mid-day; near midnight it rounds UP to
309
+ // N≈24 while the real reset is minutes away (mzgns errored 23:37, hint
310
+ // "23h", but the account was usable 23 minutes later). Trust the daily
311
+ // reset, never the literal hint, never a blind 24h.
312
+ return {
313
+ accountCooldownMs: computeQuotaCooldownMs(Date.now()),
314
+ accountCooldownReason: "RateLimited",
315
+ };
316
+ }
317
+
318
+ /**
319
+ * Milliseconds until the next UTC midnight plus a small safety margin. The
320
+ * Qwen daily quota resets at 00:00 UTC, so this is the correct "when is this
321
+ * account usable again" for a quota exhaust — regardless of what the upstream
322
+ * "Wait about N hour(s)" hint guessed.
323
+ */
324
+ // Implemented in core/account-manager.ts (shared with the @[] fallback); kept
325
+ // re-exporting here for callers that already import from retry-policy.
326
+ export { computeQuotaCooldownMs } from "../../core/account-manager.ts";
327
+
328
+ export function isChatNotExistError(err: unknown): boolean {
329
+ const message = errMessage(err).toLowerCase();
330
+ return (
331
+ message.includes("is not exist") ||
332
+ message.includes("not exist") ||
333
+ message.includes("does not exist")
334
+ );
335
+ }
336
+
337
+ export function isChatInProgressError(err: unknown): boolean {
338
+ return errMessage(err).toLowerCase().includes("in progress");
339
+ }
340
+
341
+ /**
342
+ * Qwen rejects a model the account cannot serve with Not_Found: Model not found.
343
+ * This is deterministic per request — retrying on the same (or any) account
344
+ * with the same model can never succeed, so it must terminate instead of
345
+ * burning retry attempts / account cooldowns and ending in a misleading 502.
346
+ */
347
+ export function isModelNotFoundError(err: unknown): boolean {
348
+ const code = errCode(err).toLowerCase();
349
+ const message = errMessage(err).toLowerCase();
350
+ return (
351
+ (code === "not_found" &&
352
+ message.includes("model") &&
353
+ message.includes("not found")) ||
354
+ message.includes("model not found")
355
+ );
356
+ }
357
+
358
+ /**
359
+ * Browser fetch and ReadableStream failures often arrive as plain Error
360
+ * instances, especially when the stream is consumed outside Playwright.
361
+ * Keep this matcher narrow so local programming errors are not retried as
362
+ * account/network failures.
363
+ */
364
+ export function isNetworkLikeError(err: unknown): boolean {
365
+ if (err instanceof QwenNetworkError) return true;
366
+ const message = errMessage(err).toLowerCase();
367
+ return (
368
+ message === "network error" ||
369
+ message.includes("failed to fetch") ||
370
+ message.includes("fetch failed") ||
371
+ message.includes("network connection was lost") ||
372
+ message.includes("connection reset") ||
373
+ message.includes("connection closed") ||
374
+ message.includes("socket hang up") ||
375
+ message.includes("econnreset") ||
376
+ message.includes("econnrefused") ||
377
+ message.includes("etimedout")
378
+ );
379
+ }
380
+
381
+ /**
382
+ * Corrupted chat history: Qwen rejects because the first message in the
383
+ * upstream chat thread is an assistant message (broken parent_id chain).
384
+ * Recovery: force new chat + resend full prompt + switch account.
385
+ */
386
+ export function isCorruptedChatHistoryError(err: unknown): boolean {
387
+ const message = errMessage(err).toLowerCase();
388
+ return (
389
+ message.includes("first message must not") ||
390
+ message.includes("first message must be") ||
391
+ message.includes("must not assistant message") ||
392
+ message.includes("must not be assistant")
393
+ );
394
+ }
395
+
396
+ /**
397
+ * Build a RetryAction with sane defaults so each classification branch only
398
+ * spells out the fields it actually changes. Defaults: retryable, no account
399
+ * switch, same chat, delta replay, no delay. Branch ordering below is
400
+ * load-bearing (specific recoveries must win over broad substring matches).
401
+ */
402
+ function makeRetryAction(
403
+ reason: string,
404
+ overrides: Partial<Omit<RetryAction, "reason">> = {},
405
+ ): RetryAction {
406
+ return {
407
+ retryable: true,
408
+ switchAccount: false,
409
+ forceNewChat: false,
410
+ retryWithFullPrompt: false,
411
+ retryAfterMs: 0,
412
+ ...overrides,
413
+ reason,
414
+ };
415
+ }
416
+
417
+ /**
418
+ * Generic recovery policy for create-stream + mid-stream failures.
419
+ * Unknown upstream errors are retryable by default when enabled in config.
420
+ */
421
+ export function classifyRetryAction(
422
+ err: unknown,
423
+ options?: {
424
+ clientDisconnected?: boolean;
425
+ requestAborted?: boolean;
426
+ baseDelayMs?: number;
427
+ },
428
+ ): RetryAction {
429
+ const baseDelayMs = options?.baseDelayMs ?? config.retry.baseDelayMs;
430
+ const unknownEnabled = config.retry.onUnknownUpstream !== false;
431
+
432
+ if (
433
+ isClientAbortError(
434
+ err,
435
+ options?.clientDisconnected === true,
436
+ options?.requestAborted === true,
437
+ )
438
+ ) {
439
+ return makeRetryAction("client_abort", { retryable: false });
440
+ }
441
+
442
+ if (isTerminalLocalError(err)) {
443
+ return makeRetryAction("terminal_local", { retryable: false });
444
+ }
445
+
446
+ const message = errMessage(err).toLowerCase();
447
+ const code = errCode(err).toLowerCase();
448
+ if (isAccountInitializationError(err)) {
449
+ return makeRetryAction("account_initialization_failed", {
450
+ switchAccount: true,
451
+ retryAfterMs: Math.min(baseDelayMs, 1_000),
452
+ accountCooldownMs: config.concurrency.initFailureCooldownMs,
453
+ accountCooldownReason: "AuthInitFailed",
454
+ });
455
+ }
456
+
457
+ if (
458
+ code === "account_busy" ||
459
+ message.includes("waiting for a free slot") ||
460
+ message.includes("busy: timed out")
461
+ ) {
462
+ return makeRetryAction("account_busy", {
463
+ switchAccount: true,
464
+ retryAfterMs: Math.min(baseDelayMs, 1_000),
465
+ });
466
+ }
467
+
468
+ // Agent instructions ride ONLY the account-level personalization. An
469
+ // unconfirmed sync means this account cannot serve the request as-is —
470
+ // rotate to another account (each attempt re-syncs on its own account).
471
+ if (err instanceof PersonalizationSyncError) {
472
+ return makeRetryAction("personalization_sync_failed", {
473
+ switchAccount: true,
474
+ forceNewChat: true,
475
+ retryAfterMs: baseDelayMs,
476
+ });
477
+ }
478
+
479
+ // Specialized recoveries first (even if wrapped as RetryableQwenStreamError)
480
+ // Corrupted chat history must win over broad "invalid input" matches.
481
+ // Try a fresh chat on the SAME account first — the corruption is in the
482
+ // upstream parent chain, not the account. Only rotate if the rebuild fails.
483
+ if (isCorruptedChatHistoryError(err)) {
484
+ return makeRetryAction("corrupted_chat_history", {
485
+ forceNewChat: true,
486
+ retryWithFullPrompt: true,
487
+ });
488
+ }
489
+
490
+ // Chat missing must win over broad "invalid input" substring matches.
491
+ if (isChatNotExistError(err) || isChatInProgressError(err)) {
492
+ const typed = err as RetryableStreamError;
493
+ const inProgress = isChatInProgressError(err);
494
+ // chat_in_progress: do NOT switch immediately — the account is just
495
+ // temporarily busy (escalation happens in tryCreateStreamWithRetry after
496
+ // repeated failures). chat_not_exist needs a new chat + full replay;
497
+ // in_progress retries the same chat first.
498
+ return makeRetryAction(inProgress ? "chat_in_progress" : "chat_not_exist", {
499
+ forceNewChat: !inProgress,
500
+ retryWithFullPrompt: !inProgress,
501
+ retryAfterMs: inProgress
502
+ ? (typed.retryAfterMs ?? config.retry.chatInProgressDelayMs)
503
+ : (typed.retryAfterMs ?? 0),
504
+ });
505
+ }
506
+
507
+ if (isInvalidInputError(err)) {
508
+ const typed = err as RetryableStreamError;
509
+ return makeRetryAction("invalid_input", {
510
+ switchAccount: typed.switchAccount !== false,
511
+ forceNewChat: true,
512
+ retryWithFullPrompt: true,
513
+ retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
514
+ dropFiles: typed.dropFiles,
515
+ });
516
+ }
517
+
518
+ // Content moderation rejections are deterministic — retrying on any
519
+ // account with the same content produces the same rejection. Fail fast
520
+ // instead of burning through accounts, personalization syncs and captchas.
521
+ if (isContentModerationError(err)) {
522
+ return makeRetryAction("content_moderation", { retryable: false });
523
+ }
524
+
525
+ // Model not found is equally deterministic (the account cannot serve the
526
+ // requested model). Fail fast with a clear error instead of retrying the
527
+ // same doomed request and cooldown-marking accounts for ~5 hours.
528
+ if (isModelNotFoundError(err)) {
529
+ return makeRetryAction("model_not_found", { retryable: false });
530
+ }
531
+
532
+ if (isAntiBotError(err)) {
533
+ // WAF/captcha is only identified here. Retry the same request on the
534
+ // same account immediately; recovery, cooldown and account rotation are
535
+ // intentionally left out so the failure path stays observable.
536
+ return makeRetryAction("anti_bot");
537
+ }
538
+
539
+ if (isQuotaLikeError(err)) {
540
+ const typed = err as RetryableStreamError;
541
+ const quota = classifyQuotaCooldown(errMessage(err));
542
+ const isTemporary = quota.accountCooldownReason === "RateLimitTemporary";
543
+ // Temporary load shedding: retry same account first, only switch on
544
+ // repeated failure. Real quota exhaustion: switch immediately.
545
+ return makeRetryAction("quota_or_rate_limit", {
546
+ switchAccount: isTemporary ? false : typed.switchAccount !== false,
547
+ forceNewChat: typed.forceNewChat === true,
548
+ retryWithFullPrompt: typed.retryWithFullPrompt === true,
549
+ retryAfterMs: typed.retryAfterMs ?? (isTemporary ? 3_000 : baseDelayMs),
550
+ accountCooldownMs: quota.accountCooldownMs,
551
+ accountCooldownReason: quota.accountCooldownReason,
552
+ });
553
+ }
554
+
555
+ if (
556
+ isNetworkLikeError(err) ||
557
+ err instanceof QwenUpstreamUnavailableError ||
558
+ err instanceof QwenUpstreamError ||
559
+ isAbortError(err)
560
+ ) {
561
+ const typed = err as RetryableStreamError;
562
+ return makeRetryAction(
563
+ isNetworkLikeError(err)
564
+ ? "network"
565
+ : err instanceof QwenUpstreamUnavailableError
566
+ ? "upstream_unavailable"
567
+ : isAbortError(err)
568
+ ? "stream_aborted"
569
+ : "upstream_error",
570
+ {
571
+ switchAccount: typed.switchAccount !== false,
572
+ forceNewChat: true,
573
+ retryWithFullPrompt: typed.retryWithFullPrompt === true,
574
+ retryAfterMs:
575
+ typed.retryAfterMs ??
576
+ (isNetworkLikeError(err)
577
+ ? 3000
578
+ : err instanceof QwenUpstreamUnavailableError
579
+ ? 2000
580
+ : Math.min(baseDelayMs * 2, 3000)),
581
+ },
582
+ );
583
+ }
584
+
585
+ // Preserve explicit RetryableQwenStreamError flags for remaining cases
586
+ if (err instanceof RetryableQwenStreamError) {
587
+ const typed = err as RetryableStreamError;
588
+ // Default switch unless caller explicitly set switchAccount=false
589
+ return makeRetryAction("explicit_retryable", {
590
+ switchAccount: typed.switchAccount !== false,
591
+ forceNewChat: typed.forceNewChat === true,
592
+ retryWithFullPrompt: typed.retryWithFullPrompt === true,
593
+ retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
594
+ });
595
+ }
596
+
597
+ // Default for unknown failures: retry when policy enabled
598
+ if (unknownEnabled) {
599
+ return makeRetryAction("unknown_upstream_default_retry", {
600
+ switchAccount: true,
601
+ forceNewChat: true,
602
+ retryAfterMs: baseDelayMs,
603
+ });
604
+ }
605
+
606
+ return makeRetryAction("unknown_not_retryable", { retryable: false });
607
+ }
608
+
609
+ /** Build a RetryableQwenStreamError for SSE/mid-stream failures with policy flags. */
610
+ export function toRetryableStreamError(
611
+ errCode: string,
612
+ errDetails: string,
613
+ options?: Partial<RetryAction>,
614
+ ): RetryableStreamError {
615
+ const policy = classifyRetryAction(
616
+ Object.assign(new Error(`${errCode}: ${errDetails}`), {
617
+ upstreamCode: errCode,
618
+ }),
619
+ );
620
+ const merged: RetryAction = {
621
+ ...policy,
622
+ ...options,
623
+ retryable: true,
624
+ reason: options?.reason || policy.reason,
625
+ };
626
+
627
+ const error = new RetryableQwenStreamError(
628
+ `Qwen retryable upstream error: ${errCode}: ${errDetails.substring(0, 200)}`,
629
+ merged.retryAfterMs || config.retry.baseDelayMs,
630
+ ) as RetryableStreamError;
631
+
632
+ error.upstreamCode = errCode;
633
+ error.forceNewChat = merged.forceNewChat;
634
+ error.retryWithFullPrompt = merged.retryWithFullPrompt;
635
+ error.switchAccount = merged.switchAccount;
636
+ error.dropFiles = merged.dropFiles;
637
+ return error;
638
+ }
639
+
640
+ /** For SSE error chunks: map any upstream SSE error to throw path. */
641
+ export function throwFromSseUpstreamError(
642
+ errCode: string,
643
+ errDetails: string,
644
+ ): never {
645
+ const detailsLower = errDetails.toLowerCase();
646
+ // Qwen sometimes labels the chat-state error as RateLimited. Normalize it
647
+ // before retry/logging so it cannot be mistaken for account quota exhaustion.
648
+ const normalizedErrCode =
649
+ detailsLower.includes("chat is in progress") ||
650
+ detailsLower.includes("the chat is in progress")
651
+ ? "chat_in_progress"
652
+ : errCode;
653
+
654
+ // Log upstream errors. Expected retryable codes (quota, rate limit, chat
655
+ // state) use warn level to avoid noisy stderr stack traces in production.
656
+ const expectedCodes = new Set([
657
+ "quota_limit",
658
+ "rate_limit",
659
+ "rate_limit_exceeded",
660
+ "chat_in_progress",
661
+ "invalid_input",
662
+ "data_inspection_failed",
663
+ ]);
664
+ if (expectedCodes.has(normalizedErrCode.toLowerCase())) {
665
+ logger.warn(
666
+ `[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
667
+ );
668
+ } else {
669
+ console.error(
670
+ `[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
671
+ );
672
+ }
673
+
674
+ // invalid_input keeps dedicated wording for logs/tests (not "chat is not exist")
675
+ const isChatMissing =
676
+ detailsLower.includes("is not exist") ||
677
+ detailsLower.includes("does not exist") ||
678
+ /\bnot exist\b/.test(detailsLower);
679
+ if (
680
+ !isChatMissing &&
681
+ (errCode.toLowerCase() === "invalid_input" ||
682
+ detailsLower.includes("entrada ou anexo inválido") ||
683
+ detailsLower.includes("invalid input") ||
684
+ detailsLower.includes("invalid attachment"))
685
+ ) {
686
+ logger.warn("[Upstream] invalid_input mid-stream detected", {
687
+ code: errCode,
688
+ detailsLength: errDetails.length,
689
+ messageMentionsAttachment:
690
+ detailsLower.includes("anexo") || detailsLower.includes("attachment"),
691
+ messageMentionsFile:
692
+ detailsLower.includes("file") || detailsLower.includes("arquivo"),
693
+ });
694
+
695
+ const error = new RetryableQwenStreamError(
696
+ `Qwen retryable invalid input: ${errCode}: ${errDetails.substring(0, 200)}`,
697
+ config.retry.baseDelayMs,
698
+ ) as RetryableStreamError;
699
+ error.upstreamCode = errCode;
700
+ error.forceNewChat = true;
701
+ error.retryWithFullPrompt = true;
702
+ error.switchAccount = true;
703
+ error.dropFiles = true; // Drop files on retry to isolate file-related errors
704
+ throw error;
705
+ }
706
+
707
+ // Content moderation rejections are deterministic — the same content will
708
+ // be rejected on every account. Throw as RetryableQwenStreamError so it
709
+ // propagates through the streaming catch blocks, but classifyRetryAction
710
+ // will mark it non-retryable.
711
+ if (isContentModerationError({ upstreamCode: normalizedErrCode, message: errDetails })) {
712
+ logger.warn(
713
+ `[Upstream] Content moderation rejection (not retrying): ${normalizedErrCode}`,
714
+ );
715
+ const error = new RetryableQwenStreamError(
716
+ `Qwen content moderation: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
717
+ 0,
718
+ ) as RetryableStreamError;
719
+ error.upstreamCode = normalizedErrCode;
720
+ error.switchAccount = false;
721
+ throw error;
722
+ }
723
+
724
+ // A model the account cannot serve is a deterministic rejection too — never
725
+ // transparently retrofit this doomed model request on the same/other account.
726
+ if (isModelNotFoundError({ upstreamCode: normalizedErrCode, message: errDetails })) {
727
+ logger.warn(
728
+ `[Upstream] Model not available (not retrying): ${normalizedErrCode}`,
729
+ );
730
+ const error = new RetryableQwenStreamError(
731
+ `Qwen model not found: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
732
+ 0,
733
+ ) as RetryableStreamError;
734
+ error.upstreamCode = normalizedErrCode;
735
+ error.switchAccount = false;
736
+ throw error;
737
+ }
738
+
739
+ if (
740
+ errDetails.includes("FAIL_SYS_USER_VALIDATE") ||
741
+ errDetails.includes("RGV587_ERROR") ||
742
+ errDetails.includes("user validate")
743
+ ) {
744
+ const error = new RetryableQwenStreamError(
745
+ `Qwen anti-bot: ${errCode}: ${errDetails}`,
746
+ 0,
747
+ ) as RetryableStreamError;
748
+ error.upstreamCode = errCode;
749
+ error.switchAccount = true;
750
+ throw error;
751
+ }
752
+
753
+ throw toRetryableStreamError(normalizedErrCode, errDetails);
754
+ }