qwenproxy-cli 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -14
- package/README.md +906 -906
- package/bin/qwenproxy.js +5 -1
- package/package.json +77 -78
- package/src/api/error-classifier.ts +159 -159
- package/src/api/error-helpers.ts +118 -118
- package/src/api/models.ts +261 -261
- package/src/api/server.ts +860 -859
- package/src/cache/memory-cache.ts +385 -385
- package/src/clean-cache.ts +204 -204
- package/src/core/account-concurrency.ts +671 -671
- package/src/core/account-manager.ts +301 -297
- package/src/core/account-priority.ts +163 -163
- package/src/core/accounts.ts +186 -186
- package/src/core/config.ts +383 -383
- package/src/core/crypto-utils.ts +79 -79
- package/src/core/database.ts +276 -276
- package/src/core/errors.ts +118 -118
- package/src/core/logger.ts +269 -269
- package/src/core/memory-usage.ts +84 -84
- package/src/core/metrics.ts +291 -291
- package/src/core/model-alias.ts +77 -77
- package/src/core/model-registry.ts +544 -544
- package/src/core/mutex.ts +119 -119
- package/src/core/paths.ts +199 -199
- package/src/core/prompt-limits.ts +214 -214
- package/src/core/reasoning-effort.ts +102 -102
- package/src/core/stream-registry.ts +96 -96
- package/src/core/waf-isolation.ts +117 -117
- package/src/core/watchdog.ts +195 -195
- package/src/delete-chats.ts +23 -23
- package/src/index.ts +65 -64
- package/src/login.ts +147 -147
- package/src/reset-cooldowns.ts +11 -11
- package/src/routes/anthropic/index.ts +355 -355
- package/src/routes/anthropic/translate.ts +522 -522
- package/src/routes/anthropic/types.ts +154 -154
- package/src/routes/anthropic/validation.ts +144 -144
- package/src/routes/chat/account.ts +1817 -1817
- package/src/routes/chat/context.ts +241 -241
- package/src/routes/chat/errors.ts +85 -85
- package/src/routes/chat/helpers.ts +268 -268
- package/src/routes/chat/index.ts +618 -618
- package/src/routes/chat/media.ts +285 -285
- package/src/routes/chat/retry-policy.ts +754 -754
- package/src/routes/chat/stop.ts +98 -98
- package/src/routes/chat/streaming.ts +2710 -2710
- package/src/routes/chat/validation.ts +526 -526
- package/src/routes/chat.ts +2 -2
- package/src/routes/completions.ts +290 -290
- package/src/routes/images.ts +139 -139
- package/src/routes/responses/adapter.ts +503 -503
- package/src/routes/responses/index.ts +405 -405
- package/src/routes/responses/state.ts +230 -230
- package/src/routes/responses/streaming.ts +528 -528
- package/src/routes/responses/types.ts +285 -285
- package/src/routes/responses/validation.ts +202 -202
- package/src/routes/upload.ts +731 -731
- package/src/routes/videos.ts +214 -214
- package/src/services/auth-playwright.ts +173 -173
- package/src/services/captcha-coordinator.ts +161 -161
- package/src/services/captcha-solver.ts +553 -553
- package/src/services/chat-cleanup.ts +80 -80
- package/src/services/context-meter.ts +317 -317
- package/src/services/fingerprint.ts +242 -242
- package/src/services/human-behavior.ts +173 -173
- package/src/services/media-generation.ts +1748 -1748
- package/src/services/playwright.ts +2878 -2800
- package/src/services/qwen-chat-pool.ts +345 -345
- package/src/services/qwen-errors.ts +133 -133
- package/src/services/qwen-headers.ts +79 -79
- package/src/services/qwen-thread-state.ts +393 -393
- package/src/services/qwen-url.ts +19 -19
- package/src/services/qwen.ts +3126 -3126
- package/src/services/session-keeper.ts +88 -88
- package/src/services/token-estimation-metrics.ts +118 -118
- package/src/sync/claude-code.ts +75 -75
- package/src/sync/codex.ts +123 -123
- package/src/sync/index.ts +362 -362
- package/src/sync/omp.ts +105 -105
- package/src/sync/opencode.ts +214 -214
- package/src/sync/types.ts +53 -53
- package/src/sync/utils.ts +27 -27
- package/src/sync-clients.ts +189 -189
- package/src/tools/instructions.ts +137 -137
- package/src/tools/manifest.ts +81 -81
- package/src/tools/parser.ts +2989 -2989
- package/src/tools/toolcall-tags.ts +142 -142
- package/src/tui/app.ts +259 -264
- package/src/tui/index.ts +61 -61
- package/src/tui/markdown.ts +258 -258
- package/src/tui/proxy-client.ts +331 -326
- package/src/tui/screen.ts +294 -278
- package/src/tui/server-manager.ts +270 -270
- package/src/tui/theme.ts +432 -432
- package/src/tui/types.ts +33 -33
- package/src/tui/views/accounts-view.ts +656 -656
- package/src/tui/views/chat-view.ts +1018 -823
- package/src/tui/views/logs-view.ts +479 -413
- package/src/tui/views/status-view.ts +204 -204
- package/src/tui/views/storage-view.ts +304 -291
- package/src/tui/views/sync-view.ts +409 -409
- package/src/types/ali-oss.d.ts +32 -32
- package/src/update-cli.ts +121 -0
- package/src/utils/context-truncation.ts +84 -84
- package/src/utils/json.ts +380 -380
- package/src/utils/session-id.ts +37 -37
- package/src/utils/tool-call-guard.ts +84 -84
- package/src/utils/types.ts +109 -109
|
@@ -1,754 +1,754 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Generic upstream retry / account-switch policy.
|
|
3
|
-
*
|
|
4
|
-
* Default: retry + prefer another account for unknown/upstream failures.
|
|
5
|
-
* Stop only for a small denylist of terminal local errors.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import { config } from "../../core/config.ts";
|
|
9
|
-
import { computeQuotaCooldownMs } from "../../core/account-manager.ts";
|
|
10
|
-
import { logger } from "../../core/logger.ts";
|
|
11
|
-
import {
|
|
12
|
-
PersonalizationSyncError,
|
|
13
|
-
QwenNetworkError,
|
|
14
|
-
QwenUpstreamError,
|
|
15
|
-
QwenUpstreamUnavailableError,
|
|
16
|
-
RetryableQwenStreamError,
|
|
17
|
-
} from "../../services/qwen.ts";
|
|
18
|
-
import {
|
|
19
|
-
AuthError,
|
|
20
|
-
ClientAbortedError,
|
|
21
|
-
NotFoundError,
|
|
22
|
-
ValidationError,
|
|
23
|
-
} from "../../core/errors.ts";
|
|
24
|
-
import { isAbortError } from "./helpers.ts";
|
|
25
|
-
|
|
26
|
-
export type RetryAction = {
|
|
27
|
-
/** Outer/create-stream layer should retry this failure */
|
|
28
|
-
retryable: boolean;
|
|
29
|
-
/** Prefer switching to another account when available */
|
|
30
|
-
switchAccount: boolean;
|
|
31
|
-
/** Force a new Qwen chat on retry */
|
|
32
|
-
forceNewChat: boolean;
|
|
33
|
-
/** Resend full conversation context (not just delta) */
|
|
34
|
-
retryWithFullPrompt: boolean;
|
|
35
|
-
/** Drop attached files on retry (for invalid_input caused by bad attachments) */
|
|
36
|
-
dropFiles?: boolean;
|
|
37
|
-
/** Suggested delay before next attempt */
|
|
38
|
-
retryAfterMs: number;
|
|
39
|
-
/** Optional short cooldown for the failing account */
|
|
40
|
-
accountCooldownMs?: number;
|
|
41
|
-
/** Cooldown reason label */
|
|
42
|
-
accountCooldownReason?: string;
|
|
43
|
-
/** Why this action was chosen (logging/debug) */
|
|
44
|
-
reason: string;
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
export type RetryableStreamError = RetryableQwenStreamError & {
|
|
48
|
-
upstreamCode?: string;
|
|
49
|
-
forceNewChat?: boolean;
|
|
50
|
-
retryWithFullPrompt?: boolean;
|
|
51
|
-
switchAccount?: boolean;
|
|
52
|
-
dropFiles?: boolean;
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
function errMessage(err: unknown): string {
|
|
56
|
-
if (err instanceof Error) return err.message || "";
|
|
57
|
-
return String(err ?? "");
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
function errCode(err: unknown): string {
|
|
61
|
-
const anyErr = err as { upstreamCode?: unknown; code?: unknown };
|
|
62
|
-
if (typeof anyErr?.upstreamCode === "string" && anyErr.upstreamCode) {
|
|
63
|
-
return anyErr.upstreamCode;
|
|
64
|
-
}
|
|
65
|
-
if (typeof anyErr?.code === "string" && anyErr.code) {
|
|
66
|
-
return anyErr.code;
|
|
67
|
-
}
|
|
68
|
-
return "";
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
function statusOf(err: unknown): number | undefined {
|
|
72
|
-
const anyErr = err as { upstreamStatus?: unknown; statusCode?: unknown };
|
|
73
|
-
if (typeof anyErr?.upstreamStatus === "number") return anyErr.upstreamStatus;
|
|
74
|
-
if (typeof anyErr?.statusCode === "number") return anyErr.statusCode;
|
|
75
|
-
return undefined;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/** Errors that belong to the proxy/client request itself — retrying is useless. */
|
|
79
|
-
export function isTerminalLocalError(err: unknown): boolean {
|
|
80
|
-
if (!err) return false;
|
|
81
|
-
|
|
82
|
-
if (
|
|
83
|
-
err instanceof ValidationError ||
|
|
84
|
-
err instanceof AuthError ||
|
|
85
|
-
err instanceof NotFoundError
|
|
86
|
-
) {
|
|
87
|
-
return true;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
const status = statusOf(err);
|
|
91
|
-
const code = errCode(err).toLowerCase();
|
|
92
|
-
const message = errMessage(err).toLowerCase();
|
|
93
|
-
|
|
94
|
-
// Local proxy auth / validation / not found
|
|
95
|
-
if (status === 400 || status === 401 || status === 404) {
|
|
96
|
-
// Exception: Qwen upstream can also return 404 for missing chat — that is retryable.
|
|
97
|
-
if (
|
|
98
|
-
message.includes("qwen") ||
|
|
99
|
-
message.includes("upstream") ||
|
|
100
|
-
code.includes("not_found") ||
|
|
101
|
-
message.includes("is not exist") ||
|
|
102
|
-
message.includes("does not exist")
|
|
103
|
-
) {
|
|
104
|
-
return false;
|
|
105
|
-
}
|
|
106
|
-
return true;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
if (
|
|
110
|
-
code === "invalid_api_key" ||
|
|
111
|
-
code === "authentication_error" ||
|
|
112
|
-
message.includes("missing or invalid authorization") ||
|
|
113
|
-
message.includes("invalid api key") ||
|
|
114
|
-
message.includes("messages is required") ||
|
|
115
|
-
message.includes("at least one user message") ||
|
|
116
|
-
message.includes("no qwen accounts configured")
|
|
117
|
-
) {
|
|
118
|
-
return true;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// bad_request from Qwen upstream is NOT terminal — it's a corrupted chat
|
|
122
|
-
// or invalid payload that can be recovered with a new chat + full prompt.
|
|
123
|
-
// Only treat as terminal when it's clearly a local proxy validation error.
|
|
124
|
-
if (code === "bad_request") {
|
|
125
|
-
const isQwenUpstream =
|
|
126
|
-
message.includes("qwen") ||
|
|
127
|
-
message.includes("upstream") ||
|
|
128
|
-
message.includes("invalid input") ||
|
|
129
|
-
message.includes("first message must not") ||
|
|
130
|
-
message.includes("entrada ou anexo");
|
|
131
|
-
if (!isQwenUpstream) {
|
|
132
|
-
return true;
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
return false;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
export function isClientAbortError(
|
|
140
|
-
err: unknown,
|
|
141
|
-
clientDisconnected = false,
|
|
142
|
-
requestAborted = false,
|
|
143
|
-
): boolean {
|
|
144
|
-
if (clientDisconnected || requestAborted) return true;
|
|
145
|
-
// Our own client-abort markers: the client disconnected OR a same-session
|
|
146
|
-
// retry superseded this request's lease during stream creation. A superseded
|
|
147
|
-
// request must die silently — the newer request owns the session, and
|
|
148
|
-
// retrying the old one resends full context on another account for nothing
|
|
149
|
-
// (and can queue indefinitely behind the new stream's lease).
|
|
150
|
-
if (err instanceof ClientAbortedError) return true;
|
|
151
|
-
if (err instanceof Error && err.message.includes("client aborted")) return true;
|
|
152
|
-
// Bare AbortError mid-stream is usually idle/upstream timeout (retryable).
|
|
153
|
-
return false;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
export function isInvalidInputError(err: unknown): boolean {
|
|
157
|
-
// "Invalid input the chat X is not exist" is a chat-missing error, not attachment invalid.
|
|
158
|
-
if (isChatNotExistError(err)) return false;
|
|
159
|
-
|
|
160
|
-
const code = errCode(err).toLowerCase();
|
|
161
|
-
const message = errMessage(err).toLowerCase();
|
|
162
|
-
return (
|
|
163
|
-
code === "invalid_input" ||
|
|
164
|
-
message.includes("invalid_input") ||
|
|
165
|
-
message.includes("entrada ou anexo inválido") ||
|
|
166
|
-
message.includes("invalid input") ||
|
|
167
|
-
message.includes("invalid attachment")
|
|
168
|
-
);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
/**
|
|
172
|
-
* Qwen content-safety moderation rejections (data_inspection_failed).
|
|
173
|
-
* These are deterministic: the same content will be rejected on any account,
|
|
174
|
-
* so retrying or switching accounts only wastes resources and time.
|
|
175
|
-
*/
|
|
176
|
-
export function isContentModerationError(err: unknown): boolean {
|
|
177
|
-
const code = errCode(err).toLowerCase();
|
|
178
|
-
const message = errMessage(err).toLowerCase();
|
|
179
|
-
return (
|
|
180
|
-
code === "data_inspection_failed" ||
|
|
181
|
-
message.includes("data_inspection_failed") ||
|
|
182
|
-
message.includes("conteúdo inadequado") ||
|
|
183
|
-
message.includes("inappropriate content") ||
|
|
184
|
-
message.includes("aviso de segurança do conteúdo") ||
|
|
185
|
-
message.includes("content safety")
|
|
186
|
-
);
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/** Prefer a clean chat on the current account before paying the cost of replaying
|
|
190
|
-
* the full context on another account. Callers keep their own per-request count. */
|
|
191
|
-
export function shouldRetryInvalidInputOnSameAccount(
|
|
192
|
-
reason: string,
|
|
193
|
-
alreadyRetried: boolean,
|
|
194
|
-
): boolean {
|
|
195
|
-
return (
|
|
196
|
-
(reason === "invalid_input" || reason === "corrupted_chat_history") &&
|
|
197
|
-
!alreadyRetried
|
|
198
|
-
);
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/** Keep TWO retries on the current account while an upstream generation
|
|
202
|
-
* settles. The tool loop fires the next turn the instant the previous one
|
|
203
|
-
* completes, and the upstream chat stays "in progress" for 2-4s after the
|
|
204
|
-
* terminal event — a single ~1.2s retry often loses that settle race, and
|
|
205
|
-
* escalating replays the FULL context on a cold account (~12s context reopen
|
|
206
|
-
* + captcha). Rotate only after the second failure. */
|
|
207
|
-
export function shouldRetryChatInProgressOnSameAccount(
|
|
208
|
-
reason: string,
|
|
209
|
-
alreadyRetriedCount: number,
|
|
210
|
-
): boolean {
|
|
211
|
-
// Three same-chat retries: settle is usually 2-4s but was measured >6s after
|
|
212
|
-
// huge turns, and the escalation alternative (full-context replay on a cold
|
|
213
|
-
// account) is far more expensive than one more bounded wait.
|
|
214
|
-
return reason === "chat_in_progress" && alreadyRetriedCount < 3;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
export function isAccountInitializationError(err: unknown): boolean {
|
|
218
|
-
const message = errMessage(err).toLowerCase();
|
|
219
|
-
return (
|
|
220
|
-
message.includes("header capture returned incomplete anti-fraud headers") ||
|
|
221
|
-
message.includes("required qwen anti-fraud headers are unavailable") ||
|
|
222
|
-
message.includes("playwright not initialized for account") ||
|
|
223
|
-
message.includes("playwright page unavailable") ||
|
|
224
|
-
message.includes("playwright page operation timed out") ||
|
|
225
|
-
message.includes("playwright re-initialization timed out")
|
|
226
|
-
);
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
export function isQuotaLikeError(err: unknown): boolean {
|
|
230
|
-
// Chat-not-exist / invalid attachment must never look like quota.
|
|
231
|
-
if (isChatNotExistError(err) || isInvalidInputError(err)) return false;
|
|
232
|
-
|
|
233
|
-
const code = errCode(err).toLowerCase();
|
|
234
|
-
const message = errMessage(err).toLowerCase();
|
|
235
|
-
|
|
236
|
-
// Note: RetryableQwenStreamError inherits OpenAI-style code "rate_limit_exceeded".
|
|
237
|
-
// Never treat that local code alone as quota — require message/upstream evidence.
|
|
238
|
-
return (
|
|
239
|
-
code === "quota_limit" ||
|
|
240
|
-
code === "ratelimited" ||
|
|
241
|
-
message.includes("quota_limit") ||
|
|
242
|
-
message.includes("quota exceeded") ||
|
|
243
|
-
message.includes("allocated quota") ||
|
|
244
|
-
message.includes("token-limit") ||
|
|
245
|
-
message.includes("insufficient quota") ||
|
|
246
|
-
message.includes("alta demanda") ||
|
|
247
|
-
message.includes("high demand") ||
|
|
248
|
-
message.includes("request rate increased too quickly") ||
|
|
249
|
-
message.includes("rate increased too quickly") ||
|
|
250
|
-
message.includes("upper limit for today's usage") ||
|
|
251
|
-
message.includes("you've reached the upper limit") ||
|
|
252
|
-
// Accept local rate_limit code only when message also looks like quota/rate
|
|
253
|
-
(code === "rate_limit_exceeded" &&
|
|
254
|
-
(message.includes("quota") ||
|
|
255
|
-
message.includes("rate") ||
|
|
256
|
-
message.includes("limit") ||
|
|
257
|
-
message.includes("demanda") ||
|
|
258
|
-
message.includes("demand")))
|
|
259
|
-
);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
export function isAntiBotError(err: unknown): boolean {
|
|
263
|
-
const code = errCode(err);
|
|
264
|
-
const codeLower = code.toLowerCase();
|
|
265
|
-
const message = errMessage(err).toLowerCase();
|
|
266
|
-
if (err instanceof RetryableQwenStreamError) {
|
|
267
|
-
return codeLower === "waf_challenge" || message.includes("anti-bot");
|
|
268
|
-
}
|
|
269
|
-
return (
|
|
270
|
-
code === "FAIL_SYS_USER_VALIDATE" ||
|
|
271
|
-
code === "RGV587_ERROR" ||
|
|
272
|
-
codeLower === "waf_challenge" ||
|
|
273
|
-
message.includes("fail_sys_user_validate") ||
|
|
274
|
-
message.includes("rgv587_error") ||
|
|
275
|
-
message.includes("_____tmd_____") ||
|
|
276
|
-
message.includes("tmd anti-bot") ||
|
|
277
|
-
message.includes("captcha") ||
|
|
278
|
-
message.includes("security verification") ||
|
|
279
|
-
message.includes("verify you are human") ||
|
|
280
|
-
message.includes("human verification") ||
|
|
281
|
-
message.includes("denyfromx5")
|
|
282
|
-
);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
function classifyQuotaCooldown(message: string): {
|
|
286
|
-
accountCooldownMs?: number;
|
|
287
|
-
accountCooldownReason: string;
|
|
288
|
-
} {
|
|
289
|
-
const lower = message.toLowerCase();
|
|
290
|
-
const temporary =
|
|
291
|
-
lower.includes("rate increased too quickly") ||
|
|
292
|
-
lower.includes("request rate increased too quickly") ||
|
|
293
|
-
lower.includes("alta demanda") ||
|
|
294
|
-
lower.includes("high demand") ||
|
|
295
|
-
lower.includes("tente novamente mais tarde") ||
|
|
296
|
-
lower.includes("try again later");
|
|
297
|
-
|
|
298
|
-
if (temporary) {
|
|
299
|
-
return {
|
|
300
|
-
accountCooldownMs: 2 * 60 * 1000,
|
|
301
|
-
accountCooldownReason: "RateLimitTemporary",
|
|
302
|
-
};
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
// REAL daily quota: the Qwen resets the account at the next UTC midnight
|
|
306
|
-
// (verified against 2026-08-21 production log: every proxy `until` matched
|
|
307
|
-
// the next 00:00 UTC exactly). The upstream "Wait about N hour(s)" hint is
|
|
308
|
-
// ONLY accurate when the error lands mid-day; near midnight it rounds UP to
|
|
309
|
-
// N≈24 while the real reset is minutes away (mzgns errored 23:37, hint
|
|
310
|
-
// "23h", but the account was usable 23 minutes later). Trust the daily
|
|
311
|
-
// reset, never the literal hint, never a blind 24h.
|
|
312
|
-
return {
|
|
313
|
-
accountCooldownMs: computeQuotaCooldownMs(Date.now()),
|
|
314
|
-
accountCooldownReason: "RateLimited",
|
|
315
|
-
};
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
/**
|
|
319
|
-
* Milliseconds until the next UTC midnight plus a small safety margin. The
|
|
320
|
-
* Qwen daily quota resets at 00:00 UTC, so this is the correct "when is this
|
|
321
|
-
* account usable again" for a quota exhaust — regardless of what the upstream
|
|
322
|
-
* "Wait about N hour(s)" hint guessed.
|
|
323
|
-
*/
|
|
324
|
-
// Implemented in core/account-manager.ts (shared with the @[] fallback); kept
|
|
325
|
-
// re-exporting here for callers that already import from retry-policy.
|
|
326
|
-
export { computeQuotaCooldownMs } from "../../core/account-manager.ts";
|
|
327
|
-
|
|
328
|
-
export function isChatNotExistError(err: unknown): boolean {
|
|
329
|
-
const message = errMessage(err).toLowerCase();
|
|
330
|
-
return (
|
|
331
|
-
message.includes("is not exist") ||
|
|
332
|
-
message.includes("not exist") ||
|
|
333
|
-
message.includes("does not exist")
|
|
334
|
-
);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
export function isChatInProgressError(err: unknown): boolean {
|
|
338
|
-
return errMessage(err).toLowerCase().includes("in progress");
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* Qwen rejects a model the account cannot serve with Not_Found: Model not found.
|
|
343
|
-
* This is deterministic per request — retrying on the same (or any) account
|
|
344
|
-
* with the same model can never succeed, so it must terminate instead of
|
|
345
|
-
* burning retry attempts / account cooldowns and ending in a misleading 502.
|
|
346
|
-
*/
|
|
347
|
-
export function isModelNotFoundError(err: unknown): boolean {
|
|
348
|
-
const code = errCode(err).toLowerCase();
|
|
349
|
-
const message = errMessage(err).toLowerCase();
|
|
350
|
-
return (
|
|
351
|
-
(code === "not_found" &&
|
|
352
|
-
message.includes("model") &&
|
|
353
|
-
message.includes("not found")) ||
|
|
354
|
-
message.includes("model not found")
|
|
355
|
-
);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
/**
|
|
359
|
-
* Browser fetch and ReadableStream failures often arrive as plain Error
|
|
360
|
-
* instances, especially when the stream is consumed outside Playwright.
|
|
361
|
-
* Keep this matcher narrow so local programming errors are not retried as
|
|
362
|
-
* account/network failures.
|
|
363
|
-
*/
|
|
364
|
-
export function isNetworkLikeError(err: unknown): boolean {
|
|
365
|
-
if (err instanceof QwenNetworkError) return true;
|
|
366
|
-
const message = errMessage(err).toLowerCase();
|
|
367
|
-
return (
|
|
368
|
-
message === "network error" ||
|
|
369
|
-
message.includes("failed to fetch") ||
|
|
370
|
-
message.includes("fetch failed") ||
|
|
371
|
-
message.includes("network connection was lost") ||
|
|
372
|
-
message.includes("connection reset") ||
|
|
373
|
-
message.includes("connection closed") ||
|
|
374
|
-
message.includes("socket hang up") ||
|
|
375
|
-
message.includes("econnreset") ||
|
|
376
|
-
message.includes("econnrefused") ||
|
|
377
|
-
message.includes("etimedout")
|
|
378
|
-
);
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
/**
|
|
382
|
-
* Corrupted chat history: Qwen rejects because the first message in the
|
|
383
|
-
* upstream chat thread is an assistant message (broken parent_id chain).
|
|
384
|
-
* Recovery: force new chat + resend full prompt + switch account.
|
|
385
|
-
*/
|
|
386
|
-
export function isCorruptedChatHistoryError(err: unknown): boolean {
|
|
387
|
-
const message = errMessage(err).toLowerCase();
|
|
388
|
-
return (
|
|
389
|
-
message.includes("first message must not") ||
|
|
390
|
-
message.includes("first message must be") ||
|
|
391
|
-
message.includes("must not assistant message") ||
|
|
392
|
-
message.includes("must not be assistant")
|
|
393
|
-
);
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
/**
|
|
397
|
-
* Build a RetryAction with sane defaults so each classification branch only
|
|
398
|
-
* spells out the fields it actually changes. Defaults: retryable, no account
|
|
399
|
-
* switch, same chat, delta replay, no delay. Branch ordering below is
|
|
400
|
-
* load-bearing (specific recoveries must win over broad substring matches).
|
|
401
|
-
*/
|
|
402
|
-
function makeRetryAction(
|
|
403
|
-
reason: string,
|
|
404
|
-
overrides: Partial<Omit<RetryAction, "reason">> = {},
|
|
405
|
-
): RetryAction {
|
|
406
|
-
return {
|
|
407
|
-
retryable: true,
|
|
408
|
-
switchAccount: false,
|
|
409
|
-
forceNewChat: false,
|
|
410
|
-
retryWithFullPrompt: false,
|
|
411
|
-
retryAfterMs: 0,
|
|
412
|
-
...overrides,
|
|
413
|
-
reason,
|
|
414
|
-
};
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
/**
|
|
418
|
-
* Generic recovery policy for create-stream + mid-stream failures.
|
|
419
|
-
* Unknown upstream errors are retryable by default when enabled in config.
|
|
420
|
-
*/
|
|
421
|
-
export function classifyRetryAction(
|
|
422
|
-
err: unknown,
|
|
423
|
-
options?: {
|
|
424
|
-
clientDisconnected?: boolean;
|
|
425
|
-
requestAborted?: boolean;
|
|
426
|
-
baseDelayMs?: number;
|
|
427
|
-
},
|
|
428
|
-
): RetryAction {
|
|
429
|
-
const baseDelayMs = options?.baseDelayMs ?? config.retry.baseDelayMs;
|
|
430
|
-
const unknownEnabled = config.retry.onUnknownUpstream !== false;
|
|
431
|
-
|
|
432
|
-
if (
|
|
433
|
-
isClientAbortError(
|
|
434
|
-
err,
|
|
435
|
-
options?.clientDisconnected === true,
|
|
436
|
-
options?.requestAborted === true,
|
|
437
|
-
)
|
|
438
|
-
) {
|
|
439
|
-
return makeRetryAction("client_abort", { retryable: false });
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
if (isTerminalLocalError(err)) {
|
|
443
|
-
return makeRetryAction("terminal_local", { retryable: false });
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
const message = errMessage(err).toLowerCase();
|
|
447
|
-
const code = errCode(err).toLowerCase();
|
|
448
|
-
if (isAccountInitializationError(err)) {
|
|
449
|
-
return makeRetryAction("account_initialization_failed", {
|
|
450
|
-
switchAccount: true,
|
|
451
|
-
retryAfterMs: Math.min(baseDelayMs, 1_000),
|
|
452
|
-
accountCooldownMs: config.concurrency.initFailureCooldownMs,
|
|
453
|
-
accountCooldownReason: "AuthInitFailed",
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
if (
|
|
458
|
-
code === "account_busy" ||
|
|
459
|
-
message.includes("waiting for a free slot") ||
|
|
460
|
-
message.includes("busy: timed out")
|
|
461
|
-
) {
|
|
462
|
-
return makeRetryAction("account_busy", {
|
|
463
|
-
switchAccount: true,
|
|
464
|
-
retryAfterMs: Math.min(baseDelayMs, 1_000),
|
|
465
|
-
});
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// Agent instructions ride ONLY the account-level personalization. An
|
|
469
|
-
// unconfirmed sync means this account cannot serve the request as-is —
|
|
470
|
-
// rotate to another account (each attempt re-syncs on its own account).
|
|
471
|
-
if (err instanceof PersonalizationSyncError) {
|
|
472
|
-
return makeRetryAction("personalization_sync_failed", {
|
|
473
|
-
switchAccount: true,
|
|
474
|
-
forceNewChat: true,
|
|
475
|
-
retryAfterMs: baseDelayMs,
|
|
476
|
-
});
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
// Specialized recoveries first (even if wrapped as RetryableQwenStreamError)
|
|
480
|
-
// Corrupted chat history must win over broad "invalid input" matches.
|
|
481
|
-
// Try a fresh chat on the SAME account first — the corruption is in the
|
|
482
|
-
// upstream parent chain, not the account. Only rotate if the rebuild fails.
|
|
483
|
-
if (isCorruptedChatHistoryError(err)) {
|
|
484
|
-
return makeRetryAction("corrupted_chat_history", {
|
|
485
|
-
forceNewChat: true,
|
|
486
|
-
retryWithFullPrompt: true,
|
|
487
|
-
});
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
// Chat missing must win over broad "invalid input" substring matches.
|
|
491
|
-
if (isChatNotExistError(err) || isChatInProgressError(err)) {
|
|
492
|
-
const typed = err as RetryableStreamError;
|
|
493
|
-
const inProgress = isChatInProgressError(err);
|
|
494
|
-
// chat_in_progress: do NOT switch immediately — the account is just
|
|
495
|
-
// temporarily busy (escalation happens in tryCreateStreamWithRetry after
|
|
496
|
-
// repeated failures). chat_not_exist needs a new chat + full replay;
|
|
497
|
-
// in_progress retries the same chat first.
|
|
498
|
-
return makeRetryAction(inProgress ? "chat_in_progress" : "chat_not_exist", {
|
|
499
|
-
forceNewChat: !inProgress,
|
|
500
|
-
retryWithFullPrompt: !inProgress,
|
|
501
|
-
retryAfterMs: inProgress
|
|
502
|
-
? (typed.retryAfterMs ?? config.retry.chatInProgressDelayMs)
|
|
503
|
-
: (typed.retryAfterMs ?? 0),
|
|
504
|
-
});
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
if (isInvalidInputError(err)) {
|
|
508
|
-
const typed = err as RetryableStreamError;
|
|
509
|
-
return makeRetryAction("invalid_input", {
|
|
510
|
-
switchAccount: typed.switchAccount !== false,
|
|
511
|
-
forceNewChat: true,
|
|
512
|
-
retryWithFullPrompt: true,
|
|
513
|
-
retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
|
|
514
|
-
dropFiles: typed.dropFiles,
|
|
515
|
-
});
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
// Content moderation rejections are deterministic — retrying on any
|
|
519
|
-
// account with the same content produces the same rejection. Fail fast
|
|
520
|
-
// instead of burning through accounts, personalization syncs and captchas.
|
|
521
|
-
if (isContentModerationError(err)) {
|
|
522
|
-
return makeRetryAction("content_moderation", { retryable: false });
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
// Model not found is equally deterministic (the account cannot serve the
|
|
526
|
-
// requested model). Fail fast with a clear error instead of retrying the
|
|
527
|
-
// same doomed request and cooldown-marking accounts for ~5 hours.
|
|
528
|
-
if (isModelNotFoundError(err)) {
|
|
529
|
-
return makeRetryAction("model_not_found", { retryable: false });
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
if (isAntiBotError(err)) {
|
|
533
|
-
// WAF/captcha is only identified here. Retry the same request on the
|
|
534
|
-
// same account immediately; recovery, cooldown and account rotation are
|
|
535
|
-
// intentionally left out so the failure path stays observable.
|
|
536
|
-
return makeRetryAction("anti_bot");
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
if (isQuotaLikeError(err)) {
|
|
540
|
-
const typed = err as RetryableStreamError;
|
|
541
|
-
const quota = classifyQuotaCooldown(errMessage(err));
|
|
542
|
-
const isTemporary = quota.accountCooldownReason === "RateLimitTemporary";
|
|
543
|
-
// Temporary load shedding: retry same account first, only switch on
|
|
544
|
-
// repeated failure. Real quota exhaustion: switch immediately.
|
|
545
|
-
return makeRetryAction("quota_or_rate_limit", {
|
|
546
|
-
switchAccount: isTemporary ? false : typed.switchAccount !== false,
|
|
547
|
-
forceNewChat: typed.forceNewChat === true,
|
|
548
|
-
retryWithFullPrompt: typed.retryWithFullPrompt === true,
|
|
549
|
-
retryAfterMs: typed.retryAfterMs ?? (isTemporary ? 3_000 : baseDelayMs),
|
|
550
|
-
accountCooldownMs: quota.accountCooldownMs,
|
|
551
|
-
accountCooldownReason: quota.accountCooldownReason,
|
|
552
|
-
});
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
if (
|
|
556
|
-
isNetworkLikeError(err) ||
|
|
557
|
-
err instanceof QwenUpstreamUnavailableError ||
|
|
558
|
-
err instanceof QwenUpstreamError ||
|
|
559
|
-
isAbortError(err)
|
|
560
|
-
) {
|
|
561
|
-
const typed = err as RetryableStreamError;
|
|
562
|
-
return makeRetryAction(
|
|
563
|
-
isNetworkLikeError(err)
|
|
564
|
-
? "network"
|
|
565
|
-
: err instanceof QwenUpstreamUnavailableError
|
|
566
|
-
? "upstream_unavailable"
|
|
567
|
-
: isAbortError(err)
|
|
568
|
-
? "stream_aborted"
|
|
569
|
-
: "upstream_error",
|
|
570
|
-
{
|
|
571
|
-
switchAccount: typed.switchAccount !== false,
|
|
572
|
-
forceNewChat: true,
|
|
573
|
-
retryWithFullPrompt: typed.retryWithFullPrompt === true,
|
|
574
|
-
retryAfterMs:
|
|
575
|
-
typed.retryAfterMs ??
|
|
576
|
-
(isNetworkLikeError(err)
|
|
577
|
-
? 3000
|
|
578
|
-
: err instanceof QwenUpstreamUnavailableError
|
|
579
|
-
? 2000
|
|
580
|
-
: Math.min(baseDelayMs * 2, 3000)),
|
|
581
|
-
},
|
|
582
|
-
);
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
// Preserve explicit RetryableQwenStreamError flags for remaining cases
|
|
586
|
-
if (err instanceof RetryableQwenStreamError) {
|
|
587
|
-
const typed = err as RetryableStreamError;
|
|
588
|
-
// Default switch unless caller explicitly set switchAccount=false
|
|
589
|
-
return makeRetryAction("explicit_retryable", {
|
|
590
|
-
switchAccount: typed.switchAccount !== false,
|
|
591
|
-
forceNewChat: typed.forceNewChat === true,
|
|
592
|
-
retryWithFullPrompt: typed.retryWithFullPrompt === true,
|
|
593
|
-
retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
|
|
594
|
-
});
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
// Default for unknown failures: retry when policy enabled
|
|
598
|
-
if (unknownEnabled) {
|
|
599
|
-
return makeRetryAction("unknown_upstream_default_retry", {
|
|
600
|
-
switchAccount: true,
|
|
601
|
-
forceNewChat: true,
|
|
602
|
-
retryAfterMs: baseDelayMs,
|
|
603
|
-
});
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
return makeRetryAction("unknown_not_retryable", { retryable: false });
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
/** Build a RetryableQwenStreamError for SSE/mid-stream failures with policy flags. */
|
|
610
|
-
export function toRetryableStreamError(
|
|
611
|
-
errCode: string,
|
|
612
|
-
errDetails: string,
|
|
613
|
-
options?: Partial<RetryAction>,
|
|
614
|
-
): RetryableStreamError {
|
|
615
|
-
const policy = classifyRetryAction(
|
|
616
|
-
Object.assign(new Error(`${errCode}: ${errDetails}`), {
|
|
617
|
-
upstreamCode: errCode,
|
|
618
|
-
}),
|
|
619
|
-
);
|
|
620
|
-
const merged: RetryAction = {
|
|
621
|
-
...policy,
|
|
622
|
-
...options,
|
|
623
|
-
retryable: true,
|
|
624
|
-
reason: options?.reason || policy.reason,
|
|
625
|
-
};
|
|
626
|
-
|
|
627
|
-
const error = new RetryableQwenStreamError(
|
|
628
|
-
`Qwen retryable upstream error: ${errCode}: ${errDetails.substring(0, 200)}`,
|
|
629
|
-
merged.retryAfterMs || config.retry.baseDelayMs,
|
|
630
|
-
) as RetryableStreamError;
|
|
631
|
-
|
|
632
|
-
error.upstreamCode = errCode;
|
|
633
|
-
error.forceNewChat = merged.forceNewChat;
|
|
634
|
-
error.retryWithFullPrompt = merged.retryWithFullPrompt;
|
|
635
|
-
error.switchAccount = merged.switchAccount;
|
|
636
|
-
error.dropFiles = merged.dropFiles;
|
|
637
|
-
return error;
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
/** For SSE error chunks: map any upstream SSE error to throw path. */
|
|
641
|
-
export function throwFromSseUpstreamError(
|
|
642
|
-
errCode: string,
|
|
643
|
-
errDetails: string,
|
|
644
|
-
): never {
|
|
645
|
-
const detailsLower = errDetails.toLowerCase();
|
|
646
|
-
// Qwen sometimes labels the chat-state error as RateLimited. Normalize it
|
|
647
|
-
// before retry/logging so it cannot be mistaken for account quota exhaustion.
|
|
648
|
-
const normalizedErrCode =
|
|
649
|
-
detailsLower.includes("chat is in progress") ||
|
|
650
|
-
detailsLower.includes("the chat is in progress")
|
|
651
|
-
? "chat_in_progress"
|
|
652
|
-
: errCode;
|
|
653
|
-
|
|
654
|
-
// Log upstream errors. Expected retryable codes (quota, rate limit, chat
|
|
655
|
-
// state) use warn level to avoid noisy stderr stack traces in production.
|
|
656
|
-
const expectedCodes = new Set([
|
|
657
|
-
"quota_limit",
|
|
658
|
-
"rate_limit",
|
|
659
|
-
"rate_limit_exceeded",
|
|
660
|
-
"chat_in_progress",
|
|
661
|
-
"invalid_input",
|
|
662
|
-
"data_inspection_failed",
|
|
663
|
-
]);
|
|
664
|
-
if (expectedCodes.has(normalizedErrCode.toLowerCase())) {
|
|
665
|
-
logger.warn(
|
|
666
|
-
`[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
|
|
667
|
-
);
|
|
668
|
-
} else {
|
|
669
|
-
console.error(
|
|
670
|
-
`[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
|
|
671
|
-
);
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// invalid_input keeps dedicated wording for logs/tests (not "chat is not exist")
|
|
675
|
-
const isChatMissing =
|
|
676
|
-
detailsLower.includes("is not exist") ||
|
|
677
|
-
detailsLower.includes("does not exist") ||
|
|
678
|
-
/\bnot exist\b/.test(detailsLower);
|
|
679
|
-
if (
|
|
680
|
-
!isChatMissing &&
|
|
681
|
-
(errCode.toLowerCase() === "invalid_input" ||
|
|
682
|
-
detailsLower.includes("entrada ou anexo inválido") ||
|
|
683
|
-
detailsLower.includes("invalid input") ||
|
|
684
|
-
detailsLower.includes("invalid attachment"))
|
|
685
|
-
) {
|
|
686
|
-
logger.warn("[Upstream] invalid_input mid-stream detected", {
|
|
687
|
-
code: errCode,
|
|
688
|
-
detailsLength: errDetails.length,
|
|
689
|
-
messageMentionsAttachment:
|
|
690
|
-
detailsLower.includes("anexo") || detailsLower.includes("attachment"),
|
|
691
|
-
messageMentionsFile:
|
|
692
|
-
detailsLower.includes("file") || detailsLower.includes("arquivo"),
|
|
693
|
-
});
|
|
694
|
-
|
|
695
|
-
const error = new RetryableQwenStreamError(
|
|
696
|
-
`Qwen retryable invalid input: ${errCode}: ${errDetails.substring(0, 200)}`,
|
|
697
|
-
config.retry.baseDelayMs,
|
|
698
|
-
) as RetryableStreamError;
|
|
699
|
-
error.upstreamCode = errCode;
|
|
700
|
-
error.forceNewChat = true;
|
|
701
|
-
error.retryWithFullPrompt = true;
|
|
702
|
-
error.switchAccount = true;
|
|
703
|
-
error.dropFiles = true; // Drop files on retry to isolate file-related errors
|
|
704
|
-
throw error;
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
// Content moderation rejections are deterministic — the same content will
|
|
708
|
-
// be rejected on every account. Throw as RetryableQwenStreamError so it
|
|
709
|
-
// propagates through the streaming catch blocks, but classifyRetryAction
|
|
710
|
-
// will mark it non-retryable.
|
|
711
|
-
if (isContentModerationError({ upstreamCode: normalizedErrCode, message: errDetails })) {
|
|
712
|
-
logger.warn(
|
|
713
|
-
`[Upstream] Content moderation rejection (not retrying): ${normalizedErrCode}`,
|
|
714
|
-
);
|
|
715
|
-
const error = new RetryableQwenStreamError(
|
|
716
|
-
`Qwen content moderation: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
|
|
717
|
-
0,
|
|
718
|
-
) as RetryableStreamError;
|
|
719
|
-
error.upstreamCode = normalizedErrCode;
|
|
720
|
-
error.switchAccount = false;
|
|
721
|
-
throw error;
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
// A model the account cannot serve is a deterministic rejection too — never
|
|
725
|
-
// transparently retrofit this doomed model request on the same/other account.
|
|
726
|
-
if (isModelNotFoundError({ upstreamCode: normalizedErrCode, message: errDetails })) {
|
|
727
|
-
logger.warn(
|
|
728
|
-
`[Upstream] Model not available (not retrying): ${normalizedErrCode}`,
|
|
729
|
-
);
|
|
730
|
-
const error = new RetryableQwenStreamError(
|
|
731
|
-
`Qwen model not found: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
|
|
732
|
-
0,
|
|
733
|
-
) as RetryableStreamError;
|
|
734
|
-
error.upstreamCode = normalizedErrCode;
|
|
735
|
-
error.switchAccount = false;
|
|
736
|
-
throw error;
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
if (
|
|
740
|
-
errDetails.includes("FAIL_SYS_USER_VALIDATE") ||
|
|
741
|
-
errDetails.includes("RGV587_ERROR") ||
|
|
742
|
-
errDetails.includes("user validate")
|
|
743
|
-
) {
|
|
744
|
-
const error = new RetryableQwenStreamError(
|
|
745
|
-
`Qwen anti-bot: ${errCode}: ${errDetails}`,
|
|
746
|
-
0,
|
|
747
|
-
) as RetryableStreamError;
|
|
748
|
-
error.upstreamCode = errCode;
|
|
749
|
-
error.switchAccount = true;
|
|
750
|
-
throw error;
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
throw toRetryableStreamError(normalizedErrCode, errDetails);
|
|
754
|
-
}
|
|
1
|
+
/*
|
|
2
|
+
* Generic upstream retry / account-switch policy.
|
|
3
|
+
*
|
|
4
|
+
* Default: retry + prefer another account for unknown/upstream failures.
|
|
5
|
+
* Stop only for a small denylist of terminal local errors.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { config } from "../../core/config.ts";
|
|
9
|
+
import { computeQuotaCooldownMs } from "../../core/account-manager.ts";
|
|
10
|
+
import { logger } from "../../core/logger.ts";
|
|
11
|
+
import {
|
|
12
|
+
PersonalizationSyncError,
|
|
13
|
+
QwenNetworkError,
|
|
14
|
+
QwenUpstreamError,
|
|
15
|
+
QwenUpstreamUnavailableError,
|
|
16
|
+
RetryableQwenStreamError,
|
|
17
|
+
} from "../../services/qwen.ts";
|
|
18
|
+
import {
|
|
19
|
+
AuthError,
|
|
20
|
+
ClientAbortedError,
|
|
21
|
+
NotFoundError,
|
|
22
|
+
ValidationError,
|
|
23
|
+
} from "../../core/errors.ts";
|
|
24
|
+
import { isAbortError } from "./helpers.ts";
|
|
25
|
+
|
|
26
|
+
export type RetryAction = {
|
|
27
|
+
/** Outer/create-stream layer should retry this failure */
|
|
28
|
+
retryable: boolean;
|
|
29
|
+
/** Prefer switching to another account when available */
|
|
30
|
+
switchAccount: boolean;
|
|
31
|
+
/** Force a new Qwen chat on retry */
|
|
32
|
+
forceNewChat: boolean;
|
|
33
|
+
/** Resend full conversation context (not just delta) */
|
|
34
|
+
retryWithFullPrompt: boolean;
|
|
35
|
+
/** Drop attached files on retry (for invalid_input caused by bad attachments) */
|
|
36
|
+
dropFiles?: boolean;
|
|
37
|
+
/** Suggested delay before next attempt */
|
|
38
|
+
retryAfterMs: number;
|
|
39
|
+
/** Optional short cooldown for the failing account */
|
|
40
|
+
accountCooldownMs?: number;
|
|
41
|
+
/** Cooldown reason label */
|
|
42
|
+
accountCooldownReason?: string;
|
|
43
|
+
/** Why this action was chosen (logging/debug) */
|
|
44
|
+
reason: string;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type RetryableStreamError = RetryableQwenStreamError & {
|
|
48
|
+
upstreamCode?: string;
|
|
49
|
+
forceNewChat?: boolean;
|
|
50
|
+
retryWithFullPrompt?: boolean;
|
|
51
|
+
switchAccount?: boolean;
|
|
52
|
+
dropFiles?: boolean;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
function errMessage(err: unknown): string {
|
|
56
|
+
if (err instanceof Error) return err.message || "";
|
|
57
|
+
return String(err ?? "");
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function errCode(err: unknown): string {
|
|
61
|
+
const anyErr = err as { upstreamCode?: unknown; code?: unknown };
|
|
62
|
+
if (typeof anyErr?.upstreamCode === "string" && anyErr.upstreamCode) {
|
|
63
|
+
return anyErr.upstreamCode;
|
|
64
|
+
}
|
|
65
|
+
if (typeof anyErr?.code === "string" && anyErr.code) {
|
|
66
|
+
return anyErr.code;
|
|
67
|
+
}
|
|
68
|
+
return "";
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function statusOf(err: unknown): number | undefined {
|
|
72
|
+
const anyErr = err as { upstreamStatus?: unknown; statusCode?: unknown };
|
|
73
|
+
if (typeof anyErr?.upstreamStatus === "number") return anyErr.upstreamStatus;
|
|
74
|
+
if (typeof anyErr?.statusCode === "number") return anyErr.statusCode;
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Errors that belong to the proxy/client request itself — retrying is useless. */
|
|
79
|
+
export function isTerminalLocalError(err: unknown): boolean {
|
|
80
|
+
if (!err) return false;
|
|
81
|
+
|
|
82
|
+
if (
|
|
83
|
+
err instanceof ValidationError ||
|
|
84
|
+
err instanceof AuthError ||
|
|
85
|
+
err instanceof NotFoundError
|
|
86
|
+
) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const status = statusOf(err);
|
|
91
|
+
const code = errCode(err).toLowerCase();
|
|
92
|
+
const message = errMessage(err).toLowerCase();
|
|
93
|
+
|
|
94
|
+
// Local proxy auth / validation / not found
|
|
95
|
+
if (status === 400 || status === 401 || status === 404) {
|
|
96
|
+
// Exception: Qwen upstream can also return 404 for missing chat — that is retryable.
|
|
97
|
+
if (
|
|
98
|
+
message.includes("qwen") ||
|
|
99
|
+
message.includes("upstream") ||
|
|
100
|
+
code.includes("not_found") ||
|
|
101
|
+
message.includes("is not exist") ||
|
|
102
|
+
message.includes("does not exist")
|
|
103
|
+
) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (
|
|
110
|
+
code === "invalid_api_key" ||
|
|
111
|
+
code === "authentication_error" ||
|
|
112
|
+
message.includes("missing or invalid authorization") ||
|
|
113
|
+
message.includes("invalid api key") ||
|
|
114
|
+
message.includes("messages is required") ||
|
|
115
|
+
message.includes("at least one user message") ||
|
|
116
|
+
message.includes("no qwen accounts configured")
|
|
117
|
+
) {
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// bad_request from Qwen upstream is NOT terminal — it's a corrupted chat
|
|
122
|
+
// or invalid payload that can be recovered with a new chat + full prompt.
|
|
123
|
+
// Only treat as terminal when it's clearly a local proxy validation error.
|
|
124
|
+
if (code === "bad_request") {
|
|
125
|
+
const isQwenUpstream =
|
|
126
|
+
message.includes("qwen") ||
|
|
127
|
+
message.includes("upstream") ||
|
|
128
|
+
message.includes("invalid input") ||
|
|
129
|
+
message.includes("first message must not") ||
|
|
130
|
+
message.includes("entrada ou anexo");
|
|
131
|
+
if (!isQwenUpstream) {
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function isClientAbortError(
|
|
140
|
+
err: unknown,
|
|
141
|
+
clientDisconnected = false,
|
|
142
|
+
requestAborted = false,
|
|
143
|
+
): boolean {
|
|
144
|
+
if (clientDisconnected || requestAborted) return true;
|
|
145
|
+
// Our own client-abort markers: the client disconnected OR a same-session
|
|
146
|
+
// retry superseded this request's lease during stream creation. A superseded
|
|
147
|
+
// request must die silently — the newer request owns the session, and
|
|
148
|
+
// retrying the old one resends full context on another account for nothing
|
|
149
|
+
// (and can queue indefinitely behind the new stream's lease).
|
|
150
|
+
if (err instanceof ClientAbortedError) return true;
|
|
151
|
+
if (err instanceof Error && err.message.includes("client aborted")) return true;
|
|
152
|
+
// Bare AbortError mid-stream is usually idle/upstream timeout (retryable).
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export function isInvalidInputError(err: unknown): boolean {
|
|
157
|
+
// "Invalid input the chat X is not exist" is a chat-missing error, not attachment invalid.
|
|
158
|
+
if (isChatNotExistError(err)) return false;
|
|
159
|
+
|
|
160
|
+
const code = errCode(err).toLowerCase();
|
|
161
|
+
const message = errMessage(err).toLowerCase();
|
|
162
|
+
return (
|
|
163
|
+
code === "invalid_input" ||
|
|
164
|
+
message.includes("invalid_input") ||
|
|
165
|
+
message.includes("entrada ou anexo inválido") ||
|
|
166
|
+
message.includes("invalid input") ||
|
|
167
|
+
message.includes("invalid attachment")
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Qwen content-safety moderation rejections (data_inspection_failed).
|
|
173
|
+
* These are deterministic: the same content will be rejected on any account,
|
|
174
|
+
* so retrying or switching accounts only wastes resources and time.
|
|
175
|
+
*/
|
|
176
|
+
export function isContentModerationError(err: unknown): boolean {
|
|
177
|
+
const code = errCode(err).toLowerCase();
|
|
178
|
+
const message = errMessage(err).toLowerCase();
|
|
179
|
+
return (
|
|
180
|
+
code === "data_inspection_failed" ||
|
|
181
|
+
message.includes("data_inspection_failed") ||
|
|
182
|
+
message.includes("conteúdo inadequado") ||
|
|
183
|
+
message.includes("inappropriate content") ||
|
|
184
|
+
message.includes("aviso de segurança do conteúdo") ||
|
|
185
|
+
message.includes("content safety")
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Prefer a clean chat on the current account before paying the cost of replaying
|
|
190
|
+
* the full context on another account. Callers keep their own per-request count. */
|
|
191
|
+
export function shouldRetryInvalidInputOnSameAccount(
|
|
192
|
+
reason: string,
|
|
193
|
+
alreadyRetried: boolean,
|
|
194
|
+
): boolean {
|
|
195
|
+
return (
|
|
196
|
+
(reason === "invalid_input" || reason === "corrupted_chat_history") &&
|
|
197
|
+
!alreadyRetried
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Keep TWO retries on the current account while an upstream generation
|
|
202
|
+
* settles. The tool loop fires the next turn the instant the previous one
|
|
203
|
+
* completes, and the upstream chat stays "in progress" for 2-4s after the
|
|
204
|
+
* terminal event — a single ~1.2s retry often loses that settle race, and
|
|
205
|
+
* escalating replays the FULL context on a cold account (~12s context reopen
|
|
206
|
+
* + captcha). Rotate only after the second failure. */
|
|
207
|
+
export function shouldRetryChatInProgressOnSameAccount(
|
|
208
|
+
reason: string,
|
|
209
|
+
alreadyRetriedCount: number,
|
|
210
|
+
): boolean {
|
|
211
|
+
// Three same-chat retries: settle is usually 2-4s but was measured >6s after
|
|
212
|
+
// huge turns, and the escalation alternative (full-context replay on a cold
|
|
213
|
+
// account) is far more expensive than one more bounded wait.
|
|
214
|
+
return reason === "chat_in_progress" && alreadyRetriedCount < 3;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export function isAccountInitializationError(err: unknown): boolean {
|
|
218
|
+
const message = errMessage(err).toLowerCase();
|
|
219
|
+
return (
|
|
220
|
+
message.includes("header capture returned incomplete anti-fraud headers") ||
|
|
221
|
+
message.includes("required qwen anti-fraud headers are unavailable") ||
|
|
222
|
+
message.includes("playwright not initialized for account") ||
|
|
223
|
+
message.includes("playwright page unavailable") ||
|
|
224
|
+
message.includes("playwright page operation timed out") ||
|
|
225
|
+
message.includes("playwright re-initialization timed out")
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function isQuotaLikeError(err: unknown): boolean {
|
|
230
|
+
// Chat-not-exist / invalid attachment must never look like quota.
|
|
231
|
+
if (isChatNotExistError(err) || isInvalidInputError(err)) return false;
|
|
232
|
+
|
|
233
|
+
const code = errCode(err).toLowerCase();
|
|
234
|
+
const message = errMessage(err).toLowerCase();
|
|
235
|
+
|
|
236
|
+
// Note: RetryableQwenStreamError inherits OpenAI-style code "rate_limit_exceeded".
|
|
237
|
+
// Never treat that local code alone as quota — require message/upstream evidence.
|
|
238
|
+
return (
|
|
239
|
+
code === "quota_limit" ||
|
|
240
|
+
code === "ratelimited" ||
|
|
241
|
+
message.includes("quota_limit") ||
|
|
242
|
+
message.includes("quota exceeded") ||
|
|
243
|
+
message.includes("allocated quota") ||
|
|
244
|
+
message.includes("token-limit") ||
|
|
245
|
+
message.includes("insufficient quota") ||
|
|
246
|
+
message.includes("alta demanda") ||
|
|
247
|
+
message.includes("high demand") ||
|
|
248
|
+
message.includes("request rate increased too quickly") ||
|
|
249
|
+
message.includes("rate increased too quickly") ||
|
|
250
|
+
message.includes("upper limit for today's usage") ||
|
|
251
|
+
message.includes("you've reached the upper limit") ||
|
|
252
|
+
// Accept local rate_limit code only when message also looks like quota/rate
|
|
253
|
+
(code === "rate_limit_exceeded" &&
|
|
254
|
+
(message.includes("quota") ||
|
|
255
|
+
message.includes("rate") ||
|
|
256
|
+
message.includes("limit") ||
|
|
257
|
+
message.includes("demanda") ||
|
|
258
|
+
message.includes("demand")))
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export function isAntiBotError(err: unknown): boolean {
|
|
263
|
+
const code = errCode(err);
|
|
264
|
+
const codeLower = code.toLowerCase();
|
|
265
|
+
const message = errMessage(err).toLowerCase();
|
|
266
|
+
if (err instanceof RetryableQwenStreamError) {
|
|
267
|
+
return codeLower === "waf_challenge" || message.includes("anti-bot");
|
|
268
|
+
}
|
|
269
|
+
return (
|
|
270
|
+
code === "FAIL_SYS_USER_VALIDATE" ||
|
|
271
|
+
code === "RGV587_ERROR" ||
|
|
272
|
+
codeLower === "waf_challenge" ||
|
|
273
|
+
message.includes("fail_sys_user_validate") ||
|
|
274
|
+
message.includes("rgv587_error") ||
|
|
275
|
+
message.includes("_____tmd_____") ||
|
|
276
|
+
message.includes("tmd anti-bot") ||
|
|
277
|
+
message.includes("captcha") ||
|
|
278
|
+
message.includes("security verification") ||
|
|
279
|
+
message.includes("verify you are human") ||
|
|
280
|
+
message.includes("human verification") ||
|
|
281
|
+
message.includes("denyfromx5")
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function classifyQuotaCooldown(message: string): {
|
|
286
|
+
accountCooldownMs?: number;
|
|
287
|
+
accountCooldownReason: string;
|
|
288
|
+
} {
|
|
289
|
+
const lower = message.toLowerCase();
|
|
290
|
+
const temporary =
|
|
291
|
+
lower.includes("rate increased too quickly") ||
|
|
292
|
+
lower.includes("request rate increased too quickly") ||
|
|
293
|
+
lower.includes("alta demanda") ||
|
|
294
|
+
lower.includes("high demand") ||
|
|
295
|
+
lower.includes("tente novamente mais tarde") ||
|
|
296
|
+
lower.includes("try again later");
|
|
297
|
+
|
|
298
|
+
if (temporary) {
|
|
299
|
+
return {
|
|
300
|
+
accountCooldownMs: 2 * 60 * 1000,
|
|
301
|
+
accountCooldownReason: "RateLimitTemporary",
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// REAL daily quota: the Qwen resets the account at the next UTC midnight
|
|
306
|
+
// (verified against 2026-08-21 production log: every proxy `until` matched
|
|
307
|
+
// the next 00:00 UTC exactly). The upstream "Wait about N hour(s)" hint is
|
|
308
|
+
// ONLY accurate when the error lands mid-day; near midnight it rounds UP to
|
|
309
|
+
// N≈24 while the real reset is minutes away (mzgns errored 23:37, hint
|
|
310
|
+
// "23h", but the account was usable 23 minutes later). Trust the daily
|
|
311
|
+
// reset, never the literal hint, never a blind 24h.
|
|
312
|
+
return {
|
|
313
|
+
accountCooldownMs: computeQuotaCooldownMs(Date.now()),
|
|
314
|
+
accountCooldownReason: "RateLimited",
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Milliseconds until the next UTC midnight plus a small safety margin. The
|
|
320
|
+
* Qwen daily quota resets at 00:00 UTC, so this is the correct "when is this
|
|
321
|
+
* account usable again" for a quota exhaust — regardless of what the upstream
|
|
322
|
+
* "Wait about N hour(s)" hint guessed.
|
|
323
|
+
*/
|
|
324
|
+
// Implemented in core/account-manager.ts (shared with the @[] fallback); kept
|
|
325
|
+
// re-exporting here for callers that already import from retry-policy.
|
|
326
|
+
export { computeQuotaCooldownMs } from "../../core/account-manager.ts";
|
|
327
|
+
|
|
328
|
+
export function isChatNotExistError(err: unknown): boolean {
|
|
329
|
+
const message = errMessage(err).toLowerCase();
|
|
330
|
+
return (
|
|
331
|
+
message.includes("is not exist") ||
|
|
332
|
+
message.includes("not exist") ||
|
|
333
|
+
message.includes("does not exist")
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
export function isChatInProgressError(err: unknown): boolean {
|
|
338
|
+
return errMessage(err).toLowerCase().includes("in progress");
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Qwen rejects a model the account cannot serve with Not_Found: Model not found.
|
|
343
|
+
* This is deterministic per request — retrying on the same (or any) account
|
|
344
|
+
* with the same model can never succeed, so it must terminate instead of
|
|
345
|
+
* burning retry attempts / account cooldowns and ending in a misleading 502.
|
|
346
|
+
*/
|
|
347
|
+
export function isModelNotFoundError(err: unknown): boolean {
|
|
348
|
+
const code = errCode(err).toLowerCase();
|
|
349
|
+
const message = errMessage(err).toLowerCase();
|
|
350
|
+
return (
|
|
351
|
+
(code === "not_found" &&
|
|
352
|
+
message.includes("model") &&
|
|
353
|
+
message.includes("not found")) ||
|
|
354
|
+
message.includes("model not found")
|
|
355
|
+
);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Browser fetch and ReadableStream failures often arrive as plain Error
|
|
360
|
+
* instances, especially when the stream is consumed outside Playwright.
|
|
361
|
+
* Keep this matcher narrow so local programming errors are not retried as
|
|
362
|
+
* account/network failures.
|
|
363
|
+
*/
|
|
364
|
+
export function isNetworkLikeError(err: unknown): boolean {
|
|
365
|
+
if (err instanceof QwenNetworkError) return true;
|
|
366
|
+
const message = errMessage(err).toLowerCase();
|
|
367
|
+
return (
|
|
368
|
+
message === "network error" ||
|
|
369
|
+
message.includes("failed to fetch") ||
|
|
370
|
+
message.includes("fetch failed") ||
|
|
371
|
+
message.includes("network connection was lost") ||
|
|
372
|
+
message.includes("connection reset") ||
|
|
373
|
+
message.includes("connection closed") ||
|
|
374
|
+
message.includes("socket hang up") ||
|
|
375
|
+
message.includes("econnreset") ||
|
|
376
|
+
message.includes("econnrefused") ||
|
|
377
|
+
message.includes("etimedout")
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Corrupted chat history: Qwen rejects because the first message in the
|
|
383
|
+
* upstream chat thread is an assistant message (broken parent_id chain).
|
|
384
|
+
* Recovery: force new chat + resend full prompt + switch account.
|
|
385
|
+
*/
|
|
386
|
+
export function isCorruptedChatHistoryError(err: unknown): boolean {
|
|
387
|
+
const message = errMessage(err).toLowerCase();
|
|
388
|
+
return (
|
|
389
|
+
message.includes("first message must not") ||
|
|
390
|
+
message.includes("first message must be") ||
|
|
391
|
+
message.includes("must not assistant message") ||
|
|
392
|
+
message.includes("must not be assistant")
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Build a RetryAction with sane defaults so each classification branch only
|
|
398
|
+
* spells out the fields it actually changes. Defaults: retryable, no account
|
|
399
|
+
* switch, same chat, delta replay, no delay. Branch ordering below is
|
|
400
|
+
* load-bearing (specific recoveries must win over broad substring matches).
|
|
401
|
+
*/
|
|
402
|
+
function makeRetryAction(
|
|
403
|
+
reason: string,
|
|
404
|
+
overrides: Partial<Omit<RetryAction, "reason">> = {},
|
|
405
|
+
): RetryAction {
|
|
406
|
+
return {
|
|
407
|
+
retryable: true,
|
|
408
|
+
switchAccount: false,
|
|
409
|
+
forceNewChat: false,
|
|
410
|
+
retryWithFullPrompt: false,
|
|
411
|
+
retryAfterMs: 0,
|
|
412
|
+
...overrides,
|
|
413
|
+
reason,
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Generic recovery policy for create-stream + mid-stream failures.
|
|
419
|
+
* Unknown upstream errors are retryable by default when enabled in config.
|
|
420
|
+
*/
|
|
421
|
+
export function classifyRetryAction(
|
|
422
|
+
err: unknown,
|
|
423
|
+
options?: {
|
|
424
|
+
clientDisconnected?: boolean;
|
|
425
|
+
requestAborted?: boolean;
|
|
426
|
+
baseDelayMs?: number;
|
|
427
|
+
},
|
|
428
|
+
): RetryAction {
|
|
429
|
+
const baseDelayMs = options?.baseDelayMs ?? config.retry.baseDelayMs;
|
|
430
|
+
const unknownEnabled = config.retry.onUnknownUpstream !== false;
|
|
431
|
+
|
|
432
|
+
if (
|
|
433
|
+
isClientAbortError(
|
|
434
|
+
err,
|
|
435
|
+
options?.clientDisconnected === true,
|
|
436
|
+
options?.requestAborted === true,
|
|
437
|
+
)
|
|
438
|
+
) {
|
|
439
|
+
return makeRetryAction("client_abort", { retryable: false });
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
if (isTerminalLocalError(err)) {
|
|
443
|
+
return makeRetryAction("terminal_local", { retryable: false });
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const message = errMessage(err).toLowerCase();
|
|
447
|
+
const code = errCode(err).toLowerCase();
|
|
448
|
+
if (isAccountInitializationError(err)) {
|
|
449
|
+
return makeRetryAction("account_initialization_failed", {
|
|
450
|
+
switchAccount: true,
|
|
451
|
+
retryAfterMs: Math.min(baseDelayMs, 1_000),
|
|
452
|
+
accountCooldownMs: config.concurrency.initFailureCooldownMs,
|
|
453
|
+
accountCooldownReason: "AuthInitFailed",
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
if (
|
|
458
|
+
code === "account_busy" ||
|
|
459
|
+
message.includes("waiting for a free slot") ||
|
|
460
|
+
message.includes("busy: timed out")
|
|
461
|
+
) {
|
|
462
|
+
return makeRetryAction("account_busy", {
|
|
463
|
+
switchAccount: true,
|
|
464
|
+
retryAfterMs: Math.min(baseDelayMs, 1_000),
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// Agent instructions ride ONLY the account-level personalization. An
|
|
469
|
+
// unconfirmed sync means this account cannot serve the request as-is —
|
|
470
|
+
// rotate to another account (each attempt re-syncs on its own account).
|
|
471
|
+
if (err instanceof PersonalizationSyncError) {
|
|
472
|
+
return makeRetryAction("personalization_sync_failed", {
|
|
473
|
+
switchAccount: true,
|
|
474
|
+
forceNewChat: true,
|
|
475
|
+
retryAfterMs: baseDelayMs,
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// Specialized recoveries first (even if wrapped as RetryableQwenStreamError)
|
|
480
|
+
// Corrupted chat history must win over broad "invalid input" matches.
|
|
481
|
+
// Try a fresh chat on the SAME account first — the corruption is in the
|
|
482
|
+
// upstream parent chain, not the account. Only rotate if the rebuild fails.
|
|
483
|
+
if (isCorruptedChatHistoryError(err)) {
|
|
484
|
+
return makeRetryAction("corrupted_chat_history", {
|
|
485
|
+
forceNewChat: true,
|
|
486
|
+
retryWithFullPrompt: true,
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// Chat missing must win over broad "invalid input" substring matches.
|
|
491
|
+
if (isChatNotExistError(err) || isChatInProgressError(err)) {
|
|
492
|
+
const typed = err as RetryableStreamError;
|
|
493
|
+
const inProgress = isChatInProgressError(err);
|
|
494
|
+
// chat_in_progress: do NOT switch immediately — the account is just
|
|
495
|
+
// temporarily busy (escalation happens in tryCreateStreamWithRetry after
|
|
496
|
+
// repeated failures). chat_not_exist needs a new chat + full replay;
|
|
497
|
+
// in_progress retries the same chat first.
|
|
498
|
+
return makeRetryAction(inProgress ? "chat_in_progress" : "chat_not_exist", {
|
|
499
|
+
forceNewChat: !inProgress,
|
|
500
|
+
retryWithFullPrompt: !inProgress,
|
|
501
|
+
retryAfterMs: inProgress
|
|
502
|
+
? (typed.retryAfterMs ?? config.retry.chatInProgressDelayMs)
|
|
503
|
+
: (typed.retryAfterMs ?? 0),
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
if (isInvalidInputError(err)) {
|
|
508
|
+
const typed = err as RetryableStreamError;
|
|
509
|
+
return makeRetryAction("invalid_input", {
|
|
510
|
+
switchAccount: typed.switchAccount !== false,
|
|
511
|
+
forceNewChat: true,
|
|
512
|
+
retryWithFullPrompt: true,
|
|
513
|
+
retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
|
|
514
|
+
dropFiles: typed.dropFiles,
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
// Content moderation rejections are deterministic — retrying on any
|
|
519
|
+
// account with the same content produces the same rejection. Fail fast
|
|
520
|
+
// instead of burning through accounts, personalization syncs and captchas.
|
|
521
|
+
if (isContentModerationError(err)) {
|
|
522
|
+
return makeRetryAction("content_moderation", { retryable: false });
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// Model not found is equally deterministic (the account cannot serve the
|
|
526
|
+
// requested model). Fail fast with a clear error instead of retrying the
|
|
527
|
+
// same doomed request and cooldown-marking accounts for ~5 hours.
|
|
528
|
+
if (isModelNotFoundError(err)) {
|
|
529
|
+
return makeRetryAction("model_not_found", { retryable: false });
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
if (isAntiBotError(err)) {
|
|
533
|
+
// WAF/captcha is only identified here. Retry the same request on the
|
|
534
|
+
// same account immediately; recovery, cooldown and account rotation are
|
|
535
|
+
// intentionally left out so the failure path stays observable.
|
|
536
|
+
return makeRetryAction("anti_bot");
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
if (isQuotaLikeError(err)) {
|
|
540
|
+
const typed = err as RetryableStreamError;
|
|
541
|
+
const quota = classifyQuotaCooldown(errMessage(err));
|
|
542
|
+
const isTemporary = quota.accountCooldownReason === "RateLimitTemporary";
|
|
543
|
+
// Temporary load shedding: retry same account first, only switch on
|
|
544
|
+
// repeated failure. Real quota exhaustion: switch immediately.
|
|
545
|
+
return makeRetryAction("quota_or_rate_limit", {
|
|
546
|
+
switchAccount: isTemporary ? false : typed.switchAccount !== false,
|
|
547
|
+
forceNewChat: typed.forceNewChat === true,
|
|
548
|
+
retryWithFullPrompt: typed.retryWithFullPrompt === true,
|
|
549
|
+
retryAfterMs: typed.retryAfterMs ?? (isTemporary ? 3_000 : baseDelayMs),
|
|
550
|
+
accountCooldownMs: quota.accountCooldownMs,
|
|
551
|
+
accountCooldownReason: quota.accountCooldownReason,
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
if (
|
|
556
|
+
isNetworkLikeError(err) ||
|
|
557
|
+
err instanceof QwenUpstreamUnavailableError ||
|
|
558
|
+
err instanceof QwenUpstreamError ||
|
|
559
|
+
isAbortError(err)
|
|
560
|
+
) {
|
|
561
|
+
const typed = err as RetryableStreamError;
|
|
562
|
+
return makeRetryAction(
|
|
563
|
+
isNetworkLikeError(err)
|
|
564
|
+
? "network"
|
|
565
|
+
: err instanceof QwenUpstreamUnavailableError
|
|
566
|
+
? "upstream_unavailable"
|
|
567
|
+
: isAbortError(err)
|
|
568
|
+
? "stream_aborted"
|
|
569
|
+
: "upstream_error",
|
|
570
|
+
{
|
|
571
|
+
switchAccount: typed.switchAccount !== false,
|
|
572
|
+
forceNewChat: true,
|
|
573
|
+
retryWithFullPrompt: typed.retryWithFullPrompt === true,
|
|
574
|
+
retryAfterMs:
|
|
575
|
+
typed.retryAfterMs ??
|
|
576
|
+
(isNetworkLikeError(err)
|
|
577
|
+
? 3000
|
|
578
|
+
: err instanceof QwenUpstreamUnavailableError
|
|
579
|
+
? 2000
|
|
580
|
+
: Math.min(baseDelayMs * 2, 3000)),
|
|
581
|
+
},
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// Preserve explicit RetryableQwenStreamError flags for remaining cases
|
|
586
|
+
if (err instanceof RetryableQwenStreamError) {
|
|
587
|
+
const typed = err as RetryableStreamError;
|
|
588
|
+
// Default switch unless caller explicitly set switchAccount=false
|
|
589
|
+
return makeRetryAction("explicit_retryable", {
|
|
590
|
+
switchAccount: typed.switchAccount !== false,
|
|
591
|
+
forceNewChat: typed.forceNewChat === true,
|
|
592
|
+
retryWithFullPrompt: typed.retryWithFullPrompt === true,
|
|
593
|
+
retryAfterMs: typed.retryAfterMs ?? baseDelayMs,
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// Default for unknown failures: retry when policy enabled
|
|
598
|
+
if (unknownEnabled) {
|
|
599
|
+
return makeRetryAction("unknown_upstream_default_retry", {
|
|
600
|
+
switchAccount: true,
|
|
601
|
+
forceNewChat: true,
|
|
602
|
+
retryAfterMs: baseDelayMs,
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
return makeRetryAction("unknown_not_retryable", { retryable: false });
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/** Build a RetryableQwenStreamError for SSE/mid-stream failures with policy flags. */
|
|
610
|
+
export function toRetryableStreamError(
|
|
611
|
+
errCode: string,
|
|
612
|
+
errDetails: string,
|
|
613
|
+
options?: Partial<RetryAction>,
|
|
614
|
+
): RetryableStreamError {
|
|
615
|
+
const policy = classifyRetryAction(
|
|
616
|
+
Object.assign(new Error(`${errCode}: ${errDetails}`), {
|
|
617
|
+
upstreamCode: errCode,
|
|
618
|
+
}),
|
|
619
|
+
);
|
|
620
|
+
const merged: RetryAction = {
|
|
621
|
+
...policy,
|
|
622
|
+
...options,
|
|
623
|
+
retryable: true,
|
|
624
|
+
reason: options?.reason || policy.reason,
|
|
625
|
+
};
|
|
626
|
+
|
|
627
|
+
const error = new RetryableQwenStreamError(
|
|
628
|
+
`Qwen retryable upstream error: ${errCode}: ${errDetails.substring(0, 200)}`,
|
|
629
|
+
merged.retryAfterMs || config.retry.baseDelayMs,
|
|
630
|
+
) as RetryableStreamError;
|
|
631
|
+
|
|
632
|
+
error.upstreamCode = errCode;
|
|
633
|
+
error.forceNewChat = merged.forceNewChat;
|
|
634
|
+
error.retryWithFullPrompt = merged.retryWithFullPrompt;
|
|
635
|
+
error.switchAccount = merged.switchAccount;
|
|
636
|
+
error.dropFiles = merged.dropFiles;
|
|
637
|
+
return error;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** For SSE error chunks: map any upstream SSE error to throw path. */
|
|
641
|
+
export function throwFromSseUpstreamError(
|
|
642
|
+
errCode: string,
|
|
643
|
+
errDetails: string,
|
|
644
|
+
): never {
|
|
645
|
+
const detailsLower = errDetails.toLowerCase();
|
|
646
|
+
// Qwen sometimes labels the chat-state error as RateLimited. Normalize it
|
|
647
|
+
// before retry/logging so it cannot be mistaken for account quota exhaustion.
|
|
648
|
+
const normalizedErrCode =
|
|
649
|
+
detailsLower.includes("chat is in progress") ||
|
|
650
|
+
detailsLower.includes("the chat is in progress")
|
|
651
|
+
? "chat_in_progress"
|
|
652
|
+
: errCode;
|
|
653
|
+
|
|
654
|
+
// Log upstream errors. Expected retryable codes (quota, rate limit, chat
|
|
655
|
+
// state) use warn level to avoid noisy stderr stack traces in production.
|
|
656
|
+
const expectedCodes = new Set([
|
|
657
|
+
"quota_limit",
|
|
658
|
+
"rate_limit",
|
|
659
|
+
"rate_limit_exceeded",
|
|
660
|
+
"chat_in_progress",
|
|
661
|
+
"invalid_input",
|
|
662
|
+
"data_inspection_failed",
|
|
663
|
+
]);
|
|
664
|
+
if (expectedCodes.has(normalizedErrCode.toLowerCase())) {
|
|
665
|
+
logger.warn(
|
|
666
|
+
`[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
|
|
667
|
+
);
|
|
668
|
+
} else {
|
|
669
|
+
console.error(
|
|
670
|
+
`[Upstream] Error | ${normalizedErrCode} | ${errDetails.substring(0, 200)}`,
|
|
671
|
+
);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// invalid_input keeps dedicated wording for logs/tests (not "chat is not exist")
|
|
675
|
+
const isChatMissing =
|
|
676
|
+
detailsLower.includes("is not exist") ||
|
|
677
|
+
detailsLower.includes("does not exist") ||
|
|
678
|
+
/\bnot exist\b/.test(detailsLower);
|
|
679
|
+
if (
|
|
680
|
+
!isChatMissing &&
|
|
681
|
+
(errCode.toLowerCase() === "invalid_input" ||
|
|
682
|
+
detailsLower.includes("entrada ou anexo inválido") ||
|
|
683
|
+
detailsLower.includes("invalid input") ||
|
|
684
|
+
detailsLower.includes("invalid attachment"))
|
|
685
|
+
) {
|
|
686
|
+
logger.warn("[Upstream] invalid_input mid-stream detected", {
|
|
687
|
+
code: errCode,
|
|
688
|
+
detailsLength: errDetails.length,
|
|
689
|
+
messageMentionsAttachment:
|
|
690
|
+
detailsLower.includes("anexo") || detailsLower.includes("attachment"),
|
|
691
|
+
messageMentionsFile:
|
|
692
|
+
detailsLower.includes("file") || detailsLower.includes("arquivo"),
|
|
693
|
+
});
|
|
694
|
+
|
|
695
|
+
const error = new RetryableQwenStreamError(
|
|
696
|
+
`Qwen retryable invalid input: ${errCode}: ${errDetails.substring(0, 200)}`,
|
|
697
|
+
config.retry.baseDelayMs,
|
|
698
|
+
) as RetryableStreamError;
|
|
699
|
+
error.upstreamCode = errCode;
|
|
700
|
+
error.forceNewChat = true;
|
|
701
|
+
error.retryWithFullPrompt = true;
|
|
702
|
+
error.switchAccount = true;
|
|
703
|
+
error.dropFiles = true; // Drop files on retry to isolate file-related errors
|
|
704
|
+
throw error;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// Content moderation rejections are deterministic — the same content will
|
|
708
|
+
// be rejected on every account. Throw as RetryableQwenStreamError so it
|
|
709
|
+
// propagates through the streaming catch blocks, but classifyRetryAction
|
|
710
|
+
// will mark it non-retryable.
|
|
711
|
+
if (isContentModerationError({ upstreamCode: normalizedErrCode, message: errDetails })) {
|
|
712
|
+
logger.warn(
|
|
713
|
+
`[Upstream] Content moderation rejection (not retrying): ${normalizedErrCode}`,
|
|
714
|
+
);
|
|
715
|
+
const error = new RetryableQwenStreamError(
|
|
716
|
+
`Qwen content moderation: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
|
|
717
|
+
0,
|
|
718
|
+
) as RetryableStreamError;
|
|
719
|
+
error.upstreamCode = normalizedErrCode;
|
|
720
|
+
error.switchAccount = false;
|
|
721
|
+
throw error;
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// A model the account cannot serve is a deterministic rejection too — never
|
|
725
|
+
// transparently retrofit this doomed model request on the same/other account.
|
|
726
|
+
if (isModelNotFoundError({ upstreamCode: normalizedErrCode, message: errDetails })) {
|
|
727
|
+
logger.warn(
|
|
728
|
+
`[Upstream] Model not available (not retrying): ${normalizedErrCode}`,
|
|
729
|
+
);
|
|
730
|
+
const error = new RetryableQwenStreamError(
|
|
731
|
+
`Qwen model not found: ${normalizedErrCode}: ${errDetails.substring(0, 200)}`,
|
|
732
|
+
0,
|
|
733
|
+
) as RetryableStreamError;
|
|
734
|
+
error.upstreamCode = normalizedErrCode;
|
|
735
|
+
error.switchAccount = false;
|
|
736
|
+
throw error;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
if (
|
|
740
|
+
errDetails.includes("FAIL_SYS_USER_VALIDATE") ||
|
|
741
|
+
errDetails.includes("RGV587_ERROR") ||
|
|
742
|
+
errDetails.includes("user validate")
|
|
743
|
+
) {
|
|
744
|
+
const error = new RetryableQwenStreamError(
|
|
745
|
+
`Qwen anti-bot: ${errCode}: ${errDetails}`,
|
|
746
|
+
0,
|
|
747
|
+
) as RetryableStreamError;
|
|
748
|
+
error.upstreamCode = errCode;
|
|
749
|
+
error.switchAccount = true;
|
|
750
|
+
throw error;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
throw toRetryableStreamError(normalizedErrCode, errDetails);
|
|
754
|
+
}
|