tt-help-cli-ycl 1.4.183 → 1.4.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-candidate-retry.mjs +496 -0
- package/scripts/test-crash-circuit-breaker.mjs +250 -0
- package/scripts/test-error-auto-reset.mjs +103 -0
- package/scripts/test-error-report.mjs +215 -0
- package/scripts/test-login-pool-acquire-order.mjs +92 -26
- package/scripts/test-login-pool-faceban-e2e.mjs +131 -0
- package/scripts/test-login-pool-faceban.mjs +235 -0
- package/scripts/test-login-pool-ui.mjs +2 -1
- package/scripts/test-login-pool.mjs +97 -54
- package/scripts/test-process-manager-label.mjs +145 -0
- package/src/cli/explore.js +171 -25
- package/src/cli/search.js +171 -18
- package/src/lib/browser/health-checker.js +19 -3
- package/src/lib/browser/page.js +23 -0
- package/src/lib/error-report.js +119 -0
- package/src/lib/login-pool-client.js +123 -0
- package/src/watch/data-store.js +25 -0
- package/src/watch/login-pool.js +118 -81
- package/src/watch/public/app.js +7 -2
- package/src/watch/public/index.html +2 -1
- package/src/watch/server.js +53 -8
- package/src/watchdog/process-manager.js +71 -6
- package/src/watchdog/reporter.js +2 -0
package/src/cli/search.js
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
getOrCreatePage,
|
|
27
27
|
safeCheckLogin,
|
|
28
28
|
isBrowserClosedError,
|
|
29
|
+
isRetryableRestoreFailure,
|
|
29
30
|
relaunchBrowser,
|
|
30
31
|
} from "../lib/browser/page.js";
|
|
31
32
|
import { collectSearchAuthors } from "../lib/search-collector.js";
|
|
@@ -39,16 +40,23 @@ import {
|
|
|
39
40
|
closeCaptcha,
|
|
40
41
|
} from "../scraper/modules/captcha-handler.js";
|
|
41
42
|
import { createApiClient } from "../lib/api-client.js";
|
|
43
|
+
import { createErrorReporter } from "../lib/error-report.js";
|
|
42
44
|
import {
|
|
43
45
|
HealthChecker,
|
|
44
|
-
MEMORY_THRESHOLD,
|
|
45
46
|
ROTATE_INTERVAL_MS,
|
|
46
47
|
} from "../lib/browser/health-checker.js";
|
|
47
48
|
import {
|
|
48
49
|
createLoginPoolClient,
|
|
49
50
|
injectPoolAccount,
|
|
50
51
|
ensurePoolLogin,
|
|
52
|
+
crashBackoffMs,
|
|
53
|
+
retryRestoreOnNavFailure,
|
|
54
|
+
CANDIDATE_INJECT_MAX_RETRIES,
|
|
55
|
+
CANDIDATE_INJECT_DELAY_MS,
|
|
56
|
+
decideCandidateInjectAction,
|
|
57
|
+
isRetryableInjectFailure,
|
|
51
58
|
} from "../lib/login-pool-client.js";
|
|
59
|
+
import { getClientConfig } from "../watchdog/reporter.js";
|
|
52
60
|
import {
|
|
53
61
|
server as cfgServer,
|
|
54
62
|
proxy as configuredProxy,
|
|
@@ -62,6 +70,7 @@ const STARTUP_TIKTOK_URL = "https://www.tiktok.com/@ycl5007";
|
|
|
62
70
|
const PAGE_GOTO_TIMEOUT = 60000;
|
|
63
71
|
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
64
72
|
const MAX_REBUILD_ATTEMPTS = 5; // 账户切换后导航恢复:连续重启浏览器超过该次数仍失败则上报并退出
|
|
73
|
+
const CRASH_BACKOFF_SLICE_MS = 1000; // 崩溃熔断/恢复重试等待的分片粒度(分片醒检 shuttingDown)
|
|
65
74
|
|
|
66
75
|
async function withRetry(label, fn) {
|
|
67
76
|
let backoff = 1000;
|
|
@@ -115,11 +124,14 @@ export async function handleSearch(parsed) {
|
|
|
115
124
|
// ── 初始化 ──
|
|
116
125
|
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
117
126
|
const poolMode = !searchNoPool;
|
|
127
|
+
// 内存阈值可配置覆盖(~/.tt-help.json memoryThreshold,默认 0.95):F16 类 7G 双浏览器机可调低提前重启
|
|
128
|
+
const memoryThresholdConfig = getClientConfig().memoryThreshold;
|
|
118
129
|
const healthChecker = poolMode
|
|
119
|
-
? new HealthChecker({ basePort, totalAccounts: 1 })
|
|
130
|
+
? new HealthChecker({ basePort, totalAccounts: 1, memoryThreshold: memoryThresholdConfig })
|
|
120
131
|
: new HealthChecker({
|
|
121
132
|
basePort,
|
|
122
133
|
totalAccounts: portCount,
|
|
134
|
+
memoryThreshold: memoryThresholdConfig,
|
|
123
135
|
});
|
|
124
136
|
let currentAccount = healthChecker.getCurrentAccount();
|
|
125
137
|
|
|
@@ -136,6 +148,21 @@ export async function handleSearch(parsed) {
|
|
|
136
148
|
let poolClient = null;
|
|
137
149
|
let poolAccount = null;
|
|
138
150
|
let poolAccountSince = Date.now();
|
|
151
|
+
// 崩溃熔断状态(F16 实录):浏览器级注入失败轮数 + 退避窗口截止时刻(窗口内不取号)
|
|
152
|
+
let crashStreak = 0;
|
|
153
|
+
let crashBackoffUntil = 0;
|
|
154
|
+
|
|
155
|
+
/** 分片 sleep:每 CRASH_BACKOFF_SLICE_MS 醒检 shuttingDown,被中断返回 false */
|
|
156
|
+
const sleepSliced = async (totalMs) => {
|
|
157
|
+
let remaining = totalMs;
|
|
158
|
+
while (remaining > 0) {
|
|
159
|
+
if (shuttingDown) return false;
|
|
160
|
+
const slice = Math.min(CRASH_BACKOFF_SLICE_MS, remaining);
|
|
161
|
+
await new Promise((r) => setTimeout(r, slice));
|
|
162
|
+
remaining -= slice;
|
|
163
|
+
}
|
|
164
|
+
return !shuttingDown;
|
|
165
|
+
};
|
|
139
166
|
|
|
140
167
|
const cleanup = () => {
|
|
141
168
|
if (shuttingDown) return;
|
|
@@ -167,6 +194,15 @@ export async function handleSearch(parsed) {
|
|
|
167
194
|
}
|
|
168
195
|
log(`[Search] 用户编号: ${userId}`);
|
|
169
196
|
|
|
197
|
+
// 客户端错误上报封装:自动附带 poolUsername(报错时在手池号)与 proxyNode(当前 clash 节点)
|
|
198
|
+
const errorReporter = createErrorReporter({
|
|
199
|
+
apiPost,
|
|
200
|
+
serverUrl: baseUrl,
|
|
201
|
+
userId,
|
|
202
|
+
proxyUrl: resolvedProxy,
|
|
203
|
+
getState: () => ({ poolUsername: poolAccount?.username || "" }),
|
|
204
|
+
});
|
|
205
|
+
|
|
170
206
|
setDelayConfig("normal");
|
|
171
207
|
|
|
172
208
|
try {
|
|
@@ -378,8 +414,7 @@ export async function handleSearch(parsed) {
|
|
|
378
414
|
log(
|
|
379
415
|
`[致命] 账户切换后导航持续失败,已重启浏览器 ${rebuildCount - 1} 次仍无法恢复,上报后退出`,
|
|
380
416
|
);
|
|
381
|
-
await
|
|
382
|
-
userId,
|
|
417
|
+
await errorReporter.report({
|
|
383
418
|
username: "account-switch",
|
|
384
419
|
errorType: "account_switch_failed",
|
|
385
420
|
errorMessage: `账户切换后导航持续失败: ${err.message}`,
|
|
@@ -435,8 +470,43 @@ export async function handleSearch(parsed) {
|
|
|
435
470
|
|
|
436
471
|
// ── 登录池模式辅助 ──
|
|
437
472
|
|
|
438
|
-
async function switchPoolAccount(reason) {
|
|
473
|
+
async function switchPoolAccount(reason, opts = {}) {
|
|
439
474
|
log(`\n[换号] ${reason},从账号中心取新账号...`);
|
|
475
|
+
// 崩溃熔断:退避窗口内不取号(保护账号池),分片 sleep 响应 shuttingDown
|
|
476
|
+
const backoffWaitMs = Math.max(0, crashBackoffUntil - Date.now());
|
|
477
|
+
if (backoffWaitMs > 0) {
|
|
478
|
+
log(
|
|
479
|
+
`[浏览器] 崩溃熔断中,${Math.ceil(backoffWaitMs / 1000)}s 后再试(不取号,保护账号池)`,
|
|
480
|
+
);
|
|
481
|
+
if (!(await sleepSliced(backoffWaitMs))) return false;
|
|
482
|
+
}
|
|
483
|
+
// 候选注入失败命中浏览器级崩溃签名时的处置(首判与重试中途重判共用):
|
|
484
|
+
// crashStreak++ 按轮退避 + 释放候选 + 上报 + 转同号重启;返回值由调用方直接 return
|
|
485
|
+
async function abortCandidateLoopOnCrash(crashReason, candidate, fromRecovery) {
|
|
486
|
+
// 浏览器级崩溃:不怪账号,立即中止候选循环(不消耗 inject_fail 口径),
|
|
487
|
+
// 已取候选照常 release,转同号重启(与内存超限路径同语义);按轮熔断退避
|
|
488
|
+
crashStreak++;
|
|
489
|
+
const backoffMs = crashBackoffMs(crashStreak);
|
|
490
|
+
crashBackoffUntil = Date.now() + backoffMs;
|
|
491
|
+
log(
|
|
492
|
+
`[换号] @${candidate.username} 注入失败为浏览器级崩溃(${crashReason}),中止候选循环(第 ${crashStreak} 轮,熔断 ${Math.round(backoffMs / 1000)}s),释放候选...`,
|
|
493
|
+
);
|
|
494
|
+
await poolClient.release(candidate).catch(() => {});
|
|
495
|
+
await errorReporter.report({
|
|
496
|
+
username: poolAccount ? poolAccount.username : "unknown",
|
|
497
|
+
errorType: "account_switch_failed",
|
|
498
|
+
errorMessage: `注入失败为浏览器级崩溃,中止候选循环转同号重启(第${crashStreak}轮,熔断${Math.round(backoffMs / 1000)}s): ${crashReason}`,
|
|
499
|
+
stage: "account-switch",
|
|
500
|
+
}).catch(() => {});
|
|
501
|
+
if (fromRecovery) {
|
|
502
|
+
// 恢复链中(重启后恢复失败触发的换号)不再递归重启,保留现场等熔断窗口
|
|
503
|
+
log(`[换号] 恢复链中不再递归重启浏览器,保留现场等待熔断窗口`);
|
|
504
|
+
return false;
|
|
505
|
+
}
|
|
506
|
+
return await restartPoolBrowser(
|
|
507
|
+
`注入验证遭遇浏览器级崩溃(${crashReason})`,
|
|
508
|
+
);
|
|
509
|
+
}
|
|
440
510
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
441
511
|
let next = null;
|
|
442
512
|
try {
|
|
@@ -447,18 +517,77 @@ export async function handleSearch(parsed) {
|
|
|
447
517
|
);
|
|
448
518
|
return false;
|
|
449
519
|
}
|
|
450
|
-
|
|
520
|
+
let r = await injectPoolAccount(page, next, {
|
|
451
521
|
verifyUsername: next.username,
|
|
452
522
|
}).catch((e) => ({ ok: false, reason: e.message }));
|
|
453
523
|
if (!r.ok) {
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
524
|
+
// 注入失败三路分流(与 explore.js 镜像):崩溃 → 既有熔断(中止候选循环);
|
|
525
|
+
// 确定性号死 → 立即弃号换下一个候选(现状行为);
|
|
526
|
+
// 瞬态/环境失败 → 同一候选原号重试(共 3 次尝试,间隔 15s),重试期间租约保持在手
|
|
527
|
+
let action = decideCandidateInjectAction(r.reason, isBrowserClosedError);
|
|
528
|
+
if (action === "circuit-break") {
|
|
529
|
+
return await abortCandidateLoopOnCrash(r.reason, next, opts.fromRecovery);
|
|
530
|
+
}
|
|
531
|
+
if (action === "retry") {
|
|
532
|
+
// 瞬态/环境失败(超时/AbortError/net::ERR_*/Execution context destroyed/无账号数据/导航类):
|
|
533
|
+
// 原号重试最多 2 次(共 3 次尝试),间隔 15s,重试期间不 release(租约保持在手);
|
|
534
|
+
// 确定性号死(会话过期/身份不符)不进此分支,直接弃号换下一个候选
|
|
535
|
+
r = await retryRestoreOnNavFailure(
|
|
536
|
+
() =>
|
|
537
|
+
injectPoolAccount(page, next, { verifyUsername: next.username }),
|
|
538
|
+
r.reason,
|
|
539
|
+
{
|
|
540
|
+
isRetryable: isRetryableInjectFailure,
|
|
541
|
+
maxRetries: CANDIDATE_INJECT_MAX_RETRIES,
|
|
542
|
+
delayMs: CANDIDATE_INJECT_DELAY_MS,
|
|
543
|
+
sleep: (ms) => sleepSliced(ms),
|
|
544
|
+
shouldAbort: () => shuttingDown,
|
|
545
|
+
onRetry: (n, delayMs, why) =>
|
|
546
|
+
log(
|
|
547
|
+
`[换号] @${next.username} 第${n}次注入失败(瞬态,${why}),${Math.round(delayMs / 1000)}s 后原号重试`,
|
|
548
|
+
),
|
|
549
|
+
},
|
|
550
|
+
);
|
|
551
|
+
if (r.aborted) {
|
|
552
|
+
// shuttingDown 中断:释放候选退出(租约主动还干净,服务端 TTL 兑底)
|
|
553
|
+
log(`[换号] @${next.username} 原号重试等待中被关闭信号中断,释放候选`);
|
|
554
|
+
await poolClient.release(next).catch(() => {});
|
|
555
|
+
return false;
|
|
556
|
+
}
|
|
557
|
+
if (r.ok && r.attempts > 1) {
|
|
558
|
+
log(
|
|
559
|
+
`[换号] @${next.username} 原号重试成功(第 ${r.attempts} 次尝试)`,
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
if (!r.ok) {
|
|
563
|
+
// 重试中途失败签名可能升级为浏览器级崩溃(如首败 timeout、再败 Page crashed):
|
|
564
|
+
// 对最终原因重判,命中熔断则中止候选循环(不弃号给下一候选——坏浏览器再吃一个号),
|
|
565
|
+
// 升级为确定性号死则 fall through 到下方弃号分支换候选
|
|
566
|
+
action = decideCandidateInjectAction(r.reason, isBrowserClosedError);
|
|
567
|
+
if (action === "circuit-break") {
|
|
568
|
+
return await abortCandidateLoopOnCrash(r.reason, next, opts.fromRecovery);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
if (!r.ok) {
|
|
573
|
+
if (r.attempts > 1) {
|
|
574
|
+
log(
|
|
575
|
+
`[换号] @${next.username} 原号重试 ${r.attempts - 1} 次后仍失败(共 ${r.attempts} 次尝试,${r.reason}),释放候选换下一个...`,
|
|
576
|
+
);
|
|
577
|
+
} else {
|
|
578
|
+
log(
|
|
579
|
+
`[换号] @${next.username} 注入验证失败(${r.reason}),释放候选换下一个...`,
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
await poolClient.release(next).catch(() => {});
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
457
585
|
}
|
|
458
586
|
const prev = poolAccount;
|
|
459
587
|
poolAccount = next;
|
|
460
588
|
poolAccountSince = Date.now();
|
|
461
589
|
loggedIn = true;
|
|
590
|
+
crashStreak = 0; // 注入验证成功即清零熔断轮数
|
|
462
591
|
captchaCount = 0;
|
|
463
592
|
staleRounds = 0;
|
|
464
593
|
consecutiveNetworkErrors = 0;
|
|
@@ -472,8 +601,7 @@ export async function handleSearch(parsed) {
|
|
|
472
601
|
return true;
|
|
473
602
|
}
|
|
474
603
|
log(`[换号] 连续 5 个候选注入验证失败,保留现场等待下轮重试`);
|
|
475
|
-
await
|
|
476
|
-
userId,
|
|
604
|
+
await errorReporter.report({
|
|
477
605
|
username: poolAccount ? poolAccount.username : "unknown",
|
|
478
606
|
errorType: "account_switch_failed",
|
|
479
607
|
errorMessage: "连续5个候选账号注入验证失败(详情见节点日志)",
|
|
@@ -485,17 +613,39 @@ export async function handleSearch(parsed) {
|
|
|
485
613
|
/** 浏览器重建后恢复在手账号:磁盘态一致则免注入,否则用内存 cookies 补注入 */
|
|
486
614
|
async function restorePoolLogin() {
|
|
487
615
|
if (!poolAccount) return false;
|
|
488
|
-
|
|
616
|
+
let r = await ensurePoolLogin(page, poolAccount).catch((e) => ({
|
|
489
617
|
ok: false,
|
|
490
618
|
reason: e.message,
|
|
491
619
|
}));
|
|
620
|
+
if (!r.ok) {
|
|
621
|
+
// 导航类瞬态失败(net::ERR_* / 超时 / 崩溃)重试后再弃号,重试期间不弃号;
|
|
622
|
+
// 账号级失败(身份不符/会话过期)不重试直接走换号
|
|
623
|
+
r = await retryRestoreOnNavFailure(
|
|
624
|
+
() => ensurePoolLogin(page, poolAccount),
|
|
625
|
+
r.reason,
|
|
626
|
+
{
|
|
627
|
+
isRetryable: isRetryableRestoreFailure,
|
|
628
|
+
sleep: (ms) => sleepSliced(ms),
|
|
629
|
+
shouldAbort: () => shuttingDown,
|
|
630
|
+
},
|
|
631
|
+
);
|
|
632
|
+
if (r.aborted) return false;
|
|
633
|
+
if (!r.ok && r.attempts > 1) {
|
|
634
|
+
log(`[登录池] 登录态恢复重试 ${r.attempts} 次后仍失败`);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
492
637
|
if (r.ok) {
|
|
493
638
|
loggedIn = true;
|
|
639
|
+
crashStreak = 0; // 恢复成功(含注入验证成功)即清零熔断轮数
|
|
494
640
|
log(`[登录池] 登录态已恢复 @${poolAccount.username}`);
|
|
495
641
|
return true;
|
|
496
642
|
}
|
|
497
643
|
log(`[登录池] 登录态恢复失败(${r.reason}),转入换号流程`);
|
|
498
|
-
|
|
644
|
+
// 恢复链中的换号不再嵌套浏览器重启(fromRecovery):崩溃则熔断退避,不烧池
|
|
645
|
+
return await switchPoolAccount(
|
|
646
|
+
`浏览器重启后无法恢复 @${poolAccount.username} 的登录态`,
|
|
647
|
+
{ fromRecovery: true },
|
|
648
|
+
);
|
|
499
649
|
}
|
|
500
650
|
|
|
501
651
|
/** 内存超限:同号重启浏览器(不换号——内存是浏览器进程问题,不是账号问题) */
|
|
@@ -509,7 +659,10 @@ export async function handleSearch(parsed) {
|
|
|
509
659
|
page = await setupNewPage(browser);
|
|
510
660
|
} catch (e) {
|
|
511
661
|
log(`[浏览器] 重启失败: ${e.message},转入换号流程`);
|
|
512
|
-
|
|
662
|
+
// 恢复链中的换号不再嵌套浏览器重启(fromRecovery):崩溃则熔断退避,不烧池
|
|
663
|
+
return await switchPoolAccount(`浏览器重启失败(${e.message})`, {
|
|
664
|
+
fromRecovery: true,
|
|
665
|
+
});
|
|
513
666
|
}
|
|
514
667
|
return await restorePoolLogin();
|
|
515
668
|
}
|
|
@@ -540,7 +693,7 @@ export async function handleSearch(parsed) {
|
|
|
540
693
|
// 登录池模式:内存超限→同号重启浏览器;48min→换号(原地切号)
|
|
541
694
|
const memRatio = healthChecker.memoryUsage();
|
|
542
695
|
const elapsedMs = Date.now() - poolAccountSince;
|
|
543
|
-
if (memRatio >=
|
|
696
|
+
if (memRatio >= healthChecker.memoryThreshold) {
|
|
544
697
|
await restartPoolBrowser(`系统内存 ${(memRatio * 100).toFixed(1)}% 超阈值`);
|
|
545
698
|
} else if (elapsedMs >= ROTATE_INTERVAL_MS) {
|
|
546
699
|
await handleAccountSwitch(
|
|
@@ -582,7 +735,7 @@ export async function handleSearch(parsed) {
|
|
|
582
735
|
|
|
583
736
|
// 上报
|
|
584
737
|
await withRetry("report captcha", () =>
|
|
585
|
-
|
|
738
|
+
errorReporter.report({
|
|
586
739
|
userId: clientId,
|
|
587
740
|
username: "unknown",
|
|
588
741
|
errorType: "captcha",
|
|
@@ -739,7 +892,7 @@ export async function handleSearch(parsed) {
|
|
|
739
892
|
|
|
740
893
|
const errorType = consecutiveNetworkErrors > 1 ? "network" : "other";
|
|
741
894
|
await withRetry("report error", () =>
|
|
742
|
-
|
|
895
|
+
errorReporter.report({
|
|
743
896
|
userId: clientId,
|
|
744
897
|
username: tag,
|
|
745
898
|
errorType,
|
|
@@ -772,7 +925,7 @@ export async function handleSearch(parsed) {
|
|
|
772
925
|
log(` [验证码] 搜索后检测到验证码,累计 ${captchaCount} 次`);
|
|
773
926
|
|
|
774
927
|
await withRetry("report captcha", () =>
|
|
775
|
-
|
|
928
|
+
errorReporter.report({
|
|
776
929
|
userId: clientId,
|
|
777
930
|
username: tag,
|
|
778
931
|
errorType: "captcha",
|
|
@@ -5,12 +5,28 @@ import { execSync } from "child_process";
|
|
|
5
5
|
const DEFAULT_BASE_PORT = 9222;
|
|
6
6
|
const DEFAULT_TOTAL_ACCOUNTS = 10;
|
|
7
7
|
const MEMORY_THRESHOLD = 0.95;
|
|
8
|
+
// 配置覆盖(~/.tt-help.json memoryThreshold)钳制边界:过低误触发频繁重启,过高失去保护意义
|
|
9
|
+
const MEMORY_THRESHOLD_MIN = 0.5;
|
|
10
|
+
const MEMORY_THRESHOLD_MAX = 0.98;
|
|
8
11
|
const ROTATE_INTERVAL_MS = 0.8 * 60 * 60 * 1000;
|
|
9
12
|
|
|
13
|
+
/**
|
|
14
|
+
* 内存阈值解析:缺省/非法(null/undefined/空串/非数值)回退默认 0.95,
|
|
15
|
+
* 合法值钳制 [0.5, 0.98]。纯函数,供 HealthChecker 构造与单测使用。
|
|
16
|
+
*/
|
|
17
|
+
function resolveMemoryThreshold(raw) {
|
|
18
|
+
if (raw === null || raw === undefined || raw === "") return MEMORY_THRESHOLD;
|
|
19
|
+
const n = Number(raw);
|
|
20
|
+
if (!Number.isFinite(n)) return MEMORY_THRESHOLD;
|
|
21
|
+
return Math.min(MEMORY_THRESHOLD_MAX, Math.max(MEMORY_THRESHOLD_MIN, n));
|
|
22
|
+
}
|
|
23
|
+
|
|
10
24
|
class HealthChecker {
|
|
11
25
|
constructor(options = {}) {
|
|
12
26
|
const basePort = options.basePort ?? DEFAULT_BASE_PORT;
|
|
13
27
|
const totalAccounts = options.totalAccounts ?? DEFAULT_TOTAL_ACCOUNTS;
|
|
28
|
+
// 内存阈值可配置覆盖(默认 0.95):F16 类 7G 双浏览器机可调低提前重启(explore/search 构造时传入)
|
|
29
|
+
this.memoryThreshold = resolveMemoryThreshold(options.memoryThreshold);
|
|
14
30
|
|
|
15
31
|
this.accounts = [];
|
|
16
32
|
for (let i = 0; i < totalAccounts; i++) {
|
|
@@ -83,7 +99,7 @@ class HealthChecker {
|
|
|
83
99
|
const memUsage = this.memoryUsage();
|
|
84
100
|
const memPercent = (memUsage * 100).toFixed(1);
|
|
85
101
|
|
|
86
|
-
if (memUsage >=
|
|
102
|
+
if (memUsage >= this.memoryThreshold) {
|
|
87
103
|
const currentPort = this.accounts[this.currentIndex].port;
|
|
88
104
|
return {
|
|
89
105
|
shouldSwitch: true,
|
|
@@ -104,7 +120,7 @@ class HealthChecker {
|
|
|
104
120
|
// 计算预计切换时间
|
|
105
121
|
const remainingMs = ROTATE_INTERVAL_MS - elapsed;
|
|
106
122
|
const remainingMins = Math.round(remainingMs / 60000);
|
|
107
|
-
const memHeadroom = ((
|
|
123
|
+
const memHeadroom = ((this.memoryThreshold - memUsage) * 100).toFixed(1);
|
|
108
124
|
const currentPort = this.accounts[this.currentIndex].port;
|
|
109
125
|
|
|
110
126
|
return {
|
|
@@ -114,4 +130,4 @@ class HealthChecker {
|
|
|
114
130
|
}
|
|
115
131
|
}
|
|
116
132
|
|
|
117
|
-
export { HealthChecker, MEMORY_THRESHOLD, ROTATE_INTERVAL_MS };
|
|
133
|
+
export { HealthChecker, MEMORY_THRESHOLD, MEMORY_THRESHOLD_MIN, MEMORY_THRESHOLD_MAX, resolveMemoryThreshold, ROTATE_INTERVAL_MS };
|
package/src/lib/browser/page.js
CHANGED
|
@@ -15,12 +15,20 @@ const DEFAULT_USER_DATA_DIR = path.join(
|
|
|
15
15
|
const BROWSER_CLOSED_PATTERNS = [
|
|
16
16
|
"Target page, context or browser has been closed",
|
|
17
17
|
"Target closed",
|
|
18
|
+
"Target crashed",
|
|
19
|
+
"Page crashed",
|
|
18
20
|
"Browser has been closed",
|
|
19
21
|
"Protocol error",
|
|
20
22
|
"Connection closed",
|
|
21
23
|
"net::ERR_CONNECTION",
|
|
22
24
|
];
|
|
23
25
|
|
|
26
|
+
/**
|
|
27
|
+
* 判定浏览器级故障:浏览器关闭**或渲染进程崩溃**。
|
|
28
|
+
* 渲染崩溃(Target/Target crashed / Page crashed)发生时 CDP 连接往往仍存活,
|
|
29
|
+
* 但页面已死,须与连接断开同语义处理:重建浏览器 + 重试。
|
|
30
|
+
* (F16 实录:崩溃曾被误判为「账号注入失败」驱动换号循环,单日烧 28 个账号)
|
|
31
|
+
*/
|
|
24
32
|
export function isBrowserClosedError(error) {
|
|
25
33
|
if (!error) return false;
|
|
26
34
|
const msg = error.message || error.toString() || "";
|
|
@@ -71,6 +79,21 @@ const DOM_CHECK_RETRIES = 3; // DOM 检测最大重试次数
|
|
|
71
79
|
const SAFE_CHECK_ROUNDS = 2; // 安全登录检测轮数(首次检测 + 额外1轮确认)
|
|
72
80
|
const SAFE_CHECK_INTERVAL = 5000; // 安全登录检测每轮间隔 5 秒
|
|
73
81
|
|
|
82
|
+
/**
|
|
83
|
+
* 登录态恢复失败是否属导航类瞬态失败(可重试,重试期间不弃号):
|
|
84
|
+
* - net::ERR_*(含 ERR_HTTP_RESPONSE_CODE_FAILURE / ERR_CONNECTION_* 等导航层错误)
|
|
85
|
+
* - 超时(Timeout ... exceeded / timed out)
|
|
86
|
+
* - 浏览器级关闭或渲染崩溃(isBrowserClosedError)
|
|
87
|
+
* 账号级失败(身份不符/会话过期/无账号数据等)不可重试,直接走换号流程。
|
|
88
|
+
*/
|
|
89
|
+
export function isRetryableRestoreFailure(reason) {
|
|
90
|
+
if (!reason) return false;
|
|
91
|
+
const msg = String(reason);
|
|
92
|
+
if (msg.includes("net::ERR_")) return true;
|
|
93
|
+
if (/timed?\s*out|timeout/i.test(msg)) return true;
|
|
94
|
+
return isBrowserClosedError(new Error(msg));
|
|
95
|
+
}
|
|
96
|
+
|
|
74
97
|
/**
|
|
75
98
|
* 判断登录状态:Cookie 为主,DOM 验真为辅。
|
|
76
99
|
* - 无 sessionid Cookie → 未登录
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 客户端错误上报封装(方案甲:客户端报错自带字段)
|
|
3
|
+
*
|
|
4
|
+
* 背景:/api/error-report 的旧 username 字段语义混乱(在手池号/"account-switch"
|
|
5
|
+
* 占位/被探索目标/"unknown" 四种混传),且代理节点完全没报。本封装统一附加:
|
|
6
|
+
* - poolUsername:报错时在手池号(getState 注入,无在手号空串,不传 "unknown")
|
|
7
|
+
* - proxyNode:当前 clash 节点名(三级 fallback:端口匹配通道 → 主通道 → 空串)
|
|
8
|
+
* 旧 username 字段原样透传(兼容存量页面显示),语义不动。
|
|
9
|
+
*
|
|
10
|
+
* 用法:
|
|
11
|
+
* const errorReporter = createErrorReporter({
|
|
12
|
+
* apiPost, serverUrl, userId,
|
|
13
|
+
* proxyUrl: resolvedProxy, // 可选,缺省用 ~/.tt-help.json 的 proxy
|
|
14
|
+
* getState: () => ({ poolUsername: poolAccount?.username || "" }),
|
|
15
|
+
* });
|
|
16
|
+
* await errorReporter.report({ errorType, errorMessage, stage, errorStack, username });
|
|
17
|
+
*
|
|
18
|
+
* 单测:scripts/test-error-report.mjs
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { proxy as configuredProxy } from "./constants.js";
|
|
22
|
+
import {
|
|
23
|
+
getClashHealth,
|
|
24
|
+
getClashHealthChannels,
|
|
25
|
+
} from "../watchdog/reporter.js";
|
|
26
|
+
|
|
27
|
+
/** 从代理地址提取端口(http://127.0.0.1:7897 → 7897;解析不出返回 0) */
|
|
28
|
+
export function parseProxyPort(url) {
|
|
29
|
+
if (!url) return 0;
|
|
30
|
+
try {
|
|
31
|
+
const u = new URL(url);
|
|
32
|
+
const p = parseInt(u.port, 10);
|
|
33
|
+
if (p > 0) return p;
|
|
34
|
+
} catch {
|
|
35
|
+
/* 非 URL 形态走正则兑底 */
|
|
36
|
+
}
|
|
37
|
+
const m = String(url).match(/:(\d+)/);
|
|
38
|
+
return m ? parseInt(m[1], 10) || 0 : 0;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 解析当前 clash 节点名(三级 fallback):
|
|
43
|
+
* 1. 与业务代理端口匹配且非 stale 的通道(clash-health-{port}.json)的 node
|
|
44
|
+
* 2. 主通道(clash-health.json,daemon 对主控制端双写)非 stale 的 node
|
|
45
|
+
* 3. 读不到(无 clash daemon / 文件 stale / 无 node 字段)→ ""
|
|
46
|
+
*
|
|
47
|
+
* @param {string|null} proxyUrl - 业务代理地址(--proxy 或 ~/.tt-help.json 的 proxy)
|
|
48
|
+
* @param {{channels?: Array<object>, health?: object|null}} [deps] - 测试注入用
|
|
49
|
+
* @returns {string}
|
|
50
|
+
*/
|
|
51
|
+
export function resolveProxyNode(proxyUrl, deps = {}) {
|
|
52
|
+
const channels = deps.channels ?? getClashHealthChannels();
|
|
53
|
+
const health = deps.health !== undefined ? deps.health : getClashHealth();
|
|
54
|
+
const port = parseProxyPort(proxyUrl);
|
|
55
|
+
if (port > 0) {
|
|
56
|
+
const ch = channels.find((c) => c && c.port === port && !c.stale);
|
|
57
|
+
if (ch && ch.node) return String(ch.node);
|
|
58
|
+
}
|
|
59
|
+
if (health && !health.stale && health.node) return String(health.node);
|
|
60
|
+
return "";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* 创建错误上报器
|
|
65
|
+
*
|
|
66
|
+
* @param {object} opts
|
|
67
|
+
* @param {(url: string, body: object) => Promise<any>} opts.apiPost - API 客户端 POST(createApiClient 产出)
|
|
68
|
+
* @param {string} opts.serverUrl - watch 服务端地址
|
|
69
|
+
* @param {string} opts.userId - 客户端用户编号(机器级;单次调用可用 userId 覆盖)
|
|
70
|
+
* @param {string|null} [opts.proxyUrl] - 业务代理地址(缺省读 ~/.tt-help.json 的 proxy)
|
|
71
|
+
* @param {() => {poolUsername?: string}} [opts.getState] - 报错时刻状态(CLI 闭包注入在手池号)
|
|
72
|
+
* @returns {{ report: (info: object) => Promise<any> }}
|
|
73
|
+
*/
|
|
74
|
+
export function createErrorReporter({
|
|
75
|
+
apiPost,
|
|
76
|
+
serverUrl,
|
|
77
|
+
userId,
|
|
78
|
+
proxyUrl,
|
|
79
|
+
getState,
|
|
80
|
+
}) {
|
|
81
|
+
if (!apiPost || !serverUrl) {
|
|
82
|
+
throw new Error("createErrorReporter 需要 apiPost 与 serverUrl");
|
|
83
|
+
}
|
|
84
|
+
// proxyUrl 未显式传(undefined)才回退配置文件;显式传 null 表示确无代理
|
|
85
|
+
const effectiveProxyUrl = proxyUrl === undefined ? configuredProxy : proxyUrl;
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
/**
|
|
89
|
+
* 上报一条客户端异常
|
|
90
|
+
* @param {object} info
|
|
91
|
+
* @param {string} [info.userId] - 单次覆盖(如 search 每轮标签搜索的随机 clientId)
|
|
92
|
+
* @param {string} info.errorType
|
|
93
|
+
* @param {string} [info.errorMessage]
|
|
94
|
+
* @param {string} [info.stage]
|
|
95
|
+
* @param {string} [info.errorStack]
|
|
96
|
+
* @param {string} [info.username] - 旧字段原样透传(兼容),语义不动
|
|
97
|
+
*/
|
|
98
|
+
async report({
|
|
99
|
+
userId: callUserId,
|
|
100
|
+
errorType,
|
|
101
|
+
errorMessage,
|
|
102
|
+
stage,
|
|
103
|
+
errorStack,
|
|
104
|
+
username,
|
|
105
|
+
} = {}) {
|
|
106
|
+
const state = getState ? getState() || {} : {};
|
|
107
|
+
return apiPost(`${serverUrl}/api/error-report`, {
|
|
108
|
+
userId: callUserId || userId,
|
|
109
|
+
username: username || "",
|
|
110
|
+
errorType: errorType || "other",
|
|
111
|
+
errorMessage: errorMessage || "",
|
|
112
|
+
stage: stage || "",
|
|
113
|
+
errorStack: errorStack || "",
|
|
114
|
+
poolUsername: state.poolUsername || "",
|
|
115
|
+
proxyNode: resolveProxyNode(effectiveProxyUrl),
|
|
116
|
+
});
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
@@ -193,3 +193,126 @@ export async function ensurePoolLogin(page, account) {
|
|
|
193
193
|
}));
|
|
194
194
|
return { ...r, injected: true };
|
|
195
195
|
}
|
|
196
|
+
|
|
197
|
+
// ── 崩溃熔断与恢复重试(纯函数/纯常量,2026-11-04 F16 实录驱动)──
|
|
198
|
+
// 背景:渲染进程崩溃(Target/Page crashed)曾被误判为「账号注入失败」→ 换号循环烧池
|
|
199
|
+
// (单日 28 号)。浏览器级失败须熔断退避(不取号保护账号池),账号级失败才候选轮换。
|
|
200
|
+
// 本组常量/函数保持零依赖(login-pool-client 同时被 watchdog 侧引用,勿引入 playwright 链)。
|
|
201
|
+
|
|
202
|
+
/** 崩溃熔断退避基数:第 1 轮 60s */
|
|
203
|
+
export const CRASH_BACKOFF_BASE_MS = 60 * 1000;
|
|
204
|
+
/** 崩溃熔断退避上限:15 分钟 */
|
|
205
|
+
export const CRASH_BACKOFF_MAX_MS = 15 * 60 * 1000;
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* 崩溃熔断退避时长:第 n 轮 = base × 2^(n-1),封顶 max;n 非正整数返回 0。
|
|
209
|
+
* 注入验证成功即清零轮数(清零由调用方闭包状态负责)。
|
|
210
|
+
*/
|
|
211
|
+
export function crashBackoffMs(crashStreak, base = CRASH_BACKOFF_BASE_MS, max = CRASH_BACKOFF_MAX_MS) {
|
|
212
|
+
if (!Number.isInteger(crashStreak) || crashStreak <= 0) return 0;
|
|
213
|
+
return Math.min(base * 2 ** (crashStreak - 1), max);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** 登录态恢复首败后追加重试次数(共 3 次尝试,全败才弃号转 recover_fail) */
|
|
217
|
+
export const RESTORE_RETRY_MAX_RETRIES = 2;
|
|
218
|
+
/** 恢复重试间隔 */
|
|
219
|
+
export const RESTORE_RETRY_DELAY_MS = 10 * 1000;
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* 登录态恢复重试驱动(纯逻辑,I/O 与谓词均注入,可单测;本文件保持零依赖):
|
|
223
|
+
* 首次尝试已失败(firstFailure 为其失败原因),谓词判定为导航类瞬态失败则再试
|
|
224
|
+
* maxRetries 次、间隔 delayMs;账号级失败不重试直接返回。
|
|
225
|
+
* 调用方须传入 isRetryable(客户端用 page.js 的 isRetryableRestoreFailure)。
|
|
226
|
+
* @param {() => Promise<{ok: boolean, reason?: string}>} attemptFn 重试尝试
|
|
227
|
+
* @param {string} firstFailure 首次失败原因
|
|
228
|
+
* @param {object} opts {isRetryable, maxRetries, delayMs, sleep, shouldAbort}
|
|
229
|
+
* (sleep/shouldAbort 供调用方接入分片中断)
|
|
230
|
+
* @returns {Promise<{ok: boolean, reason?: string, attempts: number, aborted?: boolean}>}
|
|
231
|
+
*/
|
|
232
|
+
export async function retryRestoreOnNavFailure(attemptFn, firstFailure, opts = {}) {
|
|
233
|
+
const isRetryable = opts.isRetryable;
|
|
234
|
+
const maxRetries = opts.maxRetries ?? RESTORE_RETRY_MAX_RETRIES;
|
|
235
|
+
const delayMs = opts.delayMs ?? RESTORE_RETRY_DELAY_MS;
|
|
236
|
+
const sleep = opts.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
237
|
+
const shouldAbort = opts.shouldAbort || (() => false);
|
|
238
|
+
let reason = firstFailure;
|
|
239
|
+
let attempts = 1; // 首次尝试已发生
|
|
240
|
+
while (typeof isRetryable === "function" && attempts <= maxRetries && isRetryable(reason)) {
|
|
241
|
+
// 每轮重试前回调(可选):attempts=当前已失败次数,供调用方打「第N次失败→间隔后重试」日志
|
|
242
|
+
if (typeof opts.onRetry === "function") opts.onRetry(attempts, delayMs, reason);
|
|
243
|
+
await sleep(delayMs);
|
|
244
|
+
if (shouldAbort()) return { ok: false, reason, attempts, aborted: true };
|
|
245
|
+
attempts++;
|
|
246
|
+
const r = await Promise.resolve()
|
|
247
|
+
.then(attemptFn)
|
|
248
|
+
.catch((e) => ({ ok: false, reason: e?.message || String(e) }));
|
|
249
|
+
if (r.ok) return { ...r, attempts };
|
|
250
|
+
reason = r.reason;
|
|
251
|
+
}
|
|
252
|
+
return { ok: false, reason, attempts };
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// ── 候选注入失败原号重试(纯函数/纯常量,2026-11-05 f11 48h 实测驱动)──
|
|
256
|
+
// 背景:注入验证失败 82.8% 为浏览器级崩溃(走既有崩溃熔断),~17% 为瞬态环境失败
|
|
257
|
+
// (page.goto 超时 / AbortError / net::ERR_* / Execution context destroyed / 无账号数据),
|
|
258
|
+
// 0 条会话过期。现状对非崩溃失败一律立即弃号换下一个候选,网络劣化窗口会连续烧掉
|
|
259
|
+
// 5 个候选且全死于同一环境。改造:瞬态失败同一候选原号重试(共 3 次尝试,间隔 15s),
|
|
260
|
+
// 重试期间租约保持在手;确定性号死(会话过期/身份不符)维持立即弃号;
|
|
261
|
+
// 浏览器级崩溃维持熔断路径(决策上崩溃优先,见 decideCandidateInjectAction)。
|
|
262
|
+
// 本组函数保持零依赖(崩溃签名判定由调用方注入 isBrowserClosedError,勿拖 playwright 链)。
|
|
263
|
+
|
|
264
|
+
/** 候选注入瞬态失败原号重试:追加次数(共 3 次尝试,全败才弃号换下一个候选) */
|
|
265
|
+
export const CANDIDATE_INJECT_MAX_RETRIES = 2;
|
|
266
|
+
/** 候选注入重试间隔 */
|
|
267
|
+
export const CANDIDATE_INJECT_DELAY_MS = 15 * 1000;
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* 确定性号死:会话过期 / 身份不符(injectPoolAccount 的账号级失败文案)。
|
|
271
|
+
* 重试无意义(号本身死了,换环境也救不活),立即弃号换下一个候选。
|
|
272
|
+
*/
|
|
273
|
+
export function isDeterministicAccountFailure(reason) {
|
|
274
|
+
if (!reason) return false;
|
|
275
|
+
const msg = String(reason);
|
|
276
|
+
return msg.includes("会话过期") || msg.includes("身份不符");
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* 候选注入失败是否可原号重试(瞬态/环境失败白名单,f11 实测清单):
|
|
281
|
+
* - 可重试:超时 / net::ERR_*(导航层) / AbortError / Execution context destroyed /
|
|
282
|
+
* 无账号数据 / 导航失败前缀
|
|
283
|
+
* - 不可重试:确定性号死(会话过期/身份不符);浏览器级崩溃签名(Page/Target crashed 等
|
|
284
|
+
* 由调用方决策先行分流走熔断,此处对不含 net::ERR_CONNECTION 前缀的纯崩溃签名兜底排除);
|
|
285
|
+
* 未知失败(既非号死也非已知瞬态签名)→ 不重试,维持现状立即弃号(保守兜底)
|
|
286
|
+
*/
|
|
287
|
+
export function isRetryableInjectFailure(reason) {
|
|
288
|
+
if (!reason) return false;
|
|
289
|
+
const msg = String(reason);
|
|
290
|
+
if (isDeterministicAccountFailure(msg)) return false;
|
|
291
|
+
if (msg.includes("net::ERR_")) return true;
|
|
292
|
+
if (/timed?\s*out|timeout/i.test(msg)) return true;
|
|
293
|
+
if (/AbortError/i.test(msg)) return true;
|
|
294
|
+
if (/Execution context (was )?destroyed/i.test(msg)) return true;
|
|
295
|
+
if (msg.includes("无账号数据")) return true;
|
|
296
|
+
if (msg.includes("导航失败")) return true;
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* 候选注入失败处置决策(纯函数,候选循环内按返回值三路分流):
|
|
302
|
+
* 返回 "circuit-break" | "retry" | "discard"
|
|
303
|
+
* - circuit-break:浏览器级崩溃(isBrowserClosedError 命中,含 Page/Target crashed、
|
|
304
|
+
* net::ERR_CONNECTION_* 等)→ 既有崩溃熔断路径(中止候选循环,已取候选照常 release)
|
|
305
|
+
* - discard:确定性号死(会话过期/身份不符)或未知失败 → 立即弃号换下一个候选(现状行为)
|
|
306
|
+
* - retry:瞬态/环境失败 → 同一候选原号重试(共 3 次尝试,间隔 15s,租约保持在手)
|
|
307
|
+
* @param {string} reason 注入失败原因
|
|
308
|
+
* @param {(error: Error) => boolean} [isBrowserClosedFn] 崩溃签名判定(客户端传
|
|
309
|
+
* page.js 的 isBrowserClosedError,SSOT 留 page.js;测试可注入假谓词)
|
|
310
|
+
*/
|
|
311
|
+
export function decideCandidateInjectAction(reason, isBrowserClosedFn) {
|
|
312
|
+
const msg = String(reason ?? "");
|
|
313
|
+
if (typeof isBrowserClosedFn === "function" && isBrowserClosedFn(new Error(msg))) {
|
|
314
|
+
return "circuit-break";
|
|
315
|
+
}
|
|
316
|
+
if (isRetryableInjectFailure(msg)) return "retry";
|
|
317
|
+
return "discard";
|
|
318
|
+
}
|