tt-help-cli-ycl 1.4.126 → 1.4.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-daemon-failover.mjs +197 -0
- package/scripts/test-daemon-switch-history.mjs +141 -0
- package/scripts/test-wmi-free-upgrade.mjs +98 -0
- package/scripts/verify-kill-edge.mjs +108 -0
- package/scripts/verify-stop-command.mjs +159 -0
- package/src/clash/daemon.js +151 -25
- package/src/cli/refresh-mini.js +40 -13
- package/src/watchdog/agent.js +34 -1
- package/src/watchdog/process-manager.js +29 -13
- package/src/watchdog/upgrader.js +154 -49
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// verify-stop-command.mjs — 211 上验证修复后的"看门狗关闭"完整时序
|
|
2
|
+
// 模拟:看门狗托管业务命令 refresh-mini --port 19223,业务启动 Edge CDP 实例。
|
|
3
|
+
// 修复后 stopCommand 顺序:先 killEdgeProcesses(端口) 杀浏览器 → 再 taskkill 业务进程树。
|
|
4
|
+
// 验证:Edge 进程树全灭(端口释放)+ 业务进程被杀。
|
|
5
|
+
import { execSync, spawn } from "node:child_process";
|
|
6
|
+
|
|
7
|
+
const PKG =
|
|
8
|
+
process.env.APPDATA.replace(/\\/g, "/") +
|
|
9
|
+
"/npm/node_modules/tt-help-cli-ycl";
|
|
10
|
+
// 包内真实函数(当前 1.4.128 的 netstat 实现)
|
|
11
|
+
const { killEdgeProcesses } = await import(
|
|
12
|
+
"file:///" + PKG + "/src/lib/browser/cdp.js"
|
|
13
|
+
);
|
|
14
|
+
|
|
15
|
+
const PORT = 19223;
|
|
16
|
+
const USER_DATA_DIR =
|
|
17
|
+
process.env.TEMP.replace(/\\/g, "/") + "/edge-test-" + PORT;
|
|
18
|
+
|
|
19
|
+
// 修复后的 extractCdpPorts(与本地 process-manager.js 修改一致)
|
|
20
|
+
const BROWSER_CMD_RE =
|
|
21
|
+
/\b(?:explore|auto|attach|scrape|comments|refresh|recheck|precheck|search|videos|videostats|open|tag|login-check)\b/;
|
|
22
|
+
function extractCdpPorts(cmdLine) {
|
|
23
|
+
const baseMatch = cmdLine.match(/--base-port\s+(\d+)/);
|
|
24
|
+
if (baseMatch) {
|
|
25
|
+
const base = parseInt(baseMatch[1], 10);
|
|
26
|
+
const countMatch = cmdLine.match(/--port-count\s+(\d+)/);
|
|
27
|
+
const count = countMatch ? parseInt(countMatch[1], 10) : 1;
|
|
28
|
+
const ports = [];
|
|
29
|
+
for (let i = 0; i < count; i++) ports.push(base + i);
|
|
30
|
+
return ports;
|
|
31
|
+
}
|
|
32
|
+
if (BROWSER_CMD_RE.test(cmdLine)) {
|
|
33
|
+
const m = cmdLine.match(/--port\s+(\d+)/);
|
|
34
|
+
if (m) return [parseInt(m[1], 10)];
|
|
35
|
+
}
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const delay = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
40
|
+
|
|
41
|
+
function listeningPids(port) {
|
|
42
|
+
const out = execSync("netstat -ano", {
|
|
43
|
+
encoding: "utf-8",
|
|
44
|
+
timeout: 5000,
|
|
45
|
+
windowsHide: true,
|
|
46
|
+
});
|
|
47
|
+
const pids = [];
|
|
48
|
+
for (const line of out.split(/\r?\n/)) {
|
|
49
|
+
if (!line.includes("LISTENING") || !line.includes(":" + port + " ")) continue;
|
|
50
|
+
const parts = line.trim().split(/\s+/);
|
|
51
|
+
const pid = parts[parts.length - 1];
|
|
52
|
+
if (/^\d+$/.test(pid) && !pids.includes(pid)) pids.push(pid);
|
|
53
|
+
}
|
|
54
|
+
return pids;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function pidAlive(pid) {
|
|
58
|
+
try {
|
|
59
|
+
const out = execSync(`tasklist /FI "PID eq ${pid}" /NH`, {
|
|
60
|
+
encoding: "utf-8",
|
|
61
|
+
timeout: 5000,
|
|
62
|
+
windowsHide: true,
|
|
63
|
+
});
|
|
64
|
+
// 解析输出文本判断(tasklist 无匹配时可能仍 exit 0,不能依赖退出码)
|
|
65
|
+
return new RegExp("\\b" + pid + "\\b").test(out);
|
|
66
|
+
} catch {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function cleanup() {
|
|
72
|
+
for (const pid of listeningPids(PORT)) {
|
|
73
|
+
try {
|
|
74
|
+
execSync(`taskkill /F /T /PID ${pid}`, {
|
|
75
|
+
stdio: "ignore",
|
|
76
|
+
windowsHide: true,
|
|
77
|
+
});
|
|
78
|
+
} catch {}
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
execSync(`rd /s /q "${USER_DATA_DIR}"`, {
|
|
82
|
+
stdio: "ignore",
|
|
83
|
+
windowsHide: true,
|
|
84
|
+
});
|
|
85
|
+
} catch {}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function main() {
|
|
89
|
+
const CMD_LINE = `tt-help refresh-mini --preset slow --port ${PORT} --rotate-min 0`;
|
|
90
|
+
|
|
91
|
+
// [1] 端口提取(修复后逻辑)
|
|
92
|
+
const ports = extractCdpPorts(CMD_LINE);
|
|
93
|
+
console.log("[1] extractCdpPorts(" + CMD_LINE + ") = " + JSON.stringify(ports));
|
|
94
|
+
if (JSON.stringify(ports) !== JSON.stringify([PORT])) {
|
|
95
|
+
console.log("RESULT: FAIL(端口提取不对)");
|
|
96
|
+
process.exit(1);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// [2] 启动业务进程(node 长驻占位)与 Edge CDP 实例(脱离进程树,模拟 start 启动)
|
|
100
|
+
const biz = spawn(
|
|
101
|
+
process.execPath,
|
|
102
|
+
["-e", "setInterval(()=>{},1000)"],
|
|
103
|
+
{ stdio: ["ignore", "ignore", "ignore"], windowsHide: true },
|
|
104
|
+
);
|
|
105
|
+
console.log("[2] 业务进程 PID = " + biz.pid);
|
|
106
|
+
|
|
107
|
+
execSync(
|
|
108
|
+
`start "" msedge --remote-debugging-port=${PORT} --user-data-dir="${USER_DATA_DIR}" --no-first-run --no-default-browser-check --disable-background-mode --disable-component-update --disable-crash-reporter --disable-breakpad --disable-sync --disable-background-networking`,
|
|
109
|
+
{ stdio: "ignore", windowsHide: true },
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
let edgePids = [];
|
|
113
|
+
for (let i = 0; i < 30; i++) {
|
|
114
|
+
await delay(1000);
|
|
115
|
+
edgePids = listeningPids(PORT);
|
|
116
|
+
if (edgePids.length > 0) break;
|
|
117
|
+
}
|
|
118
|
+
console.log("[2] Edge 监听 " + PORT + " PID = " + JSON.stringify(edgePids));
|
|
119
|
+
if (edgePids.length === 0) {
|
|
120
|
+
console.log("RESULT: SETUP_FAIL(Edge 未监听端口)");
|
|
121
|
+
await cleanup();
|
|
122
|
+
try { biz.kill(); } catch {}
|
|
123
|
+
process.exit(2);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// [3] 修复后的 stopCommand 顺序:先杀浏览器,再杀业务进程树
|
|
127
|
+
console.log("[3] 先 killEdgeProcesses(null, " + PORT + ") ...");
|
|
128
|
+
for (const port of ports) {
|
|
129
|
+
await killEdgeProcesses(null, port);
|
|
130
|
+
}
|
|
131
|
+
await delay(2000);
|
|
132
|
+
const edgeAfter = listeningPids(PORT);
|
|
133
|
+
console.log("[3] 杀浏览器后端口监听 = " + JSON.stringify(edgeAfter));
|
|
134
|
+
|
|
135
|
+
console.log("[3] 再 taskkill /F /T /PID " + biz.pid + " ...");
|
|
136
|
+
try {
|
|
137
|
+
execSync(`taskkill /F /T /PID ${biz.pid}`, {
|
|
138
|
+
stdio: "ignore",
|
|
139
|
+
timeout: 5000,
|
|
140
|
+
windowsHide: true,
|
|
141
|
+
});
|
|
142
|
+
} catch (e) {
|
|
143
|
+
console.log("[3] taskkill 业务返回: " + e.message);
|
|
144
|
+
}
|
|
145
|
+
await delay(3000);
|
|
146
|
+
const bizAlive = pidAlive(biz.pid);
|
|
147
|
+
console.log("[3] 业务进程存活: " + bizAlive);
|
|
148
|
+
|
|
149
|
+
const ok = edgeAfter.length === 0 && !bizAlive;
|
|
150
|
+
console.log(ok ? "RESULT: PASS" : "RESULT: FAIL");
|
|
151
|
+
await cleanup();
|
|
152
|
+
process.exit(ok ? 0 : 1);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
main().catch(async (e) => {
|
|
156
|
+
console.error("脚本异常: " + e.message);
|
|
157
|
+
await cleanup();
|
|
158
|
+
process.exit(3);
|
|
159
|
+
});
|
package/src/clash/daemon.js
CHANGED
|
@@ -20,8 +20,10 @@ function fmtCountries(v) {
|
|
|
20
20
|
return list.length ? list.join(",") : "无";
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
// failover
|
|
24
|
-
const FAILOVER_COOLDOWN_MS =
|
|
23
|
+
// failover 检查窗口(默认 10 分钟,与候选集 delay 初测周期对齐;可经 opts.failoverCooldownMs 覆盖)
|
|
24
|
+
const FAILOVER_COOLDOWN_MS = 600_000;
|
|
25
|
+
// 窗口内探针健康时的切换阈值(默认 3:前 2 次仅检查,第 3 次按业务级封禁轮换)
|
|
26
|
+
const FAILOVER_ESCAPE_AFTER = 3;
|
|
25
27
|
|
|
26
28
|
// ══════════════════════════════════════════════
|
|
27
29
|
// ClashDaemon
|
|
@@ -35,6 +37,10 @@ export class ClashDaemon {
|
|
|
35
37
|
opts.failoverCooldownMs > 0
|
|
36
38
|
? opts.failoverCooldownMs
|
|
37
39
|
: FAILOVER_COOLDOWN_MS;
|
|
40
|
+
this.failoverEscapeAfter =
|
|
41
|
+
opts.failoverEscapeAfter > 0
|
|
42
|
+
? opts.failoverEscapeAfter
|
|
43
|
+
: FAILOVER_ESCAPE_AFTER;
|
|
38
44
|
this.dryRun = !!opts.dryRun;
|
|
39
45
|
// 控制端发现参数(CLI 显式值优先,内部再合并 ~/.tt-help.json)
|
|
40
46
|
this._discoverOpts = {
|
|
@@ -84,6 +90,45 @@ export class ClashDaemon {
|
|
|
84
90
|
return run;
|
|
85
91
|
}
|
|
86
92
|
|
|
93
|
+
/** 记录一次成功切换(failover/巡检/手动切换共用),供候选排序使用 */
|
|
94
|
+
_markSwitch(c, nodeName) {
|
|
95
|
+
if (!c || !nodeName) return;
|
|
96
|
+
c._switchHistory = c._switchHistory || new Map();
|
|
97
|
+
c._switchHistory.set(nodeName, Date.now());
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* 壳页判定:TikTok SSR 降级响应(HTTP 200 + 无跳转 + 无 error + 无 title)。
|
|
102
|
+
* 壳页 = 加载页(data-downgrade 骨架屏),链路正常、浏览器会自行渲染,不算节点故障。
|
|
103
|
+
*/
|
|
104
|
+
_isShellPage(h) {
|
|
105
|
+
return (
|
|
106
|
+
!!h && !h.error && h.httpCode === 200 && h.numRedirects === 0 && !h.title
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* 候选排序(切换历史感知,纯时间轮换):
|
|
112
|
+
* 业务级封禁(TikTok 风控)探针无法感知,用"最近切换时间"作为唯一可靠信号,
|
|
113
|
+
* 让 failover/巡检在候选池内公平轮换,避免少数节点间反复横跳。
|
|
114
|
+
* 排序规则:
|
|
115
|
+
* - 从未切过的节点排最前(保持 delay 原序,新入池节点优先尝试)
|
|
116
|
+
* - 切过的节点按切换时间升序沉底(最早切换的先试,最近切换的最后试 = 后进后出)
|
|
117
|
+
* 轮完一圈自然回访最早切过的节点(等于重新评估),无需时间窗口过期。
|
|
118
|
+
*/
|
|
119
|
+
_sortCandidates(nodes, c) {
|
|
120
|
+
const hist = c && c._switchHistory;
|
|
121
|
+
if (!hist || !hist.size || !Array.isArray(nodes)) return nodes;
|
|
122
|
+
return [...nodes].sort((a, b) => {
|
|
123
|
+
const aLast = hist.get(a) || 0;
|
|
124
|
+
const bLast = hist.get(b) || 0;
|
|
125
|
+
if (aLast && bLast) return aLast - bLast; // 都切过:最早切换的先试
|
|
126
|
+
if (aLast) return 1; // 仅 a 切过:沉底
|
|
127
|
+
if (bLast) return -1;
|
|
128
|
+
return 0; // 都未切过:保持 delay 原序(Array.sort 稳定排序)
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
87
132
|
/** 将全局 api/checker 上下文指向指定控制器 */
|
|
88
133
|
_useController(c) {
|
|
89
134
|
api.configure({ port: c.port, secret: c.secret });
|
|
@@ -271,7 +316,7 @@ export class ClashDaemon {
|
|
|
271
316
|
const node = await api
|
|
272
317
|
.getCurrentNode(checker.CONFIG.groupName)
|
|
273
318
|
.catch(() => "?");
|
|
274
|
-
|
|
319
|
+
let h = checker.checkNodeHealth();
|
|
275
320
|
this._writeHealth(c, {
|
|
276
321
|
node,
|
|
277
322
|
healthy: h.healthy,
|
|
@@ -290,6 +335,45 @@ export class ClashDaemon {
|
|
|
290
335
|
return;
|
|
291
336
|
}
|
|
292
337
|
|
|
338
|
+
// 空壳页(TikTok SSR 降级响应):链路可用、浏览器会自行渲染,
|
|
339
|
+
// 切换无收益且会打断业务连接,不触发切换
|
|
340
|
+
if (this._isShellPage(h)) {
|
|
341
|
+
c._apiFailCount = 0;
|
|
342
|
+
this._log(
|
|
343
|
+
`~ [${c.proxyPort}] 空壳页(降级响应,链路正常),不切换 (${(h.sizeDownload / 1024).toFixed(0)}KB ${h.timeTotal.toFixed(1)}s)`,
|
|
344
|
+
);
|
|
345
|
+
return this._writeHealth(c, {
|
|
346
|
+
node,
|
|
347
|
+
healthy: true,
|
|
348
|
+
httpCode: h.httpCode,
|
|
349
|
+
timeTotal: h.timeTotal,
|
|
350
|
+
sizeDownload: h.sizeDownload,
|
|
351
|
+
title: h.title,
|
|
352
|
+
shellPage: true,
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// 真异常先复测一次:过滤瞬时抖动(截断/丢包),连续异常才进入切换
|
|
357
|
+
const firstReason =
|
|
358
|
+
h.error || `HTTP ${h.httpCode} ${(h.sizeDownload / 1024).toFixed(0)}KB`;
|
|
359
|
+
await new Promise((r) => setTimeout(r, 3000));
|
|
360
|
+
const hr = checker.checkNodeHealth();
|
|
361
|
+
if (hr.healthy || this._isShellPage(hr)) {
|
|
362
|
+
this._log(
|
|
363
|
+
`~ [${c.proxyPort}] 复测恢复(首次异常: ${firstReason}),不切换`,
|
|
364
|
+
);
|
|
365
|
+
return this._writeHealth(c, {
|
|
366
|
+
node,
|
|
367
|
+
healthy: true,
|
|
368
|
+
httpCode: hr.httpCode,
|
|
369
|
+
timeTotal: hr.timeTotal,
|
|
370
|
+
sizeDownload: hr.sizeDownload,
|
|
371
|
+
title: hr.title,
|
|
372
|
+
shellPage: this._isShellPage(hr) ? true : undefined,
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
h = hr; // 复测仍异常,以最新数据为准
|
|
376
|
+
|
|
293
377
|
const reason =
|
|
294
378
|
h.error || `HTTP ${h.httpCode} ${(h.sizeDownload / 1024).toFixed(0)}KB`;
|
|
295
379
|
const { nodes } = checker.loadHealthyList();
|
|
@@ -320,17 +404,20 @@ export class ClashDaemon {
|
|
|
320
404
|
`✗ [${c.proxyPort}] 巡检异常 [${reason}],开始从候选集切换 (当前 ${node})`,
|
|
321
405
|
);
|
|
322
406
|
// 逐个切换候选 + 内容验证,第一个满足即采用
|
|
407
|
+
// 排序:切过的节点按切换时间沉底(后进后出),优先尝试"最久没被切过"的节点
|
|
408
|
+
const ordered = this._sortCandidates(nodes, c);
|
|
323
409
|
let target = null;
|
|
324
410
|
let h2 = null;
|
|
325
|
-
for (const cand of
|
|
411
|
+
for (const cand of ordered) {
|
|
326
412
|
if (cand === node) continue;
|
|
327
413
|
this._log(`↺ [${c.proxyPort}] 尝试候选: ${cand} [原因: ${reason}]`);
|
|
328
414
|
if (!(await checker.switchToNode(cand))) {
|
|
329
415
|
this._log(` ↺ ${cand} 切换失败,下一个`);
|
|
330
416
|
continue;
|
|
331
417
|
}
|
|
418
|
+
this._markSwitch(c, cand);
|
|
332
419
|
h2 = checker.checkNodeHealth();
|
|
333
|
-
if (h2.healthy) {
|
|
420
|
+
if (h2.healthy || this._isShellPage(h2)) {
|
|
334
421
|
target = cand;
|
|
335
422
|
break;
|
|
336
423
|
}
|
|
@@ -340,6 +427,7 @@ export class ClashDaemon {
|
|
|
340
427
|
if (!target) {
|
|
341
428
|
// 全部候选不可用,恢复原节点
|
|
342
429
|
await checker.switchToNode(node).catch(() => {});
|
|
430
|
+
this._markSwitch(c, node);
|
|
343
431
|
await api.closeAllConnections().catch(() => {});
|
|
344
432
|
this._log(
|
|
345
433
|
`✗ [${c.proxyPort}] 恢复原节点 ${node}(原因: ${reason} — 候选全部不可用)`,
|
|
@@ -428,6 +516,13 @@ export class ClashDaemon {
|
|
|
428
516
|
});
|
|
429
517
|
const elapsed = ((Date.now() - start) / 1000).toFixed(0);
|
|
430
518
|
checker.saveHealthyList(r.nodes);
|
|
519
|
+
// 清理切换历史中已不在候选集的节点,防止 Map 无限增长
|
|
520
|
+
if (c._switchHistory && c._switchHistory.size) {
|
|
521
|
+
const keep = new Set(r.nodes);
|
|
522
|
+
for (const k of [...c._switchHistory.keys()]) {
|
|
523
|
+
if (!keep.has(k)) c._switchHistory.delete(k);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
431
526
|
c._meta.lastBenchmark = new Date().toISOString();
|
|
432
527
|
c._meta.healthyNodesTotal = r.total;
|
|
433
528
|
c._meta.healthyNodesCount = r.nodes.length;
|
|
@@ -489,6 +584,7 @@ export class ClashDaemon {
|
|
|
489
584
|
lastSwitch: `manual → ${cmd.node} (失败)`,
|
|
490
585
|
});
|
|
491
586
|
}
|
|
587
|
+
this._markSwitch(c, cmd.node);
|
|
492
588
|
const h = checker.checkNodeHealth();
|
|
493
589
|
// 切换后断开所有旧连接,浏览器立即重建到新节点(同巡检切换)
|
|
494
590
|
await api.closeAllConnections().catch(() => {});
|
|
@@ -512,32 +608,48 @@ export class ClashDaemon {
|
|
|
512
608
|
/**
|
|
513
609
|
* 看门狗指令: 主动故障切换(不依赖巡检发现,强制从候选集切下一个可用节点)
|
|
514
610
|
* 用于业务进程连续报错(CDN限流/网络异常/封 IP 等)时换出口节点
|
|
611
|
+
*
|
|
612
|
+
* failover 三级响应(2026-08-22,替代原 45s 冷却):
|
|
613
|
+
* 1. 每次 failover 先检查当前节点(消除冷却期外盲切);
|
|
614
|
+
* 2. 探针不健康 → 立即切换(有依据才切,无视窗口);
|
|
615
|
+
* 3. 探针健康 → 窗口内默认只检查不切换;同一窗口内 failover 到达
|
|
616
|
+
* failoverEscapeAfter 次(默认 3)→ 视为业务级封禁(curl 探针感知不到
|
|
617
|
+
* TikTok 业务风控)→ 按轮换逻辑切一次。
|
|
618
|
+
* 实际切换(成功或恢复原节点)或触发候选初测后重置窗口与计数,给新节点干净
|
|
619
|
+
* 观察窗口;窗口 = failoverCooldownMs(默认 10 分钟,与候选集初测周期对齐)。
|
|
620
|
+
* 按控制端独立记录(多控制端互不影响);对外接口(命令动作/IPC/health)不变。
|
|
515
621
|
*/
|
|
516
622
|
async _handleFailover(cmd, c) {
|
|
517
623
|
try {
|
|
518
|
-
|
|
519
|
-
// 冷却期内再次收到 failover:先验证当前节点健康,健康则跳过并续冷却
|
|
520
|
-
// (持续健康 → 持续不切,避免冷却期结束后又白切健康节点;同时过滤滞后/重连误报),
|
|
521
|
-
// 不健康才继续切换。按控制端独立记录冷却(多控制端互不影响);
|
|
522
|
-
// 对外接口(命令动作 / IPC / health 格式)保持不变。
|
|
624
|
+
const reason = (cmd && cmd.reason) || "看门狗主动故障切换";
|
|
523
625
|
const now = Date.now();
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
626
|
+
const w =
|
|
627
|
+
c._failoverWindow || (c._failoverWindow = { startAt: 0, count: 0 });
|
|
628
|
+
if (now - w.startAt >= this.failoverCooldownMs) {
|
|
629
|
+
w.startAt = now;
|
|
630
|
+
w.count = 0;
|
|
631
|
+
}
|
|
632
|
+
w.count += 1;
|
|
633
|
+
|
|
634
|
+
// 第 1 级:先检查当前节点(壳页 = 链路正常,按健康对待,与巡检口径一致)
|
|
635
|
+
const h = checker.checkNodeHealth();
|
|
636
|
+
const probeOk = !!h && (h.healthy || this._isShellPage(h));
|
|
637
|
+
if (probeOk && w.count < this.failoverEscapeAfter) {
|
|
534
638
|
this._log(
|
|
535
|
-
`🔄
|
|
639
|
+
`🔄 [${c.proxyPort}] 探针健康,窗口内第 ${w.count}/${this.failoverEscapeAfter} 次 failover 仅检查不切换 [${reason}]`,
|
|
640
|
+
);
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
if (probeOk) {
|
|
644
|
+
this._log(
|
|
645
|
+
`🔄 [${c.proxyPort}] 窗口内第 ${w.count} 次 failover:探针健康但业务持续报错,按业务级封禁轮换 [${reason}]`,
|
|
646
|
+
);
|
|
647
|
+
} else {
|
|
648
|
+
this._log(
|
|
649
|
+
`🔄 [${c.proxyPort}] 探针不健康(${(h && (h.error || `HTTP ${h.httpCode}`)) || "无响应"}),立即故障切换 [${reason}]`,
|
|
536
650
|
);
|
|
537
651
|
}
|
|
538
|
-
c._lastFailoverAt = now;
|
|
539
652
|
|
|
540
|
-
const reason = (cmd && cmd.reason) || "看门狗主动故障切换";
|
|
541
653
|
const node = await api
|
|
542
654
|
.getCurrentNode(checker.CONFIG.groupName)
|
|
543
655
|
.catch(() => "?");
|
|
@@ -547,28 +659,39 @@ export class ClashDaemon {
|
|
|
547
659
|
const { nodes } = checker.loadHealthyList();
|
|
548
660
|
if (!nodes.length) {
|
|
549
661
|
this._log("🔄 候选集为空,触发 delay 初测");
|
|
662
|
+
// 初测即处置:重置窗口,避免健康探测每轮重复触发初测
|
|
663
|
+
w.startAt = now;
|
|
664
|
+
w.count = 0;
|
|
550
665
|
await this._runDelayProbeController(c);
|
|
551
666
|
return;
|
|
552
667
|
}
|
|
553
668
|
// 逐个切换候选 + 内容验证,第一个满足即采用(跳过当前节点)
|
|
669
|
+
// 排序:切过的节点按切换时间沉底(后进后出),优先尝试"最久没被切过"的节点
|
|
670
|
+
const ordered = this._sortCandidates(nodes, c);
|
|
554
671
|
let target = null;
|
|
555
672
|
let h2 = null;
|
|
556
|
-
for (const cand of
|
|
673
|
+
for (const cand of ordered) {
|
|
557
674
|
if (cand === node) continue;
|
|
558
675
|
this._log(`↺ 尝试候选: ${cand}`);
|
|
559
676
|
if (!(await checker.switchToNode(cand))) {
|
|
560
677
|
this._log(` ↺ ${cand} 切换失败,下一个`);
|
|
561
678
|
continue;
|
|
562
679
|
}
|
|
680
|
+
this._markSwitch(c, cand);
|
|
563
681
|
h2 = checker.checkNodeHealth();
|
|
564
|
-
if (h2.healthy) {
|
|
682
|
+
if (h2.healthy || this._isShellPage(h2)) {
|
|
565
683
|
target = cand;
|
|
566
684
|
break;
|
|
567
685
|
}
|
|
568
686
|
this._log(` ↺ ${cand} 内容验证未通过,下一个`);
|
|
569
687
|
}
|
|
688
|
+
// 本次 failover 已尽力处置(成功切换或恢复原节点):重置窗口,给新节点干净观察窗口
|
|
689
|
+
w.startAt = now;
|
|
690
|
+
w.count = 0;
|
|
691
|
+
|
|
570
692
|
if (!target) {
|
|
571
693
|
await checker.switchToNode(node).catch(() => {});
|
|
694
|
+
this._markSwitch(c, node);
|
|
572
695
|
await api.closeAllConnections().catch(() => {});
|
|
573
696
|
this._log(
|
|
574
697
|
`✗ failover [${reason}] — 候选全部不可用,恢复原节点 ${node}`,
|
|
@@ -707,5 +830,8 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
|
|
707
830
|
secret: getArg("--secret") || undefined,
|
|
708
831
|
selection: getArg("--selection") || undefined,
|
|
709
832
|
configPaths: configPaths.length ? configPaths : undefined,
|
|
833
|
+
failoverCooldownMs: parseInt(getArg("--failover-cooldown-ms")) || undefined,
|
|
834
|
+
failoverEscapeAfter:
|
|
835
|
+
parseInt(getArg("--failover-escape-after")) || undefined,
|
|
710
836
|
}).start();
|
|
711
837
|
}
|
package/src/cli/refresh-mini.js
CHANGED
|
@@ -576,22 +576,49 @@ export async function handleRefreshMini(options) {
|
|
|
576
576
|
|
|
577
577
|
if (result.error) {
|
|
578
578
|
const isBlocked = result.error.startsWith("被封:");
|
|
579
|
-
|
|
579
|
+
// 导航打断(多为切换节点后的网络重建期)不计入连续任务错误,
|
|
580
|
+
// 避免"切换 → 掐断连接 → 导航打断 → 计错 → 再切换"的循环
|
|
581
|
+
const isNavBreak = result.error.includes(
|
|
582
|
+
"Execution context was destroyed",
|
|
583
|
+
);
|
|
584
|
+
if (!isNavBreak) consecutiveTaskErrors++;
|
|
580
585
|
errorCount++;
|
|
581
586
|
|
|
582
|
-
//
|
|
587
|
+
// 临时性错误:自动重试
|
|
583
588
|
if (result.retryable) {
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
589
|
+
if (isNavBreak) {
|
|
590
|
+
// 与被封同款渐退等待:30/40/50s,给切换后的网络重建时间
|
|
591
|
+
const navWaits = [30_000, 40_000, 50_000];
|
|
592
|
+
for (let retry = 0; retry < navWaits.length; retry++) {
|
|
593
|
+
console.error(
|
|
594
|
+
` [网络重建] 导航打断(切换节点后),等待 ${navWaits[retry] / 1000}s 稳定后第 ${retry + 1}/${navWaits.length} 次重试 @${username}...`,
|
|
595
|
+
);
|
|
596
|
+
await sleep(navWaits[retry]);
|
|
597
|
+
result = await processExplore(
|
|
598
|
+
page,
|
|
599
|
+
username,
|
|
600
|
+
processOptions(),
|
|
601
|
+
console.error,
|
|
602
|
+
);
|
|
603
|
+
if (!result.error) {
|
|
604
|
+
consecutiveTaskErrors = 0;
|
|
605
|
+
errorCount--;
|
|
606
|
+
break;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
} else {
|
|
610
|
+
console.error(` [临时错误] 等待 5 秒后重试 @${username}...`);
|
|
611
|
+
await sleep(5000);
|
|
612
|
+
result = await processExplore(
|
|
613
|
+
page,
|
|
614
|
+
username,
|
|
615
|
+
processOptions(),
|
|
616
|
+
console.error,
|
|
617
|
+
);
|
|
618
|
+
if (!result.error) {
|
|
619
|
+
consecutiveTaskErrors = 0;
|
|
620
|
+
errorCount--;
|
|
621
|
+
}
|
|
595
622
|
}
|
|
596
623
|
}
|
|
597
624
|
|
package/src/watchdog/agent.js
CHANGED
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
import { writeFileSync, existsSync, readFileSync } from "fs";
|
|
25
25
|
import { spawn } from "child_process";
|
|
26
26
|
import { join, dirname } from "path";
|
|
27
|
-
import { homedir } from "os";
|
|
27
|
+
import { homedir, tmpdir } from "os";
|
|
28
28
|
import { fileURLToPath } from "url";
|
|
29
29
|
|
|
30
30
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -32,6 +32,36 @@ const homeDir = homedir() || process.env.HOME || process.env.USERPROFILE || "";
|
|
|
32
32
|
const TT_HELP_CONFIG = join(homeDir, ".tt-help.json");
|
|
33
33
|
const IS_WIN = process.platform === "win32";
|
|
34
34
|
|
|
35
|
+
// ── 升级配套(绕开 WMI)──
|
|
36
|
+
// 部分 Windows 机器 WMI(WinMgmt)损坏,升级器用 Get-CimInstance 查进程表会失效,
|
|
37
|
+
// 导致旧 cmd 窗口不关、新版存活误判。故升级链改为:
|
|
38
|
+
// 1. 本进程启动时设置控制台窗口标题 → 升级器按标题匹配(tasklist /V)关旧窗
|
|
39
|
+
// 2. 本进程启动时写存活标记文件 → 升级器读文件确认新版存活(不再枚举进程表)
|
|
40
|
+
const STARTED_FILE = join(tmpdir(), "tt-help-watchdog-started.json");
|
|
41
|
+
const WINDOW_TITLE_TAG = "tt-help-watchdog";
|
|
42
|
+
|
|
43
|
+
/** 设置控制台窗口标题(仅 Windows 交互窗口生效,ESC ]0 转义;失败静默) */
|
|
44
|
+
function setConsoleTitle(clientId) {
|
|
45
|
+
if (process.platform !== "win32") return;
|
|
46
|
+
try {
|
|
47
|
+
process.stdout.write(`\x1b]0;${WINDOW_TITLE_TAG} (${clientId})\x07`);
|
|
48
|
+
} catch {}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** 写存活标记文件(升级器据此确认新版 watchdog 已启动,绕开 WMI) */
|
|
52
|
+
function writeStartedFile() {
|
|
53
|
+
try {
|
|
54
|
+
writeFileSync(
|
|
55
|
+
STARTED_FILE,
|
|
56
|
+
JSON.stringify({
|
|
57
|
+
pid: process.pid,
|
|
58
|
+
version: getVersion(),
|
|
59
|
+
ts: Date.now(),
|
|
60
|
+
}),
|
|
61
|
+
);
|
|
62
|
+
} catch {}
|
|
63
|
+
}
|
|
64
|
+
|
|
35
65
|
// ── 业务日志错误统计 → 触发节点切换 ──
|
|
36
66
|
// 识别采集程序日志里已有的错误分类标签(explore 等已打标),连续错误达标则切换出口节点
|
|
37
67
|
const NETWORK_ERROR_PATTERNS = [
|
|
@@ -173,6 +203,9 @@ export class WatchdogAgent {
|
|
|
173
203
|
}) {
|
|
174
204
|
this.serverUrl = serverUrl.replace(/\/+$/, "");
|
|
175
205
|
this.clientId = clientId;
|
|
206
|
+
// 升级配套:设窗口标题(升级器按标题关旧窗)+ 写存活文件(升级器做存活校验)
|
|
207
|
+
setConsoleTitle(clientId);
|
|
208
|
+
writeStartedFile();
|
|
176
209
|
this.intervalMs = intervalMs;
|
|
177
210
|
this.spawnDaemon = !!spawnDaemon;
|
|
178
211
|
this.loginCheckIntervalMs = Math.max(
|
|
@@ -78,16 +78,29 @@ function printProcessLine(id, msg) {
|
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
//
|
|
81
|
+
// 会启动 CDP 浏览器的采集类命令(用于识别单端口形式 --port 的归属)
|
|
82
|
+
// 排除 webserver/watch 等 --port 是服务端口的命令,避免误杀
|
|
83
|
+
const BROWSER_CMD_RE =
|
|
84
|
+
/\b(?:explore|auto|attach|scrape|comments|refresh|recheck|precheck|search|videos|videostats|open|tag|login-check)\b/;
|
|
85
|
+
|
|
86
|
+
// 从命令行中提取 CDP 端口列表(解析 --base-port/--port-count 和单端口 --port)
|
|
82
87
|
function extractCdpPorts(cmdLine) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
88
|
+
// 多端口形式:--base-port N [--port-count M] → N..N+M-1
|
|
89
|
+
const baseMatch = cmdLine.match(/--base-port\s+(\d+)/);
|
|
90
|
+
if (baseMatch) {
|
|
91
|
+
const base = parseInt(baseMatch[1], 10);
|
|
92
|
+
const countMatch = cmdLine.match(/--port-count\s+(\d+)/);
|
|
93
|
+
const count = countMatch ? parseInt(countMatch[1], 10) : 1;
|
|
94
|
+
const ports = [];
|
|
95
|
+
for (let i = 0; i < count; i++) ports.push(base + i);
|
|
96
|
+
return ports;
|
|
97
|
+
}
|
|
98
|
+
// 单端口形式(如 refresh-mini --port 5222):仅浏览器类命令提取
|
|
99
|
+
if (BROWSER_CMD_RE.test(cmdLine)) {
|
|
100
|
+
const m = cmdLine.match(/--port\s+(\d+)/);
|
|
101
|
+
if (m) return [parseInt(m[1], 10)];
|
|
102
|
+
}
|
|
103
|
+
return [];
|
|
91
104
|
}
|
|
92
105
|
|
|
93
106
|
// 引号感知的命令拆分:"a b" "c d" → ["a b", "c d"]
|
|
@@ -382,6 +395,13 @@ export class ProcessManager extends EventEmitter {
|
|
|
382
395
|
const cmdLine = info.cmdDef?.command || "";
|
|
383
396
|
const ports = extractCdpPorts(cmdLine);
|
|
384
397
|
|
|
398
|
+
// 必须先杀浏览器再杀业务进程树:业务进程被强杀后 Edge 主进程可能随之
|
|
399
|
+
// 退出、CDP 端口释放,此时按端口找监听 PID 会落空,导致 Edge 子进程残留。
|
|
400
|
+
// 先杀时端口必在监听,netstat 找 PID → taskkill /T 可杀干净整个 Edge 进程树。
|
|
401
|
+
for (const port of ports) {
|
|
402
|
+
killEdgeProcesses(null, port);
|
|
403
|
+
}
|
|
404
|
+
|
|
385
405
|
if (IS_WIN) {
|
|
386
406
|
// Windows: shell:true 产生 cmd.exe 中间层,必须杀整个进程树
|
|
387
407
|
execSync(`taskkill /PID ${info.child.pid} /T /F`, {
|
|
@@ -401,10 +421,6 @@ export class ProcessManager extends EventEmitter {
|
|
|
401
421
|
}, 2000);
|
|
402
422
|
}
|
|
403
423
|
|
|
404
|
-
// 全平台:关闭脱离进程树的浏览器(open -a / start msedge 启动的)
|
|
405
|
-
for (const port of ports) {
|
|
406
|
-
killEdgeProcesses(null, port);
|
|
407
|
-
}
|
|
408
424
|
} catch (e) {
|
|
409
425
|
console.error(`[watchdog] 停止命令 ${cmdId} 失败: ${e.message}`);
|
|
410
426
|
}
|