tt-help-cli-ycl 1.4.140 → 1.4.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-client-error-monitor.mjs +176 -0
- package/scripts/test-daemon-failover.mjs +13 -0
- package/scripts/test-proxy-switch-restart.mjs +198 -0
- package/src/clash/daemon.js +26 -10
- package/src/cli/tag.js +26 -17
- package/src/lib/api-client.js +11 -5
- package/src/lib/client-error-report.js +3 -0
- package/src/watch/data-store.js +3 -0
- package/src/watch/public/app.js +182 -8
- package/src/watch/public/index.html +41 -1
- package/src/watch/server.js +1 -0
- package/src/watch/watchdog-server.js +100 -0
- package/src/watchdog/agent.js +157 -1
- package/src/watchdog/client-error-monitor.js +131 -0
- package/src/watchdog/process-manager.js +49 -0
- package/src/webserver/server.mjs +1 -0
package/package.json
CHANGED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 客户端异常监控单测(2026-08-28)
|
|
3
|
+
*
|
|
4
|
+
* 验证:
|
|
5
|
+
* - detectClientError 检测函数:reportCount > 8 且最近一次上报在 2 分钟内
|
|
6
|
+
* - 按 hostname 过滤本机条目;hostname 为空的旧条目不参与关联
|
|
7
|
+
* - 命中 → onAlert 收到本机全部条目(含未命中条目);未命中 → 不回调
|
|
8
|
+
* - removeErrors 逐条 DELETE 并计数
|
|
9
|
+
*
|
|
10
|
+
* 运行: node --test scripts/test-client-error-monitor.mjs
|
|
11
|
+
*/
|
|
12
|
+
import { test, mock } from "node:test";
|
|
13
|
+
import assert from "node:assert/strict";
|
|
14
|
+
import os from "os";
|
|
15
|
+
import {
|
|
16
|
+
detectClientError,
|
|
17
|
+
ClientErrorMonitor,
|
|
18
|
+
} from "../src/watchdog/client-error-monitor.js";
|
|
19
|
+
|
|
20
|
+
const HOST = os.hostname();
|
|
21
|
+
|
|
22
|
+
function makeErr(overrides = {}) {
|
|
23
|
+
return {
|
|
24
|
+
userId: "u1",
|
|
25
|
+
hostname: HOST,
|
|
26
|
+
errorType: "consecutive_blocked",
|
|
27
|
+
reportCount: 10,
|
|
28
|
+
timestamp: Date.now() - 30_000,
|
|
29
|
+
...overrides,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// ── 检测函数 ──
|
|
34
|
+
|
|
35
|
+
test("D1 detectClientError:累计>8 且 2 分钟内 → 命中", () => {
|
|
36
|
+
assert.equal(
|
|
37
|
+
detectClientError(makeErr({ reportCount: 9 }), Date.now()),
|
|
38
|
+
true,
|
|
39
|
+
);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("D2 detectClientError:reportCount=8 不命中(阈值是 >8)", () => {
|
|
43
|
+
assert.equal(
|
|
44
|
+
detectClientError(makeErr({ reportCount: 8 }), Date.now()),
|
|
45
|
+
false,
|
|
46
|
+
);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("D3 detectClientError:最近上报超过 2 分钟 → 不命中", () => {
|
|
50
|
+
assert.equal(
|
|
51
|
+
detectClientError(
|
|
52
|
+
makeErr({ timestamp: Date.now() - 2 * 60_000 - 1 }),
|
|
53
|
+
Date.now(),
|
|
54
|
+
),
|
|
55
|
+
false,
|
|
56
|
+
);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("D4 detectClientError:缺 reportCount → 不命中", () => {
|
|
60
|
+
assert.equal(detectClientError({ timestamp: Date.now() }, Date.now()), false);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
// ── 监控器 ──
|
|
64
|
+
|
|
65
|
+
test("M1 按 hostname 过滤:他机条目不触发本机回调", async () => {
|
|
66
|
+
mock.method(globalThis, "fetch", async () => ({
|
|
67
|
+
ok: true,
|
|
68
|
+
json: async () => ({
|
|
69
|
+
clients: [makeErr({ hostname: "OTHER-MACHINE" })],
|
|
70
|
+
}),
|
|
71
|
+
}));
|
|
72
|
+
let alerted = false;
|
|
73
|
+
const m = new ClientErrorMonitor({
|
|
74
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
75
|
+
hostname: HOST,
|
|
76
|
+
onAlert: async () => {
|
|
77
|
+
alerted = true;
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
const hit = await m._poll();
|
|
81
|
+
assert.equal(alerted, false, "他机异常不应回调");
|
|
82
|
+
assert.equal(hit, null, "无本机条目应返回 null");
|
|
83
|
+
mock.restoreAll();
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("M2 命中:onAlert 收到本机全部条目(含未命中条目)", async () => {
|
|
87
|
+
const mineHit = makeErr({ userId: "u-hit", reportCount: 10 });
|
|
88
|
+
const mineMiss = makeErr({ userId: "u-miss", reportCount: 2 });
|
|
89
|
+
const other = makeErr({ userId: "u-other", hostname: "OTHER-MACHINE" });
|
|
90
|
+
mock.method(globalThis, "fetch", async () => ({
|
|
91
|
+
ok: true,
|
|
92
|
+
json: async () => ({ clients: [mineHit, mineMiss, other] }),
|
|
93
|
+
}));
|
|
94
|
+
let received = null;
|
|
95
|
+
const m = new ClientErrorMonitor({
|
|
96
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
97
|
+
hostname: HOST,
|
|
98
|
+
onAlert: async (mine, hit) => {
|
|
99
|
+
received = { mine, hit };
|
|
100
|
+
},
|
|
101
|
+
});
|
|
102
|
+
const hit = await m._poll();
|
|
103
|
+
assert.equal(hit.length, 1, "命中 1 条");
|
|
104
|
+
assert.equal(hit[0].userId, "u-hit");
|
|
105
|
+
assert.equal(received.mine.length, 2, "回调收到本机全部 2 条");
|
|
106
|
+
assert.deepEqual(
|
|
107
|
+
received.mine.map((e) => e.userId).sort(),
|
|
108
|
+
["u-hit", "u-miss"],
|
|
109
|
+
"回调含命中与未命中的本机条目",
|
|
110
|
+
);
|
|
111
|
+
mock.restoreAll();
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("M3 hostname 为空的旧条目不参与本机关联", async () => {
|
|
115
|
+
mock.method(globalThis, "fetch", async () => ({
|
|
116
|
+
ok: true,
|
|
117
|
+
json: async () => ({ clients: [makeErr({ hostname: "" })] }),
|
|
118
|
+
}));
|
|
119
|
+
let alerted = false;
|
|
120
|
+
const m = new ClientErrorMonitor({
|
|
121
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
122
|
+
hostname: HOST,
|
|
123
|
+
onAlert: async () => {
|
|
124
|
+
alerted = true;
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
const hit = await m._poll();
|
|
128
|
+
assert.equal(alerted, false, "空 hostname 旧条目不应回调");
|
|
129
|
+
assert.equal(hit, null);
|
|
130
|
+
mock.restoreAll();
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("M4 removeErrors:逐条 DELETE 并计数", async () => {
|
|
134
|
+
const deleted = [];
|
|
135
|
+
mock.method(globalThis, "fetch", async (url, opts) => {
|
|
136
|
+
if (opts && opts.method === "DELETE") {
|
|
137
|
+
deleted.push(String(url));
|
|
138
|
+
return { ok: true };
|
|
139
|
+
}
|
|
140
|
+
return { ok: true, json: async () => ({ clients: [] }) };
|
|
141
|
+
});
|
|
142
|
+
const m = new ClientErrorMonitor({ serverUrl: "http://127.0.0.1:3000" });
|
|
143
|
+
const removed = await m.removeErrors([
|
|
144
|
+
{ userId: "a" },
|
|
145
|
+
{ userId: "b" },
|
|
146
|
+
{ userId: "" },
|
|
147
|
+
null,
|
|
148
|
+
]);
|
|
149
|
+
assert.equal(removed, 2, "有效 userId 各删一条");
|
|
150
|
+
assert.equal(deleted.length, 2, "共 2 次 DELETE");
|
|
151
|
+
assert.ok(
|
|
152
|
+
deleted.every((u) =>
|
|
153
|
+
["a", "b"].some((id) => u.endsWith(`/api/client-error/${id}`)),
|
|
154
|
+
),
|
|
155
|
+
"DELETE 路径正确",
|
|
156
|
+
);
|
|
157
|
+
mock.restoreAll();
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("M5 拉取失败静默:fetch 抛错不回调不崩溃", async () => {
|
|
161
|
+
mock.method(globalThis, "fetch", async () => {
|
|
162
|
+
throw new Error("ECONNREFUSED");
|
|
163
|
+
});
|
|
164
|
+
let alerted = false;
|
|
165
|
+
const m = new ClientErrorMonitor({
|
|
166
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
167
|
+
hostname: HOST,
|
|
168
|
+
onAlert: async () => {
|
|
169
|
+
alerted = true;
|
|
170
|
+
},
|
|
171
|
+
});
|
|
172
|
+
const hit = await m._poll();
|
|
173
|
+
assert.equal(alerted, false);
|
|
174
|
+
assert.equal(hit, null);
|
|
175
|
+
mock.restoreAll();
|
|
176
|
+
});
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* - 探针不健康 → 立即切换(无视冷却)
|
|
6
6
|
* - 探针健康(含壳页)→ 距上次切换 < 冷却期仅检查不切换
|
|
7
7
|
* - 探针健康 → 距上次切换 >= 冷却期 → 按业务级封禁轮换一次
|
|
8
|
+
* - 强切(force=true)→ 探针健康且冷却期内仍强制轮换
|
|
8
9
|
* - 切换后(含手动/巡检/failover)更新 _lastSwitchAt 锚点,冷却期内不再切
|
|
9
10
|
* - 候选集为空 → 触发 delay 初测且不切换
|
|
10
11
|
*
|
|
@@ -229,3 +230,15 @@ test("F9 disconnectMode=full:直接全断", async () => {
|
|
|
229
230
|
assert.equal(state.idleCloses, 0, "full 模式不走优雅断连");
|
|
230
231
|
assert.equal(state.closes, 1, "full 模式直接全断");
|
|
231
232
|
});
|
|
233
|
+
|
|
234
|
+
test("F10 强切(force):冷却期内探针健康仍强制轮换", async () => {
|
|
235
|
+
resetState();
|
|
236
|
+
const { d, c } = makeCtx();
|
|
237
|
+
c._lastSwitchAt = Date.now(); // 刚切过,处于 10min 冷却期内
|
|
238
|
+
await d._handleFailover({ reason: "F10", force: true }, c);
|
|
239
|
+
assert.deepEqual(state.switches, ["🇲🇾 马来02"], "force 应无视冷却直接轮换");
|
|
240
|
+
assert.ok(
|
|
241
|
+
c._lastSwitchAt && c._lastSwitchAt > Date.now() - 5000,
|
|
242
|
+
"强切后应更新锚点(给新节点观察期)",
|
|
243
|
+
);
|
|
244
|
+
});
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 代理切换 → watchdog 强制重启命令 链路测试(2026-08-28,先打通路版本)
|
|
3
|
+
*
|
|
4
|
+
* 验证:
|
|
5
|
+
* T1 daemon _markSwitchTime 递增 restartSeq(首次为 1,连续递增)
|
|
6
|
+
* T2 daemon _writeHealth 把 restartSeq 带进 health 文件
|
|
7
|
+
* T3 daemon 未切换时 restartSeq 保持 0
|
|
8
|
+
* T4 ProcessManager.restartAll 先停后起(真实子进程,pid 变化、数量恢复)
|
|
9
|
+
* T5 restartAll 跳过 stopped / 无 child 的命令
|
|
10
|
+
* T6 无运行命令时 restartAll 返回 {restarted:0}
|
|
11
|
+
* T7 Agent._checkProxySwitch 首次读取仅建基线不触发
|
|
12
|
+
* T8 health.restartSeq 变大 → 触发强制重启(真实子进程 pid 变化)
|
|
13
|
+
* T9 health.restartSeq 变小(daemon 重启归零)→ 不触发
|
|
14
|
+
* T10 通道重启:主文件 seq 不变时不触发
|
|
15
|
+
*
|
|
16
|
+
* 运行: node --test scripts/test-proxy-switch-restart.mjs
|
|
17
|
+
*/
|
|
18
|
+
import { test } from "node:test";
|
|
19
|
+
import assert from "node:assert/strict";
|
|
20
|
+
import { writeFileSync, rmSync, existsSync, readFileSync } from "fs";
|
|
21
|
+
import { join } from "path";
|
|
22
|
+
import { tmpdir } from "os";
|
|
23
|
+
|
|
24
|
+
const HEALTH_FILE = join(tmpdir(), "clash-health.json");
|
|
25
|
+
|
|
26
|
+
/** 清理并写 health 文件(模拟 daemon 写入) */
|
|
27
|
+
function writeHealthFile(extra) {
|
|
28
|
+
writeFileSync(
|
|
29
|
+
HEALTH_FILE,
|
|
30
|
+
JSON.stringify(
|
|
31
|
+
{
|
|
32
|
+
node: "🇺🇸 美国01",
|
|
33
|
+
healthy: true,
|
|
34
|
+
lastCheck: new Date().toISOString(),
|
|
35
|
+
...extra,
|
|
36
|
+
},
|
|
37
|
+
null,
|
|
38
|
+
2,
|
|
39
|
+
),
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** 生成一个常驻假命令(测试业务进程) */
|
|
44
|
+
function fakeCmd(id) {
|
|
45
|
+
return {
|
|
46
|
+
id,
|
|
47
|
+
name: id,
|
|
48
|
+
command: process.execPath,
|
|
49
|
+
args: ["-e", "setInterval(()=>{},1000)"],
|
|
50
|
+
enabled: true,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function cleanup() {
|
|
55
|
+
try {
|
|
56
|
+
rmSync(HEALTH_FILE, { force: true });
|
|
57
|
+
} catch {}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
test("T1-T3: daemon 切换信号(_markSwitchTime 递增 + _writeHealth 带出)", async () => {
|
|
61
|
+
const { ClashDaemon } = await import("../src/clash/daemon.js");
|
|
62
|
+
const ipc = await import("../src/clash/ipc.js");
|
|
63
|
+
cleanup();
|
|
64
|
+
try {
|
|
65
|
+
const d = new ClashDaemon({ dryRun: true });
|
|
66
|
+
const c = { proxyPort: 0 };
|
|
67
|
+
|
|
68
|
+
// T3: 未切换 → restartSeq = 0
|
|
69
|
+
d._writeHealth(c, { node: "A", healthy: true });
|
|
70
|
+
let h = ipc.readHealth(0);
|
|
71
|
+
assert.equal(h.restartSeq, 0, "未切换时 restartSeq 应为 0");
|
|
72
|
+
|
|
73
|
+
// T1: 首次切换 → 1
|
|
74
|
+
d._markSwitchTime(c);
|
|
75
|
+
assert.equal(c._restartSeq, 1, "首次切换 restartSeq 应为 1");
|
|
76
|
+
d._markSwitchTime(c);
|
|
77
|
+
assert.equal(c._restartSeq, 2, "连续切换 restartSeq 递增");
|
|
78
|
+
|
|
79
|
+
// T2: _writeHealth 带出 restartSeq
|
|
80
|
+
d._writeHealth(c, { node: "B", healthy: true, lastSwitch: "A → B" });
|
|
81
|
+
h = ipc.readHealth(0);
|
|
82
|
+
assert.equal(h.restartSeq, 2, "health 文件应携带 restartSeq=2");
|
|
83
|
+
assert.equal(h.lastSwitch, "A → B");
|
|
84
|
+
} finally {
|
|
85
|
+
cleanup();
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("T4-T6: ProcessManager.restartAll 先停后起", async () => {
|
|
90
|
+
const { ProcessManager } = await import("../src/watchdog/process-manager.js");
|
|
91
|
+
const pm = new ProcessManager();
|
|
92
|
+
try {
|
|
93
|
+
// T6: 空集
|
|
94
|
+
const empty = await pm.restartAll("测试");
|
|
95
|
+
assert.deepEqual(empty, { restarted: 0 }, "无运行命令应返回 0");
|
|
96
|
+
|
|
97
|
+
pm.startCommand(fakeCmd("proc-a"));
|
|
98
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
99
|
+
assert.equal(pm.runningCount, 1);
|
|
100
|
+
const oldPid = pm.processes.get("proc-a").child.pid;
|
|
101
|
+
|
|
102
|
+
// T4: 先停后起,pid 变化、数量恢复
|
|
103
|
+
const r = await pm.restartAll("代理切换(测试)");
|
|
104
|
+
assert.equal(r.restarted, 1, "应重启 1 个命令");
|
|
105
|
+
assert.equal(pm.runningCount, 1, "重启后数量恢复");
|
|
106
|
+
const newPid = pm.processes.get("proc-a").child.pid;
|
|
107
|
+
assert.notEqual(newPid, oldPid, "重启后应是新进程(pid 不同)");
|
|
108
|
+
|
|
109
|
+
// T5: stopped 的命令不参与重启
|
|
110
|
+
pm.startCommand(fakeCmd("proc-b"));
|
|
111
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
112
|
+
pm.stopCommand("proc-b");
|
|
113
|
+
const r2 = await pm.restartAll("二次重启");
|
|
114
|
+
assert.equal(r2.restarted, 1, "只重启 running 的 1 个,stopped 的不算");
|
|
115
|
+
assert.equal(pm.runningCount, 1);
|
|
116
|
+
assert.ok(!pm.processes.has("proc-b"), "proc-b 保持停止状态");
|
|
117
|
+
} finally {
|
|
118
|
+
pm.stopAll();
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("T7-T13: Agent._checkProxySwitch 感知切换(默认禁用,开启后才重启)", async () => {
|
|
123
|
+
const { WatchdogAgent } = await import("../src/watchdog/agent.js");
|
|
124
|
+
cleanup();
|
|
125
|
+
const agent = new WatchdogAgent({
|
|
126
|
+
serverUrl: "http://127.0.0.1:9",
|
|
127
|
+
clientId: "test-switch",
|
|
128
|
+
intervalMs: 60000,
|
|
129
|
+
spawnDaemon: false,
|
|
130
|
+
});
|
|
131
|
+
try {
|
|
132
|
+
// 准备一个运行中的假业务命令
|
|
133
|
+
agent.processManager.startCommand(fakeCmd("proc-c"));
|
|
134
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
135
|
+
const pid0 = agent.processManager.processes.get("proc-c").child.pid;
|
|
136
|
+
|
|
137
|
+
// T7: 首次读取建基线,不触发
|
|
138
|
+
writeHealthFile({ restartSeq: 0, lastSwitch: "初始化" });
|
|
139
|
+
await agent._checkProxySwitch();
|
|
140
|
+
assert.equal(
|
|
141
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
142
|
+
pid0,
|
|
143
|
+
"首次读取仅建基线,不应重启",
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
// T10: seq 不变不触发
|
|
147
|
+
writeHealthFile({ restartSeq: 0, lastSwitch: "初始化" });
|
|
148
|
+
await agent._checkProxySwitch();
|
|
149
|
+
assert.equal(
|
|
150
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
151
|
+
pid0,
|
|
152
|
+
"seq 不变不应重启",
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
// T11: 默认禁用时 seq 变大 → 不触发(但基线已跟随更新)
|
|
156
|
+
writeHealthFile({ restartSeq: 1, lastSwitch: "禁用期间切换: A → B" });
|
|
157
|
+
await agent._checkProxySwitch();
|
|
158
|
+
assert.equal(
|
|
159
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
160
|
+
pid0,
|
|
161
|
+
"默认禁用时检测到切换也不应重启",
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
// T12: 开启开关后,seq 仍为 1(无新切换)→ 不补触发历史切换
|
|
165
|
+
agent._applySwitchRestartConfig({ enabled: true });
|
|
166
|
+
await agent._checkProxySwitch();
|
|
167
|
+
assert.equal(
|
|
168
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
169
|
+
pid0,
|
|
170
|
+
"开启开关后不应补触发开启前的历史切换",
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
// T8: 开启后 seq 1→2 → 触发强制重启
|
|
174
|
+
writeHealthFile({ restartSeq: 2, lastSwitch: "开启后切换: B → C" });
|
|
175
|
+
await agent._checkProxySwitch();
|
|
176
|
+
const pid1 = agent.processManager.processes.get("proc-c").child.pid;
|
|
177
|
+
assert.notEqual(pid1, pid0, "开启后检测到切换应强制重启(pid 变化)");
|
|
178
|
+
assert.equal(agent.processManager.runningCount, 1, "重启后数量恢复");
|
|
179
|
+
|
|
180
|
+
// T9: seq 变小(daemon 重启归零)→ 不触发
|
|
181
|
+
writeHealthFile({ restartSeq: 0, lastSwitch: "daemon 重启" });
|
|
182
|
+
await agent._checkProxySwitch();
|
|
183
|
+
assert.equal(
|
|
184
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
185
|
+
pid1,
|
|
186
|
+
"seq 变小(daemon 重启归零)不应触发",
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
// T13: 归零后再次真实切换(0 → 1)应重新触发
|
|
190
|
+
writeHealthFile({ restartSeq: 1, lastSwitch: "再次切换: C → D" });
|
|
191
|
+
await agent._checkProxySwitch();
|
|
192
|
+
const pid2 = agent.processManager.processes.get("proc-c").child.pid;
|
|
193
|
+
assert.notEqual(pid2, pid1, "归零后再次切换应重新触发");
|
|
194
|
+
} finally {
|
|
195
|
+
await agent.stop();
|
|
196
|
+
cleanup();
|
|
197
|
+
}
|
|
198
|
+
});
|
package/src/clash/daemon.js
CHANGED
|
@@ -104,7 +104,11 @@ export class ClashDaemon {
|
|
|
104
104
|
|
|
105
105
|
/** 记录最近一次实际切换节点的时间(failover/巡检/手动切换共用锚点,冷却期判断依据) */
|
|
106
106
|
_markSwitchTime(c) {
|
|
107
|
-
if (c)
|
|
107
|
+
if (!c) return;
|
|
108
|
+
c._lastSwitchAt = Date.now();
|
|
109
|
+
// 切换通知序号:每次实际切换 +1,watchdog 感知变化后强制重启全部命令
|
|
110
|
+
// (先打通路:发现切了就重启;冷却/开关等控制后续再加)
|
|
111
|
+
c._restartSeq = (c._restartSeq || 0) + 1;
|
|
108
112
|
}
|
|
109
113
|
|
|
110
114
|
/**
|
|
@@ -191,7 +195,12 @@ export class ClashDaemon {
|
|
|
191
195
|
|
|
192
196
|
/** 写通道健康状态(每端口一份;主控制端额外写默认文件兼容 watchdog) */
|
|
193
197
|
_writeHealth(c, state) {
|
|
194
|
-
const payload = {
|
|
198
|
+
const payload = {
|
|
199
|
+
...state,
|
|
200
|
+
// 切换通知序号(每次实际切换 +1,watchdog 比较变化感知切换)
|
|
201
|
+
restartSeq: (c && c._restartSeq) || 0,
|
|
202
|
+
meta: this._buildMeta(c),
|
|
203
|
+
};
|
|
195
204
|
ipc.writeHealth(payload, c.proxyPort || 0);
|
|
196
205
|
if (c === this._primary && c.proxyPort) ipc.writeHealth(payload, 0);
|
|
197
206
|
}
|
|
@@ -667,21 +676,28 @@ export class ClashDaemon {
|
|
|
667
676
|
try {
|
|
668
677
|
const reason = (cmd && cmd.reason) || "看门狗主动故障切换";
|
|
669
678
|
const now = Date.now();
|
|
679
|
+
// 强切(force=true,看门狗客户端异常监控触发):无视冷却期直接轮换;
|
|
680
|
+
// 切换成功后仍更新 _lastSwitchAt 锚点,给新节点干净观察期
|
|
681
|
+
const force = !!(cmd && cmd.force);
|
|
670
682
|
|
|
671
683
|
// 第 1 级:先检查当前节点(壳页 = 链路正常,按健康对待,与巡检口径一致)
|
|
672
684
|
const h = checker.checkNodeHealth();
|
|
673
685
|
const probeOk = !!h && (h.healthy || this._isShellPage(h));
|
|
674
686
|
if (probeOk) {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
this.
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
687
|
+
if (!force) {
|
|
688
|
+
// 探针健康:受冷却期约束(锚点 = 最近一次实际切换节点的时间 _lastSwitchAt)
|
|
689
|
+
const gap = now - (c._lastSwitchAt || 0);
|
|
690
|
+
if (gap < this.failoverCooldownMs) {
|
|
691
|
+
this._log(
|
|
692
|
+
`🔄 [${c.proxyPort}] 探针健康,距上次切换 ${Math.round(gap / 1000)}s(<${this.failoverCooldownMs / 60000}min 冷却),仅检查不切换 [${reason}]`,
|
|
693
|
+
);
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
682
696
|
}
|
|
683
697
|
this._log(
|
|
684
|
-
|
|
698
|
+
force
|
|
699
|
+
? `🔄 [${c.proxyPort}] 强切指令(无视冷却),按业务级封禁轮换 [${reason}]`
|
|
700
|
+
: `🔄 [${c.proxyPort}] 探针健康但业务持续报错,距上次切换已过冷却期,按业务级封禁轮换 [${reason}]`,
|
|
685
701
|
);
|
|
686
702
|
} else {
|
|
687
703
|
this._log(
|
package/src/cli/tag.js
CHANGED
|
@@ -2,6 +2,7 @@ import { writeFileSync } from "fs";
|
|
|
2
2
|
import { randomUUID } from "crypto";
|
|
3
3
|
import http from "http";
|
|
4
4
|
import readline from "readline";
|
|
5
|
+
import os from "os";
|
|
5
6
|
import {
|
|
6
7
|
getBrowserMemory,
|
|
7
8
|
formatMemoryWithBrowser,
|
|
@@ -1058,11 +1059,12 @@ export async function handleScoreAll(parsed) {
|
|
|
1058
1059
|
let retrySameTag = false; // 是否重试同一个 tag
|
|
1059
1060
|
let tag403Attempts = 0; // 当前 tag 已发生 403 的次数
|
|
1060
1061
|
let bannedCount = 0; // 连续判定“被封”的次数(成功 tag 清零)
|
|
1061
|
-
//
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
parseInt(process.env.
|
|
1065
|
-
parseInt(process.env.
|
|
1062
|
+
// 403 渐退重试间隔(秒):30/40/50s 各重试一次,仍 403 才判定“被封”一次。
|
|
1063
|
+
// 与 api-interceptor 被封重试(30/40/50s 共 3 次)保持一致
|
|
1064
|
+
const BAN_RETRY_WAITS = [
|
|
1065
|
+
(parseInt(process.env.TAG_SCOREALL_403_RETRY_1 || "30", 10) || 30) * 1000,
|
|
1066
|
+
(parseInt(process.env.TAG_SCOREALL_403_RETRY_2 || "40", 10) || 40) * 1000,
|
|
1067
|
+
(parseInt(process.env.TAG_SCOREALL_403_RETRY_3 || "50", 10) || 50) * 1000,
|
|
1066
1068
|
];
|
|
1067
1069
|
|
|
1068
1070
|
// 生成客户端 ID,用于服务端追踪
|
|
@@ -1439,26 +1441,32 @@ export async function handleScoreAll(parsed) {
|
|
|
1439
1441
|
e.message &&
|
|
1440
1442
|
(e.message.includes("验证码") || e.message.includes("captcha"));
|
|
1441
1443
|
|
|
1442
|
-
// ── 403
|
|
1444
|
+
// ── 403:渐退等待 30/40/50s 各重试一次,仍 403 判定“被封” ──
|
|
1445
|
+
// 首次被封重启浏览器(同端口换新会话),后续仍 403 只清 tab
|
|
1443
1446
|
if (is403) {
|
|
1447
|
+
const retryIndex = tag403Attempts;
|
|
1444
1448
|
tag403Attempts++;
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
Math.min(bannedCount, BAN_RETRY_BUDGETS.length - 1)
|
|
1448
|
-
];
|
|
1449
|
-
if (tag403Attempts <= budget) {
|
|
1449
|
+
if (retryIndex < BAN_RETRY_WAITS.length) {
|
|
1450
|
+
const waitMs = BAN_RETRY_WAITS[retryIndex];
|
|
1450
1451
|
retrySameTag = true;
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1452
|
+
if (retryIndex === 0) {
|
|
1453
|
+
log(
|
|
1454
|
+
` 🚫 HTTP 403,判定被封,重启浏览器后 ${waitMs / 1000}s 第 ${tag403Attempts}/${BAN_RETRY_WAITS.length} 次重试 #${currentTag} ...`,
|
|
1455
|
+
);
|
|
1456
|
+
await recycleCdpSession(`首次被封`, { switchPort: false });
|
|
1457
|
+
} else {
|
|
1458
|
+
log(
|
|
1459
|
+
` 🚫 HTTP 403,判定被封,${waitMs / 1000}s 后第 ${tag403Attempts}/${BAN_RETRY_WAITS.length} 次重试 #${currentTag} ...`,
|
|
1460
|
+
);
|
|
1461
|
+
await recyclePage();
|
|
1462
|
+
}
|
|
1463
|
+
await new Promise((r) => setTimeout(r, waitMs));
|
|
1456
1464
|
continue;
|
|
1457
1465
|
}
|
|
1458
1466
|
// 重试耗尽:判定“被封”
|
|
1459
1467
|
bannedCount++;
|
|
1460
1468
|
log(
|
|
1461
|
-
` 🚫 重试 ${
|
|
1469
|
+
` 🚫 30/40/50s 重试 ${BAN_RETRY_WAITS.length} 次仍 403,判定账号被封(连续判封 ${bannedCount} 次)`,
|
|
1462
1470
|
);
|
|
1463
1471
|
if (bannedCount >= 2) {
|
|
1464
1472
|
log(` ⚠️ 连续两次判封 → 汇报服务端 + 触发 daemon fallback`);
|
|
@@ -1511,6 +1519,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1511
1519
|
errorMessage: e.message,
|
|
1512
1520
|
stage: "tag-scoring",
|
|
1513
1521
|
errorStack: "",
|
|
1522
|
+
hostname: os.hostname(),
|
|
1514
1523
|
}),
|
|
1515
1524
|
});
|
|
1516
1525
|
} catch {}
|
package/src/lib/api-client.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import os from "os";
|
|
2
3
|
|
|
3
4
|
const MAX_RETRY_WAIT = 5 * 60 * 1000;
|
|
4
5
|
|
|
@@ -68,13 +69,18 @@ export function createApiClient(opts = {}) {
|
|
|
68
69
|
return withRetry(
|
|
69
70
|
`POST ${url}`,
|
|
70
71
|
async () => {
|
|
72
|
+
const headers = buildHeaders(
|
|
73
|
+
{ "Content-Type": "application/json" },
|
|
74
|
+
clientId,
|
|
75
|
+
meta,
|
|
76
|
+
);
|
|
77
|
+
// error-report 自动带上本机 hostname,供看门狗按机器关联异常
|
|
78
|
+
if (/\/api\/error-report(\?|$)/.test(String(url))) {
|
|
79
|
+
headers["X-Client-Hostname"] = os.hostname();
|
|
80
|
+
}
|
|
71
81
|
const res = await fetch(url, {
|
|
72
82
|
method: "POST",
|
|
73
|
-
headers
|
|
74
|
-
{ "Content-Type": "application/json" },
|
|
75
|
-
clientId,
|
|
76
|
-
meta,
|
|
77
|
-
),
|
|
83
|
+
headers,
|
|
78
84
|
body: JSON.stringify(body),
|
|
79
85
|
signal: AbortSignal.timeout(30000),
|
|
80
86
|
});
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
* 供 tag / comments 等业务命令复用,避免重复实现。
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
+
import os from "os";
|
|
7
|
+
|
|
6
8
|
/**
|
|
7
9
|
* 触发 Clash daemon 的故障切换(换出口节点),与 watchdog 的 failover 一致。
|
|
8
10
|
* 复用 ../clash/ipc.js 现成的 writeCommand 接口,不改动 daemon/ipc 模块。
|
|
@@ -47,6 +49,7 @@ export async function reportClientErrorAndFailover({
|
|
|
47
49
|
errorMessage: message,
|
|
48
50
|
stage,
|
|
49
51
|
errorStack: "",
|
|
52
|
+
hostname: os.hostname(), // 机器标识:看门狗按 hostname 关联本机异常
|
|
50
53
|
});
|
|
51
54
|
} catch {}
|
|
52
55
|
await triggerClashFailover(message, log);
|
package/src/watch/data-store.js
CHANGED
|
@@ -2477,10 +2477,12 @@ export function createStore(filePath, options = {}) {
|
|
|
2477
2477
|
username,
|
|
2478
2478
|
stage,
|
|
2479
2479
|
errorStack,
|
|
2480
|
+
hostname,
|
|
2480
2481
|
) {
|
|
2481
2482
|
const existing = clientErrors.get(userId);
|
|
2482
2483
|
if (existing) {
|
|
2483
2484
|
existing.timestamp = Date.now();
|
|
2485
|
+
if (hostname) existing.hostname = hostname;
|
|
2484
2486
|
if (errorType === "captcha") {
|
|
2485
2487
|
existing.captchaCount = (existing.captchaCount || 0) + 1;
|
|
2486
2488
|
if (!existing.captchaStage) existing.captchaStage = stage || "";
|
|
@@ -2498,6 +2500,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2498
2500
|
} else {
|
|
2499
2501
|
clientErrors.set(userId, {
|
|
2500
2502
|
userId,
|
|
2503
|
+
hostname: hostname || "",
|
|
2501
2504
|
errorType,
|
|
2502
2505
|
errorMessage: errorMessage || "",
|
|
2503
2506
|
errorStack: errorStack || "",
|