tt-help-cli-ycl 1.4.140 → 1.4.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/sync-raw-jobs.mjs +13 -7
- package/scripts/test-api-interceptor-429.mjs +122 -0
- package/scripts/test-client-error-monitor.mjs +176 -0
- package/scripts/test-daemon-failover.mjs +13 -0
- package/scripts/test-proxy-switch-restart.mjs +198 -0
- package/src/clash/daemon.js +26 -10
- package/src/cli/comments.js +5 -5
- package/src/cli/precheck.js +2 -2
- package/src/cli/refresh-mini.js +1 -1
- package/src/cli/tag.js +32 -41
- package/src/lib/api-client.js +11 -5
- package/src/lib/api-interceptor.js +26 -8
- package/src/lib/app-config.js +119 -0
- package/src/lib/args.js +3 -1
- package/src/lib/client-error-report.js +3 -0
- package/src/lib/constants.js +3 -1
- package/src/lib/target-locations.js +1 -0
- package/src/scraper/core.js +2 -2
- package/src/scraper/explore-core.js +2 -2
- package/src/watch/data-store.js +9 -8
- package/src/watch/public/app.js +362 -34
- package/src/watch/public/index.html +62 -2
- package/src/watch/public/style.css +16 -0
- package/src/watch/server.js +85 -40
- package/src/watch/watchdog-server.js +100 -0
- package/src/watchdog/agent.js +157 -1
- package/src/watchdog/client-error-monitor.js +131 -0
- package/src/watchdog/process-manager.js +49 -0
- package/src/webserver/server.mjs +1 -0
package/package.json
CHANGED
|
@@ -15,13 +15,16 @@ import Database from "better-sqlite3";
|
|
|
15
15
|
import { initDb, getDb } from "../src/watch/db-schema.js";
|
|
16
16
|
import { loadLlmConfig, reloadLlmConfig } from "../src/watch/llm-config.js";
|
|
17
17
|
import { scoreJobsBatch } from "../src/watch/llm-scoring.js";
|
|
18
|
-
import {
|
|
18
|
+
import { getTargetLocations, reloadConfig } from "../src/lib/app-config.js";
|
|
19
19
|
|
|
20
20
|
// ═══════════════════════════════════════════════════════════════════
|
|
21
21
|
// 常量
|
|
22
22
|
// ═══════════════════════════════════════════════════════════════════
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
// 目标国家:每轮 tick 前 reloadConfig() 重读配置文件,捕捉服务端(设置页)修改
|
|
25
|
+
function currentLocations() {
|
|
26
|
+
return getTargetLocations();
|
|
27
|
+
}
|
|
25
28
|
|
|
26
29
|
const FILTER = "video_count>6 AND follower_count>0 AND following_count>0";
|
|
27
30
|
|
|
@@ -161,7 +164,7 @@ const vals = (row, cols) => cols.map((c) => row[c] ?? null);
|
|
|
161
164
|
const ph = (n) => Array(n).fill("?").join(", ");
|
|
162
165
|
|
|
163
166
|
/** 构建 WHERE 子句(含国家过滤)。strict=true 时仅匹配明确国家,不含 NULL/空 */
|
|
164
|
-
function buildWhere(locations =
|
|
167
|
+
function buildWhere(locations = currentLocations(), strict = false) {
|
|
165
168
|
const locs = [
|
|
166
169
|
...new Set(
|
|
167
170
|
locations.map((l) => String(l).trim().toUpperCase()).filter(Boolean),
|
|
@@ -268,14 +271,14 @@ function countPendingByCountry() {
|
|
|
268
271
|
const map = {};
|
|
269
272
|
for (const r of rows) map[r.loc] = r.c;
|
|
270
273
|
// 确保所有目标国家都有值
|
|
271
|
-
for (const loc of
|
|
274
|
+
for (const loc of currentLocations()) {
|
|
272
275
|
if (!(loc in map)) map[loc] = 0;
|
|
273
276
|
}
|
|
274
277
|
return map;
|
|
275
278
|
}
|
|
276
279
|
|
|
277
280
|
function findNeedyCountries(byCountry, threshold) {
|
|
278
|
-
return
|
|
281
|
+
return currentLocations().filter((loc) => (byCountry[loc] || 0) < threshold);
|
|
279
282
|
}
|
|
280
283
|
|
|
281
284
|
// ═══════════════════════════════════════════════════════════════════
|
|
@@ -335,7 +338,7 @@ async function refillLLM(archiveDb, needyLocations) {
|
|
|
335
338
|
|
|
336
339
|
// 2. LLM 打分
|
|
337
340
|
const ts = Date.now();
|
|
338
|
-
const scores = await scoreJobsBatch(samples,
|
|
341
|
+
const scores = await scoreJobsBatch(samples, currentLocations());
|
|
339
342
|
log(" ", `${loc} 打分${scores.length}条 ${Date.now() - ts}ms`);
|
|
340
343
|
|
|
341
344
|
if (
|
|
@@ -406,6 +409,7 @@ async function tick(archiveDb, opts) {
|
|
|
406
409
|
const t0 = Date.now();
|
|
407
410
|
|
|
408
411
|
archiveNewUsers(archiveDb);
|
|
412
|
+
reloadConfig(); // 每轮重读目标国家配置,支持服务端(设置页)修改
|
|
409
413
|
if (opts.llm) reloadLlmConfig(); // 每轮重载,支持热切换模型
|
|
410
414
|
|
|
411
415
|
const byCountry = countPendingByCountry();
|
|
@@ -414,7 +418,9 @@ async function tick(archiveDb, opts) {
|
|
|
414
418
|
if (!needy.length) {
|
|
415
419
|
log(
|
|
416
420
|
"--",
|
|
417
|
-
`轮${tickN} 全部充足 | ${
|
|
421
|
+
`轮${tickN} 全部充足 | ${currentLocations()
|
|
422
|
+
.map((l) => `${l}:${byCountry[l]}`)
|
|
423
|
+
.join(" ")}`,
|
|
418
424
|
);
|
|
419
425
|
return;
|
|
420
426
|
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// 临时验证:fetchUserVideosAPI 对 HTTP 429 / ratelimit triggered 判定为"被封"
|
|
2
|
+
// 运行:RM_BLOCKED_WAIT_MS=1 node scripts/test-api-interceptor-429.mjs
|
|
3
|
+
import { fetchUserVideosAPI } from "../src/lib/api-interceptor.js";
|
|
4
|
+
|
|
5
|
+
// 构造一个可被 waitForResponse 返回的 429 响应
|
|
6
|
+
function make429Response() {
|
|
7
|
+
return {
|
|
8
|
+
url: () => "https://www.tiktok.com/api/post/item_list/?...",
|
|
9
|
+
status: () => 429,
|
|
10
|
+
headers: () => ({ "content-length": "19" }),
|
|
11
|
+
json: async () => {
|
|
12
|
+
throw new Error("Unexpected token 'r'...");
|
|
13
|
+
},
|
|
14
|
+
body: async () => Buffer.from("ratelimit triggered"),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// 构造一个正常响应(itemList 非空)
|
|
19
|
+
function make200Response(username) {
|
|
20
|
+
const itemList = [
|
|
21
|
+
{ id: "111", stats: { playCount: 1 }, isECVideo: 0, isAd: 0 },
|
|
22
|
+
];
|
|
23
|
+
return {
|
|
24
|
+
url: () => "https://www.tiktok.com/api/post/item_list/?...",
|
|
25
|
+
status: () => 200,
|
|
26
|
+
headers: () => ({}),
|
|
27
|
+
json: async () => ({ itemList, cursor: 0, hasMore: false }),
|
|
28
|
+
body: async () => Buffer.from(JSON.stringify({ itemList })),
|
|
29
|
+
username,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// 构造 mock page:封禁场景时 waitForResponse 返回 429 响应
|
|
34
|
+
function makePage(username, respFactory) {
|
|
35
|
+
const page = {
|
|
36
|
+
_handlers: {},
|
|
37
|
+
on(event, fn) {
|
|
38
|
+
(page._handlers[event] ||= []).push(fn);
|
|
39
|
+
return page;
|
|
40
|
+
},
|
|
41
|
+
off(event) {
|
|
42
|
+
delete page._handlers[event];
|
|
43
|
+
return page;
|
|
44
|
+
},
|
|
45
|
+
url: () => `https://www.tiktok.com/@${username}`,
|
|
46
|
+
goto: async () => {},
|
|
47
|
+
// checkCdpHealth 用 () => document.readyState(fn.length===0)→ 返回 "complete";
|
|
48
|
+
// detectPageError 用 (patterns)=>...(fn.length===1)→ 返回 null(页面无错误文案)
|
|
49
|
+
evaluate: async (fn) => (fn.length === 1 ? null : "complete"),
|
|
50
|
+
reload: async () => {},
|
|
51
|
+
waitForFunction: async () => null,
|
|
52
|
+
waitForResponse: async (pred, opts) => {
|
|
53
|
+
const resp = respFactory();
|
|
54
|
+
if (!pred(resp)) throw new Error("response 不匹配 item_list");
|
|
55
|
+
return resp;
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
return page;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
let pass = 0;
|
|
62
|
+
let fail = 0;
|
|
63
|
+
|
|
64
|
+
async function expectReject(name, fn, expectedMsg) {
|
|
65
|
+
try {
|
|
66
|
+
await fn();
|
|
67
|
+
fail++;
|
|
68
|
+
console.error(`✗ ${name}: 未抛错(期望 "${expectedMsg}")`);
|
|
69
|
+
} catch (e) {
|
|
70
|
+
if (e.message === expectedMsg || e.message.includes(expectedMsg)) {
|
|
71
|
+
pass++;
|
|
72
|
+
console.log(`✓ ${name}: 抛错 "${e.message}"`);
|
|
73
|
+
} else {
|
|
74
|
+
fail++;
|
|
75
|
+
console.error(
|
|
76
|
+
`✗ ${name}: 抛错 "${e.message}"(期望含 "${expectedMsg}")`,
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function expectResolve(name, fn) {
|
|
83
|
+
try {
|
|
84
|
+
const r = await fn();
|
|
85
|
+
if (r instanceof Map && r.size === 1) {
|
|
86
|
+
pass++;
|
|
87
|
+
console.log(`✓ ${name}: 正常返回 ${r.size} 个视频`);
|
|
88
|
+
} else {
|
|
89
|
+
fail++;
|
|
90
|
+
console.error(`✗ ${name}: 返回异常 ${JSON.stringify(r)}`);
|
|
91
|
+
}
|
|
92
|
+
} catch (e) {
|
|
93
|
+
fail++;
|
|
94
|
+
console.error(`✗ ${name}: 抛错 ${e.message}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// 用例1:429 + ratelimit triggered → 判定为"被封"
|
|
99
|
+
await expectReject(
|
|
100
|
+
"HTTP 429 ratelimit triggered 判为被封",
|
|
101
|
+
() =>
|
|
102
|
+
fetchUserVideosAPI(
|
|
103
|
+
makePage("fumolbbf08o", make429Response),
|
|
104
|
+
"fumolbbf08o",
|
|
105
|
+
5,
|
|
106
|
+
() => {},
|
|
107
|
+
),
|
|
108
|
+
"被封: fumolbbf08o",
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
// 用例2:正常 200 itemList → 不误判,正常返回视频
|
|
112
|
+
await expectResolve("HTTP 200 正常 itemList 不被误伤", () =>
|
|
113
|
+
fetchUserVideosAPI(
|
|
114
|
+
makePage("fumolbbf08o", make200Response),
|
|
115
|
+
"fumolbbf08o",
|
|
116
|
+
5,
|
|
117
|
+
() => {},
|
|
118
|
+
),
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
console.log(`\n结果: ${pass} 通过, ${fail} 失败`);
|
|
122
|
+
process.exit(fail > 0 ? 1 : 0);
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 客户端异常监控单测(2026-08-28)
|
|
3
|
+
*
|
|
4
|
+
* 验证:
|
|
5
|
+
* - detectClientError 检测函数:reportCount > 8 且最近一次上报在 2 分钟内
|
|
6
|
+
* - 按 hostname 过滤本机条目;hostname 为空的旧条目不参与关联
|
|
7
|
+
* - 命中 → onAlert 收到本机全部条目(含未命中条目);未命中 → 不回调
|
|
8
|
+
* - removeErrors 逐条 DELETE 并计数
|
|
9
|
+
*
|
|
10
|
+
* 运行: node --test scripts/test-client-error-monitor.mjs
|
|
11
|
+
*/
|
|
12
|
+
import { test, mock } from "node:test";
|
|
13
|
+
import assert from "node:assert/strict";
|
|
14
|
+
import os from "os";
|
|
15
|
+
import {
|
|
16
|
+
detectClientError,
|
|
17
|
+
ClientErrorMonitor,
|
|
18
|
+
} from "../src/watchdog/client-error-monitor.js";
|
|
19
|
+
|
|
20
|
+
const HOST = os.hostname();
|
|
21
|
+
|
|
22
|
+
function makeErr(overrides = {}) {
|
|
23
|
+
return {
|
|
24
|
+
userId: "u1",
|
|
25
|
+
hostname: HOST,
|
|
26
|
+
errorType: "consecutive_blocked",
|
|
27
|
+
reportCount: 10,
|
|
28
|
+
timestamp: Date.now() - 30_000,
|
|
29
|
+
...overrides,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// ── 检测函数 ──
|
|
34
|
+
|
|
35
|
+
test("D1 detectClientError:累计>8 且 2 分钟内 → 命中", () => {
|
|
36
|
+
assert.equal(
|
|
37
|
+
detectClientError(makeErr({ reportCount: 9 }), Date.now()),
|
|
38
|
+
true,
|
|
39
|
+
);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("D2 detectClientError:reportCount=8 不命中(阈值是 >8)", () => {
|
|
43
|
+
assert.equal(
|
|
44
|
+
detectClientError(makeErr({ reportCount: 8 }), Date.now()),
|
|
45
|
+
false,
|
|
46
|
+
);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("D3 detectClientError:最近上报超过 2 分钟 → 不命中", () => {
|
|
50
|
+
assert.equal(
|
|
51
|
+
detectClientError(
|
|
52
|
+
makeErr({ timestamp: Date.now() - 2 * 60_000 - 1 }),
|
|
53
|
+
Date.now(),
|
|
54
|
+
),
|
|
55
|
+
false,
|
|
56
|
+
);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("D4 detectClientError:缺 reportCount → 不命中", () => {
|
|
60
|
+
assert.equal(detectClientError({ timestamp: Date.now() }, Date.now()), false);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
// ── 监控器 ──
|
|
64
|
+
|
|
65
|
+
test("M1 按 hostname 过滤:他机条目不触发本机回调", async () => {
|
|
66
|
+
mock.method(globalThis, "fetch", async () => ({
|
|
67
|
+
ok: true,
|
|
68
|
+
json: async () => ({
|
|
69
|
+
clients: [makeErr({ hostname: "OTHER-MACHINE" })],
|
|
70
|
+
}),
|
|
71
|
+
}));
|
|
72
|
+
let alerted = false;
|
|
73
|
+
const m = new ClientErrorMonitor({
|
|
74
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
75
|
+
hostname: HOST,
|
|
76
|
+
onAlert: async () => {
|
|
77
|
+
alerted = true;
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
const hit = await m._poll();
|
|
81
|
+
assert.equal(alerted, false, "他机异常不应回调");
|
|
82
|
+
assert.equal(hit, null, "无本机条目应返回 null");
|
|
83
|
+
mock.restoreAll();
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("M2 命中:onAlert 收到本机全部条目(含未命中条目)", async () => {
|
|
87
|
+
const mineHit = makeErr({ userId: "u-hit", reportCount: 10 });
|
|
88
|
+
const mineMiss = makeErr({ userId: "u-miss", reportCount: 2 });
|
|
89
|
+
const other = makeErr({ userId: "u-other", hostname: "OTHER-MACHINE" });
|
|
90
|
+
mock.method(globalThis, "fetch", async () => ({
|
|
91
|
+
ok: true,
|
|
92
|
+
json: async () => ({ clients: [mineHit, mineMiss, other] }),
|
|
93
|
+
}));
|
|
94
|
+
let received = null;
|
|
95
|
+
const m = new ClientErrorMonitor({
|
|
96
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
97
|
+
hostname: HOST,
|
|
98
|
+
onAlert: async (mine, hit) => {
|
|
99
|
+
received = { mine, hit };
|
|
100
|
+
},
|
|
101
|
+
});
|
|
102
|
+
const hit = await m._poll();
|
|
103
|
+
assert.equal(hit.length, 1, "命中 1 条");
|
|
104
|
+
assert.equal(hit[0].userId, "u-hit");
|
|
105
|
+
assert.equal(received.mine.length, 2, "回调收到本机全部 2 条");
|
|
106
|
+
assert.deepEqual(
|
|
107
|
+
received.mine.map((e) => e.userId).sort(),
|
|
108
|
+
["u-hit", "u-miss"],
|
|
109
|
+
"回调含命中与未命中的本机条目",
|
|
110
|
+
);
|
|
111
|
+
mock.restoreAll();
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("M3 hostname 为空的旧条目不参与本机关联", async () => {
|
|
115
|
+
mock.method(globalThis, "fetch", async () => ({
|
|
116
|
+
ok: true,
|
|
117
|
+
json: async () => ({ clients: [makeErr({ hostname: "" })] }),
|
|
118
|
+
}));
|
|
119
|
+
let alerted = false;
|
|
120
|
+
const m = new ClientErrorMonitor({
|
|
121
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
122
|
+
hostname: HOST,
|
|
123
|
+
onAlert: async () => {
|
|
124
|
+
alerted = true;
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
const hit = await m._poll();
|
|
128
|
+
assert.equal(alerted, false, "空 hostname 旧条目不应回调");
|
|
129
|
+
assert.equal(hit, null);
|
|
130
|
+
mock.restoreAll();
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("M4 removeErrors:逐条 DELETE 并计数", async () => {
|
|
134
|
+
const deleted = [];
|
|
135
|
+
mock.method(globalThis, "fetch", async (url, opts) => {
|
|
136
|
+
if (opts && opts.method === "DELETE") {
|
|
137
|
+
deleted.push(String(url));
|
|
138
|
+
return { ok: true };
|
|
139
|
+
}
|
|
140
|
+
return { ok: true, json: async () => ({ clients: [] }) };
|
|
141
|
+
});
|
|
142
|
+
const m = new ClientErrorMonitor({ serverUrl: "http://127.0.0.1:3000" });
|
|
143
|
+
const removed = await m.removeErrors([
|
|
144
|
+
{ userId: "a" },
|
|
145
|
+
{ userId: "b" },
|
|
146
|
+
{ userId: "" },
|
|
147
|
+
null,
|
|
148
|
+
]);
|
|
149
|
+
assert.equal(removed, 2, "有效 userId 各删一条");
|
|
150
|
+
assert.equal(deleted.length, 2, "共 2 次 DELETE");
|
|
151
|
+
assert.ok(
|
|
152
|
+
deleted.every((u) =>
|
|
153
|
+
["a", "b"].some((id) => u.endsWith(`/api/client-error/${id}`)),
|
|
154
|
+
),
|
|
155
|
+
"DELETE 路径正确",
|
|
156
|
+
);
|
|
157
|
+
mock.restoreAll();
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("M5 拉取失败静默:fetch 抛错不回调不崩溃", async () => {
|
|
161
|
+
mock.method(globalThis, "fetch", async () => {
|
|
162
|
+
throw new Error("ECONNREFUSED");
|
|
163
|
+
});
|
|
164
|
+
let alerted = false;
|
|
165
|
+
const m = new ClientErrorMonitor({
|
|
166
|
+
serverUrl: "http://127.0.0.1:3000",
|
|
167
|
+
hostname: HOST,
|
|
168
|
+
onAlert: async () => {
|
|
169
|
+
alerted = true;
|
|
170
|
+
},
|
|
171
|
+
});
|
|
172
|
+
const hit = await m._poll();
|
|
173
|
+
assert.equal(alerted, false);
|
|
174
|
+
assert.equal(hit, null);
|
|
175
|
+
mock.restoreAll();
|
|
176
|
+
});
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* - 探针不健康 → 立即切换(无视冷却)
|
|
6
6
|
* - 探针健康(含壳页)→ 距上次切换 < 冷却期仅检查不切换
|
|
7
7
|
* - 探针健康 → 距上次切换 >= 冷却期 → 按业务级封禁轮换一次
|
|
8
|
+
* - 强切(force=true)→ 探针健康且冷却期内仍强制轮换
|
|
8
9
|
* - 切换后(含手动/巡检/failover)更新 _lastSwitchAt 锚点,冷却期内不再切
|
|
9
10
|
* - 候选集为空 → 触发 delay 初测且不切换
|
|
10
11
|
*
|
|
@@ -229,3 +230,15 @@ test("F9 disconnectMode=full:直接全断", async () => {
|
|
|
229
230
|
assert.equal(state.idleCloses, 0, "full 模式不走优雅断连");
|
|
230
231
|
assert.equal(state.closes, 1, "full 模式直接全断");
|
|
231
232
|
});
|
|
233
|
+
|
|
234
|
+
test("F10 强切(force):冷却期内探针健康仍强制轮换", async () => {
|
|
235
|
+
resetState();
|
|
236
|
+
const { d, c } = makeCtx();
|
|
237
|
+
c._lastSwitchAt = Date.now(); // 刚切过,处于 10min 冷却期内
|
|
238
|
+
await d._handleFailover({ reason: "F10", force: true }, c);
|
|
239
|
+
assert.deepEqual(state.switches, ["🇲🇾 马来02"], "force 应无视冷却直接轮换");
|
|
240
|
+
assert.ok(
|
|
241
|
+
c._lastSwitchAt && c._lastSwitchAt > Date.now() - 5000,
|
|
242
|
+
"强切后应更新锚点(给新节点观察期)",
|
|
243
|
+
);
|
|
244
|
+
});
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 代理切换 → watchdog 强制重启命令 链路测试(2026-08-28,先打通路版本)
|
|
3
|
+
*
|
|
4
|
+
* 验证:
|
|
5
|
+
* T1 daemon _markSwitchTime 递增 restartSeq(首次为 1,连续递增)
|
|
6
|
+
* T2 daemon _writeHealth 把 restartSeq 带进 health 文件
|
|
7
|
+
* T3 daemon 未切换时 restartSeq 保持 0
|
|
8
|
+
* T4 ProcessManager.restartAll 先停后起(真实子进程,pid 变化、数量恢复)
|
|
9
|
+
* T5 restartAll 跳过 stopped / 无 child 的命令
|
|
10
|
+
* T6 无运行命令时 restartAll 返回 {restarted:0}
|
|
11
|
+
* T7 Agent._checkProxySwitch 首次读取仅建基线不触发
|
|
12
|
+
* T8 health.restartSeq 变大 → 触发强制重启(真实子进程 pid 变化)
|
|
13
|
+
* T9 health.restartSeq 变小(daemon 重启归零)→ 不触发
|
|
14
|
+
* T10 通道重启:主文件 seq 不变时不触发
|
|
15
|
+
*
|
|
16
|
+
* 运行: node --test scripts/test-proxy-switch-restart.mjs
|
|
17
|
+
*/
|
|
18
|
+
import { test } from "node:test";
|
|
19
|
+
import assert from "node:assert/strict";
|
|
20
|
+
import { writeFileSync, rmSync, existsSync, readFileSync } from "fs";
|
|
21
|
+
import { join } from "path";
|
|
22
|
+
import { tmpdir } from "os";
|
|
23
|
+
|
|
24
|
+
const HEALTH_FILE = join(tmpdir(), "clash-health.json");
|
|
25
|
+
|
|
26
|
+
/** 清理并写 health 文件(模拟 daemon 写入) */
|
|
27
|
+
function writeHealthFile(extra) {
|
|
28
|
+
writeFileSync(
|
|
29
|
+
HEALTH_FILE,
|
|
30
|
+
JSON.stringify(
|
|
31
|
+
{
|
|
32
|
+
node: "🇺🇸 美国01",
|
|
33
|
+
healthy: true,
|
|
34
|
+
lastCheck: new Date().toISOString(),
|
|
35
|
+
...extra,
|
|
36
|
+
},
|
|
37
|
+
null,
|
|
38
|
+
2,
|
|
39
|
+
),
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** 生成一个常驻假命令(测试业务进程) */
|
|
44
|
+
function fakeCmd(id) {
|
|
45
|
+
return {
|
|
46
|
+
id,
|
|
47
|
+
name: id,
|
|
48
|
+
command: process.execPath,
|
|
49
|
+
args: ["-e", "setInterval(()=>{},1000)"],
|
|
50
|
+
enabled: true,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function cleanup() {
|
|
55
|
+
try {
|
|
56
|
+
rmSync(HEALTH_FILE, { force: true });
|
|
57
|
+
} catch {}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
test("T1-T3: daemon 切换信号(_markSwitchTime 递增 + _writeHealth 带出)", async () => {
|
|
61
|
+
const { ClashDaemon } = await import("../src/clash/daemon.js");
|
|
62
|
+
const ipc = await import("../src/clash/ipc.js");
|
|
63
|
+
cleanup();
|
|
64
|
+
try {
|
|
65
|
+
const d = new ClashDaemon({ dryRun: true });
|
|
66
|
+
const c = { proxyPort: 0 };
|
|
67
|
+
|
|
68
|
+
// T3: 未切换 → restartSeq = 0
|
|
69
|
+
d._writeHealth(c, { node: "A", healthy: true });
|
|
70
|
+
let h = ipc.readHealth(0);
|
|
71
|
+
assert.equal(h.restartSeq, 0, "未切换时 restartSeq 应为 0");
|
|
72
|
+
|
|
73
|
+
// T1: 首次切换 → 1
|
|
74
|
+
d._markSwitchTime(c);
|
|
75
|
+
assert.equal(c._restartSeq, 1, "首次切换 restartSeq 应为 1");
|
|
76
|
+
d._markSwitchTime(c);
|
|
77
|
+
assert.equal(c._restartSeq, 2, "连续切换 restartSeq 递增");
|
|
78
|
+
|
|
79
|
+
// T2: _writeHealth 带出 restartSeq
|
|
80
|
+
d._writeHealth(c, { node: "B", healthy: true, lastSwitch: "A → B" });
|
|
81
|
+
h = ipc.readHealth(0);
|
|
82
|
+
assert.equal(h.restartSeq, 2, "health 文件应携带 restartSeq=2");
|
|
83
|
+
assert.equal(h.lastSwitch, "A → B");
|
|
84
|
+
} finally {
|
|
85
|
+
cleanup();
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("T4-T6: ProcessManager.restartAll 先停后起", async () => {
|
|
90
|
+
const { ProcessManager } = await import("../src/watchdog/process-manager.js");
|
|
91
|
+
const pm = new ProcessManager();
|
|
92
|
+
try {
|
|
93
|
+
// T6: 空集
|
|
94
|
+
const empty = await pm.restartAll("测试");
|
|
95
|
+
assert.deepEqual(empty, { restarted: 0 }, "无运行命令应返回 0");
|
|
96
|
+
|
|
97
|
+
pm.startCommand(fakeCmd("proc-a"));
|
|
98
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
99
|
+
assert.equal(pm.runningCount, 1);
|
|
100
|
+
const oldPid = pm.processes.get("proc-a").child.pid;
|
|
101
|
+
|
|
102
|
+
// T4: 先停后起,pid 变化、数量恢复
|
|
103
|
+
const r = await pm.restartAll("代理切换(测试)");
|
|
104
|
+
assert.equal(r.restarted, 1, "应重启 1 个命令");
|
|
105
|
+
assert.equal(pm.runningCount, 1, "重启后数量恢复");
|
|
106
|
+
const newPid = pm.processes.get("proc-a").child.pid;
|
|
107
|
+
assert.notEqual(newPid, oldPid, "重启后应是新进程(pid 不同)");
|
|
108
|
+
|
|
109
|
+
// T5: stopped 的命令不参与重启
|
|
110
|
+
pm.startCommand(fakeCmd("proc-b"));
|
|
111
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
112
|
+
pm.stopCommand("proc-b");
|
|
113
|
+
const r2 = await pm.restartAll("二次重启");
|
|
114
|
+
assert.equal(r2.restarted, 1, "只重启 running 的 1 个,stopped 的不算");
|
|
115
|
+
assert.equal(pm.runningCount, 1);
|
|
116
|
+
assert.ok(!pm.processes.has("proc-b"), "proc-b 保持停止状态");
|
|
117
|
+
} finally {
|
|
118
|
+
pm.stopAll();
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("T7-T13: Agent._checkProxySwitch 感知切换(默认禁用,开启后才重启)", async () => {
|
|
123
|
+
const { WatchdogAgent } = await import("../src/watchdog/agent.js");
|
|
124
|
+
cleanup();
|
|
125
|
+
const agent = new WatchdogAgent({
|
|
126
|
+
serverUrl: "http://127.0.0.1:9",
|
|
127
|
+
clientId: "test-switch",
|
|
128
|
+
intervalMs: 60000,
|
|
129
|
+
spawnDaemon: false,
|
|
130
|
+
});
|
|
131
|
+
try {
|
|
132
|
+
// 准备一个运行中的假业务命令
|
|
133
|
+
agent.processManager.startCommand(fakeCmd("proc-c"));
|
|
134
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
135
|
+
const pid0 = agent.processManager.processes.get("proc-c").child.pid;
|
|
136
|
+
|
|
137
|
+
// T7: 首次读取建基线,不触发
|
|
138
|
+
writeHealthFile({ restartSeq: 0, lastSwitch: "初始化" });
|
|
139
|
+
await agent._checkProxySwitch();
|
|
140
|
+
assert.equal(
|
|
141
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
142
|
+
pid0,
|
|
143
|
+
"首次读取仅建基线,不应重启",
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
// T10: seq 不变不触发
|
|
147
|
+
writeHealthFile({ restartSeq: 0, lastSwitch: "初始化" });
|
|
148
|
+
await agent._checkProxySwitch();
|
|
149
|
+
assert.equal(
|
|
150
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
151
|
+
pid0,
|
|
152
|
+
"seq 不变不应重启",
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
// T11: 默认禁用时 seq 变大 → 不触发(但基线已跟随更新)
|
|
156
|
+
writeHealthFile({ restartSeq: 1, lastSwitch: "禁用期间切换: A → B" });
|
|
157
|
+
await agent._checkProxySwitch();
|
|
158
|
+
assert.equal(
|
|
159
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
160
|
+
pid0,
|
|
161
|
+
"默认禁用时检测到切换也不应重启",
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
// T12: 开启开关后,seq 仍为 1(无新切换)→ 不补触发历史切换
|
|
165
|
+
agent._applySwitchRestartConfig({ enabled: true });
|
|
166
|
+
await agent._checkProxySwitch();
|
|
167
|
+
assert.equal(
|
|
168
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
169
|
+
pid0,
|
|
170
|
+
"开启开关后不应补触发开启前的历史切换",
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
// T8: 开启后 seq 1→2 → 触发强制重启
|
|
174
|
+
writeHealthFile({ restartSeq: 2, lastSwitch: "开启后切换: B → C" });
|
|
175
|
+
await agent._checkProxySwitch();
|
|
176
|
+
const pid1 = agent.processManager.processes.get("proc-c").child.pid;
|
|
177
|
+
assert.notEqual(pid1, pid0, "开启后检测到切换应强制重启(pid 变化)");
|
|
178
|
+
assert.equal(agent.processManager.runningCount, 1, "重启后数量恢复");
|
|
179
|
+
|
|
180
|
+
// T9: seq 变小(daemon 重启归零)→ 不触发
|
|
181
|
+
writeHealthFile({ restartSeq: 0, lastSwitch: "daemon 重启" });
|
|
182
|
+
await agent._checkProxySwitch();
|
|
183
|
+
assert.equal(
|
|
184
|
+
agent.processManager.processes.get("proc-c").child.pid,
|
|
185
|
+
pid1,
|
|
186
|
+
"seq 变小(daemon 重启归零)不应触发",
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
// T13: 归零后再次真实切换(0 → 1)应重新触发
|
|
190
|
+
writeHealthFile({ restartSeq: 1, lastSwitch: "再次切换: C → D" });
|
|
191
|
+
await agent._checkProxySwitch();
|
|
192
|
+
const pid2 = agent.processManager.processes.get("proc-c").child.pid;
|
|
193
|
+
assert.notEqual(pid2, pid1, "归零后再次切换应重新触发");
|
|
194
|
+
} finally {
|
|
195
|
+
await agent.stop();
|
|
196
|
+
cleanup();
|
|
197
|
+
}
|
|
198
|
+
});
|
package/src/clash/daemon.js
CHANGED
|
@@ -104,7 +104,11 @@ export class ClashDaemon {
|
|
|
104
104
|
|
|
105
105
|
/** 记录最近一次实际切换节点的时间(failover/巡检/手动切换共用锚点,冷却期判断依据) */
|
|
106
106
|
_markSwitchTime(c) {
|
|
107
|
-
if (c)
|
|
107
|
+
if (!c) return;
|
|
108
|
+
c._lastSwitchAt = Date.now();
|
|
109
|
+
// 切换通知序号:每次实际切换 +1,watchdog 感知变化后强制重启全部命令
|
|
110
|
+
// (先打通路:发现切了就重启;冷却/开关等控制后续再加)
|
|
111
|
+
c._restartSeq = (c._restartSeq || 0) + 1;
|
|
108
112
|
}
|
|
109
113
|
|
|
110
114
|
/**
|
|
@@ -191,7 +195,12 @@ export class ClashDaemon {
|
|
|
191
195
|
|
|
192
196
|
/** 写通道健康状态(每端口一份;主控制端额外写默认文件兼容 watchdog) */
|
|
193
197
|
_writeHealth(c, state) {
|
|
194
|
-
const payload = {
|
|
198
|
+
const payload = {
|
|
199
|
+
...state,
|
|
200
|
+
// 切换通知序号(每次实际切换 +1,watchdog 比较变化感知切换)
|
|
201
|
+
restartSeq: (c && c._restartSeq) || 0,
|
|
202
|
+
meta: this._buildMeta(c),
|
|
203
|
+
};
|
|
195
204
|
ipc.writeHealth(payload, c.proxyPort || 0);
|
|
196
205
|
if (c === this._primary && c.proxyPort) ipc.writeHealth(payload, 0);
|
|
197
206
|
}
|
|
@@ -667,21 +676,28 @@ export class ClashDaemon {
|
|
|
667
676
|
try {
|
|
668
677
|
const reason = (cmd && cmd.reason) || "看门狗主动故障切换";
|
|
669
678
|
const now = Date.now();
|
|
679
|
+
// 强切(force=true,看门狗客户端异常监控触发):无视冷却期直接轮换;
|
|
680
|
+
// 切换成功后仍更新 _lastSwitchAt 锚点,给新节点干净观察期
|
|
681
|
+
const force = !!(cmd && cmd.force);
|
|
670
682
|
|
|
671
683
|
// 第 1 级:先检查当前节点(壳页 = 链路正常,按健康对待,与巡检口径一致)
|
|
672
684
|
const h = checker.checkNodeHealth();
|
|
673
685
|
const probeOk = !!h && (h.healthy || this._isShellPage(h));
|
|
674
686
|
if (probeOk) {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
this.
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
687
|
+
if (!force) {
|
|
688
|
+
// 探针健康:受冷却期约束(锚点 = 最近一次实际切换节点的时间 _lastSwitchAt)
|
|
689
|
+
const gap = now - (c._lastSwitchAt || 0);
|
|
690
|
+
if (gap < this.failoverCooldownMs) {
|
|
691
|
+
this._log(
|
|
692
|
+
`🔄 [${c.proxyPort}] 探针健康,距上次切换 ${Math.round(gap / 1000)}s(<${this.failoverCooldownMs / 60000}min 冷却),仅检查不切换 [${reason}]`,
|
|
693
|
+
);
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
682
696
|
}
|
|
683
697
|
this._log(
|
|
684
|
-
|
|
698
|
+
force
|
|
699
|
+
? `🔄 [${c.proxyPort}] 强切指令(无视冷却),按业务级封禁轮换 [${reason}]`
|
|
700
|
+
: `🔄 [${c.proxyPort}] 探针健康但业务持续报错,距上次切换已过冷却期,按业务级封禁轮换 [${reason}]`,
|
|
685
701
|
);
|
|
686
702
|
} else {
|
|
687
703
|
this._log(
|