tt-help-cli-ycl 1.4.140 → 1.4.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/sync-raw-jobs.mjs +13 -7
- package/scripts/test-api-interceptor-429.mjs +122 -0
- package/scripts/test-client-error-monitor.mjs +176 -0
- package/scripts/test-daemon-failover.mjs +13 -0
- package/scripts/test-proxy-switch-restart.mjs +198 -0
- package/src/clash/daemon.js +26 -10
- package/src/cli/comments.js +5 -5
- package/src/cli/precheck.js +2 -2
- package/src/cli/refresh-mini.js +1 -1
- package/src/cli/tag.js +32 -41
- package/src/lib/api-client.js +11 -5
- package/src/lib/api-interceptor.js +26 -8
- package/src/lib/app-config.js +119 -0
- package/src/lib/args.js +3 -1
- package/src/lib/client-error-report.js +3 -0
- package/src/lib/constants.js +3 -1
- package/src/lib/target-locations.js +1 -0
- package/src/scraper/core.js +2 -2
- package/src/scraper/explore-core.js +2 -2
- package/src/watch/data-store.js +9 -8
- package/src/watch/public/app.js +362 -34
- package/src/watch/public/index.html +62 -2
- package/src/watch/public/style.css +16 -0
- package/src/watch/server.js +85 -40
- package/src/watch/watchdog-server.js +100 -0
- package/src/watchdog/agent.js +157 -1
- package/src/watchdog/client-error-monitor.js +131 -0
- package/src/watchdog/process-manager.js +49 -0
- package/src/webserver/server.mjs +1 -0
package/src/watch/server.js
CHANGED
|
@@ -8,7 +8,7 @@ import { fileURLToPath } from "url";
|
|
|
8
8
|
import { spawn } from "child_process";
|
|
9
9
|
import { createStore } from "./data-store.js";
|
|
10
10
|
import { getTagDb } from "./db-schema.js";
|
|
11
|
-
import {
|
|
11
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
12
12
|
import { tryHandleWatchdogRoute } from "./watchdog-server.js";
|
|
13
13
|
|
|
14
14
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -44,11 +44,17 @@ const SLOW_STATS_TTL = 300000; // 5 分钟
|
|
|
44
44
|
const usersCache = new Map(); // key -> { result, time }
|
|
45
45
|
const USERS_CACHE_TTL = 3000; // 3 秒
|
|
46
46
|
|
|
47
|
-
// 目标国家占位符(jobs 表聚合需要 location_created IN (...)
|
|
48
|
-
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
// 目标国家占位符(jobs 表聚合需要 location_created IN (...),运行时按 getTargetLocations() 动态生成)
|
|
48
|
+
function targetPlaceholders() {
|
|
49
|
+
const locs = getTargetLocations();
|
|
50
|
+
return locs.map(() => "?").join(", ");
|
|
51
|
+
}
|
|
52
|
+
function targetInSql() {
|
|
53
|
+
const locs = getTargetLocations();
|
|
54
|
+
return locs.length
|
|
55
|
+
? `location_created IN (${targetPlaceholders()})`
|
|
56
|
+
: "1 = 0";
|
|
57
|
+
}
|
|
52
58
|
|
|
53
59
|
// 统计分组定义:每组包含若干 SQL(分片执行)与一个聚合函数
|
|
54
60
|
// sqls 项:{ key, sql, mode: 'one'|'all', params: 数组或返回数组的函数 }
|
|
@@ -80,8 +86,8 @@ const STAT_GROUPS = {
|
|
|
80
86
|
{
|
|
81
87
|
key: "jobsAgg",
|
|
82
88
|
mode: "one",
|
|
83
|
-
params:
|
|
84
|
-
sql: `SELECT
|
|
89
|
+
params: () => getTargetLocations(),
|
|
90
|
+
sql: () => `SELECT
|
|
85
91
|
SUM(jobsTotal) as jobsTotal,
|
|
86
92
|
SUM(done) as done,
|
|
87
93
|
SUM(error) as error,
|
|
@@ -94,7 +100,7 @@ const STAT_GROUPS = {
|
|
|
94
100
|
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done,
|
|
95
101
|
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) as error,
|
|
96
102
|
SUM(CASE WHEN status = 'restricted' THEN 1 ELSE 0 END) as restricted,
|
|
97
|
-
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${
|
|
103
|
+
SUM(CASE WHEN tt_seller = 1 AND verified = 0 AND ${targetInSql()} THEN 1 ELSE 0 END) as targetUsers,
|
|
98
104
|
SUM(CASE WHEN no_video = 1 THEN 1 ELSE 0 END) as noVideo
|
|
99
105
|
FROM jobs
|
|
100
106
|
UNION ALL
|
|
@@ -177,10 +183,10 @@ const STAT_GROUPS = {
|
|
|
177
183
|
{
|
|
178
184
|
key: "targetCountryStats",
|
|
179
185
|
mode: "all",
|
|
180
|
-
params:
|
|
181
|
-
sql: `SELECT location_created as country, COUNT(*) as count
|
|
186
|
+
params: () => getTargetLocations(),
|
|
187
|
+
sql: () => `SELECT location_created as country, COUNT(*) as count
|
|
182
188
|
FROM jobs
|
|
183
|
-
WHERE tt_seller = 1 AND verified = 0 AND ${
|
|
189
|
+
WHERE tt_seller = 1 AND verified = 0 AND ${targetInSql()}
|
|
184
190
|
GROUP BY location_created
|
|
185
191
|
ORDER BY count DESC`,
|
|
186
192
|
},
|
|
@@ -284,9 +290,10 @@ function refreshGroupInShards(st, name) {
|
|
|
284
290
|
if (idx < spec.sqls.length) {
|
|
285
291
|
const s = spec.sqls[idx++];
|
|
286
292
|
try {
|
|
293
|
+
const sql = typeof s.sql === "function" ? s.sql() : s.sql;
|
|
287
294
|
const params =
|
|
288
295
|
typeof s.params === "function" ? s.params() : s.params || [];
|
|
289
|
-
const r = st.rawQuery(
|
|
296
|
+
const r = st.rawQuery(sql, params);
|
|
290
297
|
if (r?.error) {
|
|
291
298
|
console.error(`[stats] ${name}.${s.key} 查询失败: ${r.error}`);
|
|
292
299
|
results[s.key] = null;
|
|
@@ -334,9 +341,10 @@ function computeGroupStats(st, name) {
|
|
|
334
341
|
try {
|
|
335
342
|
const results = {};
|
|
336
343
|
for (const s of STAT_GROUPS[name].sqls) {
|
|
344
|
+
const sql = typeof s.sql === "function" ? s.sql() : s.sql;
|
|
337
345
|
const params =
|
|
338
346
|
typeof s.params === "function" ? s.params() : s.params || [];
|
|
339
|
-
const r = st.rawQuery(
|
|
347
|
+
const r = st.rawQuery(sql, params);
|
|
340
348
|
results[s.key] =
|
|
341
349
|
s.mode === "all" ? r?.rows || [] : r?.rows?.[0] || null;
|
|
342
350
|
}
|
|
@@ -641,7 +649,7 @@ export function startWatchServer(
|
|
|
641
649
|
// 按国家维度判断:只为 new 标签不足的国家发现
|
|
642
650
|
const countriesToDiscover = [];
|
|
643
651
|
const { getTagsByStatus } = await import("./db-tags.js");
|
|
644
|
-
for (const country of
|
|
652
|
+
for (const country of getTargetLocations()) {
|
|
645
653
|
const newTags = getTagsByStatus("new", 100, 0, country, null, "llm");
|
|
646
654
|
if (newTags.length < DISCOVER_PER_COUNTRY_THRESHOLD) {
|
|
647
655
|
countriesToDiscover.push(country);
|
|
@@ -1137,7 +1145,7 @@ export function startWatchServer(
|
|
|
1137
1145
|
|
|
1138
1146
|
if (req.method === "GET" && routePath === "/api/stats") {
|
|
1139
1147
|
const stats = computeStatsIncremental(store);
|
|
1140
|
-
stats.targetLocations =
|
|
1148
|
+
stats.targetLocations = getTargetLocations();
|
|
1141
1149
|
stats.clientLoginStatus = store.getClientLoginStatus();
|
|
1142
1150
|
stats.activeClients = store.getActiveClients();
|
|
1143
1151
|
sendJSON(res, 200, stats);
|
|
@@ -1494,7 +1502,7 @@ export function startWatchServer(
|
|
|
1494
1502
|
}
|
|
1495
1503
|
|
|
1496
1504
|
if (req.method === "GET" && routePath === "/api/target-users") {
|
|
1497
|
-
const targetResult = store.getTargetUsers(
|
|
1505
|
+
const targetResult = store.getTargetUsers(getTargetLocations());
|
|
1498
1506
|
const targets = targetResult.users;
|
|
1499
1507
|
if (req.headers["accept"]?.includes("text/csv")) {
|
|
1500
1508
|
const columns = [
|
|
@@ -1549,19 +1557,16 @@ export function startWatchServer(
|
|
|
1549
1557
|
) {
|
|
1550
1558
|
// 摘要模式:只返回各国统计数
|
|
1551
1559
|
if (params.summary === "1") {
|
|
1552
|
-
const result = store.getTargetUsersByCountry(
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
);
|
|
1560
|
+
const result = store.getTargetUsersByCountry(getTargetLocations(), {
|
|
1561
|
+
summaryOnly: true,
|
|
1562
|
+
});
|
|
1556
1563
|
sendJSON(res, 200, result);
|
|
1557
1564
|
return;
|
|
1558
1565
|
}
|
|
1559
1566
|
|
|
1560
1567
|
// CSV 导出:全量数据
|
|
1561
1568
|
if (req.headers["accept"]?.includes("text/csv")) {
|
|
1562
|
-
const result = store.getTargetUsersByCountry(
|
|
1563
|
-
DEFAULT_TARGET_LOCATIONS,
|
|
1564
|
-
);
|
|
1569
|
+
const result = store.getTargetUsersByCountry(getTargetLocations());
|
|
1565
1570
|
const columns = [
|
|
1566
1571
|
"uniqueId",
|
|
1567
1572
|
"nickname",
|
|
@@ -1615,17 +1620,14 @@ export function startWatchServer(
|
|
|
1615
1620
|
|
|
1616
1621
|
// 分页模式:按国家+搜索+分页查询用户
|
|
1617
1622
|
if (params.limit) {
|
|
1618
|
-
const result = store.getTargetUsersByCountry(
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
sortOrder: params.sortOrder || "desc",
|
|
1627
|
-
},
|
|
1628
|
-
);
|
|
1623
|
+
const result = store.getTargetUsersByCountry(getTargetLocations(), {
|
|
1624
|
+
country: params.country || undefined,
|
|
1625
|
+
search: params.search || undefined,
|
|
1626
|
+
limit: parseInt(params.limit, 10),
|
|
1627
|
+
offset: parseInt(params.offset || "0", 10),
|
|
1628
|
+
sort: params.sort || undefined,
|
|
1629
|
+
sortOrder: params.sortOrder || "desc",
|
|
1630
|
+
});
|
|
1629
1631
|
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
1630
1632
|
if (result.users && Array.isArray(result.users)) {
|
|
1631
1633
|
result.users = result.users.map((u) => ({
|
|
@@ -1640,7 +1642,7 @@ export function startWatchServer(
|
|
|
1640
1642
|
}
|
|
1641
1643
|
|
|
1642
1644
|
// 默认:全量(兼容旧调用)
|
|
1643
|
-
const result = store.getTargetUsersByCountry(
|
|
1645
|
+
const result = store.getTargetUsersByCountry(getTargetLocations());
|
|
1644
1646
|
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
1645
1647
|
if (result.countries && Array.isArray(result.countries)) {
|
|
1646
1648
|
for (const country of result.countries) {
|
|
@@ -1662,9 +1664,8 @@ export function startWatchServer(
|
|
|
1662
1664
|
req.method === "GET" &&
|
|
1663
1665
|
routePath === "/api/target-users-other-countries"
|
|
1664
1666
|
) {
|
|
1665
|
-
const result =
|
|
1666
|
-
|
|
1667
|
-
);
|
|
1667
|
+
const result =
|
|
1668
|
+
store.getModifiedTargetUsersOutsideLocations(getTargetLocations());
|
|
1668
1669
|
sendJSON(res, 200, result);
|
|
1669
1670
|
return;
|
|
1670
1671
|
}
|
|
@@ -1842,6 +1843,49 @@ export function startWatchServer(
|
|
|
1842
1843
|
return;
|
|
1843
1844
|
}
|
|
1844
1845
|
|
|
1846
|
+
// ====== 目标国家配置(通用 app-config 落盘) ======
|
|
1847
|
+
|
|
1848
|
+
// GET /api/settings/target-locations — 读取当前目标国家
|
|
1849
|
+
if (
|
|
1850
|
+
req.method === "GET" &&
|
|
1851
|
+
routePath === "/api/settings/target-locations"
|
|
1852
|
+
) {
|
|
1853
|
+
try {
|
|
1854
|
+
const { getTargetLocations } = await import("../lib/app-config.js");
|
|
1855
|
+
sendJSON(res, 200, { locations: getTargetLocations() });
|
|
1856
|
+
} catch (e) {
|
|
1857
|
+
sendJSON(res, 500, { error: e.message });
|
|
1858
|
+
}
|
|
1859
|
+
return;
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
// POST /api/settings/target-locations — 保存目标国家(body: { locations: ["CZ","GR",...] })
|
|
1863
|
+
if (
|
|
1864
|
+
req.method === "POST" &&
|
|
1865
|
+
routePath === "/api/settings/target-locations"
|
|
1866
|
+
) {
|
|
1867
|
+
try {
|
|
1868
|
+
const body = await readBody(req);
|
|
1869
|
+
const { setTargetLocations } = await import("../lib/app-config.js");
|
|
1870
|
+
const saved = setTargetLocations(body.locations);
|
|
1871
|
+
sendJSON(res, 200, { locations: saved });
|
|
1872
|
+
} catch (e) {
|
|
1873
|
+
sendJSON(res, 400, { error: e.message });
|
|
1874
|
+
}
|
|
1875
|
+
return;
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
// GET /api/settings/config — 读取整份通用配置(便于后续扩展其他配置)
|
|
1879
|
+
if (req.method === "GET" && routePath === "/api/settings/config") {
|
|
1880
|
+
try {
|
|
1881
|
+
const { getConfig } = await import("../lib/app-config.js");
|
|
1882
|
+
sendJSON(res, 200, { config: getConfig() });
|
|
1883
|
+
} catch (e) {
|
|
1884
|
+
sendJSON(res, 500, { error: e.message });
|
|
1885
|
+
}
|
|
1886
|
+
return;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1845
1889
|
// GET /api/llm-test — 直接测试 LLM API 连通性
|
|
1846
1890
|
if (req.method === "GET" && routePath === "/api/llm-test") {
|
|
1847
1891
|
try {
|
|
@@ -1915,6 +1959,7 @@ export function startWatchServer(
|
|
|
1915
1959
|
body.username || "",
|
|
1916
1960
|
body.stage || "",
|
|
1917
1961
|
body.errorStack || "",
|
|
1962
|
+
body.hostname || req.headers["x-client-hostname"] || "",
|
|
1918
1963
|
);
|
|
1919
1964
|
sendJSON(res, 200, { ok: true });
|
|
1920
1965
|
} else {
|
|
@@ -1938,7 +1983,7 @@ export function startWatchServer(
|
|
|
1938
1983
|
targetLocation: params.targetLocation,
|
|
1939
1984
|
limit: params.limit,
|
|
1940
1985
|
offset: params.offset,
|
|
1941
|
-
targetLocations:
|
|
1986
|
+
targetLocations: getTargetLocations(),
|
|
1942
1987
|
});
|
|
1943
1988
|
usersCache.set(cacheKey, { result, time: Date.now() });
|
|
1944
1989
|
}
|
|
@@ -258,6 +258,46 @@ function getAllErrorTriggers() {
|
|
|
258
258
|
return cfg.errorTriggers || {};
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
+
// ── 代理切换重启配置管理(per agent,默认不启用:切换后不重启,后端下发开启才重启) ──
|
|
262
|
+
|
|
263
|
+
function getSwitchRestartStored(userId) {
|
|
264
|
+
if (!userId) return null;
|
|
265
|
+
const cfg = loadConfig();
|
|
266
|
+
return (cfg.switchRestarts && cfg.switchRestarts[userId]) || null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function setSwitchRestartStored(userId, config) {
|
|
270
|
+
const cfg = loadConfig();
|
|
271
|
+
if (!cfg.switchRestarts) cfg.switchRestarts = {};
|
|
272
|
+
cfg.switchRestarts[userId] = config;
|
|
273
|
+
saveConfig(cfg);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function getAllSwitchRestarts() {
|
|
277
|
+
const cfg = loadConfig();
|
|
278
|
+
return cfg.switchRestarts || {};
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ── 自动升级开关配置管理(per agent,默认开启:未配置或 enabled=true 均开启,后端显式关闭才停用) ──
|
|
282
|
+
|
|
283
|
+
function getAutoUpgradeStored(userId) {
|
|
284
|
+
if (!userId) return null;
|
|
285
|
+
const cfg = loadConfig();
|
|
286
|
+
return (cfg.autoUpgrades && cfg.autoUpgrades[userId]) || null;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function setAutoUpgradeStored(userId, config) {
|
|
290
|
+
const cfg = loadConfig();
|
|
291
|
+
if (!cfg.autoUpgrades) cfg.autoUpgrades = {};
|
|
292
|
+
cfg.autoUpgrades[userId] = config;
|
|
293
|
+
saveConfig(cfg);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function getAllAutoUpgrades() {
|
|
297
|
+
const cfg = loadConfig();
|
|
298
|
+
return cfg.autoUpgrades || {};
|
|
299
|
+
}
|
|
300
|
+
|
|
261
301
|
// ── 登录端口检测配置管理(per agent,后端配置要检查的 CDP 端口列表) ──
|
|
262
302
|
|
|
263
303
|
function getLoginPortsStored(userId) {
|
|
@@ -438,6 +478,8 @@ async function handleHeartbeat(params, body, res) {
|
|
|
438
478
|
logRequests: pendingLogs.length > 0 ? pendingLogs : undefined,
|
|
439
479
|
clientConfig: getClientConfigStored(userId),
|
|
440
480
|
errorTrigger: getErrorTriggerStored(userId),
|
|
481
|
+
switchRestart: getSwitchRestartStored(userId),
|
|
482
|
+
autoUpgrade: getAutoUpgradeStored(userId),
|
|
441
483
|
loginPorts: getLoginPortsStored(userId),
|
|
442
484
|
serverTime: Date.now(),
|
|
443
485
|
});
|
|
@@ -452,6 +494,8 @@ async function handleGetClients(params, body, res) {
|
|
|
452
494
|
clashCountries: getAllClientCountries(),
|
|
453
495
|
supportedCountries: SUPPORTED_COUNTRIES,
|
|
454
496
|
errorTriggers: getAllErrorTriggers(),
|
|
497
|
+
switchRestarts: getAllSwitchRestarts(),
|
|
498
|
+
autoUpgrades: getAllAutoUpgrades(),
|
|
455
499
|
loginPorts: getAllLoginPorts(),
|
|
456
500
|
configPath: WATCHDOG_CONFIG_PATH,
|
|
457
501
|
});
|
|
@@ -605,6 +649,8 @@ async function handleDeleteClient(params, body, res) {
|
|
|
605
649
|
if (cfg.clientConfigs) delete cfg.clientConfigs[userId];
|
|
606
650
|
if (cfg.clashCountries) delete cfg.clashCountries[userId];
|
|
607
651
|
if (cfg.errorTriggers) delete cfg.errorTriggers[userId];
|
|
652
|
+
if (cfg.switchRestarts) delete cfg.switchRestarts[userId];
|
|
653
|
+
if (cfg.autoUpgrades) delete cfg.autoUpgrades[userId];
|
|
608
654
|
if (cfg.loginPorts) delete cfg.loginPorts[userId];
|
|
609
655
|
saveConfig(cfg);
|
|
610
656
|
lastSentCountries.delete(userId);
|
|
@@ -725,6 +771,52 @@ async function handleSetErrorTrigger(params, body, res) {
|
|
|
725
771
|
sendJSON(res, 200, { ok: true, config: cleaned });
|
|
726
772
|
}
|
|
727
773
|
|
|
774
|
+
/** 配置某 agent 的代理切换重启(代理切换后强制重启全部命令;默认不启用) */
|
|
775
|
+
async function handleSetSwitchRestart(params, body, res) {
|
|
776
|
+
const { userId, config } = body || {};
|
|
777
|
+
if (!userId) {
|
|
778
|
+
sendJSON(res, 400, { error: "缺少 userId" });
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
if (!config || typeof config !== "object") {
|
|
782
|
+
sendJSON(res, 400, { error: "缺少 config 对象" });
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
const cleaned = {
|
|
786
|
+
enabled: !!config.enabled,
|
|
787
|
+
updatedAt: new Date().toISOString(),
|
|
788
|
+
};
|
|
789
|
+
setSwitchRestartStored(userId, cleaned);
|
|
790
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
791
|
+
console.error(
|
|
792
|
+
`[watchdog ${ts}] ${userId} 代理切换重启配置已更新: ${JSON.stringify(cleaned)}`,
|
|
793
|
+
);
|
|
794
|
+
sendJSON(res, 200, { ok: true, config: cleaned });
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/** 配置某 agent 的自动升级开关(默认开启;后端显式关闭才停用自动升级检测) */
|
|
798
|
+
async function handleSetAutoUpgrade(params, body, res) {
|
|
799
|
+
const { userId, config } = body || {};
|
|
800
|
+
if (!userId) {
|
|
801
|
+
sendJSON(res, 400, { error: "缺少 userId" });
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
if (!config || typeof config !== "object") {
|
|
805
|
+
sendJSON(res, 400, { error: "缺少 config 对象" });
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
const cleaned = {
|
|
809
|
+
enabled: !!config.enabled,
|
|
810
|
+
updatedAt: new Date().toISOString(),
|
|
811
|
+
};
|
|
812
|
+
setAutoUpgradeStored(userId, cleaned);
|
|
813
|
+
const ts = new Date().toLocaleTimeString("zh-CN", { hour12: false });
|
|
814
|
+
console.error(
|
|
815
|
+
`[watchdog ${ts}] ${userId} 自动升级配置已更新: ${JSON.stringify(cleaned)}`,
|
|
816
|
+
);
|
|
817
|
+
sendJSON(res, 200, { ok: true, config: cleaned });
|
|
818
|
+
}
|
|
819
|
+
|
|
728
820
|
/** 配置某 agent 要检测登录状态的 CDP 端口列表(逗号分隔或数组) */
|
|
729
821
|
async function handleSetLoginPorts(params, body, res) {
|
|
730
822
|
const { userId, ports } = body || {};
|
|
@@ -833,6 +925,14 @@ export async function tryHandleWatchdogRoute(
|
|
|
833
925
|
await handleSetErrorTrigger(params, body, res);
|
|
834
926
|
return true;
|
|
835
927
|
}
|
|
928
|
+
if (method === "POST" && routePath === "/api/watchdog/set-switch-restart") {
|
|
929
|
+
await handleSetSwitchRestart(params, body, res);
|
|
930
|
+
return true;
|
|
931
|
+
}
|
|
932
|
+
if (method === "POST" && routePath === "/api/watchdog/set-auto-upgrade") {
|
|
933
|
+
await handleSetAutoUpgrade(params, body, res);
|
|
934
|
+
return true;
|
|
935
|
+
}
|
|
836
936
|
if (method === "POST" && routePath === "/api/watchdog/login-ports") {
|
|
837
937
|
await handleSetLoginPorts(params, body, res);
|
|
838
938
|
return true;
|
package/src/watchdog/agent.js
CHANGED
|
@@ -24,9 +24,10 @@ import {
|
|
|
24
24
|
import { writeFileSync, existsSync, readFileSync } from "fs";
|
|
25
25
|
import { spawn } from "child_process";
|
|
26
26
|
import { join, dirname } from "path";
|
|
27
|
-
import { homedir, tmpdir } from "os";
|
|
27
|
+
import { homedir, tmpdir, hostname as osHostname } from "os";
|
|
28
28
|
import { fileURLToPath } from "url";
|
|
29
29
|
import { pushSelfLog, getSelfLogs } from "./self-log-buffer.js";
|
|
30
|
+
import { ClientErrorMonitor } from "./client-error-monitor.js";
|
|
30
31
|
|
|
31
32
|
// ── 看门狗自身日志入环(供后台页面按需拉取,经 report-logs 回传落盘) ──
|
|
32
33
|
// 包装 console.error:本进程所有 [watchdog]/[llm-analyzer]/[upgrade] 等日志
|
|
@@ -111,6 +112,7 @@ const NETWORK_ERROR_PATTERNS = [
|
|
|
111
112
|
];
|
|
112
113
|
const NETWORK_ERROR_RE = new RegExp(NETWORK_ERROR_PATTERNS.join("|"), "i");
|
|
113
114
|
const ERROR_SCAN_INTERVAL_MS = 30_000; // 日志扫描间隔
|
|
115
|
+
const SWITCH_SCAN_INTERVAL_MS = 15_000; // 代理切换感知检查间隔(daemon restartSeq 变化 → 重启全部命令)
|
|
114
116
|
const ERROR_TRIGGER_COUNT = 10; // 连续错误阈值
|
|
115
117
|
const ERROR_MIN_MS = 5 * 60 * 1000; // 最小持续时间(时间只做最小值判断)
|
|
116
118
|
const ERROR_BREAK_MS = 120_000; // 两次错误间隔超过则视为连续中断,重置计数
|
|
@@ -262,6 +264,7 @@ export class WatchdogAgent {
|
|
|
262
264
|
this.thresholdTimer = null;
|
|
263
265
|
this.llmAnalyzerTimer = null;
|
|
264
266
|
this.processManager = new ProcessManager();
|
|
267
|
+
this.clientErrorMonitor = null; // 客户端异常监控(watch 异常列表 → 本机命中强切 + 清理)
|
|
265
268
|
this.consecutiveFailures = 0;
|
|
266
269
|
this.maxFailures = 10;
|
|
267
270
|
this.upgradeChecker = null;
|
|
@@ -287,6 +290,11 @@ export class WatchdogAgent {
|
|
|
287
290
|
this._errorStates.delete(cmdId);
|
|
288
291
|
});
|
|
289
292
|
this._errorTriggerTimer = null;
|
|
293
|
+
this._switchCheckTimer = null; // 代理切换感知定时器
|
|
294
|
+
this._restartSeqBaseline = null; // 已消费的切换序号基线 Map<通道,seq>(首次读取初始化)
|
|
295
|
+
this._switchRestartEnabled = false; // 代理切换重启默认不启用(后端下发开启才生效)
|
|
296
|
+
this._switchRestartConfigKey = null; // 上次配置指纹(幂等打印用)
|
|
297
|
+
this._autoUpgradeEnabled = true; // 自动升级默认开启(后端下发关闭才停用)
|
|
290
298
|
this._errorTriggerEnabled = false; // 默认不启用(后端下发开启才生效)
|
|
291
299
|
this._errorTriggerCount = ERROR_TRIGGER_COUNT;
|
|
292
300
|
this._errorTriggerMinMs = ERROR_MIN_MS;
|
|
@@ -345,6 +353,11 @@ export class WatchdogAgent {
|
|
|
345
353
|
this._checkProcessErrors();
|
|
346
354
|
}, ERROR_SCAN_INTERVAL_MS);
|
|
347
355
|
|
|
356
|
+
// 代理切换感知:daemon 每次实际切换 restartSeq+1,变化即强制重启全部命令
|
|
357
|
+
this._switchCheckTimer = setInterval(() => {
|
|
358
|
+
this._checkProxySwitch();
|
|
359
|
+
}, SWITCH_SCAN_INTERVAL_MS);
|
|
360
|
+
|
|
348
361
|
// 登录检测调度:每 60s 检查一次是否到点(首次 60s 后检测,之后每 loginCheckIntervalMs)
|
|
349
362
|
this.loginCheckTimer = setInterval(() => {
|
|
350
363
|
this._maybeRunLoginCheck();
|
|
@@ -358,6 +371,9 @@ export class WatchdogAgent {
|
|
|
358
371
|
|
|
359
372
|
// 自动升级检测(仅全局安装模式)
|
|
360
373
|
this._startUpgradeChecker();
|
|
374
|
+
|
|
375
|
+
// 客户端异常监控(watch 服务端异常列表 → 本机命中强切 + 清理)
|
|
376
|
+
this._startClientErrorMonitor();
|
|
361
377
|
}
|
|
362
378
|
|
|
363
379
|
/**
|
|
@@ -381,6 +397,10 @@ export class WatchdogAgent {
|
|
|
381
397
|
clearInterval(this._errorTriggerTimer);
|
|
382
398
|
this._errorTriggerTimer = null;
|
|
383
399
|
}
|
|
400
|
+
if (this._switchCheckTimer) {
|
|
401
|
+
clearInterval(this._switchCheckTimer);
|
|
402
|
+
this._switchCheckTimer = null;
|
|
403
|
+
}
|
|
384
404
|
if (this.loginCheckTimer) {
|
|
385
405
|
clearInterval(this.loginCheckTimer);
|
|
386
406
|
this.loginCheckTimer = null;
|
|
@@ -393,6 +413,10 @@ export class WatchdogAgent {
|
|
|
393
413
|
this.upgradeChecker.stop();
|
|
394
414
|
this.upgradeChecker = null;
|
|
395
415
|
}
|
|
416
|
+
if (this.clientErrorMonitor) {
|
|
417
|
+
this.clientErrorMonitor.stop();
|
|
418
|
+
this.clientErrorMonitor = null;
|
|
419
|
+
}
|
|
396
420
|
// 停止 clash daemon(标记主动关闭,exit 时不再自动重启)
|
|
397
421
|
this._daemonStopping = true;
|
|
398
422
|
if (this._daemonRestartTimer) {
|
|
@@ -404,6 +428,80 @@ export class WatchdogAgent {
|
|
|
404
428
|
console.error("[watchdog] 看门狗已停止");
|
|
405
429
|
}
|
|
406
430
|
|
|
431
|
+
/**
|
|
432
|
+
* 代理切换感知:读 daemon health 的 restartSeq,任一通道变大即发生过切换,
|
|
433
|
+
* 强制重启全部被监控命令。默认不启用(后端开关开启才重启);
|
|
434
|
+
* 基线始终跟随 seq(无论开关状态):开启后只响应"新切换",不补触发历史切换。
|
|
435
|
+
* 变小 = daemon 重启序号归零,只更新基线不触发(防 watchdog/daemon 重启误伤)。
|
|
436
|
+
*/
|
|
437
|
+
async _checkProxySwitch() {
|
|
438
|
+
try {
|
|
439
|
+
const health = getClashHealth();
|
|
440
|
+
if (!health) return; // daemon 未就绪,暂不初始化基线
|
|
441
|
+
const seqMap = new Map();
|
|
442
|
+
seqMap.set("main", Number(health.restartSeq) || 0);
|
|
443
|
+
for (const ch of getClashHealthChannels()) {
|
|
444
|
+
seqMap.set(`ch${ch.port}`, Number(ch.restartSeq) || 0);
|
|
445
|
+
}
|
|
446
|
+
if (this._restartSeqBaseline === null) {
|
|
447
|
+
this._restartSeqBaseline = seqMap; // 首次读取建基线(watchdog 重启不误触发)
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
let switched = false;
|
|
451
|
+
for (const [key, seq] of seqMap) {
|
|
452
|
+
if (seq > (this._restartSeqBaseline.get(key) || 0)) {
|
|
453
|
+
switched = true;
|
|
454
|
+
break;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
this._restartSeqBaseline = seqMap;
|
|
458
|
+
if (!switched) return;
|
|
459
|
+
if (!this._switchRestartEnabled) {
|
|
460
|
+
console.error(
|
|
461
|
+
`[watchdog] 检测到代理切换但重启功能未启用(后台可开启),跳过重启`,
|
|
462
|
+
);
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
const reason = health.lastSwitch
|
|
466
|
+
? `代理切换(${health.lastSwitch})`
|
|
467
|
+
: "代理切换";
|
|
468
|
+
console.error(`[watchdog] 检测到${reason},强制重启全部命令...`);
|
|
469
|
+
await this.processManager.restartAll(reason);
|
|
470
|
+
} catch (e) {
|
|
471
|
+
console.error(`[watchdog] 代理切换感知检查失败: ${e.message}`);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** 应用服务端下发的代理切换重启配置(默认不启用,后端下发开启才生效) */
|
|
476
|
+
_applySwitchRestartConfig(cfg) {
|
|
477
|
+
this._switchRestartEnabled = !!cfg?.enabled;
|
|
478
|
+
const key = `${this._switchRestartEnabled}`;
|
|
479
|
+
if (key === this._switchRestartConfigKey) return;
|
|
480
|
+
this._switchRestartConfigKey = key;
|
|
481
|
+
console.error(
|
|
482
|
+
`[watchdog] 代理切换重启: ${this._switchRestartEnabled ? "已启用" : "已禁用"}(切换后${this._switchRestartEnabled ? "" : "不"}强制重启命令)`,
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* 应用服务端下发的自动升级开关
|
|
488
|
+
* 未配置(null/undefined)= 默认开启;后端显式 {enabled:false} 才停用升级检测器
|
|
489
|
+
*/
|
|
490
|
+
_applyAutoUpgradeConfig(cfg) {
|
|
491
|
+
const enabled = cfg ? !!cfg.enabled : true;
|
|
492
|
+
if (enabled === this._autoUpgradeEnabled) return; // 幂等,避免每次心跳重复启停
|
|
493
|
+
this._autoUpgradeEnabled = enabled;
|
|
494
|
+
console.error(
|
|
495
|
+
`[watchdog] 自动升级: ${enabled ? "已开启" : "已关闭"}${cfg ? "" : "(未配置,默认开启)"}`,
|
|
496
|
+
);
|
|
497
|
+
if (enabled) {
|
|
498
|
+
if (!this.upgradeChecker) this._startUpgradeChecker();
|
|
499
|
+
} else if (this.upgradeChecker) {
|
|
500
|
+
this.upgradeChecker.stop();
|
|
501
|
+
this.upgradeChecker = null;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
407
505
|
/** 应用服务端下发的错误触发切换配置(控制启用与参数,默认不启用) */
|
|
408
506
|
_applyErrorTriggerConfig(cfg) {
|
|
409
507
|
this._errorTriggerEnabled = !!cfg?.enabled;
|
|
@@ -511,6 +609,55 @@ export class WatchdogAgent {
|
|
|
511
609
|
}
|
|
512
610
|
}
|
|
513
611
|
|
|
612
|
+
/** 下发强切 failover(force: 无视 daemon 冷却期,直接轮换候选) */
|
|
613
|
+
async _forceFailover(reason) {
|
|
614
|
+
try {
|
|
615
|
+
const ipc = await import("../clash/ipc.js");
|
|
616
|
+
ipc.writeCommand({
|
|
617
|
+
action: "failover",
|
|
618
|
+
force: true,
|
|
619
|
+
reason: reason || "客户端异常强切",
|
|
620
|
+
});
|
|
621
|
+
console.error(
|
|
622
|
+
`[watchdog] 已下发强切命令 (原因: ${reason || "客户端异常"})`,
|
|
623
|
+
);
|
|
624
|
+
} catch (e) {
|
|
625
|
+
console.error(`[watchdog] 下发强切失败: ${e.message}`);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* 启动客户端异常监控:定时拉取 watch 服务端异常列表,
|
|
631
|
+
* 命中本机条目(reportCount > 8 且最近 2 分钟内)→ 强切代理 + 删除全部本机条目。
|
|
632
|
+
*/
|
|
633
|
+
_startClientErrorMonitor() {
|
|
634
|
+
const cfg = getClientConfig();
|
|
635
|
+
const server = cfg.server || "";
|
|
636
|
+
if (!server) {
|
|
637
|
+
console.error(
|
|
638
|
+
"[watchdog] 未配置 watch 服务端地址 (server),客户端异常监控跳过",
|
|
639
|
+
);
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
this.clientErrorMonitor = new ClientErrorMonitor({
|
|
643
|
+
serverUrl: server,
|
|
644
|
+
hostname: osHostname(),
|
|
645
|
+
intervalMs: 60_000,
|
|
646
|
+
onAlert: async (mine) => {
|
|
647
|
+
const ids = mine.map((e) => e.userId).join(",");
|
|
648
|
+
console.error(
|
|
649
|
+
`[watchdog] 客户端异常监控命中 ${mine.length} 条本机异常 (${ids}),强制切换代理并清理`,
|
|
650
|
+
);
|
|
651
|
+
await this._forceFailover("client-errors");
|
|
652
|
+
const removed = await this.clientErrorMonitor.removeErrors(mine);
|
|
653
|
+
console.error(
|
|
654
|
+
`[watchdog] 已清理本机异常条目 ${removed}/${mine.length}`,
|
|
655
|
+
);
|
|
656
|
+
},
|
|
657
|
+
});
|
|
658
|
+
this.clientErrorMonitor.start();
|
|
659
|
+
}
|
|
660
|
+
|
|
514
661
|
/** 构建心跳请求体 */
|
|
515
662
|
_buildHeartbeatBody() {
|
|
516
663
|
const sys = getSystemInfo();
|
|
@@ -609,6 +756,14 @@ export class WatchdogAgent {
|
|
|
609
756
|
if (data.errorTrigger) {
|
|
610
757
|
this._applyErrorTriggerConfig(data.errorTrigger);
|
|
611
758
|
}
|
|
759
|
+
// 服务端下发的代理切换重启配置(默认不启用,后端下发开启才生效)
|
|
760
|
+
if (data.switchRestart) {
|
|
761
|
+
this._applySwitchRestartConfig(data.switchRestart);
|
|
762
|
+
}
|
|
763
|
+
// 服务端下发的自动升级开关(未配置默认开启;后端显式关闭才停用)
|
|
764
|
+
if (data.autoUpgrade !== undefined) {
|
|
765
|
+
this._applyAutoUpgradeConfig(data.autoUpgrade);
|
|
766
|
+
}
|
|
612
767
|
// 服务端下发的登录检测端口列表
|
|
613
768
|
if (Array.isArray(data.loginPorts)) {
|
|
614
769
|
const key = data.loginPorts.join(",");
|
|
@@ -873,6 +1028,7 @@ export class WatchdogAgent {
|
|
|
873
1028
|
*/
|
|
874
1029
|
_startUpgradeChecker() {
|
|
875
1030
|
if (!isGlobalInstall()) return;
|
|
1031
|
+
if (!this._autoUpgradeEnabled) return;
|
|
876
1032
|
|
|
877
1033
|
this.upgradeChecker = startUpgradeChecker({
|
|
878
1034
|
onUpdateAvailable: async ({ current, latest }) => {
|