tt-help-cli-ycl 1.4.85 → 1.4.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -2
- package/src/clash/checker.js +53 -6
- package/src/clash/daemon.js +123 -17
- package/src/clash/ipc.js +7 -4
- package/src/cli/explore.js +69 -8
- package/src/cli/refresh.js +78 -10
- package/src/cli/search.js +69 -8
- package/src/watch/public/app.js +242 -5
- package/src/watch/public/index.html +56 -1
- package/src/watch/watchdog-server.js +157 -1
- package/src/watchdog/agent.js +286 -17
- package/src/watchdog/process-manager.js +23 -1
- package/src/watchdog/upgrader.js +39 -2
- package/src/webserver/server.mjs +32 -0
package/src/watchdog/agent.js
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
isGlobalInstall,
|
|
21
21
|
startUpgradeChecker,
|
|
22
22
|
} from "./upgrader.js";
|
|
23
|
-
import { writeFileSync, existsSync, readFileSync } from "fs";
|
|
23
|
+
import { writeFileSync, existsSync, readFileSync, openSync } from "fs";
|
|
24
24
|
import { spawn } from "child_process";
|
|
25
25
|
import { join, dirname } from "path";
|
|
26
26
|
import { homedir, tmpdir } from "os";
|
|
@@ -30,6 +30,47 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
|
30
30
|
const homeDir = homedir() || process.env.HOME || process.env.USERPROFILE || "";
|
|
31
31
|
const TT_HELP_CONFIG = join(homeDir, ".tt-help.json");
|
|
32
32
|
|
|
33
|
+
// ── 业务日志错误统计 → 触发节点切换 ──
|
|
34
|
+
// 识别采集程序日志里已有的错误分类标签(explore 等已打标),连续错误达标则切换出口节点
|
|
35
|
+
const NETWORK_ERROR_PATTERNS = [
|
|
36
|
+
"CDN限流",
|
|
37
|
+
"网络异常",
|
|
38
|
+
"net::",
|
|
39
|
+
"ERR_CONNECTION",
|
|
40
|
+
"ECONN",
|
|
41
|
+
"ETIMEDOUT",
|
|
42
|
+
"ENOTFOUND",
|
|
43
|
+
"EAI_AGAIN",
|
|
44
|
+
"ESOCKETRESET",
|
|
45
|
+
"connection.*refused",
|
|
46
|
+
"connection.*reset",
|
|
47
|
+
"Timeout",
|
|
48
|
+
"被封:",
|
|
49
|
+
];
|
|
50
|
+
const NETWORK_ERROR_RE = new RegExp(NETWORK_ERROR_PATTERNS.join("|"), "i");
|
|
51
|
+
const ERROR_SCAN_INTERVAL_MS = 30_000; // 日志扫描间隔
|
|
52
|
+
const ERROR_TRIGGER_COUNT = 10; // 连续错误阈值
|
|
53
|
+
const ERROR_MIN_MS = 5 * 60 * 1000; // 最小持续时间(时间只做最小值判断)
|
|
54
|
+
const ERROR_BREAK_MS = 120_000; // 两次错误间隔超过则视为连续中断,重置计数
|
|
55
|
+
// 只统计采集类命令,排除 webserver/watchdog 自身
|
|
56
|
+
const COLLECTOR_KEYWORDS = [
|
|
57
|
+
"explore",
|
|
58
|
+
"auto",
|
|
59
|
+
"refresh",
|
|
60
|
+
"search",
|
|
61
|
+
"precheck",
|
|
62
|
+
"recheck",
|
|
63
|
+
"attach",
|
|
64
|
+
"scrape",
|
|
65
|
+
"videos",
|
|
66
|
+
"comments",
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
function isCollectorCommand(cmdDef) {
|
|
70
|
+
const s = `${cmdDef?.command || ""} ${cmdDef?.name || ""}`.toLowerCase();
|
|
71
|
+
return COLLECTOR_KEYWORDS.some((k) => s.includes(k));
|
|
72
|
+
}
|
|
73
|
+
|
|
33
74
|
/** 自动发现 daemon.js 路径 */
|
|
34
75
|
function findDaemonPath() {
|
|
35
76
|
// 1. 内部打包路径优先(npm publish 后 src/clash/daemon.js,升级即最新)
|
|
@@ -140,9 +181,20 @@ export class WatchdogAgent {
|
|
|
140
181
|
this._lastCmdCount = -1; // 上次命令数
|
|
141
182
|
// clash daemon 集成
|
|
142
183
|
this.clashDaemon = null; // daemon 子进程引用
|
|
184
|
+
this._daemonStopping = false; // 主动关闭标记(停止/升级时不再重启)
|
|
185
|
+
this._daemonRestartCount = 0; // 连续异常退出次数
|
|
186
|
+
this._daemonRestartBackoffMs = 3000; // 重启退避(3s 起,指数递增,上限 30s)
|
|
187
|
+
this._daemonRestartTimer = null; // 重启定时器
|
|
143
188
|
this.gracePeriodMs = 120_000; // 启动宽限期 2 分钟
|
|
144
189
|
this.startTime = 0; // 启动时间戳
|
|
145
190
|
this._graceEnded = false; // 宽限期是否已结束
|
|
191
|
+
// 业务日志错误统计 → 触发节点切换
|
|
192
|
+
this._errorStates = new Map(); // cmdId -> {consecutive, firstErrorAt, lastErrorAt}
|
|
193
|
+
this._errorTriggerTimer = null;
|
|
194
|
+
this._errorTriggerEnabled = false; // 默认不启用(后端下发开启才生效)
|
|
195
|
+
this._errorTriggerCount = ERROR_TRIGGER_COUNT;
|
|
196
|
+
this._errorTriggerMinMs = ERROR_MIN_MS;
|
|
197
|
+
this._errorTriggerBreakMs = ERROR_BREAK_MS;
|
|
146
198
|
}
|
|
147
199
|
|
|
148
200
|
/**
|
|
@@ -164,6 +216,13 @@ export class WatchdogAgent {
|
|
|
164
216
|
this._setupSignalHandlers();
|
|
165
217
|
|
|
166
218
|
// 启动 clash daemon(保代理通道,--no-daemon 可关闭)
|
|
219
|
+
this._daemonStopping = false;
|
|
220
|
+
if (this._daemonRestartTimer) {
|
|
221
|
+
clearTimeout(this._daemonRestartTimer);
|
|
222
|
+
this._daemonRestartTimer = null;
|
|
223
|
+
}
|
|
224
|
+
this._daemonRestartCount = 0;
|
|
225
|
+
this._daemonRestartBackoffMs = 3000;
|
|
167
226
|
if (this.spawnDaemon) this._spawnClashDaemon();
|
|
168
227
|
|
|
169
228
|
// 首次立即发送心跳
|
|
@@ -179,11 +238,16 @@ export class WatchdogAgent {
|
|
|
179
238
|
this.processManager.checkThresholds();
|
|
180
239
|
}, 5000);
|
|
181
240
|
|
|
182
|
-
// 假死检测:每 60s 检查各进程是否有日志输出,超
|
|
241
|
+
// 假死检测:每 60s 检查各进程是否有日志输出,超 15 分钟无日志则重启
|
|
183
242
|
this.staleTimer = setInterval(() => {
|
|
184
243
|
this.processManager.checkStaleProcesses();
|
|
185
244
|
}, 60000);
|
|
186
245
|
|
|
246
|
+
// 业务日志错误统计:连续错误达标则触发节点切换(每 30s)
|
|
247
|
+
this._errorTriggerTimer = setInterval(() => {
|
|
248
|
+
this._checkProcessErrors();
|
|
249
|
+
}, ERROR_SCAN_INTERVAL_MS);
|
|
250
|
+
|
|
187
251
|
// LLM 日志分析(每 10 分钟调服务端接口)
|
|
188
252
|
this._startLlmAnalyzer();
|
|
189
253
|
|
|
@@ -208,6 +272,10 @@ export class WatchdogAgent {
|
|
|
208
272
|
clearInterval(this.staleTimer);
|
|
209
273
|
this.staleTimer = null;
|
|
210
274
|
}
|
|
275
|
+
if (this._errorTriggerTimer) {
|
|
276
|
+
clearInterval(this._errorTriggerTimer);
|
|
277
|
+
this._errorTriggerTimer = null;
|
|
278
|
+
}
|
|
211
279
|
if (this.llmAnalyzerTimer) {
|
|
212
280
|
clearInterval(this.llmAnalyzerTimer);
|
|
213
281
|
this.llmAnalyzerTimer = null;
|
|
@@ -216,12 +284,96 @@ export class WatchdogAgent {
|
|
|
216
284
|
this.upgradeChecker.stop();
|
|
217
285
|
this.upgradeChecker = null;
|
|
218
286
|
}
|
|
219
|
-
// 停止 clash daemon
|
|
287
|
+
// 停止 clash daemon(标记主动关闭,exit 时不再自动重启)
|
|
288
|
+
this._daemonStopping = true;
|
|
289
|
+
if (this._daemonRestartTimer) {
|
|
290
|
+
clearTimeout(this._daemonRestartTimer);
|
|
291
|
+
this._daemonRestartTimer = null;
|
|
292
|
+
}
|
|
220
293
|
this._killClashDaemon();
|
|
221
294
|
this.processManager.stopAll();
|
|
222
295
|
console.error("[watchdog] 看门狗已停止");
|
|
223
296
|
}
|
|
224
297
|
|
|
298
|
+
/** 应用服务端下发的错误触发切换配置(控制启用与参数,默认不启用) */
|
|
299
|
+
_applyErrorTriggerConfig(cfg) {
|
|
300
|
+
this._errorTriggerEnabled = !!cfg?.enabled;
|
|
301
|
+
if (cfg?.triggerCount > 0) this._errorTriggerCount = cfg.triggerCount;
|
|
302
|
+
if (cfg?.minMinutes > 0)
|
|
303
|
+
this._errorTriggerMinMs = cfg.minMinutes * 60 * 1000;
|
|
304
|
+
if (cfg?.breakSeconds > 0)
|
|
305
|
+
this._errorTriggerBreakMs = cfg.breakSeconds * 1000;
|
|
306
|
+
console.error(
|
|
307
|
+
`[watchdog] 错误触发切换: ${this._errorTriggerEnabled ? "已启用" : "已禁用"} (连续${this._errorTriggerCount}次 / 持续${this._errorTriggerMinMs / 60000}min / 中断${this._errorTriggerBreakMs / 1000}s)`,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* 扫描各采集进程日志,统计连续网络/限流/封禁类错误,达标则触发 daemon 故障切换。
|
|
313
|
+
* 触发条件:连续错误 ≥ _errorTriggerCount 且 持续时间 ≥ _errorTriggerMinMs(时间只做最小值判断)
|
|
314
|
+
*/
|
|
315
|
+
_checkProcessErrors() {
|
|
316
|
+
if (!this.running || !this._errorTriggerEnabled) return;
|
|
317
|
+
const now = Date.now();
|
|
318
|
+
const scanned = this.processManager.scanErrorLogs(NETWORK_ERROR_RE);
|
|
319
|
+
for (const { cmdId, cmdDef, newErrors } of scanned) {
|
|
320
|
+
if (!isCollectorCommand(cmdDef)) continue;
|
|
321
|
+
let st = this._errorStates.get(cmdId) || {
|
|
322
|
+
consecutive: 0,
|
|
323
|
+
firstErrorAt: 0,
|
|
324
|
+
lastErrorAt: 0,
|
|
325
|
+
};
|
|
326
|
+
for (const line of newErrors) {
|
|
327
|
+
// 错误间隔超阈值 → 连续中断,重新计数
|
|
328
|
+
if (
|
|
329
|
+
st.lastErrorAt &&
|
|
330
|
+
now - st.lastErrorAt > this._errorTriggerBreakMs
|
|
331
|
+
) {
|
|
332
|
+
st.consecutive = 0;
|
|
333
|
+
st.firstErrorAt = now;
|
|
334
|
+
}
|
|
335
|
+
st.consecutive++;
|
|
336
|
+
if (!st.firstErrorAt) st.firstErrorAt = now;
|
|
337
|
+
st.lastErrorAt = now;
|
|
338
|
+
// 连续错误达标 且 至少持续了最小时间 → 触发切换
|
|
339
|
+
if (
|
|
340
|
+
st.consecutive >= this._errorTriggerCount &&
|
|
341
|
+
now - st.firstErrorAt >= this._errorTriggerMinMs
|
|
342
|
+
) {
|
|
343
|
+
console.error(
|
|
344
|
+
`[watchdog] ${cmdId} 连续 ${st.consecutive} 次错误(≥${this._errorTriggerCount}次,持续≥${this._errorTriggerMinMs / 60000}min),触发节点切换`,
|
|
345
|
+
);
|
|
346
|
+
this._triggerFailover(cmdId);
|
|
347
|
+
// 切换后重新计数
|
|
348
|
+
st = { consecutive: 0, firstErrorAt: 0, lastErrorAt: 0 };
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
this._errorStates.set(cmdId, st);
|
|
352
|
+
}
|
|
353
|
+
// 无新错误的进程:长时间无错误则重置连续计数
|
|
354
|
+
for (const [cmdId, st] of this._errorStates) {
|
|
355
|
+
if (
|
|
356
|
+
st.consecutive > 0 &&
|
|
357
|
+
now - st.lastErrorAt > this._errorTriggerBreakMs
|
|
358
|
+
) {
|
|
359
|
+
st.consecutive = 0;
|
|
360
|
+
st.firstErrorAt = 0;
|
|
361
|
+
this._errorStates.set(cmdId, st);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** 下发 failover 命令给 daemon(候选集切下一个可用节点) */
|
|
367
|
+
async _triggerFailover(cmdId) {
|
|
368
|
+
try {
|
|
369
|
+
const ipc = await import("../clash/ipc.js");
|
|
370
|
+
ipc.writeCommand({ action: "failover" });
|
|
371
|
+
console.error(`[watchdog] 已下发 failover 命令 (触发源: ${cmdId})`);
|
|
372
|
+
} catch (e) {
|
|
373
|
+
console.error(`[watchdog] 下发 failover 失败: ${e.message}`);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
225
377
|
/** 构建心跳请求体 */
|
|
226
378
|
_buildHeartbeatBody() {
|
|
227
379
|
const sys = getSystemInfo();
|
|
@@ -288,19 +440,23 @@ export class WatchdogAgent {
|
|
|
288
440
|
}
|
|
289
441
|
}
|
|
290
442
|
|
|
291
|
-
// 处理 Clash daemon
|
|
443
|
+
// 处理 Clash daemon 命令(一次写入数组,daemon 轮询整批处理)
|
|
292
444
|
if (Array.isArray(data.clashCommands) && data.clashCommands.length > 0) {
|
|
293
445
|
const ipc = await import("../clash/ipc.js");
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
446
|
+
ipc.writeCommand(data.clashCommands);
|
|
447
|
+
console.error(
|
|
448
|
+
`[watchdog] 下发 Clash 命令: ${JSON.stringify(data.clashCommands)}`,
|
|
449
|
+
);
|
|
298
450
|
}
|
|
299
451
|
|
|
300
452
|
// 服务端下发的客户端配置(远程修改 ~/.tt-help.json)
|
|
301
453
|
if (data.clientConfig) {
|
|
302
454
|
applyClientConfig(data.clientConfig);
|
|
303
455
|
}
|
|
456
|
+
// 服务端下发的错误触发切换配置(默认不启用,后端下发控制)
|
|
457
|
+
if (data.errorTrigger) {
|
|
458
|
+
this._applyErrorTriggerConfig(data.errorTrigger);
|
|
459
|
+
}
|
|
304
460
|
|
|
305
461
|
// 心跳日志:仅状态变化或每 10 次打印一次
|
|
306
462
|
this._heartbeatSeq++;
|
|
@@ -398,7 +554,8 @@ export class WatchdogAgent {
|
|
|
398
554
|
this.upgradeChecker = startUpgradeChecker({
|
|
399
555
|
onUpdateAvailable: async ({ current, latest }) => {
|
|
400
556
|
console.error(`[upgrade] ${current} → ${latest}`);
|
|
401
|
-
//
|
|
557
|
+
// 升级流程:先标记 daemon 主动关闭(避免 kill 后被崩溃自愈重新拉起旧版),再停业务与 daemon
|
|
558
|
+
this._daemonStopping = true;
|
|
402
559
|
this.processManager.stopAll();
|
|
403
560
|
this._killClashDaemon();
|
|
404
561
|
|
|
@@ -412,7 +569,11 @@ export class WatchdogAgent {
|
|
|
412
569
|
this._restartSelf();
|
|
413
570
|
return;
|
|
414
571
|
}
|
|
415
|
-
|
|
572
|
+
// 升级失败:恢复 daemon 与业务,保持当前进程运行
|
|
573
|
+
console.error(
|
|
574
|
+
`[upgrade] 失败: ${result.output.substring(0, 200)},恢复运行`,
|
|
575
|
+
);
|
|
576
|
+
this._recoverAfterFailedUpgrade();
|
|
416
577
|
await this._sendHeartbeat();
|
|
417
578
|
},
|
|
418
579
|
});
|
|
@@ -456,6 +617,10 @@ export class WatchdogAgent {
|
|
|
456
617
|
`[watchdog] clash daemon 退出 (code=${code}, signal=${signal})`,
|
|
457
618
|
);
|
|
458
619
|
this.clashDaemon = null;
|
|
620
|
+
// 主动关闭(watchdog 停止/升级自重启)或已停止时不再自动重启
|
|
621
|
+
if (this._daemonStopping || !this.running || !this.spawnDaemon) return;
|
|
622
|
+
// 异常退出 → 退避自动重启,防止崩溃循环
|
|
623
|
+
this._scheduleDaemonRestart(code, signal);
|
|
459
624
|
});
|
|
460
625
|
|
|
461
626
|
child.on("error", (err) => {
|
|
@@ -463,12 +628,50 @@ export class WatchdogAgent {
|
|
|
463
628
|
this.clashDaemon = null;
|
|
464
629
|
});
|
|
465
630
|
|
|
631
|
+
// 运行稳定(60s 未退出)后重置崩溃计数与退避
|
|
632
|
+
setTimeout(() => {
|
|
633
|
+
if (this.clashDaemon === child && child.exitCode === null) {
|
|
634
|
+
this._daemonRestartCount = 0;
|
|
635
|
+
this._daemonRestartBackoffMs = 3000;
|
|
636
|
+
}
|
|
637
|
+
}, 60_000);
|
|
638
|
+
|
|
466
639
|
this.clashDaemon = child;
|
|
467
640
|
} catch (e) {
|
|
468
641
|
console.error(`[watchdog] clash daemon 启动异常: ${e.message}`);
|
|
469
642
|
}
|
|
470
643
|
}
|
|
471
644
|
|
|
645
|
+
/**
|
|
646
|
+
* daemon 异常退出后的退避自动重启(防崩溃循环)
|
|
647
|
+
* 连续异常退出超过阈值则停止自动重启,避免无限循环
|
|
648
|
+
*/
|
|
649
|
+
_scheduleDaemonRestart(code, signal) {
|
|
650
|
+
this._daemonRestartCount++;
|
|
651
|
+
if (this._daemonRestartCount > 5) {
|
|
652
|
+
console.error(
|
|
653
|
+
`[watchdog] clash daemon 连续异常退出 ${this._daemonRestartCount - 1} 次,停止自动重启(请人工排查)`,
|
|
654
|
+
);
|
|
655
|
+
this._daemonRestartCount = 0;
|
|
656
|
+
this._daemonRestartBackoffMs = 3000;
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
const delay = this._daemonRestartBackoffMs;
|
|
660
|
+
this._daemonRestartBackoffMs = Math.min(
|
|
661
|
+
this._daemonRestartBackoffMs * 2,
|
|
662
|
+
30_000,
|
|
663
|
+
);
|
|
664
|
+
console.error(
|
|
665
|
+
`[watchdog] clash daemon 异常退出,${(delay / 1000).toFixed(1)}s 后自动重启 (第 ${this._daemonRestartCount} 次, code=${code}, signal=${signal})`,
|
|
666
|
+
);
|
|
667
|
+
if (this._daemonRestartTimer) clearTimeout(this._daemonRestartTimer);
|
|
668
|
+
this._daemonRestartTimer = setTimeout(() => {
|
|
669
|
+
this._daemonRestartTimer = null;
|
|
670
|
+
if (!this.running || !this.spawnDaemon || this._daemonStopping) return;
|
|
671
|
+
this._spawnClashDaemon();
|
|
672
|
+
}, delay);
|
|
673
|
+
}
|
|
674
|
+
|
|
472
675
|
/**
|
|
473
676
|
* 停止 clash daemon 子进程
|
|
474
677
|
*/
|
|
@@ -491,7 +694,8 @@ export class WatchdogAgent {
|
|
|
491
694
|
|
|
492
695
|
/**
|
|
493
696
|
* 升级后重启自身进程(加载新代码)
|
|
494
|
-
* 用 cli.js watchdog 子进程替换当前进程,保证新版本内部 daemon
|
|
697
|
+
* 用 cli.js watchdog 子进程替换当前进程,保证新版本内部 daemon 路径逻辑生效。
|
|
698
|
+
* 带存活确认与兜底:新进程确认存活后才退出旧进程;启动失败则回退保持旧进程运行。
|
|
495
699
|
*/
|
|
496
700
|
_restartSelf() {
|
|
497
701
|
const cliPath = join(__dirname, "..", "..", "cli.js");
|
|
@@ -503,23 +707,88 @@ export class WatchdogAgent {
|
|
|
503
707
|
"--interval",
|
|
504
708
|
String(Math.max(5, Math.round(this.intervalMs / 1000))),
|
|
505
709
|
);
|
|
710
|
+
|
|
711
|
+
// 新进程日志重定向到文件,启动崩溃时有据可查
|
|
712
|
+
let logFd = undefined;
|
|
713
|
+
try {
|
|
714
|
+
const logPath = join(
|
|
715
|
+
tmpdir(),
|
|
716
|
+
`tt-help-watchdog-restart-${Date.now()}.log`,
|
|
717
|
+
);
|
|
718
|
+
logFd = openSync(logPath, "a");
|
|
719
|
+
console.error(`[upgrade] 新进程日志: ${logPath}`);
|
|
720
|
+
} catch {}
|
|
721
|
+
const out = logFd ?? "ignore";
|
|
722
|
+
|
|
506
723
|
let child;
|
|
507
724
|
try {
|
|
508
725
|
child = spawn(process.execPath, [cliPath, ...args], {
|
|
509
726
|
detached: true,
|
|
510
|
-
stdio: "ignore",
|
|
727
|
+
stdio: ["ignore", out, out],
|
|
511
728
|
windowsHide: true, // Windows 隐藏控制台窗口
|
|
512
729
|
});
|
|
513
|
-
child.unref();
|
|
514
730
|
} catch (e) {
|
|
515
|
-
console.error(`[upgrade] 重启自身失败: ${e.message}
|
|
731
|
+
console.error(`[upgrade] 重启自身失败: ${e.message},保持旧进程运行`);
|
|
732
|
+
this._recoverAfterFailedUpgrade();
|
|
516
733
|
return;
|
|
517
734
|
}
|
|
735
|
+
// 新进程独立存活(父进程退出不影响)
|
|
736
|
+
child.unref();
|
|
737
|
+
|
|
738
|
+
// 存活确认:轮询新进程是否正常存活,确认后才退出旧进程
|
|
739
|
+
let exited = false;
|
|
740
|
+
let exitCode = null;
|
|
741
|
+
let exitSignal = null;
|
|
742
|
+
child.once("exit", (c, s) => {
|
|
743
|
+
exited = true;
|
|
744
|
+
exitCode = c;
|
|
745
|
+
exitSignal = s;
|
|
746
|
+
});
|
|
747
|
+
|
|
518
748
|
console.error(
|
|
519
|
-
`[upgrade] 新版 watchdog 已启动 (pid=${child.pid})
|
|
749
|
+
`[upgrade] 新版 watchdog 已启动 (pid=${child.pid}),等待存活确认...`,
|
|
520
750
|
);
|
|
521
|
-
|
|
522
|
-
|
|
751
|
+
let attempts = 0;
|
|
752
|
+
const confirm = () => {
|
|
753
|
+
attempts++;
|
|
754
|
+
if (exited) {
|
|
755
|
+
console.error(
|
|
756
|
+
`[upgrade] 新版 watchdog 启动后即退出 (code=${exitCode}, signal=${exitSignal}),回退保持旧进程`,
|
|
757
|
+
);
|
|
758
|
+
this._recoverAfterFailedUpgrade();
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
try {
|
|
762
|
+
process.kill(child.pid, 0); // 抛错说明进程已不存在
|
|
763
|
+
} catch {
|
|
764
|
+
console.error(
|
|
765
|
+
`[upgrade] 新版 watchdog 进程不存在 (pid=${child.pid}),回退保持旧进程`,
|
|
766
|
+
);
|
|
767
|
+
this._recoverAfterFailedUpgrade();
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
if (attempts >= 6) {
|
|
771
|
+
console.error(`[upgrade] 新版 watchdog 存活确认,旧进程退出`);
|
|
772
|
+
this.stop();
|
|
773
|
+
setTimeout(() => process.exit(0), 500);
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
setTimeout(confirm, 1000);
|
|
777
|
+
};
|
|
778
|
+
setTimeout(confirm, 1000);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* 升级/重启失败后的回退恢复:重新拉起 daemon 并同步业务进程
|
|
783
|
+
*/
|
|
784
|
+
_recoverAfterFailedUpgrade() {
|
|
785
|
+
this._daemonStopping = false;
|
|
786
|
+
if (this.spawnDaemon && !this.clashDaemon) {
|
|
787
|
+
console.error(`[upgrade] 恢复 clash daemon...`);
|
|
788
|
+
this._spawnClashDaemon();
|
|
789
|
+
}
|
|
790
|
+
console.error(`[upgrade] 恢复业务进程(等待心跳同步命令)...`);
|
|
791
|
+
this._sendHeartbeat().catch(() => {});
|
|
523
792
|
}
|
|
524
793
|
|
|
525
794
|
/**
|
|
@@ -22,7 +22,7 @@ const MAX_RESTART = 3;
|
|
|
22
22
|
const MAX_LOG_LINES = 500; // 每个进程最多保留的日志行数
|
|
23
23
|
const IS_WIN = process.platform === "win32";
|
|
24
24
|
const SPAWN_STAGGER_MS = 30000; // 批量启动时每个进程间隔 30 秒,避免抢网络
|
|
25
|
-
const STALE_LOG_TIMEOUT_MS =
|
|
25
|
+
const STALE_LOG_TIMEOUT_MS = 15 * 60 * 1000; // 15 分钟无日志输出视为假死
|
|
26
26
|
|
|
27
27
|
// 从命令行中提取 CDP 端口列表(解析 --base-port 和 --port-count)
|
|
28
28
|
function extractCdpPorts(cmdLine) {
|
|
@@ -377,6 +377,28 @@ export class ProcessManager extends EventEmitter {
|
|
|
377
377
|
return result;
|
|
378
378
|
}
|
|
379
379
|
|
|
380
|
+
/**
|
|
381
|
+
* 增量扫描各进程日志中的错误行(只读,不清空日志,与 LLM 分析互不影响)
|
|
382
|
+
* 用 _errSeen 偏移记录每个进程已扫描过的日志行数,只返回新增的错误行
|
|
383
|
+
* @param {RegExp} pattern - 错误匹配正则
|
|
384
|
+
* @returns {Array<{cmdId:string, cmdDef:object, newErrors:string[]}>} 本轮新增的错误日志
|
|
385
|
+
*/
|
|
386
|
+
scanErrorLogs(pattern) {
|
|
387
|
+
const result = [];
|
|
388
|
+
for (const [cmdId, info] of this.processes) {
|
|
389
|
+
if (info.stopped || !info.logs || info.logs.length === 0) continue;
|
|
390
|
+
// 日志可能被 LLM 分析清空过,偏移需收敛到当前长度
|
|
391
|
+
const seen = Math.min(info._errSeen || 0, info.logs.length);
|
|
392
|
+
const newLines = info.logs.slice(seen);
|
|
393
|
+
info._errSeen = info.logs.length;
|
|
394
|
+
if (!newLines.length) continue;
|
|
395
|
+
const hits = newLines.filter((l) => pattern.test(l));
|
|
396
|
+
if (hits.length)
|
|
397
|
+
result.push({ cmdId, cmdDef: info.cmdDef, newErrors: hits });
|
|
398
|
+
}
|
|
399
|
+
return result;
|
|
400
|
+
}
|
|
401
|
+
|
|
380
402
|
/**
|
|
381
403
|
* 获取所有进程的日志快照(供 LLM 分析)
|
|
382
404
|
* @returns {Array<{cmdId:string, logs:string[], uptimeSeconds:number, restartCount:number}>}
|
package/src/watchdog/upgrader.js
CHANGED
|
@@ -9,11 +9,45 @@ import { exec } from "child_process";
|
|
|
9
9
|
import { readFileSync } from "fs";
|
|
10
10
|
import { fileURLToPath } from "url";
|
|
11
11
|
import { dirname, join } from "path";
|
|
12
|
+
import { homedir } from "os";
|
|
12
13
|
|
|
13
14
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
14
15
|
const PKG_PATH = join(__dirname, "..", "..", "package.json");
|
|
15
16
|
const PACKAGE = "tt-help-cli-ycl";
|
|
16
17
|
const REGISTRY = "https://registry.npmjs.org";
|
|
18
|
+
const homeDir = homedir() || process.env.HOME || process.env.USERPROFILE || "";
|
|
19
|
+
const TT_HELP_CONFIG = join(homeDir, ".tt-help.json");
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* 升级检测源:TT_HELP_REGISTRY 环境变量 > ~/.tt-help.json 的 npmRegistry > 官方 npmjs.org
|
|
23
|
+
* 注意:检测源应与本机 npm install 使用的 registry 保持一致(默认 npm 自身配置)
|
|
24
|
+
*/
|
|
25
|
+
function getRegistry() {
|
|
26
|
+
const trim = (s) => String(s).replace(/\/+$/, "");
|
|
27
|
+
if (process.env.TT_HELP_REGISTRY) return trim(process.env.TT_HELP_REGISTRY);
|
|
28
|
+
try {
|
|
29
|
+
const cfg = JSON.parse(readFileSync(TT_HELP_CONFIG, "utf-8"));
|
|
30
|
+
if (cfg.npmRegistry) return trim(cfg.npmRegistry);
|
|
31
|
+
} catch {}
|
|
32
|
+
return REGISTRY;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** 语义化版本比较:a<b → -1, a>b → 1, a==b → 0(避免字符串比较漏掉 1.4.9→1.4.10 这类升级) */
|
|
36
|
+
export function compareVersions(a, b) {
|
|
37
|
+
const pa = String(a)
|
|
38
|
+
.split(".")
|
|
39
|
+
.map((n) => parseInt(n, 10) || 0);
|
|
40
|
+
const pb = String(b)
|
|
41
|
+
.split(".")
|
|
42
|
+
.map((n) => parseInt(n, 10) || 0);
|
|
43
|
+
const len = Math.max(pa.length, pb.length);
|
|
44
|
+
for (let i = 0; i < len; i++) {
|
|
45
|
+
const x = pa[i] || 0;
|
|
46
|
+
const y = pb[i] || 0;
|
|
47
|
+
if (x !== y) return x < y ? -1 : 1;
|
|
48
|
+
}
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
17
51
|
|
|
18
52
|
/** 从 package.json 读取当前版本 */
|
|
19
53
|
export function getVersion() {
|
|
@@ -38,7 +72,7 @@ export function isGlobalInstall() {
|
|
|
38
72
|
export async function checkForUpdates() {
|
|
39
73
|
const current = getVersion();
|
|
40
74
|
try {
|
|
41
|
-
const resp = await fetch(`${
|
|
75
|
+
const resp = await fetch(`${getRegistry()}/${PACKAGE}/latest`, {
|
|
42
76
|
signal: AbortSignal.timeout(15_000),
|
|
43
77
|
});
|
|
44
78
|
if (!resp.ok)
|
|
@@ -52,7 +86,10 @@ export async function checkForUpdates() {
|
|
|
52
86
|
return {
|
|
53
87
|
current,
|
|
54
88
|
latest,
|
|
55
|
-
hasUpdate:
|
|
89
|
+
hasUpdate:
|
|
90
|
+
latest !== "?" &&
|
|
91
|
+
current !== "unknown" &&
|
|
92
|
+
compareVersions(latest, current) > 0,
|
|
56
93
|
};
|
|
57
94
|
} catch (e) {
|
|
58
95
|
return { current, latest: "?", hasUpdate: false, error: e.message };
|
package/src/webserver/server.mjs
CHANGED
|
@@ -88,6 +88,7 @@ let consecutiveErrors = 0;
|
|
|
88
88
|
const MAX_CONSECUTIVE_ERRORS = 10;
|
|
89
89
|
let warmTimer = null;
|
|
90
90
|
let lastWarmTime = 0;
|
|
91
|
+
let healthTimer = null; // 每 10 分钟健康心跳日志(始终输出,防假死检测误判重启)
|
|
91
92
|
// 滑动窗口错误率追踪(用于 /health 上报,不触发重启)
|
|
92
93
|
let recentTotal = 0;
|
|
93
94
|
let recentErrors = 0;
|
|
@@ -192,6 +193,33 @@ function stopWarmTimer() {
|
|
|
192
193
|
}
|
|
193
194
|
}
|
|
194
195
|
|
|
196
|
+
// ---- 健康心跳日志(每 10 分钟一行,始终输出,不依赖 --log) ----
|
|
197
|
+
// 目的:无 --log 时 webserver 正常无错误会长时间静默,
|
|
198
|
+
// 若被看门狗 ProcessManager 管理,15 分钟无日志会被假死检测误判重启。
|
|
199
|
+
const HEALTH_LOG_INTERVAL_MS = 10 * 60 * 1000;
|
|
200
|
+
|
|
201
|
+
function logHealth() {
|
|
202
|
+
const scraperState = scraperInstance && !scraperClosing ? "运行中" : "未就绪";
|
|
203
|
+
const windowRate =
|
|
204
|
+
recentTotal > 0 ? Math.round((recentErrors / recentTotal) * 100) : 0;
|
|
205
|
+
console.error(
|
|
206
|
+
`[webserver] 健康: scraper=${scraperState} | 连续错误=${consecutiveErrors}/${MAX_CONSECUTIVE_ERRORS} | 近1min=${recentErrors}/${recentTotal} (${windowRate}%)`,
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function startHealthTimer() {
|
|
211
|
+
if (healthTimer) return;
|
|
212
|
+
healthTimer = setInterval(logHealth, HEALTH_LOG_INTERVAL_MS);
|
|
213
|
+
healthTimer.unref?.();
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function stopHealthTimer() {
|
|
217
|
+
if (healthTimer) {
|
|
218
|
+
clearInterval(healthTimer);
|
|
219
|
+
healthTimer = null;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
195
223
|
// ---- API 处理器 ----
|
|
196
224
|
|
|
197
225
|
const AVAILABLE_APIS = [
|
|
@@ -579,6 +607,9 @@ export async function startWebserver(
|
|
|
579
607
|
console.error(`[webserver] server error: ${err.message}`);
|
|
580
608
|
});
|
|
581
609
|
|
|
610
|
+
// 启动健康心跳日志(每 10 分钟一行,防假死误判)
|
|
611
|
+
startHealthTimer();
|
|
612
|
+
|
|
582
613
|
return new Promise((resolve, reject) => {
|
|
583
614
|
server.listen(port, () => {
|
|
584
615
|
const localIP = getLocalIP();
|
|
@@ -608,6 +639,7 @@ export async function closeServer(server) {
|
|
|
608
639
|
|
|
609
640
|
export async function closeScraper() {
|
|
610
641
|
stopWarmTimer();
|
|
642
|
+
stopHealthTimer();
|
|
611
643
|
if (scraperInstance && !scraperClosing) {
|
|
612
644
|
scraperClosing = true;
|
|
613
645
|
await scraperInstance.close();
|