tt-help-cli-ycl 1.4.166 → 1.4.167
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/scripts/deploy-node.sh +83 -0
- package/src/cli/attach.js +67 -19
- package/src/lib/browser/cdp.js +182 -27
- package/src/watch/data-store.js +42 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tt-help-cli-ycl",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.167",
|
|
4
4
|
"description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -48,4 +48,4 @@
|
|
|
48
48
|
"devDependencies": {
|
|
49
49
|
"better-sqlite3": "^12.10.0"
|
|
50
50
|
}
|
|
51
|
-
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# 将本地 tiktok-cli 打包部署到 Windows 采集节点(试运行,不经 npm registry)
|
|
3
|
+
# 用法:
|
|
4
|
+
# ./scripts/deploy-node.sh tiktok-210 # 打包+传输+远程安装+校验
|
|
5
|
+
# ./scripts/deploy-node.sh tiktok-210 --bump # 先把 package.json 版本号 +1 再打包
|
|
6
|
+
# ./scripts/deploy-node.sh tiktok-210 --bump --restart # 安装后重启走 CDP 的命令进程
|
|
7
|
+
# #(watchdog 检测到退出会用新代码自动拉起)
|
|
8
|
+
# 流程: (可选)升版本号 → npm pack → scp → 远程 npm install -g → 校验 → (可选)重启命令
|
|
9
|
+
# 前提: ~/.ssh/config 已配置节点别名(如 tiktok-210,经跳板机),BatchMode 免密可用
|
|
10
|
+
# 说明:
|
|
11
|
+
# - 无需停进程再安装:采集客户端不加载 better-sqlite3(只有 videostats/watch 服务端用),无文件锁
|
|
12
|
+
# - watchdog 自升级:npm 覆盖安装后守护进程经 upgrader.js 自动重启,受管命令随后重建
|
|
13
|
+
# - 节点上保留 tgz 安装包,回滚时 npm install -g 旧版 tgz 即可
|
|
14
|
+
set -euo pipefail
|
|
15
|
+
|
|
16
|
+
ALIAS="${1:?用法: deploy-node.sh <ssh别名> [--bump] [--restart]}"
|
|
17
|
+
shift || true
|
|
18
|
+
DO_BUMP=0
|
|
19
|
+
DO_RESTART=0
|
|
20
|
+
for a in "$@"; do
|
|
21
|
+
case "$a" in
|
|
22
|
+
--bump) DO_BUMP=1 ;;
|
|
23
|
+
--restart) DO_RESTART=1 ;;
|
|
24
|
+
*) echo "未知参数: $a"; exit 1 ;;
|
|
25
|
+
esac
|
|
26
|
+
done
|
|
27
|
+
|
|
28
|
+
cd "$(dirname "$0")/.."
|
|
29
|
+
|
|
30
|
+
if [ "$DO_BUMP" = 1 ]; then
|
|
31
|
+
node -e "
|
|
32
|
+
const fs = require('fs');
|
|
33
|
+
const p = JSON.parse(fs.readFileSync('package.json', 'utf8'));
|
|
34
|
+
const m = p.version.split('.');
|
|
35
|
+
m[2] = Number(m[2]) + 1;
|
|
36
|
+
p.version = m.join('.');
|
|
37
|
+
fs.writeFileSync('package.json', JSON.stringify(p, null, 2) + '\n');
|
|
38
|
+
console.log('版本号 ->', p.version);
|
|
39
|
+
"
|
|
40
|
+
fi
|
|
41
|
+
|
|
42
|
+
NAME=$(node -p "require('./package.json').name")
|
|
43
|
+
VER=$(node -p "require('./package.json').version")
|
|
44
|
+
TGZ="${NAME}-${VER}.tgz"
|
|
45
|
+
|
|
46
|
+
echo "==> npm pack"
|
|
47
|
+
rm -f "$TGZ"
|
|
48
|
+
TGZ=$(npm pack 2>/dev/null | tail -1)
|
|
49
|
+
echo "包: $TGZ ($(du -h "$TGZ" | cut -f1))"
|
|
50
|
+
|
|
51
|
+
echo "==> scp → $ALIAS"
|
|
52
|
+
scp -o ConnectTimeout=25 -o BatchMode=yes "$TGZ" "$ALIAS:$TGZ" 2>&1 |
|
|
53
|
+
grep -vE 'post-quantum|openssh.com/pq|store now|upgraded|CLIXML|^Warning|Permanently added' >/dev/null
|
|
54
|
+
echo "已传输到 %USERPROFILE%\\$TGZ"
|
|
55
|
+
|
|
56
|
+
echo "==> 远程 npm install -g"
|
|
57
|
+
ssh -o ConnectTimeout=25 -o BatchMode=yes "$ALIAS" \
|
|
58
|
+
"npm install -g %USERPROFILE%\\$TGZ 2>&1" |
|
|
59
|
+
grep -vE 'post-quantum|openssh.com/pq|store now|upgraded|CLIXML|^Warning|Permanently added' |
|
|
60
|
+
tail -3
|
|
61
|
+
|
|
62
|
+
echo "==> 校验远程版本"
|
|
63
|
+
REMOTE_VER=$(
|
|
64
|
+
ssh -o ConnectTimeout=25 -o BatchMode=yes "$ALIAS" "tt-help --version 2>nul" 2>/dev/null |
|
|
65
|
+
grep -vE 'post-quantum|openssh.com/pq|store now|upgraded|CLIXML|^Warning|Permanently added' |
|
|
66
|
+
tail -1 | tr -d '\r'
|
|
67
|
+
)
|
|
68
|
+
if [ "$REMOTE_VER" = "$VER" ]; then
|
|
69
|
+
echo "✓ 远程版本 $REMOTE_VER"
|
|
70
|
+
else
|
|
71
|
+
echo "✗ 版本不一致: 期望 $VER, 实际 '${REMOTE_VER:-空}'"
|
|
72
|
+
exit 1
|
|
73
|
+
fi
|
|
74
|
+
|
|
75
|
+
if [ "$DO_RESTART" = 1 ]; then
|
|
76
|
+
echo "==> 重启走 CDP 的命令进程(排除 watchdog/webserver,浏览器进程保留、新客户端直接重连)"
|
|
77
|
+
ssh -o ConnectTimeout=25 -o BatchMode=yes "$ALIAS" 'powershell -NoProfile -Command "Get-CimInstance Win32_Process -Filter \"Name='node.exe'\" | Where-Object { $_.CommandLine -match \"cli\.js\" -and $_.CommandLine -notmatch \"watchdog|webserver\" } | ForEach-Object { taskkill /F /PID $_.ProcessId }"' 2>&1 |
|
|
78
|
+
grep -vE 'post-quantum|openssh.com/pq|store now|upgraded|CLIXML|^Warning|Permanently added' |
|
|
79
|
+
head -10 || true
|
|
80
|
+
echo "(watchdog 检测到异常退出后会在 5-30s 内用新代码自动拉起,Edge 进程保留、客户端直连不重启浏览器)"
|
|
81
|
+
fi
|
|
82
|
+
|
|
83
|
+
echo "==> 完成"
|
package/src/cli/attach.js
CHANGED
|
@@ -160,13 +160,26 @@ export async function handleAttach(options) {
|
|
|
160
160
|
data = await res.json();
|
|
161
161
|
} catch {}
|
|
162
162
|
if (!res.ok) {
|
|
163
|
-
// CDN 限流:抛 CDNBlockedError,外层据此冷却(webserver
|
|
163
|
+
// CDN 限流:抛 CDNBlockedError,外层据此冷却(webserver 模式同样生效)。
|
|
164
|
+
// 服务端冷却期返回 503 + error="CDN cooldown active, cooling IP"(server.mjs),
|
|
165
|
+
// 并携带 cdnCooldownSec=剩余秒数,一并识别;秒数挂到 err.serverCooldownSec
|
|
166
|
+
// 供外层精确对齐冷却窗口,避免冷却期整批空转猛冲。
|
|
164
167
|
if (
|
|
165
168
|
data &&
|
|
166
169
|
typeof data.error === "string" &&
|
|
167
|
-
data.error.includes("CDN限流")
|
|
170
|
+
(data.error.includes("CDN限流") ||
|
|
171
|
+
data.error.includes("CDN cooldown"))
|
|
168
172
|
) {
|
|
169
|
-
|
|
173
|
+
const cdnErr = new CDNBlockedError(data.error);
|
|
174
|
+
if (data.cdnCooldownSec > 0)
|
|
175
|
+
cdnErr.serverCooldownSec = data.cdnCooldownSec;
|
|
176
|
+
throw cdnErr;
|
|
177
|
+
}
|
|
178
|
+
if (data && data.cdnCooldownSec > 0) {
|
|
179
|
+
// 文案兜底:只要响应带冷却秒数字段就按 CDN 限流处理
|
|
180
|
+
const cdnErr = new CDNBlockedError("CDN限流 (cooldown)");
|
|
181
|
+
cdnErr.serverCooldownSec = data.cdnCooldownSec;
|
|
182
|
+
throw cdnErr;
|
|
170
183
|
}
|
|
171
184
|
// 其余失败(504 超时 / 503 / 502 / 500 等):抛出带状态码与错误详情的异常,
|
|
172
185
|
// 让外层日志显示真实原因(如 page.goto 超时)而非笼统"未获取到用户信息"。
|
|
@@ -221,6 +234,7 @@ export async function handleAttach(options) {
|
|
|
221
234
|
let browserRestartCount = 0;
|
|
222
235
|
let taskBatchCount = 0;
|
|
223
236
|
let consecutiveCdnBlocks = 0; // 连续 CDN 限流轮数(自适应:前几次切换节点,之后长等待)
|
|
237
|
+
let consecutiveHighFail = 0; // 连续高失败率轮数(webserver 模式通用阶梯冷却)
|
|
224
238
|
|
|
225
239
|
while (!shuttingDown) {
|
|
226
240
|
loopCount++;
|
|
@@ -299,6 +313,7 @@ export async function handleAttach(options) {
|
|
|
299
313
|
let successCount = 0;
|
|
300
314
|
let failCount = 0;
|
|
301
315
|
let cdnBlockedCount = 0;
|
|
316
|
+
let serverCdnCooldownSec = 0; // 服务端告知的剩余冷却秒数(webserver 503 响应携带)
|
|
302
317
|
let needRestart = false;
|
|
303
318
|
|
|
304
319
|
// 收集抓取成功的任务,记录抓取失败的
|
|
@@ -308,8 +323,16 @@ export async function handleAttach(options) {
|
|
|
308
323
|
const { uniqueId, info, error } = result.value;
|
|
309
324
|
if (error) {
|
|
310
325
|
if (error instanceof CDNBlockedError) {
|
|
311
|
-
attachLog(
|
|
326
|
+
attachLog(
|
|
327
|
+
error.serverCooldownSec
|
|
328
|
+
? ` ⚠ @${uniqueId} CDN限流 (服务端冷却中,剩余 ${error.serverCooldownSec}s)`
|
|
329
|
+
: ` ⚠ @${uniqueId} CDN限流 (Access Denied)`,
|
|
330
|
+
);
|
|
312
331
|
cdnBlockedCount++;
|
|
332
|
+
serverCdnCooldownSec = Math.max(
|
|
333
|
+
serverCdnCooldownSec,
|
|
334
|
+
error.serverCooldownSec || 0,
|
|
335
|
+
);
|
|
313
336
|
failCount++;
|
|
314
337
|
} else if (isBrowserClosedError(error)) {
|
|
315
338
|
needRestart = true;
|
|
@@ -384,9 +407,13 @@ export async function handleAttach(options) {
|
|
|
384
407
|
|
|
385
408
|
attachLog(` 本批结果: ${successCount} 成功, ${failCount} 失败\n`);
|
|
386
409
|
|
|
387
|
-
//
|
|
410
|
+
// 失败率冷却策略:
|
|
411
|
+
// - 本地浏览器模式(scraper):CDN 限流 >30% 触发切节点 + 冷却(主动绕过)
|
|
412
|
+
// - webserver 模式:统一按总失败率阶梯冷却(出口在远端,分类对客户端无意义,
|
|
413
|
+
// 唯一手段是等;冷却期服务端 503 会带 cdnCooldownSec,优先精确对齐窗口)
|
|
388
414
|
const cdnRatio = tasks.length > 0 ? cdnBlockedCount / tasks.length : 0;
|
|
389
|
-
|
|
415
|
+
const failRatio = tasks.length > 0 ? failCount / tasks.length : 0;
|
|
416
|
+
if (cdnRatio > 0.3 && scraper) {
|
|
390
417
|
consecutiveCdnBlocks++;
|
|
391
418
|
// 冷却指数退避:60s→120s→240s→…上限 300s(切换后短等,持续限流则长等窗口过去)
|
|
392
419
|
const coolDownSeconds = Math.min(
|
|
@@ -394,12 +421,7 @@ export async function handleAttach(options) {
|
|
|
394
421
|
MAX_CDN_COOLDOWN_SEC,
|
|
395
422
|
);
|
|
396
423
|
const shouldSwitch = consecutiveCdnBlocks <= MAX_CDN_SWITCH_ATTEMPTS;
|
|
397
|
-
if (
|
|
398
|
-
// webserver 模式:限流发生在远端 webserver 出口,本地切节点无效,仅冷却等待
|
|
399
|
-
attachLog(
|
|
400
|
-
` [Attach] CDN限流比例 ${(cdnRatio * 100).toFixed(0)}% (${cdnBlockedCount}/${tasks.length}),webserver 模式不切节点,冷却 ${coolDownSeconds} 秒...`,
|
|
401
|
-
);
|
|
402
|
-
} else if (shouldSwitch) {
|
|
424
|
+
if (shouldSwitch) {
|
|
403
425
|
// 用现有 IPC 接口通知 daemon 从候选集切下一个健康节点(换 IP 绕过 CDN 限流)
|
|
404
426
|
try {
|
|
405
427
|
writeCommand({
|
|
@@ -417,16 +439,42 @@ export async function handleAttach(options) {
|
|
|
417
439
|
);
|
|
418
440
|
}
|
|
419
441
|
await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
442
|
+
await recycleScraper(
|
|
443
|
+
scraper,
|
|
444
|
+
`CDN限流比例过高 (${cdnBlockedCount}/${tasks.length})`,
|
|
445
|
+
);
|
|
446
|
+
browserRestartCount++;
|
|
447
|
+
taskBatchCount = 0;
|
|
448
|
+
} else if (
|
|
449
|
+
webserverUrl &&
|
|
450
|
+
tasks.length >= 5 &&
|
|
451
|
+
failRatio > 0.5
|
|
452
|
+
) {
|
|
453
|
+
// webserver 模式统一阶梯:总失败率 >50% 即冷却(含 CDN 限流/goto 超时/
|
|
454
|
+
// 验证码/网络错误等所有类型)。首次 30s 低代价试探,连续则指数升级
|
|
455
|
+
// 30→60→120→240→300s 封顶;服务端告知冷却剩余秒数时优先精确对齐;
|
|
456
|
+
// 失败率回落 ≤30% 清零
|
|
457
|
+
consecutiveHighFail++;
|
|
458
|
+
const coolDownSeconds = serverCdnCooldownSec
|
|
459
|
+
? Math.min(
|
|
460
|
+
Math.max(serverCdnCooldownSec + 5, 30),
|
|
461
|
+
MAX_CDN_COOLDOWN_SEC,
|
|
462
|
+
)
|
|
463
|
+
: Math.min(
|
|
464
|
+
30 * Math.pow(2, consecutiveHighFail - 1),
|
|
465
|
+
MAX_CDN_COOLDOWN_SEC,
|
|
466
|
+
);
|
|
467
|
+
const cdnTag = cdnBlockedCount
|
|
468
|
+
? `(其中 CDN限流 ${cdnBlockedCount})`
|
|
469
|
+
: "";
|
|
470
|
+
attachLog(
|
|
471
|
+
` [Attach] 失败率 ${(failRatio * 100).toFixed(0)}% (${failCount}/${tasks.length})${cdnTag},冷却 ${coolDownSeconds} 秒...`,
|
|
472
|
+
);
|
|
473
|
+
await new Promise((r) => setTimeout(r, coolDownSeconds * 1000));
|
|
427
474
|
taskBatchCount = 0;
|
|
428
475
|
} else {
|
|
429
476
|
consecutiveCdnBlocks = 0; // 正常批次重置连续限流计数
|
|
477
|
+
if (failRatio <= 0.3) consecutiveHighFail = 0; // 失败率回落清零阶梯计数
|
|
430
478
|
}
|
|
431
479
|
|
|
432
480
|
if (scraper) {
|
package/src/lib/browser/cdp.js
CHANGED
|
@@ -222,22 +222,66 @@ function killEdgeProcesses(targetDir, port) {
|
|
|
222
222
|
});
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
+
/** 查询监听指定端口的进程 PID 列表(netstat,不依赖 WMI);查询失败返回 [] */
|
|
226
|
+
function getListeningPidsOnPort(port) {
|
|
227
|
+
try {
|
|
228
|
+
const out = execSync("netstat -ano", {
|
|
229
|
+
encoding: "utf-8",
|
|
230
|
+
timeout: 10000,
|
|
231
|
+
windowsHide: true,
|
|
232
|
+
});
|
|
233
|
+
return parseNetstatListeningPids(out, port);
|
|
234
|
+
} catch {
|
|
235
|
+
return [];
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** tasklist 验证 PID 存活且属于 msedge(不依赖 WMI):true/false;查询失败返回 null */
|
|
240
|
+
function isEdgePidAliveWin(pid) {
|
|
241
|
+
try {
|
|
242
|
+
const out = execSync(`tasklist /FI "PID eq ${pid}" /FO CSV /NH`, {
|
|
243
|
+
encoding: "utf-8",
|
|
244
|
+
timeout: 5000,
|
|
245
|
+
windowsHide: true,
|
|
246
|
+
});
|
|
247
|
+
return out.toLowerCase().includes("msedge");
|
|
248
|
+
} catch {
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
225
253
|
/**
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
254
|
+
* 读取 profile 目录下 lockfile 里的数字(ProcessSingleton 持有者 PID 候选)。
|
|
255
|
+
* Chromium 写入格式随版本有差异,返回前 5 个数字串,由调用方逐一验证。
|
|
256
|
+
*/
|
|
257
|
+
function readLockPidCandidates(userDataDir) {
|
|
258
|
+
try {
|
|
259
|
+
const content = fs.readFileSync(path.join(userDataDir, "lockfile"), "utf-8");
|
|
260
|
+
return (content.match(/\d+/g) || []).slice(0, 5);
|
|
261
|
+
} catch {
|
|
262
|
+
return [];
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* 查找"残留" Edge 主进程:占着指定 user-data-dir 的实例(调用前已确认
|
|
268
|
+
* CDP 端口未监听,故带 --remote-debugging-port 参数的僵死实例同样算残留)。
|
|
269
|
+
* 排除带 --type= 的子进程:渲染进程等子进程同样带 user-data-dir,
|
|
270
|
+
* 误杀会打断同 profile 的健康实例。
|
|
231
271
|
*
|
|
232
272
|
* 这类实例必须先杀掉再启动:否则新 msedge 会因 Chromium ProcessSingleton
|
|
233
273
|
* 把命令行转发给旧实例,变成"旧实例开一个 about:blank 新 tab",CDP 端口
|
|
234
274
|
* 永远起不来 → 等待超时 → watchdog 自动重启 → 每轮再开一个 tab。
|
|
235
275
|
*
|
|
236
276
|
* killEdgeProcesses 在 Windows 上按监听端口找 PID,对这类实例无效,
|
|
237
|
-
*
|
|
277
|
+
* 故单独按命令行/lockfile 匹配精确清理,且不依赖 WMI:
|
|
278
|
+
* 1. PowerShell 枚举 msedge 命令行按 user-data-dir 匹配(需 WMI,
|
|
279
|
+
* WMI 损坏的机器上可能返回空)
|
|
280
|
+
* 2. lockfile 兜底:读 profile 目录 lockfile 中的 PID,用 tasklist 验证
|
|
281
|
+
* 存活且是 msedge(不依赖 WMI,覆盖 WMI 损坏 + 带 CDP 参数的僵死实例)
|
|
238
282
|
*
|
|
239
283
|
* @param {string} targetDir - user-data-dir 路径
|
|
240
|
-
* @returns {string[]|null} PID
|
|
284
|
+
* @returns {string[]|null} PID 列表;命令行枚举失败且 lockfile 无可信结论时返回 null
|
|
241
285
|
*/
|
|
242
286
|
function findStaleEdgePids(targetDir) {
|
|
243
287
|
const platform = os.platform();
|
|
@@ -245,31 +289,61 @@ function findStaleEdgePids(targetDir) {
|
|
|
245
289
|
let out;
|
|
246
290
|
if (platform === "win32") {
|
|
247
291
|
// PowerShell 为主(wmic 已从新版 Win11 移除;项目其他模块同样依赖 PowerShell)
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
292
|
+
let byCmdline = null;
|
|
293
|
+
try {
|
|
294
|
+
out = decodeWinOutput(
|
|
295
|
+
execSync(
|
|
296
|
+
'powershell -NoProfile -Command "Get-CimInstance Win32_Process | Where-Object { $_.Name -eq \'msedge.exe\' } | ForEach-Object { Write-Output ($_.ProcessId.ToString() + \'|\' + $_.CommandLine) }"',
|
|
297
|
+
{ encoding: "buffer", timeout: 10000, windowsHide: true },
|
|
298
|
+
),
|
|
299
|
+
);
|
|
300
|
+
const dirLower = targetDir.toLowerCase();
|
|
301
|
+
const pids = [];
|
|
302
|
+
for (const line of out.split(/\r?\n/)) {
|
|
303
|
+
const lower = line.toLowerCase();
|
|
304
|
+
if (!lower.includes(dirLower)) continue; // 只匹配本 profile
|
|
305
|
+
// 边界保护:避免 _p9222 误匹配 _p92221 这类前缀目录
|
|
306
|
+
const idx = lower.indexOf(dirLower);
|
|
307
|
+
const next = lower[idx + dirLower.length];
|
|
308
|
+
if (next && /[a-z0-9_]/.test(next)) continue;
|
|
309
|
+
if (lower.includes("--type=")) continue; // 子进程,误杀会打断健康实例
|
|
310
|
+
// 提取 PID:win 格式 "PID|命令行"
|
|
311
|
+
const m = line.match(/^\s*(\d+)\|/);
|
|
312
|
+
if (m && !pids.includes(m[1])) pids.push(m[1]);
|
|
313
|
+
}
|
|
314
|
+
byCmdline = pids;
|
|
315
|
+
} catch (e) {
|
|
316
|
+
console.error(`[findStaleEdgePids] PowerShell 枚举失败: ${e.message}`);
|
|
317
|
+
byCmdline = null;
|
|
318
|
+
}
|
|
319
|
+
if (byCmdline && byCmdline.length > 0) return byCmdline;
|
|
320
|
+
|
|
321
|
+
// lockfile 兑底:WMI 损坏时命令行枚举为空/失败;或残留实例带 CDP
|
|
322
|
+
// 参数时命令行匹配可能误报为空,lockfile 直接给出持有者 PID
|
|
323
|
+
for (const cand of readLockPidCandidates(targetDir)) {
|
|
324
|
+
if (isEdgePidAliveWin(cand) === true) return [cand];
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// 命令行枚举失败且 lockfile 没找到活持有者 → 无法判断
|
|
328
|
+
if (byCmdline === null) return null;
|
|
329
|
+
return [];
|
|
259
330
|
}
|
|
331
|
+
|
|
332
|
+
out = execSync("ps -eo pid,args", {
|
|
333
|
+
encoding: "utf-8",
|
|
334
|
+
timeout: 10000,
|
|
335
|
+
});
|
|
260
336
|
const dirLower = targetDir.toLowerCase();
|
|
261
337
|
const pids = [];
|
|
262
338
|
for (const line of out.split(/\r?\n/)) {
|
|
263
339
|
const lower = line.toLowerCase();
|
|
264
340
|
if (!lower.includes(dirLower)) continue; // 只匹配本 profile
|
|
265
|
-
// 边界保护:避免 _p9222 误匹配 _p92221 这类前缀目录
|
|
266
341
|
const idx = lower.indexOf(dirLower);
|
|
267
342
|
const next = lower[idx + dirLower.length];
|
|
268
343
|
if (next && /[a-z0-9_]/.test(next)) continue;
|
|
269
|
-
if (lower.includes("remote-debugging-port")) continue; // 健康 CDP 实例,不动
|
|
270
344
|
if (lower.includes("--type=")) continue; // 子进程,误杀会打断健康实例
|
|
271
|
-
// 提取 PID:
|
|
272
|
-
const m = line.match(/^\s*(\d+)
|
|
345
|
+
// 提取 PID:unix 格式 " PID 命令行"
|
|
346
|
+
const m = line.match(/^\s*(\d+)\s/);
|
|
273
347
|
if (m && !pids.includes(m[1])) pids.push(m[1]);
|
|
274
348
|
}
|
|
275
349
|
return pids;
|
|
@@ -308,6 +382,66 @@ function killEdgeStale(pids) {
|
|
|
308
382
|
});
|
|
309
383
|
}
|
|
310
384
|
|
|
385
|
+
/**
|
|
386
|
+
* CDP 等待超时后的现场诊断:端口被谁占用、msedge 进程总数、profile 锁持有者。
|
|
387
|
+
* 输出随超时错误一起打印/上报,远程节点无需人工登录即可定位问题。
|
|
388
|
+
* @param {number} port
|
|
389
|
+
* @param {string} userDataDir
|
|
390
|
+
* @returns {string} 形如 "[诊断] ...; ..." 的摘要;收集失败返回 ""
|
|
391
|
+
*/
|
|
392
|
+
function collectTimeoutDiagnostics(port, userDataDir) {
|
|
393
|
+
const parts = [];
|
|
394
|
+
try {
|
|
395
|
+
if (os.platform() === "win32") {
|
|
396
|
+
const listenPids = getListeningPidsOnPort(port);
|
|
397
|
+
if (listenPids.length > 0) {
|
|
398
|
+
const names = listenPids.map((pid) => {
|
|
399
|
+
try {
|
|
400
|
+
const out = execSync(`tasklist /FI "PID eq ${pid}" /FO CSV /NH`, {
|
|
401
|
+
encoding: "utf-8",
|
|
402
|
+
timeout: 5000,
|
|
403
|
+
windowsHide: true,
|
|
404
|
+
});
|
|
405
|
+
return out.split(",")[0]?.replace(/"/g, "").trim() || pid;
|
|
406
|
+
} catch {
|
|
407
|
+
return pid;
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
parts.push(
|
|
411
|
+
`端口 ${port} 正被监听但 CDP 未响应 (PID: ${listenPids.join(", ")}, 进程: ${names.join(", ")})`,
|
|
412
|
+
);
|
|
413
|
+
} else {
|
|
414
|
+
parts.push(`端口 ${port} 无任何进程监听`);
|
|
415
|
+
}
|
|
416
|
+
try {
|
|
417
|
+
const out = execSync(
|
|
418
|
+
'tasklist /FI "IMAGENAME eq msedge.exe" /FO CSV /NH',
|
|
419
|
+
{
|
|
420
|
+
encoding: "utf-8",
|
|
421
|
+
timeout: 5000,
|
|
422
|
+
windowsHide: true,
|
|
423
|
+
},
|
|
424
|
+
);
|
|
425
|
+
const count = out
|
|
426
|
+
.split(/\r?\n/)
|
|
427
|
+
.filter((l) => l.toLowerCase().includes("msedge")).length;
|
|
428
|
+
parts.push(`msedge 进程总数: ${count}`);
|
|
429
|
+
} catch {}
|
|
430
|
+
const lockPids = readLockPidCandidates(userDataDir).filter(
|
|
431
|
+
(pid) => isEdgePidAliveWin(pid) !== false,
|
|
432
|
+
);
|
|
433
|
+
parts.push(
|
|
434
|
+
lockPids.length
|
|
435
|
+
? `profile lockfile 持有者: ${lockPids.join(", ")}`
|
|
436
|
+
: "profile lockfile 无存活持有者",
|
|
437
|
+
);
|
|
438
|
+
}
|
|
439
|
+
} catch (e) {
|
|
440
|
+
parts.push(`诊断收集失败: ${e.message}`);
|
|
441
|
+
}
|
|
442
|
+
return parts.length ? `[诊断] ${parts.join("; ")}` : "";
|
|
443
|
+
}
|
|
444
|
+
|
|
311
445
|
function launchEdgeWithCDP(port, userDataDir, proxyServer) {
|
|
312
446
|
return new Promise((resolve, reject) => {
|
|
313
447
|
const platform = os.platform();
|
|
@@ -405,14 +539,32 @@ export async function ensureBrowserReady(options = {}) {
|
|
|
405
539
|
console.error(`CDP 端口 ${port} 未就绪,正在启动 Edge 浏览器...`);
|
|
406
540
|
}
|
|
407
541
|
}
|
|
408
|
-
//
|
|
409
|
-
|
|
542
|
+
// (Windows)CDP 端口被非 Edge 进程占用 → 立即报错,避免干等 60 秒超时
|
|
543
|
+
if (os.platform() === "win32") {
|
|
544
|
+
const listenPids = getListeningPidsOnPort(port);
|
|
545
|
+
if (listenPids.length > 0) {
|
|
546
|
+
const isEdge = listenPids.map((pid) => isEdgePidAliveWin(pid));
|
|
547
|
+
if (isEdge.every((v) => v === false)) {
|
|
548
|
+
throw new Error(
|
|
549
|
+
`CDP 端口 ${port} 被非 Edge 进程占用 (PID: ${listenPids.join(", ")}),` +
|
|
550
|
+
`无法启动浏览器。请释放该端口或更换 --cdp-port`,
|
|
551
|
+
);
|
|
552
|
+
}
|
|
553
|
+
// Edge 在监听但 HTTP 未响应 → 视为僵死,按残留处理(下方清理)
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
// 残留实例清理:占着该 user-data-dir 但 CDP 端口未监听的旧 Edge 必须先杀掉
|
|
558
|
+
//(含带 --remote-debugging-port 参数的僵死实例),否则 launch 会被
|
|
559
|
+
// ProcessSingleton 转发成"旧实例开新 tab",CDP 永远起不来。
|
|
410
560
|
const stalePids = findStaleEdgePids(userDataDir);
|
|
411
561
|
if (stalePids === null) {
|
|
412
|
-
console.error(
|
|
562
|
+
console.error(
|
|
563
|
+
"[浏览器] 无法判断是否存在残留 Edge 实例(WMI/lockfile 均不可用),跳过清理",
|
|
564
|
+
);
|
|
413
565
|
} else if (stalePids.length > 0) {
|
|
414
566
|
console.error(
|
|
415
|
-
`[浏览器] 检测到占用 ${path.basename(userDataDir)}
|
|
567
|
+
`[浏览器] 检测到占用 ${path.basename(userDataDir)} 但 CDP 端口未监听的残留 Edge 实例 (PID: ${stalePids.join(", ")}),正在清理...`,
|
|
416
568
|
);
|
|
417
569
|
await killEdgeStale(stalePids);
|
|
418
570
|
await new Promise((r) => setTimeout(r, 3000));
|
|
@@ -427,8 +579,11 @@ export async function ensureBrowserReady(options = {}) {
|
|
|
427
579
|
os.platform() === "win32" ? 60000 : 30000,
|
|
428
580
|
);
|
|
429
581
|
if (!launched) {
|
|
582
|
+
const diag = collectTimeoutDiagnostics(port, userDataDir);
|
|
583
|
+
if (diag) console.error(`[浏览器] ${diag}`);
|
|
430
584
|
throw new Error(
|
|
431
|
-
`等待 CDP 端口 ${port}
|
|
585
|
+
`等待 CDP 端口 ${port} 超时。${diag ? diag + "\n" : ""}` +
|
|
586
|
+
`请确认 Edge 浏览器已安装,\n` +
|
|
432
587
|
`或手动启动: Microsoft Edge --remote-debugging-port=${port} [参见 cdp.js extraArgs]`,
|
|
433
588
|
);
|
|
434
589
|
}
|
package/src/watch/data-store.js
CHANGED
|
@@ -1260,6 +1260,10 @@ export function createStore(filePath, options = {}) {
|
|
|
1260
1260
|
return data;
|
|
1261
1261
|
}
|
|
1262
1262
|
|
|
1263
|
+
// 空级跳过记忆(claimNextJob 用):key = "level|国家列表",value = 首次全 miss 时间。
|
|
1264
|
+
// 某来源级(如 seed)全部国家分支 miss 后,TTL 内跳过该级,避免每次领任务空跑
|
|
1265
|
+
const levelEmptyCache = new Map();
|
|
1266
|
+
|
|
1263
1267
|
async function claimNextJob(
|
|
1264
1268
|
userId,
|
|
1265
1269
|
expireMs = 5 * 60 * 1000,
|
|
@@ -1391,24 +1395,42 @@ export function createStore(filePath, options = {}) {
|
|
|
1391
1395
|
// 非 pinned 优先级:seed → seller → follow → other,每级 × 3 个 location tier × (hasVideo / noVideoReq)
|
|
1392
1396
|
const priorities = [
|
|
1393
1397
|
{
|
|
1398
|
+
name: "seed",
|
|
1394
1399
|
filter: `instr(COALESCE(sources, ''), '"seed"') > 0`,
|
|
1395
1400
|
loggedInOnly: false,
|
|
1396
1401
|
},
|
|
1397
1402
|
{
|
|
1403
|
+
name: "seller",
|
|
1398
1404
|
filter: `tt_seller = 1 AND verified = 0`,
|
|
1399
1405
|
loggedInOnly: true,
|
|
1400
1406
|
sellerAnyCountry: true,
|
|
1401
1407
|
},
|
|
1402
1408
|
{
|
|
1409
|
+
name: "follow",
|
|
1403
1410
|
filter: `(instr(COALESCE(sources, ''), '"following"') > 0 OR instr(COALESCE(sources, ''), '"follower"') > 0 OR instr(COALESCE(sources, ''), '"comment"') > 0)`,
|
|
1404
1411
|
loggedInOnly: false,
|
|
1405
1412
|
},
|
|
1406
|
-
{ filter: `1 = 1`, loggedInOnly: false },
|
|
1413
|
+
{ name: "other", filter: `1 = 1`, loggedInOnly: false },
|
|
1407
1414
|
];
|
|
1408
1415
|
|
|
1416
|
+
// 空级跳过记忆:某来源级(如 seed)全部国家分支 miss 后,记录时间;
|
|
1417
|
+
// TTL 内该级整体跳过(池子空了没必要每次领任务都空跑 24国×2 条查询,
|
|
1418
|
+
// 2026-09-29 实测 seed 空级每次烧 1.4s CPU 是 watch 饱和主因)。
|
|
1419
|
+
// TTL 过期后恢复探测一次——refill 补进新任务即可自动恢复。
|
|
1420
|
+
const LEVEL_EMPTY_TTL = 60 * 1000;
|
|
1421
|
+
for (const p of priorities) {
|
|
1422
|
+
if (p.loggedInOnly && !loggedIn) continue;
|
|
1423
|
+
const levelKey = `${p.name}|${orderedCountries.join(",")}`;
|
|
1424
|
+
const emptySince = levelEmptyCache.get(levelKey);
|
|
1425
|
+
if (emptySince && now - emptySince < LEVEL_EMPTY_TTL) {
|
|
1426
|
+
p.skip = true; // 标记跳过,下面构建分支时跳过
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1409
1430
|
let basePrio = 4;
|
|
1410
1431
|
for (const p of priorities) {
|
|
1411
1432
|
if (p.loggedInOnly && !loggedIn) continue;
|
|
1433
|
+
if (p.skip) continue;
|
|
1412
1434
|
// 逐国顺序分支:配置内国家按优先级顺序;未传子集时末尾补“配置外国家”兑底。
|
|
1413
1435
|
// 空位置(loc='')只在首个国家的分支参与:首国的 noVideoReq 分支 miss 即意味着
|
|
1414
1436
|
// 空位置池已耗尽,后续分支无需再 OR ''(避免反复扫描空位置大池)
|
|
@@ -1421,6 +1443,7 @@ export function createStore(filePath, options = {}) {
|
|
|
1421
1443
|
// 传了国家子集时对应国家的商家仍优先,领完再兜任意国家(含空位置商家);
|
|
1422
1444
|
// 未传国家时同理(配置内各国 → 配置外 → 任意)。其他来源级分支不受影响。
|
|
1423
1445
|
if (p.sellerAnyCountry) locationBranches.push(null); // null → 任意国家分支
|
|
1446
|
+
p._branchStart = branches.length; // 记录该级分支区间(用于判空)
|
|
1424
1447
|
let subPrio = 0;
|
|
1425
1448
|
for (const loc of locationBranches) {
|
|
1426
1449
|
// 商家任意国家分支:不限 guessed_location(含空位置的商家)
|
|
@@ -1453,12 +1476,16 @@ export function createStore(filePath, options = {}) {
|
|
|
1453
1476
|
);
|
|
1454
1477
|
subPrio++;
|
|
1455
1478
|
}
|
|
1479
|
+
p._branchEnd = branches.length;
|
|
1456
1480
|
basePrio += 2;
|
|
1457
1481
|
}
|
|
1458
1482
|
|
|
1459
|
-
//
|
|
1483
|
+
// 逐级执行:首个命中分支的最优行即领取结果(未命中继续下一优先级分支)。
|
|
1484
|
+
// 同时记录哪些来源级全部 miss → 写入 levelEmptyCache(下次领取 TTL 内直接跳过该级)
|
|
1460
1485
|
let selectedRow = null;
|
|
1461
|
-
|
|
1486
|
+
let bi = 0;
|
|
1487
|
+
for (; bi < branches.length; bi++) {
|
|
1488
|
+
const b = branches[bi];
|
|
1462
1489
|
selectedRow =
|
|
1463
1490
|
getDb()
|
|
1464
1491
|
.prepare(
|
|
@@ -1468,6 +1495,18 @@ export function createStore(filePath, options = {}) {
|
|
|
1468
1495
|
if (selectedRow) break;
|
|
1469
1496
|
}
|
|
1470
1497
|
|
|
1498
|
+
// 记录空级:分支按优先级顺序执行,命中所处级之前的所有来源级必然全 miss;
|
|
1499
|
+
// 空级写入 levelEmptyCache(TTL 内下次领取直接跳过),命中级清除记忆
|
|
1500
|
+
// bi = 命中分支下标(未命中则停在 branches.length)
|
|
1501
|
+
const hitIdx = bi;
|
|
1502
|
+
for (const p of priorities) {
|
|
1503
|
+
if (p.loggedInOnly && !loggedIn) continue;
|
|
1504
|
+
if (p._branchStart == null) continue; // 被 skip 跳过的级无分支,保留原记忆
|
|
1505
|
+
const levelKey = `${p.name}|${orderedCountries.join(",")}`;
|
|
1506
|
+
if (hitIdx >= p._branchEnd) levelEmptyCache.set(levelKey, now);
|
|
1507
|
+
else levelEmptyCache.delete(levelKey);
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1471
1510
|
if (selectedRow) {
|
|
1472
1511
|
getDb()
|
|
1473
1512
|
.prepare(
|