tt-help-cli-ycl 1.4.144 → 1.4.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-daemon-failover.mjs +61 -4
- package/scripts/test-product-fill-cdp.mjs +168 -0
- package/src/clash/checker.js +41 -0
- package/src/clash/daemon.js +26 -0
- package/src/cli/comments.js +265 -388
- package/src/cli/product-fill.js +255 -128
- package/src/cli/tag.js +143 -334
- package/src/lib/args.js +15 -5
- package/src/lib/constants.js +14 -10
- package/src/lib/shop-scraper.mjs +69 -6
- package/src/watch/data-store.js +129 -6
- package/src/watch/public/app.js +290 -0
- package/src/watch/public/index.html +99 -1
- package/src/watch/public/style.css +202 -0
- package/src/watch/server.js +64 -1
- package/src/watchdog/process-manager.js +11 -6
- package/scripts/test-product-fill.mjs +0 -306
package/src/cli/tag.js
CHANGED
|
@@ -1,19 +1,15 @@
|
|
|
1
1
|
import { writeFileSync } from "fs";
|
|
2
2
|
import { randomUUID } from "crypto";
|
|
3
|
-
import http from "http";
|
|
4
3
|
import readline from "readline";
|
|
5
4
|
import os from "os";
|
|
6
5
|
import {
|
|
7
6
|
getBrowserMemory,
|
|
8
7
|
formatMemoryWithBrowser,
|
|
9
8
|
} from "../lib/browser-memory.js";
|
|
10
|
-
import {
|
|
11
|
-
reportConsecutiveTaskErrors,
|
|
12
|
-
reportClientErrorAndFailover,
|
|
13
|
-
} from "../lib/client-error-report.js";
|
|
9
|
+
import { reportClientErrorAndFailover } from "../lib/client-error-report.js";
|
|
14
10
|
import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
|
|
15
11
|
import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
16
|
-
import { getOrCreatePage
|
|
12
|
+
import { getOrCreatePage } from "../lib/browser/page.js";
|
|
17
13
|
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
18
14
|
import { isLocationInList } from "../lib/target-locations.js";
|
|
19
15
|
import { getTargetLocations } from "../lib/app-config.js";
|
|
@@ -38,11 +34,17 @@ function resolveTargetCountries(countries) {
|
|
|
38
34
|
}
|
|
39
35
|
|
|
40
36
|
/**
|
|
41
|
-
*
|
|
37
|
+
* 判断浏览器是否真的已断开(用户手动关窗口 / CDP 断开)。
|
|
42
38
|
* 用于让采集循环(score-all / refresh)在浏览器被关闭后自动退出,避免无限重试。
|
|
39
|
+
*
|
|
40
|
+
* 注意:page.js 的 isBrowserClosedError 会把 net::ERR_CONNECTION_* 等网络层错误
|
|
41
|
+
* 也判为“浏览器关闭”。实际上这类错误是代理掂断/连接重置,浏览器进程还活着,
|
|
42
|
+
* 属于网络错误而不是窗口关闭。此处只认两种确凿信号:
|
|
43
|
+
* 1. browser.isConnected() === false(CDP 连接已断)
|
|
44
|
+
* 2. 明确的关闭消息(Target closed / Browser has been closed)
|
|
45
|
+
* net::ERR_* 一律放行,交给错误计数处理,避免误退出循环。
|
|
43
46
|
*/
|
|
44
47
|
function isBrowserDisconnected(browser, e) {
|
|
45
|
-
if (e && isBrowserClosedError(e)) return true;
|
|
46
48
|
try {
|
|
47
49
|
if (
|
|
48
50
|
browser &&
|
|
@@ -52,39 +54,16 @@ function isBrowserDisconnected(browser, e) {
|
|
|
52
54
|
return true;
|
|
53
55
|
}
|
|
54
56
|
} catch {}
|
|
55
|
-
return false;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const req = http.get(`http://127.0.0.1:${port}/json`, (res) => {
|
|
66
|
-
res.on("data", () => {});
|
|
67
|
-
res.on("end", () => resolve(res.statusCode === 200));
|
|
68
|
-
});
|
|
69
|
-
req.on("error", () => resolve(false));
|
|
70
|
-
req.setTimeout(2000, () => {
|
|
71
|
-
resolve(false);
|
|
72
|
-
req.destroy();
|
|
73
|
-
});
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
/**
|
|
78
|
-
* 等待端口不再被监听(浏览器进程真正退出、端口释放)。
|
|
79
|
-
* @returns {Promise<boolean>} 端口已释放返回 true
|
|
80
|
-
*/
|
|
81
|
-
async function waitForPortReleased(port, timeoutMs = 10000) {
|
|
82
|
-
const deadline = Date.now() + timeoutMs;
|
|
83
|
-
while (Date.now() < deadline) {
|
|
84
|
-
if (!(await checkPortListening(port))) return true;
|
|
85
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
86
|
-
}
|
|
87
|
-
return !(await checkPortListening(port));
|
|
57
|
+
if (!e) return false;
|
|
58
|
+
const msg = e.message || "";
|
|
59
|
+
// 网络层错误(net::ERR_CONNECTION_CLOSED/RESET/REFUSED...)不代表浏览器关闭
|
|
60
|
+
if (msg.includes("net::ERR_")) return false;
|
|
61
|
+
// 明确的 CDP 关闭信号
|
|
62
|
+
return (
|
|
63
|
+
msg.includes("Target closed") ||
|
|
64
|
+
msg.includes("Browser has been closed") ||
|
|
65
|
+
msg.includes("Target page, context or browser has been closed")
|
|
66
|
+
);
|
|
88
67
|
}
|
|
89
68
|
|
|
90
69
|
// 构建带客户端追踪 header 的 fetch 封装
|
|
@@ -808,9 +787,8 @@ export async function handleScoreAll(parsed) {
|
|
|
808
787
|
} = tagScoreAll || {};
|
|
809
788
|
|
|
810
789
|
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
let cdpPort = port || defaultScoreAllPort;
|
|
790
|
+
// 固定端口:一次会话一个浏览器,全程不换端口、不重启浏览器(错误只打印,不自愈)
|
|
791
|
+
const cdpPort = port || 7222;
|
|
814
792
|
const effectiveProxy = cliProxy || configuredProxy;
|
|
815
793
|
const targetCountries = resolveTargetCountries(countries);
|
|
816
794
|
|
|
@@ -863,62 +841,7 @@ export async function handleScoreAll(parsed) {
|
|
|
863
841
|
|
|
864
842
|
await setupPageRequestBlocking(page);
|
|
865
843
|
|
|
866
|
-
//
|
|
867
|
-
const usedPorts = new Set([cdpPort]);
|
|
868
|
-
|
|
869
|
-
// 端口池起始以 CLI --port / 环境变量 / 默认 7222 为准
|
|
870
|
-
const portPoolStart = cdpPort;
|
|
871
|
-
const portPoolSize = Math.max(
|
|
872
|
-
2,
|
|
873
|
-
parseInt(process.env.TAG_SCOREALL_PORT_POOL_SIZE || "10", 10) || 10,
|
|
874
|
-
);
|
|
875
|
-
|
|
876
|
-
async function pickNextPort(currentPort) {
|
|
877
|
-
const candidates = [];
|
|
878
|
-
for (let i = 0; i < portPoolSize; i++) {
|
|
879
|
-
const p = portPoolStart + i;
|
|
880
|
-
if (p !== currentPort) candidates.push(p);
|
|
881
|
-
}
|
|
882
|
-
// 跳过已被监听的端口(其他 score-all/attach 等程序的浏览器),
|
|
883
|
-
// 避免 attach 到别人的实例、后续 close/kill 时互相干扰导致浏览器多开
|
|
884
|
-
const freePorts = [];
|
|
885
|
-
for (const p of candidates) {
|
|
886
|
-
if (!(await checkPortListening(p))) freePorts.push(p);
|
|
887
|
-
}
|
|
888
|
-
const pool = freePorts.length > 0 ? freePorts : candidates;
|
|
889
|
-
if (pool.length === 0) return currentPort;
|
|
890
|
-
return pool[Math.floor(Math.random() * pool.length)];
|
|
891
|
-
}
|
|
892
|
-
|
|
893
|
-
// ── 内存回收配置 ──
|
|
894
|
-
const RECYCLE_CONFIG = {
|
|
895
|
-
nodeRssMb: Math.max(
|
|
896
|
-
256,
|
|
897
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "2000", 10) || 2000,
|
|
898
|
-
),
|
|
899
|
-
nodeHeapMb: Math.max(
|
|
900
|
-
128,
|
|
901
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "2000", 10) || 2000,
|
|
902
|
-
),
|
|
903
|
-
edgeMb: Math.max(
|
|
904
|
-
500,
|
|
905
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_EDGE_MB || "2000", 10) || 2000,
|
|
906
|
-
),
|
|
907
|
-
cooldownMs: Math.max(
|
|
908
|
-
0,
|
|
909
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_COOLDOWN_MS || "180000", 10) ||
|
|
910
|
-
180000,
|
|
911
|
-
),
|
|
912
|
-
periodicEvery: Math.max(
|
|
913
|
-
0,
|
|
914
|
-
parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
|
|
915
|
-
),
|
|
916
|
-
switchPortOnRecycle:
|
|
917
|
-
String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "0") !== "0",
|
|
918
|
-
};
|
|
919
|
-
let lastRecycleAt = 0;
|
|
920
|
-
|
|
921
|
-
// ── 轻量回收:只清空当前 tab ──
|
|
844
|
+
// ── 轻量回收:只清空当前 tab(不重启浏览器) ──
|
|
922
845
|
async function recyclePage() {
|
|
923
846
|
if (!page || page.isClosed()) {
|
|
924
847
|
page = await getOrCreatePage(browser);
|
|
@@ -937,117 +860,29 @@ export async function handleScoreAll(parsed) {
|
|
|
937
860
|
}
|
|
938
861
|
}
|
|
939
862
|
|
|
940
|
-
// ── 深度回收:关闭浏览器 + 可选切换端口 ──
|
|
941
|
-
async function recycleCdpSession(reason, { switchPort = true } = {}) {
|
|
942
|
-
const oldPort = cdpPort;
|
|
943
|
-
const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
|
|
944
|
-
if (shouldSwitch) {
|
|
945
|
-
cdpPort = await pickNextPort(cdpPort);
|
|
946
|
-
cdpOpts.port = cdpPort;
|
|
947
|
-
clientMeta.port = cdpPort;
|
|
948
|
-
usedPorts.add(cdpPort);
|
|
949
|
-
}
|
|
950
|
-
const portHint =
|
|
951
|
-
oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
|
|
952
|
-
log(` ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
|
|
953
|
-
if (page) {
|
|
954
|
-
await page.close().catch(() => {});
|
|
955
|
-
page = null;
|
|
956
|
-
}
|
|
957
|
-
// CDP 模式下 browser.close() 只断开连接不杀进程,必须显式 kill 旧端口实例;
|
|
958
|
-
// kill 后确认端口已释放,再启动新实例,防止旧浏览器窗口残留(多开)
|
|
959
|
-
if (oldPort !== cdpPort) {
|
|
960
|
-
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
961
|
-
if (!(await waitForPortReleased(oldPort))) {
|
|
962
|
-
log(` ⚠️ 旧端口 ${oldPort} 未释放,再次尝试关闭...`);
|
|
963
|
-
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
964
|
-
if (!(await waitForPortReleased(oldPort, 5000))) {
|
|
965
|
-
log(
|
|
966
|
-
` ⚠️ 旧端口 ${oldPort} 仍未释放,继续使用新端口(旧浏览器窗口可能残留,退出时会统一清理)`,
|
|
967
|
-
);
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
await browser.close().catch(() => {});
|
|
972
|
-
browser = await ensureBrowserReady(cdpOpts);
|
|
973
|
-
page = await getOrCreatePage(browser);
|
|
974
|
-
await setupPageRequestBlocking(page);
|
|
975
|
-
lastRecycleAt = Date.now();
|
|
976
|
-
}
|
|
977
|
-
|
|
978
|
-
// ── 内存检查 + 自动回收 ──
|
|
979
|
-
// 策略:Node.js 内存超阈值 → 不换端口(只重建连接)
|
|
980
|
-
// Edge 内存超阈值 → 换端口(杀旧浏览器 + 启新实例,彻底释放)
|
|
981
|
-
// 定期轮次 → 不换端口
|
|
982
|
-
async function maybeRecycleForMemory() {
|
|
983
|
-
const mem = process.memoryUsage();
|
|
984
|
-
const rssMB = mem.rss / 1024 / 1024;
|
|
985
|
-
const heapMB = mem.heapUsed / 1024 / 1024;
|
|
986
|
-
const edgeMB = getBrowserMemory(cdpPort);
|
|
987
|
-
|
|
988
|
-
const nodeOver =
|
|
989
|
-
rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
|
|
990
|
-
const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
|
|
991
|
-
|
|
992
|
-
if (nodeOver || edgeOver) {
|
|
993
|
-
// 冷却期内跳过
|
|
994
|
-
if (
|
|
995
|
-
RECYCLE_CONFIG.cooldownMs > 0 &&
|
|
996
|
-
Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
|
|
997
|
-
)
|
|
998
|
-
return;
|
|
999
|
-
|
|
1000
|
-
const parts = [];
|
|
1001
|
-
if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
|
|
1002
|
-
parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
|
|
1003
|
-
if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
|
|
1004
|
-
parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
|
|
1005
|
-
if (edgeOver)
|
|
1006
|
-
parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
|
|
1007
|
-
|
|
1008
|
-
await recycleCdpSession(`mem ${parts.join("+")}`, {
|
|
1009
|
-
switchPort: edgeOver,
|
|
1010
|
-
});
|
|
1011
|
-
return;
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
// 定期轮次回收(默认关闭)
|
|
1015
|
-
if (
|
|
1016
|
-
RECYCLE_CONFIG.periodicEvery > 0 &&
|
|
1017
|
-
totalScored > 0 &&
|
|
1018
|
-
totalScored % RECYCLE_CONFIG.periodicEvery === 0
|
|
1019
|
-
) {
|
|
1020
|
-
await recycleCdpSession(`periodic #${totalScored}`, {
|
|
1021
|
-
switchPort: false,
|
|
1022
|
-
});
|
|
1023
|
-
}
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
863
|
let totalScored = 0;
|
|
1027
864
|
let emptyRounds = 0; // 连续无任务的轮数
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
(parseInt(process.env.TAG_SCOREALL_403_RETRY_3 || "50", 10) || 50) * 1000,
|
|
1050
|
-
];
|
|
865
|
+
|
|
866
|
+
// ── 错误计数(纯观察,不做任何自愈) ──
|
|
867
|
+
// 阈值语义:连续超过 N 次触发动作(即第 N+1 次)
|
|
868
|
+
// 403 连续 >2 次 → 上报服务端 + 强制 fallback(daemon 换节点),清零后继续跑
|
|
869
|
+
// 网络错误 连续 >10 次 → 触发 daemon fallback 换节点(本地 IPC,不依赖服务端可达)+ 退出
|
|
870
|
+
// 验证码 连续 >3 次(人工一直未解决)→ 打印退出
|
|
871
|
+
// 其他错误 连续 >10 次 → 打印退出
|
|
872
|
+
// 清零规则:成功处理一个 tag → 全部清零;fallback 触发 → 对应计数清零;
|
|
873
|
+
// 人工点掉验证码 → 验证码计数清零
|
|
874
|
+
const errorCounts = { http403: 0, network: 0, captcha: 0, other: 0 };
|
|
875
|
+
const ERROR_LIMITS = { http403: 2, network: 10, captcha: 3, other: 10 };
|
|
876
|
+
const resetAllErrorCounts = () => {
|
|
877
|
+
errorCounts.http403 = 0;
|
|
878
|
+
errorCounts.network = 0;
|
|
879
|
+
errorCounts.captcha = 0;
|
|
880
|
+
errorCounts.other = 0;
|
|
881
|
+
};
|
|
882
|
+
const bumpErrorCount = (type) => {
|
|
883
|
+
errorCounts[type]++;
|
|
884
|
+
return errorCounts[type] > ERROR_LIMITS[type];
|
|
885
|
+
};
|
|
1051
886
|
|
|
1052
887
|
// 生成客户端 ID,用于服务端追踪
|
|
1053
888
|
const clientId = randomUUID();
|
|
@@ -1080,14 +915,10 @@ export async function handleScoreAll(parsed) {
|
|
|
1080
915
|
if (enrichScraper) await enrichScraper.close().catch(() => {});
|
|
1081
916
|
log("正在关闭 CDP 浏览器...");
|
|
1082
917
|
try {
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
killEdgeProcesses(null, p),
|
|
1088
|
-
new Promise((r) => setTimeout(r, 5000)),
|
|
1089
|
-
]).catch(() => {});
|
|
1090
|
-
}
|
|
918
|
+
await Promise.race([
|
|
919
|
+
killEdgeProcesses(null, cdpPort),
|
|
920
|
+
new Promise((r) => setTimeout(r, 5000)),
|
|
921
|
+
]).catch(() => {});
|
|
1091
922
|
log("浏览器已关闭");
|
|
1092
923
|
} catch (e) {
|
|
1093
924
|
log(`关闭浏览器失败: ${e.message}`);
|
|
@@ -1116,43 +947,38 @@ export async function handleScoreAll(parsed) {
|
|
|
1116
947
|
return;
|
|
1117
948
|
}
|
|
1118
949
|
try {
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
{
|
|
1128
|
-
headers: buildClientHeaders(clientId, clientMeta),
|
|
1129
|
-
},
|
|
1130
|
-
);
|
|
1131
|
-
const tagsData = await tagsRes.json();
|
|
950
|
+
// 从服务端取下一个 new 标签(同时查剩余数量)
|
|
951
|
+
const tagsRes = await fetch(
|
|
952
|
+
`${baseUrl}/api/tags?status=new&limit=1&userId=${encodeURIComponent(userId)}`,
|
|
953
|
+
{
|
|
954
|
+
headers: buildClientHeaders(clientId, clientMeta),
|
|
955
|
+
},
|
|
956
|
+
);
|
|
957
|
+
const tagsData = await tagsRes.json();
|
|
1132
958
|
|
|
1133
|
-
|
|
1134
|
-
|
|
959
|
+
if (!tagsData.tags || tagsData.tags.length === 0) {
|
|
960
|
+
emptyRounds++;
|
|
1135
961
|
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
continue;
|
|
1144
|
-
}
|
|
1145
|
-
log(`⏳ 暂无待打分标签(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
1146
|
-
await new Promise((r) => setTimeout(r, 10000));
|
|
962
|
+
// 服务端定时发现会自动补充标签,客户端只需等待
|
|
963
|
+
if (emptyRounds >= 5) {
|
|
964
|
+
log(
|
|
965
|
+
`⏳ 连续 ${emptyRounds} 轮无标签,等待服务端发现补充(30 秒后重试)...`,
|
|
966
|
+
);
|
|
967
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
968
|
+
emptyRounds = 0;
|
|
1147
969
|
continue;
|
|
1148
970
|
}
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
tag = tagsData.tags[0].tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1153
|
-
currentTag = tag;
|
|
1154
|
-
tag403Attempts = 0;
|
|
971
|
+
log(`⏳ 暂无待打分标签(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
972
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
973
|
+
continue;
|
|
1155
974
|
}
|
|
975
|
+
|
|
976
|
+
// 有任务了,重置计数器
|
|
977
|
+
emptyRounds = 0;
|
|
978
|
+
const tag = tagsData.tags[0].tag
|
|
979
|
+
.replace(/^#+/, "")
|
|
980
|
+
.trim()
|
|
981
|
+
.toLowerCase();
|
|
1156
982
|
const startTime = Date.now();
|
|
1157
983
|
|
|
1158
984
|
log(`[${totalScored + 1}] 正在打分 #${tag} ...`);
|
|
@@ -1170,8 +996,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1170
996
|
};
|
|
1171
997
|
|
|
1172
998
|
// 锁定 tag(meta 中不放入 tag,避免非 ASCII 字符导致 header ByteString 报错)
|
|
1173
|
-
|
|
1174
|
-
if (!retrySameTag) {
|
|
999
|
+
{
|
|
1175
1000
|
const claimRes = await fetch(
|
|
1176
1001
|
`${baseUrl}/api/tags/claim?userId=${encodeURIComponent(userId)}`,
|
|
1177
1002
|
{
|
|
@@ -1233,7 +1058,6 @@ export async function handleScoreAll(parsed) {
|
|
|
1233
1058
|
totalScored++;
|
|
1234
1059
|
await randomDelay(0, 5000);
|
|
1235
1060
|
await recyclePage();
|
|
1236
|
-
await maybeRecycleForMemory();
|
|
1237
1061
|
continue;
|
|
1238
1062
|
}
|
|
1239
1063
|
// 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
|
|
@@ -1295,18 +1119,14 @@ export async function handleScoreAll(parsed) {
|
|
|
1295
1119
|
);
|
|
1296
1120
|
videos = enriched.videos;
|
|
1297
1121
|
|
|
1298
|
-
// CDN
|
|
1122
|
+
// CDN 限流:打印并计入其他错误计数(不自愈、不重启 scraper)
|
|
1299
1123
|
const cdnBlocked = enriched.cdnBlockedCount || 0;
|
|
1300
1124
|
if (cdnBlocked > 0) {
|
|
1301
1125
|
const cdnRatio = cdnBlocked / (videos.length || 1);
|
|
1302
|
-
|
|
1126
|
+
bumpErrorCount("other");
|
|
1303
1127
|
log(
|
|
1304
|
-
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)
|
|
1128
|
+
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),计入其他错误计数 (${errorCounts.other}/${ERROR_LIMITS.other})`,
|
|
1305
1129
|
);
|
|
1306
|
-
await new Promise((r) => setTimeout(r, coolSec * 1000));
|
|
1307
|
-
log(` 正在重启 TikTokScraper...`);
|
|
1308
|
-
await enrichScraper.restart();
|
|
1309
|
-
log(` ✅ TikTokScraper 已重启`);
|
|
1310
1130
|
}
|
|
1311
1131
|
}
|
|
1312
1132
|
|
|
@@ -1334,7 +1154,6 @@ export async function handleScoreAll(parsed) {
|
|
|
1334
1154
|
` ⏭️ 重试 ${COUNTRY_MAX_RETRIES} 次后国家识别率仍 ${(rate * 100).toFixed(0)}% < ${(COUNTRY_RATE_MIN * 100).toFixed(0)}%,跳过该标签,处理下一个`,
|
|
1335
1155
|
);
|
|
1336
1156
|
await recyclePage();
|
|
1337
|
-
await maybeRecycleForMemory();
|
|
1338
1157
|
await randomDelay(3000, 7000);
|
|
1339
1158
|
continue;
|
|
1340
1159
|
}
|
|
@@ -1379,10 +1198,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1379
1198
|
` ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? " " + mc : ""}${memStr}`,
|
|
1380
1199
|
);
|
|
1381
1200
|
log("");
|
|
1382
|
-
|
|
1383
|
-
retrySameTag = false; // 成功,不再重试
|
|
1384
|
-
tag403Attempts = 0;
|
|
1385
|
-
bannedCount = 0; // 成功 tag,重置连续判封计数
|
|
1201
|
+
resetAllErrorCounts(); // 成功一个 tag → 所有错误计数清零
|
|
1386
1202
|
// 每 10 个标签输出一次内存汇总
|
|
1387
1203
|
if (totalScored % 10 === 0) {
|
|
1388
1204
|
log(
|
|
@@ -1390,7 +1206,6 @@ export async function handleScoreAll(parsed) {
|
|
|
1390
1206
|
);
|
|
1391
1207
|
}
|
|
1392
1208
|
await recyclePage();
|
|
1393
|
-
await maybeRecycleForMemory();
|
|
1394
1209
|
await randomDelay(3000, 7000);
|
|
1395
1210
|
} catch (e) {
|
|
1396
1211
|
// 浏览器窗口被手动关闭(CDP 断开)→ 自动退出,避免无限重试循环
|
|
@@ -1407,9 +1222,38 @@ export async function handleScoreAll(parsed) {
|
|
|
1407
1222
|
(e.message &&
|
|
1408
1223
|
(e.message.includes("ECONNREFUSED") ||
|
|
1409
1224
|
e.message.includes("fetch failed") ||
|
|
1410
|
-
e.message.includes("network")
|
|
1225
|
+
e.message.includes("network") ||
|
|
1226
|
+
e.message.includes("net::ERR_")));
|
|
1411
1227
|
if (isNetworkError) {
|
|
1412
|
-
|
|
1228
|
+
const over = bumpErrorCount("network");
|
|
1229
|
+
log(
|
|
1230
|
+
` ⚠️ 网络错误 (${e.message}),15 秒后重试(连续网络错误 ${errorCounts.network} 次)...`,
|
|
1231
|
+
);
|
|
1232
|
+
if (over) {
|
|
1233
|
+
log(
|
|
1234
|
+
` ❌ 连续网络错误超过 ${ERROR_LIMITS.network} 次 → 触发 daemon fallback 换节点后退出`,
|
|
1235
|
+
);
|
|
1236
|
+
// 网络错误多为代理节点问题:fallback 走本地 IPC 触发 daemon 换节点
|
|
1237
|
+
// (不依赖服务端可达),上报失败会被内部忽略;随后退出由看门狗拉起
|
|
1238
|
+
await reportClientErrorAndFailover({
|
|
1239
|
+
post: (url, body) =>
|
|
1240
|
+
fetch(url, {
|
|
1241
|
+
method: "POST",
|
|
1242
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1243
|
+
"Content-Type": "application/json",
|
|
1244
|
+
}),
|
|
1245
|
+
body: JSON.stringify(body),
|
|
1246
|
+
}),
|
|
1247
|
+
baseUrl,
|
|
1248
|
+
userId,
|
|
1249
|
+
errorType: "network_failure",
|
|
1250
|
+
message: `连续 ${ERROR_LIMITS.network + 1} 次网络错误 (port=${cdpPort})`,
|
|
1251
|
+
stage: "tag-scoring",
|
|
1252
|
+
log,
|
|
1253
|
+
});
|
|
1254
|
+
await cleanup();
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1413
1257
|
await new Promise((r) => setTimeout(r, 15000));
|
|
1414
1258
|
continue;
|
|
1415
1259
|
}
|
|
@@ -1423,35 +1267,16 @@ export async function handleScoreAll(parsed) {
|
|
|
1423
1267
|
e.message &&
|
|
1424
1268
|
(e.message.includes("验证码") || e.message.includes("captcha"));
|
|
1425
1269
|
|
|
1426
|
-
// ── 403
|
|
1427
|
-
//
|
|
1270
|
+
// ── 403:当前 tag 不上报(留在服务端 scoring 状态超时回收),
|
|
1271
|
+
// 只计数;连续 >2 次上报服务端 + 强制 fallback(daemon 换节点),
|
|
1272
|
+
// 计数清零后继续跑 ──
|
|
1428
1273
|
if (is403) {
|
|
1429
|
-
const
|
|
1430
|
-
|
|
1431
|
-
if (
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
log(
|
|
1436
|
-
` 🚫 HTTP 403,判定被封,重启浏览器后 ${waitMs / 1000}s 第 ${tag403Attempts}/${BAN_RETRY_WAITS.length} 次重试 #${currentTag} ...`,
|
|
1437
|
-
);
|
|
1438
|
-
await recycleCdpSession(`首次被封`, { switchPort: false });
|
|
1439
|
-
} else {
|
|
1440
|
-
log(
|
|
1441
|
-
` 🚫 HTTP 403,判定被封,${waitMs / 1000}s 后第 ${tag403Attempts}/${BAN_RETRY_WAITS.length} 次重试 #${currentTag} ...`,
|
|
1442
|
-
);
|
|
1443
|
-
await recyclePage();
|
|
1444
|
-
}
|
|
1445
|
-
await new Promise((r) => setTimeout(r, waitMs));
|
|
1446
|
-
continue;
|
|
1447
|
-
}
|
|
1448
|
-
// 重试耗尽:判定“被封”
|
|
1449
|
-
bannedCount++;
|
|
1450
|
-
log(
|
|
1451
|
-
` 🚫 30/40/50s 重试 ${BAN_RETRY_WAITS.length} 次仍 403,判定账号被封(连续判封 ${bannedCount} 次)`,
|
|
1452
|
-
);
|
|
1453
|
-
if (bannedCount >= 2) {
|
|
1454
|
-
log(` ⚠️ 连续两次判封 → 汇报服务端 + 触发 daemon fallback`);
|
|
1274
|
+
const over = bumpErrorCount("http403");
|
|
1275
|
+
log(` 🚫 HTTP 403(连续第 ${errorCounts.http403} 次)`);
|
|
1276
|
+
if (over) {
|
|
1277
|
+
log(
|
|
1278
|
+
` ⚠️ 连续 403 超过 ${ERROR_LIMITS.http403} 次 → 上报服务端 + 触发 daemon fallback`,
|
|
1279
|
+
);
|
|
1455
1280
|
await reportClientErrorAndFailover({
|
|
1456
1281
|
post: (url, body) =>
|
|
1457
1282
|
fetch(url, {
|
|
@@ -1464,16 +1289,13 @@ export async function handleScoreAll(parsed) {
|
|
|
1464
1289
|
baseUrl,
|
|
1465
1290
|
userId,
|
|
1466
1291
|
errorType: "consecutive_blocked",
|
|
1467
|
-
message:
|
|
1292
|
+
message: `连续 ${ERROR_LIMITS.http403 + 1} 次 403 (port=${cdpPort})`,
|
|
1468
1293
|
stage: "tag-scoring",
|
|
1469
1294
|
log,
|
|
1470
1295
|
});
|
|
1471
|
-
|
|
1296
|
+
errorCounts.http403 = 0; // fallback 触发后清零
|
|
1472
1297
|
}
|
|
1473
|
-
retrySameTag = false;
|
|
1474
|
-
tag403Attempts = 0;
|
|
1475
1298
|
await recyclePage();
|
|
1476
|
-
await maybeRecycleForMemory();
|
|
1477
1299
|
await randomDelay(3000, 7000);
|
|
1478
1300
|
continue;
|
|
1479
1301
|
}
|
|
@@ -1482,9 +1304,9 @@ export async function handleScoreAll(parsed) {
|
|
|
1482
1304
|
log(` ❌ 失败: ${e.message}`);
|
|
1483
1305
|
|
|
1484
1306
|
if (isCaptcha) {
|
|
1485
|
-
|
|
1307
|
+
bumpErrorCount("captcha");
|
|
1486
1308
|
log(
|
|
1487
|
-
` 🔐
|
|
1309
|
+
` 🔐 验证码(连续第 ${errorCounts.captcha} 次),等待用户手动解决...`,
|
|
1488
1310
|
);
|
|
1489
1311
|
|
|
1490
1312
|
// 上报验证码错误
|
|
@@ -1507,24 +1329,31 @@ export async function handleScoreAll(parsed) {
|
|
|
1507
1329
|
} catch {}
|
|
1508
1330
|
|
|
1509
1331
|
// 等待用户手动解决验证码(最多 3 分钟)
|
|
1332
|
+
let captchaResolved = false;
|
|
1510
1333
|
try {
|
|
1511
1334
|
const captchaResult = await waitAndGetCaptcha(page, {
|
|
1512
1335
|
waitMs: 180000,
|
|
1513
1336
|
pollInterval: 5000,
|
|
1514
1337
|
log: () => {},
|
|
1515
1338
|
});
|
|
1339
|
+
captchaResolved = !!captchaResult.resolved;
|
|
1516
1340
|
if (captchaResult.detected && !captchaResult.resolved) {
|
|
1517
1341
|
await closeCaptcha(page).catch(() => {});
|
|
1518
1342
|
}
|
|
1519
1343
|
} catch {}
|
|
1520
1344
|
|
|
1521
|
-
if (
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1345
|
+
if (captchaResolved) {
|
|
1346
|
+
// 人工点掉验证码 → 验证码计数清零
|
|
1347
|
+
log(` ✅ 验证码已被人工处理,验证码计数清零`);
|
|
1348
|
+
errorCounts.captcha = 0;
|
|
1349
|
+
} else if (errorCounts.captcha > ERROR_LIMITS.captcha) {
|
|
1350
|
+
log(
|
|
1351
|
+
` ❌ 连续验证码超过 ${ERROR_LIMITS.captcha} 次且未解决 → 打印并退出`,
|
|
1352
|
+
);
|
|
1353
|
+
await cleanup();
|
|
1354
|
+
return;
|
|
1525
1355
|
}
|
|
1526
|
-
|
|
1527
|
-
consecutiveCaptcha = 0;
|
|
1356
|
+
continue;
|
|
1528
1357
|
}
|
|
1529
1358
|
|
|
1530
1359
|
try {
|
|
@@ -1543,34 +1372,14 @@ export async function handleScoreAll(parsed) {
|
|
|
1543
1372
|
);
|
|
1544
1373
|
} catch {}
|
|
1545
1374
|
|
|
1546
|
-
// 连续任务错误达到阈值,上报服务端 + 触发 daemon fallback 换节点
|
|
1547
|
-
consecutiveTaskErrors++;
|
|
1548
|
-
if (consecutiveTaskErrors >= CONSECUTIVE_TASK_ERRORS_MAX) {
|
|
1549
|
-
log(
|
|
1550
|
-
` ⚠️ 连续任务错误 ${consecutiveTaskErrors}/${CONSECUTIVE_TASK_ERRORS_MAX} → 汇报服务端 + 触发 fallback`,
|
|
1551
|
-
);
|
|
1552
|
-
await reportConsecutiveTaskErrors({
|
|
1553
|
-
post: (url, body) =>
|
|
1554
|
-
fetch(url, {
|
|
1555
|
-
method: "POST",
|
|
1556
|
-
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1557
|
-
"Content-Type": "application/json",
|
|
1558
|
-
}),
|
|
1559
|
-
body: JSON.stringify(body),
|
|
1560
|
-
}),
|
|
1561
|
-
baseUrl,
|
|
1562
|
-
userId,
|
|
1563
|
-
message: `连续 ${consecutiveTaskErrors} 个标签处理失败 (port=${cdpPort})`,
|
|
1564
|
-
stage: "tag-scoring",
|
|
1565
|
-
log,
|
|
1566
|
-
});
|
|
1567
|
-
consecutiveTaskErrors = 0;
|
|
1568
|
-
}
|
|
1569
1375
|
totalScored++;
|
|
1570
|
-
|
|
1571
|
-
|
|
1376
|
+
const over = bumpErrorCount("other");
|
|
1377
|
+
if (over) {
|
|
1378
|
+
log(` ❌ 连续其他错误超过 ${ERROR_LIMITS.other} 次 → 打印并退出`);
|
|
1379
|
+
await cleanup();
|
|
1380
|
+
return;
|
|
1381
|
+
}
|
|
1572
1382
|
await recyclePage();
|
|
1573
|
-
await maybeRecycleForMemory();
|
|
1574
1383
|
}
|
|
1575
1384
|
}
|
|
1576
1385
|
} finally {
|