tt-help-cli-ycl 1.4.142 → 1.4.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/sync-raw-jobs.mjs +13 -7
- package/scripts/test-api-interceptor-429.mjs +122 -0
- package/scripts/test-daemon-failover.mjs +61 -4
- package/src/clash/checker.js +41 -0
- package/src/clash/daemon.js +26 -0
- package/src/cli/comments.js +5 -5
- package/src/cli/precheck.js +2 -2
- package/src/cli/refresh-mini.js +1 -1
- package/src/cli/tag.js +110 -353
- package/src/lib/api-interceptor.js +26 -8
- package/src/lib/app-config.js +119 -0
- package/src/lib/args.js +3 -1
- package/src/lib/constants.js +9 -6
- package/src/lib/target-locations.js +1 -0
- package/src/scraper/core.js +2 -2
- package/src/scraper/explore-core.js +2 -2
- package/src/watch/data-store.js +94 -8
- package/src/watch/public/app.js +412 -26
- package/src/watch/public/index.html +74 -1
- package/src/watch/public/style.css +218 -0
- package/src/watch/server.js +144 -40
- package/src/watchdog/process-manager.js +11 -6
package/src/cli/tag.js
CHANGED
|
@@ -1,24 +1,18 @@
|
|
|
1
1
|
import { writeFileSync } from "fs";
|
|
2
2
|
import { randomUUID } from "crypto";
|
|
3
|
-
import http from "http";
|
|
4
3
|
import readline from "readline";
|
|
5
4
|
import os from "os";
|
|
6
5
|
import {
|
|
7
6
|
getBrowserMemory,
|
|
8
7
|
formatMemoryWithBrowser,
|
|
9
8
|
} from "../lib/browser-memory.js";
|
|
10
|
-
import {
|
|
11
|
-
reportConsecutiveTaskErrors,
|
|
12
|
-
reportClientErrorAndFailover,
|
|
13
|
-
} from "../lib/client-error-report.js";
|
|
9
|
+
import { reportClientErrorAndFailover } from "../lib/client-error-report.js";
|
|
14
10
|
import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
|
|
15
11
|
import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
16
12
|
import { getOrCreatePage, isBrowserClosedError } from "../lib/browser/page.js";
|
|
17
13
|
import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
|
|
18
|
-
import {
|
|
19
|
-
|
|
20
|
-
isLocationInList,
|
|
21
|
-
} from "../lib/target-locations.js";
|
|
14
|
+
import { isLocationInList } from "../lib/target-locations.js";
|
|
15
|
+
import { getTargetLocations } from "../lib/app-config.js";
|
|
22
16
|
import { delay as randomDelay } from "../lib/delay.js";
|
|
23
17
|
import { discoverTags } from "../lib/tag-discover.js";
|
|
24
18
|
import {
|
|
@@ -33,26 +27,10 @@ import {
|
|
|
33
27
|
closeCaptcha,
|
|
34
28
|
} from "../scraper/modules/captcha-handler.js";
|
|
35
29
|
|
|
36
|
-
const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
|
|
37
30
|
const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
|
|
38
|
-
const DEFAULT_SCORE_COUNTRIES = [
|
|
39
|
-
"ES",
|
|
40
|
-
"FR",
|
|
41
|
-
"DE",
|
|
42
|
-
"PT",
|
|
43
|
-
"IT",
|
|
44
|
-
"NL",
|
|
45
|
-
"BE",
|
|
46
|
-
"AT",
|
|
47
|
-
"IE",
|
|
48
|
-
"PL",
|
|
49
|
-
"CZ",
|
|
50
|
-
"GR",
|
|
51
|
-
"HU",
|
|
52
|
-
];
|
|
53
31
|
|
|
54
32
|
function resolveTargetCountries(countries) {
|
|
55
|
-
return countries ||
|
|
33
|
+
return countries || getTargetLocations();
|
|
56
34
|
}
|
|
57
35
|
|
|
58
36
|
/**
|
|
@@ -73,38 +51,6 @@ function isBrowserDisconnected(browser, e) {
|
|
|
73
51
|
return false;
|
|
74
52
|
}
|
|
75
53
|
|
|
76
|
-
/**
|
|
77
|
-
* 探测端口是否已被监听(CDP /json 端点)。
|
|
78
|
-
* 与 cdp.js 的 checkCDPPort 等价,但 tag.js 独立实现,不依赖未导出的内部函数。
|
|
79
|
-
* 用于端口池挑选时跳过被占用端口、以及关闭旧浏览器后确认端口已释放。
|
|
80
|
-
*/
|
|
81
|
-
function checkPortListening(port) {
|
|
82
|
-
return new Promise((resolve) => {
|
|
83
|
-
const req = http.get(`http://127.0.0.1:${port}/json`, (res) => {
|
|
84
|
-
res.on("data", () => {});
|
|
85
|
-
res.on("end", () => resolve(res.statusCode === 200));
|
|
86
|
-
});
|
|
87
|
-
req.on("error", () => resolve(false));
|
|
88
|
-
req.setTimeout(2000, () => {
|
|
89
|
-
resolve(false);
|
|
90
|
-
req.destroy();
|
|
91
|
-
});
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* 等待端口不再被监听(浏览器进程真正退出、端口释放)。
|
|
97
|
-
* @returns {Promise<boolean>} 端口已释放返回 true
|
|
98
|
-
*/
|
|
99
|
-
async function waitForPortReleased(port, timeoutMs = 10000) {
|
|
100
|
-
const deadline = Date.now() + timeoutMs;
|
|
101
|
-
while (Date.now() < deadline) {
|
|
102
|
-
if (!(await checkPortListening(port))) return true;
|
|
103
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
104
|
-
}
|
|
105
|
-
return !(await checkPortListening(port));
|
|
106
|
-
}
|
|
107
|
-
|
|
108
54
|
// 构建带客户端追踪 header 的 fetch 封装
|
|
109
55
|
function buildClientHeaders(clientId, meta, extra = {}) {
|
|
110
56
|
return {
|
|
@@ -465,7 +411,7 @@ export async function handleDiscover(parsed) {
|
|
|
465
411
|
countries.length === 1 &&
|
|
466
412
|
countries[0].toUpperCase() === "ALL"
|
|
467
413
|
) {
|
|
468
|
-
countries =
|
|
414
|
+
countries = getTargetLocations();
|
|
469
415
|
}
|
|
470
416
|
|
|
471
417
|
if (!countries || countries.length === 0) {
|
|
@@ -826,9 +772,8 @@ export async function handleScoreAll(parsed) {
|
|
|
826
772
|
} = tagScoreAll || {};
|
|
827
773
|
|
|
828
774
|
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
let cdpPort = port || defaultScoreAllPort;
|
|
775
|
+
// 固定端口:一次会话一个浏览器,全程不换端口、不重启浏览器(错误只打印,不自愈)
|
|
776
|
+
const cdpPort = port || 7222;
|
|
832
777
|
const effectiveProxy = cliProxy || configuredProxy;
|
|
833
778
|
const targetCountries = resolveTargetCountries(countries);
|
|
834
779
|
|
|
@@ -881,62 +826,7 @@ export async function handleScoreAll(parsed) {
|
|
|
881
826
|
|
|
882
827
|
await setupPageRequestBlocking(page);
|
|
883
828
|
|
|
884
|
-
//
|
|
885
|
-
const usedPorts = new Set([cdpPort]);
|
|
886
|
-
|
|
887
|
-
// 端口池起始以 CLI --port / 环境变量 / 默认 7222 为准
|
|
888
|
-
const portPoolStart = cdpPort;
|
|
889
|
-
const portPoolSize = Math.max(
|
|
890
|
-
2,
|
|
891
|
-
parseInt(process.env.TAG_SCOREALL_PORT_POOL_SIZE || "10", 10) || 10,
|
|
892
|
-
);
|
|
893
|
-
|
|
894
|
-
async function pickNextPort(currentPort) {
|
|
895
|
-
const candidates = [];
|
|
896
|
-
for (let i = 0; i < portPoolSize; i++) {
|
|
897
|
-
const p = portPoolStart + i;
|
|
898
|
-
if (p !== currentPort) candidates.push(p);
|
|
899
|
-
}
|
|
900
|
-
// 跳过已被监听的端口(其他 score-all/attach 等程序的浏览器),
|
|
901
|
-
// 避免 attach 到别人的实例、后续 close/kill 时互相干扰导致浏览器多开
|
|
902
|
-
const freePorts = [];
|
|
903
|
-
for (const p of candidates) {
|
|
904
|
-
if (!(await checkPortListening(p))) freePorts.push(p);
|
|
905
|
-
}
|
|
906
|
-
const pool = freePorts.length > 0 ? freePorts : candidates;
|
|
907
|
-
if (pool.length === 0) return currentPort;
|
|
908
|
-
return pool[Math.floor(Math.random() * pool.length)];
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
// ── 内存回收配置 ──
|
|
912
|
-
const RECYCLE_CONFIG = {
|
|
913
|
-
nodeRssMb: Math.max(
|
|
914
|
-
256,
|
|
915
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_RSS_MB || "2000", 10) || 2000,
|
|
916
|
-
),
|
|
917
|
-
nodeHeapMb: Math.max(
|
|
918
|
-
128,
|
|
919
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "2000", 10) || 2000,
|
|
920
|
-
),
|
|
921
|
-
edgeMb: Math.max(
|
|
922
|
-
500,
|
|
923
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_EDGE_MB || "2000", 10) || 2000,
|
|
924
|
-
),
|
|
925
|
-
cooldownMs: Math.max(
|
|
926
|
-
0,
|
|
927
|
-
parseInt(process.env.TAG_SCOREALL_RECYCLE_COOLDOWN_MS || "180000", 10) ||
|
|
928
|
-
180000,
|
|
929
|
-
),
|
|
930
|
-
periodicEvery: Math.max(
|
|
931
|
-
0,
|
|
932
|
-
parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
|
|
933
|
-
),
|
|
934
|
-
switchPortOnRecycle:
|
|
935
|
-
String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "0") !== "0",
|
|
936
|
-
};
|
|
937
|
-
let lastRecycleAt = 0;
|
|
938
|
-
|
|
939
|
-
// ── 轻量回收:只清空当前 tab ──
|
|
829
|
+
// ── 轻量回收:只清空当前 tab(不重启浏览器) ──
|
|
940
830
|
async function recyclePage() {
|
|
941
831
|
if (!page || page.isClosed()) {
|
|
942
832
|
page = await getOrCreatePage(browser);
|
|
@@ -955,117 +845,29 @@ export async function handleScoreAll(parsed) {
|
|
|
955
845
|
}
|
|
956
846
|
}
|
|
957
847
|
|
|
958
|
-
// ── 深度回收:关闭浏览器 + 可选切换端口 ──
|
|
959
|
-
async function recycleCdpSession(reason, { switchPort = true } = {}) {
|
|
960
|
-
const oldPort = cdpPort;
|
|
961
|
-
const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
|
|
962
|
-
if (shouldSwitch) {
|
|
963
|
-
cdpPort = await pickNextPort(cdpPort);
|
|
964
|
-
cdpOpts.port = cdpPort;
|
|
965
|
-
clientMeta.port = cdpPort;
|
|
966
|
-
usedPorts.add(cdpPort);
|
|
967
|
-
}
|
|
968
|
-
const portHint =
|
|
969
|
-
oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
|
|
970
|
-
log(` ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
|
|
971
|
-
if (page) {
|
|
972
|
-
await page.close().catch(() => {});
|
|
973
|
-
page = null;
|
|
974
|
-
}
|
|
975
|
-
// CDP 模式下 browser.close() 只断开连接不杀进程,必须显式 kill 旧端口实例;
|
|
976
|
-
// kill 后确认端口已释放,再启动新实例,防止旧浏览器窗口残留(多开)
|
|
977
|
-
if (oldPort !== cdpPort) {
|
|
978
|
-
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
979
|
-
if (!(await waitForPortReleased(oldPort))) {
|
|
980
|
-
log(` ⚠️ 旧端口 ${oldPort} 未释放,再次尝试关闭...`);
|
|
981
|
-
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
982
|
-
if (!(await waitForPortReleased(oldPort, 5000))) {
|
|
983
|
-
log(
|
|
984
|
-
` ⚠️ 旧端口 ${oldPort} 仍未释放,继续使用新端口(旧浏览器窗口可能残留,退出时会统一清理)`,
|
|
985
|
-
);
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
await browser.close().catch(() => {});
|
|
990
|
-
browser = await ensureBrowserReady(cdpOpts);
|
|
991
|
-
page = await getOrCreatePage(browser);
|
|
992
|
-
await setupPageRequestBlocking(page);
|
|
993
|
-
lastRecycleAt = Date.now();
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
// ── 内存检查 + 自动回收 ──
|
|
997
|
-
// 策略:Node.js 内存超阈值 → 不换端口(只重建连接)
|
|
998
|
-
// Edge 内存超阈值 → 换端口(杀旧浏览器 + 启新实例,彻底释放)
|
|
999
|
-
// 定期轮次 → 不换端口
|
|
1000
|
-
async function maybeRecycleForMemory() {
|
|
1001
|
-
const mem = process.memoryUsage();
|
|
1002
|
-
const rssMB = mem.rss / 1024 / 1024;
|
|
1003
|
-
const heapMB = mem.heapUsed / 1024 / 1024;
|
|
1004
|
-
const edgeMB = getBrowserMemory(cdpPort);
|
|
1005
|
-
|
|
1006
|
-
const nodeOver =
|
|
1007
|
-
rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
|
|
1008
|
-
const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
|
|
1009
|
-
|
|
1010
|
-
if (nodeOver || edgeOver) {
|
|
1011
|
-
// 冷却期内跳过
|
|
1012
|
-
if (
|
|
1013
|
-
RECYCLE_CONFIG.cooldownMs > 0 &&
|
|
1014
|
-
Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
|
|
1015
|
-
)
|
|
1016
|
-
return;
|
|
1017
|
-
|
|
1018
|
-
const parts = [];
|
|
1019
|
-
if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
|
|
1020
|
-
parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
|
|
1021
|
-
if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
|
|
1022
|
-
parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
|
|
1023
|
-
if (edgeOver)
|
|
1024
|
-
parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
|
|
1025
|
-
|
|
1026
|
-
await recycleCdpSession(`mem ${parts.join("+")}`, {
|
|
1027
|
-
switchPort: edgeOver,
|
|
1028
|
-
});
|
|
1029
|
-
return;
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
// 定期轮次回收(默认关闭)
|
|
1033
|
-
if (
|
|
1034
|
-
RECYCLE_CONFIG.periodicEvery > 0 &&
|
|
1035
|
-
totalScored > 0 &&
|
|
1036
|
-
totalScored % RECYCLE_CONFIG.periodicEvery === 0
|
|
1037
|
-
) {
|
|
1038
|
-
await recycleCdpSession(`periodic #${totalScored}`, {
|
|
1039
|
-
switchPort: false,
|
|
1040
|
-
});
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
848
|
let totalScored = 0;
|
|
1045
849
|
let emptyRounds = 0; // 连续无任务的轮数
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
(parseInt(process.env.TAG_SCOREALL_403_RETRY_3 || "50", 10) || 50) * 1000,
|
|
1068
|
-
];
|
|
850
|
+
|
|
851
|
+
// ── 错误计数(纯观察,不做任何自愈) ──
|
|
852
|
+
// 阈值语义:连续超过 N 次触发动作(即第 N+1 次)
|
|
853
|
+
// 403 连续 >2 次 → 上报服务端 + 强制 fallback(daemon 换节点),清零后继续跑
|
|
854
|
+
// 网络错误 连续 >10 次 → 打印退出(服务端不可达时 fallback 请求也发不出去)
|
|
855
|
+
// 验证码 连续 >3 次(人工一直未解决)→ 打印退出
|
|
856
|
+
// 其他错误 连续 >10 次 → 打印退出
|
|
857
|
+
// 清零规则:成功处理一个 tag → 全部清零;fallback 触发 → 403 计数清零;
|
|
858
|
+
// 人工点掉验证码 → 验证码计数清零
|
|
859
|
+
const errorCounts = { http403: 0, network: 0, captcha: 0, other: 0 };
|
|
860
|
+
const ERROR_LIMITS = { http403: 2, network: 10, captcha: 3, other: 10 };
|
|
861
|
+
const resetAllErrorCounts = () => {
|
|
862
|
+
errorCounts.http403 = 0;
|
|
863
|
+
errorCounts.network = 0;
|
|
864
|
+
errorCounts.captcha = 0;
|
|
865
|
+
errorCounts.other = 0;
|
|
866
|
+
};
|
|
867
|
+
const bumpErrorCount = (type) => {
|
|
868
|
+
errorCounts[type]++;
|
|
869
|
+
return errorCounts[type] > ERROR_LIMITS[type];
|
|
870
|
+
};
|
|
1069
871
|
|
|
1070
872
|
// 生成客户端 ID,用于服务端追踪
|
|
1071
873
|
const clientId = randomUUID();
|
|
@@ -1098,14 +900,10 @@ export async function handleScoreAll(parsed) {
|
|
|
1098
900
|
if (enrichScraper) await enrichScraper.close().catch(() => {});
|
|
1099
901
|
log("正在关闭 CDP 浏览器...");
|
|
1100
902
|
try {
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
killEdgeProcesses(null, p),
|
|
1106
|
-
new Promise((r) => setTimeout(r, 5000)),
|
|
1107
|
-
]).catch(() => {});
|
|
1108
|
-
}
|
|
903
|
+
await Promise.race([
|
|
904
|
+
killEdgeProcesses(null, cdpPort),
|
|
905
|
+
new Promise((r) => setTimeout(r, 5000)),
|
|
906
|
+
]).catch(() => {});
|
|
1109
907
|
log("浏览器已关闭");
|
|
1110
908
|
} catch (e) {
|
|
1111
909
|
log(`关闭浏览器失败: ${e.message}`);
|
|
@@ -1134,43 +932,38 @@ export async function handleScoreAll(parsed) {
|
|
|
1134
932
|
return;
|
|
1135
933
|
}
|
|
1136
934
|
try {
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
{
|
|
1146
|
-
headers: buildClientHeaders(clientId, clientMeta),
|
|
1147
|
-
},
|
|
1148
|
-
);
|
|
1149
|
-
const tagsData = await tagsRes.json();
|
|
935
|
+
// 从服务端取下一个 new 标签(同时查剩余数量)
|
|
936
|
+
const tagsRes = await fetch(
|
|
937
|
+
`${baseUrl}/api/tags?status=new&limit=1&userId=${encodeURIComponent(userId)}`,
|
|
938
|
+
{
|
|
939
|
+
headers: buildClientHeaders(clientId, clientMeta),
|
|
940
|
+
},
|
|
941
|
+
);
|
|
942
|
+
const tagsData = await tagsRes.json();
|
|
1150
943
|
|
|
1151
|
-
|
|
1152
|
-
|
|
944
|
+
if (!tagsData.tags || tagsData.tags.length === 0) {
|
|
945
|
+
emptyRounds++;
|
|
1153
946
|
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
continue;
|
|
1162
|
-
}
|
|
1163
|
-
log(`⏳ 暂无待打分标签(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
1164
|
-
await new Promise((r) => setTimeout(r, 10000));
|
|
947
|
+
// 服务端定时发现会自动补充标签,客户端只需等待
|
|
948
|
+
if (emptyRounds >= 5) {
|
|
949
|
+
log(
|
|
950
|
+
`⏳ 连续 ${emptyRounds} 轮无标签,等待服务端发现补充(30 秒后重试)...`,
|
|
951
|
+
);
|
|
952
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
953
|
+
emptyRounds = 0;
|
|
1165
954
|
continue;
|
|
1166
955
|
}
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
tag = tagsData.tags[0].tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1171
|
-
currentTag = tag;
|
|
1172
|
-
tag403Attempts = 0;
|
|
956
|
+
log(`⏳ 暂无待打分标签(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
957
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
958
|
+
continue;
|
|
1173
959
|
}
|
|
960
|
+
|
|
961
|
+
// 有任务了,重置计数器
|
|
962
|
+
emptyRounds = 0;
|
|
963
|
+
const tag = tagsData.tags[0].tag
|
|
964
|
+
.replace(/^#+/, "")
|
|
965
|
+
.trim()
|
|
966
|
+
.toLowerCase();
|
|
1174
967
|
const startTime = Date.now();
|
|
1175
968
|
|
|
1176
969
|
log(`[${totalScored + 1}] 正在打分 #${tag} ...`);
|
|
@@ -1188,8 +981,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1188
981
|
};
|
|
1189
982
|
|
|
1190
983
|
// 锁定 tag(meta 中不放入 tag,避免非 ASCII 字符导致 header ByteString 报错)
|
|
1191
|
-
|
|
1192
|
-
if (!retrySameTag) {
|
|
984
|
+
{
|
|
1193
985
|
const claimRes = await fetch(
|
|
1194
986
|
`${baseUrl}/api/tags/claim?userId=${encodeURIComponent(userId)}`,
|
|
1195
987
|
{
|
|
@@ -1251,7 +1043,6 @@ export async function handleScoreAll(parsed) {
|
|
|
1251
1043
|
totalScored++;
|
|
1252
1044
|
await randomDelay(0, 5000);
|
|
1253
1045
|
await recyclePage();
|
|
1254
|
-
await maybeRecycleForMemory();
|
|
1255
1046
|
continue;
|
|
1256
1047
|
}
|
|
1257
1048
|
// 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径,避免误标 dead
|
|
@@ -1313,18 +1104,14 @@ export async function handleScoreAll(parsed) {
|
|
|
1313
1104
|
);
|
|
1314
1105
|
videos = enriched.videos;
|
|
1315
1106
|
|
|
1316
|
-
// CDN
|
|
1107
|
+
// CDN 限流:打印并计入其他错误计数(不自愈、不重启 scraper)
|
|
1317
1108
|
const cdnBlocked = enriched.cdnBlockedCount || 0;
|
|
1318
1109
|
if (cdnBlocked > 0) {
|
|
1319
1110
|
const cdnRatio = cdnBlocked / (videos.length || 1);
|
|
1320
|
-
|
|
1111
|
+
bumpErrorCount("other");
|
|
1321
1112
|
log(
|
|
1322
|
-
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)
|
|
1113
|
+
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),计入其他错误计数 (${errorCounts.other}/${ERROR_LIMITS.other})`,
|
|
1323
1114
|
);
|
|
1324
|
-
await new Promise((r) => setTimeout(r, coolSec * 1000));
|
|
1325
|
-
log(` 正在重启 TikTokScraper...`);
|
|
1326
|
-
await enrichScraper.restart();
|
|
1327
|
-
log(` ✅ TikTokScraper 已重启`);
|
|
1328
1115
|
}
|
|
1329
1116
|
}
|
|
1330
1117
|
|
|
@@ -1352,7 +1139,6 @@ export async function handleScoreAll(parsed) {
|
|
|
1352
1139
|
` ⏭️ 重试 ${COUNTRY_MAX_RETRIES} 次后国家识别率仍 ${(rate * 100).toFixed(0)}% < ${(COUNTRY_RATE_MIN * 100).toFixed(0)}%,跳过该标签,处理下一个`,
|
|
1353
1140
|
);
|
|
1354
1141
|
await recyclePage();
|
|
1355
|
-
await maybeRecycleForMemory();
|
|
1356
1142
|
await randomDelay(3000, 7000);
|
|
1357
1143
|
continue;
|
|
1358
1144
|
}
|
|
@@ -1397,10 +1183,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1397
1183
|
` ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? " " + mc : ""}${memStr}`,
|
|
1398
1184
|
);
|
|
1399
1185
|
log("");
|
|
1400
|
-
|
|
1401
|
-
retrySameTag = false; // 成功,不再重试
|
|
1402
|
-
tag403Attempts = 0;
|
|
1403
|
-
bannedCount = 0; // 成功 tag,重置连续判封计数
|
|
1186
|
+
resetAllErrorCounts(); // 成功一个 tag → 所有错误计数清零
|
|
1404
1187
|
// 每 10 个标签输出一次内存汇总
|
|
1405
1188
|
if (totalScored % 10 === 0) {
|
|
1406
1189
|
log(
|
|
@@ -1408,7 +1191,6 @@ export async function handleScoreAll(parsed) {
|
|
|
1408
1191
|
);
|
|
1409
1192
|
}
|
|
1410
1193
|
await recyclePage();
|
|
1411
|
-
await maybeRecycleForMemory();
|
|
1412
1194
|
await randomDelay(3000, 7000);
|
|
1413
1195
|
} catch (e) {
|
|
1414
1196
|
// 浏览器窗口被手动关闭(CDP 断开)→ 自动退出,避免无限重试循环
|
|
@@ -1427,7 +1209,17 @@ export async function handleScoreAll(parsed) {
|
|
|
1427
1209
|
e.message.includes("fetch failed") ||
|
|
1428
1210
|
e.message.includes("network")));
|
|
1429
1211
|
if (isNetworkError) {
|
|
1430
|
-
|
|
1212
|
+
const over = bumpErrorCount("network");
|
|
1213
|
+
log(
|
|
1214
|
+
` ⚠️ 服务端连接失败 (${e.message}),15 秒后重试(连续网络错误 ${errorCounts.network} 次)...`,
|
|
1215
|
+
);
|
|
1216
|
+
if (over) {
|
|
1217
|
+
log(
|
|
1218
|
+
` ❌ 连续网络错误超过 ${ERROR_LIMITS.network} 次 → 打印并退出(服务端不可达,fallback 发不出去)`,
|
|
1219
|
+
);
|
|
1220
|
+
await cleanup();
|
|
1221
|
+
return;
|
|
1222
|
+
}
|
|
1431
1223
|
await new Promise((r) => setTimeout(r, 15000));
|
|
1432
1224
|
continue;
|
|
1433
1225
|
}
|
|
@@ -1441,35 +1233,16 @@ export async function handleScoreAll(parsed) {
|
|
|
1441
1233
|
e.message &&
|
|
1442
1234
|
(e.message.includes("验证码") || e.message.includes("captcha"));
|
|
1443
1235
|
|
|
1444
|
-
// ── 403
|
|
1445
|
-
//
|
|
1236
|
+
// ── 403:当前 tag 不上报(留在服务端 scoring 状态超时回收),
|
|
1237
|
+
// 只计数;连续 >2 次上报服务端 + 强制 fallback(daemon 换节点),
|
|
1238
|
+
// 计数清零后继续跑 ──
|
|
1446
1239
|
if (is403) {
|
|
1447
|
-
const
|
|
1448
|
-
|
|
1449
|
-
if (
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
log(
|
|
1454
|
-
` 🚫 HTTP 403,判定被封,重启浏览器后 ${waitMs / 1000}s 第 ${tag403Attempts}/${BAN_RETRY_WAITS.length} 次重试 #${currentTag} ...`,
|
|
1455
|
-
);
|
|
1456
|
-
await recycleCdpSession(`首次被封`, { switchPort: false });
|
|
1457
|
-
} else {
|
|
1458
|
-
log(
|
|
1459
|
-
` 🚫 HTTP 403,判定被封,${waitMs / 1000}s 后第 ${tag403Attempts}/${BAN_RETRY_WAITS.length} 次重试 #${currentTag} ...`,
|
|
1460
|
-
);
|
|
1461
|
-
await recyclePage();
|
|
1462
|
-
}
|
|
1463
|
-
await new Promise((r) => setTimeout(r, waitMs));
|
|
1464
|
-
continue;
|
|
1465
|
-
}
|
|
1466
|
-
// 重试耗尽:判定“被封”
|
|
1467
|
-
bannedCount++;
|
|
1468
|
-
log(
|
|
1469
|
-
` 🚫 30/40/50s 重试 ${BAN_RETRY_WAITS.length} 次仍 403,判定账号被封(连续判封 ${bannedCount} 次)`,
|
|
1470
|
-
);
|
|
1471
|
-
if (bannedCount >= 2) {
|
|
1472
|
-
log(` ⚠️ 连续两次判封 → 汇报服务端 + 触发 daemon fallback`);
|
|
1240
|
+
const over = bumpErrorCount("http403");
|
|
1241
|
+
log(` 🚫 HTTP 403(连续第 ${errorCounts.http403} 次)`);
|
|
1242
|
+
if (over) {
|
|
1243
|
+
log(
|
|
1244
|
+
` ⚠️ 连续 403 超过 ${ERROR_LIMITS.http403} 次 → 上报服务端 + 触发 daemon fallback`,
|
|
1245
|
+
);
|
|
1473
1246
|
await reportClientErrorAndFailover({
|
|
1474
1247
|
post: (url, body) =>
|
|
1475
1248
|
fetch(url, {
|
|
@@ -1482,16 +1255,13 @@ export async function handleScoreAll(parsed) {
|
|
|
1482
1255
|
baseUrl,
|
|
1483
1256
|
userId,
|
|
1484
1257
|
errorType: "consecutive_blocked",
|
|
1485
|
-
message:
|
|
1258
|
+
message: `连续 ${ERROR_LIMITS.http403 + 1} 次 403 (port=${cdpPort})`,
|
|
1486
1259
|
stage: "tag-scoring",
|
|
1487
1260
|
log,
|
|
1488
1261
|
});
|
|
1489
|
-
|
|
1262
|
+
errorCounts.http403 = 0; // fallback 触发后清零
|
|
1490
1263
|
}
|
|
1491
|
-
retrySameTag = false;
|
|
1492
|
-
tag403Attempts = 0;
|
|
1493
1264
|
await recyclePage();
|
|
1494
|
-
await maybeRecycleForMemory();
|
|
1495
1265
|
await randomDelay(3000, 7000);
|
|
1496
1266
|
continue;
|
|
1497
1267
|
}
|
|
@@ -1500,9 +1270,9 @@ export async function handleScoreAll(parsed) {
|
|
|
1500
1270
|
log(` ❌ 失败: ${e.message}`);
|
|
1501
1271
|
|
|
1502
1272
|
if (isCaptcha) {
|
|
1503
|
-
|
|
1273
|
+
bumpErrorCount("captcha");
|
|
1504
1274
|
log(
|
|
1505
|
-
` 🔐
|
|
1275
|
+
` 🔐 验证码(连续第 ${errorCounts.captcha} 次),等待用户手动解决...`,
|
|
1506
1276
|
);
|
|
1507
1277
|
|
|
1508
1278
|
// 上报验证码错误
|
|
@@ -1525,24 +1295,31 @@ export async function handleScoreAll(parsed) {
|
|
|
1525
1295
|
} catch {}
|
|
1526
1296
|
|
|
1527
1297
|
// 等待用户手动解决验证码(最多 3 分钟)
|
|
1298
|
+
let captchaResolved = false;
|
|
1528
1299
|
try {
|
|
1529
1300
|
const captchaResult = await waitAndGetCaptcha(page, {
|
|
1530
1301
|
waitMs: 180000,
|
|
1531
1302
|
pollInterval: 5000,
|
|
1532
1303
|
log: () => {},
|
|
1533
1304
|
});
|
|
1305
|
+
captchaResolved = !!captchaResult.resolved;
|
|
1534
1306
|
if (captchaResult.detected && !captchaResult.resolved) {
|
|
1535
1307
|
await closeCaptcha(page).catch(() => {});
|
|
1536
1308
|
}
|
|
1537
1309
|
} catch {}
|
|
1538
1310
|
|
|
1539
|
-
if (
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1311
|
+
if (captchaResolved) {
|
|
1312
|
+
// 人工点掉验证码 → 验证码计数清零
|
|
1313
|
+
log(` ✅ 验证码已被人工处理,验证码计数清零`);
|
|
1314
|
+
errorCounts.captcha = 0;
|
|
1315
|
+
} else if (errorCounts.captcha > ERROR_LIMITS.captcha) {
|
|
1316
|
+
log(
|
|
1317
|
+
` ❌ 连续验证码超过 ${ERROR_LIMITS.captcha} 次且未解决 → 打印并退出`,
|
|
1318
|
+
);
|
|
1319
|
+
await cleanup();
|
|
1320
|
+
return;
|
|
1543
1321
|
}
|
|
1544
|
-
|
|
1545
|
-
consecutiveCaptcha = 0;
|
|
1322
|
+
continue;
|
|
1546
1323
|
}
|
|
1547
1324
|
|
|
1548
1325
|
try {
|
|
@@ -1561,34 +1338,14 @@ export async function handleScoreAll(parsed) {
|
|
|
1561
1338
|
);
|
|
1562
1339
|
} catch {}
|
|
1563
1340
|
|
|
1564
|
-
// 连续任务错误达到阈值,上报服务端 + 触发 daemon fallback 换节点
|
|
1565
|
-
consecutiveTaskErrors++;
|
|
1566
|
-
if (consecutiveTaskErrors >= CONSECUTIVE_TASK_ERRORS_MAX) {
|
|
1567
|
-
log(
|
|
1568
|
-
` ⚠️ 连续任务错误 ${consecutiveTaskErrors}/${CONSECUTIVE_TASK_ERRORS_MAX} → 汇报服务端 + 触发 fallback`,
|
|
1569
|
-
);
|
|
1570
|
-
await reportConsecutiveTaskErrors({
|
|
1571
|
-
post: (url, body) =>
|
|
1572
|
-
fetch(url, {
|
|
1573
|
-
method: "POST",
|
|
1574
|
-
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1575
|
-
"Content-Type": "application/json",
|
|
1576
|
-
}),
|
|
1577
|
-
body: JSON.stringify(body),
|
|
1578
|
-
}),
|
|
1579
|
-
baseUrl,
|
|
1580
|
-
userId,
|
|
1581
|
-
message: `连续 ${consecutiveTaskErrors} 个标签处理失败 (port=${cdpPort})`,
|
|
1582
|
-
stage: "tag-scoring",
|
|
1583
|
-
log,
|
|
1584
|
-
});
|
|
1585
|
-
consecutiveTaskErrors = 0;
|
|
1586
|
-
}
|
|
1587
1341
|
totalScored++;
|
|
1588
|
-
|
|
1589
|
-
|
|
1342
|
+
const over = bumpErrorCount("other");
|
|
1343
|
+
if (over) {
|
|
1344
|
+
log(` ❌ 连续其他错误超过 ${ERROR_LIMITS.other} 次 → 打印并退出`);
|
|
1345
|
+
await cleanup();
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1590
1348
|
await recyclePage();
|
|
1591
|
-
await maybeRecycleForMemory();
|
|
1592
1349
|
}
|
|
1593
1350
|
}
|
|
1594
1351
|
} finally {
|
|
@@ -1649,7 +1406,7 @@ export async function handleTag(parsed) {
|
|
|
1649
1406
|
.split(",")
|
|
1650
1407
|
.map((s) => s.trim().toUpperCase())
|
|
1651
1408
|
.filter(Boolean)
|
|
1652
|
-
:
|
|
1409
|
+
: getTargetLocations();
|
|
1653
1410
|
|
|
1654
1411
|
const autoEnrich = enrich || !!discover;
|
|
1655
1412
|
|
|
@@ -2129,7 +1886,7 @@ export async function handleTagRefresh(parsed) {
|
|
|
2129
1886
|
}
|
|
2130
1887
|
|
|
2131
1888
|
// 过滤目标国家的视频和作者
|
|
2132
|
-
const targetLocations =
|
|
1889
|
+
const targetLocations = getTargetLocations();
|
|
2133
1890
|
const filteredVideos = videos.filter((v) =>
|
|
2134
1891
|
isLocationInList(v.locationCreated, targetLocations),
|
|
2135
1892
|
);
|