tt-help-cli-ycl 1.4.136 → 1.4.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/test-webserver-error-classify.mjs +379 -0
- package/src/clash/checker.js +14 -2
- package/src/clash/daemon.js +7 -1
- package/src/cli/comments.js +40 -125
- package/src/cli/explore.js +24 -0
- package/src/cli/tag.js +261 -210
- package/src/cli/webserver.js +6 -0
- package/src/lib/api-interceptor-comment.js +14 -2
- package/src/lib/args.js +24 -0
- package/src/lib/browser/page.js +34 -6
- package/src/lib/browser-memory.js +95 -0
- package/src/lib/client-error-report.js +82 -0
- package/src/lib/constants.js +4 -0
- package/src/lib/parse-ssr.mjs +13 -1
- package/src/lib/port.js +72 -0
- package/src/lib/tiktok-scraper.mjs +104 -4
- package/src/watch/public/app.js +67 -23
- package/src/watch/public/style.css +18 -0
- package/src/webserver/server.mjs +354 -39
package/src/cli/tag.js
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
-
import { writeFileSync
|
|
1
|
+
import { writeFileSync } from "fs";
|
|
2
2
|
import { randomUUID } from "crypto";
|
|
3
|
-
import { execSync } from "child_process";
|
|
4
3
|
import http from "http";
|
|
5
|
-
import os from "os";
|
|
6
|
-
import path from "path";
|
|
7
4
|
import readline from "readline";
|
|
5
|
+
import {
|
|
6
|
+
getBrowserMemory,
|
|
7
|
+
formatMemoryWithBrowser,
|
|
8
|
+
} from "../lib/browser-memory.js";
|
|
9
|
+
import {
|
|
10
|
+
reportConsecutiveTaskErrors,
|
|
11
|
+
reportClientErrorAndFailover,
|
|
12
|
+
} from "../lib/client-error-report.js";
|
|
8
13
|
import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
|
|
9
14
|
import { killEdgeProcesses, ensureBrowserReady } from "../lib/browser/cdp.js";
|
|
10
15
|
import { getOrCreatePage, isBrowserClosedError } from "../lib/browser/page.js";
|
|
@@ -99,92 +104,6 @@ async function waitForPortReleased(port, timeoutMs = 10000) {
|
|
|
99
104
|
return !(await checkPortListening(port));
|
|
100
105
|
}
|
|
101
106
|
|
|
102
|
-
function formatMemoryUsage(mem = process.memoryUsage()) {
|
|
103
|
-
return `rss:${(mem.rss / 1024 / 1024).toFixed(0)}MB heap:${(mem.heapUsed / 1024 / 1024).toFixed(0)}MB ext:${(mem.external / 1024 / 1024).toFixed(0)}MB ab:${(mem.arrayBuffers / 1024 / 1024).toFixed(0)}MB`;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/**
|
|
107
|
-
* 获取 Edge 浏览器进程的内存占用(macOS/Linux)
|
|
108
|
-
*/
|
|
109
|
-
function getBrowserMemory(port = null) {
|
|
110
|
-
try {
|
|
111
|
-
const platform = os.platform();
|
|
112
|
-
let cmd;
|
|
113
|
-
if (platform === "darwin") {
|
|
114
|
-
// 按端口筛选:匹配 --remote-debugging-port=PORT 的进程
|
|
115
|
-
if (port) {
|
|
116
|
-
cmd = `ps aux | grep "[M]icrosoft Edge" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
|
|
117
|
-
} else {
|
|
118
|
-
cmd =
|
|
119
|
-
'ps aux | grep "[M]icrosoft Edge" | awk \'{sum += $6} END {printf "%d", sum/1024}\'';
|
|
120
|
-
}
|
|
121
|
-
} else if (platform === "linux") {
|
|
122
|
-
if (port) {
|
|
123
|
-
cmd = `ps aux | grep "[m]sedge" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
|
|
124
|
-
} else {
|
|
125
|
-
cmd =
|
|
126
|
-
'ps aux | grep "[m]sedge" | awk \'{sum += $6} END {printf "%d", sum/1024}\'';
|
|
127
|
-
}
|
|
128
|
-
} else if (platform === "win32") {
|
|
129
|
-
// Windows: 用 PowerShell 查 msedge 进程内存(WorkingSet64 → MB)
|
|
130
|
-
// 用临时 .ps1 文件避免命令行转义问题
|
|
131
|
-
const ps1Path = path.join(os.tmpdir(), `edge-mem-${Date.now()}.ps1`);
|
|
132
|
-
try {
|
|
133
|
-
let ps1Content;
|
|
134
|
-
if (port) {
|
|
135
|
-
ps1Content =
|
|
136
|
-
`$procs = Get-CimInstance Win32_Process -Filter "Name LIKE 'msedge%'" -ErrorAction SilentlyContinue; ` +
|
|
137
|
-
`$sum = 0; ` +
|
|
138
|
-
`foreach ($p in $procs) { ` +
|
|
139
|
-
` if ($p.CommandLine -and $p.CommandLine -like "*--remote-debugging-port=${port}*") { ` +
|
|
140
|
-
` $sum += $p.WorkingSetSize ` +
|
|
141
|
-
` } ` +
|
|
142
|
-
`}; ` +
|
|
143
|
-
`Write-Output $sum`;
|
|
144
|
-
} else {
|
|
145
|
-
// 无端口场景:Get-Process(.NET API,不依赖 WMI)统计全部 msedge 工作集,
|
|
146
|
-
// 与原 Get-CimInstance 全量求和行为等价;WMI 损坏/禁用的机器也能返回
|
|
147
|
-
ps1Content =
|
|
148
|
-
`$procs = Get-Process msedge -ErrorAction SilentlyContinue; ` +
|
|
149
|
-
`$sum = 0; ` +
|
|
150
|
-
`foreach ($p in $procs) { $sum += $p.WorkingSet64 }; ` +
|
|
151
|
-
`Write-Output $sum`;
|
|
152
|
-
}
|
|
153
|
-
writeFileSync(ps1Path, ps1Content);
|
|
154
|
-
cmd = `powershell.exe -NoProfile -ExecutionPolicy Bypass -File "${ps1Path}"`;
|
|
155
|
-
const result = execSync(cmd, { timeout: 5000 }).toString().trim();
|
|
156
|
-
try {
|
|
157
|
-
unlinkSync(ps1Path);
|
|
158
|
-
} catch {}
|
|
159
|
-
// PowerShell 返回 WorkingSetSize(字节),需除以 1048576 得 MB
|
|
160
|
-
const bytes = parseInt(result, 10);
|
|
161
|
-
const mb = bytes > 0 ? Math.round(bytes / 1048576) : 0;
|
|
162
|
-
return mb > 0 ? mb : null;
|
|
163
|
-
} catch {
|
|
164
|
-
try {
|
|
165
|
-
unlinkSync(ps1Path);
|
|
166
|
-
} catch {}
|
|
167
|
-
return null;
|
|
168
|
-
}
|
|
169
|
-
} else {
|
|
170
|
-
return null;
|
|
171
|
-
}
|
|
172
|
-
// macOS/Linux: ps RSS 是 KB,awk 已除以 1024 得 MB
|
|
173
|
-
const result = execSync(cmd, { timeout: 5000 }).toString().trim();
|
|
174
|
-
const mb = parseInt(result, 10);
|
|
175
|
-
return mb > 0 ? mb : null;
|
|
176
|
-
} catch {
|
|
177
|
-
return null;
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
function formatMemoryWithBrowser(port = null) {
|
|
182
|
-
const nodeMem = formatMemoryUsage();
|
|
183
|
-
const browserMB = getBrowserMemory(port);
|
|
184
|
-
const portLabel = port ? `:${port}` : "";
|
|
185
|
-
return browserMB ? `${nodeMem} edge${portLabel}:${browserMB}MB` : nodeMem;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
107
|
// 构建带客户端追踪 header 的 fetch 封装
|
|
189
108
|
function buildClientHeaders(clientId, meta, extra = {}) {
|
|
190
109
|
return {
|
|
@@ -266,9 +185,10 @@ async function enrichWithServerCache(
|
|
|
266
185
|
}
|
|
267
186
|
}
|
|
268
187
|
|
|
269
|
-
// Step 3:
|
|
188
|
+
// Step 3: 给已知国家的视频赋值,收集未知视频(已有国家的跳过,避免重复 enrich)
|
|
270
189
|
const unknownVideos = [];
|
|
271
190
|
for (const v of videos) {
|
|
191
|
+
if (v.locationCreated) continue; // 已识别到国家,不再重复补
|
|
272
192
|
const loc = knownLocations[v.authorUniqueId];
|
|
273
193
|
if (loc) {
|
|
274
194
|
v.locationCreated = loc;
|
|
@@ -1011,7 +931,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1011
931
|
parseInt(process.env.TAG_SCOREALL_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
|
|
1012
932
|
),
|
|
1013
933
|
switchPortOnRecycle:
|
|
1014
|
-
String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "
|
|
934
|
+
String(process.env.TAG_SCOREALL_SWITCH_PORT_ON_RECYCLE || "0") !== "0",
|
|
1015
935
|
};
|
|
1016
936
|
let lastRecycleAt = 0;
|
|
1017
937
|
|
|
@@ -1124,8 +1044,26 @@ export async function handleScoreAll(parsed) {
|
|
|
1124
1044
|
let emptyRounds = 0; // 连续无任务的轮数
|
|
1125
1045
|
let consecutiveCaptcha = 0;
|
|
1126
1046
|
const MAX_CONSECUTIVE_CAPTCHA = 3;
|
|
1127
|
-
let
|
|
1128
|
-
const
|
|
1047
|
+
let consecutiveTaskErrors = 0; // 连续任务处理错误计数(达阈值后上报服务端)
|
|
1048
|
+
const CONSECUTIVE_TASK_ERRORS_MAX = Math.max(
|
|
1049
|
+
1,
|
|
1050
|
+
parseInt(
|
|
1051
|
+
process.env.TAG_SCOREALL_CONSECUTIVE_TASK_ERRORS_MAX || "20",
|
|
1052
|
+
10,
|
|
1053
|
+
) || 20,
|
|
1054
|
+
); // 连续任务错误阈值,达到后上报 /api/error-report
|
|
1055
|
+
|
|
1056
|
+
// ── 403 判封 + 重试机制 ──
|
|
1057
|
+
let currentTag = null; // 当前正在处理的 tag(403 重试时复用,不消耗下一个)
|
|
1058
|
+
let retrySameTag = false; // 是否重试同一个 tag
|
|
1059
|
+
let tag403Attempts = 0; // 当前 tag 已发生 403 的次数
|
|
1060
|
+
let bannedCount = 0; // 连续判定“被封”的次数(成功 tag 清零)
|
|
1061
|
+
// 按判封次数递增的重试预算:第 1 次判封前重试 30、第 2 次 40、第 3 次及以后 50
|
|
1062
|
+
const BAN_RETRY_BUDGETS = [
|
|
1063
|
+
parseInt(process.env.TAG_SCOREALL_403_RETRY_1 || "30", 10) || 30,
|
|
1064
|
+
parseInt(process.env.TAG_SCOREALL_403_RETRY_2 || "40", 10) || 40,
|
|
1065
|
+
parseInt(process.env.TAG_SCOREALL_403_RETRY_3 || "50", 10) || 50,
|
|
1066
|
+
];
|
|
1129
1067
|
|
|
1130
1068
|
// 生成客户端 ID,用于服务端追踪
|
|
1131
1069
|
const clientId = randomUUID();
|
|
@@ -1194,39 +1132,43 @@ export async function handleScoreAll(parsed) {
|
|
|
1194
1132
|
return;
|
|
1195
1133
|
}
|
|
1196
1134
|
try {
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
emptyRounds
|
|
1135
|
+
let tag;
|
|
1136
|
+
if (retrySameTag) {
|
|
1137
|
+
// 403 重试:复用当前 tag,不消耗队列里的下一个
|
|
1138
|
+
tag = currentTag;
|
|
1139
|
+
} else {
|
|
1140
|
+
// 从服务端取下一个 new 标签(同时查剩余数量)
|
|
1141
|
+
const tagsRes = await fetch(
|
|
1142
|
+
`${baseUrl}/api/tags?status=new&limit=1&userId=${encodeURIComponent(userId)}`,
|
|
1143
|
+
{
|
|
1144
|
+
headers: buildClientHeaders(clientId, clientMeta),
|
|
1145
|
+
},
|
|
1146
|
+
);
|
|
1147
|
+
const tagsData = await tagsRes.json();
|
|
1148
|
+
|
|
1149
|
+
if (!tagsData.tags || tagsData.tags.length === 0) {
|
|
1150
|
+
emptyRounds++;
|
|
1151
|
+
|
|
1152
|
+
// 服务端定时发现会自动补充标签,客户端只需等待
|
|
1153
|
+
if (emptyRounds >= 5) {
|
|
1154
|
+
log(
|
|
1155
|
+
`⏳ 连续 ${emptyRounds} 轮无标签,等待服务端发现补充(30 秒后重试)...`,
|
|
1156
|
+
);
|
|
1157
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
1158
|
+
emptyRounds = 0;
|
|
1159
|
+
continue;
|
|
1160
|
+
}
|
|
1161
|
+
log(`⏳ 暂无待打分标签(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
1162
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1216
1163
|
continue;
|
|
1217
1164
|
}
|
|
1218
|
-
log(`⏳ 暂无待打分标签(连续 ${emptyRounds} 轮),10 秒后重试...`);
|
|
1219
|
-
await new Promise((r) => setTimeout(r, 10000));
|
|
1220
|
-
continue;
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
// 有任务了,重置计数器
|
|
1224
|
-
emptyRounds = 0;
|
|
1225
1165
|
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
.trim()
|
|
1229
|
-
|
|
1166
|
+
// 有任务了,重置计数器
|
|
1167
|
+
emptyRounds = 0;
|
|
1168
|
+
tag = tagsData.tags[0].tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1169
|
+
currentTag = tag;
|
|
1170
|
+
tag403Attempts = 0;
|
|
1171
|
+
}
|
|
1230
1172
|
const startTime = Date.now();
|
|
1231
1173
|
|
|
1232
1174
|
log(`[${totalScored + 1}] 正在打分 #${tag} ...`);
|
|
@@ -1244,31 +1186,37 @@ export async function handleScoreAll(parsed) {
|
|
|
1244
1186
|
};
|
|
1245
1187
|
|
|
1246
1188
|
// 锁定 tag(meta 中不放入 tag,避免非 ASCII 字符导致 header ByteString 报错)
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
if (claimData.
|
|
1261
|
-
|
|
1189
|
+
// 403 重试时 tag 已被本客户端锁定,跳过 claim,直接复用
|
|
1190
|
+
if (!retrySameTag) {
|
|
1191
|
+
const claimRes = await fetch(
|
|
1192
|
+
`${baseUrl}/api/tags/claim?userId=${encodeURIComponent(userId)}`,
|
|
1193
|
+
{
|
|
1194
|
+
method: "POST",
|
|
1195
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1196
|
+
"Content-Type": "application/json",
|
|
1197
|
+
}),
|
|
1198
|
+
body: JSON.stringify({ tag }),
|
|
1199
|
+
},
|
|
1200
|
+
);
|
|
1201
|
+
const claimData = await claimRes.json();
|
|
1202
|
+
if (!claimData.ok) {
|
|
1203
|
+
// already claimed: 其他机器抢先了,跳过不标 dead
|
|
1204
|
+
if (
|
|
1205
|
+
claimData.error &&
|
|
1206
|
+
claimData.error.includes("already claimed")
|
|
1207
|
+
) {
|
|
1208
|
+
log(` ⏭️ 已被其他客户端锁定,跳过`);
|
|
1209
|
+
await recyclePage();
|
|
1210
|
+
continue;
|
|
1211
|
+
}
|
|
1212
|
+
log(` ⚠️ 无法锁定 (${claimData.error}),标记为 dead 并跳过`);
|
|
1213
|
+
result.error = claimData.error;
|
|
1214
|
+
result.status = "dead";
|
|
1215
|
+
await reportToServer(baseUrl, result, clientId, clientMeta, userId);
|
|
1216
|
+
totalScored++;
|
|
1262
1217
|
await recyclePage();
|
|
1263
1218
|
continue;
|
|
1264
1219
|
}
|
|
1265
|
-
log(` ⚠️ 无法锁定 (${claimData.error}),标记为 dead 并跳过`);
|
|
1266
|
-
result.error = claimData.error;
|
|
1267
|
-
result.status = "dead";
|
|
1268
|
-
await reportToServer(baseUrl, result, clientId, clientMeta, userId);
|
|
1269
|
-
totalScored++;
|
|
1270
|
-
await recyclePage();
|
|
1271
|
-
continue;
|
|
1272
1220
|
}
|
|
1273
1221
|
|
|
1274
1222
|
// 抓取视频(CDP 连接已登录 Edge)
|
|
@@ -1309,55 +1257,102 @@ export async function handleScoreAll(parsed) {
|
|
|
1309
1257
|
throw new Error("no videos returned (network/captcha/slow-load)");
|
|
1310
1258
|
}
|
|
1311
1259
|
|
|
1312
|
-
//
|
|
1313
|
-
|
|
1260
|
+
// 国家识别率阈值:低于则等待后重新识别国家(复用同一批视频,不重抓列表)
|
|
1261
|
+
const COUNTRY_RATE_MIN =
|
|
1262
|
+
parseFloat(process.env.TAG_SCOREALL_MIN_COUNTRY_RATE || "0.8") || 0.8;
|
|
1263
|
+
const COUNTRY_RETRY_WAIT_MS =
|
|
1264
|
+
parseInt(
|
|
1265
|
+
process.env.TAG_SCOREALL_COUNTRY_RETRY_WAIT_MS || "120000",
|
|
1266
|
+
10,
|
|
1267
|
+
) || 120000;
|
|
1268
|
+
const COUNTRY_MAX_RETRIES = Math.max(
|
|
1269
|
+
0,
|
|
1270
|
+
parseInt(process.env.TAG_SCOREALL_COUNTRY_MAX_RETRIES || "2", 10) ||
|
|
1271
|
+
2,
|
|
1272
|
+
);
|
|
1273
|
+
|
|
1274
|
+
let countryRetries = 0;
|
|
1275
|
+
let rate = 0;
|
|
1276
|
+
while (true) {
|
|
1277
|
+
// enrich: webserver API 或本地 HTTP view-source
|
|
1278
|
+
if (webserver) {
|
|
1279
|
+
// 只补尚未识别到国家的视频,已有国家的不重复查
|
|
1280
|
+
const pendingVideos = videos.filter((v) => !v.locationCreated);
|
|
1281
|
+
if (pendingVideos.length > 0) {
|
|
1282
|
+
log(
|
|
1283
|
+
` ⏳ 补充国家信息 (${pendingVideos.length} 个待补视频, via webserver ${webserver})...`,
|
|
1284
|
+
);
|
|
1285
|
+
await enrichViaWebserver(
|
|
1286
|
+
pendingVideos,
|
|
1287
|
+
webserver,
|
|
1288
|
+
({ done, total, current, locationCreated }) => {
|
|
1289
|
+
if (done % 10 === 0 || done === total) {
|
|
1290
|
+
const name = current.split("/").pop().slice(0, 20);
|
|
1291
|
+
process.stderr.write(
|
|
1292
|
+
`\r [${done}/${total}] ${name} → ${locationCreated || "-"}\x1b[K`,
|
|
1293
|
+
);
|
|
1294
|
+
}
|
|
1295
|
+
},
|
|
1296
|
+
);
|
|
1297
|
+
process.stderr.write("\n");
|
|
1298
|
+
}
|
|
1299
|
+
} else {
|
|
1300
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1301
|
+
const enriched = await enrichWithServerCache(
|
|
1302
|
+
videos,
|
|
1303
|
+
baseUrl,
|
|
1304
|
+
enrichScraper,
|
|
1305
|
+
{
|
|
1306
|
+
clientId,
|
|
1307
|
+
clientMeta,
|
|
1308
|
+
effectiveProxy,
|
|
1309
|
+
log,
|
|
1310
|
+
},
|
|
1311
|
+
);
|
|
1312
|
+
videos = enriched.videos;
|
|
1313
|
+
|
|
1314
|
+
// CDN 限流检测:有拦截则冷却 + 重启 scraper
|
|
1315
|
+
const cdnBlocked = enriched.cdnBlockedCount || 0;
|
|
1316
|
+
if (cdnBlocked > 0) {
|
|
1317
|
+
const cdnRatio = cdnBlocked / (videos.length || 1);
|
|
1318
|
+
const coolSec = cdnRatio > 0.3 ? 120 : 60;
|
|
1319
|
+
log(
|
|
1320
|
+
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),冷却 ${coolSec} 秒后重启 scraper`,
|
|
1321
|
+
);
|
|
1322
|
+
await new Promise((r) => setTimeout(r, coolSec * 1000));
|
|
1323
|
+
log(` 正在重启 TikTokScraper...`);
|
|
1324
|
+
await enrichScraper.restart();
|
|
1325
|
+
log(` ✅ TikTokScraper 已重启`);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// 统计国家信息获取情况:多少视频真实拿到国家,多少没有
|
|
1330
|
+
const withLoc = videos.filter((v) => v.locationCreated).length;
|
|
1331
|
+
const withoutLoc = videos.length - withLoc;
|
|
1332
|
+
rate = videos.length > 0 ? withLoc / videos.length : 0;
|
|
1314
1333
|
log(
|
|
1315
|
-
`
|
|
1316
|
-
);
|
|
1317
|
-
await enrichViaWebserver(
|
|
1318
|
-
videos,
|
|
1319
|
-
webserver,
|
|
1320
|
-
({ done, total, current, locationCreated }) => {
|
|
1321
|
-
if (done % 10 === 0 || done === total) {
|
|
1322
|
-
const name = current.split("/").pop().slice(0, 20);
|
|
1323
|
-
process.stderr.write(
|
|
1324
|
-
`\r [${done}/${total}] ${name} → ${locationCreated || "-"}\x1b[K`,
|
|
1325
|
-
);
|
|
1326
|
-
}
|
|
1327
|
-
},
|
|
1334
|
+
` ✓ 国家信息: ${withLoc} 个有国家 / ${withoutLoc} 个未获取到(共 ${videos.length},识别率 ${(rate * 100).toFixed(0)}%)`,
|
|
1328
1335
|
);
|
|
1329
|
-
|
|
1336
|
+
|
|
1337
|
+
if (rate >= COUNTRY_RATE_MIN) break;
|
|
1338
|
+
if (countryRetries >= COUNTRY_MAX_RETRIES) break;
|
|
1339
|
+
|
|
1340
|
+
countryRetries++;
|
|
1330
1341
|
log(
|
|
1331
|
-
`
|
|
1332
|
-
);
|
|
1333
|
-
} else {
|
|
1334
|
-
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1335
|
-
const enriched = await enrichWithServerCache(
|
|
1336
|
-
videos,
|
|
1337
|
-
baseUrl,
|
|
1338
|
-
enrichScraper,
|
|
1339
|
-
{
|
|
1340
|
-
clientId,
|
|
1341
|
-
clientMeta,
|
|
1342
|
-
effectiveProxy,
|
|
1343
|
-
log,
|
|
1344
|
-
},
|
|
1342
|
+
` ⚠️ 国家识别率 ${(rate * 100).toFixed(0)}% < ${(COUNTRY_RATE_MIN * 100).toFixed(0)}%,等待 ${Math.round(COUNTRY_RETRY_WAIT_MS / 1000)} 秒后重新识别国家(第 ${countryRetries}/${COUNTRY_MAX_RETRIES} 次)...`,
|
|
1345
1343
|
);
|
|
1346
|
-
|
|
1344
|
+
await new Promise((r) => setTimeout(r, COUNTRY_RETRY_WAIT_MS));
|
|
1345
|
+
}
|
|
1347
1346
|
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
log(` 正在重启 TikTokScraper...`);
|
|
1358
|
-
await enrichScraper.restart();
|
|
1359
|
-
log(` ✅ TikTokScraper 已重启`);
|
|
1360
|
-
}
|
|
1347
|
+
// 重试耗尽仍低于阈值:跳过该标签,处理下一个任务
|
|
1348
|
+
if (rate < COUNTRY_RATE_MIN) {
|
|
1349
|
+
log(
|
|
1350
|
+
` ⏭️ 重试 ${COUNTRY_MAX_RETRIES} 次后国家识别率仍 ${(rate * 100).toFixed(0)}% < ${(COUNTRY_RATE_MIN * 100).toFixed(0)}%,跳过该标签,处理下一个`,
|
|
1351
|
+
);
|
|
1352
|
+
await recyclePage();
|
|
1353
|
+
await maybeRecycleForMemory();
|
|
1354
|
+
await randomDelay(3000, 7000);
|
|
1355
|
+
continue;
|
|
1361
1356
|
}
|
|
1362
1357
|
|
|
1363
1358
|
// 过滤 + 算分 (共用函数)
|
|
@@ -1400,7 +1395,10 @@ export async function handleScoreAll(parsed) {
|
|
|
1400
1395
|
` ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? " " + mc : ""}${memStr}`,
|
|
1401
1396
|
);
|
|
1402
1397
|
log("");
|
|
1403
|
-
|
|
1398
|
+
consecutiveTaskErrors = 0; // 成功完成,重置连续任务错误计数
|
|
1399
|
+
retrySameTag = false; // 成功,不再重试
|
|
1400
|
+
tag403Attempts = 0;
|
|
1401
|
+
bannedCount = 0; // 成功 tag,重置连续判封计数
|
|
1404
1402
|
// 每 10 个标签输出一次内存汇总
|
|
1405
1403
|
if (totalScored % 10 === 0) {
|
|
1406
1404
|
log(
|
|
@@ -1437,23 +1435,62 @@ export async function handleScoreAll(parsed) {
|
|
|
1437
1435
|
(e.message.includes("HTTP 403") ||
|
|
1438
1436
|
e.message.includes("status 403") ||
|
|
1439
1437
|
e.message.includes("status code 403"));
|
|
1438
|
+
const isCaptcha =
|
|
1439
|
+
e.message &&
|
|
1440
|
+
(e.message.includes("验证码") || e.message.includes("captcha"));
|
|
1440
1441
|
|
|
1442
|
+
// ── 403:重试同一 tag,预算按判封次数递增 ──
|
|
1441
1443
|
if (is403) {
|
|
1442
|
-
|
|
1444
|
+
tag403Attempts++;
|
|
1445
|
+
const budget =
|
|
1446
|
+
BAN_RETRY_BUDGETS[
|
|
1447
|
+
Math.min(bannedCount, BAN_RETRY_BUDGETS.length - 1)
|
|
1448
|
+
];
|
|
1449
|
+
if (tag403Attempts <= budget) {
|
|
1450
|
+
retrySameTag = true;
|
|
1451
|
+
log(
|
|
1452
|
+
` 🚫 HTTP 403 (${tag403Attempts}/${budget}) — 重试同一 tag #${currentTag} ...`,
|
|
1453
|
+
);
|
|
1454
|
+
await recyclePage();
|
|
1455
|
+
await randomDelay(2000, 5000);
|
|
1456
|
+
continue;
|
|
1457
|
+
}
|
|
1458
|
+
// 重试耗尽:判定“被封”
|
|
1459
|
+
bannedCount++;
|
|
1443
1460
|
log(
|
|
1444
|
-
` 🚫
|
|
1461
|
+
` 🚫 重试 ${tag403Attempts} 次仍 403,判定账号被封(连续判封 ${bannedCount} 次)`,
|
|
1445
1462
|
);
|
|
1446
|
-
|
|
1447
|
-
|
|
1463
|
+
if (bannedCount >= 2) {
|
|
1464
|
+
log(` ⚠️ 连续两次判封 → 汇报服务端 + 触发 daemon fallback`);
|
|
1465
|
+
await reportClientErrorAndFailover({
|
|
1466
|
+
post: (url, body) =>
|
|
1467
|
+
fetch(url, {
|
|
1468
|
+
method: "POST",
|
|
1469
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1470
|
+
"Content-Type": "application/json",
|
|
1471
|
+
}),
|
|
1472
|
+
body: JSON.stringify(body),
|
|
1473
|
+
}),
|
|
1474
|
+
baseUrl,
|
|
1475
|
+
userId,
|
|
1476
|
+
errorType: "consecutive_blocked",
|
|
1477
|
+
message: `连续两次判定账号被封 (port=${cdpPort})`,
|
|
1478
|
+
stage: "tag-scoring",
|
|
1479
|
+
log,
|
|
1480
|
+
});
|
|
1481
|
+
bannedCount = 0;
|
|
1482
|
+
}
|
|
1483
|
+
retrySameTag = false;
|
|
1484
|
+
tag403Attempts = 0;
|
|
1485
|
+
await recyclePage();
|
|
1486
|
+
await maybeRecycleForMemory();
|
|
1487
|
+
await randomDelay(3000, 7000);
|
|
1488
|
+
continue;
|
|
1448
1489
|
}
|
|
1449
1490
|
|
|
1491
|
+
// ── 非 403:保持原有错误处理 ──
|
|
1450
1492
|
log(` ❌ 失败: ${e.message}`);
|
|
1451
1493
|
|
|
1452
|
-
// ── 验证码检测 ──
|
|
1453
|
-
const isCaptcha =
|
|
1454
|
-
e.message &&
|
|
1455
|
-
(e.message.includes("验证码") || e.message.includes("captcha"));
|
|
1456
|
-
|
|
1457
1494
|
if (isCaptcha) {
|
|
1458
1495
|
consecutiveCaptcha++;
|
|
1459
1496
|
log(
|
|
@@ -1491,7 +1528,7 @@ export async function handleScoreAll(parsed) {
|
|
|
1491
1528
|
} catch {}
|
|
1492
1529
|
|
|
1493
1530
|
if (consecutiveCaptcha >= MAX_CONSECUTIVE_CAPTCHA) {
|
|
1494
|
-
log(` ♻️ 连续${MAX_CONSECUTIVE_CAPTCHA}次验证码 →
|
|
1531
|
+
log(` ♻️ 连续${MAX_CONSECUTIVE_CAPTCHA}次验证码 → 重启浏览器`);
|
|
1495
1532
|
await recycleCdpSession("连续验证码", { switchPort: true });
|
|
1496
1533
|
consecutiveCaptcha = 0;
|
|
1497
1534
|
}
|
|
@@ -1514,19 +1551,33 @@ export async function handleScoreAll(parsed) {
|
|
|
1514
1551
|
clientMeta,
|
|
1515
1552
|
);
|
|
1516
1553
|
} catch {}
|
|
1517
|
-
totalScored++;
|
|
1518
1554
|
|
|
1519
|
-
//
|
|
1520
|
-
|
|
1555
|
+
// 连续任务错误达到阈值,上报服务端 + 触发 daemon fallback 换节点
|
|
1556
|
+
consecutiveTaskErrors++;
|
|
1557
|
+
if (consecutiveTaskErrors >= CONSECUTIVE_TASK_ERRORS_MAX) {
|
|
1521
1558
|
log(
|
|
1522
|
-
`
|
|
1559
|
+
` ⚠️ 连续任务错误 ${consecutiveTaskErrors}/${CONSECUTIVE_TASK_ERRORS_MAX} → 汇报服务端 + 触发 fallback`,
|
|
1523
1560
|
);
|
|
1524
|
-
await
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1561
|
+
await reportConsecutiveTaskErrors({
|
|
1562
|
+
post: (url, body) =>
|
|
1563
|
+
fetch(url, {
|
|
1564
|
+
method: "POST",
|
|
1565
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1566
|
+
"Content-Type": "application/json",
|
|
1567
|
+
}),
|
|
1568
|
+
body: JSON.stringify(body),
|
|
1569
|
+
}),
|
|
1570
|
+
baseUrl,
|
|
1571
|
+
userId,
|
|
1572
|
+
message: `连续 ${consecutiveTaskErrors} 个标签处理失败 (port=${cdpPort})`,
|
|
1573
|
+
stage: "tag-scoring",
|
|
1574
|
+
log,
|
|
1575
|
+
});
|
|
1576
|
+
consecutiveTaskErrors = 0;
|
|
1528
1577
|
}
|
|
1529
|
-
|
|
1578
|
+
totalScored++;
|
|
1579
|
+
retrySameTag = false;
|
|
1580
|
+
tag403Attempts = 0;
|
|
1530
1581
|
await recyclePage();
|
|
1531
1582
|
await maybeRecycleForMemory();
|
|
1532
1583
|
}
|
package/src/cli/webserver.js
CHANGED
|
@@ -11,6 +11,9 @@ export async function handleWebserver(options) {
|
|
|
11
11
|
webserverProxy,
|
|
12
12
|
webserverLog,
|
|
13
13
|
webserverNotify,
|
|
14
|
+
webserverRate,
|
|
15
|
+
webserverBurst,
|
|
16
|
+
webserverCooldown,
|
|
14
17
|
} = options;
|
|
15
18
|
|
|
16
19
|
const { server, port } = await startWebserver(
|
|
@@ -19,6 +22,9 @@ export async function handleWebserver(options) {
|
|
|
19
22
|
webserverProxy,
|
|
20
23
|
webserverNotify,
|
|
21
24
|
webserverLog,
|
|
25
|
+
webserverRate,
|
|
26
|
+
webserverBurst,
|
|
27
|
+
webserverCooldown,
|
|
22
28
|
);
|
|
23
29
|
|
|
24
30
|
let shuttingDown = false;
|
|
@@ -57,8 +57,17 @@ async function fetchUserCommentsAPI(
|
|
|
57
57
|
// 点击评论 tab 触发 API
|
|
58
58
|
log(" [API拦截] 点击评论 tab...");
|
|
59
59
|
const tabs = page.locator('[class*="tabbar-item"]');
|
|
60
|
-
|
|
61
|
-
|
|
60
|
+
|
|
61
|
+
// 优先精确匹配:文本必须是"评论"或"Comment",避免命中"商品/猜你喜欢/Shop/Related"等 tab
|
|
62
|
+
let commentTab = tabs.filter({ hasText: /^\s*(评论|Comment)\s*$/i });
|
|
63
|
+
let count = await commentTab.count();
|
|
64
|
+
if (count === 0) {
|
|
65
|
+
// 兜底:宽松匹配"评论|Comment",但明确排除非评论 tab(商品/购物/猜你喜欢/推荐)
|
|
66
|
+
commentTab = tabs.filter({ hasText: /评论|Comment/ }).filter({
|
|
67
|
+
hasNotText: /商品|购物|带货|Shop|Shopping|橱窗|猜你喜欢|Related|推荐/i,
|
|
68
|
+
});
|
|
69
|
+
count = await commentTab.count();
|
|
70
|
+
}
|
|
62
71
|
|
|
63
72
|
if (count === 0) {
|
|
64
73
|
return {
|
|
@@ -69,6 +78,9 @@ async function fetchUserCommentsAPI(
|
|
|
69
78
|
};
|
|
70
79
|
}
|
|
71
80
|
|
|
81
|
+
// 只在候选只有一个/第一个确为评论 tab 时点击,避免点击到其它 tab
|
|
82
|
+
await commentTab.first().click({ force: true });
|
|
83
|
+
|
|
72
84
|
await commentTab.first().click({ force: true });
|
|
73
85
|
|
|
74
86
|
// 等待 API 响应
|
package/src/lib/args.js
CHANGED
|
@@ -412,6 +412,10 @@ function parseWebserverArgs(args) {
|
|
|
412
412
|
let webserverPoolSize = 0;
|
|
413
413
|
let webserverProxy = null;
|
|
414
414
|
let webserverLog = false;
|
|
415
|
+
// 默认安全限速:3 请求/秒。限速突发容量自动跟随 pool-size(见下方返回)。
|
|
416
|
+
let webserverRate = 3;
|
|
417
|
+
// CDN 限流冷却时长(秒),默认 100(2 次重启 + 切节点流程节奏)
|
|
418
|
+
let webserverCooldown = 100;
|
|
415
419
|
// 默认汇报到 ~/.tt-help.json 中配置的 server 地址
|
|
416
420
|
let webserverNotify = defaultServer;
|
|
417
421
|
|
|
@@ -423,6 +427,22 @@ function parseWebserverArgs(args) {
|
|
|
423
427
|
webserverPoolSize = parseInt(args[++i]) || 0;
|
|
424
428
|
} else if (arg === "--proxy") {
|
|
425
429
|
webserverProxy = args[++i] || null;
|
|
430
|
+
} else if (arg === "--rate") {
|
|
431
|
+
const next = args[i + 1];
|
|
432
|
+
if (next && !next.startsWith("--")) {
|
|
433
|
+
webserverRate = parseInt(next) || 0;
|
|
434
|
+
i++;
|
|
435
|
+
} else {
|
|
436
|
+
webserverRate = 3; // 缺省回退到安全默认值
|
|
437
|
+
}
|
|
438
|
+
} else if (arg === "--cooldown") {
|
|
439
|
+
const next = args[i + 1];
|
|
440
|
+
if (next && !next.startsWith("--")) {
|
|
441
|
+
webserverCooldown = parseInt(next) || 100;
|
|
442
|
+
i++;
|
|
443
|
+
} else {
|
|
444
|
+
webserverCooldown = 100; // 缺省回退到默认值
|
|
445
|
+
}
|
|
426
446
|
} else if (arg === "--log") {
|
|
427
447
|
webserverLog = true;
|
|
428
448
|
} else if (arg === "--notify") {
|
|
@@ -443,6 +463,10 @@ function parseWebserverArgs(args) {
|
|
|
443
463
|
webserverProxy,
|
|
444
464
|
webserverLog,
|
|
445
465
|
webserverNotify,
|
|
466
|
+
webserverRate,
|
|
467
|
+
// burst 自动跟随 pool-size:未显式指定时与最终生效的页面池大小一致
|
|
468
|
+
webserverBurst: webserverPoolSize > 0 ? webserverPoolSize : 8,
|
|
469
|
+
webserverCooldown,
|
|
446
470
|
urls: [],
|
|
447
471
|
outputFile: null,
|
|
448
472
|
outputFormat: "json",
|