tt-help-cli-ycl 1.4.12 → 1.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/comments.js +10 -4
- package/src/cli/tag.js +513 -13
- package/src/lib/api-interceptor.js +2 -0
- package/src/lib/args.js +35 -0
- package/src/lib/constants.js +5 -2
- package/src/main.js +3 -0
- package/src/scraper/explore-core.js +1 -0
- package/src/watch/data-store.js +90 -4
- package/src/watch/db-columns.js +2 -0
- package/src/watch/db-crud.js +28 -0
- package/src/watch/db-schema.js +85 -0
- package/src/watch/db-tags.js +114 -0
- package/src/watch/public/app.js +80 -13
- package/src/watch/public/index.html +7 -1
- package/src/watch/public/style.css +43 -0
- package/src/watch/server.js +422 -19
- package/src/watch/tag-selflearn.js +986 -0
package/package.json
CHANGED
package/src/cli/comments.js
CHANGED
|
@@ -528,6 +528,7 @@ async function runAutoMode(options) {
|
|
|
528
528
|
maxComments,
|
|
529
529
|
basePort,
|
|
530
530
|
proxy: proxyOption,
|
|
531
|
+
locations,
|
|
531
532
|
} = options;
|
|
532
533
|
|
|
533
534
|
const actualParallel = Math.max(1, parallel || CFG.parallel);
|
|
@@ -535,6 +536,7 @@ async function runAutoMode(options) {
|
|
|
535
536
|
const actualMaxComments = maxComments || CFG.maxComments;
|
|
536
537
|
const effectiveProxy = proxyOption || configuredProxy;
|
|
537
538
|
const portPoolStart = basePort || CFG.portPoolStart;
|
|
539
|
+
const jobLocations = locations || null;
|
|
538
540
|
|
|
539
541
|
loadEnvConfig();
|
|
540
542
|
|
|
@@ -597,8 +599,9 @@ async function runAutoMode(options) {
|
|
|
597
599
|
console.error(`\n╔══════════════════════════════════════════╗`);
|
|
598
600
|
console.error(`║ 自动评论采集模式(CDP 连接已登录浏览器)`);
|
|
599
601
|
console.error(
|
|
600
|
-
`║ 目标国家: ${DEFAULT_TARGET_LOCATIONS.slice(0, 6).join(",")}
|
|
602
|
+
`║ 目标国家: ${jobLocations || DEFAULT_TARGET_LOCATIONS.slice(0, 6).join(",") + "..."}`,
|
|
601
603
|
);
|
|
604
|
+
if (jobLocations) console.error(`║ 任务过滤: ${jobLocations}`);
|
|
602
605
|
console.error(`║ 服务端: ${serverUrl}`);
|
|
603
606
|
console.error(
|
|
604
607
|
`║ CDP 端口池: ${portPoolStart}-${portPoolStart + CFG.portPoolSize - 1}(当前: ${session.port})`,
|
|
@@ -635,9 +638,10 @@ async function runAutoMode(options) {
|
|
|
635
638
|
// ── 领取任务 ──
|
|
636
639
|
let tasks;
|
|
637
640
|
try {
|
|
638
|
-
const
|
|
639
|
-
`${serverUrl}/api/comment-tasks?limit=${actualParallel}
|
|
640
|
-
|
|
641
|
+
const apiUrl = jobLocations
|
|
642
|
+
? `${serverUrl}/api/comment-tasks?limit=${actualParallel}&locations=${encodeURIComponent(jobLocations)}`
|
|
643
|
+
: `${serverUrl}/api/comment-tasks?limit=${actualParallel}`;
|
|
644
|
+
const resp = await apiGet(apiUrl);
|
|
641
645
|
tasks = resp.tasks || [];
|
|
642
646
|
} catch (err) {
|
|
643
647
|
console.error(` × 获取任务失败: ${err.message}`);
|
|
@@ -1031,6 +1035,7 @@ export async function handleComments(options) {
|
|
|
1031
1035
|
commentsServer,
|
|
1032
1036
|
commentsBasePort,
|
|
1033
1037
|
commentsProxy,
|
|
1038
|
+
commentsLocations,
|
|
1034
1039
|
} = options;
|
|
1035
1040
|
|
|
1036
1041
|
// 自动模式:无 URL
|
|
@@ -1042,6 +1047,7 @@ export async function handleComments(options) {
|
|
|
1042
1047
|
maxComments: commentsMax,
|
|
1043
1048
|
basePort: commentsBasePort,
|
|
1044
1049
|
proxy: commentsProxy,
|
|
1050
|
+
locations: commentsLocations,
|
|
1045
1051
|
});
|
|
1046
1052
|
}
|
|
1047
1053
|
|
package/src/cli/tag.js
CHANGED
|
@@ -171,6 +171,82 @@ async function pushToServer(
|
|
|
171
171
|
return data;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
/**
|
|
175
|
+
* 带服务端缓存的批量 enrich:先从服务端查询已有国家信息,只对未知的做 HTTP enrich
|
|
176
|
+
* @returns {{ videos: Array, cdnBlockedCount: number }}
|
|
177
|
+
*/
|
|
178
|
+
async function enrichWithServerCache(
|
|
179
|
+
videos,
|
|
180
|
+
baseUrl,
|
|
181
|
+
enrichScraper,
|
|
182
|
+
options = {},
|
|
183
|
+
) {
|
|
184
|
+
const { clientId, clientMeta, effectiveProxy, log } = options;
|
|
185
|
+
const logFn = log || (() => {});
|
|
186
|
+
|
|
187
|
+
// Step 1: 收集所有唯一作者 ID
|
|
188
|
+
const uniqueAuthors = [
|
|
189
|
+
...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
|
|
190
|
+
];
|
|
191
|
+
|
|
192
|
+
// Step 2: 批量查询服务端已知国家
|
|
193
|
+
let knownLocations = {};
|
|
194
|
+
if (uniqueAuthors.length > 0 && baseUrl) {
|
|
195
|
+
try {
|
|
196
|
+
const res = await fetch(`${baseUrl}/api/bulk-author-locations`, {
|
|
197
|
+
method: "POST",
|
|
198
|
+
headers: buildClientHeaders(clientId || "", clientMeta || {}, {
|
|
199
|
+
"Content-Type": "application/json",
|
|
200
|
+
}),
|
|
201
|
+
body: JSON.stringify({ uniqueIds: uniqueAuthors }),
|
|
202
|
+
});
|
|
203
|
+
const data = await res.json();
|
|
204
|
+
knownLocations = data.locations || {};
|
|
205
|
+
const knownCount = Object.keys(knownLocations).length;
|
|
206
|
+
if (knownCount > 0) {
|
|
207
|
+
logFn(
|
|
208
|
+
` 📋 服务端已知 ${knownCount}/${uniqueAuthors.length} 个作者的国家`,
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
} catch {
|
|
212
|
+
// 查询失败则全部走 enrich,不影响主流程
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Step 3: 给已知国家的视频赋值,收集未知视频
|
|
217
|
+
const unknownVideos = [];
|
|
218
|
+
for (const v of videos) {
|
|
219
|
+
const loc = knownLocations[v.authorUniqueId];
|
|
220
|
+
if (loc) {
|
|
221
|
+
v.locationCreated = loc;
|
|
222
|
+
} else {
|
|
223
|
+
unknownVideos.push(v);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Step 4: 只对未知视频做 HTTP enrich
|
|
228
|
+
let cdnBlockedCount = 0;
|
|
229
|
+
if (unknownVideos.length > 0) {
|
|
230
|
+
const enrichOpts = {
|
|
231
|
+
mode: "videos",
|
|
232
|
+
existingScraper: enrichScraper,
|
|
233
|
+
onProgress: ({ done, total, current, locationCreated }) => {
|
|
234
|
+
if (done % 10 === 0 || done === total) {
|
|
235
|
+
process.stderr.write(
|
|
236
|
+
`\r [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}\x1b[K`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
|
|
242
|
+
const enriched = await enrichVideosWithLocation(unknownVideos, enrichOpts);
|
|
243
|
+
// 将 enrich 结果写回原数组(v 是引用,已经被 enrichVideosWithLocation 修改)
|
|
244
|
+
cdnBlockedCount = enriched.cdnBlockedCount || 0;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return { videos, cdnBlockedCount };
|
|
248
|
+
}
|
|
249
|
+
|
|
174
250
|
// 共享算分逻辑(handleScore / handleScoreAll 共用)
|
|
175
251
|
function calcScore(authorCount, matchedAuthors, filteredVideoCount) {
|
|
176
252
|
const densityScore =
|
|
@@ -1002,20 +1078,18 @@ export async function handleScoreAll(parsed) {
|
|
|
1002
1078
|
continue;
|
|
1003
1079
|
}
|
|
1004
1080
|
|
|
1005
|
-
// enrich:
|
|
1006
|
-
const
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1081
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1082
|
+
const enriched = await enrichWithServerCache(
|
|
1083
|
+
videos,
|
|
1084
|
+
baseUrl,
|
|
1085
|
+
enrichScraper,
|
|
1086
|
+
{
|
|
1087
|
+
clientId,
|
|
1088
|
+
clientMeta,
|
|
1089
|
+
effectiveProxy,
|
|
1090
|
+
log,
|
|
1015
1091
|
},
|
|
1016
|
-
|
|
1017
|
-
if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
|
|
1018
|
-
const enriched = await enrichVideosWithLocation(videos, enrichOpts);
|
|
1092
|
+
);
|
|
1019
1093
|
videos = enriched.videos;
|
|
1020
1094
|
|
|
1021
1095
|
// CDN 限流检测:有拦截则冷却 + 重启 scraper
|
|
@@ -1324,3 +1398,429 @@ export async function handleTag(parsed) {
|
|
|
1324
1398
|
console.log(json);
|
|
1325
1399
|
}
|
|
1326
1400
|
}
|
|
1401
|
+
|
|
1402
|
+
// ====== Productive 标签刷新(重新抓取,不打分) ======
|
|
1403
|
+
|
|
1404
|
+
export async function handleTagRefresh(parsed) {
|
|
1405
|
+
const { tagRefresh } = parsed;
|
|
1406
|
+
const { serverUrl, port, proxy: cliProxy } = tagRefresh || {};
|
|
1407
|
+
|
|
1408
|
+
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
1409
|
+
const defaultRefreshPort =
|
|
1410
|
+
parseInt(process.env.TAG_REFRESH_PORT_POOL_START || "8222", 10) || 8222;
|
|
1411
|
+
let cdpPort = port || defaultRefreshPort;
|
|
1412
|
+
const effectiveProxy = cliProxy || configuredProxy;
|
|
1413
|
+
|
|
1414
|
+
const log = (...args) => process.stderr.write(args.join(" ") + "\n");
|
|
1415
|
+
|
|
1416
|
+
log("");
|
|
1417
|
+
log("╔══════════════════════════════════════════╗");
|
|
1418
|
+
log("║ Productive 标签刷新模式(重新抓取)");
|
|
1419
|
+
log("║ 服务端: " + baseUrl);
|
|
1420
|
+
log(
|
|
1421
|
+
"║ CDP 端口: " +
|
|
1422
|
+
cdpPort +
|
|
1423
|
+
(effectiveProxy ? " 代理: " + effectiveProxy : ""),
|
|
1424
|
+
);
|
|
1425
|
+
log("║ 只抓取不重新打分,score>70 的标签");
|
|
1426
|
+
log("╚══════════════════════════════════════════╝");
|
|
1427
|
+
log("");
|
|
1428
|
+
|
|
1429
|
+
log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
|
|
1430
|
+
log("");
|
|
1431
|
+
|
|
1432
|
+
// ── 设备标识 ──
|
|
1433
|
+
let userId = configuredUserId;
|
|
1434
|
+
if (!userId) {
|
|
1435
|
+
userId = await getMacOrUuid();
|
|
1436
|
+
saveUserId(userId);
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
// 连接 CDP 浏览器
|
|
1440
|
+
const cdpOpts = { port: cdpPort };
|
|
1441
|
+
if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
|
|
1442
|
+
let browser = await ensureBrowserReady(cdpOpts);
|
|
1443
|
+
let page = await getOrCreatePage(browser);
|
|
1444
|
+
const blockedRoutePages = new WeakSet();
|
|
1445
|
+
|
|
1446
|
+
async function setupPageRequestBlocking(targetPage) {
|
|
1447
|
+
if (!targetPage || blockedRoutePages.has(targetPage)) return;
|
|
1448
|
+
await targetPage.route("**/*", (route) => {
|
|
1449
|
+
const resourceType = route.request().resourceType();
|
|
1450
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
1451
|
+
route.abort();
|
|
1452
|
+
} else {
|
|
1453
|
+
route.continue();
|
|
1454
|
+
}
|
|
1455
|
+
});
|
|
1456
|
+
blockedRoutePages.add(targetPage);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
await setupPageRequestBlocking(page);
|
|
1460
|
+
|
|
1461
|
+
// 端口池
|
|
1462
|
+
const portPoolStart = cdpPort;
|
|
1463
|
+
const portPoolSize = Math.max(
|
|
1464
|
+
2,
|
|
1465
|
+
parseInt(process.env.TAG_REFRESH_PORT_POOL_SIZE || "10", 10) || 10,
|
|
1466
|
+
);
|
|
1467
|
+
|
|
1468
|
+
function pickNextPort(currentPort) {
|
|
1469
|
+
const candidates = [];
|
|
1470
|
+
for (let i = 0; i < portPoolSize; i++) {
|
|
1471
|
+
const p = portPoolStart + i;
|
|
1472
|
+
if (p !== currentPort) candidates.push(p);
|
|
1473
|
+
}
|
|
1474
|
+
if (candidates.length === 0) return currentPort;
|
|
1475
|
+
return candidates[Math.floor(Math.random() * candidates.length)];
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
// ── 内存回收配置 ──
|
|
1479
|
+
const RECYCLE_CONFIG = {
|
|
1480
|
+
nodeRssMb: Math.max(
|
|
1481
|
+
256,
|
|
1482
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_RSS_MB || "2000", 10) || 2000,
|
|
1483
|
+
),
|
|
1484
|
+
nodeHeapMb: Math.max(
|
|
1485
|
+
128,
|
|
1486
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_HEAP_MB || "2000", 10) || 2000,
|
|
1487
|
+
),
|
|
1488
|
+
edgeMb: Math.max(
|
|
1489
|
+
500,
|
|
1490
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_EDGE_MB || "2000", 10) || 2000,
|
|
1491
|
+
),
|
|
1492
|
+
cooldownMs: Math.max(
|
|
1493
|
+
0,
|
|
1494
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_COOLDOWN_MS || "180000", 10) ||
|
|
1495
|
+
180000,
|
|
1496
|
+
),
|
|
1497
|
+
periodicEvery: Math.max(
|
|
1498
|
+
0,
|
|
1499
|
+
parseInt(process.env.TAG_REFRESH_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
|
|
1500
|
+
),
|
|
1501
|
+
switchPortOnRecycle:
|
|
1502
|
+
String(process.env.TAG_REFRESH_SWITCH_PORT_ON_RECYCLE || "1") !== "0",
|
|
1503
|
+
};
|
|
1504
|
+
let lastRecycleAt = 0;
|
|
1505
|
+
|
|
1506
|
+
// ── 轻量回收 ──
|
|
1507
|
+
async function recyclePage() {
|
|
1508
|
+
if (!page || page.isClosed()) {
|
|
1509
|
+
page = await getOrCreatePage(browser);
|
|
1510
|
+
await setupPageRequestBlocking(page);
|
|
1511
|
+
return;
|
|
1512
|
+
}
|
|
1513
|
+
try {
|
|
1514
|
+
await page.goto("about:blank", {
|
|
1515
|
+
waitUntil: "domcontentloaded",
|
|
1516
|
+
timeout: 5000,
|
|
1517
|
+
});
|
|
1518
|
+
} catch {
|
|
1519
|
+
await page.close().catch(() => {});
|
|
1520
|
+
page = await getOrCreatePage(browser);
|
|
1521
|
+
await setupPageRequestBlocking(page);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
// ── 深度回收 ──
|
|
1526
|
+
async function recycleCdpSession(reason, { switchPort = true } = {}) {
|
|
1527
|
+
const oldPort = cdpPort;
|
|
1528
|
+
const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
|
|
1529
|
+
if (shouldSwitch) {
|
|
1530
|
+
cdpPort = pickNextPort(cdpPort);
|
|
1531
|
+
cdpOpts.port = cdpPort;
|
|
1532
|
+
clientMeta.port = cdpPort;
|
|
1533
|
+
}
|
|
1534
|
+
const portHint =
|
|
1535
|
+
oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
|
|
1536
|
+
log(` ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
|
|
1537
|
+
if (page) {
|
|
1538
|
+
await page.close().catch(() => {});
|
|
1539
|
+
page = null;
|
|
1540
|
+
}
|
|
1541
|
+
await browser.close().catch(() => {});
|
|
1542
|
+
if (oldPort !== cdpPort) {
|
|
1543
|
+
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
1544
|
+
}
|
|
1545
|
+
browser = await ensureBrowserReady(cdpOpts);
|
|
1546
|
+
page = await getOrCreatePage(browser);
|
|
1547
|
+
await setupPageRequestBlocking(page);
|
|
1548
|
+
lastRecycleAt = Date.now();
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
async function maybeRecycleForMemory() {
|
|
1552
|
+
const mem = process.memoryUsage();
|
|
1553
|
+
const rssMB = mem.rss / 1024 / 1024;
|
|
1554
|
+
const heapMB = mem.heapUsed / 1024 / 1024;
|
|
1555
|
+
const edgeMB = getBrowserMemory(cdpPort);
|
|
1556
|
+
|
|
1557
|
+
const nodeOver =
|
|
1558
|
+
rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
|
|
1559
|
+
const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
|
|
1560
|
+
|
|
1561
|
+
if (nodeOver || edgeOver) {
|
|
1562
|
+
if (
|
|
1563
|
+
RECYCLE_CONFIG.cooldownMs > 0 &&
|
|
1564
|
+
Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
|
|
1565
|
+
)
|
|
1566
|
+
return;
|
|
1567
|
+
|
|
1568
|
+
const parts = [];
|
|
1569
|
+
if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
|
|
1570
|
+
parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
|
|
1571
|
+
if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
|
|
1572
|
+
parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
|
|
1573
|
+
if (edgeOver)
|
|
1574
|
+
parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
|
|
1575
|
+
|
|
1576
|
+
await recycleCdpSession(`mem ${parts.join("+")}`, {
|
|
1577
|
+
switchPort: edgeOver,
|
|
1578
|
+
});
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
if (
|
|
1583
|
+
RECYCLE_CONFIG.periodicEvery > 0 &&
|
|
1584
|
+
totalRefreshed > 0 &&
|
|
1585
|
+
totalRefreshed % RECYCLE_CONFIG.periodicEvery === 0
|
|
1586
|
+
) {
|
|
1587
|
+
await recycleCdpSession(`periodic #${totalRefreshed}`, {
|
|
1588
|
+
switchPort: false,
|
|
1589
|
+
});
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
let totalRefreshed = 0;
|
|
1594
|
+
let emptyRounds = 0;
|
|
1595
|
+
let consecutiveCaptcha = 0;
|
|
1596
|
+
const MAX_CONSECUTIVE_CAPTCHA = 3;
|
|
1597
|
+
|
|
1598
|
+
const clientId = randomUUID();
|
|
1599
|
+
const clientMeta = { type: "tag-refresh", port: cdpPort };
|
|
1600
|
+
|
|
1601
|
+
// 复用 TikTokScraper
|
|
1602
|
+
const enrichScraper = new TikTokScraper({
|
|
1603
|
+
poolSize: 3,
|
|
1604
|
+
proxyServer: effectiveProxy || undefined,
|
|
1605
|
+
});
|
|
1606
|
+
await enrichScraper.init();
|
|
1607
|
+
log("✅ TikTokScraper 已就绪 (enrich 复用)");
|
|
1608
|
+
log(` 客户端 ID: ${clientId.substring(0, 8)}...`);
|
|
1609
|
+
log("");
|
|
1610
|
+
|
|
1611
|
+
const cleanup = () => {
|
|
1612
|
+
log("\n正在清理资源...");
|
|
1613
|
+
enrichScraper.close().catch(() => {});
|
|
1614
|
+
killEdgeProcesses(null, cdpPort);
|
|
1615
|
+
process.exit(0);
|
|
1616
|
+
};
|
|
1617
|
+
process.on("SIGINT", cleanup);
|
|
1618
|
+
|
|
1619
|
+
try {
|
|
1620
|
+
while (true) {
|
|
1621
|
+
try {
|
|
1622
|
+
// 从服务端获取下一个 productive 标签
|
|
1623
|
+
const taskRes = await fetch(`${baseUrl}/api/tags/refresh-task`, {
|
|
1624
|
+
headers: buildClientHeaders(clientId, clientMeta),
|
|
1625
|
+
});
|
|
1626
|
+
const taskData = await taskRes.json();
|
|
1627
|
+
|
|
1628
|
+
if (!taskData.tag) {
|
|
1629
|
+
emptyRounds++;
|
|
1630
|
+
|
|
1631
|
+
if (emptyRounds >= 5) {
|
|
1632
|
+
log(
|
|
1633
|
+
`⏳ 连续 ${emptyRounds} 轮无 productive 标签可刷新(30 秒后重试)...`,
|
|
1634
|
+
);
|
|
1635
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
1636
|
+
emptyRounds = 0;
|
|
1637
|
+
continue;
|
|
1638
|
+
}
|
|
1639
|
+
log(
|
|
1640
|
+
`⏳ 暂无 productive 标签可刷新(连续 ${emptyRounds} 轮),10 秒后重试...`,
|
|
1641
|
+
);
|
|
1642
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
emptyRounds = 0;
|
|
1647
|
+
|
|
1648
|
+
const tag = taskData.tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1649
|
+
const startTime = Date.now();
|
|
1650
|
+
|
|
1651
|
+
log(
|
|
1652
|
+
`[${totalRefreshed + 1}] 正在刷新 #${tag} (score: ${taskData.score})...`,
|
|
1653
|
+
);
|
|
1654
|
+
|
|
1655
|
+
const result = {
|
|
1656
|
+
tag,
|
|
1657
|
+
pushedUsers: 0,
|
|
1658
|
+
error: null,
|
|
1659
|
+
};
|
|
1660
|
+
|
|
1661
|
+
// 锁定标签
|
|
1662
|
+
const claimRes = await fetch(`${baseUrl}/api/tags/refresh-claim`, {
|
|
1663
|
+
method: "POST",
|
|
1664
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1665
|
+
"Content-Type": "application/json",
|
|
1666
|
+
}),
|
|
1667
|
+
body: JSON.stringify({ tag }),
|
|
1668
|
+
});
|
|
1669
|
+
const claimData = await claimRes.json();
|
|
1670
|
+
if (!claimData.ok) {
|
|
1671
|
+
if (
|
|
1672
|
+
claimData.error &&
|
|
1673
|
+
claimData.error.includes("already being refreshed")
|
|
1674
|
+
) {
|
|
1675
|
+
log(` ⏭️ 已被其他客户端锁定,跳过`);
|
|
1676
|
+
await recyclePage();
|
|
1677
|
+
continue;
|
|
1678
|
+
}
|
|
1679
|
+
log(` ⚠️ 无法锁定 (${claimData.error}),重试下一个`);
|
|
1680
|
+
await recyclePage();
|
|
1681
|
+
continue;
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
// 抓取视频
|
|
1685
|
+
const fetchStart = Date.now();
|
|
1686
|
+
log(` 抓取 TikTok 标签页...`);
|
|
1687
|
+
const tagResult = await fetchTagData(tag, {
|
|
1688
|
+
page,
|
|
1689
|
+
onProgress: ({ videos, authors }) => {
|
|
1690
|
+
process.stderr.write(
|
|
1691
|
+
`\r 抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
|
|
1692
|
+
);
|
|
1693
|
+
},
|
|
1694
|
+
});
|
|
1695
|
+
const fetchSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1696
|
+
log(
|
|
1697
|
+
`\r 完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者 (${fetchSec}s)\x1b[K`,
|
|
1698
|
+
);
|
|
1699
|
+
|
|
1700
|
+
let videos = tagResult.videos;
|
|
1701
|
+
|
|
1702
|
+
if (!videos || videos.length === 0) {
|
|
1703
|
+
const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1704
|
+
log(` ⚠️ 无视频 (${deadSec}s),保留 productive 状态,等待下次刷新`);
|
|
1705
|
+
result.error = "no videos found";
|
|
1706
|
+
await fetch(`${baseUrl}/api/tags/refresh-result`, {
|
|
1707
|
+
method: "POST",
|
|
1708
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1709
|
+
"Content-Type": "application/json",
|
|
1710
|
+
}),
|
|
1711
|
+
body: JSON.stringify({ tag, pushedUsers: 0, error: result.error }),
|
|
1712
|
+
}).catch(() => {});
|
|
1713
|
+
totalRefreshed++;
|
|
1714
|
+
await randomDelay(0, 5000);
|
|
1715
|
+
await recyclePage();
|
|
1716
|
+
await maybeRecycleForMemory();
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1721
|
+
const enriched = await enrichWithServerCache(
|
|
1722
|
+
videos,
|
|
1723
|
+
baseUrl,
|
|
1724
|
+
enrichScraper,
|
|
1725
|
+
{
|
|
1726
|
+
clientId,
|
|
1727
|
+
clientMeta,
|
|
1728
|
+
effectiveProxy,
|
|
1729
|
+
log,
|
|
1730
|
+
},
|
|
1731
|
+
);
|
|
1732
|
+
videos = enriched.videos;
|
|
1733
|
+
|
|
1734
|
+
// CDN 限流检测
|
|
1735
|
+
const cdnBlocked = enriched.cdnBlockedCount || 0;
|
|
1736
|
+
if (cdnBlocked > 0) {
|
|
1737
|
+
const cdnRatio = cdnBlocked / (videos.length || 1);
|
|
1738
|
+
const coolSec = cdnRatio > 0.3 ? 120 : 60;
|
|
1739
|
+
log(
|
|
1740
|
+
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),冷却 ${coolSec} 秒后重启 scraper`,
|
|
1741
|
+
);
|
|
1742
|
+
await new Promise((r) => setTimeout(r, coolSec * 1000));
|
|
1743
|
+
log(` 正在重启 TikTokScraper...`);
|
|
1744
|
+
await enrichScraper.restart();
|
|
1745
|
+
log(` ✅ TikTokScraper 已重启`);
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
// 过滤目标国家的视频和作者
|
|
1749
|
+
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
1750
|
+
const filteredVideos = videos.filter((v) =>
|
|
1751
|
+
isLocationInList(v.locationCreated, targetLocations),
|
|
1752
|
+
);
|
|
1753
|
+
const filteredAuthors = [
|
|
1754
|
+
...new Set(
|
|
1755
|
+
filteredVideos.map((v) => v.authorUniqueId).filter(Boolean),
|
|
1756
|
+
),
|
|
1757
|
+
];
|
|
1758
|
+
const filteredCount = videos.length - filteredVideos.length;
|
|
1759
|
+
if (filteredCount > 0) {
|
|
1760
|
+
log(` 过滤非目标国家: ${filteredCount}/${videos.length} 视频被排除`);
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// 推送过滤后的作者到服务端
|
|
1764
|
+
if (filteredAuthors.length > 0) {
|
|
1765
|
+
const pushResult = await pushToServer(
|
|
1766
|
+
baseUrl,
|
|
1767
|
+
filteredAuthors,
|
|
1768
|
+
filteredVideos,
|
|
1769
|
+
clientId,
|
|
1770
|
+
clientMeta,
|
|
1771
|
+
);
|
|
1772
|
+
result.pushedUsers = pushResult.added || 0;
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
// 上报刷新结果(只更新 pushed_users,不改变 score/status)
|
|
1776
|
+
await fetch(`${baseUrl}/api/tags/refresh-result`, {
|
|
1777
|
+
method: "POST",
|
|
1778
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1779
|
+
"Content-Type": "application/json",
|
|
1780
|
+
}),
|
|
1781
|
+
body: JSON.stringify({
|
|
1782
|
+
tag,
|
|
1783
|
+
pushedUsers: result.pushedUsers,
|
|
1784
|
+
error: result.error,
|
|
1785
|
+
}),
|
|
1786
|
+
}).catch(() => {});
|
|
1787
|
+
|
|
1788
|
+
totalRefreshed++;
|
|
1789
|
+
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
1790
|
+
const memStr = ` mem=${formatMemoryWithBrowser(cdpPort)}`;
|
|
1791
|
+
log(
|
|
1792
|
+
` 🔄 #${tag} 完成 (${elapsed}s) — ${videos.length} 视频(${filteredVideos.length} 匹配), ${filteredAuthors.length} 作者, 推送 ${result.pushedUsers} 用户${memStr}`,
|
|
1793
|
+
);
|
|
1794
|
+
log("");
|
|
1795
|
+
|
|
1796
|
+
if (totalRefreshed % 10 === 0) {
|
|
1797
|
+
log(
|
|
1798
|
+
`📊 [${totalRefreshed}] 内存汇总: ${formatMemoryWithBrowser(cdpPort)}`,
|
|
1799
|
+
);
|
|
1800
|
+
}
|
|
1801
|
+
await recyclePage();
|
|
1802
|
+
await maybeRecycleForMemory();
|
|
1803
|
+
await randomDelay(3000, 7000);
|
|
1804
|
+
} catch (e) {
|
|
1805
|
+
const isNetworkError =
|
|
1806
|
+
e.code === "ECONNREFUSED" ||
|
|
1807
|
+
e.code === "ENOTFOUND" ||
|
|
1808
|
+
e.code === "ECONNRESET" ||
|
|
1809
|
+
(e.message &&
|
|
1810
|
+
(e.message.includes("ECONNREFUSED") ||
|
|
1811
|
+
e.message.includes("fetch failed") ||
|
|
1812
|
+
e.message.includes("network")));
|
|
1813
|
+
if (isNetworkError) {
|
|
1814
|
+
log(` ⚠️ 服务端连接失败 (${e.message}),15 秒后重试...`);
|
|
1815
|
+
await new Promise((r) => setTimeout(r, 15000));
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
log(` ⚠️ 处理出错 (${e.message}),10 秒后重试...`);
|
|
1819
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1820
|
+
await recyclePage().catch(() => {});
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
} finally {
|
|
1824
|
+
cleanup();
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
@@ -24,6 +24,7 @@ async function processAPIResponse(
|
|
|
24
24
|
createTime: item.createTime || null,
|
|
25
25
|
playCount: item.stats?.playCount || 0,
|
|
26
26
|
isECVideo: item.isECVideo ? 1 : 0,
|
|
27
|
+
isAd: item.isAd ? 1 : 0,
|
|
27
28
|
});
|
|
28
29
|
}
|
|
29
30
|
|
|
@@ -74,6 +75,7 @@ async function processAPIResponse(
|
|
|
74
75
|
createTime: item.createTime || null,
|
|
75
76
|
playCount: item.stats?.playCount || 0,
|
|
76
77
|
isECVideo: item.isECVideo ? 1 : 0,
|
|
78
|
+
isAd: item.isAd ? 1 : 0,
|
|
77
79
|
});
|
|
78
80
|
}
|
|
79
81
|
}
|
package/src/lib/args.js
CHANGED
|
@@ -670,6 +670,7 @@ function parseCommentsArgs(args) {
|
|
|
670
670
|
let commentsServer = defaultServer;
|
|
671
671
|
let commentsBasePort = 6222;
|
|
672
672
|
let commentsProxy = null;
|
|
673
|
+
let commentsLocations = null;
|
|
673
674
|
|
|
674
675
|
const positional = [];
|
|
675
676
|
|
|
@@ -689,6 +690,12 @@ function parseCommentsArgs(args) {
|
|
|
689
690
|
commentsBasePort = parseInt(args[++i]) || 6222;
|
|
690
691
|
} else if (arg === "--proxy") {
|
|
691
692
|
commentsProxy = args[++i];
|
|
693
|
+
} else if (
|
|
694
|
+
arg === "-l" ||
|
|
695
|
+
arg === "--location" ||
|
|
696
|
+
arg === "--job-locations"
|
|
697
|
+
) {
|
|
698
|
+
commentsLocations = args[++i];
|
|
692
699
|
} else {
|
|
693
700
|
positional.push(arg);
|
|
694
701
|
}
|
|
@@ -709,6 +716,7 @@ function parseCommentsArgs(args) {
|
|
|
709
716
|
commentsServer,
|
|
710
717
|
commentsBasePort,
|
|
711
718
|
commentsProxy,
|
|
719
|
+
commentsLocations,
|
|
712
720
|
urls: [],
|
|
713
721
|
outputFormat: "json",
|
|
714
722
|
exploreCount: 0,
|
|
@@ -738,7 +746,9 @@ function parseTagArgs(args) {
|
|
|
738
746
|
let isDiscover = false;
|
|
739
747
|
let isScore = false;
|
|
740
748
|
let isScoreAll = false;
|
|
749
|
+
let isRefresh = false;
|
|
741
750
|
let scoreAllPort = 7222;
|
|
751
|
+
let refreshPort = 8222;
|
|
742
752
|
let scoreProxy = null;
|
|
743
753
|
let scoreTag = null;
|
|
744
754
|
let scoreCountries = null;
|
|
@@ -786,6 +796,8 @@ function parseTagArgs(args) {
|
|
|
786
796
|
} else if (arg === "--port") {
|
|
787
797
|
if (isScoreAll) {
|
|
788
798
|
scoreAllPort = parseInt(args[++i]) || 7222;
|
|
799
|
+
} else if (isRefresh) {
|
|
800
|
+
refreshPort = parseInt(args[++i]) || 8222;
|
|
789
801
|
} else {
|
|
790
802
|
scorePort = parseInt(args[++i]) || 9222;
|
|
791
803
|
}
|
|
@@ -818,6 +830,8 @@ function parseTagArgs(args) {
|
|
|
818
830
|
if (scoreCountries) i += scoreCountries.length;
|
|
819
831
|
} else if (cleaned.toLowerCase() === "score-all") {
|
|
820
832
|
isScoreAll = true;
|
|
833
|
+
} else if (cleaned.toLowerCase() === "refresh") {
|
|
834
|
+
isRefresh = true;
|
|
821
835
|
} else {
|
|
822
836
|
tags.push(cleaned.toLowerCase());
|
|
823
837
|
}
|
|
@@ -891,6 +905,27 @@ function parseTagArgs(args) {
|
|
|
891
905
|
};
|
|
892
906
|
}
|
|
893
907
|
|
|
908
|
+
if (isRefresh) {
|
|
909
|
+
return {
|
|
910
|
+
subcommand: "tag-refresh",
|
|
911
|
+
tagRefresh: {
|
|
912
|
+
serverUrl,
|
|
913
|
+
port: refreshPort,
|
|
914
|
+
proxy: scoreProxy,
|
|
915
|
+
},
|
|
916
|
+
urls: [],
|
|
917
|
+
outputFormat: "json",
|
|
918
|
+
exploreCount: 0,
|
|
919
|
+
showConfig: false,
|
|
920
|
+
showHelp: false,
|
|
921
|
+
customProxy: null,
|
|
922
|
+
configAction: null,
|
|
923
|
+
configValue: null,
|
|
924
|
+
pipeMode: false,
|
|
925
|
+
filterStr: null,
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
|
|
894
929
|
return {
|
|
895
930
|
subcommand: "tag",
|
|
896
931
|
tagTags: {
|