tt-help-cli-ycl 1.4.13 → 1.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/tag.js +513 -13
- package/src/lib/api-interceptor.js +2 -0
- package/src/lib/args.js +27 -0
- package/src/lib/constants.js +16 -0
- package/src/main.js +3 -0
- package/src/scraper/explore-core.js +1 -0
- package/src/watch/data-store.js +62 -0
- package/src/watch/db-columns.js +2 -0
- package/src/watch/db-crud.js +28 -0
- package/src/watch/db-schema.js +3 -0
- package/src/watch/db-tags.js +95 -0
- package/src/watch/public/app.js +80 -13
- package/src/watch/public/index.html +7 -1
- package/src/watch/public/style.css +43 -0
- package/src/watch/server.js +90 -17
package/package.json
CHANGED
package/src/cli/tag.js
CHANGED
|
@@ -171,6 +171,82 @@ async function pushToServer(
|
|
|
171
171
|
return data;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
/**
|
|
175
|
+
* 带服务端缓存的批量 enrich:先从服务端查询已有国家信息,只对未知的做 HTTP enrich
|
|
176
|
+
* @returns {{ videos: Array, cdnBlockedCount: number }}
|
|
177
|
+
*/
|
|
178
|
+
async function enrichWithServerCache(
|
|
179
|
+
videos,
|
|
180
|
+
baseUrl,
|
|
181
|
+
enrichScraper,
|
|
182
|
+
options = {},
|
|
183
|
+
) {
|
|
184
|
+
const { clientId, clientMeta, effectiveProxy, log } = options;
|
|
185
|
+
const logFn = log || (() => {});
|
|
186
|
+
|
|
187
|
+
// Step 1: 收集所有唯一作者 ID
|
|
188
|
+
const uniqueAuthors = [
|
|
189
|
+
...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
|
|
190
|
+
];
|
|
191
|
+
|
|
192
|
+
// Step 2: 批量查询服务端已知国家
|
|
193
|
+
let knownLocations = {};
|
|
194
|
+
if (uniqueAuthors.length > 0 && baseUrl) {
|
|
195
|
+
try {
|
|
196
|
+
const res = await fetch(`${baseUrl}/api/bulk-author-locations`, {
|
|
197
|
+
method: "POST",
|
|
198
|
+
headers: buildClientHeaders(clientId || "", clientMeta || {}, {
|
|
199
|
+
"Content-Type": "application/json",
|
|
200
|
+
}),
|
|
201
|
+
body: JSON.stringify({ uniqueIds: uniqueAuthors }),
|
|
202
|
+
});
|
|
203
|
+
const data = await res.json();
|
|
204
|
+
knownLocations = data.locations || {};
|
|
205
|
+
const knownCount = Object.keys(knownLocations).length;
|
|
206
|
+
if (knownCount > 0) {
|
|
207
|
+
logFn(
|
|
208
|
+
` 📋 服务端已知 ${knownCount}/${uniqueAuthors.length} 个作者的国家`,
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
} catch {
|
|
212
|
+
// 查询失败则全部走 enrich,不影响主流程
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Step 3: 给已知国家的视频赋值,收集未知视频
|
|
217
|
+
const unknownVideos = [];
|
|
218
|
+
for (const v of videos) {
|
|
219
|
+
const loc = knownLocations[v.authorUniqueId];
|
|
220
|
+
if (loc) {
|
|
221
|
+
v.locationCreated = loc;
|
|
222
|
+
} else {
|
|
223
|
+
unknownVideos.push(v);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Step 4: 只对未知视频做 HTTP enrich
|
|
228
|
+
let cdnBlockedCount = 0;
|
|
229
|
+
if (unknownVideos.length > 0) {
|
|
230
|
+
const enrichOpts = {
|
|
231
|
+
mode: "videos",
|
|
232
|
+
existingScraper: enrichScraper,
|
|
233
|
+
onProgress: ({ done, total, current, locationCreated }) => {
|
|
234
|
+
if (done % 10 === 0 || done === total) {
|
|
235
|
+
process.stderr.write(
|
|
236
|
+
`\r [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}\x1b[K`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
|
|
242
|
+
const enriched = await enrichVideosWithLocation(unknownVideos, enrichOpts);
|
|
243
|
+
// 将 enrich 结果写回原数组(v 是引用,已经被 enrichVideosWithLocation 修改)
|
|
244
|
+
cdnBlockedCount = enriched.cdnBlockedCount || 0;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return { videos, cdnBlockedCount };
|
|
248
|
+
}
|
|
249
|
+
|
|
174
250
|
// 共享算分逻辑(handleScore / handleScoreAll 共用)
|
|
175
251
|
function calcScore(authorCount, matchedAuthors, filteredVideoCount) {
|
|
176
252
|
const densityScore =
|
|
@@ -1002,20 +1078,18 @@ export async function handleScoreAll(parsed) {
|
|
|
1002
1078
|
continue;
|
|
1003
1079
|
}
|
|
1004
1080
|
|
|
1005
|
-
// enrich:
|
|
1006
|
-
const
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1081
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1082
|
+
const enriched = await enrichWithServerCache(
|
|
1083
|
+
videos,
|
|
1084
|
+
baseUrl,
|
|
1085
|
+
enrichScraper,
|
|
1086
|
+
{
|
|
1087
|
+
clientId,
|
|
1088
|
+
clientMeta,
|
|
1089
|
+
effectiveProxy,
|
|
1090
|
+
log,
|
|
1015
1091
|
},
|
|
1016
|
-
|
|
1017
|
-
if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
|
|
1018
|
-
const enriched = await enrichVideosWithLocation(videos, enrichOpts);
|
|
1092
|
+
);
|
|
1019
1093
|
videos = enriched.videos;
|
|
1020
1094
|
|
|
1021
1095
|
// CDN 限流检测:有拦截则冷却 + 重启 scraper
|
|
@@ -1324,3 +1398,429 @@ export async function handleTag(parsed) {
|
|
|
1324
1398
|
console.log(json);
|
|
1325
1399
|
}
|
|
1326
1400
|
}
|
|
1401
|
+
|
|
1402
|
+
// ====== Productive 标签刷新(重新抓取,不打分) ======
|
|
1403
|
+
|
|
1404
|
+
export async function handleTagRefresh(parsed) {
|
|
1405
|
+
const { tagRefresh } = parsed;
|
|
1406
|
+
const { serverUrl, port, proxy: cliProxy } = tagRefresh || {};
|
|
1407
|
+
|
|
1408
|
+
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
1409
|
+
const defaultRefreshPort =
|
|
1410
|
+
parseInt(process.env.TAG_REFRESH_PORT_POOL_START || "4222", 10) || 4222;
|
|
1411
|
+
let cdpPort = port || defaultRefreshPort;
|
|
1412
|
+
const effectiveProxy = cliProxy || configuredProxy;
|
|
1413
|
+
|
|
1414
|
+
const log = (...args) => process.stderr.write(args.join(" ") + "\n");
|
|
1415
|
+
|
|
1416
|
+
log("");
|
|
1417
|
+
log("╔══════════════════════════════════════════╗");
|
|
1418
|
+
log("║ Productive 标签刷新模式(重新抓取)");
|
|
1419
|
+
log("║ 服务端: " + baseUrl);
|
|
1420
|
+
log(
|
|
1421
|
+
"║ CDP 端口: " +
|
|
1422
|
+
cdpPort +
|
|
1423
|
+
(effectiveProxy ? " 代理: " + effectiveProxy : ""),
|
|
1424
|
+
);
|
|
1425
|
+
log("║ 只抓取不重新打分,score>70 的标签");
|
|
1426
|
+
log("╚══════════════════════════════════════════╝");
|
|
1427
|
+
log("");
|
|
1428
|
+
|
|
1429
|
+
log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
|
|
1430
|
+
log("");
|
|
1431
|
+
|
|
1432
|
+
// ── 设备标识 ──
|
|
1433
|
+
let userId = configuredUserId;
|
|
1434
|
+
if (!userId) {
|
|
1435
|
+
userId = await getMacOrUuid();
|
|
1436
|
+
saveUserId(userId);
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
// 连接 CDP 浏览器
|
|
1440
|
+
const cdpOpts = { port: cdpPort };
|
|
1441
|
+
if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
|
|
1442
|
+
let browser = await ensureBrowserReady(cdpOpts);
|
|
1443
|
+
let page = await getOrCreatePage(browser);
|
|
1444
|
+
const blockedRoutePages = new WeakSet();
|
|
1445
|
+
|
|
1446
|
+
async function setupPageRequestBlocking(targetPage) {
|
|
1447
|
+
if (!targetPage || blockedRoutePages.has(targetPage)) return;
|
|
1448
|
+
await targetPage.route("**/*", (route) => {
|
|
1449
|
+
const resourceType = route.request().resourceType();
|
|
1450
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
1451
|
+
route.abort();
|
|
1452
|
+
} else {
|
|
1453
|
+
route.continue();
|
|
1454
|
+
}
|
|
1455
|
+
});
|
|
1456
|
+
blockedRoutePages.add(targetPage);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
await setupPageRequestBlocking(page);
|
|
1460
|
+
|
|
1461
|
+
// 端口池
|
|
1462
|
+
const portPoolStart = cdpPort;
|
|
1463
|
+
const portPoolSize = Math.max(
|
|
1464
|
+
2,
|
|
1465
|
+
parseInt(process.env.TAG_REFRESH_PORT_POOL_SIZE || "10", 10) || 10,
|
|
1466
|
+
);
|
|
1467
|
+
|
|
1468
|
+
function pickNextPort(currentPort) {
|
|
1469
|
+
const candidates = [];
|
|
1470
|
+
for (let i = 0; i < portPoolSize; i++) {
|
|
1471
|
+
const p = portPoolStart + i;
|
|
1472
|
+
if (p !== currentPort) candidates.push(p);
|
|
1473
|
+
}
|
|
1474
|
+
if (candidates.length === 0) return currentPort;
|
|
1475
|
+
return candidates[Math.floor(Math.random() * candidates.length)];
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
// ── 内存回收配置 ──
|
|
1479
|
+
const RECYCLE_CONFIG = {
|
|
1480
|
+
nodeRssMb: Math.max(
|
|
1481
|
+
256,
|
|
1482
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_RSS_MB || "2000", 10) || 2000,
|
|
1483
|
+
),
|
|
1484
|
+
nodeHeapMb: Math.max(
|
|
1485
|
+
128,
|
|
1486
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_HEAP_MB || "2000", 10) || 2000,
|
|
1487
|
+
),
|
|
1488
|
+
edgeMb: Math.max(
|
|
1489
|
+
500,
|
|
1490
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_EDGE_MB || "2000", 10) || 2000,
|
|
1491
|
+
),
|
|
1492
|
+
cooldownMs: Math.max(
|
|
1493
|
+
0,
|
|
1494
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_COOLDOWN_MS || "180000", 10) ||
|
|
1495
|
+
180000,
|
|
1496
|
+
),
|
|
1497
|
+
periodicEvery: Math.max(
|
|
1498
|
+
0,
|
|
1499
|
+
parseInt(process.env.TAG_REFRESH_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
|
|
1500
|
+
),
|
|
1501
|
+
switchPortOnRecycle:
|
|
1502
|
+
String(process.env.TAG_REFRESH_SWITCH_PORT_ON_RECYCLE || "1") !== "0",
|
|
1503
|
+
};
|
|
1504
|
+
let lastRecycleAt = 0;
|
|
1505
|
+
|
|
1506
|
+
// ── 轻量回收 ──
|
|
1507
|
+
async function recyclePage() {
|
|
1508
|
+
if (!page || page.isClosed()) {
|
|
1509
|
+
page = await getOrCreatePage(browser);
|
|
1510
|
+
await setupPageRequestBlocking(page);
|
|
1511
|
+
return;
|
|
1512
|
+
}
|
|
1513
|
+
try {
|
|
1514
|
+
await page.goto("about:blank", {
|
|
1515
|
+
waitUntil: "domcontentloaded",
|
|
1516
|
+
timeout: 5000,
|
|
1517
|
+
});
|
|
1518
|
+
} catch {
|
|
1519
|
+
await page.close().catch(() => {});
|
|
1520
|
+
page = await getOrCreatePage(browser);
|
|
1521
|
+
await setupPageRequestBlocking(page);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
// ── 深度回收 ──
|
|
1526
|
+
async function recycleCdpSession(reason, { switchPort = true } = {}) {
|
|
1527
|
+
const oldPort = cdpPort;
|
|
1528
|
+
const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
|
|
1529
|
+
if (shouldSwitch) {
|
|
1530
|
+
cdpPort = pickNextPort(cdpPort);
|
|
1531
|
+
cdpOpts.port = cdpPort;
|
|
1532
|
+
clientMeta.port = cdpPort;
|
|
1533
|
+
}
|
|
1534
|
+
const portHint =
|
|
1535
|
+
oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
|
|
1536
|
+
log(` ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
|
|
1537
|
+
if (page) {
|
|
1538
|
+
await page.close().catch(() => {});
|
|
1539
|
+
page = null;
|
|
1540
|
+
}
|
|
1541
|
+
await browser.close().catch(() => {});
|
|
1542
|
+
if (oldPort !== cdpPort) {
|
|
1543
|
+
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
1544
|
+
}
|
|
1545
|
+
browser = await ensureBrowserReady(cdpOpts);
|
|
1546
|
+
page = await getOrCreatePage(browser);
|
|
1547
|
+
await setupPageRequestBlocking(page);
|
|
1548
|
+
lastRecycleAt = Date.now();
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
async function maybeRecycleForMemory() {
|
|
1552
|
+
const mem = process.memoryUsage();
|
|
1553
|
+
const rssMB = mem.rss / 1024 / 1024;
|
|
1554
|
+
const heapMB = mem.heapUsed / 1024 / 1024;
|
|
1555
|
+
const edgeMB = getBrowserMemory(cdpPort);
|
|
1556
|
+
|
|
1557
|
+
const nodeOver =
|
|
1558
|
+
rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
|
|
1559
|
+
const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
|
|
1560
|
+
|
|
1561
|
+
if (nodeOver || edgeOver) {
|
|
1562
|
+
if (
|
|
1563
|
+
RECYCLE_CONFIG.cooldownMs > 0 &&
|
|
1564
|
+
Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
|
|
1565
|
+
)
|
|
1566
|
+
return;
|
|
1567
|
+
|
|
1568
|
+
const parts = [];
|
|
1569
|
+
if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
|
|
1570
|
+
parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
|
|
1571
|
+
if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
|
|
1572
|
+
parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
|
|
1573
|
+
if (edgeOver)
|
|
1574
|
+
parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
|
|
1575
|
+
|
|
1576
|
+
await recycleCdpSession(`mem ${parts.join("+")}`, {
|
|
1577
|
+
switchPort: edgeOver,
|
|
1578
|
+
});
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
if (
|
|
1583
|
+
RECYCLE_CONFIG.periodicEvery > 0 &&
|
|
1584
|
+
totalRefreshed > 0 &&
|
|
1585
|
+
totalRefreshed % RECYCLE_CONFIG.periodicEvery === 0
|
|
1586
|
+
) {
|
|
1587
|
+
await recycleCdpSession(`periodic #${totalRefreshed}`, {
|
|
1588
|
+
switchPort: false,
|
|
1589
|
+
});
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
let totalRefreshed = 0;
|
|
1594
|
+
let emptyRounds = 0;
|
|
1595
|
+
let consecutiveCaptcha = 0;
|
|
1596
|
+
const MAX_CONSECUTIVE_CAPTCHA = 3;
|
|
1597
|
+
|
|
1598
|
+
const clientId = randomUUID();
|
|
1599
|
+
const clientMeta = { type: "tag-refresh", port: cdpPort };
|
|
1600
|
+
|
|
1601
|
+
// 复用 TikTokScraper
|
|
1602
|
+
const enrichScraper = new TikTokScraper({
|
|
1603
|
+
poolSize: 3,
|
|
1604
|
+
proxyServer: effectiveProxy || undefined,
|
|
1605
|
+
});
|
|
1606
|
+
await enrichScraper.init();
|
|
1607
|
+
log("✅ TikTokScraper 已就绪 (enrich 复用)");
|
|
1608
|
+
log(` 客户端 ID: ${clientId.substring(0, 8)}...`);
|
|
1609
|
+
log("");
|
|
1610
|
+
|
|
1611
|
+
const cleanup = () => {
|
|
1612
|
+
log("\n正在清理资源...");
|
|
1613
|
+
enrichScraper.close().catch(() => {});
|
|
1614
|
+
killEdgeProcesses(null, cdpPort);
|
|
1615
|
+
process.exit(0);
|
|
1616
|
+
};
|
|
1617
|
+
process.on("SIGINT", cleanup);
|
|
1618
|
+
|
|
1619
|
+
try {
|
|
1620
|
+
while (true) {
|
|
1621
|
+
try {
|
|
1622
|
+
// 从服务端获取下一个 productive 标签
|
|
1623
|
+
const taskRes = await fetch(`${baseUrl}/api/tags/refresh-task`, {
|
|
1624
|
+
headers: buildClientHeaders(clientId, clientMeta),
|
|
1625
|
+
});
|
|
1626
|
+
const taskData = await taskRes.json();
|
|
1627
|
+
|
|
1628
|
+
if (!taskData.tag) {
|
|
1629
|
+
emptyRounds++;
|
|
1630
|
+
|
|
1631
|
+
if (emptyRounds >= 5) {
|
|
1632
|
+
log(
|
|
1633
|
+
`⏳ 连续 ${emptyRounds} 轮无 productive 标签可刷新(30 秒后重试)...`,
|
|
1634
|
+
);
|
|
1635
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
1636
|
+
emptyRounds = 0;
|
|
1637
|
+
continue;
|
|
1638
|
+
}
|
|
1639
|
+
log(
|
|
1640
|
+
`⏳ 暂无 productive 标签可刷新(连续 ${emptyRounds} 轮),10 秒后重试...`,
|
|
1641
|
+
);
|
|
1642
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
emptyRounds = 0;
|
|
1647
|
+
|
|
1648
|
+
const tag = taskData.tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1649
|
+
const startTime = Date.now();
|
|
1650
|
+
|
|
1651
|
+
log(
|
|
1652
|
+
`[${totalRefreshed + 1}] 正在刷新 #${tag} (score: ${taskData.score})...`,
|
|
1653
|
+
);
|
|
1654
|
+
|
|
1655
|
+
const result = {
|
|
1656
|
+
tag,
|
|
1657
|
+
pushedUsers: 0,
|
|
1658
|
+
error: null,
|
|
1659
|
+
};
|
|
1660
|
+
|
|
1661
|
+
// 锁定标签
|
|
1662
|
+
const claimRes = await fetch(`${baseUrl}/api/tags/refresh-claim`, {
|
|
1663
|
+
method: "POST",
|
|
1664
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1665
|
+
"Content-Type": "application/json",
|
|
1666
|
+
}),
|
|
1667
|
+
body: JSON.stringify({ tag }),
|
|
1668
|
+
});
|
|
1669
|
+
const claimData = await claimRes.json();
|
|
1670
|
+
if (!claimData.ok) {
|
|
1671
|
+
if (
|
|
1672
|
+
claimData.error &&
|
|
1673
|
+
claimData.error.includes("already being refreshed")
|
|
1674
|
+
) {
|
|
1675
|
+
log(` ⏭️ 已被其他客户端锁定,跳过`);
|
|
1676
|
+
await recyclePage();
|
|
1677
|
+
continue;
|
|
1678
|
+
}
|
|
1679
|
+
log(` ⚠️ 无法锁定 (${claimData.error}),重试下一个`);
|
|
1680
|
+
await recyclePage();
|
|
1681
|
+
continue;
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
// 抓取视频
|
|
1685
|
+
const fetchStart = Date.now();
|
|
1686
|
+
log(` 抓取 TikTok 标签页...`);
|
|
1687
|
+
const tagResult = await fetchTagData(tag, {
|
|
1688
|
+
page,
|
|
1689
|
+
onProgress: ({ videos, authors }) => {
|
|
1690
|
+
process.stderr.write(
|
|
1691
|
+
`\r 抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
|
|
1692
|
+
);
|
|
1693
|
+
},
|
|
1694
|
+
});
|
|
1695
|
+
const fetchSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1696
|
+
log(
|
|
1697
|
+
`\r 完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者 (${fetchSec}s)\x1b[K`,
|
|
1698
|
+
);
|
|
1699
|
+
|
|
1700
|
+
let videos = tagResult.videos;
|
|
1701
|
+
|
|
1702
|
+
if (!videos || videos.length === 0) {
|
|
1703
|
+
const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1704
|
+
log(` ⚠️ 无视频 (${deadSec}s),保留 productive 状态,等待下次刷新`);
|
|
1705
|
+
result.error = "no videos found";
|
|
1706
|
+
await fetch(`${baseUrl}/api/tags/refresh-result`, {
|
|
1707
|
+
method: "POST",
|
|
1708
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1709
|
+
"Content-Type": "application/json",
|
|
1710
|
+
}),
|
|
1711
|
+
body: JSON.stringify({ tag, pushedUsers: 0, error: result.error }),
|
|
1712
|
+
}).catch(() => {});
|
|
1713
|
+
totalRefreshed++;
|
|
1714
|
+
await randomDelay(0, 5000);
|
|
1715
|
+
await recyclePage();
|
|
1716
|
+
await maybeRecycleForMemory();
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1721
|
+
const enriched = await enrichWithServerCache(
|
|
1722
|
+
videos,
|
|
1723
|
+
baseUrl,
|
|
1724
|
+
enrichScraper,
|
|
1725
|
+
{
|
|
1726
|
+
clientId,
|
|
1727
|
+
clientMeta,
|
|
1728
|
+
effectiveProxy,
|
|
1729
|
+
log,
|
|
1730
|
+
},
|
|
1731
|
+
);
|
|
1732
|
+
videos = enriched.videos;
|
|
1733
|
+
|
|
1734
|
+
// CDN 限流检测
|
|
1735
|
+
const cdnBlocked = enriched.cdnBlockedCount || 0;
|
|
1736
|
+
if (cdnBlocked > 0) {
|
|
1737
|
+
const cdnRatio = cdnBlocked / (videos.length || 1);
|
|
1738
|
+
const coolSec = cdnRatio > 0.3 ? 120 : 60;
|
|
1739
|
+
log(
|
|
1740
|
+
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),冷却 ${coolSec} 秒后重启 scraper`,
|
|
1741
|
+
);
|
|
1742
|
+
await new Promise((r) => setTimeout(r, coolSec * 1000));
|
|
1743
|
+
log(` 正在重启 TikTokScraper...`);
|
|
1744
|
+
await enrichScraper.restart();
|
|
1745
|
+
log(` ✅ TikTokScraper 已重启`);
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
// 过滤目标国家的视频和作者
|
|
1749
|
+
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
1750
|
+
const filteredVideos = videos.filter((v) =>
|
|
1751
|
+
isLocationInList(v.locationCreated, targetLocations),
|
|
1752
|
+
);
|
|
1753
|
+
const filteredAuthors = [
|
|
1754
|
+
...new Set(
|
|
1755
|
+
filteredVideos.map((v) => v.authorUniqueId).filter(Boolean),
|
|
1756
|
+
),
|
|
1757
|
+
];
|
|
1758
|
+
const filteredCount = videos.length - filteredVideos.length;
|
|
1759
|
+
if (filteredCount > 0) {
|
|
1760
|
+
log(` 过滤非目标国家: ${filteredCount}/${videos.length} 视频被排除`);
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// 推送过滤后的作者到服务端
|
|
1764
|
+
if (filteredAuthors.length > 0) {
|
|
1765
|
+
const pushResult = await pushToServer(
|
|
1766
|
+
baseUrl,
|
|
1767
|
+
filteredAuthors,
|
|
1768
|
+
filteredVideos,
|
|
1769
|
+
clientId,
|
|
1770
|
+
clientMeta,
|
|
1771
|
+
);
|
|
1772
|
+
result.pushedUsers = pushResult.added || 0;
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
// 上报刷新结果(只更新 pushed_users,不改变 score/status)
|
|
1776
|
+
await fetch(`${baseUrl}/api/tags/refresh-result`, {
|
|
1777
|
+
method: "POST",
|
|
1778
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1779
|
+
"Content-Type": "application/json",
|
|
1780
|
+
}),
|
|
1781
|
+
body: JSON.stringify({
|
|
1782
|
+
tag,
|
|
1783
|
+
pushedUsers: result.pushedUsers,
|
|
1784
|
+
error: result.error,
|
|
1785
|
+
}),
|
|
1786
|
+
}).catch(() => {});
|
|
1787
|
+
|
|
1788
|
+
totalRefreshed++;
|
|
1789
|
+
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
1790
|
+
const memStr = ` mem=${formatMemoryWithBrowser(cdpPort)}`;
|
|
1791
|
+
log(
|
|
1792
|
+
` 🔄 #${tag} 完成 (${elapsed}s) — ${videos.length} 视频(${filteredVideos.length} 匹配), ${filteredAuthors.length} 作者, 推送 ${result.pushedUsers} 用户${memStr}`,
|
|
1793
|
+
);
|
|
1794
|
+
log("");
|
|
1795
|
+
|
|
1796
|
+
if (totalRefreshed % 10 === 0) {
|
|
1797
|
+
log(
|
|
1798
|
+
`📊 [${totalRefreshed}] 内存汇总: ${formatMemoryWithBrowser(cdpPort)}`,
|
|
1799
|
+
);
|
|
1800
|
+
}
|
|
1801
|
+
await recyclePage();
|
|
1802
|
+
await maybeRecycleForMemory();
|
|
1803
|
+
await randomDelay(3000, 7000);
|
|
1804
|
+
} catch (e) {
|
|
1805
|
+
const isNetworkError =
|
|
1806
|
+
e.code === "ECONNREFUSED" ||
|
|
1807
|
+
e.code === "ENOTFOUND" ||
|
|
1808
|
+
e.code === "ECONNRESET" ||
|
|
1809
|
+
(e.message &&
|
|
1810
|
+
(e.message.includes("ECONNREFUSED") ||
|
|
1811
|
+
e.message.includes("fetch failed") ||
|
|
1812
|
+
e.message.includes("network")));
|
|
1813
|
+
if (isNetworkError) {
|
|
1814
|
+
log(` ⚠️ 服务端连接失败 (${e.message}),15 秒后重试...`);
|
|
1815
|
+
await new Promise((r) => setTimeout(r, 15000));
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
log(` ⚠️ 处理出错 (${e.message}),10 秒后重试...`);
|
|
1819
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1820
|
+
await recyclePage().catch(() => {});
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
} finally {
|
|
1824
|
+
cleanup();
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
@@ -24,6 +24,7 @@ async function processAPIResponse(
|
|
|
24
24
|
createTime: item.createTime || null,
|
|
25
25
|
playCount: item.stats?.playCount || 0,
|
|
26
26
|
isECVideo: item.isECVideo ? 1 : 0,
|
|
27
|
+
isAd: item.isAd ? 1 : 0,
|
|
27
28
|
});
|
|
28
29
|
}
|
|
29
30
|
|
|
@@ -74,6 +75,7 @@ async function processAPIResponse(
|
|
|
74
75
|
createTime: item.createTime || null,
|
|
75
76
|
playCount: item.stats?.playCount || 0,
|
|
76
77
|
isECVideo: item.isECVideo ? 1 : 0,
|
|
78
|
+
isAd: item.isAd ? 1 : 0,
|
|
77
79
|
});
|
|
78
80
|
}
|
|
79
81
|
}
|
package/src/lib/args.js
CHANGED
|
@@ -746,7 +746,9 @@ function parseTagArgs(args) {
|
|
|
746
746
|
let isDiscover = false;
|
|
747
747
|
let isScore = false;
|
|
748
748
|
let isScoreAll = false;
|
|
749
|
+
let isRefresh = false;
|
|
749
750
|
let scoreAllPort = 7222;
|
|
751
|
+
let refreshPort = 4222;
|
|
750
752
|
let scoreProxy = null;
|
|
751
753
|
let scoreTag = null;
|
|
752
754
|
let scoreCountries = null;
|
|
@@ -794,6 +796,8 @@ function parseTagArgs(args) {
|
|
|
794
796
|
} else if (arg === "--port") {
|
|
795
797
|
if (isScoreAll) {
|
|
796
798
|
scoreAllPort = parseInt(args[++i]) || 7222;
|
|
799
|
+
} else if (isRefresh) {
|
|
800
|
+
refreshPort = parseInt(args[++i]) || 8222;
|
|
797
801
|
} else {
|
|
798
802
|
scorePort = parseInt(args[++i]) || 9222;
|
|
799
803
|
}
|
|
@@ -826,6 +830,8 @@ function parseTagArgs(args) {
|
|
|
826
830
|
if (scoreCountries) i += scoreCountries.length;
|
|
827
831
|
} else if (cleaned.toLowerCase() === "score-all") {
|
|
828
832
|
isScoreAll = true;
|
|
833
|
+
} else if (cleaned.toLowerCase() === "refresh") {
|
|
834
|
+
isRefresh = true;
|
|
829
835
|
} else {
|
|
830
836
|
tags.push(cleaned.toLowerCase());
|
|
831
837
|
}
|
|
@@ -899,6 +905,27 @@ function parseTagArgs(args) {
|
|
|
899
905
|
};
|
|
900
906
|
}
|
|
901
907
|
|
|
908
|
+
if (isRefresh) {
|
|
909
|
+
return {
|
|
910
|
+
subcommand: "tag-refresh",
|
|
911
|
+
tagRefresh: {
|
|
912
|
+
serverUrl,
|
|
913
|
+
port: refreshPort,
|
|
914
|
+
proxy: scoreProxy,
|
|
915
|
+
},
|
|
916
|
+
urls: [],
|
|
917
|
+
outputFormat: "json",
|
|
918
|
+
exploreCount: 0,
|
|
919
|
+
showConfig: false,
|
|
920
|
+
showHelp: false,
|
|
921
|
+
customProxy: null,
|
|
922
|
+
configAction: null,
|
|
923
|
+
configValue: null,
|
|
924
|
+
pipeMode: false,
|
|
925
|
+
filterStr: null,
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
|
|
902
929
|
return {
|
|
903
930
|
subcommand: "tag",
|
|
904
931
|
tagTags: {
|
package/src/lib/constants.js
CHANGED
|
@@ -276,6 +276,22 @@ const HELP_TEXT = [
|
|
|
276
276
|
" tt-help tag score-all --countries ES -s http://127.0.0.1:3001",
|
|
277
277
|
" tt-help tag score-all --port 9222 -s http://127.0.0.1:3001 --discover",
|
|
278
278
|
"",
|
|
279
|
+
" tag refresh [选项]",
|
|
280
|
+
" 轮询刷新 productive 标签(score>70):重新抓取并推送用户,不重新打分",
|
|
281
|
+
" enrich 浏览器实例在整个循环中复用",
|
|
282
|
+
" 选项:",
|
|
283
|
+
" -s, --server <URL> 服务端地址(默认 http://127.0.0.1:3000)",
|
|
284
|
+
" --port <N> 起始 CDP 端口,端口池默认为 N~N+9(默认 4222)",
|
|
285
|
+
" 行为:",
|
|
286
|
+
" 按 refreshed_at ASC 轮询 productive 标签,防重锁保证多客户端不重复",
|
|
287
|
+
" 环境变量:",
|
|
288
|
+
" TAG_REFRESH_PORT_POOL_SIZE=N 端口池大小(默认 10)",
|
|
289
|
+
" TAG_REFRESH_RECYCLE_RSS_MB=N 内存回收阈值 RSS MB(默认 2000)",
|
|
290
|
+
" TAG_REFRESH_RECYCLE_HEAP_MB=N 内存回收阈值 Heap MB(默认 2000)",
|
|
291
|
+
" 示例: tt-help tag refresh",
|
|
292
|
+
" tt-help tag refresh -s http://127.0.0.1:3001",
|
|
293
|
+
" tt-help tag refresh --port 8222 -s http://127.0.0.1:3001",
|
|
294
|
+
"",
|
|
279
295
|
" config [show|set|unset|reset]",
|
|
280
296
|
" config 查看当前配置",
|
|
281
297
|
" config set <key> <value> 设置配置(key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments)",
|
package/src/main.js
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
handleDiscover,
|
|
17
17
|
handleScore,
|
|
18
18
|
handleScoreAll,
|
|
19
|
+
handleTagRefresh,
|
|
19
20
|
} from "./cli/tag.js";
|
|
20
21
|
|
|
21
22
|
async function main() {
|
|
@@ -50,6 +51,8 @@ async function main() {
|
|
|
50
51
|
return handleScore(parsed);
|
|
51
52
|
case "tag-score-all":
|
|
52
53
|
return handleScoreAll(parsed);
|
|
54
|
+
case "tag-refresh":
|
|
55
|
+
return handleTagRefresh(parsed);
|
|
53
56
|
}
|
|
54
57
|
|
|
55
58
|
const {
|
|
@@ -105,6 +105,7 @@ async function processExplore(page, username, options, log) {
|
|
|
105
105
|
href: topVideo.href,
|
|
106
106
|
playCount: topVideo.playCount,
|
|
107
107
|
createTime: topVideo.createTime,
|
|
108
|
+
isAd: topVideo.isAd,
|
|
108
109
|
};
|
|
109
110
|
log(
|
|
110
111
|
` 7天内 EC视频最高播放: ${topVideo.playCount} 次播放 (${recentVideos.length} 个EC候选)`,
|