tt-help-cli-ycl 1.4.13 → 1.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/tag.js +513 -13
- package/src/lib/api-interceptor.js +2 -0
- package/src/lib/args.js +27 -0
- package/src/main.js +3 -0
- package/src/scraper/explore-core.js +1 -0
- package/src/watch/data-store.js +62 -0
- package/src/watch/db-columns.js +2 -0
- package/src/watch/db-crud.js +28 -0
- package/src/watch/db-schema.js +3 -0
- package/src/watch/db-tags.js +95 -0
- package/src/watch/public/app.js +80 -13
- package/src/watch/public/index.html +7 -1
- package/src/watch/public/style.css +43 -0
- package/src/watch/server.js +90 -17
package/package.json
CHANGED
package/src/cli/tag.js
CHANGED
|
@@ -171,6 +171,82 @@ async function pushToServer(
|
|
|
171
171
|
return data;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
/**
|
|
175
|
+
* 带服务端缓存的批量 enrich:先从服务端查询已有国家信息,只对未知的做 HTTP enrich
|
|
176
|
+
* @returns {{ videos: Array, cdnBlockedCount: number }}
|
|
177
|
+
*/
|
|
178
|
+
async function enrichWithServerCache(
|
|
179
|
+
videos,
|
|
180
|
+
baseUrl,
|
|
181
|
+
enrichScraper,
|
|
182
|
+
options = {},
|
|
183
|
+
) {
|
|
184
|
+
const { clientId, clientMeta, effectiveProxy, log } = options;
|
|
185
|
+
const logFn = log || (() => {});
|
|
186
|
+
|
|
187
|
+
// Step 1: 收集所有唯一作者 ID
|
|
188
|
+
const uniqueAuthors = [
|
|
189
|
+
...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
|
|
190
|
+
];
|
|
191
|
+
|
|
192
|
+
// Step 2: 批量查询服务端已知国家
|
|
193
|
+
let knownLocations = {};
|
|
194
|
+
if (uniqueAuthors.length > 0 && baseUrl) {
|
|
195
|
+
try {
|
|
196
|
+
const res = await fetch(`${baseUrl}/api/bulk-author-locations`, {
|
|
197
|
+
method: "POST",
|
|
198
|
+
headers: buildClientHeaders(clientId || "", clientMeta || {}, {
|
|
199
|
+
"Content-Type": "application/json",
|
|
200
|
+
}),
|
|
201
|
+
body: JSON.stringify({ uniqueIds: uniqueAuthors }),
|
|
202
|
+
});
|
|
203
|
+
const data = await res.json();
|
|
204
|
+
knownLocations = data.locations || {};
|
|
205
|
+
const knownCount = Object.keys(knownLocations).length;
|
|
206
|
+
if (knownCount > 0) {
|
|
207
|
+
logFn(
|
|
208
|
+
` 📋 服务端已知 ${knownCount}/${uniqueAuthors.length} 个作者的国家`,
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
} catch {
|
|
212
|
+
// 查询失败则全部走 enrich,不影响主流程
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Step 3: 给已知国家的视频赋值,收集未知视频
|
|
217
|
+
const unknownVideos = [];
|
|
218
|
+
for (const v of videos) {
|
|
219
|
+
const loc = knownLocations[v.authorUniqueId];
|
|
220
|
+
if (loc) {
|
|
221
|
+
v.locationCreated = loc;
|
|
222
|
+
} else {
|
|
223
|
+
unknownVideos.push(v);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Step 4: 只对未知视频做 HTTP enrich
|
|
228
|
+
let cdnBlockedCount = 0;
|
|
229
|
+
if (unknownVideos.length > 0) {
|
|
230
|
+
const enrichOpts = {
|
|
231
|
+
mode: "videos",
|
|
232
|
+
existingScraper: enrichScraper,
|
|
233
|
+
onProgress: ({ done, total, current, locationCreated }) => {
|
|
234
|
+
if (done % 10 === 0 || done === total) {
|
|
235
|
+
process.stderr.write(
|
|
236
|
+
`\r [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}\x1b[K`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
|
|
242
|
+
const enriched = await enrichVideosWithLocation(unknownVideos, enrichOpts);
|
|
243
|
+
// 将 enrich 结果写回原数组(v 是引用,已经被 enrichVideosWithLocation 修改)
|
|
244
|
+
cdnBlockedCount = enriched.cdnBlockedCount || 0;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return { videos, cdnBlockedCount };
|
|
248
|
+
}
|
|
249
|
+
|
|
174
250
|
// 共享算分逻辑(handleScore / handleScoreAll 共用)
|
|
175
251
|
function calcScore(authorCount, matchedAuthors, filteredVideoCount) {
|
|
176
252
|
const densityScore =
|
|
@@ -1002,20 +1078,18 @@ export async function handleScoreAll(parsed) {
|
|
|
1002
1078
|
continue;
|
|
1003
1079
|
}
|
|
1004
1080
|
|
|
1005
|
-
// enrich:
|
|
1006
|
-
const
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1081
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1082
|
+
const enriched = await enrichWithServerCache(
|
|
1083
|
+
videos,
|
|
1084
|
+
baseUrl,
|
|
1085
|
+
enrichScraper,
|
|
1086
|
+
{
|
|
1087
|
+
clientId,
|
|
1088
|
+
clientMeta,
|
|
1089
|
+
effectiveProxy,
|
|
1090
|
+
log,
|
|
1015
1091
|
},
|
|
1016
|
-
|
|
1017
|
-
if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
|
|
1018
|
-
const enriched = await enrichVideosWithLocation(videos, enrichOpts);
|
|
1092
|
+
);
|
|
1019
1093
|
videos = enriched.videos;
|
|
1020
1094
|
|
|
1021
1095
|
// CDN 限流检测:有拦截则冷却 + 重启 scraper
|
|
@@ -1324,3 +1398,429 @@ export async function handleTag(parsed) {
|
|
|
1324
1398
|
console.log(json);
|
|
1325
1399
|
}
|
|
1326
1400
|
}
|
|
1401
|
+
|
|
1402
|
+
// ====== Productive 标签刷新(重新抓取,不打分) ======
|
|
1403
|
+
|
|
1404
|
+
export async function handleTagRefresh(parsed) {
|
|
1405
|
+
const { tagRefresh } = parsed;
|
|
1406
|
+
const { serverUrl, port, proxy: cliProxy } = tagRefresh || {};
|
|
1407
|
+
|
|
1408
|
+
const baseUrl = serverUrl || DEFAULT_SERVER;
|
|
1409
|
+
const defaultRefreshPort =
|
|
1410
|
+
parseInt(process.env.TAG_REFRESH_PORT_POOL_START || "8222", 10) || 8222;
|
|
1411
|
+
let cdpPort = port || defaultRefreshPort;
|
|
1412
|
+
const effectiveProxy = cliProxy || configuredProxy;
|
|
1413
|
+
|
|
1414
|
+
const log = (...args) => process.stderr.write(args.join(" ") + "\n");
|
|
1415
|
+
|
|
1416
|
+
log("");
|
|
1417
|
+
log("╔══════════════════════════════════════════╗");
|
|
1418
|
+
log("║ Productive 标签刷新模式(重新抓取)");
|
|
1419
|
+
log("║ 服务端: " + baseUrl);
|
|
1420
|
+
log(
|
|
1421
|
+
"║ CDP 端口: " +
|
|
1422
|
+
cdpPort +
|
|
1423
|
+
(effectiveProxy ? " 代理: " + effectiveProxy : ""),
|
|
1424
|
+
);
|
|
1425
|
+
log("║ 只抓取不重新打分,score>70 的标签");
|
|
1426
|
+
log("╚══════════════════════════════════════════╝");
|
|
1427
|
+
log("");
|
|
1428
|
+
|
|
1429
|
+
log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
|
|
1430
|
+
log("");
|
|
1431
|
+
|
|
1432
|
+
// ── 设备标识 ──
|
|
1433
|
+
let userId = configuredUserId;
|
|
1434
|
+
if (!userId) {
|
|
1435
|
+
userId = await getMacOrUuid();
|
|
1436
|
+
saveUserId(userId);
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
// 连接 CDP 浏览器
|
|
1440
|
+
const cdpOpts = { port: cdpPort };
|
|
1441
|
+
if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
|
|
1442
|
+
let browser = await ensureBrowserReady(cdpOpts);
|
|
1443
|
+
let page = await getOrCreatePage(browser);
|
|
1444
|
+
const blockedRoutePages = new WeakSet();
|
|
1445
|
+
|
|
1446
|
+
async function setupPageRequestBlocking(targetPage) {
|
|
1447
|
+
if (!targetPage || blockedRoutePages.has(targetPage)) return;
|
|
1448
|
+
await targetPage.route("**/*", (route) => {
|
|
1449
|
+
const resourceType = route.request().resourceType();
|
|
1450
|
+
if (resourceType === "image" || resourceType === "stylesheet") {
|
|
1451
|
+
route.abort();
|
|
1452
|
+
} else {
|
|
1453
|
+
route.continue();
|
|
1454
|
+
}
|
|
1455
|
+
});
|
|
1456
|
+
blockedRoutePages.add(targetPage);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
await setupPageRequestBlocking(page);
|
|
1460
|
+
|
|
1461
|
+
// 端口池
|
|
1462
|
+
const portPoolStart = cdpPort;
|
|
1463
|
+
const portPoolSize = Math.max(
|
|
1464
|
+
2,
|
|
1465
|
+
parseInt(process.env.TAG_REFRESH_PORT_POOL_SIZE || "10", 10) || 10,
|
|
1466
|
+
);
|
|
1467
|
+
|
|
1468
|
+
function pickNextPort(currentPort) {
|
|
1469
|
+
const candidates = [];
|
|
1470
|
+
for (let i = 0; i < portPoolSize; i++) {
|
|
1471
|
+
const p = portPoolStart + i;
|
|
1472
|
+
if (p !== currentPort) candidates.push(p);
|
|
1473
|
+
}
|
|
1474
|
+
if (candidates.length === 0) return currentPort;
|
|
1475
|
+
return candidates[Math.floor(Math.random() * candidates.length)];
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
// ── 内存回收配置 ──
|
|
1479
|
+
const RECYCLE_CONFIG = {
|
|
1480
|
+
nodeRssMb: Math.max(
|
|
1481
|
+
256,
|
|
1482
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_RSS_MB || "2000", 10) || 2000,
|
|
1483
|
+
),
|
|
1484
|
+
nodeHeapMb: Math.max(
|
|
1485
|
+
128,
|
|
1486
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_HEAP_MB || "2000", 10) || 2000,
|
|
1487
|
+
),
|
|
1488
|
+
edgeMb: Math.max(
|
|
1489
|
+
500,
|
|
1490
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_EDGE_MB || "2000", 10) || 2000,
|
|
1491
|
+
),
|
|
1492
|
+
cooldownMs: Math.max(
|
|
1493
|
+
0,
|
|
1494
|
+
parseInt(process.env.TAG_REFRESH_RECYCLE_COOLDOWN_MS || "180000", 10) ||
|
|
1495
|
+
180000,
|
|
1496
|
+
),
|
|
1497
|
+
periodicEvery: Math.max(
|
|
1498
|
+
0,
|
|
1499
|
+
parseInt(process.env.TAG_REFRESH_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
|
|
1500
|
+
),
|
|
1501
|
+
switchPortOnRecycle:
|
|
1502
|
+
String(process.env.TAG_REFRESH_SWITCH_PORT_ON_RECYCLE || "1") !== "0",
|
|
1503
|
+
};
|
|
1504
|
+
let lastRecycleAt = 0;
|
|
1505
|
+
|
|
1506
|
+
// ── 轻量回收 ──
|
|
1507
|
+
async function recyclePage() {
|
|
1508
|
+
if (!page || page.isClosed()) {
|
|
1509
|
+
page = await getOrCreatePage(browser);
|
|
1510
|
+
await setupPageRequestBlocking(page);
|
|
1511
|
+
return;
|
|
1512
|
+
}
|
|
1513
|
+
try {
|
|
1514
|
+
await page.goto("about:blank", {
|
|
1515
|
+
waitUntil: "domcontentloaded",
|
|
1516
|
+
timeout: 5000,
|
|
1517
|
+
});
|
|
1518
|
+
} catch {
|
|
1519
|
+
await page.close().catch(() => {});
|
|
1520
|
+
page = await getOrCreatePage(browser);
|
|
1521
|
+
await setupPageRequestBlocking(page);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
// ── 深度回收 ──
|
|
1526
|
+
async function recycleCdpSession(reason, { switchPort = true } = {}) {
|
|
1527
|
+
const oldPort = cdpPort;
|
|
1528
|
+
const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
|
|
1529
|
+
if (shouldSwitch) {
|
|
1530
|
+
cdpPort = pickNextPort(cdpPort);
|
|
1531
|
+
cdpOpts.port = cdpPort;
|
|
1532
|
+
clientMeta.port = cdpPort;
|
|
1533
|
+
}
|
|
1534
|
+
const portHint =
|
|
1535
|
+
oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
|
|
1536
|
+
log(` ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
|
|
1537
|
+
if (page) {
|
|
1538
|
+
await page.close().catch(() => {});
|
|
1539
|
+
page = null;
|
|
1540
|
+
}
|
|
1541
|
+
await browser.close().catch(() => {});
|
|
1542
|
+
if (oldPort !== cdpPort) {
|
|
1543
|
+
await killEdgeProcesses(null, oldPort).catch(() => {});
|
|
1544
|
+
}
|
|
1545
|
+
browser = await ensureBrowserReady(cdpOpts);
|
|
1546
|
+
page = await getOrCreatePage(browser);
|
|
1547
|
+
await setupPageRequestBlocking(page);
|
|
1548
|
+
lastRecycleAt = Date.now();
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
async function maybeRecycleForMemory() {
|
|
1552
|
+
const mem = process.memoryUsage();
|
|
1553
|
+
const rssMB = mem.rss / 1024 / 1024;
|
|
1554
|
+
const heapMB = mem.heapUsed / 1024 / 1024;
|
|
1555
|
+
const edgeMB = getBrowserMemory(cdpPort);
|
|
1556
|
+
|
|
1557
|
+
const nodeOver =
|
|
1558
|
+
rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
|
|
1559
|
+
const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
|
|
1560
|
+
|
|
1561
|
+
if (nodeOver || edgeOver) {
|
|
1562
|
+
if (
|
|
1563
|
+
RECYCLE_CONFIG.cooldownMs > 0 &&
|
|
1564
|
+
Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
|
|
1565
|
+
)
|
|
1566
|
+
return;
|
|
1567
|
+
|
|
1568
|
+
const parts = [];
|
|
1569
|
+
if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
|
|
1570
|
+
parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
|
|
1571
|
+
if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
|
|
1572
|
+
parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
|
|
1573
|
+
if (edgeOver)
|
|
1574
|
+
parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
|
|
1575
|
+
|
|
1576
|
+
await recycleCdpSession(`mem ${parts.join("+")}`, {
|
|
1577
|
+
switchPort: edgeOver,
|
|
1578
|
+
});
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
if (
|
|
1583
|
+
RECYCLE_CONFIG.periodicEvery > 0 &&
|
|
1584
|
+
totalRefreshed > 0 &&
|
|
1585
|
+
totalRefreshed % RECYCLE_CONFIG.periodicEvery === 0
|
|
1586
|
+
) {
|
|
1587
|
+
await recycleCdpSession(`periodic #${totalRefreshed}`, {
|
|
1588
|
+
switchPort: false,
|
|
1589
|
+
});
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
let totalRefreshed = 0;
|
|
1594
|
+
let emptyRounds = 0;
|
|
1595
|
+
let consecutiveCaptcha = 0;
|
|
1596
|
+
const MAX_CONSECUTIVE_CAPTCHA = 3;
|
|
1597
|
+
|
|
1598
|
+
const clientId = randomUUID();
|
|
1599
|
+
const clientMeta = { type: "tag-refresh", port: cdpPort };
|
|
1600
|
+
|
|
1601
|
+
// 复用 TikTokScraper
|
|
1602
|
+
const enrichScraper = new TikTokScraper({
|
|
1603
|
+
poolSize: 3,
|
|
1604
|
+
proxyServer: effectiveProxy || undefined,
|
|
1605
|
+
});
|
|
1606
|
+
await enrichScraper.init();
|
|
1607
|
+
log("✅ TikTokScraper 已就绪 (enrich 复用)");
|
|
1608
|
+
log(` 客户端 ID: ${clientId.substring(0, 8)}...`);
|
|
1609
|
+
log("");
|
|
1610
|
+
|
|
1611
|
+
const cleanup = () => {
|
|
1612
|
+
log("\n正在清理资源...");
|
|
1613
|
+
enrichScraper.close().catch(() => {});
|
|
1614
|
+
killEdgeProcesses(null, cdpPort);
|
|
1615
|
+
process.exit(0);
|
|
1616
|
+
};
|
|
1617
|
+
process.on("SIGINT", cleanup);
|
|
1618
|
+
|
|
1619
|
+
try {
|
|
1620
|
+
while (true) {
|
|
1621
|
+
try {
|
|
1622
|
+
// 从服务端获取下一个 productive 标签
|
|
1623
|
+
const taskRes = await fetch(`${baseUrl}/api/tags/refresh-task`, {
|
|
1624
|
+
headers: buildClientHeaders(clientId, clientMeta),
|
|
1625
|
+
});
|
|
1626
|
+
const taskData = await taskRes.json();
|
|
1627
|
+
|
|
1628
|
+
if (!taskData.tag) {
|
|
1629
|
+
emptyRounds++;
|
|
1630
|
+
|
|
1631
|
+
if (emptyRounds >= 5) {
|
|
1632
|
+
log(
|
|
1633
|
+
`⏳ 连续 ${emptyRounds} 轮无 productive 标签可刷新(30 秒后重试)...`,
|
|
1634
|
+
);
|
|
1635
|
+
await new Promise((r) => setTimeout(r, 30000));
|
|
1636
|
+
emptyRounds = 0;
|
|
1637
|
+
continue;
|
|
1638
|
+
}
|
|
1639
|
+
log(
|
|
1640
|
+
`⏳ 暂无 productive 标签可刷新(连续 ${emptyRounds} 轮),10 秒后重试...`,
|
|
1641
|
+
);
|
|
1642
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
emptyRounds = 0;
|
|
1647
|
+
|
|
1648
|
+
const tag = taskData.tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1649
|
+
const startTime = Date.now();
|
|
1650
|
+
|
|
1651
|
+
log(
|
|
1652
|
+
`[${totalRefreshed + 1}] 正在刷新 #${tag} (score: ${taskData.score})...`,
|
|
1653
|
+
);
|
|
1654
|
+
|
|
1655
|
+
const result = {
|
|
1656
|
+
tag,
|
|
1657
|
+
pushedUsers: 0,
|
|
1658
|
+
error: null,
|
|
1659
|
+
};
|
|
1660
|
+
|
|
1661
|
+
// 锁定标签
|
|
1662
|
+
const claimRes = await fetch(`${baseUrl}/api/tags/refresh-claim`, {
|
|
1663
|
+
method: "POST",
|
|
1664
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1665
|
+
"Content-Type": "application/json",
|
|
1666
|
+
}),
|
|
1667
|
+
body: JSON.stringify({ tag }),
|
|
1668
|
+
});
|
|
1669
|
+
const claimData = await claimRes.json();
|
|
1670
|
+
if (!claimData.ok) {
|
|
1671
|
+
if (
|
|
1672
|
+
claimData.error &&
|
|
1673
|
+
claimData.error.includes("already being refreshed")
|
|
1674
|
+
) {
|
|
1675
|
+
log(` ⏭️ 已被其他客户端锁定,跳过`);
|
|
1676
|
+
await recyclePage();
|
|
1677
|
+
continue;
|
|
1678
|
+
}
|
|
1679
|
+
log(` ⚠️ 无法锁定 (${claimData.error}),重试下一个`);
|
|
1680
|
+
await recyclePage();
|
|
1681
|
+
continue;
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
// 抓取视频
|
|
1685
|
+
const fetchStart = Date.now();
|
|
1686
|
+
log(` 抓取 TikTok 标签页...`);
|
|
1687
|
+
const tagResult = await fetchTagData(tag, {
|
|
1688
|
+
page,
|
|
1689
|
+
onProgress: ({ videos, authors }) => {
|
|
1690
|
+
process.stderr.write(
|
|
1691
|
+
`\r 抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
|
|
1692
|
+
);
|
|
1693
|
+
},
|
|
1694
|
+
});
|
|
1695
|
+
const fetchSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1696
|
+
log(
|
|
1697
|
+
`\r 完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者 (${fetchSec}s)\x1b[K`,
|
|
1698
|
+
);
|
|
1699
|
+
|
|
1700
|
+
let videos = tagResult.videos;
|
|
1701
|
+
|
|
1702
|
+
if (!videos || videos.length === 0) {
|
|
1703
|
+
const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
|
|
1704
|
+
log(` ⚠️ 无视频 (${deadSec}s),保留 productive 状态,等待下次刷新`);
|
|
1705
|
+
result.error = "no videos found";
|
|
1706
|
+
await fetch(`${baseUrl}/api/tags/refresh-result`, {
|
|
1707
|
+
method: "POST",
|
|
1708
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1709
|
+
"Content-Type": "application/json",
|
|
1710
|
+
}),
|
|
1711
|
+
body: JSON.stringify({ tag, pushedUsers: 0, error: result.error }),
|
|
1712
|
+
}).catch(() => {});
|
|
1713
|
+
totalRefreshed++;
|
|
1714
|
+
await randomDelay(0, 5000);
|
|
1715
|
+
await recyclePage();
|
|
1716
|
+
await maybeRecycleForMemory();
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
// enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
|
|
1721
|
+
const enriched = await enrichWithServerCache(
|
|
1722
|
+
videos,
|
|
1723
|
+
baseUrl,
|
|
1724
|
+
enrichScraper,
|
|
1725
|
+
{
|
|
1726
|
+
clientId,
|
|
1727
|
+
clientMeta,
|
|
1728
|
+
effectiveProxy,
|
|
1729
|
+
log,
|
|
1730
|
+
},
|
|
1731
|
+
);
|
|
1732
|
+
videos = enriched.videos;
|
|
1733
|
+
|
|
1734
|
+
// CDN 限流检测
|
|
1735
|
+
const cdnBlocked = enriched.cdnBlockedCount || 0;
|
|
1736
|
+
if (cdnBlocked > 0) {
|
|
1737
|
+
const cdnRatio = cdnBlocked / (videos.length || 1);
|
|
1738
|
+
const coolSec = cdnRatio > 0.3 ? 120 : 60;
|
|
1739
|
+
log(
|
|
1740
|
+
` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),冷却 ${coolSec} 秒后重启 scraper`,
|
|
1741
|
+
);
|
|
1742
|
+
await new Promise((r) => setTimeout(r, coolSec * 1000));
|
|
1743
|
+
log(` 正在重启 TikTokScraper...`);
|
|
1744
|
+
await enrichScraper.restart();
|
|
1745
|
+
log(` ✅ TikTokScraper 已重启`);
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
// 过滤目标国家的视频和作者
|
|
1749
|
+
const targetLocations = DEFAULT_TARGET_LOCATIONS;
|
|
1750
|
+
const filteredVideos = videos.filter((v) =>
|
|
1751
|
+
isLocationInList(v.locationCreated, targetLocations),
|
|
1752
|
+
);
|
|
1753
|
+
const filteredAuthors = [
|
|
1754
|
+
...new Set(
|
|
1755
|
+
filteredVideos.map((v) => v.authorUniqueId).filter(Boolean),
|
|
1756
|
+
),
|
|
1757
|
+
];
|
|
1758
|
+
const filteredCount = videos.length - filteredVideos.length;
|
|
1759
|
+
if (filteredCount > 0) {
|
|
1760
|
+
log(` 过滤非目标国家: ${filteredCount}/${videos.length} 视频被排除`);
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// 推送过滤后的作者到服务端
|
|
1764
|
+
if (filteredAuthors.length > 0) {
|
|
1765
|
+
const pushResult = await pushToServer(
|
|
1766
|
+
baseUrl,
|
|
1767
|
+
filteredAuthors,
|
|
1768
|
+
filteredVideos,
|
|
1769
|
+
clientId,
|
|
1770
|
+
clientMeta,
|
|
1771
|
+
);
|
|
1772
|
+
result.pushedUsers = pushResult.added || 0;
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
// 上报刷新结果(只更新 pushed_users,不改变 score/status)
|
|
1776
|
+
await fetch(`${baseUrl}/api/tags/refresh-result`, {
|
|
1777
|
+
method: "POST",
|
|
1778
|
+
headers: buildClientHeaders(clientId, clientMeta, {
|
|
1779
|
+
"Content-Type": "application/json",
|
|
1780
|
+
}),
|
|
1781
|
+
body: JSON.stringify({
|
|
1782
|
+
tag,
|
|
1783
|
+
pushedUsers: result.pushedUsers,
|
|
1784
|
+
error: result.error,
|
|
1785
|
+
}),
|
|
1786
|
+
}).catch(() => {});
|
|
1787
|
+
|
|
1788
|
+
totalRefreshed++;
|
|
1789
|
+
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
1790
|
+
const memStr = ` mem=${formatMemoryWithBrowser(cdpPort)}`;
|
|
1791
|
+
log(
|
|
1792
|
+
` 🔄 #${tag} 完成 (${elapsed}s) — ${videos.length} 视频(${filteredVideos.length} 匹配), ${filteredAuthors.length} 作者, 推送 ${result.pushedUsers} 用户${memStr}`,
|
|
1793
|
+
);
|
|
1794
|
+
log("");
|
|
1795
|
+
|
|
1796
|
+
if (totalRefreshed % 10 === 0) {
|
|
1797
|
+
log(
|
|
1798
|
+
`📊 [${totalRefreshed}] 内存汇总: ${formatMemoryWithBrowser(cdpPort)}`,
|
|
1799
|
+
);
|
|
1800
|
+
}
|
|
1801
|
+
await recyclePage();
|
|
1802
|
+
await maybeRecycleForMemory();
|
|
1803
|
+
await randomDelay(3000, 7000);
|
|
1804
|
+
} catch (e) {
|
|
1805
|
+
const isNetworkError =
|
|
1806
|
+
e.code === "ECONNREFUSED" ||
|
|
1807
|
+
e.code === "ENOTFOUND" ||
|
|
1808
|
+
e.code === "ECONNRESET" ||
|
|
1809
|
+
(e.message &&
|
|
1810
|
+
(e.message.includes("ECONNREFUSED") ||
|
|
1811
|
+
e.message.includes("fetch failed") ||
|
|
1812
|
+
e.message.includes("network")));
|
|
1813
|
+
if (isNetworkError) {
|
|
1814
|
+
log(` ⚠️ 服务端连接失败 (${e.message}),15 秒后重试...`);
|
|
1815
|
+
await new Promise((r) => setTimeout(r, 15000));
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
log(` ⚠️ 处理出错 (${e.message}),10 秒后重试...`);
|
|
1819
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
1820
|
+
await recyclePage().catch(() => {});
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
} finally {
|
|
1824
|
+
cleanup();
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
@@ -24,6 +24,7 @@ async function processAPIResponse(
|
|
|
24
24
|
createTime: item.createTime || null,
|
|
25
25
|
playCount: item.stats?.playCount || 0,
|
|
26
26
|
isECVideo: item.isECVideo ? 1 : 0,
|
|
27
|
+
isAd: item.isAd ? 1 : 0,
|
|
27
28
|
});
|
|
28
29
|
}
|
|
29
30
|
|
|
@@ -74,6 +75,7 @@ async function processAPIResponse(
|
|
|
74
75
|
createTime: item.createTime || null,
|
|
75
76
|
playCount: item.stats?.playCount || 0,
|
|
76
77
|
isECVideo: item.isECVideo ? 1 : 0,
|
|
78
|
+
isAd: item.isAd ? 1 : 0,
|
|
77
79
|
});
|
|
78
80
|
}
|
|
79
81
|
}
|
package/src/lib/args.js
CHANGED
|
@@ -746,7 +746,9 @@ function parseTagArgs(args) {
|
|
|
746
746
|
let isDiscover = false;
|
|
747
747
|
let isScore = false;
|
|
748
748
|
let isScoreAll = false;
|
|
749
|
+
let isRefresh = false;
|
|
749
750
|
let scoreAllPort = 7222;
|
|
751
|
+
let refreshPort = 8222;
|
|
750
752
|
let scoreProxy = null;
|
|
751
753
|
let scoreTag = null;
|
|
752
754
|
let scoreCountries = null;
|
|
@@ -794,6 +796,8 @@ function parseTagArgs(args) {
|
|
|
794
796
|
} else if (arg === "--port") {
|
|
795
797
|
if (isScoreAll) {
|
|
796
798
|
scoreAllPort = parseInt(args[++i]) || 7222;
|
|
799
|
+
} else if (isRefresh) {
|
|
800
|
+
refreshPort = parseInt(args[++i]) || 8222;
|
|
797
801
|
} else {
|
|
798
802
|
scorePort = parseInt(args[++i]) || 9222;
|
|
799
803
|
}
|
|
@@ -826,6 +830,8 @@ function parseTagArgs(args) {
|
|
|
826
830
|
if (scoreCountries) i += scoreCountries.length;
|
|
827
831
|
} else if (cleaned.toLowerCase() === "score-all") {
|
|
828
832
|
isScoreAll = true;
|
|
833
|
+
} else if (cleaned.toLowerCase() === "refresh") {
|
|
834
|
+
isRefresh = true;
|
|
829
835
|
} else {
|
|
830
836
|
tags.push(cleaned.toLowerCase());
|
|
831
837
|
}
|
|
@@ -899,6 +905,27 @@ function parseTagArgs(args) {
|
|
|
899
905
|
};
|
|
900
906
|
}
|
|
901
907
|
|
|
908
|
+
if (isRefresh) {
|
|
909
|
+
return {
|
|
910
|
+
subcommand: "tag-refresh",
|
|
911
|
+
tagRefresh: {
|
|
912
|
+
serverUrl,
|
|
913
|
+
port: refreshPort,
|
|
914
|
+
proxy: scoreProxy,
|
|
915
|
+
},
|
|
916
|
+
urls: [],
|
|
917
|
+
outputFormat: "json",
|
|
918
|
+
exploreCount: 0,
|
|
919
|
+
showConfig: false,
|
|
920
|
+
showHelp: false,
|
|
921
|
+
customProxy: null,
|
|
922
|
+
configAction: null,
|
|
923
|
+
configValue: null,
|
|
924
|
+
pipeMode: false,
|
|
925
|
+
filterStr: null,
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
|
|
902
929
|
return {
|
|
903
930
|
subcommand: "tag",
|
|
904
931
|
tagTags: {
|
package/src/main.js
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
handleDiscover,
|
|
17
17
|
handleScore,
|
|
18
18
|
handleScoreAll,
|
|
19
|
+
handleTagRefresh,
|
|
19
20
|
} from "./cli/tag.js";
|
|
20
21
|
|
|
21
22
|
async function main() {
|
|
@@ -50,6 +51,8 @@ async function main() {
|
|
|
50
51
|
return handleScore(parsed);
|
|
51
52
|
case "tag-score-all":
|
|
52
53
|
return handleScoreAll(parsed);
|
|
54
|
+
case "tag-refresh":
|
|
55
|
+
return handleTagRefresh(parsed);
|
|
53
56
|
}
|
|
54
57
|
|
|
55
58
|
const {
|
|
@@ -105,6 +105,7 @@ async function processExplore(page, username, options, log) {
|
|
|
105
105
|
href: topVideo.href,
|
|
106
106
|
playCount: topVideo.playCount,
|
|
107
107
|
createTime: topVideo.createTime,
|
|
108
|
+
isAd: topVideo.isAd,
|
|
108
109
|
};
|
|
109
110
|
log(
|
|
110
111
|
` 7天内 EC视频最高播放: ${topVideo.playCount} 次播放 (${recentVideos.length} 个EC候选)`,
|
package/src/watch/data-store.js
CHANGED
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
getJobRow,
|
|
51
51
|
getJobBaseRow,
|
|
52
52
|
getJob,
|
|
53
|
+
getBulkAuthorLocations,
|
|
53
54
|
getAllJobs,
|
|
54
55
|
getVideoRow,
|
|
55
56
|
getAllVideoRows,
|
|
@@ -96,6 +97,9 @@ import {
|
|
|
96
97
|
rawQuery,
|
|
97
98
|
normalizeTags,
|
|
98
99
|
clearTags,
|
|
100
|
+
getProductiveRefreshTask,
|
|
101
|
+
claimRefreshTag,
|
|
102
|
+
reportRefreshResult,
|
|
99
103
|
} from "./db-tags.js";
|
|
100
104
|
|
|
101
105
|
// LLM 打分
|
|
@@ -391,6 +395,7 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
|
|
|
391
395
|
refresh_time,
|
|
392
396
|
top_video_play_count,
|
|
393
397
|
top_video_href,
|
|
398
|
+
top_video_is_ad,
|
|
394
399
|
status,
|
|
395
400
|
sources
|
|
396
401
|
FROM jobs
|
|
@@ -490,6 +495,54 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
|
|
|
490
495
|
};
|
|
491
496
|
}
|
|
492
497
|
|
|
498
|
+
/**
|
|
499
|
+
* 查询已修改(modified_at IS NOT NULL)且 location_created 不在 targetLocations 中的目标商家,
|
|
500
|
+
* 按国家分组返回。
|
|
501
|
+
*/
|
|
502
|
+
function getModifiedTargetUsersOutsideLocations(targetLocations = []) {
|
|
503
|
+
if (!getDb()) return { countries: [], total: 0 };
|
|
504
|
+
|
|
505
|
+
const placeholders = targetLocations.map(() => "?").join(", ");
|
|
506
|
+
const rows = getDb()
|
|
507
|
+
.prepare(
|
|
508
|
+
`
|
|
509
|
+
SELECT unique_id, nickname, follower_count, video_count,
|
|
510
|
+
location_created, confirmed_location, modified_at, latest_video_time, refresh_time
|
|
511
|
+
FROM jobs
|
|
512
|
+
WHERE tt_seller = 1
|
|
513
|
+
AND verified = 0
|
|
514
|
+
AND modified_at IS NOT NULL
|
|
515
|
+
AND location_created IS NOT NULL
|
|
516
|
+
AND location_created NOT IN (${placeholders})
|
|
517
|
+
ORDER BY location_created ASC, COALESCE(latest_video_time, 0) DESC
|
|
518
|
+
`,
|
|
519
|
+
)
|
|
520
|
+
.all(...targetLocations)
|
|
521
|
+
.map(mapJobRow);
|
|
522
|
+
|
|
523
|
+
const countryMap = new Map();
|
|
524
|
+
for (const row of rows) {
|
|
525
|
+
const country = row.locationCreated || "未知";
|
|
526
|
+
if (!countryMap.has(country)) {
|
|
527
|
+
countryMap.set(country, { users: [], userCount: 0 });
|
|
528
|
+
}
|
|
529
|
+
const entry = countryMap.get(country);
|
|
530
|
+
entry.users.push(row);
|
|
531
|
+
entry.userCount++;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
const countries = [];
|
|
535
|
+
for (const [country, entry] of countryMap) {
|
|
536
|
+
countries.push({
|
|
537
|
+
country,
|
|
538
|
+
count: entry.userCount,
|
|
539
|
+
users: entry.users,
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
return { total: rows.length, countries };
|
|
544
|
+
}
|
|
545
|
+
|
|
493
546
|
export function createStore(filePath, options = {}) {
|
|
494
547
|
if (!filePath) {
|
|
495
548
|
throw new Error("createStore requires an explicit .db path");
|
|
@@ -2292,6 +2345,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2292
2345
|
if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
|
|
2293
2346
|
user.topVideoPlayCount = result.topRecentVideo.playCount || null;
|
|
2294
2347
|
user.topVideoHref = result.topRecentVideo.href || null;
|
|
2348
|
+
user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
|
|
2295
2349
|
}
|
|
2296
2350
|
user.sources = [...new Set([...(user.sources || []), "processed"])];
|
|
2297
2351
|
}
|
|
@@ -2518,6 +2572,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2518
2572
|
if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
|
|
2519
2573
|
user.topVideoPlayCount = result.topRecentVideo.playCount || null;
|
|
2520
2574
|
user.topVideoHref = result.topRecentVideo.href || null;
|
|
2575
|
+
user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
|
|
2521
2576
|
}
|
|
2522
2577
|
const newUsers = processDiscoveredUsers(result);
|
|
2523
2578
|
const ret = updateJobInfo(uniqueId, user, false);
|
|
@@ -2543,6 +2598,7 @@ export function createStore(filePath, options = {}) {
|
|
|
2543
2598
|
if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
|
|
2544
2599
|
user.topVideoPlayCount = result.topRecentVideo.playCount || null;
|
|
2545
2600
|
user.topVideoHref = result.topRecentVideo.href || null;
|
|
2601
|
+
user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
|
|
2546
2602
|
}
|
|
2547
2603
|
const newUsers = processDiscoveredUsers(result);
|
|
2548
2604
|
return { saved: true, newUsers };
|
|
@@ -3241,6 +3297,7 @@ export function createStore(filePath, options = {}) {
|
|
|
3241
3297
|
getProcessedUsers,
|
|
3242
3298
|
getAllUsers,
|
|
3243
3299
|
getUserDbCount,
|
|
3300
|
+
getBulkAuthorLocations,
|
|
3244
3301
|
getJobsCount,
|
|
3245
3302
|
getRawJobsCount,
|
|
3246
3303
|
getPendingJobsCount,
|
|
@@ -3260,6 +3317,8 @@ export function createStore(filePath, options = {}) {
|
|
|
3260
3317
|
getRawJobsPage: getRawJobsPageFromDb,
|
|
3261
3318
|
getTargetUsers: getTargetUsersFromDb,
|
|
3262
3319
|
getTargetUsersByCountry: getTargetUsersByCountryFromDb,
|
|
3320
|
+
getModifiedTargetUsersOutsideLocations: (loc) =>
|
|
3321
|
+
getModifiedTargetUsersOutsideLocations(loc),
|
|
3263
3322
|
getStats,
|
|
3264
3323
|
getStatusGroups,
|
|
3265
3324
|
markGroupsDirty,
|
|
@@ -3310,6 +3369,9 @@ export function createStore(filePath, options = {}) {
|
|
|
3310
3369
|
getTagStats,
|
|
3311
3370
|
normalizeTags,
|
|
3312
3371
|
clearTags,
|
|
3372
|
+
getProductiveRefreshTask,
|
|
3373
|
+
claimRefreshTag,
|
|
3374
|
+
reportRefreshResult,
|
|
3313
3375
|
data,
|
|
3314
3376
|
};
|
|
3315
3377
|
|
package/src/watch/db-columns.js
CHANGED
|
@@ -48,6 +48,7 @@ export const JOB_EXT_COLUMNS = [
|
|
|
48
48
|
"user_create_time",
|
|
49
49
|
"top_video_play_count",
|
|
50
50
|
"top_video_href",
|
|
51
|
+
"top_video_is_ad",
|
|
51
52
|
];
|
|
52
53
|
|
|
53
54
|
// 完整的 jobs 列 = 核心列 + 扩展列
|
|
@@ -88,6 +89,7 @@ export const WRITABLE_JOB_COLUMNS = new Set([
|
|
|
88
89
|
"latest_video_time",
|
|
89
90
|
"top_video_play_count",
|
|
90
91
|
"top_video_href",
|
|
92
|
+
"top_video_is_ad",
|
|
91
93
|
"user_create_time",
|
|
92
94
|
]);
|
|
93
95
|
|
package/src/watch/db-crud.js
CHANGED
|
@@ -327,6 +327,34 @@ export function getJob(uniqueId) {
|
|
|
327
327
|
return mapJobRow(getJobRow(uniqueId));
|
|
328
328
|
}
|
|
329
329
|
|
|
330
|
+
/**
|
|
331
|
+
* 批量查询作者的国家信息
|
|
332
|
+
* 从 jobs 和 jobs_base 表中查找已知的 location_created
|
|
333
|
+
* @param {string[]} uniqueIds
|
|
334
|
+
* @returns {Record<string, string>} { uniqueId: locationCreated }
|
|
335
|
+
*/
|
|
336
|
+
export function getBulkAuthorLocations(uniqueIds) {
|
|
337
|
+
const db = getDb();
|
|
338
|
+
if (!db || uniqueIds.length === 0) return {};
|
|
339
|
+
// SQLite 的 IN 子句不支持数组参数,动态构建占位符
|
|
340
|
+
const placeholders = uniqueIds.map(() => "?").join(",");
|
|
341
|
+
const sql = `
|
|
342
|
+
SELECT unique_id, location_created FROM jobs
|
|
343
|
+
WHERE unique_id IN (${placeholders}) AND location_created IS NOT NULL AND location_created != ''
|
|
344
|
+
UNION
|
|
345
|
+
SELECT unique_id, location_created FROM jobs_base
|
|
346
|
+
WHERE unique_id IN (${placeholders}) AND location_created IS NOT NULL AND location_created != ''
|
|
347
|
+
`;
|
|
348
|
+
const rows = db.prepare(sql).all(...uniqueIds, ...uniqueIds);
|
|
349
|
+
const result = {};
|
|
350
|
+
for (const r of rows) {
|
|
351
|
+
if (!result[r.unique_id]) {
|
|
352
|
+
result[r.unique_id] = r.location_created;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return result;
|
|
356
|
+
}
|
|
357
|
+
|
|
330
358
|
export function getAllJobs() {
|
|
331
359
|
const db = getDb();
|
|
332
360
|
if (!db) return [];
|
package/src/watch/db-schema.js
CHANGED
|
@@ -127,6 +127,7 @@ export function initDb(filePath) {
|
|
|
127
127
|
bio_link: "TEXT",
|
|
128
128
|
top_video_play_count: "INTEGER",
|
|
129
129
|
top_video_href: "TEXT",
|
|
130
|
+
top_video_is_ad: "INTEGER",
|
|
130
131
|
user_create_time: "INTEGER",
|
|
131
132
|
});
|
|
132
133
|
|
|
@@ -280,6 +281,8 @@ export function initDb(filePath) {
|
|
|
280
281
|
strategy: "TEXT",
|
|
281
282
|
dead_reason: "TEXT",
|
|
282
283
|
direction_id: "INTEGER",
|
|
284
|
+
refreshing: "INTEGER DEFAULT 0",
|
|
285
|
+
refreshed_at: "TEXT",
|
|
283
286
|
});
|
|
284
287
|
|
|
285
288
|
// ===== tag_directions 表(分层探索:通用词/分类/产品) =====
|
package/src/watch/db-tags.js
CHANGED
|
@@ -1234,3 +1234,98 @@ function getLang(country) {
|
|
|
1234
1234
|
};
|
|
1235
1235
|
return COUNTRY_LANG[country] || "en";
|
|
1236
1236
|
}
|
|
1237
|
+
|
|
1238
|
+
// ===== Tag 刷新 CRUD(productive 标签轮询重新抓取) =====
|
|
1239
|
+
|
|
1240
|
+
/**
|
|
1241
|
+
* 获取一个待刷新的 productive 标签(score > 70 且未被刷新锁定)
|
|
1242
|
+
* 按 refreshed_at ASC 排序(最久未刷新的优先),NULL 排最前
|
|
1243
|
+
*/
|
|
1244
|
+
export function getProductiveRefreshTask() {
|
|
1245
|
+
const db = getDb();
|
|
1246
|
+
if (!db) return null;
|
|
1247
|
+
|
|
1248
|
+
// 先清理超时的 refreshing 锁(超过 30 分钟未释放)
|
|
1249
|
+
db.prepare(
|
|
1250
|
+
`UPDATE tags SET refreshing = 0
|
|
1251
|
+
WHERE refreshing = 1 AND refreshed_at IS NOT NULL
|
|
1252
|
+
AND refreshed_at < datetime('now', '-30 minutes')`,
|
|
1253
|
+
).run();
|
|
1254
|
+
|
|
1255
|
+
const row = db
|
|
1256
|
+
.prepare(
|
|
1257
|
+
`SELECT * FROM tags
|
|
1258
|
+
WHERE status = 'productive' AND score > 70 AND refreshing = 0
|
|
1259
|
+
ORDER BY refreshed_at IS NULL DESC, refreshed_at ASC
|
|
1260
|
+
LIMIT 1`,
|
|
1261
|
+
)
|
|
1262
|
+
.get();
|
|
1263
|
+
return row ? parseTagRow(row) : null;
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
/**
|
|
1267
|
+
* 锁定一个 productive 标签用于刷新(防并发冲突)
|
|
1268
|
+
*/
|
|
1269
|
+
export function claimRefreshTag(tag) {
|
|
1270
|
+
const db = getDb();
|
|
1271
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
1272
|
+
|
|
1273
|
+
const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1274
|
+
if (!normalized || normalized.length < 2) {
|
|
1275
|
+
return { ok: false, error: "invalid tag" };
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
// 同时尝试匹配带 # 和不带 # 的格式(兼容旧数据)
|
|
1279
|
+
const result = db
|
|
1280
|
+
.prepare(
|
|
1281
|
+
`UPDATE tags SET refreshing = 1, refreshed_at = datetime('now')
|
|
1282
|
+
WHERE (tag = ? OR tag = ?) AND status = 'productive' AND score > 70 AND refreshing = 0`,
|
|
1283
|
+
)
|
|
1284
|
+
.run(normalized, `#${normalized}`);
|
|
1285
|
+
if (result.changes === 0) {
|
|
1286
|
+
const row = db
|
|
1287
|
+
.prepare("SELECT status, refreshing FROM tags WHERE tag = ? OR tag = ?")
|
|
1288
|
+
.get(normalized, `#${normalized}`);
|
|
1289
|
+
if (!row) return { ok: false, error: "tag not found" };
|
|
1290
|
+
if (row.refreshing) return { ok: false, error: "already being refreshed" };
|
|
1291
|
+
return {
|
|
1292
|
+
ok: false,
|
|
1293
|
+
error: `tag status is ${row.status}, not eligible for refresh`,
|
|
1294
|
+
};
|
|
1295
|
+
}
|
|
1296
|
+
return { ok: true, tag: normalized };
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
/**
|
|
1300
|
+
* 报告刷新结果并解锁标签(不改变 score/status,只更新 refreshed_at 和 pushed_users)
|
|
1301
|
+
*/
|
|
1302
|
+
export function reportRefreshResult(tag, fields) {
|
|
1303
|
+
const db = getDb();
|
|
1304
|
+
if (!db) return { ok: false, error: "db not ready" };
|
|
1305
|
+
const { pushedUsers, error } = fields;
|
|
1306
|
+
const now = new Date().toISOString();
|
|
1307
|
+
const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
|
|
1308
|
+
|
|
1309
|
+
try {
|
|
1310
|
+
const result = db
|
|
1311
|
+
.prepare(
|
|
1312
|
+
`UPDATE tags SET
|
|
1313
|
+
refreshing = 0,
|
|
1314
|
+
refreshed_at = ?,
|
|
1315
|
+
pushed_users = COALESCE(pushed_users, 0) + COALESCE(?, 0),
|
|
1316
|
+
last_error = COALESCE(?, last_error),
|
|
1317
|
+
score_count = score_count + 1
|
|
1318
|
+
WHERE tag = ? OR tag = ?`,
|
|
1319
|
+
)
|
|
1320
|
+
.run(
|
|
1321
|
+
now,
|
|
1322
|
+
pushedUsers ?? null,
|
|
1323
|
+
error ?? null,
|
|
1324
|
+
normalized,
|
|
1325
|
+
`#${normalized}`,
|
|
1326
|
+
);
|
|
1327
|
+
return { ok: result.changes > 0, tag: normalized };
|
|
1328
|
+
} catch (e) {
|
|
1329
|
+
return { ok: false, error: e.message };
|
|
1330
|
+
}
|
|
1331
|
+
}
|
package/src/watch/public/app.js
CHANGED
|
@@ -735,6 +735,9 @@ async function selectLocation(uniqueId, location) {
|
|
|
735
735
|
userInMem.locationCreated = location;
|
|
736
736
|
userInMem.modifiedAt = Date.now();
|
|
737
737
|
}
|
|
738
|
+
|
|
739
|
+
// 刷新"其他国家"区域(修改后可能需要更新统计)
|
|
740
|
+
fetchTargetOtherCountries();
|
|
738
741
|
} catch (e) {
|
|
739
742
|
showToast("更新失败: " + e.message, true);
|
|
740
743
|
} finally {
|
|
@@ -1023,6 +1026,30 @@ document.getElementById("statTargetCard").addEventListener("click", () => {
|
|
|
1023
1026
|
window.addEventListener("hashchange", handleRoute);
|
|
1024
1027
|
window.addEventListener("DOMContentLoaded", handleRoute);
|
|
1025
1028
|
|
|
1029
|
+
// 回顶部按钮:仅在 targetPage 激活时监听滚动
|
|
1030
|
+
window.addEventListener(
|
|
1031
|
+
"scroll",
|
|
1032
|
+
function () {
|
|
1033
|
+
const btn = document.getElementById("backToTop");
|
|
1034
|
+
if (!btn) return;
|
|
1035
|
+
const targetPage = document.getElementById("targetPage");
|
|
1036
|
+
if (!targetPage || !targetPage.classList.contains("active")) {
|
|
1037
|
+
btn.classList.remove("visible");
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
if (window.scrollY > 400) {
|
|
1041
|
+
btn.classList.add("visible");
|
|
1042
|
+
} else {
|
|
1043
|
+
btn.classList.remove("visible");
|
|
1044
|
+
}
|
|
1045
|
+
},
|
|
1046
|
+
{ passive: true },
|
|
1047
|
+
);
|
|
1048
|
+
|
|
1049
|
+
function scrollToTop() {
|
|
1050
|
+
window.scrollTo({ top: 0, behavior: "smooth" });
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1026
1053
|
function handleRoute() {
|
|
1027
1054
|
const hash = window.location.hash;
|
|
1028
1055
|
if (hash === "#pending") {
|
|
@@ -1410,18 +1437,20 @@ function renderTagsTable() {
|
|
|
1410
1437
|
const created = t.created_at || "-";
|
|
1411
1438
|
const score = t.score != null ? t.score.toFixed(1) : "-";
|
|
1412
1439
|
const authorCount = t.author_count ?? "-";
|
|
1413
|
-
const totalPosts = t.total_posts ?? "-";
|
|
1414
1440
|
const matchedAuthors = t.matched_authors ?? "-";
|
|
1415
1441
|
const pushedUsers = t.pushed_users ?? "-";
|
|
1442
|
+
const refreshedAt = t.refreshed_at
|
|
1443
|
+
? t.refreshed_at.slice(0, 16).replace("T", " ")
|
|
1444
|
+
: "-";
|
|
1416
1445
|
|
|
1417
1446
|
return `<tr>
|
|
1418
1447
|
<td style="color:#9ca3af;font-size:12px;text-align:center">${i + 1}</td>
|
|
1419
1448
|
<td style="font-weight:600;color:#e0e0e0">#${escapeHtml(t.tag)}</td>
|
|
1420
|
-
<td style="color:${t.score >= 5 ? "#22c55e" : t.score >= 3 ? "#facc15" : "#888"}">${score}</td>
|
|
1421
|
-
<td>${authorCount}</td>
|
|
1422
|
-
<td>${
|
|
1423
|
-
<td>${matchedAuthors}</td>
|
|
1424
|
-
<td>${pushedUsers}</td>
|
|
1449
|
+
<td style="text-align:center;color:${t.score >= 5 ? "#22c55e" : t.score >= 3 ? "#facc15" : "#888"}">${score}</td>
|
|
1450
|
+
<td style="text-align:center">${authorCount}</td>
|
|
1451
|
+
<td style="text-align:center;font-size:11px;color:#9ca3af">${refreshedAt}</td>
|
|
1452
|
+
<td style="text-align:center">${matchedAuthors}</td>
|
|
1453
|
+
<td style="text-align:center">${pushedUsers}</td>
|
|
1425
1454
|
<td style="font-size:11px">${countries}</td>
|
|
1426
1455
|
<td>${statusTag}</td>
|
|
1427
1456
|
<td style="font-size:11px;color:#888">${source}</td>
|
|
@@ -1517,6 +1546,7 @@ function showTargetPage() {
|
|
|
1517
1546
|
document.getElementById("tagsPage").classList.remove("active");
|
|
1518
1547
|
showLoading("正在加载目标商家数据...");
|
|
1519
1548
|
fetchTargetByCountry();
|
|
1549
|
+
fetchTargetOtherCountries();
|
|
1520
1550
|
// 同步统计
|
|
1521
1551
|
if (currentStats) {
|
|
1522
1552
|
document.getElementById("targetPageStatTotal").textContent = formatStatNum(
|
|
@@ -1535,6 +1565,44 @@ let currentTargetSeq = 0;
|
|
|
1535
1565
|
let currentTargetSort = { key: null, asc: true };
|
|
1536
1566
|
const TARGET_PAGE_SIZE = 200;
|
|
1537
1567
|
|
|
1568
|
+
async function fetchTargetOtherCountries() {
|
|
1569
|
+
try {
|
|
1570
|
+
const res = await fetch("/api/target-users-other-countries");
|
|
1571
|
+
const data = await res.json();
|
|
1572
|
+
renderTargetOtherCountries(data.countries || []);
|
|
1573
|
+
} catch (e) {
|
|
1574
|
+
console.error("获取其他国家数据失败:", e);
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
function renderTargetOtherCountries(countries) {
|
|
1579
|
+
const section = document.getElementById("targetOtherSection");
|
|
1580
|
+
const grid = document.getElementById("targetOtherGrid");
|
|
1581
|
+
if (!section || !grid) return;
|
|
1582
|
+
|
|
1583
|
+
if (!countries.length) {
|
|
1584
|
+
section.style.display = "none";
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1587
|
+
section.style.display = "block";
|
|
1588
|
+
const total = countries.reduce((sum, c) => sum + (c.count || 0), 0);
|
|
1589
|
+
grid.innerHTML = countries
|
|
1590
|
+
.map((c) => {
|
|
1591
|
+
const cnt = c.count || 0;
|
|
1592
|
+
const pct = total > 0 ? ((cnt / total) * 100).toFixed(1) : "0.0";
|
|
1593
|
+
const safeCountry = escapeJsString(c.country);
|
|
1594
|
+
const sel = currentTargetLocation === c.country ? " selected" : "";
|
|
1595
|
+
return `
|
|
1596
|
+
<div class="pending-country-item other-country-item${sel}"
|
|
1597
|
+
onclick="filterTargetByCountry('${safeCountry}')">
|
|
1598
|
+
<div class="country-name">${c.country}</div>
|
|
1599
|
+
<div class="country-count">${cnt}</div>
|
|
1600
|
+
<div class="country-label">${pct}%</div>
|
|
1601
|
+
</div>`;
|
|
1602
|
+
})
|
|
1603
|
+
.join("");
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1538
1606
|
async function fetchTargetByCountry() {
|
|
1539
1607
|
try {
|
|
1540
1608
|
const res = await fetch("/api/target-users-by-country?summary=1");
|
|
@@ -1712,9 +1780,12 @@ function renderTargetTable() {
|
|
|
1712
1780
|
u.topVideoPlayCount != null && u.topVideoPlayCount >= 1000
|
|
1713
1781
|
? formatNum(u.topVideoPlayCount)
|
|
1714
1782
|
: "-";
|
|
1783
|
+
const adTag = u.topVideoIsAd
|
|
1784
|
+
? ' <span style="color:red">(AD)</span>'
|
|
1785
|
+
: "";
|
|
1715
1786
|
const topPlayCountCell = u.topVideoHref
|
|
1716
|
-
? `<td data-label="最大播放量" style="font-size:11px;color:#888"><a href="${u.topVideoHref}" target="_blank" style="color:#3b82f6;text-decoration:none" title="点击查看视频">${topPlayCount}</a></td>`
|
|
1717
|
-
: `<td data-label="最大播放量" style="font-size:11px;color:#888">${topPlayCount}</td>`;
|
|
1787
|
+
? `<td data-label="最大播放量" style="font-size:11px;color:#888"><a href="${u.topVideoHref}" target="_blank" style="color:#3b82f6;text-decoration:none" title="点击查看视频">${topPlayCount}${adTag}</a></td>`
|
|
1788
|
+
: `<td data-label="最大播放量" style="font-size:11px;color:#888">${topPlayCount}${adTag}</td>`;
|
|
1718
1789
|
|
|
1719
1790
|
const editIcon = ' <span style="font-size:10px;opacity:0.5">✏️</span>';
|
|
1720
1791
|
const locationCell = u.modifiedAt
|
|
@@ -1782,13 +1853,9 @@ function filterTargetByCountry(country) {
|
|
|
1782
1853
|
}
|
|
1783
1854
|
|
|
1784
1855
|
function clearTargetFilters() {
|
|
1785
|
-
|
|
1856
|
+
// 只清空搜索关键词,保留当前选中的国家筛选
|
|
1786
1857
|
const searchInput = document.getElementById("targetSearchInput");
|
|
1787
|
-
const locationFilter = document.getElementById("targetLocationFilter");
|
|
1788
1858
|
if (searchInput) searchInput.value = "";
|
|
1789
|
-
if (locationFilter) locationFilter.value = "";
|
|
1790
|
-
const btn = document.getElementById("targetReprocessBtn");
|
|
1791
|
-
if (btn) btn.style.display = "none";
|
|
1792
1859
|
if (currentTargetSummary) {
|
|
1793
1860
|
renderTargetCountryGrid(currentTargetSummary.countries || []);
|
|
1794
1861
|
loadTargetUsersPage();
|
|
@@ -328,6 +328,11 @@
|
|
|
328
328
|
<span style="color:#666;font-size:12px">加载中...</span>
|
|
329
329
|
</div>
|
|
330
330
|
<div class="muted-tip">点击国家卡片可筛选列表。</div>
|
|
331
|
+
<div class="target-other-section" id="targetOtherSection" style="display:none">
|
|
332
|
+
<h3 style="color:#f59e0b;margin-top:20px;margin-bottom:12px">🌍 其他国家 (已修改)</h3>
|
|
333
|
+
<div class="pending-country-grid" id="targetOtherGrid"></div>
|
|
334
|
+
<div class="muted-tip">这些商家已被手动修改到非目标国家。</div>
|
|
335
|
+
</div>
|
|
331
336
|
</div>
|
|
332
337
|
<div class="target-table-card">
|
|
333
338
|
<h3>目标商家列表</h3>
|
|
@@ -361,6 +366,7 @@
|
|
|
361
366
|
</div>
|
|
362
367
|
</div>
|
|
363
368
|
</div>
|
|
369
|
+
<div id="backToTop" title="回到顶部" onclick="scrollToTop()">↑</div>
|
|
364
370
|
</div>
|
|
365
371
|
<div id="tagsPage">
|
|
366
372
|
<div class="stats" style="margin-bottom:16px">
|
|
@@ -415,7 +421,7 @@
|
|
|
415
421
|
<th style="min-width:120px">关键词</th>
|
|
416
422
|
<th class="sortable-tag" data-sort="score">评分 <span class="sort-icon">↕</span></th>
|
|
417
423
|
<th class="sortable-tag" data-sort="author_count">作者数 <span class="sort-icon">↕</span></th>
|
|
418
|
-
<th class="sortable-tag" data-sort="
|
|
424
|
+
<th class="sortable-tag" data-sort="refreshed_at">最近刷新 <span class="sort-icon">↕</span></th>
|
|
419
425
|
<th class="sortable-tag" data-sort="matched_authors">匹配作者 <span class="sort-icon">↕</span></th>
|
|
420
426
|
<th class="sortable-tag" data-sort="pushed_users">推送用户 <span class="sort-icon">↕</span></th>
|
|
421
427
|
<th>国家</th>
|
|
@@ -839,6 +839,36 @@ td.user-id:hover {
|
|
|
839
839
|
display: block;
|
|
840
840
|
}
|
|
841
841
|
|
|
842
|
+
#backToTop {
|
|
843
|
+
position: fixed;
|
|
844
|
+
bottom: 32px;
|
|
845
|
+
right: 32px;
|
|
846
|
+
width: 44px;
|
|
847
|
+
height: 44px;
|
|
848
|
+
border-radius: 50%;
|
|
849
|
+
background: rgba(254, 44, 85, 0.85);
|
|
850
|
+
color: #fff;
|
|
851
|
+
font-size: 20px;
|
|
852
|
+
display: none;
|
|
853
|
+
align-items: center;
|
|
854
|
+
justify-content: center;
|
|
855
|
+
cursor: pointer;
|
|
856
|
+
z-index: 999;
|
|
857
|
+
box-shadow: 0 4px 16px rgba(254, 44, 85, 0.35);
|
|
858
|
+
transition: opacity 0.25s, transform 0.25s, background 0.2s;
|
|
859
|
+
user-select: none;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
#backToTop:hover {
|
|
863
|
+
background: #e61944;
|
|
864
|
+
transform: translateY(-2px);
|
|
865
|
+
box-shadow: 0 6px 20px rgba(254, 44, 85, 0.5);
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
#backToTop.visible {
|
|
869
|
+
display: flex;
|
|
870
|
+
}
|
|
871
|
+
|
|
842
872
|
#mainPage.hidden {
|
|
843
873
|
display: none;
|
|
844
874
|
}
|
|
@@ -1004,6 +1034,19 @@ td.user-id:hover {
|
|
|
1004
1034
|
color: #ddd6fe;
|
|
1005
1035
|
}
|
|
1006
1036
|
|
|
1037
|
+
.other-country-item {
|
|
1038
|
+
border-color: #f59e0b;
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
.other-country-item .country-name {
|
|
1042
|
+
color: #f59e0b;
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
.other-country-item:hover {
|
|
1046
|
+
border-color: #fbbf24;
|
|
1047
|
+
background: #3a3520;
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1007
1050
|
.back-btn {
|
|
1008
1051
|
padding: 6px 14px;
|
|
1009
1052
|
border: 1px solid #333;
|
package/src/watch/server.js
CHANGED
|
@@ -93,6 +93,7 @@ function inferClientType(routePath) {
|
|
|
93
93
|
if (routePath.startsWith("/api/redo-job")) return "refresh";
|
|
94
94
|
if (routePath.startsWith("/api/user-update-tasks")) return "attach";
|
|
95
95
|
if (routePath.startsWith("/api/comment-task")) return "comments";
|
|
96
|
+
if (routePath.startsWith("/api/tags/refresh")) return "tag-refresh";
|
|
96
97
|
if (routePath.startsWith("/api/tags")) return "scoring";
|
|
97
98
|
if (
|
|
98
99
|
routePath.startsWith("/api/job") ||
|
|
@@ -142,9 +143,12 @@ export function startWatchServer(
|
|
|
142
143
|
let discoverRunning = false;
|
|
143
144
|
let lastDiscoverTime = 0;
|
|
144
145
|
|
|
145
|
-
//
|
|
146
|
-
const
|
|
147
|
-
|
|
146
|
+
// 每国标签不足阈值:某国 new 标签低于此值时触发 discover
|
|
147
|
+
const DISCOVER_PER_COUNTRY_THRESHOLD = Math.max(
|
|
148
|
+
3,
|
|
149
|
+
parseInt(process.env.TAG_DISCOVER_PER_COUNTRY_THRESHOLD || "10", 10) ||
|
|
150
|
+
10,
|
|
151
|
+
);
|
|
148
152
|
|
|
149
153
|
async function runAutoDiscover(trigger = "timer") {
|
|
150
154
|
if (discoverRunning) return;
|
|
@@ -156,23 +160,14 @@ export function startWatchServer(
|
|
|
156
160
|
);
|
|
157
161
|
return;
|
|
158
162
|
}
|
|
159
|
-
// 先看全局总量:标签池充足时直接跳过,不浪费 LLM token
|
|
160
|
-
const globalStats = store.getTagStats();
|
|
161
|
-
const totalNew = globalStats?.new || 0;
|
|
162
|
-
if (totalNew >= DISCOVER_SUFFICIENT_TOTAL) {
|
|
163
|
-
console.error(
|
|
164
|
-
`[discover] 标签池充足 (${totalNew} 个 new ≥ ${DISCOVER_SUFFICIENT_TOTAL}),跳过 [触发: ${trigger}]`,
|
|
165
|
-
);
|
|
166
|
-
return;
|
|
167
|
-
}
|
|
168
163
|
discoverRunning = true;
|
|
169
164
|
try {
|
|
170
|
-
//
|
|
165
|
+
// 按国家维度判断:只为 new 标签不足的国家发现
|
|
171
166
|
const countriesToDiscover = [];
|
|
172
167
|
const { getTagsByStatus } = await import("./db-tags.js");
|
|
173
168
|
for (const country of DEFAULT_TARGET_LOCATIONS) {
|
|
174
169
|
const newTags = getTagsByStatus("new", 100, 0, country);
|
|
175
|
-
if (newTags.length <
|
|
170
|
+
if (newTags.length < DISCOVER_PER_COUNTRY_THRESHOLD) {
|
|
176
171
|
countriesToDiscover.push(country);
|
|
177
172
|
}
|
|
178
173
|
}
|
|
@@ -860,12 +855,13 @@ export function startWatchServer(
|
|
|
860
855
|
offset: parseInt(params.offset || "0", 10),
|
|
861
856
|
},
|
|
862
857
|
);
|
|
863
|
-
// 确保每个用户对象包含 topVideoPlayCount 和
|
|
858
|
+
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
864
859
|
if (result.users && Array.isArray(result.users)) {
|
|
865
860
|
result.users = result.users.map((u) => ({
|
|
866
861
|
...u,
|
|
867
862
|
topVideoPlayCount: u.topVideoPlayCount || null,
|
|
868
863
|
topVideoHref: u.topVideoHref || null,
|
|
864
|
+
topVideoIsAd: u.topVideoIsAd ?? null,
|
|
869
865
|
}));
|
|
870
866
|
}
|
|
871
867
|
sendJSON(res, 200, result);
|
|
@@ -874,7 +870,7 @@ export function startWatchServer(
|
|
|
874
870
|
|
|
875
871
|
// 默认:全量(兼容旧调用)
|
|
876
872
|
const result = store.getTargetUsersByCountry(DEFAULT_TARGET_LOCATIONS);
|
|
877
|
-
// 确保每个用户对象包含 topVideoPlayCount 和
|
|
873
|
+
// 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
|
|
878
874
|
if (result.countries && Array.isArray(result.countries)) {
|
|
879
875
|
for (const country of result.countries) {
|
|
880
876
|
if (country.users && Array.isArray(country.users)) {
|
|
@@ -882,6 +878,7 @@ export function startWatchServer(
|
|
|
882
878
|
...u,
|
|
883
879
|
topVideoPlayCount: u.topVideoPlayCount || null,
|
|
884
880
|
topVideoHref: u.topVideoHref || null,
|
|
881
|
+
topVideoIsAd: u.topVideoIsAd ?? null,
|
|
885
882
|
}));
|
|
886
883
|
}
|
|
887
884
|
}
|
|
@@ -890,6 +887,17 @@ export function startWatchServer(
|
|
|
890
887
|
return;
|
|
891
888
|
}
|
|
892
889
|
|
|
890
|
+
if (
|
|
891
|
+
req.method === "GET" &&
|
|
892
|
+
routePath === "/api/target-users-other-countries"
|
|
893
|
+
) {
|
|
894
|
+
const result = store.getModifiedTargetUsersOutsideLocations(
|
|
895
|
+
DEFAULT_TARGET_LOCATIONS,
|
|
896
|
+
);
|
|
897
|
+
sendJSON(res, 200, result);
|
|
898
|
+
return;
|
|
899
|
+
}
|
|
900
|
+
|
|
893
901
|
if (req.method === "GET" && routePath === "/api/client-errors") {
|
|
894
902
|
sendJSON(res, 200, { clients: store.getClientErrors() });
|
|
895
903
|
return;
|
|
@@ -1327,6 +1335,71 @@ export function startWatchServer(
|
|
|
1327
1335
|
return;
|
|
1328
1336
|
}
|
|
1329
1337
|
|
|
1338
|
+
// POST /api/bulk-author-locations — 批量查询作者国家(用于 tag 客户端 enrich 前跳过已知数据)
|
|
1339
|
+
if (req.method === "POST" && routePath === "/api/bulk-author-locations") {
|
|
1340
|
+
try {
|
|
1341
|
+
const body = await readBody(req);
|
|
1342
|
+
const uniqueIds = Array.isArray(body.uniqueIds) ? body.uniqueIds : [];
|
|
1343
|
+
const locations = store.getBulkAuthorLocations(uniqueIds);
|
|
1344
|
+
sendJSON(res, 200, { locations });
|
|
1345
|
+
} catch (e) {
|
|
1346
|
+
sendJSON(res, 500, { error: e.message });
|
|
1347
|
+
}
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
// GET /api/tags/refresh-task — 领取一个 productive 标签用于刷新(score>70,未被锁定)
|
|
1352
|
+
if (req.method === "GET" && routePath === "/api/tags/refresh-task") {
|
|
1353
|
+
const task = store.getProductiveRefreshTask();
|
|
1354
|
+
if (!task) {
|
|
1355
|
+
sendJSON(res, 200, {
|
|
1356
|
+
tag: null,
|
|
1357
|
+
message: "no productive tags available for refresh",
|
|
1358
|
+
});
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
sendJSON(res, 200, {
|
|
1362
|
+
tag: task.tag,
|
|
1363
|
+
score: task.score,
|
|
1364
|
+
refreshedAt: task.refreshed_at,
|
|
1365
|
+
});
|
|
1366
|
+
return;
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
// POST /api/tags/refresh-claim { tag } — 锁定 productive 标签用于刷新(防并发冲突)
|
|
1370
|
+
if (req.method === "POST" && routePath === "/api/tags/refresh-claim") {
|
|
1371
|
+
try {
|
|
1372
|
+
const body = await readBody(req);
|
|
1373
|
+
const tag = (body.tag || "").trim().toLowerCase();
|
|
1374
|
+
if (!tag) {
|
|
1375
|
+
sendJSON(res, 400, { error: "tag 不能为空" });
|
|
1376
|
+
return;
|
|
1377
|
+
}
|
|
1378
|
+
const result = store.claimRefreshTag(tag);
|
|
1379
|
+
sendJSON(res, result.ok ? 200 : 409, result);
|
|
1380
|
+
} catch (e) {
|
|
1381
|
+
sendJSON(res, 500, { error: e.message });
|
|
1382
|
+
}
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
// POST /api/tags/refresh-result { tag, pushedUsers, error } — 报告刷新结果并解锁
|
|
1387
|
+
if (req.method === "POST" && routePath === "/api/tags/refresh-result") {
|
|
1388
|
+
try {
|
|
1389
|
+
const body = await readBody(req);
|
|
1390
|
+
const tag = (body.tag || "").trim().toLowerCase();
|
|
1391
|
+
if (!tag) {
|
|
1392
|
+
sendJSON(res, 400, { error: "tag 不能为空" });
|
|
1393
|
+
return;
|
|
1394
|
+
}
|
|
1395
|
+
const result = store.reportRefreshResult(tag, body);
|
|
1396
|
+
sendJSON(res, result.ok ? 200 : 400, result);
|
|
1397
|
+
} catch (e) {
|
|
1398
|
+
sendJSON(res, 500, { error: e.message });
|
|
1399
|
+
}
|
|
1400
|
+
return;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1330
1403
|
// GET /api/tags/recent-generated?country=ES&lookback=10 — 最近N轮生成的标签(去重)
|
|
1331
1404
|
if (req.method === "GET" && routePath === "/api/tags/recent-generated") {
|
|
1332
1405
|
try {
|
|
@@ -1915,7 +1988,7 @@ export function startWatchServer(
|
|
|
1915
1988
|
);
|
|
1916
1989
|
setTimeout(() => runAutoDiscover("startup"), 30000);
|
|
1917
1990
|
console.error(
|
|
1918
|
-
`[discover] 定时+按需发现已启动: 定时间隔 ${DISCOVER_INTERVAL_MS / 60000} 分钟, 每国 ${DISCOVER_COUNT_PER_COUNTRY} 个标签,
|
|
1991
|
+
`[discover] 定时+按需发现已启动: 定时间隔 ${DISCOVER_INTERVAL_MS / 60000} 分钟, 每国 ${DISCOVER_COUNT_PER_COUNTRY} 个标签, 每国不足阈值 ${DISCOVER_PER_COUNTRY_THRESHOLD}, 按需触发阈值 ${DISCOVER_LOW_THRESHOLD}, 冷却期 ${DISCOVER_COOLDOWN_MS / 1000}s`,
|
|
1919
1992
|
);
|
|
1920
1993
|
|
|
1921
1994
|
_resolve({ server, port });
|