tt-help-cli-ycl 1.4.13 → 1.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tt-help-cli-ycl",
3
- "version": "1.4.13",
3
+ "version": "1.4.14",
4
4
  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
5
5
  "type": "module",
6
6
  "bin": {
package/src/cli/tag.js CHANGED
@@ -171,6 +171,82 @@ async function pushToServer(
171
171
  return data;
172
172
  }
173
173
 
174
+ /**
175
+ * 带服务端缓存的批量 enrich:先从服务端查询已有国家信息,只对未知的做 HTTP enrich
176
+ * @returns {{ videos: Array, cdnBlockedCount: number }}
177
+ */
178
+ async function enrichWithServerCache(
179
+ videos,
180
+ baseUrl,
181
+ enrichScraper,
182
+ options = {},
183
+ ) {
184
+ const { clientId, clientMeta, effectiveProxy, log } = options;
185
+ const logFn = log || (() => {});
186
+
187
+ // Step 1: 收集所有唯一作者 ID
188
+ const uniqueAuthors = [
189
+ ...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
190
+ ];
191
+
192
+ // Step 2: 批量查询服务端已知国家
193
+ let knownLocations = {};
194
+ if (uniqueAuthors.length > 0 && baseUrl) {
195
+ try {
196
+ const res = await fetch(`${baseUrl}/api/bulk-author-locations`, {
197
+ method: "POST",
198
+ headers: buildClientHeaders(clientId || "", clientMeta || {}, {
199
+ "Content-Type": "application/json",
200
+ }),
201
+ body: JSON.stringify({ uniqueIds: uniqueAuthors }),
202
+ });
203
+ const data = await res.json();
204
+ knownLocations = data.locations || {};
205
+ const knownCount = Object.keys(knownLocations).length;
206
+ if (knownCount > 0) {
207
+ logFn(
208
+ ` 📋 服务端已知 ${knownCount}/${uniqueAuthors.length} 个作者的国家`,
209
+ );
210
+ }
211
+ } catch {
212
+ // 查询失败则全部走 enrich,不影响主流程
213
+ }
214
+ }
215
+
216
+ // Step 3: 给已知国家的视频赋值,收集未知视频
217
+ const unknownVideos = [];
218
+ for (const v of videos) {
219
+ const loc = knownLocations[v.authorUniqueId];
220
+ if (loc) {
221
+ v.locationCreated = loc;
222
+ } else {
223
+ unknownVideos.push(v);
224
+ }
225
+ }
226
+
227
+ // Step 4: 只对未知视频做 HTTP enrich
228
+ let cdnBlockedCount = 0;
229
+ if (unknownVideos.length > 0) {
230
+ const enrichOpts = {
231
+ mode: "videos",
232
+ existingScraper: enrichScraper,
233
+ onProgress: ({ done, total, current, locationCreated }) => {
234
+ if (done % 10 === 0 || done === total) {
235
+ process.stderr.write(
236
+ `\r [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}\x1b[K`,
237
+ );
238
+ }
239
+ },
240
+ };
241
+ if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
242
+ const enriched = await enrichVideosWithLocation(unknownVideos, enrichOpts);
243
+ // 将 enrich 结果写回原数组(v 是引用,已经被 enrichVideosWithLocation 修改)
244
+ cdnBlockedCount = enriched.cdnBlockedCount || 0;
245
+ }
246
+
247
+ return { videos, cdnBlockedCount };
248
+ }
249
+
174
250
  // 共享算分逻辑(handleScore / handleScoreAll 共用)
175
251
  function calcScore(authorCount, matchedAuthors, filteredVideoCount) {
176
252
  const densityScore =
@@ -1002,20 +1078,18 @@ export async function handleScoreAll(parsed) {
1002
1078
  continue;
1003
1079
  }
1004
1080
 
1005
- // enrich: 逐个视频查 view-source 获取国家
1006
- const enrichOpts = {
1007
- mode: "videos",
1008
- existingScraper: enrichScraper,
1009
- onProgress: ({ done, total, current, locationCreated }) => {
1010
- if (done % 10 === 0 || done === total) {
1011
- process.stderr.write(
1012
- `\r [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}\x1b[K`,
1013
- );
1014
- }
1081
+ // enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
1082
+ const enriched = await enrichWithServerCache(
1083
+ videos,
1084
+ baseUrl,
1085
+ enrichScraper,
1086
+ {
1087
+ clientId,
1088
+ clientMeta,
1089
+ effectiveProxy,
1090
+ log,
1015
1091
  },
1016
- };
1017
- if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
1018
- const enriched = await enrichVideosWithLocation(videos, enrichOpts);
1092
+ );
1019
1093
  videos = enriched.videos;
1020
1094
 
1021
1095
  // CDN 限流检测:有拦截则冷却 + 重启 scraper
@@ -1324,3 +1398,429 @@ export async function handleTag(parsed) {
1324
1398
  console.log(json);
1325
1399
  }
1326
1400
  }
1401
+
1402
+ // ====== Productive 标签刷新(重新抓取,不打分) ======
1403
+
1404
+ export async function handleTagRefresh(parsed) {
1405
+ const { tagRefresh } = parsed;
1406
+ const { serverUrl, port, proxy: cliProxy } = tagRefresh || {};
1407
+
1408
+ const baseUrl = serverUrl || DEFAULT_SERVER;
1409
+ const defaultRefreshPort =
1410
+ parseInt(process.env.TAG_REFRESH_PORT_POOL_START || "8222", 10) || 8222;
1411
+ let cdpPort = port || defaultRefreshPort;
1412
+ const effectiveProxy = cliProxy || configuredProxy;
1413
+
1414
+ const log = (...args) => process.stderr.write(args.join(" ") + "\n");
1415
+
1416
+ log("");
1417
+ log("╔══════════════════════════════════════════╗");
1418
+ log("║ Productive 标签刷新模式(重新抓取)");
1419
+ log("║ 服务端: " + baseUrl);
1420
+ log(
1421
+ "║ CDP 端口: " +
1422
+ cdpPort +
1423
+ (effectiveProxy ? " 代理: " + effectiveProxy : ""),
1424
+ );
1425
+ log("║ 只抓取不重新打分,score>70 的标签");
1426
+ log("╚══════════════════════════════════════════╝");
1427
+ log("");
1428
+
1429
+ log(`📊 启动内存: ${formatMemoryWithBrowser(cdpPort)}`);
1430
+ log("");
1431
+
1432
+ // ── 设备标识 ──
1433
+ let userId = configuredUserId;
1434
+ if (!userId) {
1435
+ userId = await getMacOrUuid();
1436
+ saveUserId(userId);
1437
+ }
1438
+
1439
+ // 连接 CDP 浏览器
1440
+ const cdpOpts = { port: cdpPort };
1441
+ if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
1442
+ let browser = await ensureBrowserReady(cdpOpts);
1443
+ let page = await getOrCreatePage(browser);
1444
+ const blockedRoutePages = new WeakSet();
1445
+
1446
+ async function setupPageRequestBlocking(targetPage) {
1447
+ if (!targetPage || blockedRoutePages.has(targetPage)) return;
1448
+ await targetPage.route("**/*", (route) => {
1449
+ const resourceType = route.request().resourceType();
1450
+ if (resourceType === "image" || resourceType === "stylesheet") {
1451
+ route.abort();
1452
+ } else {
1453
+ route.continue();
1454
+ }
1455
+ });
1456
+ blockedRoutePages.add(targetPage);
1457
+ }
1458
+
1459
+ await setupPageRequestBlocking(page);
1460
+
1461
+ // 端口池
1462
+ const portPoolStart = cdpPort;
1463
+ const portPoolSize = Math.max(
1464
+ 2,
1465
+ parseInt(process.env.TAG_REFRESH_PORT_POOL_SIZE || "10", 10) || 10,
1466
+ );
1467
+
1468
+ function pickNextPort(currentPort) {
1469
+ const candidates = [];
1470
+ for (let i = 0; i < portPoolSize; i++) {
1471
+ const p = portPoolStart + i;
1472
+ if (p !== currentPort) candidates.push(p);
1473
+ }
1474
+ if (candidates.length === 0) return currentPort;
1475
+ return candidates[Math.floor(Math.random() * candidates.length)];
1476
+ }
1477
+
1478
+ // ── 内存回收配置 ──
1479
+ const RECYCLE_CONFIG = {
1480
+ nodeRssMb: Math.max(
1481
+ 256,
1482
+ parseInt(process.env.TAG_REFRESH_RECYCLE_RSS_MB || "2000", 10) || 2000,
1483
+ ),
1484
+ nodeHeapMb: Math.max(
1485
+ 128,
1486
+ parseInt(process.env.TAG_REFRESH_RECYCLE_HEAP_MB || "2000", 10) || 2000,
1487
+ ),
1488
+ edgeMb: Math.max(
1489
+ 500,
1490
+ parseInt(process.env.TAG_REFRESH_RECYCLE_EDGE_MB || "2000", 10) || 2000,
1491
+ ),
1492
+ cooldownMs: Math.max(
1493
+ 0,
1494
+ parseInt(process.env.TAG_REFRESH_RECYCLE_COOLDOWN_MS || "180000", 10) ||
1495
+ 180000,
1496
+ ),
1497
+ periodicEvery: Math.max(
1498
+ 0,
1499
+ parseInt(process.env.TAG_REFRESH_PERIODIC_RECYCLE_EVERY || "0", 10) || 0,
1500
+ ),
1501
+ switchPortOnRecycle:
1502
+ String(process.env.TAG_REFRESH_SWITCH_PORT_ON_RECYCLE || "1") !== "0",
1503
+ };
1504
+ let lastRecycleAt = 0;
1505
+
1506
+ // ── 轻量回收 ──
1507
+ async function recyclePage() {
1508
+ if (!page || page.isClosed()) {
1509
+ page = await getOrCreatePage(browser);
1510
+ await setupPageRequestBlocking(page);
1511
+ return;
1512
+ }
1513
+ try {
1514
+ await page.goto("about:blank", {
1515
+ waitUntil: "domcontentloaded",
1516
+ timeout: 5000,
1517
+ });
1518
+ } catch {
1519
+ await page.close().catch(() => {});
1520
+ page = await getOrCreatePage(browser);
1521
+ await setupPageRequestBlocking(page);
1522
+ }
1523
+ }
1524
+
1525
+ // ── 深度回收 ──
1526
+ async function recycleCdpSession(reason, { switchPort = true } = {}) {
1527
+ const oldPort = cdpPort;
1528
+ const shouldSwitch = switchPort && RECYCLE_CONFIG.switchPortOnRecycle;
1529
+ if (shouldSwitch) {
1530
+ cdpPort = pickNextPort(cdpPort);
1531
+ cdpOpts.port = cdpPort;
1532
+ clientMeta.port = cdpPort;
1533
+ }
1534
+ const portHint =
1535
+ oldPort === cdpPort ? `port=${cdpPort}` : `port ${oldPort}→${cdpPort}`;
1536
+ log(` ♻️ 重建 CDP 会话 (${reason}; ${portHint})...`);
1537
+ if (page) {
1538
+ await page.close().catch(() => {});
1539
+ page = null;
1540
+ }
1541
+ await browser.close().catch(() => {});
1542
+ if (oldPort !== cdpPort) {
1543
+ await killEdgeProcesses(null, oldPort).catch(() => {});
1544
+ }
1545
+ browser = await ensureBrowserReady(cdpOpts);
1546
+ page = await getOrCreatePage(browser);
1547
+ await setupPageRequestBlocking(page);
1548
+ lastRecycleAt = Date.now();
1549
+ }
1550
+
1551
+ async function maybeRecycleForMemory() {
1552
+ const mem = process.memoryUsage();
1553
+ const rssMB = mem.rss / 1024 / 1024;
1554
+ const heapMB = mem.heapUsed / 1024 / 1024;
1555
+ const edgeMB = getBrowserMemory(cdpPort);
1556
+
1557
+ const nodeOver =
1558
+ rssMB >= RECYCLE_CONFIG.nodeRssMb || heapMB >= RECYCLE_CONFIG.nodeHeapMb;
1559
+ const edgeOver = edgeMB !== null && edgeMB >= RECYCLE_CONFIG.edgeMb;
1560
+
1561
+ if (nodeOver || edgeOver) {
1562
+ if (
1563
+ RECYCLE_CONFIG.cooldownMs > 0 &&
1564
+ Date.now() - lastRecycleAt < RECYCLE_CONFIG.cooldownMs
1565
+ )
1566
+ return;
1567
+
1568
+ const parts = [];
1569
+ if (rssMB >= RECYCLE_CONFIG.nodeRssMb)
1570
+ parts.push(`rss=${rssMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeRssMb}`);
1571
+ if (heapMB >= RECYCLE_CONFIG.nodeHeapMb)
1572
+ parts.push(`heap=${heapMB.toFixed(0)}≥${RECYCLE_CONFIG.nodeHeapMb}`);
1573
+ if (edgeOver)
1574
+ parts.push(`edge:${cdpPort}=${edgeMB}≥${RECYCLE_CONFIG.edgeMb}`);
1575
+
1576
+ await recycleCdpSession(`mem ${parts.join("+")}`, {
1577
+ switchPort: edgeOver,
1578
+ });
1579
+ return;
1580
+ }
1581
+
1582
+ if (
1583
+ RECYCLE_CONFIG.periodicEvery > 0 &&
1584
+ totalRefreshed > 0 &&
1585
+ totalRefreshed % RECYCLE_CONFIG.periodicEvery === 0
1586
+ ) {
1587
+ await recycleCdpSession(`periodic #${totalRefreshed}`, {
1588
+ switchPort: false,
1589
+ });
1590
+ }
1591
+ }
1592
+
1593
+ let totalRefreshed = 0;
1594
+ let emptyRounds = 0;
1595
+ let consecutiveCaptcha = 0;
1596
+ const MAX_CONSECUTIVE_CAPTCHA = 3;
1597
+
1598
+ const clientId = randomUUID();
1599
+ const clientMeta = { type: "tag-refresh", port: cdpPort };
1600
+
1601
+ // 复用 TikTokScraper
1602
+ const enrichScraper = new TikTokScraper({
1603
+ poolSize: 3,
1604
+ proxyServer: effectiveProxy || undefined,
1605
+ });
1606
+ await enrichScraper.init();
1607
+ log("✅ TikTokScraper 已就绪 (enrich 复用)");
1608
+ log(` 客户端 ID: ${clientId.substring(0, 8)}...`);
1609
+ log("");
1610
+
1611
+ const cleanup = () => {
1612
+ log("\n正在清理资源...");
1613
+ enrichScraper.close().catch(() => {});
1614
+ killEdgeProcesses(null, cdpPort);
1615
+ process.exit(0);
1616
+ };
1617
+ process.on("SIGINT", cleanup);
1618
+
1619
+ try {
1620
+ while (true) {
1621
+ try {
1622
+ // 从服务端获取下一个 productive 标签
1623
+ const taskRes = await fetch(`${baseUrl}/api/tags/refresh-task`, {
1624
+ headers: buildClientHeaders(clientId, clientMeta),
1625
+ });
1626
+ const taskData = await taskRes.json();
1627
+
1628
+ if (!taskData.tag) {
1629
+ emptyRounds++;
1630
+
1631
+ if (emptyRounds >= 5) {
1632
+ log(
1633
+ `⏳ 连续 ${emptyRounds} 轮无 productive 标签可刷新(30 秒后重试)...`,
1634
+ );
1635
+ await new Promise((r) => setTimeout(r, 30000));
1636
+ emptyRounds = 0;
1637
+ continue;
1638
+ }
1639
+ log(
1640
+ `⏳ 暂无 productive 标签可刷新(连续 ${emptyRounds} 轮),10 秒后重试...`,
1641
+ );
1642
+ await new Promise((r) => setTimeout(r, 10000));
1643
+ continue;
1644
+ }
1645
+
1646
+ emptyRounds = 0;
1647
+
1648
+ const tag = taskData.tag.replace(/^#+/, "").trim().toLowerCase();
1649
+ const startTime = Date.now();
1650
+
1651
+ log(
1652
+ `[${totalRefreshed + 1}] 正在刷新 #${tag} (score: ${taskData.score})...`,
1653
+ );
1654
+
1655
+ const result = {
1656
+ tag,
1657
+ pushedUsers: 0,
1658
+ error: null,
1659
+ };
1660
+
1661
+ // 锁定标签
1662
+ const claimRes = await fetch(`${baseUrl}/api/tags/refresh-claim`, {
1663
+ method: "POST",
1664
+ headers: buildClientHeaders(clientId, clientMeta, {
1665
+ "Content-Type": "application/json",
1666
+ }),
1667
+ body: JSON.stringify({ tag }),
1668
+ });
1669
+ const claimData = await claimRes.json();
1670
+ if (!claimData.ok) {
1671
+ if (
1672
+ claimData.error &&
1673
+ claimData.error.includes("already being refreshed")
1674
+ ) {
1675
+ log(` ⏭️ 已被其他客户端锁定,跳过`);
1676
+ await recyclePage();
1677
+ continue;
1678
+ }
1679
+ log(` ⚠️ 无法锁定 (${claimData.error}),重试下一个`);
1680
+ await recyclePage();
1681
+ continue;
1682
+ }
1683
+
1684
+ // 抓取视频
1685
+ const fetchStart = Date.now();
1686
+ log(` 抓取 TikTok 标签页...`);
1687
+ const tagResult = await fetchTagData(tag, {
1688
+ page,
1689
+ onProgress: ({ videos, authors }) => {
1690
+ process.stderr.write(
1691
+ `\r 抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
1692
+ );
1693
+ },
1694
+ });
1695
+ const fetchSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
1696
+ log(
1697
+ `\r 完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者 (${fetchSec}s)\x1b[K`,
1698
+ );
1699
+
1700
+ let videos = tagResult.videos;
1701
+
1702
+ if (!videos || videos.length === 0) {
1703
+ const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
1704
+ log(` ⚠️ 无视频 (${deadSec}s),保留 productive 状态,等待下次刷新`);
1705
+ result.error = "no videos found";
1706
+ await fetch(`${baseUrl}/api/tags/refresh-result`, {
1707
+ method: "POST",
1708
+ headers: buildClientHeaders(clientId, clientMeta, {
1709
+ "Content-Type": "application/json",
1710
+ }),
1711
+ body: JSON.stringify({ tag, pushedUsers: 0, error: result.error }),
1712
+ }).catch(() => {});
1713
+ totalRefreshed++;
1714
+ await randomDelay(0, 5000);
1715
+ await recyclePage();
1716
+ await maybeRecycleForMemory();
1717
+ continue;
1718
+ }
1719
+
1720
+ // enrich: 先查服务端缓存,只对未知视频发 HTTP 请求
1721
+ const enriched = await enrichWithServerCache(
1722
+ videos,
1723
+ baseUrl,
1724
+ enrichScraper,
1725
+ {
1726
+ clientId,
1727
+ clientMeta,
1728
+ effectiveProxy,
1729
+ log,
1730
+ },
1731
+ );
1732
+ videos = enriched.videos;
1733
+
1734
+ // CDN 限流检测
1735
+ const cdnBlocked = enriched.cdnBlockedCount || 0;
1736
+ if (cdnBlocked > 0) {
1737
+ const cdnRatio = cdnBlocked / (videos.length || 1);
1738
+ const coolSec = cdnRatio > 0.3 ? 120 : 60;
1739
+ log(
1740
+ ` ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%),冷却 ${coolSec} 秒后重启 scraper`,
1741
+ );
1742
+ await new Promise((r) => setTimeout(r, coolSec * 1000));
1743
+ log(` 正在重启 TikTokScraper...`);
1744
+ await enrichScraper.restart();
1745
+ log(` ✅ TikTokScraper 已重启`);
1746
+ }
1747
+
1748
+ // 过滤目标国家的视频和作者
1749
+ const targetLocations = DEFAULT_TARGET_LOCATIONS;
1750
+ const filteredVideos = videos.filter((v) =>
1751
+ isLocationInList(v.locationCreated, targetLocations),
1752
+ );
1753
+ const filteredAuthors = [
1754
+ ...new Set(
1755
+ filteredVideos.map((v) => v.authorUniqueId).filter(Boolean),
1756
+ ),
1757
+ ];
1758
+ const filteredCount = videos.length - filteredVideos.length;
1759
+ if (filteredCount > 0) {
1760
+ log(` 过滤非目标国家: ${filteredCount}/${videos.length} 视频被排除`);
1761
+ }
1762
+
1763
+ // 推送过滤后的作者到服务端
1764
+ if (filteredAuthors.length > 0) {
1765
+ const pushResult = await pushToServer(
1766
+ baseUrl,
1767
+ filteredAuthors,
1768
+ filteredVideos,
1769
+ clientId,
1770
+ clientMeta,
1771
+ );
1772
+ result.pushedUsers = pushResult.added || 0;
1773
+ }
1774
+
1775
+ // 上报刷新结果(只更新 pushed_users,不改变 score/status)
1776
+ await fetch(`${baseUrl}/api/tags/refresh-result`, {
1777
+ method: "POST",
1778
+ headers: buildClientHeaders(clientId, clientMeta, {
1779
+ "Content-Type": "application/json",
1780
+ }),
1781
+ body: JSON.stringify({
1782
+ tag,
1783
+ pushedUsers: result.pushedUsers,
1784
+ error: result.error,
1785
+ }),
1786
+ }).catch(() => {});
1787
+
1788
+ totalRefreshed++;
1789
+ const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
1790
+ const memStr = ` mem=${formatMemoryWithBrowser(cdpPort)}`;
1791
+ log(
1792
+ ` 🔄 #${tag} 完成 (${elapsed}s) — ${videos.length} 视频(${filteredVideos.length} 匹配), ${filteredAuthors.length} 作者, 推送 ${result.pushedUsers} 用户${memStr}`,
1793
+ );
1794
+ log("");
1795
+
1796
+ if (totalRefreshed % 10 === 0) {
1797
+ log(
1798
+ `📊 [${totalRefreshed}] 内存汇总: ${formatMemoryWithBrowser(cdpPort)}`,
1799
+ );
1800
+ }
1801
+ await recyclePage();
1802
+ await maybeRecycleForMemory();
1803
+ await randomDelay(3000, 7000);
1804
+ } catch (e) {
1805
+ const isNetworkError =
1806
+ e.code === "ECONNREFUSED" ||
1807
+ e.code === "ENOTFOUND" ||
1808
+ e.code === "ECONNRESET" ||
1809
+ (e.message &&
1810
+ (e.message.includes("ECONNREFUSED") ||
1811
+ e.message.includes("fetch failed") ||
1812
+ e.message.includes("network")));
1813
+ if (isNetworkError) {
1814
+ log(` ⚠️ 服务端连接失败 (${e.message}),15 秒后重试...`);
1815
+ await new Promise((r) => setTimeout(r, 15000));
1816
+ continue;
1817
+ }
1818
+ log(` ⚠️ 处理出错 (${e.message}),10 秒后重试...`);
1819
+ await new Promise((r) => setTimeout(r, 10000));
1820
+ await recyclePage().catch(() => {});
1821
+ }
1822
+ }
1823
+ } finally {
1824
+ cleanup();
1825
+ }
1826
+ }
@@ -24,6 +24,7 @@ async function processAPIResponse(
24
24
  createTime: item.createTime || null,
25
25
  playCount: item.stats?.playCount || 0,
26
26
  isECVideo: item.isECVideo ? 1 : 0,
27
+ isAd: item.isAd ? 1 : 0,
27
28
  });
28
29
  }
29
30
 
@@ -74,6 +75,7 @@ async function processAPIResponse(
74
75
  createTime: item.createTime || null,
75
76
  playCount: item.stats?.playCount || 0,
76
77
  isECVideo: item.isECVideo ? 1 : 0,
78
+ isAd: item.isAd ? 1 : 0,
77
79
  });
78
80
  }
79
81
  }
package/src/lib/args.js CHANGED
@@ -746,7 +746,9 @@ function parseTagArgs(args) {
746
746
  let isDiscover = false;
747
747
  let isScore = false;
748
748
  let isScoreAll = false;
749
+ let isRefresh = false;
749
750
  let scoreAllPort = 7222;
751
+ let refreshPort = 8222;
750
752
  let scoreProxy = null;
751
753
  let scoreTag = null;
752
754
  let scoreCountries = null;
@@ -794,6 +796,8 @@ function parseTagArgs(args) {
794
796
  } else if (arg === "--port") {
795
797
  if (isScoreAll) {
796
798
  scoreAllPort = parseInt(args[++i]) || 7222;
799
+ } else if (isRefresh) {
800
+ refreshPort = parseInt(args[++i]) || 8222;
797
801
  } else {
798
802
  scorePort = parseInt(args[++i]) || 9222;
799
803
  }
@@ -826,6 +830,8 @@ function parseTagArgs(args) {
826
830
  if (scoreCountries) i += scoreCountries.length;
827
831
  } else if (cleaned.toLowerCase() === "score-all") {
828
832
  isScoreAll = true;
833
+ } else if (cleaned.toLowerCase() === "refresh") {
834
+ isRefresh = true;
829
835
  } else {
830
836
  tags.push(cleaned.toLowerCase());
831
837
  }
@@ -899,6 +905,27 @@ function parseTagArgs(args) {
899
905
  };
900
906
  }
901
907
 
908
+ if (isRefresh) {
909
+ return {
910
+ subcommand: "tag-refresh",
911
+ tagRefresh: {
912
+ serverUrl,
913
+ port: refreshPort,
914
+ proxy: scoreProxy,
915
+ },
916
+ urls: [],
917
+ outputFormat: "json",
918
+ exploreCount: 0,
919
+ showConfig: false,
920
+ showHelp: false,
921
+ customProxy: null,
922
+ configAction: null,
923
+ configValue: null,
924
+ pipeMode: false,
925
+ filterStr: null,
926
+ };
927
+ }
928
+
902
929
  return {
903
930
  subcommand: "tag",
904
931
  tagTags: {
package/src/main.js CHANGED
@@ -16,6 +16,7 @@ import {
16
16
  handleDiscover,
17
17
  handleScore,
18
18
  handleScoreAll,
19
+ handleTagRefresh,
19
20
  } from "./cli/tag.js";
20
21
 
21
22
  async function main() {
@@ -50,6 +51,8 @@ async function main() {
50
51
  return handleScore(parsed);
51
52
  case "tag-score-all":
52
53
  return handleScoreAll(parsed);
54
+ case "tag-refresh":
55
+ return handleTagRefresh(parsed);
53
56
  }
54
57
 
55
58
  const {
@@ -105,6 +105,7 @@ async function processExplore(page, username, options, log) {
105
105
  href: topVideo.href,
106
106
  playCount: topVideo.playCount,
107
107
  createTime: topVideo.createTime,
108
+ isAd: topVideo.isAd,
108
109
  };
109
110
  log(
110
111
  ` 7天内 EC视频最高播放: ${topVideo.playCount} 次播放 (${recentVideos.length} 个EC候选)`,
@@ -50,6 +50,7 @@ import {
50
50
  getJobRow,
51
51
  getJobBaseRow,
52
52
  getJob,
53
+ getBulkAuthorLocations,
53
54
  getAllJobs,
54
55
  getVideoRow,
55
56
  getAllVideoRows,
@@ -96,6 +97,9 @@ import {
96
97
  rawQuery,
97
98
  normalizeTags,
98
99
  clearTags,
100
+ getProductiveRefreshTask,
101
+ claimRefreshTag,
102
+ reportRefreshResult,
99
103
  } from "./db-tags.js";
100
104
 
101
105
  // LLM 打分
@@ -391,6 +395,7 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
391
395
  refresh_time,
392
396
  top_video_play_count,
393
397
  top_video_href,
398
+ top_video_is_ad,
394
399
  status,
395
400
  sources
396
401
  FROM jobs
@@ -490,6 +495,54 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
490
495
  };
491
496
  }
492
497
 
498
+ /**
499
+ * 查询已修改(modified_at IS NOT NULL)且 location_created 不在 targetLocations 中的目标商家,
500
+ * 按国家分组返回。
501
+ */
502
+ function getModifiedTargetUsersOutsideLocations(targetLocations = []) {
503
+ if (!getDb()) return { countries: [], total: 0 };
504
+
505
+ const placeholders = targetLocations.map(() => "?").join(", ");
506
+ const rows = getDb()
507
+ .prepare(
508
+ `
509
+ SELECT unique_id, nickname, follower_count, video_count,
510
+ location_created, confirmed_location, modified_at, latest_video_time, refresh_time
511
+ FROM jobs
512
+ WHERE tt_seller = 1
513
+ AND verified = 0
514
+ AND modified_at IS NOT NULL
515
+ AND location_created IS NOT NULL
516
+ AND location_created NOT IN (${placeholders})
517
+ ORDER BY location_created ASC, COALESCE(latest_video_time, 0) DESC
518
+ `,
519
+ )
520
+ .all(...targetLocations)
521
+ .map(mapJobRow);
522
+
523
+ const countryMap = new Map();
524
+ for (const row of rows) {
525
+ const country = row.locationCreated || "未知";
526
+ if (!countryMap.has(country)) {
527
+ countryMap.set(country, { users: [], userCount: 0 });
528
+ }
529
+ const entry = countryMap.get(country);
530
+ entry.users.push(row);
531
+ entry.userCount++;
532
+ }
533
+
534
+ const countries = [];
535
+ for (const [country, entry] of countryMap) {
536
+ countries.push({
537
+ country,
538
+ count: entry.userCount,
539
+ users: entry.users,
540
+ });
541
+ }
542
+
543
+ return { total: rows.length, countries };
544
+ }
545
+
493
546
  export function createStore(filePath, options = {}) {
494
547
  if (!filePath) {
495
548
  throw new Error("createStore requires an explicit .db path");
@@ -2292,6 +2345,7 @@ export function createStore(filePath, options = {}) {
2292
2345
  if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
2293
2346
  user.topVideoPlayCount = result.topRecentVideo.playCount || null;
2294
2347
  user.topVideoHref = result.topRecentVideo.href || null;
2348
+ user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
2295
2349
  }
2296
2350
  user.sources = [...new Set([...(user.sources || []), "processed"])];
2297
2351
  }
@@ -2518,6 +2572,7 @@ export function createStore(filePath, options = {}) {
2518
2572
  if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
2519
2573
  user.topVideoPlayCount = result.topRecentVideo.playCount || null;
2520
2574
  user.topVideoHref = result.topRecentVideo.href || null;
2575
+ user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
2521
2576
  }
2522
2577
  const newUsers = processDiscoveredUsers(result);
2523
2578
  const ret = updateJobInfo(uniqueId, user, false);
@@ -2543,6 +2598,7 @@ export function createStore(filePath, options = {}) {
2543
2598
  if (result.topRecentVideo && typeof result.topRecentVideo === "object") {
2544
2599
  user.topVideoPlayCount = result.topRecentVideo.playCount || null;
2545
2600
  user.topVideoHref = result.topRecentVideo.href || null;
2601
+ user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
2546
2602
  }
2547
2603
  const newUsers = processDiscoveredUsers(result);
2548
2604
  return { saved: true, newUsers };
@@ -3241,6 +3297,7 @@ export function createStore(filePath, options = {}) {
3241
3297
  getProcessedUsers,
3242
3298
  getAllUsers,
3243
3299
  getUserDbCount,
3300
+ getBulkAuthorLocations,
3244
3301
  getJobsCount,
3245
3302
  getRawJobsCount,
3246
3303
  getPendingJobsCount,
@@ -3260,6 +3317,8 @@ export function createStore(filePath, options = {}) {
3260
3317
  getRawJobsPage: getRawJobsPageFromDb,
3261
3318
  getTargetUsers: getTargetUsersFromDb,
3262
3319
  getTargetUsersByCountry: getTargetUsersByCountryFromDb,
3320
+ getModifiedTargetUsersOutsideLocations: (loc) =>
3321
+ getModifiedTargetUsersOutsideLocations(loc),
3263
3322
  getStats,
3264
3323
  getStatusGroups,
3265
3324
  markGroupsDirty,
@@ -3310,6 +3369,9 @@ export function createStore(filePath, options = {}) {
3310
3369
  getTagStats,
3311
3370
  normalizeTags,
3312
3371
  clearTags,
3372
+ getProductiveRefreshTask,
3373
+ claimRefreshTag,
3374
+ reportRefreshResult,
3313
3375
  data,
3314
3376
  };
3315
3377
 
@@ -48,6 +48,7 @@ export const JOB_EXT_COLUMNS = [
48
48
  "user_create_time",
49
49
  "top_video_play_count",
50
50
  "top_video_href",
51
+ "top_video_is_ad",
51
52
  ];
52
53
 
53
54
  // 完整的 jobs 列 = 核心列 + 扩展列
@@ -88,6 +89,7 @@ export const WRITABLE_JOB_COLUMNS = new Set([
88
89
  "latest_video_time",
89
90
  "top_video_play_count",
90
91
  "top_video_href",
92
+ "top_video_is_ad",
91
93
  "user_create_time",
92
94
  ]);
93
95
 
@@ -327,6 +327,34 @@ export function getJob(uniqueId) {
327
327
  return mapJobRow(getJobRow(uniqueId));
328
328
  }
329
329
 
330
+ /**
331
+ * 批量查询作者的国家信息
332
+ * 从 jobs 和 jobs_base 表中查找已知的 location_created
333
+ * @param {string[]} uniqueIds
334
+ * @returns {Record<string, string>} { uniqueId: locationCreated }
335
+ */
336
+ export function getBulkAuthorLocations(uniqueIds) {
337
+ const db = getDb();
338
+ if (!db || uniqueIds.length === 0) return {};
339
+ // SQLite 的 IN 子句不支持数组参数,动态构建占位符
340
+ const placeholders = uniqueIds.map(() => "?").join(",");
341
+ const sql = `
342
+ SELECT unique_id, location_created FROM jobs
343
+ WHERE unique_id IN (${placeholders}) AND location_created IS NOT NULL AND location_created != ''
344
+ UNION
345
+ SELECT unique_id, location_created FROM jobs_base
346
+ WHERE unique_id IN (${placeholders}) AND location_created IS NOT NULL AND location_created != ''
347
+ `;
348
+ const rows = db.prepare(sql).all(...uniqueIds, ...uniqueIds);
349
+ const result = {};
350
+ for (const r of rows) {
351
+ if (!result[r.unique_id]) {
352
+ result[r.unique_id] = r.location_created;
353
+ }
354
+ }
355
+ return result;
356
+ }
357
+
330
358
  export function getAllJobs() {
331
359
  const db = getDb();
332
360
  if (!db) return [];
@@ -127,6 +127,7 @@ export function initDb(filePath) {
127
127
  bio_link: "TEXT",
128
128
  top_video_play_count: "INTEGER",
129
129
  top_video_href: "TEXT",
130
+ top_video_is_ad: "INTEGER",
130
131
  user_create_time: "INTEGER",
131
132
  });
132
133
 
@@ -280,6 +281,8 @@ export function initDb(filePath) {
280
281
  strategy: "TEXT",
281
282
  dead_reason: "TEXT",
282
283
  direction_id: "INTEGER",
284
+ refreshing: "INTEGER DEFAULT 0",
285
+ refreshed_at: "TEXT",
283
286
  });
284
287
 
285
288
  // ===== tag_directions 表(分层探索:通用词/分类/产品) =====
@@ -1234,3 +1234,98 @@ function getLang(country) {
1234
1234
  };
1235
1235
  return COUNTRY_LANG[country] || "en";
1236
1236
  }
1237
+
1238
+ // ===== Tag 刷新 CRUD(productive 标签轮询重新抓取) =====
1239
+
1240
+ /**
1241
+ * 获取一个待刷新的 productive 标签(score > 70 且未被刷新锁定)
1242
+ * 按 refreshed_at ASC 排序(最久未刷新的优先),NULL 排最前
1243
+ */
1244
+ export function getProductiveRefreshTask() {
1245
+ const db = getDb();
1246
+ if (!db) return null;
1247
+
1248
+ // 先清理超时的 refreshing 锁(超过 30 分钟未释放)
1249
+ db.prepare(
1250
+ `UPDATE tags SET refreshing = 0
1251
+ WHERE refreshing = 1 AND refreshed_at IS NOT NULL
1252
+ AND refreshed_at < datetime('now', '-30 minutes')`,
1253
+ ).run();
1254
+
1255
+ const row = db
1256
+ .prepare(
1257
+ `SELECT * FROM tags
1258
+ WHERE status = 'productive' AND score > 70 AND refreshing = 0
1259
+ ORDER BY refreshed_at IS NULL DESC, refreshed_at ASC
1260
+ LIMIT 1`,
1261
+ )
1262
+ .get();
1263
+ return row ? parseTagRow(row) : null;
1264
+ }
1265
+
1266
+ /**
1267
+ * 锁定一个 productive 标签用于刷新(防并发冲突)
1268
+ */
1269
+ export function claimRefreshTag(tag) {
1270
+ const db = getDb();
1271
+ if (!db) return { ok: false, error: "db not ready" };
1272
+
1273
+ const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
1274
+ if (!normalized || normalized.length < 2) {
1275
+ return { ok: false, error: "invalid tag" };
1276
+ }
1277
+
1278
+ // 同时尝试匹配带 # 和不带 # 的格式(兼容旧数据)
1279
+ const result = db
1280
+ .prepare(
1281
+ `UPDATE tags SET refreshing = 1, refreshed_at = datetime('now')
1282
+ WHERE (tag = ? OR tag = ?) AND status = 'productive' AND score > 70 AND refreshing = 0`,
1283
+ )
1284
+ .run(normalized, `#${normalized}`);
1285
+ if (result.changes === 0) {
1286
+ const row = db
1287
+ .prepare("SELECT status, refreshing FROM tags WHERE tag = ? OR tag = ?")
1288
+ .get(normalized, `#${normalized}`);
1289
+ if (!row) return { ok: false, error: "tag not found" };
1290
+ if (row.refreshing) return { ok: false, error: "already being refreshed" };
1291
+ return {
1292
+ ok: false,
1293
+ error: `tag status is ${row.status}, not eligible for refresh`,
1294
+ };
1295
+ }
1296
+ return { ok: true, tag: normalized };
1297
+ }
1298
+
1299
+ /**
1300
+ * 报告刷新结果并解锁标签(不改变 score/status,只更新 refreshed_at 和 pushed_users)
1301
+ */
1302
+ export function reportRefreshResult(tag, fields) {
1303
+ const db = getDb();
1304
+ if (!db) return { ok: false, error: "db not ready" };
1305
+ const { pushedUsers, error } = fields;
1306
+ const now = new Date().toISOString();
1307
+ const normalized = tag.replace(/^#+/, "").trim().toLowerCase();
1308
+
1309
+ try {
1310
+ const result = db
1311
+ .prepare(
1312
+ `UPDATE tags SET
1313
+ refreshing = 0,
1314
+ refreshed_at = ?,
1315
+ pushed_users = COALESCE(pushed_users, 0) + COALESCE(?, 0),
1316
+ last_error = COALESCE(?, last_error),
1317
+ score_count = score_count + 1
1318
+ WHERE tag = ? OR tag = ?`,
1319
+ )
1320
+ .run(
1321
+ now,
1322
+ pushedUsers ?? null,
1323
+ error ?? null,
1324
+ normalized,
1325
+ `#${normalized}`,
1326
+ );
1327
+ return { ok: result.changes > 0, tag: normalized };
1328
+ } catch (e) {
1329
+ return { ok: false, error: e.message };
1330
+ }
1331
+ }
@@ -735,6 +735,9 @@ async function selectLocation(uniqueId, location) {
735
735
  userInMem.locationCreated = location;
736
736
  userInMem.modifiedAt = Date.now();
737
737
  }
738
+
739
+ // 刷新"其他国家"区域(修改后可能需要更新统计)
740
+ fetchTargetOtherCountries();
738
741
  } catch (e) {
739
742
  showToast("更新失败: " + e.message, true);
740
743
  } finally {
@@ -1023,6 +1026,30 @@ document.getElementById("statTargetCard").addEventListener("click", () => {
1023
1026
  window.addEventListener("hashchange", handleRoute);
1024
1027
  window.addEventListener("DOMContentLoaded", handleRoute);
1025
1028
 
1029
+ // 回顶部按钮:仅在 targetPage 激活时监听滚动
1030
+ window.addEventListener(
1031
+ "scroll",
1032
+ function () {
1033
+ const btn = document.getElementById("backToTop");
1034
+ if (!btn) return;
1035
+ const targetPage = document.getElementById("targetPage");
1036
+ if (!targetPage || !targetPage.classList.contains("active")) {
1037
+ btn.classList.remove("visible");
1038
+ return;
1039
+ }
1040
+ if (window.scrollY > 400) {
1041
+ btn.classList.add("visible");
1042
+ } else {
1043
+ btn.classList.remove("visible");
1044
+ }
1045
+ },
1046
+ { passive: true },
1047
+ );
1048
+
1049
+ function scrollToTop() {
1050
+ window.scrollTo({ top: 0, behavior: "smooth" });
1051
+ }
1052
+
1026
1053
  function handleRoute() {
1027
1054
  const hash = window.location.hash;
1028
1055
  if (hash === "#pending") {
@@ -1410,18 +1437,20 @@ function renderTagsTable() {
1410
1437
  const created = t.created_at || "-";
1411
1438
  const score = t.score != null ? t.score.toFixed(1) : "-";
1412
1439
  const authorCount = t.author_count ?? "-";
1413
- const totalPosts = t.total_posts ?? "-";
1414
1440
  const matchedAuthors = t.matched_authors ?? "-";
1415
1441
  const pushedUsers = t.pushed_users ?? "-";
1442
+ const refreshedAt = t.refreshed_at
1443
+ ? t.refreshed_at.slice(0, 16).replace("T", " ")
1444
+ : "-";
1416
1445
 
1417
1446
  return `<tr>
1418
1447
  <td style="color:#9ca3af;font-size:12px;text-align:center">${i + 1}</td>
1419
1448
  <td style="font-weight:600;color:#e0e0e0">#${escapeHtml(t.tag)}</td>
1420
- <td style="color:${t.score >= 5 ? "#22c55e" : t.score >= 3 ? "#facc15" : "#888"}">${score}</td>
1421
- <td>${authorCount}</td>
1422
- <td>${totalPosts}</td>
1423
- <td>${matchedAuthors}</td>
1424
- <td>${pushedUsers}</td>
1449
+ <td style="text-align:center;color:${t.score >= 5 ? "#22c55e" : t.score >= 3 ? "#facc15" : "#888"}">${score}</td>
1450
+ <td style="text-align:center">${authorCount}</td>
1451
+ <td style="text-align:center;font-size:11px;color:#9ca3af">${refreshedAt}</td>
1452
+ <td style="text-align:center">${matchedAuthors}</td>
1453
+ <td style="text-align:center">${pushedUsers}</td>
1425
1454
  <td style="font-size:11px">${countries}</td>
1426
1455
  <td>${statusTag}</td>
1427
1456
  <td style="font-size:11px;color:#888">${source}</td>
@@ -1517,6 +1546,7 @@ function showTargetPage() {
1517
1546
  document.getElementById("tagsPage").classList.remove("active");
1518
1547
  showLoading("正在加载目标商家数据...");
1519
1548
  fetchTargetByCountry();
1549
+ fetchTargetOtherCountries();
1520
1550
  // 同步统计
1521
1551
  if (currentStats) {
1522
1552
  document.getElementById("targetPageStatTotal").textContent = formatStatNum(
@@ -1535,6 +1565,44 @@ let currentTargetSeq = 0;
1535
1565
  let currentTargetSort = { key: null, asc: true };
1536
1566
  const TARGET_PAGE_SIZE = 200;
1537
1567
 
1568
+ async function fetchTargetOtherCountries() {
1569
+ try {
1570
+ const res = await fetch("/api/target-users-other-countries");
1571
+ const data = await res.json();
1572
+ renderTargetOtherCountries(data.countries || []);
1573
+ } catch (e) {
1574
+ console.error("获取其他国家数据失败:", e);
1575
+ }
1576
+ }
1577
+
1578
+ function renderTargetOtherCountries(countries) {
1579
+ const section = document.getElementById("targetOtherSection");
1580
+ const grid = document.getElementById("targetOtherGrid");
1581
+ if (!section || !grid) return;
1582
+
1583
+ if (!countries.length) {
1584
+ section.style.display = "none";
1585
+ return;
1586
+ }
1587
+ section.style.display = "block";
1588
+ const total = countries.reduce((sum, c) => sum + (c.count || 0), 0);
1589
+ grid.innerHTML = countries
1590
+ .map((c) => {
1591
+ const cnt = c.count || 0;
1592
+ const pct = total > 0 ? ((cnt / total) * 100).toFixed(1) : "0.0";
1593
+ const safeCountry = escapeJsString(c.country);
1594
+ const sel = currentTargetLocation === c.country ? " selected" : "";
1595
+ return `
1596
+ <div class="pending-country-item other-country-item${sel}"
1597
+ onclick="filterTargetByCountry('${safeCountry}')">
1598
+ <div class="country-name">${c.country}</div>
1599
+ <div class="country-count">${cnt}</div>
1600
+ <div class="country-label">${pct}%</div>
1601
+ </div>`;
1602
+ })
1603
+ .join("");
1604
+ }
1605
+
1538
1606
  async function fetchTargetByCountry() {
1539
1607
  try {
1540
1608
  const res = await fetch("/api/target-users-by-country?summary=1");
@@ -1712,9 +1780,12 @@ function renderTargetTable() {
1712
1780
  u.topVideoPlayCount != null && u.topVideoPlayCount >= 1000
1713
1781
  ? formatNum(u.topVideoPlayCount)
1714
1782
  : "-";
1783
+ const adTag = u.topVideoIsAd
1784
+ ? ' <span style="color:red">(AD)</span>'
1785
+ : "";
1715
1786
  const topPlayCountCell = u.topVideoHref
1716
- ? `<td data-label="最大播放量" style="font-size:11px;color:#888"><a href="${u.topVideoHref}" target="_blank" style="color:#3b82f6;text-decoration:none" title="点击查看视频">${topPlayCount}</a></td>`
1717
- : `<td data-label="最大播放量" style="font-size:11px;color:#888">${topPlayCount}</td>`;
1787
+ ? `<td data-label="最大播放量" style="font-size:11px;color:#888"><a href="${u.topVideoHref}" target="_blank" style="color:#3b82f6;text-decoration:none" title="点击查看视频">${topPlayCount}${adTag}</a></td>`
1788
+ : `<td data-label="最大播放量" style="font-size:11px;color:#888">${topPlayCount}${adTag}</td>`;
1718
1789
 
1719
1790
  const editIcon = ' <span style="font-size:10px;opacity:0.5">✏️</span>';
1720
1791
  const locationCell = u.modifiedAt
@@ -1782,13 +1853,9 @@ function filterTargetByCountry(country) {
1782
1853
  }
1783
1854
 
1784
1855
  function clearTargetFilters() {
1785
- currentTargetLocation = "";
1856
+ // 只清空搜索关键词,保留当前选中的国家筛选
1786
1857
  const searchInput = document.getElementById("targetSearchInput");
1787
- const locationFilter = document.getElementById("targetLocationFilter");
1788
1858
  if (searchInput) searchInput.value = "";
1789
- if (locationFilter) locationFilter.value = "";
1790
- const btn = document.getElementById("targetReprocessBtn");
1791
- if (btn) btn.style.display = "none";
1792
1859
  if (currentTargetSummary) {
1793
1860
  renderTargetCountryGrid(currentTargetSummary.countries || []);
1794
1861
  loadTargetUsersPage();
@@ -328,6 +328,11 @@
328
328
  <span style="color:#666;font-size:12px">加载中...</span>
329
329
  </div>
330
330
  <div class="muted-tip">点击国家卡片可筛选列表。</div>
331
+ <div class="target-other-section" id="targetOtherSection" style="display:none">
332
+ <h3 style="color:#f59e0b;margin-top:20px;margin-bottom:12px">🌍 其他国家 (已修改)</h3>
333
+ <div class="pending-country-grid" id="targetOtherGrid"></div>
334
+ <div class="muted-tip">这些商家已被手动修改到非目标国家。</div>
335
+ </div>
331
336
  </div>
332
337
  <div class="target-table-card">
333
338
  <h3>目标商家列表</h3>
@@ -361,6 +366,7 @@
361
366
  </div>
362
367
  </div>
363
368
  </div>
369
+ <div id="backToTop" title="回到顶部" onclick="scrollToTop()">↑</div>
364
370
  </div>
365
371
  <div id="tagsPage">
366
372
  <div class="stats" style="margin-bottom:16px">
@@ -415,7 +421,7 @@
415
421
  <th style="min-width:120px">关键词</th>
416
422
  <th class="sortable-tag" data-sort="score">评分 <span class="sort-icon">↕</span></th>
417
423
  <th class="sortable-tag" data-sort="author_count">作者数 <span class="sort-icon">↕</span></th>
418
- <th class="sortable-tag" data-sort="total_posts">帖子数 <span class="sort-icon">↕</span></th>
424
+ <th class="sortable-tag" data-sort="refreshed_at">最近刷新 <span class="sort-icon">↕</span></th>
419
425
  <th class="sortable-tag" data-sort="matched_authors">匹配作者 <span class="sort-icon">↕</span></th>
420
426
  <th class="sortable-tag" data-sort="pushed_users">推送用户 <span class="sort-icon">↕</span></th>
421
427
  <th>国家</th>
@@ -839,6 +839,36 @@ td.user-id:hover {
839
839
  display: block;
840
840
  }
841
841
 
842
+ #backToTop {
843
+ position: fixed;
844
+ bottom: 32px;
845
+ right: 32px;
846
+ width: 44px;
847
+ height: 44px;
848
+ border-radius: 50%;
849
+ background: rgba(254, 44, 85, 0.85);
850
+ color: #fff;
851
+ font-size: 20px;
852
+ display: none;
853
+ align-items: center;
854
+ justify-content: center;
855
+ cursor: pointer;
856
+ z-index: 999;
857
+ box-shadow: 0 4px 16px rgba(254, 44, 85, 0.35);
858
+ transition: opacity 0.25s, transform 0.25s, background 0.2s;
859
+ user-select: none;
860
+ }
861
+
862
+ #backToTop:hover {
863
+ background: #e61944;
864
+ transform: translateY(-2px);
865
+ box-shadow: 0 6px 20px rgba(254, 44, 85, 0.5);
866
+ }
867
+
868
+ #backToTop.visible {
869
+ display: flex;
870
+ }
871
+
842
872
  #mainPage.hidden {
843
873
  display: none;
844
874
  }
@@ -1004,6 +1034,19 @@ td.user-id:hover {
1004
1034
  color: #ddd6fe;
1005
1035
  }
1006
1036
 
1037
+ .other-country-item {
1038
+ border-color: #f59e0b;
1039
+ }
1040
+
1041
+ .other-country-item .country-name {
1042
+ color: #f59e0b;
1043
+ }
1044
+
1045
+ .other-country-item:hover {
1046
+ border-color: #fbbf24;
1047
+ background: #3a3520;
1048
+ }
1049
+
1007
1050
  .back-btn {
1008
1051
  padding: 6px 14px;
1009
1052
  border: 1px solid #333;
@@ -93,6 +93,7 @@ function inferClientType(routePath) {
93
93
  if (routePath.startsWith("/api/redo-job")) return "refresh";
94
94
  if (routePath.startsWith("/api/user-update-tasks")) return "attach";
95
95
  if (routePath.startsWith("/api/comment-task")) return "comments";
96
+ if (routePath.startsWith("/api/tags/refresh")) return "tag-refresh";
96
97
  if (routePath.startsWith("/api/tags")) return "scoring";
97
98
  if (
98
99
  routePath.startsWith("/api/job") ||
@@ -142,9 +143,12 @@ export function startWatchServer(
142
143
  let discoverRunning = false;
143
144
  let lastDiscoverTime = 0;
144
145
 
145
- // 标签池充足阈值:new 标签总数超过此值时,定时触发直接跳过(不浪费 LLM token)
146
- const DISCOVER_SUFFICIENT_TOTAL =
147
- parseInt(process.env.TAG_DISCOVER_SUFFICIENT || "30", 10) || 30;
146
+ // 每国标签不足阈值:某国 new 标签低于此值时触发 discover
147
+ const DISCOVER_PER_COUNTRY_THRESHOLD = Math.max(
148
+ 3,
149
+ parseInt(process.env.TAG_DISCOVER_PER_COUNTRY_THRESHOLD || "10", 10) ||
150
+ 10,
151
+ );
148
152
 
149
153
  async function runAutoDiscover(trigger = "timer") {
150
154
  if (discoverRunning) return;
@@ -156,23 +160,14 @@ export function startWatchServer(
156
160
  );
157
161
  return;
158
162
  }
159
- // 先看全局总量:标签池充足时直接跳过,不浪费 LLM token
160
- const globalStats = store.getTagStats();
161
- const totalNew = globalStats?.new || 0;
162
- if (totalNew >= DISCOVER_SUFFICIENT_TOTAL) {
163
- console.error(
164
- `[discover] 标签池充足 (${totalNew} 个 new ≥ ${DISCOVER_SUFFICIENT_TOTAL}),跳过 [触发: ${trigger}]`,
165
- );
166
- return;
167
- }
168
163
  discoverRunning = true;
169
164
  try {
170
- // 只为 new 标签不足的国家发现
165
+ // 按国家维度判断:只为 new 标签不足的国家发现
171
166
  const countriesToDiscover = [];
172
167
  const { getTagsByStatus } = await import("./db-tags.js");
173
168
  for (const country of DEFAULT_TARGET_LOCATIONS) {
174
169
  const newTags = getTagsByStatus("new", 100, 0, country);
175
- if (newTags.length < 10) {
170
+ if (newTags.length < DISCOVER_PER_COUNTRY_THRESHOLD) {
176
171
  countriesToDiscover.push(country);
177
172
  }
178
173
  }
@@ -860,12 +855,13 @@ export function startWatchServer(
860
855
  offset: parseInt(params.offset || "0", 10),
861
856
  },
862
857
  );
863
- // 确保每个用户对象包含 topVideoPlayCount 和 topVideoHref
858
+ // 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
864
859
  if (result.users && Array.isArray(result.users)) {
865
860
  result.users = result.users.map((u) => ({
866
861
  ...u,
867
862
  topVideoPlayCount: u.topVideoPlayCount || null,
868
863
  topVideoHref: u.topVideoHref || null,
864
+ topVideoIsAd: u.topVideoIsAd ?? null,
869
865
  }));
870
866
  }
871
867
  sendJSON(res, 200, result);
@@ -874,7 +870,7 @@ export function startWatchServer(
874
870
 
875
871
  // 默认:全量(兼容旧调用)
876
872
  const result = store.getTargetUsersByCountry(DEFAULT_TARGET_LOCATIONS);
877
- // 确保每个用户对象包含 topVideoPlayCount 和 topVideoHref
873
+ // 确保每个用户对象包含 topVideoPlayCount、topVideoHref 和 topVideoIsAd
878
874
  if (result.countries && Array.isArray(result.countries)) {
879
875
  for (const country of result.countries) {
880
876
  if (country.users && Array.isArray(country.users)) {
@@ -882,6 +878,7 @@ export function startWatchServer(
882
878
  ...u,
883
879
  topVideoPlayCount: u.topVideoPlayCount || null,
884
880
  topVideoHref: u.topVideoHref || null,
881
+ topVideoIsAd: u.topVideoIsAd ?? null,
885
882
  }));
886
883
  }
887
884
  }
@@ -890,6 +887,17 @@ export function startWatchServer(
890
887
  return;
891
888
  }
892
889
 
890
+ if (
891
+ req.method === "GET" &&
892
+ routePath === "/api/target-users-other-countries"
893
+ ) {
894
+ const result = store.getModifiedTargetUsersOutsideLocations(
895
+ DEFAULT_TARGET_LOCATIONS,
896
+ );
897
+ sendJSON(res, 200, result);
898
+ return;
899
+ }
900
+
893
901
  if (req.method === "GET" && routePath === "/api/client-errors") {
894
902
  sendJSON(res, 200, { clients: store.getClientErrors() });
895
903
  return;
@@ -1327,6 +1335,71 @@ export function startWatchServer(
1327
1335
  return;
1328
1336
  }
1329
1337
 
1338
+ // POST /api/bulk-author-locations — 批量查询作者国家(用于 tag 客户端 enrich 前跳过已知数据)
1339
+ if (req.method === "POST" && routePath === "/api/bulk-author-locations") {
1340
+ try {
1341
+ const body = await readBody(req);
1342
+ const uniqueIds = Array.isArray(body.uniqueIds) ? body.uniqueIds : [];
1343
+ const locations = store.getBulkAuthorLocations(uniqueIds);
1344
+ sendJSON(res, 200, { locations });
1345
+ } catch (e) {
1346
+ sendJSON(res, 500, { error: e.message });
1347
+ }
1348
+ return;
1349
+ }
1350
+
1351
+ // GET /api/tags/refresh-task — 领取一个 productive 标签用于刷新(score>70,未被锁定)
1352
+ if (req.method === "GET" && routePath === "/api/tags/refresh-task") {
1353
+ const task = store.getProductiveRefreshTask();
1354
+ if (!task) {
1355
+ sendJSON(res, 200, {
1356
+ tag: null,
1357
+ message: "no productive tags available for refresh",
1358
+ });
1359
+ return;
1360
+ }
1361
+ sendJSON(res, 200, {
1362
+ tag: task.tag,
1363
+ score: task.score,
1364
+ refreshedAt: task.refreshed_at,
1365
+ });
1366
+ return;
1367
+ }
1368
+
1369
+ // POST /api/tags/refresh-claim { tag } — 锁定 productive 标签用于刷新(防并发冲突)
1370
+ if (req.method === "POST" && routePath === "/api/tags/refresh-claim") {
1371
+ try {
1372
+ const body = await readBody(req);
1373
+ const tag = (body.tag || "").trim().toLowerCase();
1374
+ if (!tag) {
1375
+ sendJSON(res, 400, { error: "tag 不能为空" });
1376
+ return;
1377
+ }
1378
+ const result = store.claimRefreshTag(tag);
1379
+ sendJSON(res, result.ok ? 200 : 409, result);
1380
+ } catch (e) {
1381
+ sendJSON(res, 500, { error: e.message });
1382
+ }
1383
+ return;
1384
+ }
1385
+
1386
+ // POST /api/tags/refresh-result { tag, pushedUsers, error } — 报告刷新结果并解锁
1387
+ if (req.method === "POST" && routePath === "/api/tags/refresh-result") {
1388
+ try {
1389
+ const body = await readBody(req);
1390
+ const tag = (body.tag || "").trim().toLowerCase();
1391
+ if (!tag) {
1392
+ sendJSON(res, 400, { error: "tag 不能为空" });
1393
+ return;
1394
+ }
1395
+ const result = store.reportRefreshResult(tag, body);
1396
+ sendJSON(res, result.ok ? 200 : 400, result);
1397
+ } catch (e) {
1398
+ sendJSON(res, 500, { error: e.message });
1399
+ }
1400
+ return;
1401
+ }
1402
+
1330
1403
  // GET /api/tags/recent-generated?country=ES&lookback=10 — 最近N轮生成的标签(去重)
1331
1404
  if (req.method === "GET" && routePath === "/api/tags/recent-generated") {
1332
1405
  try {
@@ -1915,7 +1988,7 @@ export function startWatchServer(
1915
1988
  );
1916
1989
  setTimeout(() => runAutoDiscover("startup"), 30000);
1917
1990
  console.error(
1918
- `[discover] 定时+按需发现已启动: 定时间隔 ${DISCOVER_INTERVAL_MS / 60000} 分钟, 每国 ${DISCOVER_COUNT_PER_COUNTRY} 个标签, 按需阈值 ${DISCOVER_LOW_THRESHOLD}, 冷却期 ${DISCOVER_COOLDOWN_MS / 1000}s`,
1991
+ `[discover] 定时+按需发现已启动: 定时间隔 ${DISCOVER_INTERVAL_MS / 60000} 分钟, 每国 ${DISCOVER_COUNT_PER_COUNTRY} 个标签, 每国不足阈值 ${DISCOVER_PER_COUNTRY_THRESHOLD}, 按需触发阈值 ${DISCOVER_LOW_THRESHOLD}, 冷却期 ${DISCOVER_COOLDOWN_MS / 1000}s`,
1919
1992
  );
1920
1993
 
1921
1994
  _resolve({ server, port });