tt-help-cli-ycl 1.3.95 → 1.3.97

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.95",
+  "version": "1.3.97",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -9,6 +9,7 @@ import {
   DEFAULT_TARGET_LOCATIONS,
   isLocationInList,
 } from "../lib/target-locations.js";
+import { delay as randomDelay } from "../lib/delay.js";
 import { discoverTags } from "../lib/tag-discover.js";
 import {
   server as cfgServer,
@@ -121,13 +122,13 @@ async function processTag(
       port: port || 9222,
       onProgress: ({ videos, authors }) => {
         process.stderr.write(
-          `\r${prefix} #${tag}: ${videos} 视频, ${authors} 作者`,
+          `\r${prefix} #${tag}: ${videos} 视频, ${authors} 作者\x1b[K`,
         );
       },
     });
 
     process.stderr.write(
-      `\r${prefix} #${tag}: ${result.videoCount} 视频, ${result.uniqueAuthorCount} 作者`,
+      `\r${prefix} #${tag}: ${result.videoCount} 视频, ${result.uniqueAuthorCount} 作者\x1b[K`,
     );
 
     let videos = result.videos;
@@ -154,7 +155,7 @@ async function processTag(
             locationCreated &&
             isLocationInList(locationCreated, targetLocations);
           process.stderr.write(
-            `\r  [${done}/${total}] ${label} → ${loc}${hit ? " ✓" : ""}`,
+            `\r  [${done}/${total}] ${label} → ${loc}${hit ? " ✓" : ""}\x1b[K`,
           );
         },
       });
@@ -295,8 +296,8 @@ async function scoreSingleTag(
   { baseUrl, cdpPort, targetCountries, effectiveProxy },
 ) {
   const log = (...args) => process.stderr.write(args.join(" ") + "\n");
-  const progress = (msg) => process.stderr.write(`\r  ${msg}`);
-  const clearLine = () => process.stderr.write("\r" + " ".repeat(80) + "\r");
+  const progress = (msg) => process.stderr.write(`\r  ${msg}\x1b[K`);
+  const clearLine = () => process.stderr.write("\r\x1b[K");
 
   const startTime = Date.now();
 
@@ -613,7 +614,7 @@ export async function handleScoreAll(parsed) {
   const cdpOpts = { port: cdpPort };
   if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
   const browser = await ensureBrowserReady(cdpOpts);
-  const page = await getOrCreatePage(browser);
+  let page = await getOrCreatePage(browser);
 
   let totalScored = 0;
   let emptyRounds = 0; // 连续无任务的轮数
@@ -633,6 +634,15 @@ export async function handleScoreAll(parsed) {
   log(`  客户端 ID: ${clientId.substring(0, 8)}...`);
   log("");
 
+  // Ctrl+C 时关闭浏览器和 scraper
+  const cleanup = () => {
+    log("\n正在清理资源...");
+    enrichScraper.close().catch(() => {});
+    killEdgeProcesses(null, cdpPort);
+    process.exit(0);
+  };
+  process.on("SIGINT", cleanup);
+
   try {
     while (true) {
       try {
@@ -720,15 +730,19 @@ export async function handleScoreAll(parsed) {
         }
 
         // 抓取视频（CDP 连接已登录 Edge）
+        const fetchStart = Date.now();
         log(`  抓取 TikTok 标签页...`);
         const tagResult = await fetchTagData(tag, {
           port: cdpPort,
           onProgress: ({ videos, authors }) => {
-            process.stderr.write(`\r  抓取中: ${videos} 视频, ${authors} 作者`);
+            process.stderr.write(
+              `\r  抓取中: ${videos} 视频, ${authors} 作者\x1b[K`,
+            );
           },
         });
+        const fetchSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
         log(
-          `\r  完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者`,
+          `\r  完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者 (${fetchSec}s)\x1b[K`,
         );
 
         result.totalPosts = tagResult.totalPosts || 0;
@@ -736,11 +750,23 @@ export async function handleScoreAll(parsed) {
         let videos = tagResult.videos;
 
         if (!videos || videos.length === 0) {
-          log("  ⚠️ 无视频，标记 dead");
+          const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
+          const memMB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(
+            0,
+          );
+          log(`  ⚠️ 无视频 (${deadSec}s) mem=${memMB}MB，标记 dead`);
           result.status = "dead";
           result.error = "no videos found";
           await reportToServer(baseUrl, result, clientId, clientMeta);
           totalScored++;
+          // 导航到 about:blank 释放页面状态再跳过
+          await page
+            .goto("about:blank", {
+              waitUntil: "domcontentloaded",
+              timeout: 5000,
+            })
+            .catch(() => {});
+          process.stderr.write(`  → page=${page.url()}\n`);
           continue;
         }
 
@@ -751,7 +777,7 @@ export async function handleScoreAll(parsed) {
           onProgress: ({ done, total, current, locationCreated }) => {
             if (done % 10 === 0 || done === total) {
               process.stderr.write(
-                `\r    [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}`,
+                `\r    [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}\x1b[K`,
               );
             }
           },
@@ -760,6 +786,20 @@ export async function handleScoreAll(parsed) {
         const enriched = await enrichVideosWithLocation(videos, enrichOpts);
         videos = enriched.videos;
 
+        // CDN 限流检测：有拦截则冷却 + 重启 scraper
+        const cdnBlocked = enriched.cdnBlockedCount || 0;
+        if (cdnBlocked > 0) {
+          const cdnRatio = cdnBlocked / (videos.length || 1);
+          const coolSec = cdnRatio > 0.3 ? 120 : 60;
+          log(
+            `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
+          );
+          await new Promise((r) => setTimeout(r, coolSec * 1000));
+          log(`  正在重启 TikTokScraper...`);
+          await enrichScraper.restart();
+          log(`  ✅ TikTokScraper 已重启`);
+        }
+
         // 过滤 + 算分 (共用函数)
         const { matchedAuthorSet } = applyFilterAndScore(
           videos,
@@ -795,10 +835,25 @@ export async function handleScoreAll(parsed) {
         const mc = result.matchedCountries
           .map((c) => `${c.c}:${c.n}`)
           .join(" ");
+        // Node.js 进程内存占用
+        const memMB = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(0);
+        const memStr = ` mem=${memMB}MB`;
         log(
-          `  ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? "  " + mc : ""}`,
+          `  ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)${mc ? "  " + mc : ""}${memStr}`,
         );
         log("");
+
+        // 随机等待 3-7 秒，避免连续访问 TikTok 触发风控
+        await randomDelay(3000, 7000);
+
+        // 导航到 about:blank 卸载页面，状态清零，下次 goto 重新初始化
+        await page
+          .goto("about:blank", { waitUntil: "domcontentloaded", timeout: 5000 })
+          .catch((e) => {
+            log(`  ⚠️ about:blank 跳转失败: ${e.message}`);
+          });
+        process.stderr.write(`  → page=${page.url()}\n`);
+        await randomDelay(3000, 7000);
       } catch (e) {
         // 区分网络错误和业务错误
         const isNetworkError =

package/src/lib/api-interceptor.js

@@ -23,6 +23,7 @@ async function processAPIResponse(
       href,
       createTime: item.createTime || null,
       playCount: item.stats?.playCount || 0,
+      isECVideo: item.isECVideo ? 1 : 0,
     });
   }
 
@@ -72,6 +73,7 @@ async function processAPIResponse(
             href,
             createTime: item.createTime || null,
             playCount: item.stats?.playCount || 0,
+            isECVideo: item.isECVideo ? 1 : 0,
           });
         }
       }

package/src/lib/tag-fetcher.js

@@ -2,6 +2,7 @@ import { chromium } from "playwright";
 import { ensureBrowserReady } from "./browser/cdp.js";
 import { getOrCreatePage } from "./browser/page.js";
 import { TikTokScraper } from "./tiktok-scraper.mjs";
+import { CDNBlockedError } from "./parse-ssr.mjs";
 
 const TAG_URL = "https://www.tiktok.com/tag";
 const SCROLL_INTERVAL = 3000;
@@ -56,43 +57,45 @@ export async function fetchTagData(tag, options = {}) {
   const browser = await ensureBrowserReady(cdpOptions);
   const page = await getOrCreatePage(browser);
 
-  try {
-    let challengeInfo = null;
-    const rawVideos = [];
-    const seenVideoIds = new Set();
-    const authors = new Set();
-
-    page.on("response", async (resp) => {
-      try {
-        const url = resp.url();
-        const ct = resp.headers()["content-type"] || "";
-
-        if (url.includes("/api/challenge/detail/") && ct.includes("json")) {
-          const body = await resp.json();
-          if (body?.challengeInfo?.challenge) {
-            challengeInfo = body.challengeInfo.challenge;
-          }
+  let challengeInfo = null;
+  const rawVideos = [];
+  const seenVideoIds = new Set();
+  const authors = new Set();
+
+  const responseHandler = async (resp) => {
+    try {
+      const url = resp.url();
+      const ct = resp.headers()["content-type"] || "";
+
+      if (url.includes("/api/challenge/detail/") && ct.includes("json")) {
+        const body = await resp.json();
+        if (body?.challengeInfo?.challenge) {
+          challengeInfo = body.challengeInfo.challenge;
         }
+      }
 
-        if (url.includes("/api/challenge/item_list/") && ct.includes("json")) {
-          const body = await resp.json();
-          if (!body?.itemList) return;
-          for (const item of body.itemList) {
-            const vid = item.id || "";
-            if (vid && !seenVideoIds.has(vid)) {
-              seenVideoIds.add(vid);
-              const uid = item.author?.uniqueId || "";
-              if (uid) authors.add(uid);
-              rawVideos.push(extractItemData(item));
-            }
-          }
-          if (onProgress) {
-            onProgress({ videos: rawVideos.length, authors: authors.size });
+      if (url.includes("/api/challenge/item_list/") && ct.includes("json")) {
+        const body = await resp.json();
+        if (!body?.itemList) return;
+        for (const item of body.itemList) {
+          const vid = item.id || "";
+          if (vid && !seenVideoIds.has(vid)) {
+            seenVideoIds.add(vid);
+            const uid = item.author?.uniqueId || "";
+            if (uid) authors.add(uid);
+            rawVideos.push(extractItemData(item));
           }
         }
-      } catch {}
-    });
+        if (onProgress) {
+          onProgress({ videos: rawVideos.length, authors: authors.size });
+        }
+      }
+    } catch {}
+  };
+
+  page.on("response", responseHandler);
 
+  try {
     const tagUrl = `${TAG_URL}/${encodeURIComponent(tag)}`;
     const resp = await page.goto(tagUrl, {
       waitUntil: "domcontentloaded",
@@ -185,7 +188,7 @@ export async function fetchTagData(tag, options = {}) {
       uniqueAuthors: [...authors],
     };
   } finally {
-    // 不关闭 page 和 browser，由用户自行关闭
+    page.off("response", responseHandler);
   }
 }
 
@@ -197,7 +200,7 @@ export async function fetchTagData(tag, options = {}) {
  * @param {number} [options.poolSize=3] - 并发页面数
  * @param {number} [options.maxRetries=3] - 单个请求最大重试次数
  * @param {Function} [options.onProgress] - 进度回调 ({ done, total, current, locationCreated })
- * @returns {Promise<{ videos: Array, locationMap: Record<string, string|null> }>}
+ * @returns {Promise<{ videos: Array, locationMap: Record<string, string|null>, cdnBlockedCount: number }>}
  */
 export async function enrichVideosWithLocation(videos, options = {}) {
   const {
@@ -219,6 +222,8 @@ export async function enrichVideosWithLocation(videos, options = {}) {
     const locationMap = {};
     let done = 0;
 
+    let cdnBlockedCount = 0;
+
     if (mode === "users") {
       const uniqueAuthors = [
         ...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
@@ -239,7 +244,10 @@ export async function enrichVideosWithLocation(videos, options = {}) {
                 current: uniqueId,
                 locationCreated: location,
               });
-          } catch {
+          } catch (err) {
+            if (err instanceof CDNBlockedError) {
+              cdnBlockedCount++;
+            }
             locationMap[uniqueId] = null;
             done++;
             if (onProgress)
@@ -277,7 +285,10 @@ export async function enrichVideosWithLocation(videos, options = {}) {
                 current: videoUrl,
                 locationCreated: location,
               });
-          } catch {
+          } catch (err) {
+            if (err instanceof CDNBlockedError) {
+              cdnBlockedCount++;
+            }
             v.locationCreated = null;
             locationMap[v.id] = null;
             done++;
@@ -295,7 +306,7 @@ export async function enrichVideosWithLocation(videos, options = {}) {
       await Promise.allSettled(tasks);
     }
 
-    return { videos: enriched, locationMap };
+    return { videos: enriched, locationMap, cdnBlockedCount };
   } finally {
     if (ownsScraper) await scraper.close();
   }

package/src/lib/tiktok-scraper.mjs

@@ -267,6 +267,14 @@ export class TikTokScraper {
     const slot = this._pickSlot();
     return slot.lock.run(async () => {
       let rawHtml = await this._fetchViewSource(videoUrl, slot);
+      // CDN 限流立即抛出，不重试
+      if (detectAccessDenied(rawHtml)) {
+        const denied = detectAccessDenied(rawHtml);
+        throw new CDNBlockedError(
+          `CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
+          denied.reference,
+        );
+      }
       let result = parseVideoInfo(rawHtml);
       for (let attempt = 1; !result && attempt <= maxRetries; attempt++) {
         // 检查是否值得重试
@@ -278,6 +286,14 @@ export class TikTokScraper {
         } catch {}
         await delay(500 * attempt);
         rawHtml = await this._fetchViewSource(videoUrl, slot);
+        // 重试中也检查 CDN 限流
+        if (detectAccessDenied(rawHtml)) {
+          const denied = detectAccessDenied(rawHtml);
+          throw new CDNBlockedError(
+            `CDN限流 (Access Denied, ref:${denied.reference || "N/A"})`,
+            denied.reference,
+          );
+        }
         result = parseVideoInfo(rawHtml);
       }
       return result || null;

package/src/scraper/explore-core.js

@@ -87,11 +87,14 @@ async function processExplore(page, username, options, log) {
       if (result.userInfo) result.userInfo.latestVideoTime = latestCreateTime;
     }
 
-    // 找出 7 天内发布且播放量最大的视频
+    // 找出 7 天内发布且 isECVideo=1 且播放量最大的视频
     const SEVEN_DAYS_SECONDS = 7 * 24 * 60 * 60;
     const nowSeconds = Math.floor(Date.now() / 1000);
     const recentVideos = videoArray.filter(
-      (v) => v.createTime && nowSeconds - v.createTime <= SEVEN_DAYS_SECONDS,
+      (v) =>
+        v.isECVideo === 1 &&
+        v.createTime &&
+        nowSeconds - v.createTime <= SEVEN_DAYS_SECONDS,
     );
     if (recentVideos.length > 0) {
       const topVideo = recentVideos.reduce((max, v) =>
@@ -104,7 +107,7 @@ async function processExplore(page, username, options, log) {
         createTime: topVideo.createTime,
       };
       log(
-        `  7天内最高播放视频: ${topVideo.playCount} 次播放 (${recentVideos.length} 个候选)`,
+        `  7天内 EC视频最高播放: ${topVideo.playCount} 次播放 (${recentVideos.length} 个EC候选)`,
       );
     }
 

package/src/watch/data-store.js

@@ -90,7 +90,9 @@ import {
   getDeadTags,
   claimTag,
   reportTagScore,
+  resetStaleScoringTags,
   getAllTags,
+  getTagStats,
   rawQuery,
   normalizeTags,
   clearTags,
@@ -1415,6 +1417,7 @@ export function createStore(filePath, options = {}) {
               `(
                 instr(COALESCE(sources, ''), '"following"') > 0
                 OR instr(COALESCE(sources, ''), '"follower"') > 0
+                OR instr(COALESCE(sources, ''), '"comment"') > 0
               )`,
             ],
           });
@@ -1639,7 +1642,8 @@ export function createStore(filePath, options = {}) {
             (u) =>
               u.sources &&
               (u.sources.includes("following") ||
-                u.sources.includes("follower")),
+                u.sources.includes("follower") ||
+                u.sources.includes("comment")),
           );
           follow.sort((a, b) => locationTier(a) - locationTier(b));
           next = follow[0] || null;
@@ -2674,6 +2678,30 @@ export function createStore(filePath, options = {}) {
     return { ok: true, location, modifiedAt: user.modifiedAt };
   }
 
+  function setNonSeller(uniqueId) {
+    if (getDb()) {
+      const existing = getDb()
+        .prepare("SELECT * FROM jobs WHERE unique_id = ?")
+        .get(uniqueId);
+      if (!existing) return { error: "user not found" };
+      const now = Date.now();
+      getDb()
+        .prepare(
+          "UPDATE jobs SET tt_seller = 0, updated_at = ? WHERE unique_id = ?",
+        )
+        .run(now, uniqueId);
+      console.error(`[DB] setNonSeller: ${uniqueId} → tt_seller=0`);
+      return { ok: true };
+    }
+
+    const user = getUser(uniqueId);
+    if (!user) return { error: "user not found" };
+    user.ttSeller = false;
+    user.updatedAt = Date.now();
+    save();
+    return { ok: true };
+  }
+
   // 将单个 job 移动到 raw_jobs 表（完整字段复制 + 删除原记录）
   function moveJobToRaw(uniqueId) {
     if (!getDb()) return false;
@@ -3127,6 +3155,7 @@ export function createStore(filePath, options = {}) {
     getPendingUserUpdateTasks,
     updateUserInfo,
     updateUserLocation,
+    setNonSeller,
     batchUpdateUserInfo,
     reportClientError,
     deleteClientError,
@@ -3154,7 +3183,9 @@ export function createStore(filePath, options = {}) {
     getDeadTags,
     claimTag,
     reportTagScore,
+    resetStaleScoringTags,
     getAllTags,
+    getTagStats,
     normalizeTags,
     clearTags,
     data,

package/src/watch/db-stats.js

@@ -93,8 +93,11 @@ export function getDashboardStatsFromDb(targetLocations = []) {
     .prepare("SELECT COUNT(*) as total FROM jobs_base")
     .get().total;
 
+  const tagCount = db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
+
   return {
     totalUsers: aggregateRow.total,
+    tagCount,
     rawJobs: getRawJobsCount(),
     dbTotalUsers: getUserDbCount(),
     jobsTotal: aggregateRow.total,

package/src/watch/db-tags.js

@@ -33,17 +33,61 @@ export function insertTag(tag, countries, source = "llm") {
   }
 }
 
-export function getTagsByStatus(status, limit = 100) {
+export function getTagsByStatus(
+  status,
+  limit = 100,
+  offset = 0,
+  country = null,
+) {
   const db = getDb();
   if (!db) return [];
-  const rows = db
-    .prepare(
-      "SELECT * FROM tags WHERE status = ? ORDER BY score ASC, created_at ASC LIMIT ?",
-    )
-    .all(status, limit);
+  let sql = "SELECT * FROM tags WHERE status = ?";
+  const params = [status];
+  if (country) {
+    sql += " AND countries LIKE ?";
+    params.push(`%"${country}"%`);
+  }
+  sql += " ORDER BY score ASC, created_at ASC LIMIT ? OFFSET ?";
+  params.push(limit, offset);
+  const rows = db.prepare(sql).all(...params);
   return rows.map(parseTagRow);
 }
 
+export function getTagStats(country = null) {
+  const db = getDb();
+  if (!db) return null;
+  let sql = `SELECT
+    COUNT(*) as total,
+    SUM(CASE WHEN status = 'productive' THEN 1 ELSE 0 END) as productive,
+    SUM(CASE WHEN status = 'dead' THEN 1 ELSE 0 END) as dead,
+    SUM(CASE WHEN status = 'new' THEN 1 ELSE 0 END) as newCount,
+    SUM(CASE WHEN status = 'scoring' THEN 1 ELSE 0 END) as scoring
+  FROM tags`;
+  const params = [];
+  if (country) {
+    sql += " WHERE countries LIKE ?";
+    params.push(`%"${country}"%`);
+  }
+  const row = db.prepare(sql).get(...params);
+  // 获取所有出现过的国家
+  const allRows = db.prepare("SELECT countries FROM tags").all();
+  const countrySet = new Set();
+  for (const r of allRows) {
+    try {
+      const arr = JSON.parse(r.countries || "[]");
+      for (const c of arr) countrySet.add(c);
+    } catch {}
+  }
+  return {
+    total: row.total,
+    productive: row.productive || 0,
+    dead: row.dead || 0,
+    new: row.newCount || 0,
+    scoring: row.scoring || 0,
+    countries: [...countrySet].sort(),
+  };
+}
+
 export function getTagsByCountry(country, minScore = 0) {
   const db = getDb();
   if (!db) return [];
@@ -64,12 +108,33 @@ export function getDeadTags(country) {
   return rows.map(parseTagRow).filter((r) => r.countries.includes(country));
 }
 
+export function resetStaleScoringTags(minutes = 30) {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+  // 清理超时的 scoring 标签：有时间戳的按时间，没时间戳的（旧数据）直接清
+  const result = db
+    .prepare(
+      "UPDATE tags SET status = 'new', scored_at = NULL WHERE status = 'scoring' AND (scored_at IS NULL OR scored_at < datetime('now', ?))",
+    )
+    .run(`-${minutes} minutes`);
+  if (result.changes > 0) {
+    console.error(
+      `[tags] 清理了 ${result.changes} 个超时 scoring 标签（>${minutes}分钟）`,
+    );
+  }
+  return { ok: true, reset: result.changes };
+}
+
 export function claimTag(tag) {
   const db = getDb();
   if (!db) return { ok: false, error: "db not ready" };
+
+  // 先清理超时的 scoring 标签，防止死任务堆积
+  resetStaleScoringTags();
+
   const result = db
     .prepare(
-      "UPDATE tags SET status = 'scoring' WHERE tag = ? AND status = 'new'",
+      "UPDATE tags SET status = 'scoring', scored_at = datetime('now') WHERE tag = ? AND status = 'new'",
     )
     .run(tag);
   if (result.changes === 0) {
@@ -132,12 +197,18 @@ export function reportTagScore(tag, fields) {
   }
 }
 
-export function getAllTags(limit = 200) {
+export function getAllTags(limit = 200, offset = 0, country = null) {
   const db = getDb();
   if (!db) return [];
-  const rows = db
-    .prepare("SELECT * FROM tags ORDER BY score DESC, created_at DESC LIMIT ?")
-    .all(limit);
+  let sql = "SELECT * FROM tags";
+  const params = [];
+  if (country) {
+    sql += " WHERE countries LIKE ?";
+    params.push(`%"${country}"%`);
+  }
+  sql += " ORDER BY score DESC, created_at DESC LIMIT ? OFFSET ?";
+  params.push(limit, offset);
+  const rows = db.prepare(sql).all(...params);
   return rows.map(parseTagRow);
 }