tt-help-cli-ycl 1.4.16 → 1.4.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.16",
+  "version": "1.4.17",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/refresh.js

@@ -54,6 +54,7 @@ export async function handleRefresh(options) {
     exploreMaxFollowing,
     exploreMaxFollowers,
     exploreLocation,
+    exploreJobLocations,
     exploreMaxUsers,
     serverUrl,
     explorePort,
@@ -106,6 +107,7 @@ export async function handleRefresh(options) {
       `关注/粉丝: ${exploreEnableFollow ? "启用" : "禁用"} (${exploreMaxFollowing}/${exploreMaxFollowers})`,
     );
     console.error(`国家筛选: ${exploreLocation}`);
+    if (exploreJobLocations) console.error(`任务国家: ${exploreJobLocations}`);
     console.error(`空闲间隔: ${exploreInterval || 30} 秒`);
     if (effectiveProxy) {
       console.error(
@@ -274,6 +276,9 @@ export async function handleRefresh(options) {
       if (exploreRedoMaxAge > 0) {
         redoUrl += `&maxAge=${exploreRedoMaxAge}`;
       }
+      if (exploreJobLocations) {
+        redoUrl += `&locations=${encodeURIComponent(exploreJobLocations)}`;
+      }
       let job = null;
       try {
         job = await apiGet(redoUrl);

package/src/cli/tag.js

@@ -528,13 +528,11 @@ async function scoreSingleTag(
         log(`  ✗ 标签不存在（TikTok 提示"找不到此话题标签"），标记为 dead`);
         result.status = "dead";
         result.error = "tag_not_found";
-      } else {
-        log(`  ✗ 没有视频数据，标记为 dead（可能是网络问题或页面未加载）`);
-        result.status = "dead";
-        result.error = "no videos found";
+        await reportToServer(baseUrl, result);
+        return result;
       }
-      await reportToServer(baseUrl, result);
-      return result;
+      // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径，避免误标 dead
+      throw new Error("no videos returned — possible network/captcha/slow-load, will retry");
     }
 
     // Step 2: 补充国家信息
@@ -1064,18 +1062,24 @@ export async function handleScoreAll(parsed) {
 
         if (!videos || videos.length === 0) {
           const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
+          if (tagResult.error === "tag_not_found") {
+            log(
+              `  ⚠️ 标签不存在 (${deadSec}s)，标记 dead`,
+            );
+            result.status = "dead";
+            result.error = "tag_not_found";
+            await reportToServer(baseUrl, result, clientId, clientMeta);
+            totalScored++;
+            await randomDelay(0, 5000);
+            await recyclePage();
+            await maybeRecycleForMemory();
+            continue;
+          }
+          // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径，避免误标 dead
           log(
-            `  ⚠️ 无视频 (${deadSec}s) mem=${formatMemoryWithBrowser(cdpPort)}，标记 dead`,
+            `  ⚠️ 无视频 (${deadSec}s)，可能网络/风控/加载慢 → 抛异常重试`,
           );
-          result.status = "dead";
-          result.error = "no videos found";
-          await reportToServer(baseUrl, result, clientId, clientMeta);
-          totalScored++;
-          // 随机等待 3-7 秒，避免连续访问 TikTok 触发风控
-          await randomDelay(0, 5000);
-          await recyclePage();
-          await maybeRecycleForMemory();
-          continue;
+          throw new Error("no videos returned (network/captcha/slow-load)");
         }
 
         // enrich: 先查服务端缓存，只对未知视频发 HTTP 请求

package/src/lib/args.js

@@ -471,6 +471,7 @@ function parseRefreshArgs(args) {
   let exploreMaxVideos = 16;
   let exploreRedoMaxAge = 0;
   let exploreProxy = null;
+  let exploreJobLocations = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -502,6 +503,8 @@ function parseRefreshArgs(args) {
       exploreProxy = args[++i];
     } else if (arg === "--location") {
       exploreLocation = args[++i];
+    } else if (arg === "--job-locations") {
+      exploreJobLocations = args[++i];
     } else if (arg === "--enable-follow") {
       exploreEnableFollow = true;
     } else if (arg === "--disable-follow") {
@@ -519,6 +522,7 @@ function parseRefreshArgs(args) {
     exploreMaxFollowing,
     exploreMaxFollowers,
     exploreLocation,
+    exploreJobLocations,
     exploreMaxUsers,
     explorePort,
     exploreBasePort,

package/src/lib/constants.js

@@ -145,6 +145,7 @@ const HELP_TEXT = [
   "    选项:",
   "      --server <URL>            服务端地址，默认 http://127.0.0.1:3001",
   `      --location <国家代码>     国家筛选，逗号分隔，默认 ${DEFAULT_TARGET_LOCATIONS_CSV}`,
+  "      --job-locations <国家>    任务国家筛选，逗号分隔（仅筛选服务端任务），基于真实国家字段 location_created",
   "      --max-videos <数量>       每用户最大视频数，默认 16",
   "      --enable-follow           启用关注/粉丝提取（默认启用）",
   "      --disable-follow          禁用关注/粉丝提取",

package/src/lib/tag-fetcher.js

@@ -6,7 +6,7 @@ import { CDNBlockedError } from "./parse-ssr.mjs";
 
 const TAG_URL = "https://www.tiktok.com/tag";
 const SCROLL_INTERVAL = 3000;
-const MAX_STALE_ROUNDS = 3;
+const MAX_STALE_ROUNDS = 5;
 
 function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));
@@ -113,6 +113,12 @@ export async function fetchTagData(tag, options = {}) {
 
     await page.waitForTimeout(3000);
 
+    // 等待首屏 API 数据到达（避免网络慢时 0 视频进入滚动循环）
+    const videoWaitStart = Date.now();
+    while (seenVideoIds.size === 0 && Date.now() - videoWaitStart < 15000) {
+      await sleep(1000);
+    }
+
     // 检测页面异常：验证码、网络错误、标签不存在
     const pageError = await page.evaluate(() => {
       const text = document.body?.innerText || "";

package/src/watch/data-store.js

@@ -380,6 +380,9 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
 
   // 分页模式：按国家或全局分页查询用户
   if (limit !== undefined) {
+    const filterIsTarget =
+      filterCountry && targetLocations.includes(filterCountry);
+
     let sql = `
       SELECT
         unique_id,
@@ -401,13 +404,22 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
       FROM jobs
       WHERE tt_seller = 1
         AND verified = 0
-        AND location_created IN (${placeholders})
     `;
-    const params = [...targetLocations];
+    const params = [];
+
+    // 目标国家筛选：仅当 filterCountry 是目标国家或不筛选时才加 IN 限制
+    if (filterIsTarget || !filterCountry) {
+      sql += ` AND location_created IN (${placeholders})`;
+      params.push(...targetLocations);
+    }
 
     if (filterCountry) {
       sql += ` AND location_created = ?`;
       params.push(filterCountry);
+      // 筛选非目标国家时，仅返回已修改的用户（与“其他国家”语义一致）
+      if (!filterIsTarget) {
+        sql += ` AND modified_at IS NOT NULL`;
+      }
     }
 
     if (search) {
@@ -2476,12 +2488,12 @@ export function createStore(filePath, options = {}) {
     return { saved: true, pinned: user.pinned };
   }
 
-  function getNextRedoJob(userId, maxAgeSeconds = 43200) {
+  function getNextRedoJob(userId, maxAgeSeconds = 43200, locations = null) {
     if (getDb()) {
       const now = Date.now();
       const threshold = now - maxAgeSeconds * 1000;
       const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
-      const targetLocations = DEFAULT_TARGET_LOCATIONS;
+      const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
       const placeholders = targetLocations.map(() => "?").join(",");
       const row = getDb()
         .prepare(
@@ -2515,7 +2527,7 @@ export function createStore(filePath, options = {}) {
     const defaultTime = new Date("2016-01-01T00:00:00Z").getTime();
 
     // 筛选目标国家用户，按 refreshTime 升序取最远的（没有则默认 2016-01-01）
-    const targetLocations = DEFAULT_TARGET_LOCATIONS;
+    const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
     const targetUsers = data.filter(
       (u) =>
         u.ttSeller &&
@@ -2699,13 +2711,15 @@ export function createStore(filePath, options = {}) {
 
     if (getDb()) {
       const l = Math.max(1, parseInt(limit) || 5);
+      const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
 
       let sql = `
         SELECT *
         FROM jobs_base
         WHERE COALESCE(user_update_count, 0) <= 0
+           OR (COALESCE(user_update_count, 0) > 0 AND updated_at < ?)
       `;
-      const sqlParams = [];
+      const sqlParams = [fiveMinutesAgo];
 
       if (hasCountryFilter) {
         const placeholders = targetCountries.map(() => "?").join(", ");
@@ -2750,14 +2764,17 @@ export function createStore(filePath, options = {}) {
     }
 
     const l = Math.max(1, parseInt(limit) || 5);
+    const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
     const pending = data
       .filter((u) => {
         const updateCount = u.userUpdateCount;
-        // jobs_base 中所有 user_update_count <= 0 的记录都可以领取
+        // user_update_count <= 0 的记录可直接领取
+        // user_update_count > 0 但 updatedAt 超过 5 分钟的，视为客户端失败/超时，重新回收
         if (
           updateCount === null ||
           updateCount === undefined ||
-          updateCount <= 0
+          updateCount <= 0 ||
+          (u.updatedAt && u.updatedAt < fiveMinutesAgo)
         ) {
           if (hasCountryFilter) {
             const loc = (u.guessedLocation || "").toUpperCase();

package/src/watch/db-stats.js

@@ -95,9 +95,17 @@ export function getDashboardStatsFromDb(targetLocations = []) {
 
   const tagCount = db.prepare("SELECT COUNT(*) as total FROM tags").get().total;
 
+  const commentTaskCount = db
+    .prepare(
+      `SELECT COUNT(*) as total FROM videos
+       WHERE user_update_count IS NULL OR user_update_count <= 0`,
+    )
+    .get().total;
+
   return {
     totalUsers: aggregateRow.total,
     tagCount,
+    commentTaskCount,
     rawJobs: getRawJobsCount(),
     dbTotalUsers: getUserDbCount(),
     jobsTotal: aggregateRow.total,

package/src/watch/public/app.js

@@ -166,6 +166,7 @@ function renderStats() {
   flashEl("statTarget", d.targetUsers, { full: true });
   flashEl("statUserUpdateTasks", d.userUpdateTasks || 0, { full: true });
   flashEl("statRawJobs", d.rawJobs || 0);
+  flashEl("statCommentTask", d.commentTaskCount || 0, { full: true });
   flashEl("statTags", d.tagCount || 0, { full: true });
   // 同步子页面 stats
   const pendingTotal = document.getElementById("pendingStatTotal");

package/src/watch/public/index.html

@@ -53,6 +53,10 @@
         <div class="label">目标商家</div>
         <div class="value target" id="statTarget">0</div>
       </div>
+      <div class="stat-card clickable pending-card" id="statCommentTaskCard">
+        <div class="label">待评论任务</div>
+        <div class="value target" id="statCommentTask">0</div>
+      </div>
       <div class="stat-card clickable pending-card" id="statTagsCard" onclick="navigateToTags()" style="cursor:pointer">
         <div class="label">关键词</div>
         <div class="value target" id="statTags">0</div>

package/src/watch/server.js

@@ -551,12 +551,27 @@ export function startWatchServer(
       if (req.method === "GET" && routePath === "/api/redo-job") {
         const userId = params.userId || "";
         const maxAge = parseInt(params.maxAge) || 43200;
-        const job = store.getNextRedoJob(userId, maxAge);
+        const locationsParam = params.locations || "";
+        const locations = locationsParam
+          ? locationsParam
+              .split(",")
+              .map((s) => s.trim().toUpperCase())
+              .filter(Boolean)
+          : null;
+        const job = store.getNextRedoJob(userId, maxAge, locations);
         if (job) {
-          logJob("REDO-CLAIM", { user: job.uniqueId, clientId: userId });
+          logJob("REDO-CLAIM", {
+            user: job.uniqueId,
+            clientId: userId,
+            locations,
+          });
           sendJSON(res, 200, { hasJob: true, user: job });
         } else {
-          logJob("REDO-CLAIM", { result: "no-job", clientId: userId });
+          logJob("REDO-CLAIM", {
+            result: "no-job",
+            clientId: userId,
+            locations,
+          });
           sendJSON(res, 200, { hasJob: false });
         }
         return;