tt-help-cli-ycl 1.3.80 → 1.3.81

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.80",
+  "version": "1.3.81",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/lib/api-interceptor.js

@@ -32,7 +32,7 @@ async function processAPIResponse(
 
     try {
       const pageData = await (() => {
-        // 重试包装：处理页面导航导致的执行上下文销毁
+        // 重试包装：处理页面导航导致的执行上下文销毁和 CDP 断连
         const tryEval = async (retries = 3) => {
           for (let i = 0; i < retries; i++) {
             try {
@@ -42,8 +42,11 @@ async function processAPIResponse(
               }, newUrl);
             } catch (e) {
               if (
-                e.message.includes("Execution context was destroyed") &&
-                i < retries - 1
+                e.message &&
+                (e.message.includes("Execution context was destroyed") ||
+                  e.message.includes("Target closed") ||
+                  e.message.includes("Connection closed") ||
+                  e.message.includes("Protocol error"))
               ) {
                 await delay(500 * (i + 1), 500 * (i + 1));
               } else {
@@ -89,6 +92,14 @@ async function processAPIResponse(
 async function fetchUserVideosAPI(page, username, maxVideos, log) {
   log(`  [API拦截] 获取 @${username} 视频 ...`);
 
+  // CDP 健康检查：确保 page 可用
+  try {
+    await page.evaluate(() => 1);
+  } catch (e) {
+    log(`  [API拦截] CDP 连接异常: ${e.message}`);
+    throw new Error(`CDP 连接异常: ${e.message}`);
+  }
+
   let apiRequestUrl = null;
   let sawApiRequest = false;
 

package/src/lib/page-error-detector.js

@@ -54,23 +54,40 @@ const PATTERNS = {
   service_error: ["出错了", "很抱歉"],
 };
 
-export async function detectPageError(page) {
-  return page.evaluate((patterns) => {
-    const body = document.body;
-    if (!body) return null;
-    const bodyText = body.innerText;
-    const lower = bodyText.toLowerCase();
+export async function detectPageError(page, timeout = 10000) {
+  try {
+    return await page.evaluate(
+      (patterns) => {
+        const body = document.body;
+        if (!body) return null;
+        const bodyText = body.innerText;
+        const lower = bodyText.toLowerCase();
 
-    for (const [type, phrases] of Object.entries(patterns)) {
-      for (const phrase of phrases) {
-        if (lower.includes(phrase.toLowerCase())) {
-          return type;
+        for (const [type, phrases] of Object.entries(patterns)) {
+          for (const phrase of phrases) {
+            if (lower.includes(phrase.toLowerCase())) {
+              return type;
+            }
+          }
         }
-      }
-    }
 
-    return null;
-  }, PATTERNS);
+        return null;
+      },
+      PATTERNS,
+    );
+  } catch (e) {
+    // CDP 断连或超时：返回 null 而非永久挂起
+    if (
+      e.message &&
+      (e.message.includes("Timeout") ||
+        e.message.includes("Target closed") ||
+        e.message.includes("Connection closed") ||
+        e.message.includes("Protocol error"))
+    ) {
+      return null;
+    }
+    throw e;
+  }
 }
 
 /**

package/src/lib/scroll-collector.js

@@ -30,7 +30,7 @@ async function doCollect(
       }
 
       const fn = eval("(" + fnStr + ")");
-      return fn(el, args);
+      return args !== undefined ? fn(el, args) : fn(el);
     },
     { fn: fnStr, containerSelector: container, findScrollableFlag: findScrollable, args: extraArgs },
   );

package/src/videos/core.js

@@ -7,14 +7,18 @@ import {
 import { fetchUserVideosAPI } from "../lib/api-interceptor.js";
 
 async function getUserInfo(page) {
-  // 重试包装：处理页面导航导致的执行上下文销毁
+  // 重试包装：处理页面导航导致的执行上下文销毁和 CDP 断连
   const evaluateWithRetry = async (fn, retries = 3) => {
     for (let i = 0; i < retries; i++) {
       try {
         return await page.evaluate(fn);
       } catch (e) {
         if (
-          e.message.includes("Execution context was destroyed") &&
+          e.message &&
+          (e.message.includes("Execution context was destroyed") ||
+            e.message.includes("Target closed") ||
+            e.message.includes("Connection closed") ||
+            e.message.includes("Protocol error")) &&
           i < retries - 1
         ) {
           await new Promise((r) => setTimeout(r, 500 * (i + 1)));

package/src/watch/data-store.js

@@ -650,13 +650,24 @@ function getDashboardStatsFromDb(targetLocations = []) {
             AND instr(COALESCE(sources, ''), '"guess"') = 0
             AND instr(COALESCE(sources, ''), '"following"') = 0
             AND instr(COALESCE(sources, ''), '"follower"') = 0
-          THEN 1 ELSE 0 END) as seed,
-        SUM(CASE WHEN COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0 THEN 1 ELSE 0 END) as userUpdateTasks
+          THEN 1 ELSE 0 END) as seed
       FROM jobs
     `,
     )
     .get(...targetParams);
 
+  // userUpdateTasks 单独从 jobs_base 统计
+  const userUpdateTasksRow = db
+    .prepare(
+      `
+      SELECT COUNT(*) as userUpdateTasks
+      FROM jobs_base
+      WHERE COALESCE(tt_seller, '') = ''
+        AND COALESCE(user_update_count, 0) <= 0
+    `,
+    )
+    .get();
+
   // countryStats 和 targetCountryStats 需要 GROUP BY，保留为独立查询
   const countryStats = db
     .prepare(
@@ -712,7 +723,7 @@ function getDashboardStatsFromDb(targetLocations = []) {
     restrictedUsers: aggregateRow.restricted,
     errorUsers: aggregateRow.error,
     targetUsers: aggregateRow.targetUsers,
-    userUpdateTasks: aggregateRow.userUpdateTasks,
+    userUpdateTasks: userUpdateTasksRow.userUpdateTasks,
     targetCountryStats,
     countryStats,
     sourceStats: {
@@ -761,7 +772,7 @@ function getUserUpdateByCountryFromDb() {
       SELECT
         COALESCE(guessed_location, '未知') as country,
         COUNT(*) as count
-      FROM jobs
+      FROM jobs_base
       WHERE COALESCE(tt_seller, '') = ''
         AND COALESCE(user_update_count, 0) <= 0
       GROUP BY COALESCE(guessed_location, '未知')
@@ -782,7 +793,7 @@ function getAttachStuckByCountryFromDb() {
       SELECT
         COALESCE(guessed_location, '未知') as country,
         COUNT(*) as count
-      FROM jobs
+      FROM jobs_base
       WHERE COALESCE(tt_seller, '') = ''
         AND COALESCE(user_update_count, 0) = 1
       GROUP BY COALESCE(guessed_location, '未知')
@@ -816,7 +827,7 @@ function restoreAttachStuckByCountry(country) {
       .prepare(
         `
         SELECT COUNT(*) as c
-        FROM jobs
+        FROM jobs_base
         WHERE ${whereSql}
       `,
       )
@@ -828,7 +839,7 @@ function restoreAttachStuckByCountry(country) {
 
   db.prepare(
     `
-    UPDATE jobs
+    UPDATE jobs_base
     SET user_update_count = 0,
         updated_at = ?,
         claimed_by = NULL,
@@ -943,7 +954,7 @@ function moveJobsToRawByCountry(scope, country) {
       .prepare(
         `
         SELECT COUNT(*) as c
-        FROM jobs
+        FROM jobs_base
         WHERE ${whereSql}
       `,
       )
@@ -1017,14 +1028,14 @@ function moveJobsToRawByCountry(scope, country) {
         signature,
         sec_uid,
         latest_video_time
-      FROM jobs
+      FROM jobs_base
       WHERE ${whereSql}
     `,
     ).run(targetCountry);
 
     db.prepare(
       `
-      DELETE FROM jobs
+      DELETE FROM jobs_base
       WHERE ${whereSql}
     `,
     ).run(targetCountry);
@@ -1718,6 +1729,13 @@ function getJobRow(uniqueId) {
   return db.prepare("SELECT * FROM jobs WHERE unique_id = ?").get(uniqueId);
 }
 
+function getJobBaseRow(uniqueId) {
+  if (!db) return null;
+  return db
+    .prepare("SELECT * FROM jobs_base WHERE unique_id = ?")
+    .get(uniqueId);
+}
+
 function getJob(uniqueId) {
   return mapJobRow(getJobRow(uniqueId));
 }
@@ -1795,6 +1813,43 @@ function inferStatus(u) {
   return "pending";
 }
 
+function updateJobBaseInfo(uniqueId, info, incrementCount = true) {
+  if (!db) return { error: "db not initialized" };
+  const existing = getJobBaseRow(uniqueId);
+  if (!existing) return { error: "user not found" };
+
+  const nextValues = {};
+  for (const [key, value] of Object.entries(info || {})) {
+    if (key === "uniqueId" || key === "unique_id") continue;
+    if (value === undefined || value === "") continue;
+    let column = camelToSnake(key);
+    // 字段别名：bio → signature
+    if (column === "bio") column = "signature";
+    if (!writableJobColumns.has(column)) continue;
+    nextValues[column] = normalizeJobValue(column, value);
+  }
+
+  nextValues.updated_at = Date.now();
+  if (incrementCount) {
+    nextValues.user_update_count = (existing.user_update_count || 0) + 1;
+  }
+
+  const columns = Object.keys(nextValues);
+  if (columns.length > 0) {
+    const sql = `UPDATE jobs_base SET ${columns.map((column) => `${column} = ?`).join(", ")} WHERE unique_id = ?`;
+    db.prepare(sql).run(
+      ...columns.map((column) => nextValues[column]),
+      uniqueId,
+    );
+  }
+
+  return {
+    ok: true,
+    userUpdateCount:
+      nextValues.user_update_count ?? existing.user_update_count ?? 0,
+  };
+}
+
 function addJobBaseToDb(user) {
   if (!db) return;
   const now = Date.now();
@@ -3375,7 +3430,7 @@ export function createStore(filePath) {
 
       let sql = `
         SELECT *
-        FROM jobs
+        FROM jobs_base
         WHERE COALESCE(tt_seller, '') = ''
           AND COALESCE(user_update_count, 0) <= 0
       `;
@@ -3395,7 +3450,7 @@ export function createStore(filePath) {
       const now = Date.now();
       const bumpStmt = db.prepare(
         `
-        UPDATE jobs
+        UPDATE jobs_base
         SET user_update_count = COALESCE(user_update_count, 0) + 1,
             updated_at = ?
         WHERE unique_id = ?
@@ -3526,7 +3581,8 @@ export function createStore(filePath) {
   function batchUpdateUserInfo(updates) {
     if (db) {
       const results = [];
-      const moveList = [];
+      const rawMoveList = [];
+      const sellerMoveList = [];
 
       const txn = db.transaction((items) => {
         items.forEach((item) => {
@@ -3536,13 +3592,13 @@ export function createStore(filePath) {
           let updateResult;
           if (info && info.error && info.statusCode !== undefined) {
             // 只更新 status_code，不更新其他字段
-            updateResult = updateJobInfo(
+            updateResult = updateJobBaseInfo(
               uniqueId,
               { statusCode: info.statusCode },
               true,
             );
           } else {
-            updateResult = updateJobInfo(uniqueId, info, true);
+            updateResult = updateJobBaseInfo(uniqueId, info, true);
           }
 
           if (updateResult.error) {
@@ -3550,34 +3606,66 @@ export function createStore(filePath) {
             return;
           }
 
-          // 检查 tt_seller：非商家则标记为需要移动到毛料表
-          const row = getJobRow(uniqueId);
+          // 检查 tt_seller：商家移到 jobs，非商家移到 raw_jobs
+          const row = getJobBaseRow(uniqueId);
           const ttSeller = row ? row.tt_seller : null;
           if (ttSeller) {
-            // 商家：保持当前逻辑
+            // 商家：标记移动到 jobs
             results.push({
               uniqueId,
               ok: true,
               userUpdateCount: updateResult.userUpdateCount,
+              _movedToJobs: true,
             });
+            sellerMoveList.push(uniqueId);
           } else {
-            // 非商家：标记移动
+            // 非商家：标记移动到 raw_jobs
             results.push({
               uniqueId,
               ok: true,
               userUpdateCount: updateResult.userUpdateCount,
               _movedToRaw: true,
             });
-            moveList.push(uniqueId);
+            rawMoveList.push(uniqueId);
           }
         });
       });
       txn(updates);
 
-      // 批量移动非商家用户到 raw_jobs（优化：一次 SQL 搞定）
-      if (moveList.length > 0) {
-        const placeholders = moveList.map(() => "?").join(",");
-        // 批量 INSERT 到 raw_jobs
+      // 批量移动商家用户到 jobs
+      if (sellerMoveList.length > 0) {
+        const placeholders = sellerMoveList.map(() => "?").join(",");
+        db.prepare(
+          `
+          INSERT OR REPLACE INTO jobs (
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          )
+          SELECT
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          FROM jobs_base WHERE unique_id IN (${placeholders})
+        `,
+        ).run(...sellerMoveList);
+
+        db.prepare(
+          `DELETE FROM jobs_base WHERE unique_id IN (${placeholders})`,
+        ).run(...sellerMoveList);
+      }
+
+      // 批量移动非商家用户到 raw_jobs
+      if (rawMoveList.length > 0) {
+        const placeholders = rawMoveList.map(() => "?").join(",");
         db.prepare(
           `
           INSERT OR REPLACE INTO raw_jobs (
@@ -3597,19 +3685,18 @@ export function createStore(filePath) {
             follower_count, following_count, heart_count, refresh_time,
             processed, processed_at, created_at, updated_at,
             region, signature, bio_link, sec_uid, status_code, latest_video_time
-          FROM jobs WHERE unique_id IN (${placeholders})
+          FROM jobs_base WHERE unique_id IN (${placeholders})
         `,
-        ).run(...moveList);
+        ).run(...rawMoveList);
 
-        // 批量 DELETE 从 jobs
-        db.prepare(`DELETE FROM jobs WHERE unique_id IN (${placeholders})`).run(
-          ...moveList,
-        );
+        db.prepare(
+          `DELETE FROM jobs_base WHERE unique_id IN (${placeholders})`,
+        ).run(...rawMoveList);
       }
 
       // 清理内部标记
       return results.map((r) => {
-        const { _movedToRaw, ...rest } = r;
+        const { _movedToRaw, _movedToJobs, ...rest } = r;
         return rest;
       });
     }