tt-help-cli-ycl 1.4.10 → 1.4.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.10",
+  "version": "1.4.12",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/comments.js

@@ -5,7 +5,9 @@
 //   手动模式: tt-help comments <URL> [N] [--save]            单视频评论提取
 // ────────────────────────────────────────
 
+import { writeFileSync, unlinkSync } from "fs";
 import os from "os";
+import path from "path";
 import { execSync } from "child_process";
 
 import { fetchUserCommentsAPI } from "../lib/api-interceptor-comment.js";
@@ -65,6 +67,10 @@ const CFG = {
   periodicRecycleEvery: 0,
   /** 内存回收 — 回收时是否切换端口 */
   switchPortOnRecycle: true,
+  /** 连续任务处理错误阈值（达到后上报服务端） */
+  consecutiveTaskErrorsMax: 20,
+  /** 单端口最大运行时间(ms)，超时自动切换端口（默认 48 分钟） */
+  maxSessionRunTimeMs: 48 * 60 * 1000,
 };
 
 /** 从环境变量覆写配置 */
@@ -97,6 +103,14 @@ function loadEnvConfig() {
   );
   CFG.switchPortOnRecycle =
     String(process.env.COMMENTS_SWITCH_PORT_ON_RECYCLE || "1") !== "0";
+  CFG.consecutiveTaskErrorsMax = Math.max(
+    1,
+    int("COMMENTS_CONSECUTIVE_TASK_ERRORS_MAX", CFG.consecutiveTaskErrorsMax),
+  );
+  CFG.maxSessionRunTimeMs = Math.max(
+    60000,
+    int("COMMENTS_MAX_SESSION_RUN_TIME_MS", CFG.maxSessionRunTimeMs),
+  );
 }
 
 // ────────────────────────────────────────
@@ -108,32 +122,82 @@ function formatMemoryUsage(mem = process.memoryUsage()) {
   return `rss:${mb(mem.rss)}MB heap:${mb(mem.heapUsed)}MB ext:${mb(mem.external)}MB ab:${mb(mem.arrayBuffers)}MB`;
 }
 
-function getBrowserMemory(port) {
+/**
+ * 获取 Edge 浏览器进程的内存占用（跨平台：macOS/Linux/Windows）
+ */
+function getBrowserMemory(port = null) {
   try {
     const platform = os.platform();
-    if (platform !== "darwin" && platform !== "linux") return null;
-    const procName = platform === "darwin" ? "Microsoft Edge" : "msedge";
-    const c = procName[0];
-    const rest = procName.slice(1);
     let cmd;
-    if (port) {
-      cmd = `ps aux | grep "[${c}]${rest}" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
-    } else {
-      cmd = `ps aux | grep "[${c}]${rest}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
+    if (platform === "darwin") {
+      if (port) {
+        cmd = `ps aux | grep "[M]icrosoft Edge" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
+      } else {
+        cmd =
+          'ps aux | grep "[M]icrosoft Edge" | awk \'{sum += $6} END {printf "%d", sum/1024}\'';
+      }
+      const result = execSync(cmd, { timeout: 5000 }).toString().trim();
+      const mb = parseInt(result, 10);
+      return mb > 0 ? mb : null;
+    } else if (platform === "linux") {
+      if (port) {
+        cmd = `ps aux | grep "[m]sedge" | grep "remote-debugging-port=${port}" | awk '{sum += $6} END {printf "%d", sum/1024}'`;
+      } else {
+        cmd =
+          'ps aux | grep "[m]sedge" | awk \'{sum += $6} END {printf "%d", sum/1024}\'';
+      }
+      const result = execSync(cmd, { timeout: 5000 }).toString().trim();
+      const mb = parseInt(result, 10);
+      return mb > 0 ? mb : null;
+    } else if (platform === "win32") {
+      const ps1Path = path.join(os.tmpdir(), `edge-mem-${Date.now()}.ps1`);
+      try {
+        let ps1Content;
+        if (port) {
+          ps1Content =
+            `$procs = Get-CimInstance Win32_Process -Filter "Name LIKE 'msedge%'" -ErrorAction SilentlyContinue; ` +
+            `$sum = 0; ` +
+            `foreach ($p in $procs) { ` +
+            `  if ($p.CommandLine -and $p.CommandLine -like "*--remote-debugging-port=${port}*") { ` +
+            `    $sum += $p.WorkingSetSize ` +
+            `  } ` +
+            `}; ` +
+            `Write-Output $sum`;
+        } else {
+          ps1Content =
+            `$procs = Get-CimInstance Win32_Process -Filter "Name LIKE 'msedge%'" -ErrorAction SilentlyContinue; ` +
+            `$sum = 0; ` +
+            `foreach ($p in $procs) { $sum += $p.WorkingSetSize }; ` +
+            `Write-Output $sum`;
+        }
+        writeFileSync(ps1Path, ps1Content);
+        cmd = `powershell.exe -NoProfile -ExecutionPolicy Bypass -File "${ps1Path}"`;
+        const result = execSync(cmd, { timeout: 5000 }).toString().trim();
+        try {
+          unlinkSync(ps1Path);
+        } catch {}
+        const bytes = parseInt(result, 10);
+        const mb = bytes > 0 ? Math.round(bytes / 1048576) : 0;
+        return mb > 0 ? mb : null;
+      } catch {
+        try {
+          unlinkSync(ps1Path);
+        } catch {}
+        return null;
+      }
     }
-    const mb = parseInt(execSync(cmd, { timeout: 5000 }).toString().trim(), 10);
-    return mb > 0 ? mb : null;
+    return null;
   } catch {
     return null;
   }
 }
 
-function formatMemoryWithBrowser(port) {
+function formatMemoryWithBrowser(port = null) {
   const nodeMem = formatMemoryUsage();
   const edgeMB = getBrowserMemory(port);
   if (!edgeMB) return nodeMem;
-  const label = port ? ` edge:${port}:${edgeMB}MB` : ` edge:${edgeMB}MB`;
-  return nodeMem + label;
+  const portLabel = port ? `:${port}` : "";
+  return `${nodeMem} edge${portLabel}:${edgeMB}MB`;
 }
 
 // ────────────────────────────────────────
@@ -225,6 +289,7 @@ class CdpSessionManager {
     this.browser = null;
     this.page = null;
     this.lastRecycleAt = 0;
+    this.sessionStartTime = Date.now();
     this._lastMemCheck = 0;
     this._shuttingDown = false;
   }
@@ -304,14 +369,28 @@ class CdpSessionManager {
     this.browser = await ensureBrowserReady(opts);
     this.page = await this._ensurePage();
     this.lastRecycleAt = Date.now();
+    // 端口切换时重置运行计时
+    if (oldPort !== undefined && oldPort !== this.port) {
+      this.sessionStartTime = Date.now();
+    }
   }
 
-  /** 内存检查 + 按需回收（每 30s 最多采样一次） */
+  /** 内存检查 + 定时轮换 + 按需回收（每 30s 最多采样一次） */
   async maybeRecycleForMemory(processedCount) {
     const now = Date.now();
     if (now - this._lastMemCheck < 30000) return;
     this._lastMemCheck = now;
 
+    // 定时轮换：单端口运行超时自动切换
+    const sessionElapsed = now - this.sessionStartTime;
+    if (sessionElapsed >= CFG.maxSessionRunTimeMs) {
+      const mins = Math.round(sessionElapsed / 60000);
+      await this.recycleSession(`定时轮换 (运行 ${mins} 分钟)`, {
+        switchPort: true,
+      });
+      return;
+    }
+
     const mem = process.memoryUsage();
     const rssMB = mem.rss / 1024 / 1024;
     const heapMB = mem.heapUsed / 1024 / 1024;
@@ -481,7 +560,8 @@ async function runAutoMode(options) {
     browserRestartCount = 0;
   let consecutiveErrors = 0,
     consecutive403 = 0,
-    consecutiveCaptchaCount = 0;
+    consecutiveCaptchaCount = 0,
+    consecutiveTaskErrors = 0;
 
   // ── CDP 会话 ──
   const session = new CdpSessionManager({
@@ -720,15 +800,21 @@ async function runAutoMode(options) {
             await apiPut(`${serverUrl}/api/comment-task/${id}`);
             processedCount++;
             consecutive403 = 0;
+            consecutiveTaskErrors = 0;
 
             // 每 10 个任务输出内存汇总
             if (totalTasks() % 10 === 0) {
+              const remainingMs =
+                session.sessionStartTime + CFG.maxSessionRunTimeMs - Date.now();
+              const remainingMins =
+                remainingMs > 0 ? Math.round(remainingMs / 60000) : 0;
               console.error(
-                `── #${totalTasks()} ── ${formatMemoryWithBrowser(session.port)}  处理=${processedCount} 跳过=${skippedCount} 错误=${errorCount}`,
+                `── #${totalTasks()} ── ${formatMemoryWithBrowser(session.port)}  处理=${processedCount} 跳过=${skippedCount} 错误=${errorCount}  距切换≈${remainingMins}min`,
               );
             }
           } catch (err) {
             errorCount++;
+            consecutiveTaskErrors++;
             const isBrowserClosed = isBrowserClosedError(err);
             const is403 =
               err.message &&
@@ -754,6 +840,22 @@ async function runAutoMode(options) {
             } else {
               console.error(`        ${err.message}`);
             }
+
+            // 连续任务错误达到阈值，汇报服务端
+            if (consecutiveTaskErrors >= CFG.consecutiveTaskErrorsMax) {
+              console.error(
+                `  ⚠️ 连续任务错误 ${consecutiveTaskErrors}/${CFG.consecutiveTaskErrorsMax} → 汇报服务端`,
+              );
+              apiPost(`${serverUrl}/api/error-report`, {
+                userId,
+                username: "",
+                errorType: "consecutive_task_errors",
+                errorMessage: `连续 ${consecutiveTaskErrors} 个任务处理失败 (port=${session.port})`,
+                stage: "comments",
+                errorStack: "",
+              }).catch(() => {});
+              consecutiveTaskErrors = 0;
+            }
           }
 
           // 任务间内存检查
@@ -761,6 +863,7 @@ async function runAutoMode(options) {
         }
       } catch (err) {
         errorCount++;
+        consecutiveTaskErrors++;
         if (isBrowserClosedError(err)) {
           browserRestartCount++;
           console.error(`  ⚠ 浏览器断开 (#${browserRestartCount}) → 重启`);
@@ -768,6 +871,22 @@ async function runAutoMode(options) {
         } else {
           console.error(`  × ${err.message}`);
         }
+
+        // 连续轮间错误达到阈值，汇报服务端
+        if (consecutiveTaskErrors >= CFG.consecutiveTaskErrorsMax) {
+          console.error(
+            `  ⚠️ 连续轮间错误 ${consecutiveTaskErrors}/${CFG.consecutiveTaskErrorsMax} → 汇报服务端`,
+          );
+          apiPost(`${serverUrl}/api/error-report`, {
+            userId,
+            username: "",
+            errorType: "consecutive_task_errors",
+            errorMessage: `连续 ${consecutiveTaskErrors} 个轮次处理失败 (port=${session.port})`,
+            stage: "comments",
+            errorStack: "",
+          }).catch(() => {});
+          consecutiveTaskErrors = 0;
+        }
       }
 
       // 轮间内存检查 + 等待
@@ -794,7 +913,9 @@ async function runManualMode(url, options) {
     save: shouldSave,
     serverUrl = defaultServer,
     basePort = 9222,
+    proxy: proxyOption,
   } = options;
+  const effectiveProxy = proxyOption || configuredProxy;
 
   let browser;
   try {
@@ -930,5 +1051,6 @@ export async function handleComments(options) {
     save: commentsSave,
     serverUrl: commentsServer || defaultServer,
     basePort: commentsBasePort,
+    proxy: commentsProxy,
   });
 }

package/src/watch/data-store.js

@@ -2647,8 +2647,7 @@ export function createStore(filePath, options = {}) {
       let sql = `
         SELECT *
         FROM jobs_base
-        WHERE (COALESCE(tt_seller, '') = '' OR tt_seller = 1)
-          AND COALESCE(user_update_count, 0) <= 0
+        WHERE COALESCE(user_update_count, 0) <= 0
       `;
       const sqlParams = [];
 
@@ -2698,13 +2697,7 @@ export function createStore(filePath, options = {}) {
     const pending = data
       .filter((u) => {
         const updateCount = u.userUpdateCount;
-        // ttSeller 为空 或 ttSeller=1（商家重处理）都可以领取
-        const ttSellerEligible =
-          u.ttSeller === null ||
-          u.ttSeller === undefined ||
-          u.ttSeller === "" ||
-          u.ttSeller === 1;
-        if (!ttSellerEligible) return false;
+        // jobs_base 中所有 user_update_count <= 0 的记录都可以领取
         if (
           updateCount === null ||
           updateCount === undefined ||
@@ -3395,14 +3388,13 @@ export function createStore(filePath, options = {}) {
       };
     }
 
-    // 5. 验证：统计 jobs_base 中可被 attach 领取的记录数（tt_seller=1 且 user_update_count<=0）
+    // 5. 验证：统计 jobs_base 中可被 attach 领取的记录数（user_update_count<=0）
     let available = 0;
     try {
       const row = db
         .prepare(
           `SELECT COUNT(*) as total FROM jobs_base
-           WHERE tt_seller = 1
-             AND COALESCE(user_update_count, 0) <= 0
+           WHERE COALESCE(user_update_count, 0) <= 0
              AND video_count = 0`,
         )
         .get();

package/src/watch/db-stats.js

@@ -50,7 +50,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
 
   const userUpdateTasksRow = db
     .prepare(
-      `SELECT COUNT(*) as userUpdateTasks FROM jobs_base WHERE COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) <= 0`,
+      `SELECT COUNT(*) as userUpdateTasks FROM jobs_base WHERE COALESCE(user_update_count, 0) <= 0`,
     )
     .get();
 
@@ -147,7 +147,7 @@ export function getUserUpdateByCountryFromDb() {
     .prepare(
       `SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
        FROM jobs_base
-       WHERE tt_seller IS NULL AND COALESCE(user_update_count, 0) <= 0
+       WHERE COALESCE(user_update_count, 0) <= 0
        GROUP BY COALESCE(guessed_location, '未知')
        ORDER BY count DESC`,
     )
@@ -161,7 +161,7 @@ export function getAttachStuckByCountryFromDb() {
     .prepare(
       `SELECT COALESCE(guessed_location, '未知') as country, COUNT(*) as count
        FROM jobs_base
-       WHERE tt_seller IS NULL AND COALESCE(user_update_count, 0) = 1
+       WHERE COALESCE(user_update_count, 0) = 1
        GROUP BY COALESCE(guessed_location, '未知')
        ORDER BY count DESC`,
     )
@@ -194,7 +194,7 @@ export function restoreAttachStuckByCountry(country) {
     };
   }
 
-  const whereSql = `COALESCE(tt_seller, '') = '' AND COALESCE(user_update_count, 0) = 1 AND COALESCE(guessed_location, '未知') = ?`;
+  const whereSql = `COALESCE(user_update_count, 0) = 1 AND COALESCE(guessed_location, '未知') = ?`;
   const count =
     db
       .prepare(`SELECT COUNT(*) as c FROM jobs_base WHERE ${whereSql}`)

package/src/watch/public/app.js

@@ -163,7 +163,6 @@ function renderStats() {
   if (statJobsBase)
     statJobsBase.textContent = `jobs_base: ${formatStatNum(d.jobsBaseTotal || 0)}`;
   flashEl("statError", d.errorUsers);
-  flashEl("statRestricted", d.restrictedUsers);
   flashEl("statTarget", d.targetUsers, { full: true });
   flashEl("statUserUpdateTasks", d.userUpdateTasks || 0, { full: true });
   flashEl("statRawJobs", d.rawJobs || 0);
@@ -376,7 +375,6 @@ function renderSourceChart(sources) {
     following: "关注",
     follower: "粉丝",
     processed: "已处理",
-    restricted: "受限(跳过)",
     error: "错误(待重试)",
     noVideo: "无视频",
   };
@@ -419,13 +417,10 @@ function renderTable(users) {
       const nowStatus = u.status;
       const changed =
         wasStatus !== nowStatus &&
-        (nowStatus === "done" ||
-          nowStatus === "restricted" ||
-          nowStatus === "error");
+        (nowStatus === "done" || nowStatus === "error");
       const rowClass = changed ? ' class="row-flash"' : "";
 
       const statusTags = {
-        restricted: '<span class="tag error">受限(跳过)</span>',
         error: '<span class="tag error">错误(待重试)</span>',
         done: '<span class="tag processed">已完成</span>',
         processing: '<span class="tag processing">处理中</span>',
@@ -991,33 +986,6 @@ async function resetJob(uniqueId) {
   }
 }
 
-async function moveRestrictedToBase() {
-  const count = currentStats?.restrictedUsers || 0;
-  if (count === 0) {
-    showToast("没有受限用户");
-    return;
-  }
-  if (!confirm(`确定将 ${count} 个受限用户移到 jobs_base 吗？`)) return;
-  showLoading("迁移中...");
-  try {
-    const res = await fetch("/api/jobs/move-restricted-to-base", {
-      method: "POST",
-    });
-    const data = await res.json();
-    if (data.error) {
-      showToast(data.error, true);
-      return;
-    }
-    showToast(`已迁移 ${data.moved} 个受限用户`);
-    fetchStats();
-    fetchUsers();
-  } catch (e) {
-    showToast("迁移失败: " + e.message, true);
-  } finally {
-    hideLoading();
-  }
-}
-
 async function batchResetErrors() {
   const errorUsers = currentUsers.filter((u) => u.status === "error");
   if (errorUsers.length === 0) {
@@ -1762,8 +1730,6 @@ function renderTargetTable() {
         statusTag = '<span class="tag pending">待处理</span>';
       else if (u.status === "error")
         statusTag = '<span class="tag error">错误</span>';
-      else if (u.status === "restricted")
-        statusTag = '<span class="tag error">受限</span>';
       else statusTag = u.status || "-";
 
       return `<tr data-user="${u.uniqueId}">

package/src/watch/public/index.html

@@ -41,10 +41,6 @@
         <div class="label">错误</div>
         <div class="value error" id="statError">0</div>
       </div>
-      <div class="stat-card clickable" onclick="moveRestrictedToBase()">
-        <div class="label">受限</div>
-        <div class="value error" id="statRestricted">0</div>
-      </div>
       <div class="stat-card clickable pending-card" id="statUserUpdateCard" onclick="navigateToUserUpdate()">
         <div class="label">待补资料</div>
         <div class="value target" id="statUserUpdateTasks">0</div>
@@ -122,7 +118,6 @@
         <button data-filter="pending" onclick="setFilter('pending')">待处理</button>
         <button data-filter="done" onclick="setFilter('done')">已完成</button>
         <button data-filter="error" onclick="setFilter('error')">错误</button>
-        <button data-filter="restricted" onclick="setFilter('restricted')">受限</button>
         <button data-filter="target" onclick="setFilter('target')" style="background:#7c3aed;color:#fff">目标商家</button>
         <select id="targetLocationFilter" onchange="onTargetLocationChange()"
           style="padding:6px 10px;border:1px solid #7c3aed;border-radius:6px;background:#2a2a3a;color:#ccc;font-size:12px;cursor:pointer;outline:none;display:none">