tt-help-cli-ycl 1.4.42 → 1.4.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.42",
+  "version": "1.4.44",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/explore.js

@@ -170,9 +170,6 @@ export async function handleExplore(options) {
 
     const page = await getOrCreatePage(browser);
 
-    // 创建专用商品页（后台复用，避免重复打开/关闭闪屏）
-    const shopPage = await browser.contexts()[0].newPage();
-
     // 先导航到 TikTok 页面，再检测登录状态
     await withRetry("启动页面导航", async () => {
       await page.goto(STARTUP_TIKTOK_URL, {
@@ -222,9 +219,6 @@ export async function handleExplore(options) {
       browser = newBrowser;
       const newPage = await setupNewPage(browser);
       Object.assign(page, newPage);
-      // 重建商品页（旧 browser 已关闭）
-      await shopPage.close().catch(() => {});
-      Object.assign(shopPage, await newBrowser.contexts()[0].newPage());
       Object.assign(cdpOptions, {
         port: nextAccount.port,
         userDataDir: nextAccount.userDataDir,
@@ -394,14 +388,13 @@ export async function handleExplore(options) {
           {
             maxVideos: exploreMaxVideos,
             enableFollow: exploreEnableFollow,
+            enableVideos: false,
             loggedIn, // 传入登录状态，避免每次调用 isLoggedIn(page)
             maxFollowing: exploreMaxFollowing,
             maxFollowers: exploreMaxFollowers,
             location: exploreLocation,
             proxyServer: cdpOptions.proxyServer || null,
             webserverUrl: exploreWebserver || null,
-            browser,
-            shopPage,
           },
           console.error,
         );
@@ -415,9 +408,6 @@ export async function handleExplore(options) {
           browser = newBrowser;
           const newPage = await setupNewPage(browser);
           Object.assign(page, newPage);
-          // 重建商品页
-          await shopPage.close().catch(() => {});
-          Object.assign(shopPage, await newBrowser.contexts()[0].newPage());
           // 重试当前用户
           result = await processExplore(
             page,
@@ -426,13 +416,13 @@ export async function handleExplore(options) {
               maxVideos: exploreMaxVideos,
               enableFollow: exploreEnableFollow,
               loggedIn, // 传入登录状态
+              enableVideos: false,
               maxFollowing: exploreMaxFollowing,
               maxFollowers: exploreMaxFollowers,
               location: exploreLocation,
               proxyServer: cdpOptions.proxyServer || null,
               webserverUrl: exploreWebserver || null,
               browser,
-              shopPage,
             },
             console.error,
           );
@@ -475,13 +465,13 @@ export async function handleExplore(options) {
                 maxVideos: exploreMaxVideos,
                 enableFollow: exploreEnableFollow,
                 loggedIn,
+                enableVideos: false,
                 maxFollowing: exploreMaxFollowing,
                 maxFollowers: exploreMaxFollowers,
                 location: exploreLocation,
                 proxyServer: cdpOptions.proxyServer || null,
                 webserverUrl: exploreWebserver || null,
                 browser,
-                shopPage,
               },
               console.error,
             );

package/src/cli/refresh.js

@@ -172,9 +172,6 @@ export async function handleRefresh(options) {
 
     const page = await getOrCreatePage(browser);
 
-    // 创建专用商品页（后台复用，避免重复打开/关闭闪屏）
-    const shopPage = await browser.contexts()[0].newPage();
-
     // 导航到 TikTok 页面
     await withRetry("启动页面导航", async () => {
       await page.goto(STARTUP_TIKTOK_URL, {
@@ -223,9 +220,6 @@ export async function handleRefresh(options) {
       browser = newBrowser;
       const newPage = await setupNewPage(browser);
       Object.assign(page, newPage);
-      // 重建商品页（旧 browser 已关闭）
-      await shopPage.close().catch(() => {});
-      Object.assign(shopPage, await newBrowser.contexts()[0].newPage());
       Object.assign(cdpOptions, {
         port: nextAccount.port,
         userDataDir: nextAccount.userDataDir,
@@ -404,13 +398,13 @@ export async function handleRefresh(options) {
         {
           maxVideos: exploreMaxVideos || 16,
           enableFollow: exploreEnableFollow !== false,
+          enableVideos: false,
           loggedIn,
           maxFollowing: exploreMaxFollowing || 100,
           maxFollowers: exploreMaxFollowers || 100,
           location: exploreLocation,
           locationMode: "refresh",
           browser,
-          shopPage,
           proxyServer: cdpOptions.proxyServer || null,
           webserverUrl: exploreWebserver || null,
         },
@@ -426,22 +420,19 @@ export async function handleRefresh(options) {
         browser = newBrowser;
         const newPage = await setupNewPage(browser);
         Object.assign(page, newPage);
-        // 重建商品页
-        await shopPage.close().catch(() => {});
-        Object.assign(shopPage, await newBrowser.contexts()[0].newPage());
         result = await processExplore(
           page,
           username,
           {
             maxVideos: exploreMaxVideos || 16,
             enableFollow: exploreEnableFollow !== false,
+            enableVideos: false,
             loggedIn,
             maxFollowing: exploreMaxFollowing || 100,
             maxFollowers: exploreMaxFollowers || 100,
             location: exploreLocation,
             locationMode: "refresh",
             browser,
-            shopPage,
             proxyServer: cdpOptions.proxyServer || null,
             webserverUrl: exploreWebserver || null,
           },
@@ -485,13 +476,13 @@ export async function handleRefresh(options) {
             {
               maxVideos: exploreMaxVideos || 16,
               enableFollow: exploreEnableFollow !== false,
+              enableVideos: false,
               loggedIn,
               maxFollowing: exploreMaxFollowing || 100,
               maxFollowers: exploreMaxFollowers || 100,
               location: exploreLocation,
               locationMode: "refresh",
               browser,
-              shopPage,
               proxyServer: cdpOptions.proxyServer || null,
               webserverUrl: exploreWebserver || null,
             },

package/src/cli/webserver.js

@@ -5,12 +5,14 @@ import {
 } from "../webserver/server.mjs";
 
 export async function handleWebserver(options) {
-  const { webserverPort, webserverPoolSize, webserverProxy } = options;
+  const { webserverPort, webserverPoolSize, webserverProxy, webserverNotify } =
+    options;
 
   const { server, port } = await startWebserver(
     webserverPort,
     webserverPoolSize,
     webserverProxy,
+    webserverNotify,
   );
 
   let shuttingDown = false;

package/src/lib/args.js

@@ -181,6 +181,7 @@ function parseExploreArgs(args) {
   let exploreUserId = null;
   let exploreMaxVideos = 16;
   let exploreProxy = null;
+  let exploreWebserver = null;
 
   const positional = [];
   const PRESETS = ["fast", "normal", "slow", "stealth"];
@@ -221,6 +222,8 @@ function parseExploreArgs(args) {
       exploreMaxVideos = parseInt(args[++i]) || 16;
     } else if (arg === "--proxy") {
       exploreProxy = args[++i];
+    } else if (arg === "--webserver") {
+      exploreWebserver = args[++i];
     } else {
       positional.push(arg);
     }
@@ -262,6 +265,7 @@ function parseExploreArgs(args) {
     exploreUserId,
     exploreMaxVideos,
     exploreProxy,
+    exploreWebserver,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,
@@ -403,6 +407,7 @@ function parseWebserverArgs(args) {
   let webserverPort = 3000;
   let webserverPoolSize = 0;
   let webserverProxy = null;
+  let webserverNotify = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -412,6 +417,8 @@ function parseWebserverArgs(args) {
       webserverPoolSize = parseInt(args[++i]) || 0;
     } else if (arg === "--proxy") {
       webserverProxy = args[++i] || null;
+    } else if (arg === "--notify") {
+      webserverNotify = args[++i] || null;
     }
   }
 
@@ -420,6 +427,7 @@ function parseWebserverArgs(args) {
     webserverPort,
     webserverPoolSize,
     webserverProxy,
+    webserverNotify,
     urls: [],
     outputFile: null,
     outputFormat: "json",

package/src/lib/shop-scraper.mjs

@@ -26,6 +26,16 @@ const PDP_HEADERS = {
   "Accept-Language": "en-US,en;q=0.9",
 };
 
+// 浏览器解验证码后缓存的 Cookie，给 undici 复用
+let cachedCookies = null;
+let cookiesExpireAt = 0;
+const COOKIE_TTL_MS = 30 * 60 * 1000; // Cookie 有效期 30 分钟
+
+function getCookieHeader() {
+  if (!cachedCookies || Date.now() > cookiesExpireAt) return null;
+  return cachedCookies.map((c) => `${c.name}=${c.value}`).join("; ");
+}
+
 /**
  * Extract a JSON object by its key name, handling nested braces and strings.
  */
@@ -71,15 +81,11 @@ function extractJsonByKey(html, keyPattern) {
 }
 
 /**
- * Open a headed browser for the user to manually solve TikTok captcha.
- * Returns true once the user closes the browser (captcha assumed solved).
- */
-/**
- * Open a headed browser for manual captcha solving. Polls the page until
- * product content appears, then auto-closes the browser.
+ * Open a headed browser for manual captcha solving, extract cookies
+ * so undici can reuse them for a while.
  * @param {string} url - TikTok Shop PDP URL to open in browser
  * @param {string} [proxyUrl]
- * @returns {Promise<boolean>} true if captcha was solved, false if timed out
+ * @returns {Promise<boolean>} true if captcha was solved and cookies cached
  */
 export async function resolveCaptchaWithBrowser(url, proxyUrl) {
   const { chromium } = await import("playwright");
@@ -110,10 +116,10 @@ export async function resolveCaptchaWithBrowser(url, proxyUrl) {
     "[shop-scraper] Browser opened, waiting for captcha to be solved...",
   );
 
-  // Poll until captcha is solved (page loads real content) or user closes browser
+  // Poll until captcha is solved (page loads real content) or browser closed
   let solved = false;
-  const POLL_MS = 1000;
-  const MAX_WAIT_MS = 120000;
+  const POLL_MS = 2000;
+  const MAX_WAIT_MS = 3600_000; // 最长等 1 小时
   const start = Date.now();
 
   while (!solved && Date.now() - start < MAX_WAIT_MS) {
@@ -126,19 +132,29 @@ export async function resolveCaptchaWithBrowser(url, proxyUrl) {
         solved = true;
         break;
       }
-      // Check if browser was closed by user
       if (!browser.isConnected()) break;
     } catch (e) {
-      // Page might be closed, check browser
       if (!browser.isConnected()) break;
     }
     await new Promise((r) => setTimeout(r, POLL_MS));
   }
 
   if (solved) {
-    console.error("[shop-scraper] Captcha solved, closing browser...");
+    // 提取 Cookie 给 undici 复用
+    try {
+      const cookies = await page.context().cookies();
+      if (cookies.length > 0) {
+        cachedCookies = cookies;
+        cookiesExpireAt = Date.now() + COOKIE_TTL_MS;
+        console.error(
+          `[shop-scraper] Captcha solved, cached ${cookies.length} cookies for undici reuse`,
+        );
+      }
+    } catch (e) {
+      console.error("[shop-scraper] Failed to extract cookies:", e.message);
+    }
   } else {
-    console.error("[shop-scraper] Timed out or browser closed, proceeding...");
+    console.error("[shop-scraper] Timed out or browser closed");
   }
 
   await browser.close().catch(() => {});
@@ -160,8 +176,14 @@ export async function getProductInfo(url, proxyUrl, options = {}) {
   let lastError;
   for (let attempt = 1; attempt <= 3; attempt++) {
     try {
+      const headers = { ...PDP_HEADERS };
+      const cookieStr = getCookieHeader();
+      if (cookieStr) {
+        headers["Cookie"] = cookieStr;
+      }
+
       const opts = {
-        headers: PDP_HEADERS,
+        headers,
         ...(agent ? { dispatcher: agent } : {}),
       };
 
@@ -173,15 +195,18 @@ export async function getProductInfo(url, proxyUrl, options = {}) {
     } catch (err) {
       lastError = err;
 
-      // Captcha: try browser fallback if enabled
+      // Captcha：清除过期 Cookie，弹浏览器重新获取
       if (
         err.message?.includes("Blocked by TikTok") &&
         options.captchaFallback
       ) {
-        console.error("[shop-scraper] Captcha hit, opening browser...");
+        console.error("[shop-scraper] Captcha hit, clearing cached cookies...");
+        cachedCookies = null;
+        cookiesExpireAt = 0;
+        console.error("[shop-scraper] Opening browser for manual captcha...");
         await resolveCaptchaWithBrowser(url, proxy);
-        // Retry HTTP after captcha solved
-        attempt = 0; // reset counter for fresh retries
+        // 重试（带着新 Cookie）
+        attempt = 0;
         continue;
       }
 

package/src/scraper/explore-core.js

@@ -60,7 +60,10 @@ async function processExplore(page, username, options, log) {
 
   try {
     log(`  访问 @${username} 主页...`);
-    const videoList = await collectVideos(page, username, maxVideos, log);
+    const videoList =
+      options.enableVideos !== false
+        ? await collectVideos(page, username, maxVideos, log)
+        : null;
 
     log("  获取用户信息...");
     const info = await getUserInfo(page);
@@ -130,19 +133,22 @@ async function processExplore(page, username, options, log) {
         seoUrl: topVideo.seoUrl || null,
       };
 
-      // 如果有商品链接，抓取商品详情页（评分、销量等）
-      if (topVideo.seoUrl && options.shopPage && !options.shopPage.isClosed()) {
+      // 如果有商品链接，通过 WebService API 抓取商品详情页（评分、销量等）
+      if (topVideo.seoUrl && options.webserverUrl) {
         try {
-          const { getProductInfoCDP } = await import("../lib/shop-scraper.mjs");
-          const shopUrl = topVideo.seoUrl.replace(
-            "www.tiktok.com/shop/pdp",
-            "shop.tiktok.com/us/pdp",
-          );
-
-          const productDetail = await getProductInfoCDP(
-            options.shopPage,
-            shopUrl,
+          const resp = await fetch(
+            `${options.webserverUrl}/api/tiktok/product`,
+            {
+              method: "POST",
+              headers: { "Content-Type": "application/json" },
+              body: JSON.stringify({ url: topVideo.seoUrl }),
+            },
           );
+          if (!resp.ok) {
+            const err = await resp.json().catch(() => ({}));
+            throw new Error(err.error || `HTTP ${resp.status}`);
+          }
+          const productDetail = await resp.json();
           result.productDetail = productDetail;
           log(
             `  商品详情: 评分${productDetail.rating ?? "-"} | 销量${productDetail.soldCount ?? "-"} | 评论${productDetail.reviewCount ?? "-"}`,

package/src/webserver/server.mjs

@@ -1,6 +1,7 @@
 import http from "http";
 import os from "os";
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import { getProductInfo } from "../lib/shop-scraper.mjs";
 
 // ---- 工具函数 ----
 
@@ -61,6 +62,7 @@ const SCRAPER_CONFIG = {
   wafTTL: 120000,
   maxRequestsPerPage: 50,
   proxyServer: null,
+  captchaNotifyUrl: null,
 };
 
 let scraperInstance = null;
@@ -144,6 +146,7 @@ const AVAILABLE_APIS = [
   "/api/tiktok/user",
   "/api/tiktok/video",
   "/api/tiktok/lookup",
+  "/api/tiktok/product",
   "/health",
 ];
 
@@ -192,6 +195,82 @@ async function handleTiktokLookup(req, res) {
   return sendJSON(res, 200, result);
 }
 
+async function handleTiktokProduct(req, res) {
+  const body = await readBody(req);
+  if (!body.url) {
+    return sendJSON(res, 400, { error: "missing url" });
+  }
+  // URL 标准化：www.tiktok.com/shop/pdp → shop.tiktok.com/us/pdp
+  const normalizedUrl = body.url.replace(
+    "www.tiktok.com/shop/pdp",
+    "shop.tiktok.com/us/pdp",
+  );
+  try {
+    const result = await getProductInfo(
+      normalizedUrl,
+      SCRAPER_CONFIG.proxyServer,
+      {
+        captchaFallback: false,
+      },
+    );
+    return sendJSON(res, 200, result);
+  } catch (e) {
+    // 先返回错误，不阻塞请求
+    sendJSON(res, 502, { error: e.message });
+
+    // 如果是 captcha，异步弹浏览器（不阻塞其他请求）
+    if (e.message?.includes("Blocked by TikTok")) {
+      scheduleCaptchaRecovery(normalizedUrl, SCRAPER_CONFIG.proxyServer);
+    }
+    return;
+  }
+}
+
+// 防止重复弹浏览器
+let captchaRecoveryRunning = false;
+
+async function scheduleCaptchaRecovery(url, proxy) {
+  if (captchaRecoveryRunning) {
+    console.error("[webserver] Captcha recovery already in progress, skipping");
+    return;
+  }
+  captchaRecoveryRunning = true;
+
+  // 通知 Watch Server：这台机器出现验证码需要人工处理
+  if (SCRAPER_CONFIG.captchaNotifyUrl) {
+    try {
+      await fetch(`${SCRAPER_CONFIG.captchaNotifyUrl}/api/error-report`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          userId: `webserver-${process.env.HOSTNAME || os.hostname() || "unknown"}`,
+          errorType: "captcha_blocked",
+          errorMessage:
+            "TikTok Shop captcha detected, browser opened - manual intervention required",
+          stage: "product_info",
+          errorStack: url,
+        }),
+      });
+      console.error("[webserver] Captcha alert sent to watch server");
+    } catch (e) {
+      console.error("[webserver] Failed to notify watch server:", e.message);
+    }
+  }
+
+  try {
+    const { getProductInfo } = await import("../lib/shop-scraper.mjs");
+    console.error(
+      "[webserver] Starting async captcha recovery, browser will open...",
+    );
+    await getProductInfo(url, proxy, { captchaFallback: true });
+    console.error("[webserver] Captcha recovery completed, cookies cached");
+  } catch (err) {
+    console.error("[webserver] Captcha recovery failed:", err.message);
+  } finally {
+    captchaRecoveryRunning = false;
+  }
+}
+
 function handleHealth(req, res) {
   return sendJSON(res, 200, {
     status: "ok",
@@ -219,6 +298,7 @@ const ROUTES = [
   { method: "POST", path: "/api/tiktok/user", handler: handleTiktokUser },
   { method: "POST", path: "/api/tiktok/video", handler: handleTiktokVideo },
   { method: "POST", path: "/api/tiktok/lookup", handler: handleTiktokLookup },
+  { method: "POST", path: "/api/tiktok/product", handler: handleTiktokProduct },
 ];
 
 // ---- 服务器 ----
@@ -227,6 +307,7 @@ export async function startWebserver(
   port = 3000,
   poolSize = 0,
   proxyServer = null,
+  captchaNotifyUrl = null,
 ) {
   if (poolSize > 0) {
     SCRAPER_CONFIG.poolSize = poolSize;
@@ -234,6 +315,9 @@ export async function startWebserver(
   if (proxyServer) {
     SCRAPER_CONFIG.proxyServer = proxyServer;
   }
+  if (captchaNotifyUrl) {
+    SCRAPER_CONFIG.captchaNotifyUrl = captchaNotifyUrl;
+  }
   const server = http.createServer(async (req, res) => {
     const url = new URL(req.url, `http://127.0.0.1:${port}`);