tt-help-cli-ycl 1.4.35 → 1.4.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.35",
+  "version": "1.4.37",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/explore.js

@@ -398,6 +398,7 @@ export async function handleExplore(options) {
             maxFollowers: exploreMaxFollowers,
             location: exploreLocation,
             proxyServer: cdpOptions.proxyServer || null,
+            webserverUrl: exploreWebserver || null,
             browser,
             shopPage,
           },
@@ -428,6 +429,7 @@ export async function handleExplore(options) {
               maxFollowers: exploreMaxFollowers,
               location: exploreLocation,
               proxyServer: cdpOptions.proxyServer || null,
+              webserverUrl: exploreWebserver || null,
               browser,
               shopPage,
             },
@@ -476,6 +478,7 @@ export async function handleExplore(options) {
                 maxFollowers: exploreMaxFollowers,
                 location: exploreLocation,
                 proxyServer: cdpOptions.proxyServer || null,
+                webserverUrl: exploreWebserver || null,
                 browser,
                 shopPage,
               },

package/src/cli/refresh.js

@@ -411,6 +411,7 @@ export async function handleRefresh(options) {
           browser,
           shopPage,
           proxyServer: cdpOptions.proxyServer || null,
+          webserverUrl: exploreWebserver || null,
         },
         console.error,
       );
@@ -441,6 +442,7 @@ export async function handleRefresh(options) {
             browser,
             shopPage,
             proxyServer: cdpOptions.proxyServer || null,
+            webserverUrl: exploreWebserver || null,
           },
           console.error,
         );
@@ -490,6 +492,7 @@ export async function handleRefresh(options) {
               browser,
               shopPage,
               proxyServer: cdpOptions.proxyServer || null,
+              webserverUrl: exploreWebserver || null,
             },
             console.error,
           );

package/src/cli/webserver.js

@@ -1,19 +1,42 @@
-import { startWebserver, closeScraper } from "../webserver/server.mjs";
+import {
+  startWebserver,
+  closeScraper,
+  closeServer,
+} from "../webserver/server.mjs";
 
 export async function handleWebserver(options) {
-  const { webserverPort } = options;
+  const { webserverPort, webserverPoolSize, webserverProxy } = options;
 
-  const { server, port } = await startWebserver(webserverPort);
+  const { server, port } = await startWebserver(
+    webserverPort,
+    webserverPoolSize,
+    webserverProxy,
+  );
 
-  process.once("SIGINT", async () => {
-    await closeScraper();
-    server.close();
-    process.exit(0);
-  });
+  let shuttingDown = false;
+
+  async function gracefulShutdown(signal) {
+    if (shuttingDown) return;
+    shuttingDown = true;
+
+    console.error(`\n[webserver] Received ${signal}, shutting down...`);
 
-  process.once("SIGTERM", async () => {
-    await closeScraper();
-    server.close();
+    try {
+      await closeScraper();
+    } catch (e) {
+      console.error(`[webserver] scraper close error: ${e.message}`);
+    }
+
+    try {
+      await closeServer(server);
+    } catch (e) {
+      console.error(`[webserver] server close error: ${e.message}`);
+    }
+
+    console.error("[webserver] Shutdown complete.");
     process.exit(0);
-  });
+  }
+
+  process.once("SIGINT", () => gracefulShutdown("SIGINT"));
+  process.once("SIGTERM", () => gracefulShutdown("SIGTERM"));
 }

package/src/lib/args.js

@@ -401,17 +401,25 @@ function parseWatchArgs(args) {
 
 function parseWebserverArgs(args) {
   let webserverPort = 3000;
+  let webserverPoolSize = 0;
+  let webserverProxy = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
     if (arg === "-p" || arg === "--port") {
       webserverPort = parseInt(args[++i]) || 3000;
+    } else if (arg === "--pool-size") {
+      webserverPoolSize = parseInt(args[++i]) || 0;
+    } else if (arg === "--proxy") {
+      webserverProxy = args[++i] || null;
     }
   }
 
   return {
     subcommand: "webserver",
     webserverPort,
+    webserverPoolSize,
+    webserverProxy,
     urls: [],
     outputFile: null,
     outputFormat: "json",
@@ -482,6 +490,7 @@ function parseRefreshArgs(args) {
   let exploreRedoMaxAge = 0;
   let exploreProxy = null;
   let exploreJobLocations = null;
+  let exploreWebserver = null;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -511,6 +520,8 @@ function parseRefreshArgs(args) {
       exploreRedoMaxAge = parseInt(args[++i]) || 43200;
     } else if (arg === "--proxy") {
       exploreProxy = args[++i];
+    } else if (arg === "--webserver") {
+      exploreWebserver = args[++i];
     } else if (arg === "--location") {
       exploreLocation = args[++i];
     } else if (arg === "--job-locations") {
@@ -542,6 +553,7 @@ function parseRefreshArgs(args) {
     exploreMaxVideos,
     exploreRedoMaxAge,
     exploreProxy,
+    exploreWebserver,
     serverUrl,
     urls: [],
     outputFormat: "json",

package/src/lib/constants.js

@@ -138,6 +138,7 @@ const HELP_TEXT = [
   "      --port-count <数量>       端口数量（账户数），默认 10",
   "      --user-id <编号>          客户端编号（设备ID），默认自动生成",
   "--proxy <地址>            浏览器代理（如 socks5://127.0.0.1:1080），未指定时使用配置文件 proxy",
+  "--webserver <URL>         集中式视频国家查询服务（如 http://127.0.0.1:3000），省去本地 Playwright 浏览器",
   "",
   "  tt-help refresh [选项]",
   "    对目标商家用户进行轮回刷新，重新采集视频 + 关注 + 粉丝",
@@ -159,6 +160,7 @@ const HELP_TEXT = [
   "      --port-count <数量>       端口数量（账户数），默认 10",
   "      --user-id <编号>          客户端编号（设备ID），默认自动生成",
   "--proxy <地址>            浏览器代理（如 socks5://127.0.0.1:1080），未指定时使用配置文件 proxy",
+  "--webserver <URL>         集中式视频国家查询服务（如 http://127.0.0.1:3000），省去本地 Playwright 浏览器",
   "",
   "  tt-help precheck [选项]",
   "    对目标商家用户进行轻量预检，通过 HTTP 获取 videoCount 判断是否有新视频",
@@ -240,11 +242,15 @@ const HELP_TEXT = [
   "    启动 HTTP 服务，对外提供 TikTok 数据抓取 API",
   "    选项:",
   "      -p, --port <端口号>    监听端口，默认 3000",
+  "      --pool-size <数量>     浏览器页面池大小，默认 8",
+  "      --proxy <地址>         HTTP 代理（如 127.0.0.1:7897）",
   "    API:",
+  "      GET  /health            健康检查",
+  "      GET  /                  服务信息",
   "      POST /api/tiktok/user   获取用户信息 { uniqueId: string }",
   "      POST /api/tiktok/video  获取视频信息 { videoUrl: string }",
   "      POST /api/tiktok/lookup 同时获取视频和作者信息 { videoUrl: string }",
-  "    示例: tt-help webserver -p 3000",
+  "    示例: tt-help webserver -p 3000 --pool-size 8 --proxy 127.0.0.1:7897",
   "",
   "  comments [选项]",
   "    评论采集：从视频评论中发现新用户，来源标记为 comment",
@@ -377,7 +383,9 @@ const HELP_TEXT = [
   "",
   "  示例: tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@adidas",
   "        tt-help explore qiqi23280 fast --location ES --max-comments 50",
+  "        tt-help explore --webserver http://127.0.0.1:3000 --location DE,ES,FR",
   "        tt-help refresh --server http://127.0.0.1:3001 --port 9222",
+  "        tt-help webserver -p 3000 --pool-size 8 --proxy 127.0.0.1:7897",
   "        tt-help config set server http://127.0.0.1:3001",
   "        tt-help attach -p 5 -i 10",
   "        tt-help watch -o data/result.db",

package/src/npm-main.js

@@ -9,6 +9,7 @@ import { handleOpen } from "./cli/open.js";
 import { handleComments } from "./cli/comments.js";
 import { handleRefresh } from "./cli/refresh.js";
 import { handlePrecheck } from "./cli/precheck.js";
+import { handleWebserver } from "./cli/webserver.js";
 import {
   handleTag,
   handleDiscover,
@@ -38,8 +39,9 @@ async function main() {
       return handleAttach(parsed);
     case "recheck":
       return handleRecheck(parsed);
-    case "watch":
     case "webserver":
+      return handleWebserver(parsed);
+    case "watch":
     case "videostats":
     case "db-import":
       return exitUnsupportedCommand(parsed.subcommand);

package/src/scraper/explore-core.js

@@ -26,6 +26,7 @@ async function processExplore(page, username, options, log) {
     location = DEFAULT_TARGET_LOCATIONS_CSV,
     locationMode = "explore", // "explore" | "refresh"
     proxyServer = null,
+    webserverUrl = null,
   } = options;
 
   const result = {
@@ -172,9 +173,31 @@ async function processExplore(page, username, options, log) {
       const sampleUrls = sampleVideos.map((v) =>
         v.href.startsWith("http") ? v.href : `https://www.tiktok.com${v.href}`,
       );
-      const locations = await Promise.all(
-        sampleUrls.map((url) => extractVideoLocation(url)),
-      );
+      let locations;
+      if (webserverUrl) {
+        // 走集中式 webserver，不再启动本地 TikTokScraper
+        locations = await Promise.all(
+          sampleUrls.map(async (url) => {
+            try {
+              const res = await fetch(`${webserverUrl}/api/tiktok/video`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ videoUrl: url }),
+                signal: AbortSignal.timeout(30000),
+              });
+              const data = await res.json();
+              return data?.locationCreated || null;
+            } catch (e) {
+              log(`  [webserver] 视频查询失败: ${e.message}`);
+              return null;
+            }
+          }),
+        );
+      } else {
+        locations = await Promise.all(
+          sampleUrls.map((url) => extractVideoLocation(url)),
+        );
+      }
       log(
         `  国家采样(${locations.length}个): [${locations.filter(Boolean).join(", ") || "无数据"}]`,
       );

package/src/watch/data-store.js

@@ -2335,11 +2335,10 @@ export function createStore(filePath, options = {}) {
     const targetLocations = locations || DEFAULT_TARGET_LOCATIONS;
     const placeholders = targetLocations.map(() => "?").join(",");
 
-    // 事务：SELECT + 立即 UPDATE refresh_time，防止多客户端重复领取
-    const rows = getDb().transaction(() => {
-      const selected = getDb()
-        .prepare(
-          `
+    // SELECT 候选用户
+    const selected = getDb()
+      .prepare(
+        `
         SELECT unique_id, nickname, video_count, COALESCE(refresh_time, ?) AS refresh_time
         FROM jobs
         WHERE tt_seller = 1
@@ -2350,30 +2349,28 @@ export function createStore(filePath, options = {}) {
         ORDER BY COALESCE(refresh_time, ?) ASC
         LIMIT ?
       `,
-        )
-        .all(
-          defaultTime,
-          ...targetLocations,
-          defaultTime,
-          threshold,
-          defaultTime,
-          limit,
-        );
-
-      if (selected.length > 0) {
-        const ids = selected.map((r) => r.unique_id);
-        const idPlaceholders = ids.map(() => "?").join(",");
-        getDb()
-          .prepare(
-            `UPDATE jobs SET refresh_time = ?, updated_at = ? WHERE unique_id IN (${idPlaceholders})`,
-          )
-          .run(now, now, ...ids);
-      }
+      )
+      .all(
+        defaultTime,
+        ...targetLocations,
+        defaultTime,
+        threshold,
+        defaultTime,
+        limit,
+      );
 
-      return selected;
-    })();
+    // 立即更新 refresh_time 防重复领取（非事务，小概率并发重复可接受）
+    if (selected.length > 0) {
+      const ids = selected.map((r) => r.unique_id);
+      const idPlaceholders = ids.map(() => "?").join(",");
+      getDb()
+        .prepare(
+          `UPDATE jobs SET refresh_time = ?, updated_at = ? WHERE unique_id IN (${idPlaceholders})`,
+        )
+        .run(now, now, ...ids);
+    }
 
-    return rows.map((r) => ({
+    return selected.map((r) => ({
       uniqueId: r.unique_id,
       nickname: r.nickname,
       videoCount: r.video_count || 0,

package/src/watch/db-schema.js

@@ -70,6 +70,7 @@ export function initDb(filePath) {
   fs.mkdirSync(path.dirname(dbPath), { recursive: true });
   db = new Database(dbPath);
   db.pragma("journal_mode = WAL");
+  db.pragma("busy_timeout = 5000");
 
   // ===== users 表 =====
   db.exec(`

package/src/webserver/server.mjs

@@ -2,6 +2,8 @@ import http from "http";
 import os from "os";
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 
+// ---- 工具函数 ----
+
 function getLocalIP() {
   const ifaces = os.networkInterfaces();
   for (const name of Object.keys(ifaces)) {
@@ -14,23 +16,29 @@ function getLocalIP() {
   return "127.0.0.1";
 }
 
+const CORS_HEADERS = {
+  "Access-Control-Allow-Origin": "*",
+  "Access-Control-Allow-Methods": "POST, GET, OPTIONS",
+  "Access-Control-Allow-Headers": "Content-Type",
+};
+
 function sendJSON(res, status, data) {
   const body = JSON.stringify(data, null, 2);
   res.writeHead(status, {
     "Content-Type": "application/json; charset=utf-8",
-    "Access-Control-Allow-Origin": "*",
-    "Access-Control-Allow-Methods": "POST, GET, OPTIONS",
-    "Access-Control-Allow-Headers": "Content-Type",
+    ...CORS_HEADERS,
   });
   res.end(body);
 }
 
+const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10MB
+
 function readBody(req) {
   return new Promise((resolve, reject) => {
     let body = "";
     req.on("data", (chunk) => {
       body += chunk.toString();
-      if (body.length > 10 * 1024 * 1024) {
+      if (body.length > MAX_BODY_SIZE) {
         req.destroy();
         reject(new Error("Request body too large"));
       }
@@ -46,8 +54,18 @@ function readBody(req) {
   });
 }
 
+// ---- Scraper 生命周期 ----
+
+const SCRAPER_CONFIG = {
+  poolSize: 8,
+  wafTTL: 120000,
+  maxRequestsPerPage: 50,
+  proxyServer: null,
+};
+
 let scraperInstance = null;
 let scraperClosing = false;
+let scraperInitPromise = null;
 
 async function getScraper() {
   if (scraperInstance && !scraperClosing) {
@@ -56,115 +74,188 @@ async function getScraper() {
   if (scraperClosing) {
     throw new Error("Server is shutting down");
   }
-  scraperInstance = new TikTokScraper({
-    poolSize: 3,
-    wafTTL: 120000,
-    maxRequestsPerPage: 50,
-  });
-  await scraperInstance.init();
-  return scraperInstance;
+  // 防止并发调用重复初始化
+  if (scraperInitPromise) {
+    return scraperInitPromise;
+  }
+  scraperInitPromise = (async () => {
+    const scraper = new TikTokScraper(SCRAPER_CONFIG);
+    await scraper.init();
+    scraperInstance = scraper;
+    return scraper;
+  })();
+  try {
+    return await scraperInitPromise;
+  } finally {
+    scraperInitPromise = null;
+  }
 }
 
+// ---- API 处理器 ----
+
+const AVAILABLE_APIS = [
+  "/api/tiktok/user",
+  "/api/tiktok/video",
+  "/api/tiktok/lookup",
+  "/health",
+];
+
 async function handleTiktokUser(req, res) {
-  try {
-    const body = await readBody(req);
-    if (!body.uniqueId) {
-      return sendJSON(res, 400, { error: "missing uniqueId" });
-    }
-    const scraper = await getScraper();
-    const result = await scraper.getUserInfo(body.uniqueId);
-    if (!result) {
-      return sendJSON(res, 404, { error: "User not found or failed to parse data" });
-    }
-    return sendJSON(res, 200, result);
-  } catch (e) {
-    return sendJSON(res, 500, { error: e.message });
+  const body = await readBody(req);
+  if (!body.uniqueId) {
+    return sendJSON(res, 400, { error: "missing uniqueId" });
   }
+  const scraper = await getScraper();
+  const result = await scraper.getUserInfo(body.uniqueId);
+  if (!result) {
+    return sendJSON(res, 404, {
+      error: "User not found or failed to parse data",
+    });
+  }
+  return sendJSON(res, 200, result);
 }
 
 async function handleTiktokVideo(req, res) {
-  try {
-    const body = await readBody(req);
-    if (!body.videoUrl) {
-      return sendJSON(res, 400, { error: "missing videoUrl" });
-    }
-    const scraper = await getScraper();
-    const result = await scraper.getVideoInfo(body.videoUrl);
-    if (!result) {
-      return sendJSON(res, 404, { error: "Video not found or failed to parse data" });
-    }
-    return sendJSON(res, 200, result);
-  } catch (e) {
-    return sendJSON(res, 500, { error: e.message });
+  const body = await readBody(req);
+  if (!body.videoUrl) {
+    return sendJSON(res, 400, { error: "missing videoUrl" });
+  }
+  const scraper = await getScraper();
+  const result = await scraper.getVideoInfo(body.videoUrl);
+  if (!result) {
+    return sendJSON(res, 404, {
+      error: "Video not found or failed to parse data",
+    });
   }
+  return sendJSON(res, 200, result);
 }
 
 async function handleTiktokLookup(req, res) {
-  try {
-    const body = await readBody(req);
-    if (!body.videoUrl) {
-      return sendJSON(res, 400, { error: "missing videoUrl" });
-    }
-    const scraper = await getScraper();
-    const result = await scraper.getUserAndVideo(body.videoUrl);
-    if (!result) {
-      return sendJSON(res, 404, { error: "Video not found or failed to parse data" });
-    }
-    return sendJSON(res, 200, result);
-  } catch (e) {
-    return sendJSON(res, 500, { error: e.message });
+  const body = await readBody(req);
+  if (!body.videoUrl) {
+    return sendJSON(res, 400, { error: "missing videoUrl" });
   }
+  const scraper = await getScraper();
+  const result = await scraper.getUserAndVideo(body.videoUrl);
+  if (!result) {
+    return sendJSON(res, 404, {
+      error: "Video not found or failed to parse data",
+    });
+  }
+  return sendJSON(res, 200, result);
+}
+
+function handleHealth(req, res) {
+  return sendJSON(res, 200, {
+    status: "ok",
+    scraperReady: scraperInstance !== null && !scraperClosing,
+    uptime: process.uptime(),
+  });
 }
 
-export async function startWebserver(port = 3000) {
+function handleRoot(req, res) {
+  return sendJSON(res, 200, {
+    service: "TikTok Scraper WebService",
+    available: AVAILABLE_APIS,
+  });
+}
+
+// ---- 路由表 ----
+
+const ROUTES = [
+  { method: "GET", path: "/", handler: handleRoot },
+  { method: "GET", path: "/health", handler: handleHealth },
+  { method: "POST", path: "/api/tiktok/user", handler: handleTiktokUser },
+  { method: "POST", path: "/api/tiktok/video", handler: handleTiktokVideo },
+  { method: "POST", path: "/api/tiktok/lookup", handler: handleTiktokLookup },
+];
+
+// ---- 服务器 ----
+
+export async function startWebserver(
+  port = 3000,
+  poolSize = 0,
+  proxyServer = null,
+) {
+  if (poolSize > 0) {
+    SCRAPER_CONFIG.poolSize = poolSize;
+  }
+  if (proxyServer) {
+    SCRAPER_CONFIG.proxyServer = proxyServer;
+  }
   const server = http.createServer(async (req, res) => {
     const url = new URL(req.url, `http://127.0.0.1:${port}`);
 
+    // CORS 预检
     if (req.method === "OPTIONS") {
-      res.writeHead(204, {
-        "Access-Control-Allow-Origin": "*",
-        "Access-Control-Allow-Methods": "POST, GET, OPTIONS",
-        "Access-Control-Allow-Headers": "Content-Type",
-      });
+      res.writeHead(204, CORS_HEADERS);
       res.end();
       return;
     }
 
+    // 路由匹配
+    const route = ROUTES.find(
+      (r) => r.method === req.method && r.path === url.pathname,
+    );
+
+    if (!route) {
+      return sendJSON(res, 404, {
+        error: "Not found",
+        available: AVAILABLE_APIS,
+      });
+    }
+
     try {
-      if (url.pathname === "/api/tiktok/user" && req.method === "POST") {
-        return await handleTiktokUser(req, res);
-      }
-      if (url.pathname === "/api/tiktok/video" && req.method === "POST") {
-        return await handleTiktokVideo(req, res);
+      return await route.handler(req, res);
+    } catch (e) {
+      if (!res.headersSent) {
+        return sendJSON(res, 500, { error: e.message });
       }
-      if (url.pathname === "/api/tiktok/lookup" && req.method === "POST") {
-        return await handleTiktokLookup(req, res);
+      // 响应已发送（流式等场景），记录日志并销毁 socket
+      console.error(
+        `[webserver] unhandled error after response sent: ${e.message}`,
+      );
+      if (!res.writableEnded) {
+        res.destroy();
       }
-
-      sendJSON(res, 404, { error: "Not found", available: ["/api/tiktok/user", "/api/tiktok/video", "/api/tiktok/lookup"] });
-    } catch (e) {
-      sendJSON(res, 500, { error: e.message });
     }
   });
 
+  // 连接超时配置
+  server.keepAliveTimeout = 65000;
+  server.headersTimeout = 66000;
+  server.requestTimeout = 120000;
+
+  server.on("error", (err) => {
+    console.error(`[webserver] server error: ${err.message}`);
+  });
+
   return new Promise((resolve, reject) => {
     server.listen(port, () => {
       const localIP = getLocalIP();
       console.error(`\nTikTok Scraper WebService started`);
       console.error(`  Local:    http://127.0.0.1:${port}`);
       console.error(`  Network:  http://${localIP}:${port}`);
-      console.error(`  APIs:     POST /api/tiktok/user, /api/tiktok/video, /api/tiktok/lookup`);
+      console.error(`  APIs:     ${AVAILABLE_APIS.join(", ")}`);
       console.error(`  Press Ctrl+C to stop.\n`);
       resolve({ server, port });
     });
-    server.on("error", (err) => {
-      reject(err);
+    server.once("error", reject);
+  });
+}
+
+export async function closeServer(server) {
+  if (!server || !server.listening) return;
+  return new Promise((resolve, reject) => {
+    server.close((err) => {
+      if (err) reject(err);
+      else resolve();
     });
   });
 }
 
 export async function closeScraper() {
-  if (scraperInstance) {
+  if (scraperInstance && !scraperClosing) {
     scraperClosing = true;
     await scraperInstance.close();
     scraperInstance = null;