tt-help-cli-ycl 1.4.40 → 1.4.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.40",
+  "version": "1.4.41",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -297,11 +297,64 @@ function applyFilterAndScore(videos, targetCountries, result) {
   return { filtered, matchedAuthorSet };
 }
 
+/**
+ * 通过 webserver API 获取视频的国家信息（不走本地 HTTP view-source）
+ */
+async function enrichViaWebserver(videos, webserverUrl, onProgress) {
+  const total = videos.length;
+  let done = 0;
+  const CONCURRENCY = 16;
+
+  async function worker(queue) {
+    while (queue.length > 0) {
+      const v = queue.shift();
+      if (!v) break;
+      const videoUrl = `https://www.tiktok.com/@${v.authorUniqueId}/video/${v.id}`;
+      try {
+        const res = await fetch(`${webserverUrl}/api/tiktok/video`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ videoUrl }),
+          signal: AbortSignal.timeout(30000),
+        });
+        const data = await res.json();
+        v.locationCreated = data?.locationCreated || null;
+      } catch {
+        v.locationCreated = null;
+      }
+      done++;
+      if (onProgress) {
+        onProgress({
+          done,
+          total,
+          current: videoUrl,
+          locationCreated: v.locationCreated,
+        });
+      }
+    }
+  }
+
+  const queue = [...videos];
+  const workers = Array.from(
+    { length: Math.min(CONCURRENCY, queue.length) },
+    () => worker(queue),
+  );
+  await Promise.all(workers);
+}
+
 async function processTag(
   tag,
   index,
   total,
-  { enrich, targetLocations, noFilter, serverUrl, recordTags, port },
+  {
+    enrich,
+    targetLocations,
+    noFilter,
+    serverUrl,
+    recordTags,
+    port,
+    webserverUrl,
+  },
 ) {
   const prefix = total > 1 ? `[${index + 1}/${total}]` : "";
   process.stderr.write(`${prefix} 正在获取 #${tag} ... `);
@@ -323,33 +376,53 @@ async function processTag(
     let videos = result.videos;
     let filteredAuthors = result.uniqueAuthors;
 
-    if (enrich) {
-      const enrichMode = enrich === true ? "videos" : enrich;
-      if (noFilter) {
+    if (webserverUrl || enrich) {
+      if (webserverUrl) {
         process.stderr.write(
-          `\n  正在补充国家信息 (${enrichMode} 模式，不过滤)...\n`,
+          `\n  正在补充国家信息 (webserver API，目标: ${targetLocations.join(",")})...\n`,
         );
-      } else {
-        process.stderr.write(
-          `\n  正在补充国家信息 (${enrichMode} 模式，目标: ${targetLocations.join(",")})...\n`,
+        await enrichViaWebserver(
+          videos,
+          webserverUrl,
+          ({ done, total, current, locationCreated }) => {
+            const loc = locationCreated || "-";
+            const hit =
+              locationCreated &&
+              isLocationInList(locationCreated, targetLocations);
+            process.stderr.write(
+              `\r  [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${loc}${hit ? " ✓" : ""}\x1b[K`,
+            );
+          },
         );
-      }
-
-      const enriched = await enrichVideosWithLocation(videos, {
-        mode: enrichMode,
-        onProgress: ({ done, total, current, locationCreated }) => {
-          const label = enrichMode === "users" ? `@${current}` : current;
-          const loc = locationCreated || "-";
-          const hit =
-            locationCreated &&
-            isLocationInList(locationCreated, targetLocations);
+        process.stderr.write("\n");
+      } else {
+        const enrichMode = enrich === true ? "videos" : enrich;
+        if (noFilter) {
           process.stderr.write(
-            `\r  [${done}/${total}] ${label} → ${loc}${hit ? " ✓" : ""}\x1b[K`,
+            `\n  正在补充国家信息 (${enrichMode} 模式，不过滤)...\n`,
           );
-        },
-      });
-      videos = enriched.videos;
-      process.stderr.write("\n");
+        } else {
+          process.stderr.write(
+            `\n  正在补充国家信息 (${enrichMode} 模式，目标: ${targetLocations.join(",")})...\n`,
+          );
+        }
+
+        const enriched = await enrichVideosWithLocation(videos, {
+          mode: enrichMode,
+          onProgress: ({ done, total, current, locationCreated }) => {
+            const label = enrichMode === "users" ? `@${current}` : current;
+            const loc = locationCreated || "-";
+            const hit =
+              locationCreated &&
+              isLocationInList(locationCreated, targetLocations);
+            process.stderr.write(
+              `\r  [${done}/${total}] ${label} → ${loc}${hit ? " ✓" : ""}\x1b[K`,
+            );
+          },
+        });
+        videos = enriched.videos;
+        process.stderr.write("\n");
+      }
 
       if (!noFilter) {
         const before = videos.length;
@@ -482,7 +555,7 @@ function askTagQuestion(question) {
 
 async function scoreSingleTag(
   tag,
-  { baseUrl, cdpPort, targetCountries, effectiveProxy },
+  { baseUrl, cdpPort, targetCountries, effectiveProxy, webserverUrl },
 ) {
   const log = (...args) => process.stderr.write(args.join(" ") + "\n");
   const progress = (msg) => process.stderr.write(`\r  ${msg}\x1b[K`);
@@ -538,22 +611,41 @@ async function scoreSingleTag(
     }
 
     // Step 2: 补充国家信息
-    log(`  ⏳ 补充国家信息 (${videos.length} 个视频)...`);
-    const enrichOpts = {
-      mode: "videos",
-      onProgress: ({ done, total, current, locationCreated }) => {
-        if (done % 10 === 0 || done === total) {
-          const name = current.split("/").pop().slice(0, 20);
-          progress(`[${done}/${total}] ${name} → ${locationCreated || "-"}`);
-        }
-      },
-    };
-    if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
-    const enriched = await enrichVideosWithLocation(videos, enrichOpts);
-    clearLine();
-    videos = enriched.videos;
-    const withLoc = videos.filter((v) => v.locationCreated).length;
-    log(`  ✓ 国家信息: ${withLoc}/${videos.length} 个视频有国家`);
+    if (webserverUrl) {
+      log(
+        `  ⏳ 补充国家信息 (${videos.length} 个视频, via webserver ${webserverUrl})...`,
+      );
+      await enrichViaWebserver(
+        videos,
+        webserverUrl,
+        ({ done, total, current, locationCreated }) => {
+          if (done % 10 === 0 || done === total) {
+            const name = current.split("/").pop().slice(0, 20);
+            progress(`[${done}/${total}] ${name} → ${locationCreated || "-"}`);
+          }
+        },
+      );
+      clearLine();
+      const withLoc = videos.filter((v) => v.locationCreated).length;
+      log(`  ✓ 国家信息: ${withLoc}/${videos.length} 个视频有国家`);
+    } else {
+      log(`  ⏳ 补充国家信息 (${videos.length} 个视频)...`);
+      const enrichOpts = {
+        mode: "videos",
+        onProgress: ({ done, total, current, locationCreated }) => {
+          if (done % 10 === 0 || done === total) {
+            const name = current.split("/").pop().slice(0, 20);
+            progress(`[${done}/${total}] ${name} → ${locationCreated || "-"}`);
+          }
+        },
+      };
+      if (effectiveProxy) enrichOpts.proxyServer = effectiveProxy;
+      const enriched = await enrichVideosWithLocation(videos, enrichOpts);
+      clearLine();
+      videos = enriched.videos;
+      const withLoc = videos.filter((v) => v.locationCreated).length;
+      log(`  ✓ 国家信息: ${withLoc}/${videos.length} 个视频有国家`);
+    }
 
     // Step 3: 过滤 + 算分
     const { matchedAuthorSet } = applyFilterAndScore(
@@ -610,7 +702,14 @@ async function scoreSingleTag(
 
 export async function handleScore(parsed) {
   const { tagScore } = parsed;
-  const { tag, countries, serverUrl, port, proxy: cliProxy } = tagScore || {};
+  const {
+    tag,
+    countries,
+    serverUrl,
+    port,
+    proxy: cliProxy,
+    webserver,
+  } = tagScore || {};
 
   const baseUrl = serverUrl || DEFAULT_SERVER;
   const cdpPort = port || 9222;
@@ -680,6 +779,7 @@ export async function handleScore(parsed) {
       cdpPort,
       targetCountries,
       effectiveProxy,
+      webserverUrl: webserver,
     });
 
     // 如果队列还有 tag，自动继续；否则询问
@@ -743,7 +843,13 @@ async function reportToServer(baseUrl, result, clientId, meta, userId = "") {
 
 export async function handleScoreAll(parsed) {
   const { tagScoreAll } = parsed;
-  let { countries, serverUrl, port, proxy: cliProxy } = tagScoreAll || {};
+  let {
+    countries,
+    serverUrl,
+    port,
+    proxy: cliProxy,
+    webserver,
+  } = tagScoreAll || {};
 
   const baseUrl = serverUrl || DEFAULT_SERVER;
   const defaultScoreAllPort =
@@ -951,19 +1057,24 @@ export async function handleScoreAll(parsed) {
   const clientMeta = { type: "scoring", port: cdpPort };
 
   // 复用 TikTokScraper 实例，避免每次 enrich 都启动/关闭 headless 浏览器
-  const enrichScraper = new TikTokScraper({
-    poolSize: 3,
-    proxyServer: effectiveProxy || undefined,
-  });
-  await enrichScraper.init();
-  log("✅ TikTokScraper 已就绪 (enrich 复用)");
+  // webserver 模式下不需要本地 scraper
+  const enrichScraper = webserver
+    ? null
+    : new TikTokScraper({
+        poolSize: 3,
+        proxyServer: effectiveProxy || undefined,
+      });
+  if (enrichScraper) {
+    await enrichScraper.init();
+    log("✅ TikTokScraper 已就绪 (enrich 复用)");
+  }
   log(`  客户端 ID: ${clientId.substring(0, 8)}...`);
   log("");
 
   // Ctrl+C 时关闭浏览器和 scraper
   const cleanup = () => {
     log("\n正在清理资源...");
-    enrichScraper.close().catch(() => {});
+    if (enrichScraper) enrichScraper.close().catch(() => {});
     killEdgeProcesses(null, cdpPort);
     process.exit(0);
   };
@@ -1087,32 +1198,55 @@ export async function handleScoreAll(parsed) {
           throw new Error("no videos returned (network/captcha/slow-load)");
         }
 
-        // enrich: 先查服务端缓存，只对未知视频发 HTTP 请求
-        const enriched = await enrichWithServerCache(
-          videos,
-          baseUrl,
-          enrichScraper,
-          {
-            clientId,
-            clientMeta,
-            effectiveProxy,
-            log,
-          },
-        );
-        videos = enriched.videos;
-
-        // CDN 限流检测：有拦截则冷却 + 重启 scraper
-        const cdnBlocked = enriched.cdnBlockedCount || 0;
-        if (cdnBlocked > 0) {
-          const cdnRatio = cdnBlocked / (videos.length || 1);
-          const coolSec = cdnRatio > 0.3 ? 120 : 60;
+        // enrich: webserver API 或本地 HTTP view-source
+        if (webserver) {
           log(
-            `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
+            `  ⏳ 补充国家信息 (${videos.length} 个视频, via webserver ${webserver})...`,
           );
-          await new Promise((r) => setTimeout(r, coolSec * 1000));
-          log(`  正在重启 TikTokScraper...`);
-          await enrichScraper.restart();
-          log(`  ✅ TikTokScraper 已重启`);
+          await enrichViaWebserver(
+            videos,
+            webserver,
+            ({ done, total, current, locationCreated }) => {
+              if (done % 10 === 0 || done === total) {
+                const name = current.split("/").pop().slice(0, 20);
+                process.stderr.write(
+                  `\r    [${done}/${total}] ${name} → ${locationCreated || "-"}\x1b[K`,
+                );
+              }
+            },
+          );
+          process.stderr.write("\n");
+          log(
+            `  ✓ 国家信息: ${videos.filter((v) => v.locationCreated).length}/${videos.length} 个视频有国家`,
+          );
+        } else {
+          // enrich: 先查服务端缓存，只对未知视频发 HTTP 请求
+          const enriched = await enrichWithServerCache(
+            videos,
+            baseUrl,
+            enrichScraper,
+            {
+              clientId,
+              clientMeta,
+              effectiveProxy,
+              log,
+            },
+          );
+          videos = enriched.videos;
+
+          // CDN 限流检测：有拦截则冷却 + 重启 scraper
+          const cdnBlocked = enriched.cdnBlockedCount || 0;
+          if (cdnBlocked > 0) {
+            const cdnRatio = cdnBlocked / (videos.length || 1);
+            const coolSec = cdnRatio > 0.3 ? 120 : 60;
+            log(
+              `  ⚠️ CDN 限流: ${cdnBlocked}/${videos.length} (${(cdnRatio * 100).toFixed(0)}%)，冷却 ${coolSec} 秒后重启 scraper`,
+            );
+            await new Promise((r) => setTimeout(r, coolSec * 1000));
+            log(`  正在重启 TikTokScraper...`);
+            await enrichScraper.restart();
+            log(`  ✅ TikTokScraper 已重启`);
+          }
         }
 
         // 过滤 + 算分 (共用函数)
@@ -1281,7 +1415,7 @@ export async function handleScoreAll(parsed) {
       }
     }
   } finally {
-    await enrichScraper.close();
+    if (enrichScraper) await enrichScraper.close();
     log("✅ TikTokScraper 已关闭");
     log("正在关闭 CDP 浏览器...");
     try {
@@ -1336,6 +1470,7 @@ export async function handleTag(parsed) {
     noFilter,
     serverUrl,
     discover,
+    webserver,
   } = tagTags;
 
   const targetLocations = locations
@@ -1373,6 +1508,7 @@ export async function handleTag(parsed) {
       noFilter,
       serverUrl,
       recordTags: !!discover,
+      webserverUrl: webserver,
     });
 
     const output = { tag: result.tag };

package/src/lib/args.js

@@ -983,6 +983,7 @@ function parseTagArgs(args) {
   let scoreTag = null;
   let scoreCountries = null;
   let scorePort = 9222;
+  let webserverUrl = null;
   // 延迟路由：--port 的值先暂存，等循环结束后根据已确定的 subcommand 分配
   let rawTagPort = null;
 
@@ -1033,6 +1034,8 @@ function parseTagArgs(args) {
       scoreProxy = args[++i];
     } else if (arg === "-p" || arg === "--prompt") {
       discoverPrompt = args[++i];
+    } else if (arg === "--webserver") {
+      webserverUrl = args[++i];
     } else if (!arg.startsWith("-")) {
       const cleaned = arg.replace("#", "").trim();
       if (cleaned.toLowerCase() === "discover") {
@@ -1107,6 +1110,7 @@ function parseTagArgs(args) {
         serverUrl,
         port: scorePort,
         proxy: scoreProxy,
+        webserver: webserverUrl,
       },
       urls: [],
       outputFormat: "json",
@@ -1129,6 +1133,7 @@ function parseTagArgs(args) {
         serverUrl,
         port: scoreAllPort,
         proxy: scoreProxy,
+        webserver: webserverUrl,
       },
       urls: [],
       outputFormat: "json",
@@ -1198,6 +1203,7 @@ function parseTagArgs(args) {
       noFilter,
       serverUrl,
       discover,
+      webserver: webserverUrl,
     },
     urls: [],
     outputFormat: "json",