tt-help-cli-ycl 1.3.83 → 1.3.85

package/src/lib/tag-fetcher.js

@@ -0,0 +1,296 @@
+import { chromium } from "playwright";
+import { detectBrowser } from "./browser/launch.js";
+import { getAntiDetectScript } from "./browser/anti-detect.js";
+import { TikTokScraper } from "./tiktok-scraper.mjs";
+
+const TAG_URL = "https://www.tiktok.com/tag";
+const USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+const SCROLL_INTERVAL = 3000;
+const MAX_STALE_ROUNDS = 3;
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function findBrowser() {
+  return detectBrowser();
+}
+
+async function launchBrowser(browserPath) {
+  const opts = {
+    headless: true,
+    args: [
+      "--no-sandbox",
+      "--disable-blink-features=AutomationControlled",
+      "--disable-dev-shm-usage",
+    ],
+  };
+  if (browserPath) opts.executablePath = browserPath;
+
+  try {
+    return await chromium.launch(opts);
+  } catch {
+    if (browserPath) {
+      opts.executablePath = undefined;
+      return await chromium.launch(opts);
+    }
+    throw new Error("无法启动浏览器");
+  }
+}
+
+async function extractItemData(item) {
+  return {
+    id: item.id || "",
+    desc: (item.desc || "").trim(),
+    authorUniqueId: item.author?.uniqueId || "",
+    authorId: item.author?.id || "",
+    authorNickname: item.author?.nickname || "",
+    authorSecUid: item.author?.secUid || "",
+    createTime: item.createTime || 0,
+    playCount: item.stats?.playCount || 0,
+    diggCount: item.stats?.diggCount || 0,
+    shareCount: item.stats?.shareCount || 0,
+    commentCount: item.stats?.commentCount || 0,
+    musicTitle: item.music?.title || "",
+    isAd: item.isAd || false,
+    duration: item.video?.duration || 0,
+  };
+}
+
+/**
+ * 获取 TikTok 标签页下的所有视频和作者
+ * @param {string} tag - 标签名称（不含 # 号）
+ * @param {object} [options]
+ * @param {number} [options.timeout=300000] - 最大等待时间 (ms)
+ * @param {string} [options.browserPath] - 浏览器可执行文件路径，不传则自动探测
+ * @param {string} [options.locale='en-US'] - 页面语言
+ * @param {Function} [options.onProgress] - 进度回调 ({ videos, authors })
+ * @returns {Promise<{ tag: string, challengeId: string, totalPosts: number, videos: Array, uniqueAuthors: string[] }>}
+ */
+export async function fetchTagData(tag, options = {}) {
+  const {
+    timeout = 300000,
+    browserPath: customBrowserPath,
+    locale = "en-US",
+    onProgress,
+  } = options;
+
+  const browserPath = customBrowserPath || findBrowser();
+  if (!browserPath) {
+    throw new Error(
+      "未找到可用的浏览器，请设置 browserPath 或安装 Chrome/Edge",
+    );
+  }
+
+  const browser = await launchBrowser(browserPath);
+
+  try {
+    const context = await browser.newContext({
+      viewport: { width: 1280, height: 900 },
+      userAgent: USER_AGENT,
+      locale,
+    });
+    await context.addInitScript(getAntiDetectScript());
+    const page = await context.newPage();
+
+    let challengeInfo = null;
+    const rawVideos = [];
+    const authors = new Set();
+
+    page.on("response", async (resp) => {
+      try {
+        const url = resp.url();
+        const ct = resp.headers()["content-type"] || "";
+
+        if (url.includes("/api/challenge/detail/") && ct.includes("json")) {
+          const body = await resp.json();
+          if (body?.challengeInfo?.challenge) {
+            challengeInfo = body.challengeInfo.challenge;
+          }
+        }
+
+        if (url.includes("/api/challenge/item_list/") && ct.includes("json")) {
+          const body = await resp.json();
+          if (!body?.itemList) return;
+          for (const item of body.itemList) {
+            const uid = item.author?.uniqueId || "";
+            if (uid) authors.add(uid);
+            rawVideos.push(await extractItemData(item));
+          }
+          if (onProgress) {
+            onProgress({ videos: rawVideos.length, authors: authors.size });
+          }
+        }
+      } catch {}
+    });
+
+    const tagUrl = `${TAG_URL}/${encodeURIComponent(tag)}`;
+    const resp = await page.goto(tagUrl, {
+      waitUntil: "domcontentloaded",
+      timeout: 30000,
+    });
+
+    if (resp.status() !== 200) {
+      throw new Error(`标签页返回 HTTP ${resp.status()}`);
+    }
+
+    await page.waitForTimeout(3000);
+
+    const pageError = await page.evaluate(() => {
+      const text = document.body?.innerText || "";
+      if (text.includes("Something went wrong")) return "page_error";
+      return null;
+    });
+    if (pageError) {
+      throw new Error("标签页加载失败，TikTok 返回了错误页面");
+    }
+
+    let lastCount = 0;
+    let staleRounds = 0;
+    const startTime = Date.now();
+
+    while (staleRounds < MAX_STALE_ROUNDS) {
+      if (Date.now() - startTime > timeout) break;
+
+      await page.evaluate(() => window.scrollBy(0, 3000));
+      await sleep(SCROLL_INTERVAL);
+
+      if (rawVideos.length === lastCount) {
+        staleRounds++;
+      } else {
+        staleRounds = 0;
+        lastCount = rawVideos.length;
+      }
+    }
+
+    const seen = new Set();
+    const uniqueVideos = rawVideos.filter((v) =>
+      seen.has(v.id) ? false : (seen.add(v.id), true),
+    );
+
+    const totalPosts = challengeInfo?.stats?.videoCount || 0;
+
+    return {
+      tag,
+      challengeId: challengeInfo?.id || "",
+      totalPosts,
+      videoCount: uniqueVideos.length,
+      uniqueAuthorCount: authors.size,
+      videos: uniqueVideos,
+      uniqueAuthors: [...authors],
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * 为视频列表补充国家/地区信息（通过 TikTokScraper view-source 方式）
+ * @param {Array} videos - fetchTagData 返回的 videos 数组
+ * @param {object} [options]
+ * @param {string} [options.mode='users'] - 'users': 按作者查用户信息 / 'videos': 按视频查视频信息
+ * @param {number} [options.poolSize=3] - 并发页面数
+ * @param {number} [options.maxRetries=3] - 单个请求最大重试次数
+ * @param {Function} [options.onProgress] - 进度回调 ({ done, total, current, locationCreated })
+ * @returns {Promise<{ videos: Array, locationMap: Record<string, string|null> }>}
+ */
+export async function enrichVideosWithLocation(videos, options = {}) {
+  const {
+    mode = "videos",
+    poolSize = 3,
+    maxRetries = 3,
+    onProgress,
+    existingScraper,
+  } = options;
+
+  const scraper = existingScraper || new TikTokScraper({ poolSize });
+  const ownsScraper = !existingScraper;
+  if (ownsScraper) await scraper.init();
+
+  try {
+    const enriched = [...videos];
+    const locationMap = {};
+    let done = 0;
+
+    if (mode === "users") {
+      const uniqueAuthors = [
+        ...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
+      ];
+      const total = uniqueAuthors.length;
+
+      const tasks = uniqueAuthors.map((uniqueId) =>
+        (async () => {
+          try {
+            const userInfo = await scraper.getUserInfo(uniqueId, maxRetries);
+            const location = userInfo?.locationCreated || null;
+            locationMap[uniqueId] = location;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: uniqueId,
+                locationCreated: location,
+              });
+          } catch {
+            locationMap[uniqueId] = null;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: uniqueId,
+                locationCreated: null,
+              });
+          }
+        })(),
+      );
+
+      await Promise.allSettled(tasks);
+
+      for (const v of enriched) {
+        v.locationCreated = locationMap[v.authorUniqueId] || null;
+      }
+    } else {
+      const total = videos.length;
+
+      const tasks = enriched.map((v) =>
+        (async () => {
+          const videoUrl = `https://www.tiktok.com/@${v.authorUniqueId}/video/${v.id}`;
+          try {
+            const videoInfo = await scraper.getVideoInfo(videoUrl, maxRetries);
+            const location = videoInfo?.locationCreated || null;
+            v.locationCreated = location;
+            locationMap[v.id] = location;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: videoUrl,
+                locationCreated: location,
+              });
+          } catch {
+            v.locationCreated = null;
+            locationMap[v.id] = null;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: videoUrl,
+                locationCreated: null,
+              });
+          }
+        })(),
+      );
+
+      await Promise.allSettled(tasks);
+    }
+
+    return { videos: enriched, locationMap };
+  } finally {
+    if (ownsScraper) await scraper.close();
+  }
+}

package/src/lib/target-locations.js

@@ -51,10 +51,28 @@ function findFirstMatchingLocation(
   );
 }
 
+/**
+ * 从按频率排序的 entries 中，找第一个属于目标国家的。
+ * @param {Array<[string, number]>} entries - 已按频率降序排列的 [国家, 次数] 数组
+ * @param {string[]} targetLocations - 目标国家列表
+ * @returns {string|null} 频率最高的目标国家，如都不匹配则返回 null
+ */
+function findBestMatchingLocation(
+  entries,
+  targetLocations = DEFAULT_TARGET_LOCATIONS,
+) {
+  const normalizedTarget = normalizeLocationList(targetLocations);
+  for (const [loc] of entries) {
+    if (normalizedTarget.includes(loc)) return loc;
+  }
+  return null;
+}
+
 export {
   DEFAULT_TARGET_LOCATIONS,
   DEFAULT_TARGET_LOCATIONS_CSV,
   findFirstMatchingLocation,
+  findBestMatchingLocation,
   isLocationInList,
   normalizeLocation,
   normalizeLocationList,

package/src/main.js

@@ -11,6 +11,12 @@ import { handleVideoStats } from "./cli/videostats.js";
 import { handleDbImport } from "./cli/db-import.js";
 import { handleWebserver } from "./cli/webserver.js";
 import { handleRefresh } from "./cli/refresh.js";
+import {
+  handleTag,
+  handleDiscover,
+  handleScore,
+  handleScoreAll,
+} from "./cli/tag.js";
 
 async function main() {
   const parsed = parseArgs();
@@ -36,6 +42,14 @@ async function main() {
       return handleDbImport(parsed);
     case "refresh":
       return handleRefresh(parsed);
+    case "tag":
+      return handleTag(parsed);
+    case "tag-discover":
+      return handleDiscover(parsed);
+    case "tag-score":
+      return handleScore(parsed);
+    case "tag-score-all":
+      return handleScoreAll(parsed);
   }
 
   const {

package/src/npm-main.js

@@ -7,6 +7,7 @@ import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
 import { handleOpen } from "./cli/open.js";
 import { handleComments } from "./cli/comments.js";
 import { handleRefresh } from "./cli/refresh.js";
+import { handleTag } from "./cli/tag.js";
 
 function exitUnsupportedCommand(command) {
   console.error(
@@ -36,6 +37,8 @@ async function main() {
       return handleComments(parsed);
     case "refresh":
       return handleRefresh(parsed);
+    case "tag":
+      return handleTag(parsed);
   }
 
   const {

package/src/scraper/explore-core.js

@@ -6,7 +6,7 @@ import { extractFollowAndFollowers } from "./modules/follow-extractor.js";
 import { extractVideoLocation, setScraperProxy } from "../lib/scrape.js";
 import {
   DEFAULT_TARGET_LOCATIONS_CSV,
-  findFirstMatchingLocation,
+  findBestMatchingLocation,
   isLocationInList,
   normalizeLocation,
   normalizeLocationList,
@@ -152,13 +152,13 @@ async function processExplore(page, username, options, log) {
           locationDecision = `众数 (${entries[0][1]}次)`;
         }
       } else {
-        // explore 模式：优先命中目标国家，不匹配则回退众数
-        const matchedTargetLocation = findFirstMatchingLocation(
-          normalizedLocations,
+        // explore 模式：取频率最高的目标国家，不匹配则回退众数
+        const bestTargetLocation = findBestMatchingLocation(
+          entries,
           locationList,
         );
-        if (matchedTargetLocation) {
-          locationCreated = matchedTargetLocation;
+        if (bestTargetLocation) {
+          locationCreated = bestTargetLocation;
           locationDecision = "命中目标国家";
         } else if (entries.length > 0) {
           locationCreated = entries[0][0];