tt-help-cli-ycl 1.3.84 → 1.3.86

package/src/lib/args.js

@@ -714,6 +714,186 @@ function parseCommentsArgs(args) {
   };
 }
 
+function parseTagArgs(args) {
+  const tags = [];
+  let outputFile = null;
+  let authorsOnly = false;
+  let videosOnly = false;
+  let enrich = null;
+  let locations = null;
+  let noFilter = false;
+  let serverUrl = null;
+  let discover = null;
+  let discoverCountries = [];
+  let discoverCount = 4;
+  let discoverPrompt = null;
+  let isDiscover = false;
+  let isScore = false;
+  let isScoreAll = false;
+  let scoreTag = null;
+  let scoreCountries = null;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "-o" || arg === "--output") {
+      outputFile = args[++i];
+    } else if (arg === "-s" || arg === "--server") {
+      serverUrl = args[++i];
+    } else if (arg === "--authors-only") {
+      authorsOnly = true;
+    } else if (arg === "--videos-only") {
+      videosOnly = true;
+    } else if (arg === "--enrich") {
+      const next = args[i + 1];
+      if (next === "users" || next === "videos") {
+        enrich = next;
+        i++;
+      } else {
+        enrich = true;
+      }
+    } else if (arg === "--locations") {
+      locations = args[++i];
+    } else if (arg === "--no-filter") {
+      noFilter = true;
+    } else if (arg === "--discover") {
+      // 旧版 --discover 兼容：跟在 tag 命令后面无子命令
+      const next = args[i + 1];
+      const count = parseInt(next);
+      if (!isNaN(count) && count > 0) {
+        discover = count;
+        i++;
+      } else {
+        discover = true;
+      }
+    } else if (arg === "--count") {
+      discoverCount = parseInt(args[++i]) || 4;
+    } else if (arg === "--countries") {
+      scoreCountries = args[++i]
+        .split(",")
+        .map((s) => s.trim().toUpperCase())
+        .filter(Boolean);
+    } else if (arg === "-p" || arg === "--prompt") {
+      discoverPrompt = args[++i];
+    } else if (!arg.startsWith("-")) {
+      const cleaned = arg.replace("#", "").trim();
+      if (cleaned.toLowerCase() === "discover") {
+        isDiscover = true;
+        for (let j = i + 1; j < args.length; j++) {
+          if (args[j].startsWith("-")) break;
+          discoverCountries.push(args[j].trim().toUpperCase());
+        }
+        i += discoverCountries.length;
+      } else if (cleaned.toLowerCase() === "score") {
+        isScore = true;
+        // score 后的第一个位置参数为 tag 名
+        if (i + 1 < args.length && !args[i + 1].startsWith("-")) {
+          scoreTag = args[i + 1].replace("#", "").trim().toLowerCase();
+          i++;
+        }
+        // 后续位置参数为 countries
+        for (let j = i + 1; j < args.length; j++) {
+          if (args[j].startsWith("-")) break;
+          if (!scoreCountries) scoreCountries = [];
+          scoreCountries.push(args[j].trim().toUpperCase());
+        }
+        if (scoreCountries) i += scoreCountries.length;
+      } else if (cleaned.toLowerCase() === "score-all") {
+        isScoreAll = true;
+      } else {
+        tags.push(cleaned.toLowerCase());
+      }
+    }
+  }
+
+  if (isDiscover) {
+    return {
+      subcommand: "tag-discover",
+      tagDiscover: {
+        countries: discoverCountries,
+        count: discoverCount,
+        prompt: discoverPrompt,
+        serverUrl,
+      },
+      urls: [],
+      outputFormat: "json",
+      exploreCount: 0,
+      showConfig: false,
+      showHelp: false,
+      customProxy: null,
+      configAction: null,
+      configValue: null,
+      pipeMode: false,
+      filterStr: null,
+    };
+  }
+
+  if (isScore) {
+    return {
+      subcommand: "tag-score",
+      tagScore: {
+        tag: scoreTag,
+        countries: scoreCountries,
+        serverUrl,
+      },
+      urls: [],
+      outputFormat: "json",
+      exploreCount: 0,
+      showConfig: false,
+      showHelp: false,
+      customProxy: null,
+      configAction: null,
+      configValue: null,
+      pipeMode: false,
+      filterStr: null,
+    };
+  }
+
+  if (isScoreAll) {
+    return {
+      subcommand: "tag-score-all",
+      tagScoreAll: {
+        countries: scoreCountries,
+        serverUrl,
+      },
+      urls: [],
+      outputFormat: "json",
+      exploreCount: 0,
+      showConfig: false,
+      showHelp: false,
+      customProxy: null,
+      configAction: null,
+      configValue: null,
+      pipeMode: false,
+      filterStr: null,
+    };
+  }
+
+  return {
+    subcommand: "tag",
+    tagTags: {
+      tags,
+      outputFile,
+      authorsOnly,
+      videosOnly,
+      enrich,
+      locations,
+      noFilter,
+      serverUrl,
+      discover,
+    },
+    urls: [],
+    outputFormat: "json",
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 export function parseArgs() {
   const args = process.argv.slice(2);
 
@@ -780,12 +960,8 @@ export function parseArgs() {
     return parseCommentsArgs(args.slice(1));
   }
 
-  if (args.length > 0 && args[0] === "videostats") {
-    return parseVideoStatsArgs(args.slice(1));
-  }
-
-  if (args.length > 0 && args[0] === "db-import") {
-    return parseDbImportArgs(args.slice(1));
+  if (args.length > 0 && args[0] === "tag") {
+    return parseTagArgs(args.slice(1));
   }
 
   if (args.length > 0 && args[0] === "refresh") {

package/src/lib/constants.js

@@ -208,6 +208,46 @@ const HELP_TEXT = [
   "      POST /api/tiktok/lookup 同时获取视频和作者信息 { videoUrl: string }",
   "    示例: tt-help webserver -p 3000",
   "",
+  "  tag  <标签名> [...] [选项]",
+  "    抓取标签页视频和作者（旧版 CLI 模式）",
+  "    选项:",
+  "      -s, --server <URL>        推送到 watch 服务端",
+  "      --enrich [users|videos]   补充国家/地区信息（默认 videos）",
+  `      --locations <国家代码>      目标国家，逗号分隔（默认 ${DEFAULT_TARGET_LOCATIONS_CSV}）`,
+  "      --no-filter               不过滤国家",
+  "      --discover [数量]          LLM 自动发现标签 + 记录有效标签",
+  "      -o, --output <file>       输出到 JSON 文件",
+  "      --authors-only            只输出作者列表",
+  "      --videos-only             只输出视频列表",
+  "    示例: tt-help tag ventas --enrich -s http://127.0.0.1:3001",
+  "",
+  "  tag discover <国家> [国家...] [选项]",
+  "    LLM 生成对应语言的 TikTok 电商标签，存入 tags 表",
+  "    选项:",
+  "      --count <N>               每个国家生成标签数（默认 4）",
+  "      -p, --prompt <文本>       用户自定义领域提示",
+  "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
+  "    示例: tt-help tag discover ES",
+  "          tt-help tag discover ES FR DE --count 5",
+  '          tt-help tag discover DE -p "手工首饰卖家"',
+  "",
+  "  tag score <标签名> [选项]",
+  "    客户端本地打分：抓取标签页 → 查作者国家 → 算分 → 推送用户 → 上报服务端",
+  "    选项:",
+  "      --countries <CSV>         目标国家，逗号分隔（默认 13 个欧洲国家）",
+  "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
+  "    示例: tt-help tag score ventas",
+  "          tt-help tag score ventas --countries ES",
+  "",
+  "  tag score-all [选项]",
+  "    自动循环打分：从服务端 tags 表取 new 标签，逐个本地打分并上报",
+  "    enrich 浏览器实例在整个循环中复用",
+  "    选项:",
+  "      --countries <CSV>         目标国家，逗号分隔（默认 13 个欧洲国家）",
+  "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
+  "    示例: tt-help tag score-all",
+  "          tt-help tag score-all --countries ES -s http://127.0.0.1:3001",
+  "",
   "  config  [show|set|unset|reset]",
   "    config                    查看当前配置",
   "    config set <key> <value>  设置配置（key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments）",
@@ -225,6 +265,9 @@ const HELP_TEXT = [
   "        tt-help attach -p 5 -i 10",
   "        tt-help watch -o data/result.db",
   "        tt-help videostats data/result.db -p 3",
+  "        tt-help tag discover ES FR --count 5",
+  "        tt-help tag score ventas --countries ES",
+  "        tt-help tag score-all --countries ES,FR -s http://127.0.0.1:3001",
 ];
 
 const PUBLIC_HELP_HIDDEN_HEADERS = new Set([

package/src/lib/parse-ssr.mjs

@@ -116,6 +116,7 @@ export function parseUserInfo(rawHtml) {
     secUid: u.secUid,
     ttSeller: u.ttSeller || false,
     locationCreated: u.locationCreated || null,
+    createTime: u.createTime || null,
     statusCode: 0,
   };
 }

package/src/lib/tag-discover.js

@@ -0,0 +1,150 @@
+import { readFileSync, writeFileSync, existsSync } from "fs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const TAGS_FILE = resolve(
+  __dirname,
+  "..",
+  "..",
+  "data",
+  "productive-tags.json",
+);
+
+function loadTags() {
+  try {
+    if (existsSync(TAGS_FILE)) {
+      return JSON.parse(readFileSync(TAGS_FILE, "utf-8"));
+    }
+  } catch {}
+  return { tags: [], lastUpdated: null };
+}
+
+function saveTags(data) {
+  const dir = dirname(TAGS_FILE);
+  if (!existsSync(dir)) {
+    const { mkdirSync } = require("fs");
+    mkdirSync(dir, { recursive: true });
+  }
+  writeFileSync(TAGS_FILE, JSON.stringify(data, null, 2), "utf-8");
+}
+
+export function getProductiveTags() {
+  return loadTags().tags;
+}
+
+export function recordProductiveTag(tag, country, userCount) {
+  const data = loadTags();
+  const existing = data.tags.find((t) => t.tag === tag);
+  if (existing) {
+    if (!existing.countries.includes(country)) {
+      existing.countries.push(country);
+    }
+    existing.userCount += userCount;
+    existing.lastUsed = new Date().toISOString();
+  } else {
+    data.tags.push({
+      tag,
+      countries: [country],
+      userCount,
+      firstSeen: new Date().toISOString(),
+      lastUsed: new Date().toISOString(),
+    });
+  }
+  data.lastUpdated = new Date().toISOString();
+  saveTags(data);
+}
+
+async function callLLM(prompt) {
+  const apiKey = process.env.APIKEY || "";
+  const { fetch } = await import("undici");
+
+  const response = await fetch(
+    "http://82.156.52.214:18000/v1/chat/completions",
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        model: "zc-fast",
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: 1024,
+        temperature: 0.7,
+      }),
+    },
+  );
+
+  const result = await response.json();
+  const content = result.choices?.[0]?.message?.content || "";
+  return content;
+}
+
+function normalizeTag(t) {
+  return t.replace(/^#+/, "").trim().toLowerCase();
+}
+
+function parseTagsFromResponse(content) {
+  try {
+    const parsed = JSON.parse(content);
+    if (Array.isArray(parsed)) {
+      return parsed.map(normalizeTag).filter((t) => t && t.length >= 2);
+    }
+    if (Array.isArray(parsed.tags)) {
+      return parsed.tags.map(normalizeTag).filter((t) => t && t.length >= 2);
+    }
+  } catch {}
+
+  const lines = content.split(/[\n,]+/);
+  const tags = [];
+  for (const line of lines) {
+    const cleaned = normalizeTag(line.replace(/^[-\d.\s]+/, ""));
+    if (cleaned && /^[a-z0-9_]+$/.test(cleaned) && cleaned.length >= 2) {
+      tags.push(cleaned);
+    }
+  }
+  return tags;
+}
+
+export async function discoverTags(countries, options = {}) {
+  const { language = "auto", count = 10 } = options;
+
+  const productiveTags = getProductiveTags();
+  const countryStr = Array.isArray(countries)
+    ? countries.join(", ")
+    : countries;
+  const langHint =
+    language === "auto" ? "" : `Tags should be in ${language} language.`;
+
+  const historyHint =
+    productiveTags.length > 0
+      ? `Previously productive tags for these countries: ${productiveTags
+          .filter((t) => t.countries.some((c) => countries.includes(c)))
+          .map((t) => `#${t.tag}`)
+          .join(", ")}. Generate new ones, don't repeat these.`
+      : "";
+
+  const prompt = `Generate ${count} TikTok hashtags (lowercase, no spaces, no # symbol) that are likely to be used by online sellers, shop owners, e-commerce merchants, and small businesses in these countries: ${countryStr}.
+
+Requirements:
+- Focus on tags that sellers/merchants actually use to promote their products
+- Include local language commerce tags (sell, shop, store, online, vendor, etc. in the local language)
+- Mix broad commerce tags with country-specific tags
+${langHint}
+${historyHint}
+
+Return ONLY a JSON array of tag strings, nothing else. Example: ["ventas","tiendaonline","vender"]`;
+
+  process.stderr.write(
+    `  [LLM] 正在生成 ${count} 个标签 (目标: ${countryStr})...\n`,
+  );
+  const content = await callLLM(prompt);
+  const tags = parseTagsFromResponse(content);
+
+  const unique = [...new Set(tags)].slice(0, count);
+  process.stderr.write(
+    `  [LLM] 生成 ${unique.length} 个标签: ${unique.join(", ")}\n`,
+  );
+  return unique;
+}

package/src/lib/tag-fetcher.js

@@ -0,0 +1,296 @@
+import { chromium } from "playwright";
+import { detectBrowser } from "./browser/launch.js";
+import { getAntiDetectScript } from "./browser/anti-detect.js";
+import { TikTokScraper } from "./tiktok-scraper.mjs";
+
+const TAG_URL = "https://www.tiktok.com/tag";
+const USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+const SCROLL_INTERVAL = 3000;
+const MAX_STALE_ROUNDS = 3;
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function findBrowser() {
+  return detectBrowser();
+}
+
+async function launchBrowser(browserPath) {
+  const opts = {
+    headless: true,
+    args: [
+      "--no-sandbox",
+      "--disable-blink-features=AutomationControlled",
+      "--disable-dev-shm-usage",
+    ],
+  };
+  if (browserPath) opts.executablePath = browserPath;
+
+  try {
+    return await chromium.launch(opts);
+  } catch {
+    if (browserPath) {
+      opts.executablePath = undefined;
+      return await chromium.launch(opts);
+    }
+    throw new Error("无法启动浏览器");
+  }
+}
+
+async function extractItemData(item) {
+  return {
+    id: item.id || "",
+    desc: (item.desc || "").trim(),
+    authorUniqueId: item.author?.uniqueId || "",
+    authorId: item.author?.id || "",
+    authorNickname: item.author?.nickname || "",
+    authorSecUid: item.author?.secUid || "",
+    createTime: item.createTime || 0,
+    playCount: item.stats?.playCount || 0,
+    diggCount: item.stats?.diggCount || 0,
+    shareCount: item.stats?.shareCount || 0,
+    commentCount: item.stats?.commentCount || 0,
+    musicTitle: item.music?.title || "",
+    isAd: item.isAd || false,
+    duration: item.video?.duration || 0,
+  };
+}
+
+/**
+ * 获取 TikTok 标签页下的所有视频和作者
+ * @param {string} tag - 标签名称（不含 # 号）
+ * @param {object} [options]
+ * @param {number} [options.timeout=300000] - 最大等待时间 (ms)
+ * @param {string} [options.browserPath] - 浏览器可执行文件路径，不传则自动探测
+ * @param {string} [options.locale='en-US'] - 页面语言
+ * @param {Function} [options.onProgress] - 进度回调 ({ videos, authors })
+ * @returns {Promise<{ tag: string, challengeId: string, totalPosts: number, videos: Array, uniqueAuthors: string[] }>}
+ */
+export async function fetchTagData(tag, options = {}) {
+  const {
+    timeout = 300000,
+    browserPath: customBrowserPath,
+    locale = "en-US",
+    onProgress,
+  } = options;
+
+  const browserPath = customBrowserPath || findBrowser();
+  if (!browserPath) {
+    throw new Error(
+      "未找到可用的浏览器，请设置 browserPath 或安装 Chrome/Edge",
+    );
+  }
+
+  const browser = await launchBrowser(browserPath);
+
+  try {
+    const context = await browser.newContext({
+      viewport: { width: 1280, height: 900 },
+      userAgent: USER_AGENT,
+      locale,
+    });
+    await context.addInitScript(getAntiDetectScript());
+    const page = await context.newPage();
+
+    let challengeInfo = null;
+    const rawVideos = [];
+    const authors = new Set();
+
+    page.on("response", async (resp) => {
+      try {
+        const url = resp.url();
+        const ct = resp.headers()["content-type"] || "";
+
+        if (url.includes("/api/challenge/detail/") && ct.includes("json")) {
+          const body = await resp.json();
+          if (body?.challengeInfo?.challenge) {
+            challengeInfo = body.challengeInfo.challenge;
+          }
+        }
+
+        if (url.includes("/api/challenge/item_list/") && ct.includes("json")) {
+          const body = await resp.json();
+          if (!body?.itemList) return;
+          for (const item of body.itemList) {
+            const uid = item.author?.uniqueId || "";
+            if (uid) authors.add(uid);
+            rawVideos.push(await extractItemData(item));
+          }
+          if (onProgress) {
+            onProgress({ videos: rawVideos.length, authors: authors.size });
+          }
+        }
+      } catch {}
+    });
+
+    const tagUrl = `${TAG_URL}/${encodeURIComponent(tag)}`;
+    const resp = await page.goto(tagUrl, {
+      waitUntil: "domcontentloaded",
+      timeout: 30000,
+    });
+
+    if (resp.status() !== 200) {
+      throw new Error(`标签页返回 HTTP ${resp.status()}`);
+    }
+
+    await page.waitForTimeout(3000);
+
+    const pageError = await page.evaluate(() => {
+      const text = document.body?.innerText || "";
+      if (text.includes("Something went wrong")) return "page_error";
+      return null;
+    });
+    if (pageError) {
+      throw new Error("标签页加载失败，TikTok 返回了错误页面");
+    }
+
+    let lastCount = 0;
+    let staleRounds = 0;
+    const startTime = Date.now();
+
+    while (staleRounds < MAX_STALE_ROUNDS) {
+      if (Date.now() - startTime > timeout) break;
+
+      await page.evaluate(() => window.scrollBy(0, 3000));
+      await sleep(SCROLL_INTERVAL);
+
+      if (rawVideos.length === lastCount) {
+        staleRounds++;
+      } else {
+        staleRounds = 0;
+        lastCount = rawVideos.length;
+      }
+    }
+
+    const seen = new Set();
+    const uniqueVideos = rawVideos.filter((v) =>
+      seen.has(v.id) ? false : (seen.add(v.id), true),
+    );
+
+    const totalPosts = challengeInfo?.stats?.videoCount || 0;
+
+    return {
+      tag,
+      challengeId: challengeInfo?.id || "",
+      totalPosts,
+      videoCount: uniqueVideos.length,
+      uniqueAuthorCount: authors.size,
+      videos: uniqueVideos,
+      uniqueAuthors: [...authors],
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * 为视频列表补充国家/地区信息（通过 TikTokScraper view-source 方式）
+ * @param {Array} videos - fetchTagData 返回的 videos 数组
+ * @param {object} [options]
+ * @param {string} [options.mode='users'] - 'users': 按作者查用户信息 / 'videos': 按视频查视频信息
+ * @param {number} [options.poolSize=3] - 并发页面数
+ * @param {number} [options.maxRetries=3] - 单个请求最大重试次数
+ * @param {Function} [options.onProgress] - 进度回调 ({ done, total, current, locationCreated })
+ * @returns {Promise<{ videos: Array, locationMap: Record<string, string|null> }>}
+ */
+export async function enrichVideosWithLocation(videos, options = {}) {
+  const {
+    mode = "videos",
+    poolSize = 3,
+    maxRetries = 3,
+    onProgress,
+    existingScraper,
+  } = options;
+
+  const scraper = existingScraper || new TikTokScraper({ poolSize });
+  const ownsScraper = !existingScraper;
+  if (ownsScraper) await scraper.init();
+
+  try {
+    const enriched = [...videos];
+    const locationMap = {};
+    let done = 0;
+
+    if (mode === "users") {
+      const uniqueAuthors = [
+        ...new Set(videos.map((v) => v.authorUniqueId).filter(Boolean)),
+      ];
+      const total = uniqueAuthors.length;
+
+      const tasks = uniqueAuthors.map((uniqueId) =>
+        (async () => {
+          try {
+            const userInfo = await scraper.getUserInfo(uniqueId, maxRetries);
+            const location = userInfo?.locationCreated || null;
+            locationMap[uniqueId] = location;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: uniqueId,
+                locationCreated: location,
+              });
+          } catch {
+            locationMap[uniqueId] = null;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: uniqueId,
+                locationCreated: null,
+              });
+          }
+        })(),
+      );
+
+      await Promise.allSettled(tasks);
+
+      for (const v of enriched) {
+        v.locationCreated = locationMap[v.authorUniqueId] || null;
+      }
+    } else {
+      const total = videos.length;
+
+      const tasks = enriched.map((v) =>
+        (async () => {
+          const videoUrl = `https://www.tiktok.com/@${v.authorUniqueId}/video/${v.id}`;
+          try {
+            const videoInfo = await scraper.getVideoInfo(videoUrl, maxRetries);
+            const location = videoInfo?.locationCreated || null;
+            v.locationCreated = location;
+            locationMap[v.id] = location;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: videoUrl,
+                locationCreated: location,
+              });
+          } catch {
+            v.locationCreated = null;
+            locationMap[v.id] = null;
+            done++;
+            if (onProgress)
+              onProgress({
+                done,
+                total,
+                current: videoUrl,
+                locationCreated: null,
+              });
+          }
+        })(),
+      );
+
+      await Promise.allSettled(tasks);
+    }
+
+    return { videos: enriched, locationMap };
+  } finally {
+    if (ownsScraper) await scraper.close();
+  }
+}