tt-help-cli-ycl 1.4.27 → 1.4.29

package/src/lib/args.js

@@ -556,6 +556,41 @@ function parseRefreshArgs(args) {
   };
 }
 
+function parseSearchArgs(args) {
+  let serverUrl = null;
+  let basePort = 3333;
+  let portCount = 10;
+  let proxy = null;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "-s" || arg === "--server") {
+      serverUrl = args[++i];
+    } else if (arg === "--base-port") {
+      basePort = parseInt(args[++i]) || 3333;
+    } else if (arg === "--port-count") {
+      portCount = parseInt(args[++i]) || 10;
+    } else if (arg === "--proxy") {
+      proxy = args[++i];
+    }
+  }
+
+  return {
+    subcommand: "search",
+    search: { serverUrl, basePort, portCount, proxy },
+    urls: [],
+    outputFormat: "json",
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 function parseAttachArgs(args) {
   let parallel = 1;
   let interval = 10;
@@ -690,8 +725,10 @@ function parseRawSyncArgs(args) {
   let rawSyncDb = null;
   let rawSyncArchive = null;
   let rawSyncInterval = null;
-  let rawSyncMax = null;
-  let rawSyncBatch = null;
+  let rawSyncThreshold = null;
+  let rawSyncLlm = false;
+  let rawSyncLlmMin = null;
+  let rawSyncLlmSample = null;
   let rawSyncOnce = false;
 
   for (let i = 0; i < args.length; i++) {
@@ -702,10 +739,17 @@ function parseRawSyncArgs(args) {
       rawSyncArchive = args[++i];
     } else if (arg === "--interval" && i + 1 < args.length) {
       rawSyncInterval = parseInt(args[++i]) || null;
-    } else if (arg === "--max" && i + 1 < args.length) {
-      rawSyncMax = parseInt(args[++i]) || null;
-    } else if (arg === "--batch" && i + 1 < args.length) {
-      rawSyncBatch = parseInt(args[++i]) || null;
+    } else if (
+      (arg === "--threshold" || arg === "--max") &&
+      i + 1 < args.length
+    ) {
+      rawSyncThreshold = parseInt(args[++i]) || null;
+    } else if (arg === "--llm") {
+      rawSyncLlm = true;
+    } else if (arg === "--llm-min" && i + 1 < args.length) {
+      rawSyncLlmMin = parseInt(args[++i]) || null;
+    } else if (arg === "--llm-sample" && i + 1 < args.length) {
+      rawSyncLlmSample = parseInt(args[++i]) || null;
     } else if (arg === "--once") {
       rawSyncOnce = true;
     }
@@ -716,8 +760,10 @@ function parseRawSyncArgs(args) {
     rawSyncDb,
     rawSyncArchive,
     rawSyncInterval,
-    rawSyncMax,
-    rawSyncBatch,
+    rawSyncThreshold,
+    rawSyncLlm,
+    rawSyncLlmMin,
+    rawSyncLlmSample,
     rawSyncOnce,
     urls: [],
     outputFormat: "json",
@@ -855,8 +901,10 @@ function parseTagArgs(args) {
   let isScore = false;
   let isScoreAll = false;
   let isRefresh = false;
+  let isSearch = false;
   let scoreAllPort = 7222;
   let refreshPort = 4222;
+  let searchPort = 3333;
   let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
@@ -938,6 +986,8 @@ function parseTagArgs(args) {
         isScoreAll = true;
       } else if (cleaned.toLowerCase() === "refresh") {
         isRefresh = true;
+      } else if (cleaned.toLowerCase() === "search") {
+        isSearch = true;
       } else {
         tags.push(cleaned.toLowerCase());
       }
@@ -948,6 +998,7 @@ function parseTagArgs(args) {
   if (rawTagPort !== null && !isNaN(rawTagPort)) {
     if (isScoreAll) scoreAllPort = rawTagPort;
     else if (isRefresh) refreshPort = rawTagPort;
+    else if (isSearch) searchPort = rawTagPort;
     else scorePort = rawTagPort;
   }
 
@@ -1039,6 +1090,28 @@ function parseTagArgs(args) {
     };
   }
 
+  if (isSearch) {
+    return {
+      subcommand: "search",
+      search: {
+        serverUrl,
+        basePort: searchPort,
+        portCount: 10,
+        proxy: scoreProxy,
+      },
+      urls: [],
+      outputFormat: "json",
+      exploreCount: 0,
+      showConfig: false,
+      showHelp: false,
+      customProxy: null,
+      configAction: null,
+      configValue: null,
+      pipeMode: false,
+      filterStr: null,
+    };
+  }
+
   return {
     subcommand: "tag",
     tagTags: {
@@ -1143,6 +1216,10 @@ export function parseArgs() {
     return parseRefreshArgs(args.slice(1));
   }
 
+  if (args.length > 0 && args[0] === "search") {
+    return parseSearchArgs(args.slice(1));
+  }
+
   if (args.length > 0 && args[0] === "raw-sync") {
     return parseRawSyncArgs(args.slice(1));
   }

package/src/lib/constants.js

@@ -314,18 +314,38 @@ const HELP_TEXT = [
   "          tt-help tag refresh -s http://127.0.0.1:3001",
   "          tt-help tag refresh --port 8222 -s http://127.0.0.1:3001",
   "",
+  "  search [选项]",
+  "    基于 productive 标签搜索 TikTok 作者（score>70 的标签作为搜索词）",
+  "    5 个搜索 Tab 并行采集（综合/视频/用户/直播/照片），去重后提交到 job_base",
+  "    选项:",
+  "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
+  "      --base-port <端口号>      起始 CDP 端口，默认 3333",
+  "      --port-count <数量>       端口数量（账户数），默认 10",
+  "      --proxy <地址>            浏览器代理",
+  "    行为:",
+  "      - 从 tags 表取 score>70 的 productive 标签",
+  "      - 去掉 # 号后作为搜索关键词",
+  "      - 并行搜索 5 个 tab，去重后提交到 job_base（来源=search）",
+  "      - 猜测国家取标签 countries 数组的第一个",
+  "    示例: tt-help search",
+  "          tt-help search -s http://127.0.0.1:3001",
+  "          tt-help search --base-port 3333 -s http://127.0.0.1:3001",
+  "          tt-help search --proxy http://127.0.0.1:7890",
+  "",
   "  raw-sync --db <主库路径> --archive <归档库路径> [选项]",
-  "    从归档库向主库 raw_jobs 同步数据，保持主库总量不超上限",
+  "    归档新用户 + 按需补充任务（从归档库向主库 jobs 移入）",
   "    选项:",
   "      --db <路径>            主库路径（必填）",
   "      --archive <路径>       归档库路径（必填）",
-  "      --interval <秒>        检测间隔，默认 30",
-  "      --max <条>             主库 raw_jobs 保有量上限，默认 10000",
-  "      --batch <条>           每次搬移条数，默认 500",
+  "  --interval <秒>        检测间隔，默认 120（2分钟）",
+  "  --threshold <条>       pending 低于此值时触发补任务，默认 50",
+  "      --llm                  启用 LLM 打分模式",
+  "      --llm-min <分>         LLM 最低分阈值，默认 60",
+  "      --llm-sample <条>      LLM 每批采样数，默认 100",
   "      --once                 只执行一次后退出",
   "    示例: tt-help raw-sync --db data/result.db --archive data/result-archive.db",
   "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --once",
-  "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --interval 60 --max 20000",
+  "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --llm --threshold 300",
   "",
   "  config  [show|set|unset|reset]",
   "    config                    查看当前配置",

package/src/lib/search-collector.js

@@ -0,0 +1,203 @@
+/**
+ * TikTok 搜索页作者采集 — DOM 滚动版
+ *
+ * 用法:
+ *   import { collectSearchAuthors } from "../lib/search-collector.js";
+ *   const ids = await collectSearchAuthors(browser, "tienda");
+ */
+
+const TAB_URLS = {
+  综合: (q) => `https://www.tiktok.com/search?q=${encodeURIComponent(q)}`,
+  视频: (q) => `https://www.tiktok.com/search/video?q=${encodeURIComponent(q)}`,
+  用户: (q) => `https://www.tiktok.com/search/user?q=${encodeURIComponent(q)}`,
+  直播: (q) => `https://www.tiktok.com/search/live?q=${encodeURIComponent(q)}`,
+  照片: (q) => `https://www.tiktok.com/search/photo?q=${encodeURIComponent(q)}`,
+};
+
+const DEFAULT_TABS = ["综合", "视频", "用户", "直播", "照片"];
+
+const SCROLL = {
+  maxRounds: 60,
+  staleThreshold: 3,
+  delayMin: 800,
+  delayMax: 1500,
+  waitAfterScroll: 4000,
+};
+const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+const randDelay = () =>
+  delay(
+    Math.floor(Math.random() * (SCROLL.delayMax - SCROLL.delayMin + 1)) +
+      SCROLL.delayMin,
+  );
+
+async function extractAuthors(page, excludeSet) {
+  return page.evaluate(
+    (exclude) => {
+      const ids = new Set();
+      const skip = new Set(exclude || []);
+      for (const a of document.querySelectorAll('a[href*="/@"]')) {
+        const m = (a.getAttribute("href") || "").match(/\/@([^/?]+)/);
+        if (m && m[1] !== "undefined" && m[1].length <= 30 && !skip.has(m[1])) {
+          ids.add(m[1]);
+        }
+      }
+      return [...ids];
+    },
+    [...excludeSet],
+  );
+}
+
+async function scrollPage(page) {
+  await page.evaluate(() => {
+    const main = document.querySelector("main");
+    if (main && main.scrollHeight > main.clientHeight + 10) {
+      main.scrollBy(0, main.clientHeight * 0.8);
+    } else {
+      window.scrollBy(0, window.innerHeight * 0.8);
+    }
+  });
+}
+
+async function processOneTab(page, tabName, keyword, allIds, excludeSet, log) {
+  const urlFn = TAB_URLS[tabName];
+  if (!urlFn) {
+    log(`[${tabName}] 不支持`);
+    return;
+  }
+
+  const tag = `[${tabName}]`;
+  const url = urlFn(keyword);
+
+  // 首屏正常加载
+  log(`${tag} 加载...`);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
+  } catch (e) {
+    log(`${tag} 导航失败: ${e.message}`);
+    return;
+  }
+  await delay(4000);
+
+  // 首屏渲染完成后屏蔽图片/字体/媒体，加速后续滚动
+  await page.route("**/*", (route) => {
+    const t = route.request().resourceType();
+    if (t === "image" || t === "font" || t === "media") route.abort();
+    else route.continue();
+  });
+
+  const tabSet = new Set();
+  let stale = 0;
+
+  for (let round = 0; round <= SCROLL.maxRounds; round++) {
+    if (round > 0) {
+      await scrollPage(page);
+      await randDelay();
+      await delay(SCROLL.waitAfterScroll);
+    }
+
+    const ids = await extractAuthors(page, excludeSet);
+    let added = 0;
+    for (const id of ids) {
+      if (!tabSet.has(id)) {
+        tabSet.add(id);
+        added++;
+      }
+    }
+
+    if (round === 0) {
+      log(`${tag} 初始: ${tabSet.size}`);
+      continue;
+    }
+
+    if (added === 0) {
+      stale++;
+      if (stale >= SCROLL.staleThreshold) {
+        log(`${tag} ${SCROLL.staleThreshold}轮无新`);
+        break;
+      }
+    } else {
+      stale = 0;
+      if (round % 5 === 0 || added >= 5)
+        log(`${tag} R${round}: +${added}, 计${tabSet.size}`);
+    }
+    if (tabSet.size >= 500) break;
+  }
+
+  let added = 0;
+  for (const id of tabSet) {
+    if (!allIds.has(id)) {
+      allIds.add(id);
+      added++;
+    }
+  }
+  log(`${tag} 完成: ${tabSet.size}个, 新增${added} (累计${allIds.size})`);
+}
+
+/**
+ * @param {import("playwright").Browser} browser
+ * @param {string} keyword
+ * @param {object} [opts]
+ * @param {string[]} [opts.tabs]
+ * @param {function} [opts.log] - 日志函数，默认 console.error
+ * @returns {Promise<string[]>}
+ */
+export async function collectSearchAuthors(browser, keyword, opts = {}) {
+  const tabs = opts.tabs || DEFAULT_TABS;
+  const log = opts.log || ((...args) => console.error(...args));
+  const excludeSet = new Set(opts.exclude || []);
+
+  log(`[collect] "${keyword}"  Tab: [${tabs.join(", ")}]`);
+  if (excludeSet.size > 0)
+    log(`[collect] 排除: [${[...excludeSet].join(", ")}]`);
+
+  const context = browser.contexts()[0];
+  const existingPage = context.pages()[0];
+  const allIds = new Set();
+
+  // 并行：每个 tab 独立页面同时跑
+  const newPages = []; // 追踪所有新建页面，确保清理
+  const tasks = tabs.map(async (tabName, i) => {
+    let page = null;
+    try {
+      page = i === 0 ? existingPage : await context.newPage();
+      if (i > 0) newPages.push(page);
+      await processOneTab(page, tabName, keyword, allIds, excludeSet, log);
+    } catch (e) {
+      log(`[${tabName}] 异常: ${e.message}`);
+    } finally {
+      if (i > 0 && page) {
+        try {
+          await page.close();
+        } catch {}
+      }
+    }
+  });
+
+  await Promise.all(tasks);
+
+  // 兜底：确保所有新建页面都已关闭
+  for (const p of newPages) {
+    try {
+      if (!p.isClosed()) await p.close();
+    } catch {}
+  }
+
+  return [...allIds].sort();
+}
+
+/**
+ * 采集并保存到文件（便捷函数）
+ * @param {import("playwright").Browser} browser
+ * @param {string} keyword
+ * @param {string} outputFile - 输出文件路径
+ * @param {object} [opts] - 同 collectSearchAuthors
+ * @returns {Promise<string[]>}
+ */
+export async function searchAndSave(browser, keyword, outputFile, opts = {}) {
+  const fs = await import("fs");
+  const path = await import("path");
+  const ids = await collectSearchAuthors(browser, keyword, opts);
+  fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+  fs.writeFileSync(outputFile, ids.join("\n") + "\n", "utf-8");
+  return ids;
+}

package/src/lib/target-locations.js

@@ -12,6 +12,7 @@ const DEFAULT_TARGET_LOCATIONS = [
   "IT",
   "IE",
   "AT",
+  "US",
 ];
 
 const DEFAULT_TARGET_LOCATIONS_CSV = DEFAULT_TARGET_LOCATIONS.join(",");

package/src/main.js

@@ -20,6 +20,7 @@ import {
   handleTagRefresh,
 } from "./cli/tag.js";
 import { handleRawSync } from "./cli/raw-sync.js";
+import { handleSearch } from "./cli/search.js";
 
 async function main() {
   const parsed = parseArgs();
@@ -59,6 +60,8 @@ async function main() {
       return handleScoreAll(parsed);
     case "tag-refresh":
       return handleTagRefresh(parsed);
+    case "search":
+      return handleSearch(parsed);
   }
 
   const {

package/src/npm-main.js

@@ -16,6 +16,7 @@ import {
   handleTagRefresh,
 } from "./cli/tag.js";
 import { handleRawSync } from "./cli/raw-sync.js";
+import { handleSearch } from "./cli/search.js";
 
 function exitUnsupportedCommand(command) {
   console.error(
@@ -59,6 +60,8 @@ async function main() {
       return handleTagRefresh(parsed);
     case "raw-sync":
       return handleRawSync(parsed);
+    case "search":
+      return handleSearch(parsed);
   }
 
   const {