tt-help-cli-ycl 1.4.26 → 1.4.28

package/src/cli/raw-sync.js

@@ -1,7 +1,7 @@
 /**
  * raw-sync CLI 入口
  *
- * 启动 raw_jobs 同步程序（独立子进程）。
+ * 启动 raw_jobs 同步与任务补充程序（独立子进程）。
  */
 
 import { spawn } from "child_process";
@@ -16,10 +16,7 @@ export function handleRawSync(parsed) {
 
   if (parsed.rawSyncInterval != null)
     args.push("--interval", String(parsed.rawSyncInterval));
-  if (parsed.rawSyncThreshold != null)
-    args.push("--threshold", String(parsed.rawSyncThreshold));
-  if (parsed.rawSyncBatch != null)
-    args.push("--batch", String(parsed.rawSyncBatch));
+  if (parsed.rawSyncLlm) args.push("--llm");
   if (parsed.rawSyncOnce) args.push("--once");
 
   console.error(`[raw-sync] 启动同步程序: ${SCRIPT}`);

package/src/cli/search.js

@@ -0,0 +1,160 @@
+/**
+ * handleSearch — 基于 productive 标签搜索 TikTok 作者
+ *
+ * 流程:
+ *   1. GET /api/tags/search-task   → 领取标签
+ *   2. POST /api/tags/search-claim → 锁定标签
+ *   3. collectSearchAuthors()      → 并行搜索 5 个 tab
+ *   4. POST /api/tags/search-result → 提交结果
+ */
+
+import { randomUUID } from "crypto";
+import { ensureBrowserReady } from "../lib/browser/cdp.js";
+import { getOrCreatePage, safeCheckLogin } from "../lib/browser/page.js";
+import { collectSearchAuthors } from "../lib/search-collector.js";
+import {
+  server as cfgServer,
+  proxy as configuredProxy,
+} from "../lib/constants.js";
+
+const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
+
+export async function handleSearch(parsed) {
+  const { search: searchOpts } = parsed;
+  const { serverUrl, port, proxy: cliProxy } = searchOpts || {};
+
+  const baseUrl = serverUrl || DEFAULT_SERVER;
+  const cdpPort = port || 3333;
+  const effectiveProxy = cliProxy || configuredProxy;
+
+  const log = (...args) => process.stderr.write(args.join(" ") + "\n");
+
+  log("");
+  log("╔════════════════════════════════════╗");
+  log("║    TikTok 搜索作者采集（标签驱动）");
+  log("║    服务端: " + baseUrl);
+  log(
+    "║    CDP 端口: " +
+      cdpPort +
+      (effectiveProxy ? "  代理: " + effectiveProxy : ""),
+  );
+  log("║    score>70 的 productive 标签");
+  log("╚════════════════════════════════════╝");
+  log("");
+
+  // 连接 CDP 浏览器
+  const cdpOpts = { port: cdpPort };
+  if (effectiveProxy) cdpOpts.proxyServer = effectiveProxy;
+  const browser = await ensureBrowserReady(cdpOpts);
+  const page = await getOrCreatePage(browser);
+
+  // 先导航到 TikTok 页面，再检测登录（参照 explore.js 模式）
+  log("导航到 TikTok...");
+  try {
+    await page.goto("https://www.tiktok.com/@ycl5007", {
+      waitUntil: "domcontentloaded",
+      timeout: 30000,
+    });
+  } catch (e) {
+    log(`⚠️ 导航失败: ${e.message}`);
+  }
+
+  const loggedIn = await safeCheckLogin(page);
+  if (!loggedIn) {
+    log("❌ 未登录 TikTok，请在浏览器中登录后重试");
+    await browser.close();
+    process.exit(1);
+  }
+  log("✅ 已登录 TikTok");
+
+  const clientId = randomUUID();
+  const clientMeta = { type: "tag-search", port: cdpPort };
+  const headers = (extra = {}) => ({
+    "X-Client-Id": clientId,
+    "X-Client-Info": JSON.stringify(clientMeta),
+    ...extra,
+  });
+  log(`客户端 ID: ${clientId.substring(0, 8)}...`);
+  log("");
+
+  let totalSearched = 0;
+  let emptyRounds = 0;
+
+  const cleanup = () => {
+    log("\n正在清理...");
+    browser.close().catch(() => {});
+    process.exit(0);
+  };
+  process.on("SIGINT", cleanup);
+
+  while (true) {
+    try {
+      // 1. 获取搜索任务
+      const taskRes = await fetch(`${baseUrl}/api/tags/search-task`, {
+        headers: headers(),
+      });
+      const task = await taskRes.json();
+
+      if (!task.tag) {
+        emptyRounds++;
+        if (emptyRounds >= 5) {
+          log(`⏳ 连续 ${emptyRounds} 轮无标签可搜索（30 秒后重试）...`);
+          await new Promise((r) => setTimeout(r, 30000));
+          emptyRounds = 0;
+          continue;
+        }
+        log(`⏳ 暂无标签可搜索（连续 ${emptyRounds} 轮），10 秒后重试...`);
+        await new Promise((r) => setTimeout(r, 10000));
+        continue;
+      }
+      emptyRounds = 0;
+
+      const tag = task.tag;
+      const country = task.country;
+      log(
+        `[${totalSearched + 1}] 搜索: "${tag}" (score: ${task.score}, 国家: ${country})`,
+      );
+
+      // 2. 锁定标签
+      const claimRes = await fetch(`${baseUrl}/api/tags/search-claim`, {
+        method: "POST",
+        headers: headers({ "Content-Type": "application/json" }),
+        body: JSON.stringify({ tag }),
+      });
+      const claimData = await claimRes.json();
+      if (!claimData.ok) {
+        log(`  ⏭️ ${claimData.error}，跳过`);
+        continue;
+      }
+
+      // 3. 并行搜索
+      const startTime = Date.now();
+      const ids = await collectSearchAuthors(browser, tag, { log });
+      const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+      log(`  搜索完成: ${ids.length} 个作者 (${elapsed}s)`);
+
+      // 4. 提交结果
+      if (ids.length > 0) {
+        const resultRes = await fetch(`${baseUrl}/api/tags/search-result`, {
+          method: "POST",
+          headers: headers({ "Content-Type": "application/json" }),
+          body: JSON.stringify({ tag, authors: ids, country }),
+        });
+        const result = await resultRes.json();
+        log(`  提交: 新增 ${result.pushed} 个, 跳过 ${result.skipped} 个`);
+      } else {
+        await fetch(`${baseUrl}/api/tags/search-result`, {
+          method: "POST",
+          headers: headers({ "Content-Type": "application/json" }),
+          body: JSON.stringify({ tag, authors: [], country }),
+        });
+        log(`  提交: 0 个作者`);
+      }
+
+      totalSearched++;
+    } catch (e) {
+      log(`❌ 异常: ${e.message}`);
+      await new Promise((r) => setTimeout(r, 5000));
+    }
+  }
+}

package/src/lib/args.js

@@ -556,6 +556,38 @@ function parseRefreshArgs(args) {
   };
 }
 
+function parseSearchArgs(args) {
+  let serverUrl = null;
+  let port = 3333;
+  let proxy = null;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "-s" || arg === "--server") {
+      serverUrl = args[++i];
+    } else if (arg === "--port") {
+      port = parseInt(args[++i]) || 3333;
+    } else if (arg === "--proxy") {
+      proxy = args[++i];
+    }
+  }
+
+  return {
+    subcommand: "search",
+    search: { serverUrl, port, proxy },
+    urls: [],
+    outputFormat: "json",
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 function parseAttachArgs(args) {
   let parallel = 1;
   let interval = 10;
@@ -691,7 +723,9 @@ function parseRawSyncArgs(args) {
   let rawSyncArchive = null;
   let rawSyncInterval = null;
   let rawSyncThreshold = null;
-  let rawSyncBatch = null;
+  let rawSyncLlm = false;
+  let rawSyncLlmMin = null;
+  let rawSyncLlmSample = null;
   let rawSyncOnce = false;
 
   for (let i = 0; i < args.length; i++) {
@@ -702,10 +736,17 @@ function parseRawSyncArgs(args) {
       rawSyncArchive = args[++i];
     } else if (arg === "--interval" && i + 1 < args.length) {
       rawSyncInterval = parseInt(args[++i]) || null;
-    } else if (arg === "--threshold" && i + 1 < args.length) {
+    } else if (
+      (arg === "--threshold" || arg === "--max") &&
+      i + 1 < args.length
+    ) {
       rawSyncThreshold = parseInt(args[++i]) || null;
-    } else if (arg === "--batch" && i + 1 < args.length) {
-      rawSyncBatch = parseInt(args[++i]) || null;
+    } else if (arg === "--llm") {
+      rawSyncLlm = true;
+    } else if (arg === "--llm-min" && i + 1 < args.length) {
+      rawSyncLlmMin = parseInt(args[++i]) || null;
+    } else if (arg === "--llm-sample" && i + 1 < args.length) {
+      rawSyncLlmSample = parseInt(args[++i]) || null;
     } else if (arg === "--once") {
       rawSyncOnce = true;
     }
@@ -717,7 +758,9 @@ function parseRawSyncArgs(args) {
     rawSyncArchive,
     rawSyncInterval,
     rawSyncThreshold,
-    rawSyncBatch,
+    rawSyncLlm,
+    rawSyncLlmMin,
+    rawSyncLlmSample,
     rawSyncOnce,
     urls: [],
     outputFormat: "json",
@@ -855,8 +898,10 @@ function parseTagArgs(args) {
   let isScore = false;
   let isScoreAll = false;
   let isRefresh = false;
+  let isSearch = false;
   let scoreAllPort = 7222;
   let refreshPort = 4222;
+  let searchPort = 3333;
   let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
@@ -938,6 +983,8 @@ function parseTagArgs(args) {
         isScoreAll = true;
       } else if (cleaned.toLowerCase() === "refresh") {
         isRefresh = true;
+      } else if (cleaned.toLowerCase() === "search") {
+        isSearch = true;
       } else {
         tags.push(cleaned.toLowerCase());
       }
@@ -948,6 +995,7 @@ function parseTagArgs(args) {
   if (rawTagPort !== null && !isNaN(rawTagPort)) {
     if (isScoreAll) scoreAllPort = rawTagPort;
     else if (isRefresh) refreshPort = rawTagPort;
+    else if (isSearch) searchPort = rawTagPort;
     else scorePort = rawTagPort;
   }
 
@@ -1039,6 +1087,27 @@ function parseTagArgs(args) {
     };
   }
 
+  if (isSearch) {
+    return {
+      subcommand: "search",
+      search: {
+        serverUrl,
+        port: searchPort,
+        proxy: scoreProxy,
+      },
+      urls: [],
+      outputFormat: "json",
+      exploreCount: 0,
+      showConfig: false,
+      showHelp: false,
+      customProxy: null,
+      configAction: null,
+      configValue: null,
+      pipeMode: false,
+      filterStr: null,
+    };
+  }
+
   return {
     subcommand: "tag",
     tagTags: {
@@ -1143,6 +1212,10 @@ export function parseArgs() {
     return parseRefreshArgs(args.slice(1));
   }
 
+  if (args.length > 0 && args[0] === "search") {
+    return parseSearchArgs(args.slice(1));
+  }
+
   if (args.length > 0 && args[0] === "raw-sync") {
     return parseRawSyncArgs(args.slice(1));
   }

package/src/lib/constants.js

@@ -314,18 +314,37 @@ const HELP_TEXT = [
   "          tt-help tag refresh -s http://127.0.0.1:3001",
   "          tt-help tag refresh --port 8222 -s http://127.0.0.1:3001",
   "",
+  "  search [选项]",
+  "    基于 productive 标签搜索 TikTok 作者（score>70 的标签作为搜索词）",
+  "    5 个搜索 Tab 并行采集（综合/视频/用户/直播/照片），去重后提交到 job_base",
+  "    选项:",
+  "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
+  "      --port <N>                CDP 端口（默认 3333）",
+  "      --proxy <地址>            浏览器代理",
+  "    行为:",
+  "      - 从 tags 表取 score>70 的 productive 标签",
+  "      - 去掉 # 号后作为搜索关键词",
+  "      - 并行搜索 5 个 tab，去重后提交到 job_base（来源=search）",
+  "      - 猜测国家取标签 countries 数组的第一个",
+  "    示例: tt-help search",
+  "          tt-help search -s http://127.0.0.1:3001",
+  "          tt-help search --port 3333 -s http://127.0.0.1:3001",
+  "          tt-help search --proxy http://127.0.0.1:7890",
+  "",
   "  raw-sync --db <主库路径> --archive <归档库路径> [选项]",
-  "    从归档库向主库 raw_jobs 同步数据，按国家保持最低保有量",
+  "    归档新用户 + 按需补充任务（从归档库向主库 jobs 移入）",
   "    选项:",
   "      --db <路径>            主库路径（必填）",
   "      --archive <路径>       归档库路径（必填）",
-  "      --interval <秒>        检测间隔，默认 30",
-  "      --threshold <条>       每个国家最低保有量，默认 1000",
-  "      --batch <条>           每次搬移上限，默认 500",
+  "  --interval <秒>        检测间隔，默认 120（2分钟）",
+  "  --threshold <条>       pending 低于此值时触发补任务，默认 50",
+  "      --llm                  启用 LLM 打分模式",
+  "      --llm-min <分>         LLM 最低分阈值，默认 60",
+  "      --llm-sample <条>      LLM 每批采样数，默认 100",
   "      --once                 只执行一次后退出",
   "    示例: tt-help raw-sync --db data/result.db --archive data/result-archive.db",
   "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --once",
-  "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --interval 60 --threshold 2000",
+  "          tt-help raw-sync --db data/result.db --archive data/result-archive.db --llm --threshold 300",
   "",
   "  config  [show|set|unset|reset]",
   "    config                    查看当前配置",

package/src/lib/search-collector.js

@@ -0,0 +1,194 @@
+/**
+ * TikTok 搜索页作者采集 — DOM 滚动版
+ *
+ * 用法:
+ *   import { collectSearchAuthors } from "../lib/search-collector.js";
+ *   const ids = await collectSearchAuthors(browser, "tienda");
+ */
+
+const TAB_URLS = {
+  综合: (q) => `https://www.tiktok.com/search?q=${encodeURIComponent(q)}`,
+  视频: (q) => `https://www.tiktok.com/search/video?q=${encodeURIComponent(q)}`,
+  用户: (q) => `https://www.tiktok.com/search/user?q=${encodeURIComponent(q)}`,
+  直播: (q) => `https://www.tiktok.com/search/live?q=${encodeURIComponent(q)}`,
+  照片: (q) => `https://www.tiktok.com/search/photo?q=${encodeURIComponent(q)}`,
+};
+
+const DEFAULT_TABS = ["综合", "视频", "用户", "直播", "照片"];
+
+const SCROLL = {
+  maxRounds: 60,
+  staleThreshold: 8,
+  delayMin: 800,
+  delayMax: 1500,
+  waitAfterScroll: 4000,
+};
+const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+const randDelay = () =>
+  delay(
+    Math.floor(Math.random() * (SCROLL.delayMax - SCROLL.delayMin + 1)) +
+      SCROLL.delayMin,
+  );
+
+async function extractAuthors(page) {
+  return page.evaluate(() => {
+    const ids = new Set();
+    for (const a of document.querySelectorAll('a[href*="/@"]')) {
+      const m = (a.getAttribute("href") || "").match(/\/@([^/?]+)/);
+      if (m && m[1] !== "undefined" && m[1].length <= 30) ids.add(m[1]);
+    }
+    return [...ids];
+  });
+}
+
+async function scrollPage(page) {
+  await page.evaluate(() => {
+    const main = document.querySelector("main");
+    if (main && main.scrollHeight > main.clientHeight + 10) {
+      main.scrollBy(0, main.clientHeight * 0.8);
+    } else {
+      window.scrollBy(0, window.innerHeight * 0.8);
+    }
+  });
+}
+
+async function processOneTab(page, tabName, keyword, allIds, log) {
+  const urlFn = TAB_URLS[tabName];
+  if (!urlFn) {
+    log(`[${tabName}] 不支持`);
+    return;
+  }
+
+  const tag = `[${tabName}]`;
+  const url = urlFn(keyword);
+
+  // 首屏正常加载
+  log(`${tag} 加载...`);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
+  } catch (e) {
+    log(`${tag} 导航失败: ${e.message}`);
+    return;
+  }
+  await delay(4000);
+
+  // 首屏渲染完成后屏蔽图片/字体/媒体，加速后续滚动
+  await page.route("**/*", (route) => {
+    const t = route.request().resourceType();
+    if (t === "image" || t === "font" || t === "media") route.abort();
+    else route.continue();
+  });
+
+  const tabSet = new Set();
+  let stale = 0;
+
+  for (let round = 0; round <= SCROLL.maxRounds; round++) {
+    if (round > 0) {
+      await scrollPage(page);
+      await randDelay();
+      await delay(SCROLL.waitAfterScroll);
+    }
+
+    const ids = await extractAuthors(page);
+    let added = 0;
+    for (const id of ids) {
+      if (!tabSet.has(id)) {
+        tabSet.add(id);
+        added++;
+      }
+    }
+
+    if (round === 0) {
+      log(`${tag} 初始: ${tabSet.size}`);
+      continue;
+    }
+
+    if (added === 0) {
+      stale++;
+      if (stale >= SCROLL.staleThreshold) {
+        log(`${tag} ${SCROLL.staleThreshold}轮无新`);
+        break;
+      }
+    } else {
+      stale = 0;
+      if (round % 5 === 0 || added >= 5)
+        log(`${tag} R${round}: +${added}, 计${tabSet.size}`);
+    }
+    if (tabSet.size >= 500) break;
+  }
+
+  let added = 0;
+  for (const id of tabSet) {
+    if (!allIds.has(id)) {
+      allIds.add(id);
+      added++;
+    }
+  }
+  log(`${tag} 完成: ${tabSet.size}个, 新增${added} (累计${allIds.size})`);
+}
+
+/**
+ * @param {import("playwright").Browser} browser
+ * @param {string} keyword
+ * @param {object} [opts]
+ * @param {string[]} [opts.tabs]
+ * @param {function} [opts.log] - 日志函数，默认 console.error
+ * @returns {Promise<string[]>}
+ */
+export async function collectSearchAuthors(browser, keyword, opts = {}) {
+  const tabs = opts.tabs || DEFAULT_TABS;
+  const log = opts.log || ((...args) => console.error(...args));
+
+  log(`[collect] "${keyword}"  Tab: [${tabs.join(", ")}]`);
+
+  const context = browser.contexts()[0];
+  const existingPage = context.pages()[0];
+  const allIds = new Set();
+
+  // 并行：每个 tab 独立页面同时跑
+  const newPages = []; // 追踪所有新建页面，确保清理
+  const tasks = tabs.map(async (tabName, i) => {
+    let page = null;
+    try {
+      page = i === 0 ? existingPage : await context.newPage();
+      if (i > 0) newPages.push(page);
+      await processOneTab(page, tabName, keyword, allIds, log);
+    } catch (e) {
+      log(`[${tabName}] 异常: ${e.message}`);
+    } finally {
+      if (i > 0 && page) {
+        try {
+          await page.close();
+        } catch {}
+      }
+    }
+  });
+
+  await Promise.all(tasks);
+
+  // 兜底：确保所有新建页面都已关闭
+  for (const p of newPages) {
+    try {
+      if (!p.isClosed()) await p.close();
+    } catch {}
+  }
+
+  return [...allIds].sort();
+}
+
+/**
+ * 采集并保存到文件（便捷函数）
+ * @param {import("playwright").Browser} browser
+ * @param {string} keyword
+ * @param {string} outputFile - 输出文件路径
+ * @param {object} [opts] - 同 collectSearchAuthors
+ * @returns {Promise<string[]>}
+ */
+export async function searchAndSave(browser, keyword, outputFile, opts = {}) {
+  const fs = await import("fs");
+  const path = await import("path");
+  const ids = await collectSearchAuthors(browser, keyword, opts);
+  fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+  fs.writeFileSync(outputFile, ids.join("\n") + "\n", "utf-8");
+  return ids;
+}

package/src/lib/target-locations.js

@@ -12,6 +12,7 @@ const DEFAULT_TARGET_LOCATIONS = [
   "IT",
   "IE",
   "AT",
+  "US",
 ];
 
 const DEFAULT_TARGET_LOCATIONS_CSV = DEFAULT_TARGET_LOCATIONS.join(",");

package/src/main.js

@@ -20,6 +20,7 @@ import {
   handleTagRefresh,
 } from "./cli/tag.js";
 import { handleRawSync } from "./cli/raw-sync.js";
+import { handleSearch } from "./cli/search.js";
 
 async function main() {
   const parsed = parseArgs();
@@ -59,6 +60,8 @@ async function main() {
       return handleScoreAll(parsed);
     case "tag-refresh":
       return handleTagRefresh(parsed);
+    case "search":
+      return handleSearch(parsed);
   }
 
   const {

package/src/npm-main.js

@@ -16,6 +16,7 @@ import {
   handleTagRefresh,
 } from "./cli/tag.js";
 import { handleRawSync } from "./cli/raw-sync.js";
+import { handleSearch } from "./cli/search.js";
 
 function exitUnsupportedCommand(command) {
   console.error(
@@ -59,6 +60,8 @@ async function main() {
       return handleTagRefresh(parsed);
     case "raw-sync":
       return handleRawSync(parsed);
+    case "search":
+      return handleSearch(parsed);
   }
 
   const {