tt-help-cli-ycl 1.3.84 → 1.3.86

package/src/cli/tag.js

@@ -0,0 +1,736 @@
+import { writeFileSync } from "fs";
+import { fetchTagData, enrichVideosWithLocation } from "../lib/tag-fetcher.js";
+import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
+import {
+  DEFAULT_TARGET_LOCATIONS,
+  isLocationInList,
+} from "../lib/target-locations.js";
+import { discoverTags, recordProductiveTag } from "../lib/tag-discover.js";
+import { server as cfgServer } from "../lib/constants.js";
+
+const ALL_COUNTRIES = DEFAULT_TARGET_LOCATIONS;
+const DEFAULT_SERVER = cfgServer || "http://127.0.0.1:3000";
+
+async function pushToServer(serverUrl, filteredAuthors, videos) {
+  const users = filteredAuthors.map((author) => {
+    const video = videos.find((v) => v.authorUniqueId === author);
+    return {
+      uniqueId: author,
+      sources: ["tag"],
+      locationCreated: video?.locationCreated || null,
+    };
+  });
+
+  const res = await fetch(`${serverUrl}/api/raw-users`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ users }),
+  });
+  const data = await res.json();
+  process.stderr.write(
+    `  已推送 ${data.added} 个用户到 jobs_base (来源: tag, 跳过: ${data.skipped})\n`,
+  );
+  return data;
+}
+
+// 共享算分逻辑（handleScore / handleScoreAll 共用）
+function calcScore(authorCount, matchedAuthors, filteredVideoCount) {
+  const densityScore =
+    authorCount > 0 ? (matchedAuthors / authorCount) * 50 : 0;
+  const absoluteScore = Math.min(matchedAuthors / 10, 1) * 30;
+  const videoBonus =
+    filteredVideoCount > 0 ? Math.min(filteredVideoCount / 20, 1) * 20 : 0;
+  const score = Math.round(
+    Math.min(densityScore + absoluteScore + videoBonus, 100),
+  );
+  let status;
+  if (score >= 70) status = "productive";
+  else if (score >= 50) status = "scored";
+  else if (score < 10) status = "dead";
+  else status = "scored";
+  return { score, status };
+}
+
+// 通用的过滤 + 算分流程（handleScore / handleScoreAll 共用）
+function applyFilterAndScore(videos, targetCountries, result) {
+  const filtered = videos.filter((v) =>
+    isLocationInList(v.locationCreated, targetCountries),
+  );
+  const matchedAuthorSet = new Set(
+    filtered.map((v) => v.authorUniqueId).filter(Boolean),
+  );
+  result.matchedAuthors = matchedAuthorSet.size;
+
+  const countryStats = {};
+  for (const v of filtered) {
+    if (v.locationCreated)
+      countryStats[v.locationCreated] =
+        (countryStats[v.locationCreated] || 0) + 1;
+  }
+  result.matchedCountries = Object.entries(countryStats).map(([c, n]) => ({
+    c,
+    n,
+  }));
+
+  const { score, status } = calcScore(
+    result.authorCount,
+    result.matchedAuthors,
+    filtered.length,
+  );
+  result.score = score;
+  result.status = status;
+
+  return { filtered, matchedAuthorSet };
+}
+
+async function processTag(
+  tag,
+  index,
+  total,
+  { enrich, targetLocations, noFilter, serverUrl, recordTags },
+) {
+  const prefix = total > 1 ? `[${index + 1}/${total}]` : "";
+  process.stderr.write(`${prefix} 正在获取 #${tag} ... `);
+
+  try {
+    const result = await fetchTagData(tag, {
+      onProgress: ({ videos, authors }) => {
+        process.stderr.write(
+          `\r${prefix} #${tag}: ${videos} 视频, ${authors} 作者`,
+        );
+      },
+    });
+
+    process.stderr.write(
+      `\r${prefix} #${tag}: ${result.videoCount} 视频, ${result.uniqueAuthorCount} 作者`,
+    );
+
+    let videos = result.videos;
+    let filteredAuthors = result.uniqueAuthors;
+
+    if (enrich) {
+      const enrichMode = enrich === true ? "videos" : enrich;
+      if (noFilter) {
+        process.stderr.write(
+          `\n  正在补充国家信息 (${enrichMode} 模式，不过滤)...\n`,
+        );
+      } else {
+        process.stderr.write(
+          `\n  正在补充国家信息 (${enrichMode} 模式，目标: ${targetLocations.join(",")})...\n`,
+        );
+      }
+
+      const enriched = await enrichVideosWithLocation(videos, {
+        mode: enrichMode,
+        onProgress: ({ done, total, current, locationCreated }) => {
+          const label = enrichMode === "users" ? `@${current}` : current;
+          const loc = locationCreated || "-";
+          const hit =
+            locationCreated &&
+            isLocationInList(locationCreated, targetLocations);
+          process.stderr.write(
+            `\r  [${done}/${total}] ${label} → ${loc}${hit ? " ✓" : ""}`,
+          );
+        },
+      });
+      videos = enriched.videos;
+      process.stderr.write("\n");
+
+      if (!noFilter) {
+        const before = videos.length;
+        videos = videos.filter((v) =>
+          isLocationInList(v.locationCreated, targetLocations),
+        );
+        const filteredAuthorsSet = new Set(
+          videos.map((v) => v.authorUniqueId).filter(Boolean),
+        );
+        filteredAuthors = [...filteredAuthorsSet];
+        process.stderr.write(
+          `  过滤后: ${before} → ${videos.length} 视频, ${filteredAuthors.length} 作者\n`,
+        );
+      }
+    }
+
+    if (serverUrl && filteredAuthors.length > 0) {
+      const pushResult = await pushToServer(serverUrl, filteredAuthors, videos);
+      if (recordTags && pushResult.added > 0) {
+        const countries = [
+          ...new Set(videos.map((v) => v.locationCreated).filter(Boolean)),
+        ];
+        for (const c of countries) {
+          recordProductiveTag(tag, c, pushResult.added);
+        }
+        process.stderr.write(
+          `  已记录标签 #${tag} (${countries.join(",")}, ${pushResult.added} 用户)\n`,
+        );
+      }
+    }
+
+    process.stderr.write(
+      `\r${prefix} #${tag}: ${videos.length} 视频, ${filteredAuthors.length} 作者\n`,
+    );
+
+    return {
+      tag,
+      totalPosts: result.totalPosts,
+      videoCount: videos.length,
+      authorCount: filteredAuthors.length,
+      authors: filteredAuthors,
+      videos,
+    };
+  } catch (err) {
+    process.stderr.write(`\r${prefix} #${tag}: 失败 - ${err.message}\n`);
+    return { tag, error: err.message };
+  }
+}
+
+export async function handleDiscover(parsed) {
+  const { tagDiscover } = parsed;
+  let { countries, count = 4, prompt, serverUrl } = tagDiscover || {};
+
+  // 支持 'all' 展开为全部目标国家
+  if (
+    countries &&
+    countries.length === 1 &&
+    countries[0].toUpperCase() === "ALL"
+  ) {
+    countries = ALL_COUNTRIES;
+  }
+
+  if (!countries || countries.length === 0) {
+    console.error(
+      "用法: tt-help tag discover <国家|all> [国家...] [--count <n>] [--prompt <文本>] [-s <服务端>]",
+    );
+    console.error("");
+    console.error("示例:");
+    console.error(
+      "  tt-help tag discover all --count 10             # 为全部 13 个国家各生成 10 个标签",
+    );
+    console.error(
+      "  tt-help tag discover ES                          # 为西班牙生成 4 个标签",
+    );
+    console.error(
+      "  tt-help tag discover ES FR --count 5             # 各生成 5 个",
+    );
+    console.error(
+      '  tt-help tag discover DE -p "卖手工首饰"           # 带用户提示',
+    );
+    console.error(
+      "  tt-help tag discover ES -s http://127.0.0.1:3001 # 指定服务端",
+    );
+    process.exit(1);
+  }
+
+  const baseUrl = serverUrl || "http://127.0.0.1:3000";
+
+  for (const country of countries) {
+    const params = new URLSearchParams({ country, count: String(count) });
+    if (prompt) params.set("prompt", prompt);
+
+    try {
+      const res = await fetch(`${baseUrl}/api/tags/discover?${params}`);
+      const data = await res.json();
+      if (data.error) {
+        console.error(`${country}: 错误 - ${data.error}`);
+      } else {
+        console.log(
+          `${country}: 新增 ${data.added}/${data.total} 个标签: ${(data.tags || []).join(", ")}`,
+        );
+      }
+    } catch (e) {
+      console.error(`${country}: 请求失败 - ${e.message}`);
+    }
+  }
+}
+
+export async function handleScore(parsed) {
+  const { tagScore } = parsed;
+  const { tag, countries, serverUrl } = tagScore || {};
+
+  if (!tag) {
+    console.error(
+      "用法: tt-help tag score <tag名称> [--countries <CSV>] [-s <服务端>]",
+    );
+    console.error("");
+    console.error("示例:");
+    console.error(
+      "  tt-help tag score ventas                                        # 打分单个标签",
+    );
+    console.error(
+      "  tt-help tag score ventas --countries ES,FR                      # 指定目标国家",
+    );
+    console.error(
+      "  tt-help tag score ventas -s http://127.0.0.1:3001              # 指定服务端",
+    );
+    process.exit(1);
+  }
+
+  const baseUrl = serverUrl || DEFAULT_SERVER;
+  const targetCountries = countries || [
+    "ES",
+    "FR",
+    "DE",
+    "PT",
+    "IT",
+    "NL",
+    "BE",
+    "AT",
+    "IE",
+    "PL",
+    "CZ",
+    "GR",
+    "HU",
+  ];
+
+  const log = (...args) => process.stderr.write(args.join(" ") + "\n");
+  const startTime = Date.now();
+
+  log("");
+  log("========================================");
+  log(`  标签打分: #${tag}`);
+  log(`  目标国家: ${targetCountries.join(", ")}`);
+  log(`  服务端:   ${baseUrl}`);
+  log("  模式:     客户端本地打分（Playwright → enrich → 算分 → 上报）");
+  log("========================================");
+  log("");
+
+  const result = {
+    tag,
+    status: "error",
+    score: 0,
+    totalPosts: 0,
+    authorCount: 0,
+    matchedAuthors: 0,
+    matchedCountries: [],
+    pushedUsers: 0,
+    error: null,
+  };
+
+  try {
+    // Step 1: 打开标签页抓取视频
+    log("Step 1/4: 打开 TikTok 标签页抓取视频...");
+    const tagResult = await fetchTagData(tag, {
+      onProgress: ({ videos, authors }) => {
+        process.stderr.write(`\r  已抓取: ${videos} 视频, ${authors} 作者`);
+      },
+    });
+    log(
+      `\r  完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者`,
+    );
+    result.totalPosts = tagResult.totalPosts || 0;
+    result.authorCount = tagResult.uniqueAuthorCount || 0;
+
+    let videos = tagResult.videos;
+    if (!videos || videos.length === 0) {
+      log("  ⚠️ 没有视频，标记为 dead");
+      result.status = "dead";
+      result.error = "no videos found";
+      await reportToServer(baseUrl, result);
+      return;
+    }
+
+    // Step 2/4: 通过 TikTokScraper.getVideoInfo 逐个视频获取国家
+    log(`Step 2/4: 补充国家信息 (${videos.length} 个视频)...`);
+    const enriched = await enrichVideosWithLocation(videos, {
+      mode: "videos",
+      onProgress: ({ done, total, current, locationCreated }) => {
+        if (done % 10 === 0 || done === total) {
+          process.stderr.write(
+            `\r  [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}`,
+          );
+        }
+      },
+    });
+    videos = enriched.videos;
+    const withLoc = videos.filter((v) => v.locationCreated).length;
+    log(`\r  完成: ${withLoc}/${videos.length} 个视频有国家信息`);
+
+    // Step 3/4: 过滤 + 算分
+    log("Step 3/4: 过滤目标国家 + 计算分数...");
+    const { matchedAuthorSet } = applyFilterAndScore(
+      videos,
+      targetCountries,
+      result,
+    );
+
+    log(
+      `  算分: ${result.score}/100 → ${result.status} (匹配 ${result.matchedAuthors}/${result.authorCount} 作者)`,
+    );
+    if (result.matchedCountries.length > 0) {
+      log(
+        `  国家: ${result.matchedCountries.map((c) => `${c.c}:${c.n}`).join(", ")}`,
+      );
+    }
+
+    // Step 4/4: 推送用户 + 上报结果
+    log("Step 4/4: 推送用户到服务端 + 上报打分结果...");
+    if (result.matchedAuthors > 0) {
+      const pushResult = await pushToServer(
+        baseUrl,
+        [...matchedAuthorSet],
+        videos,
+      );
+      result.pushedUsers = pushResult.added || 0;
+    }
+    await reportToServer(baseUrl, result);
+  } catch (e) {
+    log(`❌ 错误: ${e.message}`);
+    result.error = e.message;
+    try {
+      await reportToServer(baseUrl, result);
+    } catch {}
+    return;
+  }
+
+  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  const icon =
+    result.status === "productive"
+      ? "🟢"
+      : result.status === "scored"
+        ? "🟡"
+        : result.status === "dead"
+          ? "🔴"
+          : "⚪";
+  log("");
+  log("----------------------------------------");
+  log(`  ${icon} 打分完成 (${elapsed}s)`);
+  log(`  状态: ${result.status}  分数: ${result.score}/100`);
+  log(
+    `  视频作者: ${result.authorCount}  匹配: ${result.matchedAuthors}  推送: ${result.pushedUsers}`,
+  );
+  log("----------------------------------------");
+
+  console.log(JSON.stringify(result, null, 2));
+}
+
+async function reportToServer(baseUrl, result) {
+  try {
+    const res = await fetch(`${baseUrl}/api/tags/score-result`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(result),
+    });
+    const data = await res.json();
+    if (!data.ok) process.stderr.write(`  ⚠️ 上报失败: ${data.error}\n`);
+  } catch (e) {
+    process.stderr.write(`  ⚠️ 上报请求失败: ${e.message}\n`);
+  }
+}
+
+export async function handleScoreAll(parsed) {
+  const { tagScoreAll } = parsed;
+  let { countries, serverUrl } = tagScoreAll || {};
+
+  const baseUrl = serverUrl || DEFAULT_SERVER;
+  const targetCountries = countries || [
+    "ES",
+    "FR",
+    "DE",
+    "PT",
+    "IT",
+    "NL",
+    "BE",
+    "AT",
+    "IE",
+    "PL",
+    "CZ",
+    "GR",
+    "HU",
+  ];
+
+  const log = (...args) => process.stderr.write(args.join(" ") + "\n");
+
+  log("");
+  log("========================================");
+  log("  自动循环打分模式（客户端本地执行）");
+  log(`  目标国家: ${targetCountries.join(", ")}`);
+  log(`  服务端:   ${baseUrl}`);
+  log("  流程: 从服务端拉 tag → 本地 Playwright 抓取 → enrich → 算分 → 上报");
+  log("  每个标签约 1-2 分钟");
+  log("========================================");
+  log("");
+
+  let totalScored = 0;
+  let totalNew = null;
+
+  // 复用 TikTokScraper 实例，避免每次 enrich 都启动/关闭 headless 浏览器
+  const enrichScraper = new TikTokScraper({ poolSize: 3 });
+  await enrichScraper.init();
+  log("✅ TikTokScraper 已就绪 (enrich 复用)");
+  log("");
+
+  try {
+    while (true) {
+      // 查剩余数量
+      if (totalNew === null) {
+        try {
+          const statsRes = await fetch(
+            `${baseUrl}/api/tags?status=new&limit=1000`,
+          );
+          const statsData = await statsRes.json();
+          totalNew = statsData.total || 0;
+          log(`📋 待打分标签: ${totalNew} 个`);
+          log("");
+        } catch (e) {
+          log(`⚠️ 无法连接服务端: ${e.message}`);
+          break;
+        }
+      }
+
+      // 从服务端取下一个 new 标签
+      const tagsRes = await fetch(`${baseUrl}/api/tags?status=new&limit=1`);
+      const tagsData = await tagsRes.json();
+      if (!tagsData.tags || tagsData.tags.length === 0) {
+        log(`  ⏳ 暂无待打分标签，10 秒后重试...`);
+        totalNew = null; // 重置计数，下次新标签到达时重新查询
+        await new Promise((r) => setTimeout(r, 10000));
+        continue;
+      }
+
+      const tag = tagsData.tags[0].tag.replace(/^#+/, "").trim().toLowerCase();
+      const startTime = Date.now();
+
+      log(`[${totalScored + 1}/${totalNew || "?"}] 正在打分 #${tag} ...`);
+
+      const result = {
+        tag,
+        status: "error",
+        score: 0,
+        totalPosts: 0,
+        authorCount: 0,
+        matchedAuthors: 0,
+        matchedCountries: [],
+        pushedUsers: 0,
+        error: null,
+      };
+
+      try {
+        // 锁定 tag
+        const claimRes = await fetch(`${baseUrl}/api/tags/claim`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ tag }),
+        });
+        const claimData = await claimRes.json();
+        if (!claimData.ok) {
+          // already claimed: 其他机器抢先了，跳过不标 dead
+          if (claimData.error && claimData.error.includes("already claimed")) {
+            log(`  ⏭️ 已被其他客户端锁定，跳过`);
+            continue;
+          }
+          log(`  ⚠️ 无法锁定 (${claimData.error})，标记为 dead 并跳过`);
+          result.error = claimData.error;
+          result.status = "dead";
+          await reportToServer(baseUrl, result);
+          totalScored++;
+          continue;
+        }
+
+        // 抓取视频
+        log(`  抓取 TikTok 标签页...`);
+        const tagResult = await fetchTagData(tag, {
+          onProgress: ({ videos, authors }) => {
+            process.stderr.write(`\r  抓取中: ${videos} 视频, ${authors} 作者`);
+          },
+        });
+        log(
+          `\r  完成: ${tagResult.videoCount} 视频, ${tagResult.uniqueAuthorCount} 作者`,
+        );
+
+        result.totalPosts = tagResult.totalPosts || 0;
+        result.authorCount = tagResult.uniqueAuthorCount || 0;
+        let videos = tagResult.videos;
+
+        if (!videos || videos.length === 0) {
+          log("  ⚠️ 无视频，标记 dead");
+          result.status = "dead";
+          result.error = "no videos found";
+          await reportToServer(baseUrl, result);
+          totalScored++;
+          continue;
+        }
+
+        // enrich: 逐个视频查 view-source 获取国家
+        log(`  补充国家信息...`);
+        const enriched = await enrichVideosWithLocation(videos, {
+          mode: "videos",
+          existingScraper: enrichScraper,
+          onProgress: ({ done, total, current, locationCreated }) => {
+            if (done % 10 === 0 || done === total) {
+              process.stderr.write(
+                `\r    [${done}/${total}] ${current.split("/").pop().slice(0, 20)} → ${locationCreated || "-"}`,
+              );
+            }
+          },
+        });
+        videos = enriched.videos;
+        const withLoc = videos.filter((v) => v.locationCreated).length;
+        log(`  完成: ${withLoc}/${videos.length} 个视频有国家信息`);
+
+        // 过滤 + 算分 (共用函数)
+        const { matchedAuthorSet } = applyFilterAndScore(
+          videos,
+          targetCountries,
+          result,
+        );
+
+        // 推送用户
+        if (result.matchedAuthors > 0) {
+          const pushResult = await pushToServer(
+            baseUrl,
+            [...matchedAuthorSet],
+            videos,
+          );
+          result.pushedUsers = pushResult.added || 0;
+        }
+
+        // 上报结果
+        await reportToServer(baseUrl, result);
+
+        totalScored++;
+        const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+        const icon =
+          result.status === "productive"
+            ? "🟢"
+            : result.status === "scored"
+              ? "🟡"
+              : result.status === "dead"
+                ? "🔴"
+                : "⚪";
+        const mc = result.matchedCountries
+          .map((c) => `${c.c}:${c.n}`)
+          .join(" ");
+        log(
+          `  ${icon} ${result.status} score=${result.score} authors=${result.authorCount} matched=${result.matchedAuthors} (${elapsed}s)`,
+        );
+        if (mc) log(`     国家: ${mc}`);
+        log(`     剩余: ~${Math.max(0, (totalNew || 0) - totalScored)} 个`);
+        log("");
+      } catch (e) {
+        log(`  ❌ 失败: ${e.message}`);
+        result.error = e.message;
+        try {
+          await reportToServer(baseUrl, result);
+        } catch {}
+        totalScored++;
+      }
+    }
+  } finally {
+    await enrichScraper.close();
+    log("✅ TikTokScraper 已关闭");
+  }
+}
+
+export async function handleTag(parsed) {
+  const { tagTags } = parsed;
+
+  if (!tagTags || tagTags.length === 0) {
+    console.error(
+      "用法: tt-help tag <tag名称> [...] [-s <服务端>] [--enrich] [--locations <国家>] [--no-filter] [--discover]",
+    );
+    console.error("");
+    console.error("选项:");
+    console.error("  -o, --output <file>       输出到 JSON 文件");
+    console.error("  -s, --server <url>        推送到 watch 服务端");
+    console.error(
+      "  --enrich [users|videos]   补充国家/地区信息（默认 videos）",
+    );
+    console.error(
+      "  --locations <国家代码>      目标国家，逗号分隔（默认欧洲13国）",
+    );
+    console.error("  --no-filter               不过滤国家");
+    console.error(
+      "  --discover [数量]          LLM 自动发现标签 + 记录有效标签",
+    );
+    console.error("  --authors-only            只输出作者列表");
+    console.error("  --videos-only             只输出视频列表");
+    console.error("");
+    console.error("示例:");
+    console.error("  tt-help tag ventas --enrich -s http://127.0.0.1:3000");
+    console.error("  tt-help tag --discover --enrich -s http://127.0.0.1:3000");
+    console.error(
+      "  tt-help tag --discover 20 --locations ES,FR -s http://127.0.0.1:3000",
+    );
+    process.exit(1);
+  }
+
+  const {
+    tags,
+    outputFile,
+    authorsOnly,
+    videosOnly,
+    enrich,
+    locations,
+    noFilter,
+    serverUrl,
+    discover,
+  } = tagTags;
+
+  const targetLocations = locations
+    ? locations
+        .split(",")
+        .map((s) => s.trim().toUpperCase())
+        .filter(Boolean)
+    : DEFAULT_TARGET_LOCATIONS;
+
+  const autoEnrich = enrich || !!discover;
+
+  let finalTags = tags || [];
+
+  if (discover) {
+    const discoverCount = typeof discover === "number" ? discover : 10;
+    const generatedTags = await discoverTags(targetLocations, {
+      count: discoverCount,
+    });
+    finalTags = [...new Set([...finalTags, ...generatedTags])];
+    process.stderr.write(`  共 ${finalTags.length} 个标签待处理\n\n`);
+  }
+
+  if (finalTags.length === 0) {
+    console.error("没有标签可处理，请提供标签或使用 --discover");
+    process.exit(1);
+  }
+
+  const allResults = [];
+
+  for (let i = 0; i < finalTags.length; i++) {
+    const result = await processTag(finalTags[i], i, finalTags.length, {
+      enrich: autoEnrich,
+      targetLocations,
+      noFilter,
+      serverUrl,
+      recordTags: !!discover,
+    });
+
+    const output = { tag: result.tag };
+    if (result.error) {
+      output.error = result.error;
+    } else if (authorsOnly) {
+      output.authors = result.authors;
+    } else if (videosOnly) {
+      output.videos = result.videos;
+    } else {
+      Object.assign(output, {
+        totalPosts: result.totalPosts,
+        videoCount: result.videoCount,
+        authorCount: result.authorCount,
+        authors: result.authors,
+        videos: result.videos,
+      });
+    }
+    allResults.push(output);
+  }
+
+  const json = JSON.stringify(
+    allResults.length === 1 ? allResults[0] : allResults,
+    null,
+    2,
+  );
+
+  if (outputFile) {
+    writeFileSync(outputFile, json, "utf-8");
+    process.stderr.write(`\n已保存到 ${outputFile}\n`);
+  } else {
+    console.log(json);
+  }
+}