tt-help-cli-ycl 1.4.1 → 1.4.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/tag.js

@@ -580,6 +580,7 @@ export async function handleScoreAll(parsed) {
     countries,
     serverUrl,
     autoDiscover,
+    discoverCount = 10,
     port,
     proxy: cliProxy,
   } = tagScoreAll || {};
@@ -654,7 +655,7 @@ export async function handleScoreAll(parsed) {
   );
   const memHeapRecycleMb = Math.max(
     128,
-    parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "320", 10) || 320,
+    parseInt(process.env.TAG_SCOREALL_RECYCLE_HEAP_MB || "512", 10) || 512,
   );
   const recycleCooldownMs = Math.max(
     0,
@@ -687,9 +688,9 @@ export async function handleScoreAll(parsed) {
     }
   }
 
-  async function recycleCdpSession(reason) {
+  async function recycleCdpSession(reason, { switchPort = true } = {}) {
     const oldPort = cdpPort;
-    if (switchPortOnRecycle) {
+    if (switchPort && switchPortOnRecycle) {
       cdpPort = pickNextPort(cdpPort);
       cdpOpts.port = cdpPort;
       clientMeta.port = cdpPort;
@@ -722,6 +723,7 @@ export async function handleScoreAll(parsed) {
       }
       await recycleCdpSession(
         `mem rss=${rssMB.toFixed(0)}MB heap=${heapMB.toFixed(0)}MB (threshold rss=${memRssRecycleMb} heap=${memHeapRecycleMb})`,
+        { switchPort: false },
       );
       return;
     }
@@ -730,15 +732,21 @@ export async function handleScoreAll(parsed) {
       totalScored > 0 &&
       totalScored % periodicRecycleEvery === 0
     ) {
-      await recycleCdpSession(`periodic every ${totalScored} tasks`);
+      await recycleCdpSession(`periodic every ${totalScored} tasks`, {
+        switchPort: false,
+      });
     }
   }
 
   let totalScored = 0;
   let emptyRounds = 0; // 连续无任务的轮数
   let consecutive403 = 0; // 连续 HTTP 403 次数，达 3 次认为被封
-  const DISCOVER_AFTER_EMPTY = 3; // 连续 3 轮无任务时触发 discover
+  const DISCOVER_AFTER_EMPTY = 3; // 连续 3 轮无任务时触发 discover（兜底）
   const CONSECUTIVE_403_BLOCKED = 3; // 连续 3 次 403 认为账号被封
+  const DISCOVER_LOW_THRESHOLD = 5; // 剩余 new 标签 < 5 时提前触发发现
+  const DISCOVER_COOLDOWN_MS = 5 * 60 * 1000; // 发现冷却时间 5 分钟，防止频繁调用 LLM
+  let discoverPromise = null; // 正在后台执行的发现 Promise
+  let lastDiscoverTime = 0; // 上次发现完成的时间戳
 
   // 生成客户端 ID，用于服务端追踪
   const clientId = randomUUID();
@@ -763,38 +771,82 @@ export async function handleScoreAll(parsed) {
   };
   process.on("SIGINT", cleanup);
 
+  // 异步发现函数：后台为所有目标国家生成标签，不阻塞打分循环
+  async function runDiscoverInBackground() {
+    if (discoverPromise) return; // 已在执行中，不重复触发
+    const elapsed = Date.now() - lastDiscoverTime;
+    if (elapsed < DISCOVER_COOLDOWN_MS) {
+      const remainMin = Math.ceil((DISCOVER_COOLDOWN_MS - elapsed) / 60000);
+      log(`🔍 发现冷却中，还需 ${remainMin} 分钟后才能再次触发`);
+      return;
+    }
+    log(`🔍 剩余标签不足，后台为 ${targetCountries.length} 个国家生成标签...`);
+    discoverPromise = (async () => {
+      let totalInserted = 0;
+      for (const country of targetCountries) {
+        try {
+          const discRes = await fetch(
+            `${baseUrl}/api/tags/discover?country=${country}&count=${discoverCount}`,
+            { headers: buildClientHeaders(clientId, clientMeta) },
+          );
+          const discData = await discRes.json();
+          if (discData.added) {
+            totalInserted += discData.added;
+            log(`  ${country}: 新增 ${discData.added} 个`);
+          }
+        } catch (e) {
+          log(`  ${country}: 请求失败 (${e.message})`);
+        }
+      }
+      log(`  ✅ 发现完成，共入库 ${totalInserted} 个新标签`);
+      lastDiscoverTime = Date.now(); // 记录完成时间，开始冷却
+      discoverPromise = null;
+    })();
+  }
+
   try {
     while (true) {
       try {
-        // 从服务端取下一个 new 标签
+        // 从服务端取下一个 new 标签（同时查剩余数量）
         const tagsRes = await fetch(`${baseUrl}/api/tags?status=new&limit=1`, {
           headers: buildClientHeaders(clientId, clientMeta),
         });
         const tagsData = await tagsRes.json();
+
+        // 提前触发发现：剩余 new 标签低于阈值且不在冷却期时后台生成
+        if (
+          autoDiscover &&
+          !discoverPromise &&
+          Date.now() - lastDiscoverTime >= DISCOVER_COOLDOWN_MS &&
+          (tagsData.total || 0) < DISCOVER_LOW_THRESHOLD
+        ) {
+          runDiscoverInBackground();
+        }
+
         if (!tagsData.tags || tagsData.tags.length === 0) {
           emptyRounds++;
 
-          // 自动发现：连续 N 轮无任务时自动生成标签
+          // 兜底发现：连续 N 轮无任务（提前发现可能没生效）
           if (autoDiscover && emptyRounds >= DISCOVER_AFTER_EMPTY) {
-            log(
-              `🔍 连续 ${emptyRounds} 轮无待打分标签，自动为 ${targetCountries.length} 个国家生成标签...`,
-            );
-            for (const country of targetCountries) {
-              try {
-                const discRes = await fetch(
-                  `${baseUrl}/api/tags/discover?country=${country}&count=5`,
-                  { headers: buildClientHeaders(clientId, clientMeta) },
-                );
-                const discData = await discRes.json();
-                if (discData.inserted) {
-                  log(`  ${country}: 新增 ${discData.inserted} 个`);
-                }
-              } catch (e) {
-                log(`  ${country}: 请求失败 (${e.message})`);
-              }
+            // 等后台发现完成（如果正在跑）
+            if (discoverPromise) {
+              log(`  ⏳ 等待后台发现完成...`);
+              await discoverPromise;
+            } else if (Date.now() - lastDiscoverTime >= DISCOVER_COOLDOWN_MS) {
+              // 冷却期已过，触发一次同步发现
+              log(`🔍 连续 ${emptyRounds} 轮无待打分标签，同步生成标签...`);
+              await runDiscoverInBackground();
+              await discoverPromise;
+            } else {
+              const remainMin = Math.ceil(
+                (DISCOVER_COOLDOWN_MS - (Date.now() - lastDiscoverTime)) /
+                  60000,
+              );
+              log(
+                `🔍 连续 ${emptyRounds} 轮空，但发现冷却中（还需 ${remainMin} 分钟），继续等待...`,
+              );
             }
-            emptyRounds = 0; // 重置计数器
-            // 等 3 秒让服务端处理完
+            emptyRounds = 0;
             await new Promise((r) => setTimeout(r, 3000));
             continue;
           }

package/src/lib/args.js

@@ -740,6 +740,7 @@ function parseTagArgs(args) {
   let isScore = false;
   let isScoreAll = false;
   let scoreAllPort = 7222;
+  let discoverCountScoreAll = 10;
   let scoreProxy = null;
   let scoreTag = null;
   let scoreCountries = null;
@@ -781,6 +782,8 @@ function parseTagArgs(args) {
       discoverCount = parseInt(args[++i]) || 4;
     } else if (arg === "--auto-discover") {
       autoDiscover = true;
+    } else if (arg === "--discover-count") {
+      discoverCountScoreAll = parseInt(args[++i]) || 10;
     } else if (arg === "--countries") {
       scoreCountries = args[++i]
         .split(",")
@@ -879,6 +882,7 @@ function parseTagArgs(args) {
         countries: scoreCountries,
         serverUrl,
         autoDiscover,
+        discoverCount: discoverCountScoreAll,
         port: scoreAllPort,
         proxy: scoreProxy,
       },

package/src/lib/constants.js

@@ -261,13 +261,14 @@ const HELP_TEXT = [
   "      --countries <CSV>         目标国家，逗号分隔（默认 13 个欧洲国家）",
   "      -s, --server <URL>        服务端地址（默认 http://127.0.0.1:3000）",
   "      --port <N>                起始 CDP 端口，端口池默认为 N~N+9（默认 7222）",
-  "      --discover                开启自动发现：连续 3 轮无新标签时自动生成",
+  "      --discover                开启自动发现：剩余 new 标签 < 5 时后台异步生成，连续 3 轮空兜底",
+  "      --discover-count <N>      每次发现每个国家生成标签数（默认 10，上限 20）",
   "    行为:",
   "      连续 3 次 HTTP 403 → 判断账号被封 → 等 2 分钟后自动切换到池中另一端口",
   "    环境变量:",
   "      TAG_SCOREALL_PORT_POOL_SIZE=N  端口池大小（默认 10）",
   "      TAG_SCOREALL_RECYCLE_RSS_MB=N  内存回收阈值 RSS MB（默认 900）",
-  "      TAG_SCOREALL_RECYCLE_HEAP_MB=N 内存回收阈值 Heap MB（默认 320）",
+  "      TAG_SCOREALL_RECYCLE_HEAP_MB=N 内存回收阈值 Heap MB（默认 512）",
   "    示例: tt-help tag score-all",
   "          tt-help tag score-all --countries ES -s http://127.0.0.1:3001",
   "          tt-help tag score-all --port 9222 -s http://127.0.0.1:3001 --discover",

package/src/lib/tag-discover.js

@@ -66,16 +66,26 @@ async function discoverTagsForCountryCli(
 
   // 从服务端获取历史 tag
   const history = await fetchTagHistory(serverUrl, country);
-
-  // 使用统一的 prompt 组装
-  const prompt = buildDiscoverPrompt(country, count, history, userPrompt);
+  const { productive, dead, allExisting: existingSet } = history;
+  const existingArr = Array.isArray(existingSet) ? existingSet : [];
+
+  // 使用统一的 prompt 组装（不传 allExisting，由应用层去重）
+  const prompt = buildDiscoverPrompt(
+    country,
+    count,
+    { productive, dead },
+    userPrompt,
+  );
 
   process.stderr.write(
-    `  [LLM] 正在生成 ${count} 个标签 (国家: ${country}, 语言: ${getLang(country)})...\n`,
+    `  [LLM] 正在生成标签 (国家: ${country}, 语言: ${getLang(country)}, 目标: ${count} 个)...\n`,
   );
   const content = await callLLM(prompt);
   const tags = parseTagsFromResponse(content);
-  const unique = [...new Set(tags)].slice(0, count);
+  // 两阶段筛选：先内部去重，再过滤已存在，最后取 count 个
+  const unique = [...new Set(tags)]
+    .filter((t) => !existingArr.includes(t))
+    .slice(0, count);
 
   process.stderr.write(
     `  [LLM] 生成 ${unique.length} 个标签: ${unique.join(", ")}\n`,

package/src/watch/db-tags.js

@@ -22,11 +22,19 @@ export function insertTag(tag, countries, source = "llm") {
     return { inserted: false, error: "invalid tag" };
   }
   try {
+    // ON CONFLICT: 合并 countries 数组，避免跨国家共享 tag 时信息丢失
     const result = db
       .prepare(
-        "INSERT OR IGNORE INTO tags (tag, countries, source) VALUES (?, ?, ?)",
+        `INSERT INTO tags (tag, countries, source) VALUES (@tag, @countries, @source)
+         ON CONFLICT(tag) DO UPDATE SET countries = (
+           SELECT json_group_array(DISTINCT value) FROM (
+             SELECT value FROM json_each(countries)
+             UNION
+             SELECT value FROM json_each(@countries)
+           )
+         )`,
       )
-      .run(normalized, JSON.stringify(countries), source);
+      .run({ tag: normalized, countries: JSON.stringify(countries), source });
     return { inserted: result.changes > 0, tag: normalized };
   } catch (e) {
     return { inserted: false, error: e.message };

package/src/watch/server.js

@@ -1169,9 +1169,18 @@ export function startWatchServer(
           tags = store.getAllTags(limit, offset, upperCountry);
         }
 
-        // 获取总数（带国家过滤）
+        // 获取总数（带国家过滤，按 status 返回对应计数）
         const stats = store.getTagStats(upperCountry);
-        const total = stats ? stats.total : tags.length;
+        let total;
+        if (stats) {
+          if (status === "new") total = stats.new;
+          else if (status === "productive") total = stats.productive;
+          else if (status === "dead") total = stats.dead;
+          else if (status === "scoring") total = stats.scoring;
+          else total = stats.total;
+        } else {
+          total = tags.length;
+        }
         sendJSON(res, 200, { tags, total, offset, limit });
         return;
       }

package/src/watch/tag-service.js

@@ -45,7 +45,7 @@ export async function callLLM(prompt) {
       model: LLM_MODEL,
       messages: [{ role: "user", content: prompt }],
       max_tokens: 1024,
-      temperature: 0.3,
+      temperature: 0.7,
     }),
   });
 
@@ -182,13 +182,6 @@ export function buildDiscoverPrompt(
       ? `\nWhy previous tags failed: ${errorPatterns.join("; ")}. Avoid tags likely to have same issues.`
       : "";
 
-  // 已存在的所有 tag（防止重复生成）
-  const allExisting = history.allExisting || [];
-  const existingHint =
-    allExisting.length > 0
-      ? `\nTags already in database (DO NOT generate these again): ${allExisting.slice(0, 50).join(", ")}.`
-      : "";
-
   const userHint = userPrompt
     ? `\nAdditional focus: ${userPrompt}. Generate tags specifically for this niche.`
     : "";
@@ -244,7 +237,7 @@ Think about the MOST COMMON hashtags that real sellers in ${country} use on TikT
 - Common seller identities (shop, boutique, small business, creator...)
 - Well-known community tags (support local, marketplace, second round...)
 
-${qualityWarning}${productiveHint}${deadHint}${errorHint}${existingHint}${userHint}${strategyReview}
+${qualityWarning}${productiveHint}${deadHint}${errorHint}${userHint}${strategyReview}
 
 ## Quality check before responding
 
@@ -255,15 +248,16 @@ For each tag you consider, ask yourself: "Have I actually seen this hashtag used
 Return ONLY a JSON object with two fields:
 {
   "strategy": "Your analysis (2-3 sentences). Acknowledge what worked/failed before and explain why your chosen tags are likely real and commonly used.",
-  "tags": ["tag1", "tag2", "tag3", "tag4"]
+  "tags": ["tag1", "tag2", "tag3", "tag4", "..."]
 }
 
 Rules:
+- Brainstorm broadly — generate 12-15 distinct, diverse tag ideas covering different categories (selling verbs, product types, community tags, marketplace terms, seasonal trends)
+- We will filter duplicates and existing tags on our end, so explore freely
 - QUALITY over creativity — 3 real, commonly used tags beat 4 invented ones
 - NEVER invent compound words or translate concepts into hashtags
 - Prefer simpler, more common tags over hyper-specific ones
-- Look for tags with broad appeal (many potential posters)
-- Do NOT generate tags that already exist`;
+- Look for tags with broad appeal (many potential posters)`;
 }
 
 // ====== discover: 单国家标签发现 ======
@@ -281,10 +275,10 @@ export async function discoverTagsForCountry(
   // 读取历史打分记录
   const productive = store.getTagsByCountry(country, 50);
   const dead = store.getDeadTags(country);
-  // 获取该国所有已存在的 tag 名（防止重复生成）
+  // 获取该国所有已存在的 tag 名（应用层去重用）
   const allTags = store.getTagsByCountry(country, 0);
-  const allExisting = allTags.map((t) => t.tag);
-  const history = { productive, dead, allExisting };
+  const allExisting = new Set(allTags.map((t) => t.tag));
+  const history = { productive, dead };
 
   // 读取历史策略效果（自学习）
   const { getDiscoverRound, getDiscoverEffectiveness } =
@@ -305,9 +299,11 @@ export async function discoverTagsForCountry(
   );
   const content = await callLLM(prompt);
 
-  // 解析响应：提取 strategy 和 tags
+  // 解析响应：提取 strategy 和 tags，筛选掉数据库中已存在的
   const { strategy, tags } = parseDiscoverResponse(content);
-  const unique = [...new Set(tags)].slice(0, count);
+  const unique = [...new Set(tags)]
+    .filter((t) => !allExisting.has(t))
+    .slice(0, count);
 
   console.error(
     `[tag-service] LLM strategy: ${strategy?.substring(0, 150) || "(none)"}`,