tt-help-cli-ycl 1.4.20 → 1.4.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.20",
+  "version": "1.4.22",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/comments.js

@@ -35,6 +35,7 @@ import {
 } from "../scraper/modules/captcha-handler.js";
 import { detectModals, closeAllModals } from "../lib/modal-handler.js";
 import { getMacOrUuid } from "../lib/mac-or-uuid.js";
+import { parseVideoInfo } from "../lib/parse-ssr.mjs";
 
 // ────────────────────────────────────────
 // Section 1: 常量 & 配置默认值
@@ -492,6 +493,49 @@ function extractVideoInfo() {
   return result;
 }
 
+/**
+ * 从视频页 SSR 数据中提取 challenges（标签/hashtags）
+ * @param {Page} page - Playwright page
+ * @returns {Promise<{ tagNames: string[], challenges: Array }>}
+ */
+async function extractVideoTags(page) {
+  try {
+    const html = await page.evaluate(() => document.documentElement.outerHTML);
+    const videoInfo = parseVideoInfo(html);
+    if (
+      !videoInfo ||
+      !videoInfo.challenges ||
+      videoInfo.challenges.length === 0
+    ) {
+      return { tagNames: [], challenges: [] };
+    }
+    const tagNames = videoInfo.challenges.map((c) => c.title).filter(Boolean);
+    return { tagNames, challenges: videoInfo.challenges };
+  } catch (e) {
+    console.error(`        [tags] 提取失败: ${e.message}`);
+    return { tagNames: [], challenges: [] };
+  }
+}
+
+/**
+ * 将视频 tag 保存到服务端（只插入不存在的，不覆盖已有）
+ * @returns {Promise<{ inserted: number }>}
+ */
+async function saveVideoTags(apiPost, serverUrl, tagNames, country) {
+  if (!tagNames || tagNames.length === 0) return { inserted: 0 };
+  try {
+    const result = await apiPost(`${serverUrl}/api/tags/batch-add-if-new`, {
+      tags: tagNames,
+      countries: [country],
+      source: "video",
+    });
+    return { inserted: result.inserted || 0 };
+  } catch (e) {
+    console.error(`        [tags] 保存失败: ${e.message}`);
+    return { inserted: 0 };
+  }
+}
+
 function extractUniqueAuthors(comments) {
   return [
     ...new Set(
@@ -668,7 +712,7 @@ async function runAutoMode(options) {
           if (shuttingDown) break;
 
           try {
-            const { id, href, locationCreated } = task;
+            const { id, href, locationCreated, ttSeller } = task;
             const loc = normalizeLocation(locationCreated);
             const taskNum = totalTasks() + 1;
 
@@ -691,6 +735,24 @@ async function runAutoMode(options) {
             // 导航到视频页
             await prepareVideoPage(session.page, href);
 
+            // ── 提取视频 tag（仅 ttSeller 视频）──
+            if (ttSeller && loc) {
+              const { tagNames } = await extractVideoTags(session.page);
+              if (tagNames.length > 0) {
+                const { inserted } = await saveVideoTags(
+                  apiPost,
+                  serverUrl,
+                  tagNames,
+                  loc,
+                );
+                if (inserted > 0) {
+                  console.error(
+                    `        [tags] 提取到 ${tagNames.length} 个: ${tagNames.join(", ")} → 新增 ${inserted} 个`,
+                  );
+                }
+              }
+            }
+
             // 获取评论
             const result = await fetchUserCommentsAPI(session.page, {
               maxComments: actualMaxComments,

package/src/lib/parse-ssr.mjs

@@ -138,11 +138,17 @@ export function parseVideoInfo(rawHtml) {
   const item = vd.itemInfo.itemStruct;
   const author = item.author || {};
   const stats = item.stats || {};
+  const challenges = (item.challenges || []).map((c) => ({
+    id: c.id,
+    title: c.title,
+    desc: c.desc || "",
+  }));
   return {
     id: item.id,
     desc: item.desc || "",
     createTime: item.createTime || null,
     locationCreated: item.locationCreated || null,
+    challenges,
     author: {
       uniqueId: author.uniqueId,
       nickname: author.nickname,

package/src/lib/tag-discover.js

@@ -158,11 +158,15 @@ async function discoverTagsForCountryCli(
       `  🏥 ${country} health: ${health.healthStatus} | deadRatio=${health.deadRatio}% | productiveRatio=${health.productiveRatio}% | saturation=${health.saturation}%\n`,
     );
 
-    // 动态温度调整
-    if (health.healthStatus === "critical") {
-      temperature = 0.3;
-    } else if (health.healthStatus === "warning") {
+    // 动态温度调整：卡死时升温打破 LLM 确定性重复
+    if (health.consecutiveLowProductiveRounds >= 3) {
+      temperature = 1.0;
+    } else if (health.consecutiveLowProductiveRounds >= 2) {
+      temperature = 0.9;
+    } else if (health.healthStatus === "critical") {
       temperature = 0.5;
+    } else if (health.healthStatus === "warning") {
+      temperature = 0.6;
     }
 
     // 自动 niche 切换

package/src/lib/tag-fetcher.js

@@ -166,6 +166,11 @@ export async function fetchTagData(tag, options = {}) {
       throw new Error("标签页加载失败，TikTok 返回了错误页面");
     }
 
+    // 检测 CDN/网络拦截：15 秒后仍无任何 API 数据到达，说明被拦截而非真没数据
+    if (seenVideoIds.size === 0 && challengeInfo === null) {
+      throw new Error("无法获取标签页 API 数据（可能被 CDN 拦截或网络异常）");
+    }
+
     let lastCount = 0;
     let staleRounds = 0;
     const startTime = Date.now();

package/src/npm-main.js

@@ -3,6 +3,7 @@ import { PUBLIC_HELP_TEXT } from "./lib/constants.js";
 import { handleInfo } from "./cli/info.js";
 import { handleExplore } from "./cli/explore.js";
 import { handleAttach } from "./cli/attach.js";
+import { handleRecheck } from "./cli/recheck.js";
 import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
 import { handleOpen } from "./cli/open.js";
 import { handleComments } from "./cli/comments.js";
@@ -32,6 +33,8 @@ async function main() {
       return handleInfo(parsed);
     case "attach":
       return handleAttach(parsed);
+    case "recheck":
+      return handleRecheck(parsed);
     case "watch":
     case "webserver":
     case "videostats":

package/src/watch/data-store.js

@@ -86,6 +86,7 @@ import {
 // Tag CRUD
 import {
   insertTag,
+  insertTagsIfNew,
   getTagsByStatus,
   getTagsByCountry,
   getDeadTags,
@@ -3601,6 +3602,7 @@ export function createStore(filePath, options = {}) {
     getLlmSampleOffsets, // 获取 LLM 采样偏移量状态
     // Tag 发现与打分
     insertTag,
+    insertTagsIfNew,
     getTagsByStatus,
     getTagsByCountry,
     getDeadTags,

package/src/watch/db-tags.js

@@ -50,11 +50,59 @@ export function insertTag(tag, countries, source = "llm") {
   }
 }
 
+/**
+ * 批量插入 tag，只插入不存在的（INSERT OR IGNORE）。
+ * 用于非 LLM 来源（如视频页采集），不覆盖已有 tag 的任何字段。
+ *
+ * @param {string[]} tags - 原始 tag 名列表（可带 # 前缀）
+ * @param {string[]} countries - 国家列表
+ * @param {string} source - 来源标识，默认 'video'
+ * @returns {{ inserted: number, tags: string[] }}
+ */
+export function insertTagsIfNew(tags, countries, source = "video") {
+  const db = getDb();
+  if (!db) return { inserted: 0, tags: [] };
+  if (!Array.isArray(tags) || tags.length === 0)
+    return { inserted: 0, tags: [] };
+  if (!Array.isArray(countries) || countries.length === 0)
+    return { inserted: 0, tags: [] };
+
+  const normalizedList = tags
+    .map((t) => t.replace(/^#+/, "").trim().toLowerCase())
+    .filter((t) => t.length >= 2);
+
+  if (normalizedList.length === 0) return { inserted: 0, tags: [] };
+
+  const countriesJson = JSON.stringify(countries);
+  let inserted = 0;
+  const insertedTags = [];
+
+  try {
+    const stmt = db.prepare(
+      `INSERT OR IGNORE INTO tags (tag, countries, source) VALUES (?, ?, ?)`,
+    );
+    const txn = db.transaction((items) => {
+      for (const tag of items) {
+        const result = stmt.run(tag, countriesJson, source);
+        if (result.changes > 0) {
+          inserted++;
+          insertedTags.push(tag);
+        }
+      }
+    });
+    txn(normalizedList);
+    return { inserted, tags: insertedTags };
+  } catch (e) {
+    return { inserted, error: e.message };
+  }
+}
+
 export function getTagsByStatus(
   status,
   limit = 100,
   offset = 0,
   country = null,
+  search = null,
 ) {
   const db = getDb();
   if (!db) return [];
@@ -64,6 +112,10 @@ export function getTagsByStatus(
     sql += " AND countries LIKE ?";
     params.push(`%"${country}"%`);
   }
+  if (search) {
+    sql += " AND tag LIKE ?";
+    params.push(`%${search}%`);
+  }
   sql +=
     " ORDER BY refreshed_at DESC NULLS LAST, score ASC, created_at ASC LIMIT ? OFFSET ?";
   params.push(limit, offset);
@@ -71,21 +123,29 @@ export function getTagsByStatus(
   return rows.map(parseTagRow);
 }
 
-export function getTagStats(country = null) {
+export function getTagStats(country = null, search = null) {
   const db = getDb();
   if (!db) return null;
+  const conditions = [];
+  const params = [];
+  if (country) {
+    conditions.push("countries LIKE ?");
+    params.push(`%"${country}"%`);
+  }
+  if (search) {
+    conditions.push("tag LIKE ?");
+    params.push(`%${search}%`);
+  }
+  const whereClause =
+    conditions.length > 0 ? " WHERE " + conditions.join(" AND ") : "";
+
   let sql = `SELECT
     COUNT(*) as total,
     SUM(CASE WHEN status = 'productive' THEN 1 ELSE 0 END) as productive,
     SUM(CASE WHEN status = 'dead' THEN 1 ELSE 0 END) as dead,
     SUM(CASE WHEN status = 'new' THEN 1 ELSE 0 END) as newCount,
     SUM(CASE WHEN status = 'scoring' THEN 1 ELSE 0 END) as scoring
-  FROM tags`;
-  const params = [];
-  if (country) {
-    sql += " WHERE countries LIKE ?";
-    params.push(`%"${country}"%`);
-  }
+  FROM tags${whereClause}`;
   const row = db.prepare(sql).get(...params);
   // 获取所有出现过的国家
   const allRows = db.prepare("SELECT countries FROM tags").all();
@@ -276,15 +336,28 @@ export function reportTagScore(tag, fields) {
   }
 }
 
-export function getAllTags(limit = 200, offset = 0, country = null) {
+export function getAllTags(
+  limit = 200,
+  offset = 0,
+  country = null,
+  search = null,
+) {
   const db = getDb();
   if (!db) return [];
   let sql = "SELECT * FROM tags";
   const params = [];
+  const conditions = [];
   if (country) {
-    sql += " WHERE countries LIKE ?";
+    conditions.push("countries LIKE ?");
     params.push(`%"${country}"%`);
   }
+  if (search) {
+    conditions.push("tag LIKE ?");
+    params.push(`%${search}%`);
+  }
+  if (conditions.length > 0) {
+    sql += " WHERE " + conditions.join(" AND ");
+  }
   sql +=
     " ORDER BY refreshed_at DESC NULLS LAST, score DESC, created_at DESC LIMIT ? OFFSET ?";
   params.push(limit, offset);
@@ -371,6 +444,7 @@ export function insertDiscoverLog({
   country,
   round,
   strategy,
+  niche,
   tagsGenerated,
   tagsRaw,
   tagsAdded,
@@ -382,18 +456,26 @@ export function insertDiscoverLog({
   const db = getDb();
   if (!db) return { ok: false, error: "db not ready" };
 
+  // 确保 niche 列存在（兼容旧表）
+  try {
+    db.prepare("ALTER TABLE discover_log ADD COLUMN niche TEXT").run();
+  } catch {
+    // 列已存在则忽略
+  }
+
   try {
     const result = db
       .prepare(
         `INSERT INTO discover_log
-         (country, round, strategy, tags_generated, tags_raw, tags_added,
+         (country, round, strategy, niche, tags_generated, tags_raw, tags_added,
           productive_sample, dead_sample, avg_productive_score, avg_dead_score)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       )
       .run(
         country,
         round,
         strategy || null,
+        niche || null,
         JSON.stringify(tagsGenerated || []),
         tagsRaw ? JSON.stringify(tagsRaw) : null,
         tagsAdded || 0,
@@ -714,6 +796,9 @@ export function getCountryHealth(country) {
   // 推荐的 niche 方向（基于现有 productive tag 的分类分析）
   const nicheRecommendation = recommendNiche(country, productive);
 
+  // 追踪 LLM 当前在攻击哪个 niche（取最近3轮 discover_log 的 niche 字段，众数）
+  const currentNiche = getCurrentNiche(country);
+
   return {
     country,
     total,
@@ -732,9 +817,43 @@ export function getCountryHealth(country) {
     healthStatus,
     issues,
     nicheRecommendation,
+    currentNiche,
   };
 }
 
+/**
+ * 获取 LLM 当前正在攻击的 niche（最近3轮 discover_log 的 niche 众数）
+ */
+function getCurrentNiche(country) {
+  const db = getDb();
+  if (!db) return null;
+
+  try {
+    const rows = db
+      .prepare(
+        `SELECT niche FROM discover_log
+         WHERE country = ? AND niche IS NOT NULL
+         ORDER BY round DESC
+         LIMIT 3`,
+      )
+      .all(country);
+
+    if (rows.length === 0) return null;
+
+    // 取出现次数最多的 niche
+    const counts = {};
+    for (const r of rows) {
+      const n = r.niche;
+      if (n) counts[n] = (counts[n] || 0) + 1;
+    }
+    const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
+    return sorted.length > 0 ? sorted[0][0] : null;
+  } catch {
+    // niche 列尚不存在（旧表），返回 null
+    return null;
+  }
+}
+
 /**
  * 基于现有 productive tag 推荐新的 niche 方向
  * 分析已有 tag 的类别分布，找出未覆盖或覆盖不足的方向
@@ -1121,13 +1240,68 @@ export function insertReflectionRules(rules) {
   return { ok: true, inserted: inserted.length };
 }
 
+/**
+ * 计算两段规则文本的相似度（简单的词重叠率）
+ * 返回 0-1，越高越相似
+ */
+function ruleTextSimilarity(a, b) {
+  const wordsA = new Set(
+    a
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((w) => w.length > 2),
+  );
+  const wordsB = new Set(
+    b
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((w) => w.length > 2),
+  );
+  if (wordsA.size === 0 || wordsB.size === 0) return 0;
+  const intersection = [...wordsA].filter((w) => wordsB.has(w)).length;
+  return intersection / Math.max(wordsA.size, wordsB.size);
+}
+
 /**
  * 废弃旧规则（当新反思覆盖旧规则时）
+ * 加入去重：如果新规则与旧规则语义相似（>70% 词重叠），跳过 supersede
  */
-export function supersedeRules(country, reason = "new reflection") {
+export function supersedeRules(
+  country,
+  newRules = [],
+  reason = "new reflection",
+) {
   const db = getDb();
   if (!db) return { ok: false, error: "db not ready" };
 
+  // 获取当前活跃规则
+  const oldRules = db
+    .prepare(
+      "SELECT id, rule_text FROM reflection_rules WHERE country = ? AND is_active = 1",
+    )
+    .all(country);
+
+  // 规则去重：如果所有新规则与所有旧规则高度相似，跳过
+  if (oldRules.length > 0 && newRules.length > 0) {
+    let allSimilar = true;
+    for (const nr of newRules) {
+      const newText = typeof nr === "string" ? nr : nr.ruleText || "";
+      const hasSimilar = oldRules.some(
+        (or) => ruleTextSimilarity(newText, or.rule_text) > 0.7,
+      );
+      if (!hasSimilar) {
+        allSimilar = false;
+        break;
+      }
+    }
+    if (allSimilar) {
+      console.error(
+        `[reflection] 跳过 supersede: ${newRules.length} 条新规则与 ${oldRules.length} 条旧规则高度相似`,
+      );
+      return { ok: true, superseded: 0, skipped: "rules too similar" };
+    }
+  }
+
   const result = db
     .prepare(
       `UPDATE reflection_rules
@@ -1141,13 +1315,45 @@ export function supersedeRules(country, reason = "new reflection") {
 
 /**
  * 检查是否需要触发反思
- * 条件：最近 N 轮 productive_cnt ≤ threshold
+ * 条件：最近 N 轮 productive_cnt ≤ threshold，且距上次反思 ≥ cooldownRounds 轮
  * @returns {{ shouldReflect: boolean, reason: string, recentRounds: array }}
  */
-export function shouldTriggerReflection(country, lookback = 5, threshold = 1) {
+export function shouldTriggerReflection(
+  country,
+  lookback = 5,
+  threshold = 1,
+  cooldownRounds = 10,
+) {
   const db = getDb();
   if (!db) return { shouldReflect: false, reason: "db not ready" };
 
+  // 冷却期检查：距上次反思不足 cooldownRounds 轮则跳过
+  const lastReflection = db
+    .prepare(
+      `SELECT MAX(rr.created_at) as lastReflectAt
+       FROM reflection_rules rr
+       WHERE rr.country = ? AND rr.is_active = 0 AND rr.superseded_at IS NOT NULL
+       ORDER BY rr.superseded_at DESC
+       LIMIT 1`,
+    )
+    .get(country);
+
+  if (lastReflection?.lastReflectAt) {
+    const roundsSince = db
+      .prepare(
+        `SELECT COUNT(*) as cnt FROM discover_log
+         WHERE country = ? AND created_at > ?`,
+      )
+      .get(country, lastReflection.lastReflectAt);
+    if ((roundsSince?.cnt || 0) < cooldownRounds) {
+      return {
+        shouldReflect: false,
+        reason: `冷却期: 距上次反思仅${roundsSince?.cnt || 0}轮，需≥${cooldownRounds}轮`,
+        recentRounds: [],
+      };
+    }
+  }
+
   // 获取最近 lookback 轮的 discover_log
   const recentLogs = db
     .prepare(
@@ -1202,14 +1408,14 @@ export function shouldTriggerReflection(country, lookback = 5, threshold = 1) {
   }
 
   const avgProductive = totalProductive / lookback;
-  // 触发条件：平均 productive ≤ threshold，或连续 ≥2 轮 productive ≤ 1
+  // 触发条件：平均 productive ≤ threshold，或连续 ≥3 轮 productive ≤ 1
   const shouldReflect =
-    avgProductive <= threshold || consecutiveLowProductiveRounds >= 2;
+    avgProductive <= threshold || consecutiveLowProductiveRounds >= 3;
 
   return {
     shouldReflect,
     reason: shouldReflect
-      ? consecutiveLowProductiveRounds >= 2
+      ? consecutiveLowProductiveRounds >= 3
         ? `连续${consecutiveLowProductiveRounds}轮productive≤1，需要反思换方向`
         : `最近${lookback}轮平均productive=${avgProductive.toFixed(1)} ≤ ${threshold}`
       : `最近${lookback}轮平均productive=${avgProductive.toFixed(1)} > ${threshold}`,
@@ -1222,6 +1428,20 @@ export function shouldTriggerReflection(country, lookback = 5, threshold = 1) {
   };
 }
 
+/**
+ * 清理所有已废弃的反思规则（is_active=0）
+ */
+export function cleanupInactiveReflectionRules() {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+
+  const result = db
+    .prepare("DELETE FROM reflection_rules WHERE is_active = 0")
+    .run();
+  console.error(`[reflection] 🧹 清理了 ${result.changes} 条废弃规则`);
+  return { ok: true, deleted: result.changes };
+}
+
 /**
  * 获取反思所需的数据摘要（供 LLM 分析）
  */