tt-help-cli-ycl 1.4.3 → 1.4.5

package/src/watch/db-tags.js

@@ -357,6 +357,37 @@ export function getRecentDiscoverLogs(country, limit = 10) {
     }));
 }
 
+/**
+ * 获取最近 N 轮生成的所有标签（去重），用于检测重复生成
+ */
+export function getAllExistingTagNames() {
+  const db = getDb();
+  if (!db) return new Set();
+  const rows = db.prepare("SELECT tag FROM tags").all();
+  return new Set(rows.map((r) => r.tag));
+}
+
+export function getRecentGeneratedTags(country, lookback = 10) {
+  const db = getDb();
+  if (!db) return [];
+
+  const logs = db
+    .prepare(
+      `SELECT tags_generated FROM discover_log
+       WHERE country = ?
+       ORDER BY round DESC
+       LIMIT ?`,
+    )
+    .all(country, lookback);
+
+  const allTags = new Set();
+  for (const log of logs) {
+    const tags = JSON.parse(log.tags_generated || "[]");
+    for (const t of tags) allTags.add(t);
+  }
+  return [...allTags];
+}
+
 /**
  * 获取某国家的 discover 轮次（用于递增）
  */
@@ -425,3 +456,762 @@ export function getDiscoverEffectiveness(country) {
     };
   });
 }
+
+// ====== 自闭环：健康指标 & 数据清理 ======
+
+/**
+ * 获取某国家的 tag 健康指标
+ * 返回：总数、各状态数、dead比率、productive比率、饱和度、最近轮次效果
+ */
+export function getCountryHealth(country) {
+  const db = getDb();
+  if (!db) return null;
+
+  // 基础统计
+  const allTags = db
+    .prepare(`SELECT tag, status, score FROM tags WHERE countries LIKE ?`)
+    .all(`%"${country}"%`);
+
+  const total = allTags.length;
+  const productive = allTags.filter((t) => t.status === "productive");
+  const dead = allTags.filter((t) => t.status === "dead");
+  const scored = allTags.filter((t) => t.status === "scored");
+  const lowScored = allTags.filter(
+    (t) => t.status === "scored" && t.score >= 10 && t.score < 20,
+  );
+
+  const deadRatio = total > 0 ? (dead.length / total) * 100 : 0;
+  const productiveRatio = total > 0 ? (productive.length / total) * 100 : 0;
+  const avgProductiveScore =
+    productive.length > 0
+      ? productive.reduce((s, t) => s + t.score, 0) / productive.length
+      : 0;
+
+  // 最近5轮 discover 效果
+  const recentLogs = db
+    .prepare(
+      `SELECT round, tags_added, tags_generated FROM discover_log
+       WHERE country = ? ORDER BY round DESC LIMIT 5`,
+    )
+    .all(country);
+
+  // 计算最近5轮的 productive 和 dead 比例
+  let recentProductiveCount = 0;
+  let recentDeadCount = 0;
+  let recentTotalGenerated = 0;
+  let recentEmptyRounds = 0;
+  let consecutiveLowProductiveRounds = 0; // 连续 productive=0 的轮数
+
+  for (const log of recentLogs) {
+    const tagsGen = JSON.parse(log.tags_generated || "[]");
+    recentTotalGenerated += tagsGen.length;
+    if (tagsGen.length === 0) recentEmptyRounds++;
+
+    if (tagsGen.length > 0) {
+      const placeholders = tagsGen.map(() => "?").join(",");
+      const tagResults = db
+        .prepare(`SELECT status FROM tags WHERE tag IN (${placeholders})`)
+        .all(...tagsGen);
+      const productiveInRound = tagResults.filter(
+        (t) => t.status === "productive",
+      ).length;
+      recentProductiveCount += productiveInRound;
+      recentDeadCount += tagResults.filter((t) => t.status === "dead").length;
+      // 连续低效轮计数：productive=0 则累加，否则重置
+      if (productiveInRound === 0) {
+        consecutiveLowProductiveRounds++;
+      } else {
+        consecutiveLowProductiveRounds = 0;
+      }
+    } else {
+      // 空轮也算低效轮
+      consecutiveLowProductiveRounds++;
+    }
+  }
+
+  const recentProductiveRatio =
+    recentTotalGenerated > 0
+      ? (recentProductiveCount / recentTotalGenerated) * 100
+      : 0;
+  const recentDeadRatio =
+    recentTotalGenerated > 0
+      ? (recentDeadCount / recentTotalGenerated) * 100
+      : 0;
+
+  // 饱和度：最近轮次生成空或极少 tag 的比例
+  const saturation =
+    recentLogs.length > 0 ? (recentEmptyRounds / recentLogs.length) * 100 : 0;
+
+  // 判断健康状态
+  let healthStatus = "healthy";
+  const issues = [];
+
+  if (deadRatio > 60) {
+    healthStatus = "critical";
+    issues.push(
+      `dead_ratio=${deadRatio.toFixed(1)}% (>60%): 太多无效tag，需要降低生成激进度`,
+    );
+  } else if (deadRatio > 40) {
+    healthStatus = "warning";
+    issues.push(
+      `dead_ratio=${deadRatio.toFixed(1)}% (>40%): 无效tag偏多，需要更保守`,
+    );
+  }
+
+  if (productiveRatio < 8) {
+    healthStatus = healthStatus === "critical" ? "critical" : "warning";
+    issues.push(
+      `productive_ratio=${productiveRatio.toFixed(1)}% (<8%): 有效tag太少，可能需要切换策略方向`,
+    );
+  }
+
+  if (saturation >= 40) {
+    healthStatus = healthStatus === "critical" ? "critical" : "warning";
+    issues.push(
+      `saturation=${saturation.toFixed(0)}%: tag空间接近饱和，需要切换到新niche`,
+    );
+  }
+
+  if (recentEmptyRounds > 0) {
+    issues.push(
+      `recent_empty_rounds=${recentEmptyRounds}: LLM无法生成新不重复tag`,
+    );
+  }
+
+  // 连续低效轮警告：连续多轮 productive=0，需要强制换方向
+  if (consecutiveLowProductiveRounds >= 2) {
+    healthStatus = healthStatus === "critical" ? "critical" : "warning";
+    issues.push(
+      `consecutive_low_productive_rounds=${consecutiveLowProductiveRounds}: 连续${consecutiveLowProductiveRounds}轮无有效产出，必须切换商品分类方向`,
+    );
+  }
+
+  if (lowScored.length > productive.length * 2) {
+    issues.push(
+      `low_scored=${lowScored.length} vs productive=${productive.length}: 低分tag远多于有效tag，应清理`,
+    );
+  }
+
+  // 推荐的 niche 方向（基于现有 productive tag 的分类分析）
+  const nicheRecommendation = recommendNiche(country, productive);
+
+  return {
+    country,
+    total,
+    productive: productive.length,
+    dead: dead.length,
+    scored: scored.length,
+    lowScored: lowScored.length,
+    deadRatio: deadRatio.toFixed(1),
+    productiveRatio: productiveRatio.toFixed(1),
+    avgProductiveScore: avgProductiveScore.toFixed(1),
+    recentProductiveRatio: recentProductiveRatio.toFixed(1),
+    recentDeadRatio: recentDeadRatio.toFixed(1),
+    recentEmptyRounds,
+    consecutiveLowProductiveRounds,
+    saturation: saturation.toFixed(0),
+    healthStatus,
+    issues,
+    nicheRecommendation,
+  };
+}
+
+/**
+ * 基于现有 productive tag 推荐新的 niche 方向
+ * 分析已有 tag 的类别分布，找出未覆盖或覆盖不足的方向
+ */
+function recommendNiche(country, productiveTags) {
+  // 定义 niche 类别及其关键词模式
+  const niches = {
+    fashion: [
+      "clothes",
+      "kleidung",
+      "kleding",
+      "vestiti",
+      "ropa",
+      "outfit",
+      "mode",
+      "shoes",
+      "sneakers",
+      "jurken",
+      "obleceni",
+      "ruha",
+      "zapatos",
+      "chaussures",
+      "vintage",
+      "secondhand",
+      "thrift",
+      "friperie",
+      "tweedehands",
+      "hasznalt",
+      "segundamano",
+      "secondamano",
+    ],
+    food: [
+      "food",
+      "bakery",
+      "brot",
+      "brood",
+      "pane",
+      "comida",
+      "restaurant",
+      "cafe",
+      "kitchen",
+      "konyha",
+      "cukrar",
+      "patisserie",
+      "boulangerie",
+      "wypieki",
+      "sauerteig",
+      "etterm",
+      "kaviarnia",
+    ],
+    handmade_crafts: [
+      "handmade",
+      "handwerk",
+      "handgemaakt",
+      "artigianato",
+      "hechoamano",
+      "kézműves",
+      "kreativ",
+      "craft",
+      "diy",
+      "knutselen",
+      "ceramic",
+      "keramiek",
+      "kerzen",
+      "gyertya",
+      "harskunst",
+      "resin",
+      "macrame",
+      "pottery",
+      "weaving",
+    ],
+    home_living: [
+      "home",
+      "wohnung",
+      "huis",
+      "casa",
+      "casa",
+      "lakberendezes",
+      "interior",
+      "deco",
+      "decoration",
+      "einrichten",
+      "furniture",
+      "möbel",
+      "meubilair",
+      "epipla",
+      "butor",
+      "nabytek",
+      "tapic",
+      "reinigung",
+      "cleaning",
+    ],
+    electronics_tech: [
+      "tech",
+      "elektronik",
+      "electronica",
+      "gadget",
+      "phone",
+      "smartphone",
+      "laptop",
+      "computer",
+      "audio",
+      "cuffie",
+      "auricolari",
+      "headphones",
+      "caricatore",
+      "charger",
+      "usb",
+      "playstation",
+      "gaming",
+    ],
+    pets_animals: [
+      "pet",
+      "hund",
+      "dog",
+      "cat",
+      "kat",
+      "kuty",
+      "konia",
+      "pets",
+      "tier",
+      "dieren",
+      "animale",
+      "animal",
+      "kutyakonzerv",
+      "akcesoriadlapsow",
+    ],
+    beauty_health: [
+      "beauty",
+      "kosmetik",
+      "cosmetica",
+      "beaute",
+      "belleza",
+      "szepseg",
+      "kapper",
+      "hair",
+      "haj",
+      "skincare",
+      "makeup",
+      "parfum",
+      "perfume",
+      "wellness",
+      "gesundheit",
+      "salud",
+      "sante",
+    ],
+    books_collectibles: [
+      "book",
+      "buch",
+      "boek",
+      "libro",
+      "knihy",
+      "konyv",
+      "collectible",
+      "sammlung",
+      "verzameling",
+      "collezionismo",
+      "coleccion",
+      "antique",
+      "antiquariat",
+      "vintage",
+      "vinyl",
+      "schallplatten",
+      "pokemon",
+      "kaarten",
+      "moedas",
+      "mince",
+    ],
+    services_professional: [
+      "service",
+      "dienst",
+      "servizio",
+      "servicio",
+      "szolgaltatas",
+      "handwerker",
+      "craftsman",
+      "repair",
+      "reparatur",
+      "reparatie",
+      "riparazione",
+      "reparacion",
+      "cleaning",
+      "reinigung",
+      "moving",
+      "umzug",
+      "verhuur",
+      "mudanca",
+      "florist",
+      "photography",
+      "fotograf",
+    ],
+    marketplace_local: [
+      "market",
+      "markt",
+      "mercato",
+      "mercado",
+      "piac",
+      "trh",
+      "flohmarkt",
+      "rommelmarkt",
+      "brocante",
+      "bazar",
+      "mercadillo",
+      "vlooienmarkt",
+      "braderie",
+      "feira",
+      "local",
+      "lokal",
+      "locale",
+      "winkel",
+      "boutique",
+      "shop",
+      "tienda",
+      "negozio",
+      "bol",
+      "webshop",
+      "eshop",
+    ],
+  };
+
+  // 统计每个 niche 在 productive tag 中的覆盖数
+  const nicheCoverage = {};
+  for (const [niche, keywords] of Object.entries(niches)) {
+    const matched = productiveTags.filter((t) =>
+      keywords.some((kw) => t.tag.includes(kw)),
+    ).length;
+    nicheCoverage[niche] = matched;
+  }
+
+  // 找出覆盖不足的 niche（productive < 5）作为推荐方向
+  const uncovered = Object.entries(nicheCoverage)
+    .filter(([, count]) => count < 5)
+    .sort(([, a], [, b]) => a - b)
+    .map(([niche, count]) => ({ niche, currentCoverage: count }));
+
+  // 找出覆盖良好的 niche（productive >= 10）作为已饱和方向
+  const saturated = Object.entries(nicheCoverage)
+    .filter(([, count]) => count >= 10)
+    .sort(([, a], [, b]) => b - a)
+    .map(([niche, count]) => ({ niche, currentCoverage: count }));
+
+  return { uncovered, saturated, nicheCoverage };
+}
+
+/**
+ * 清理低效 tag：将 score < 20 的 scored tag 降级为 dead
+ * @param {number} minScore - 低于此分数的 scored tag 降级为 dead（默认 20）
+ * @returns {object} { cleaned: number }
+ */
+export function cleanupLowScoredTags(minScore = 20) {
+  const db = getDb();
+  if (!db) return { cleaned: 0, error: "db not ready" };
+
+  const result = db
+    .prepare(
+      `UPDATE tags SET status = 'dead', last_error = 'auto_cleanup: scored<20' 
+       WHERE status = 'scored' AND score < ?`,
+    )
+    .run(minScore);
+
+  if (result.changes > 0) {
+    console.error(
+      `[tags] 自闭环清理: ${result.changes} 个低分scored tag降级为dead (score<${minScore})`,
+    );
+  }
+  return { cleaned: result.changes };
+}
+
+/**
+ * 清理长时间卡在 scoring 状态的 tag（超时未打分）
+ * @param {number} minutes - 超时分钟数（默认 60）
+ * @returns {object} { reset: number }
+ */
+export function cleanupStaleScoringTags(minutes = 60) {
+  const db = getDb();
+  if (!db) return { reset: 0, error: "db not ready" };
+
+  const result = db
+    .prepare(
+      `UPDATE tags SET status = 'new', scored_at = NULL 
+       WHERE status = 'scoring' AND scored_at IS NOT NULL 
+       AND scored_at < datetime('now', '-${minutes} minutes')`,
+    )
+    .run();
+
+  if (result.changes > 0) {
+    console.error(
+      `[tags] 自闭环清理: ${result.changes} 个超时scoring tag重置为new (>${minutes}min)`,
+    );
+  }
+  return { reset: result.changes };
+}
+
+/**
+ * 批量清理 error 状态的 tag（重置为 new 以重新打分）
+ * @returns {object} { reset: number }
+ */
+export function cleanupErrorTags() {
+  const db = getDb();
+  if (!db) return { reset: 0, error: "db not ready" };
+
+  const result = db
+    .prepare(
+      `UPDATE tags SET status = 'new', scored_at = NULL, last_error = NULL 
+       WHERE status = 'error'`,
+    )
+    .run();
+
+  if (result.changes > 0) {
+    console.error(`[tags] 自闭环清理: ${result.changes} 个error tag重置为new`);
+  }
+  return { reset: result.changes };
+}
+
+// ====== 反思闭环：reflection_rules CRUD ======
+
+/**
+ * 获取某国家的所有活跃硬规则
+ */
+export function getActiveRules(country) {
+  const db = getDb();
+  if (!db) return [];
+
+  return db
+    .prepare(
+      `SELECT id, country, rule_type, rule_text, trigger_reason, created_at
+       FROM reflection_rules
+       WHERE country = ? AND is_active = 1
+       ORDER BY created_at DESC`,
+    )
+    .all(country);
+}
+
+/**
+ * 插入一条反思规则
+ */
+export function insertReflectionRule({
+  country,
+  ruleType = "hard",
+  ruleText,
+  triggerReason,
+  productiveSample,
+  deadSample,
+  analysisSummary,
+}) {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+
+  try {
+    const result = db
+      .prepare(
+        `INSERT INTO reflection_rules
+         (country, rule_type, rule_text, source, trigger_reason,
+          productive_sample, dead_sample, analysis_summary)
+         VALUES (?, ?, ?, 'reflection', ?, ?, ?, ?)`,
+      )
+      .run(
+        country,
+        ruleType,
+        ruleText,
+        triggerReason || null,
+        JSON.stringify(productiveSample || []),
+        JSON.stringify(deadSample || []),
+        analysisSummary || null,
+      );
+    return { ok: true, id: result.lastInsertRowid };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+/**
+ * 批量插入反思规则（一次反思产出多条规则）
+ */
+export function insertReflectionRules(rules) {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+
+  const inserted = [];
+  for (const rule of rules) {
+    const result = insertReflectionRule(rule);
+    if (result.ok) inserted.push(result.id);
+  }
+  return { ok: true, inserted: inserted.length };
+}
+
+/**
+ * 废弃旧规则（当新反思覆盖旧规则时）
+ */
+export function supersedeRules(country, reason = "new reflection") {
+  const db = getDb();
+  if (!db) return { ok: false, error: "db not ready" };
+
+  const result = db
+    .prepare(
+      `UPDATE reflection_rules
+       SET is_active = 0, superseded_at = datetime('now')
+       WHERE country = ? AND is_active = 1`,
+    )
+    .run(country);
+
+  return { ok: true, superseded: result.changes };
+}
+
+/**
+ * 检查是否需要触发反思
+ * 条件：最近 N 轮 productive_cnt ≤ threshold
+ * @returns {{ shouldReflect: boolean, reason: string, recentRounds: array }}
+ */
+export function shouldTriggerReflection(country, lookback = 5, threshold = 1) {
+  const db = getDb();
+  if (!db) return { shouldReflect: false, reason: "db not ready" };
+
+  // 获取最近 lookback 轮的 discover_log
+  const recentLogs = db
+    .prepare(
+      `SELECT round, tags_generated, tags_added, strategy, created_at
+       FROM discover_log
+       WHERE country = ?
+       ORDER BY round DESC
+       LIMIT ?`,
+    )
+    .all(country, lookback);
+
+  if (recentLogs.length < lookback) {
+    return {
+      shouldReflect: false,
+      reason: `only ${recentLogs.length} rounds, need ${lookback}`,
+      recentRounds: recentLogs,
+    };
+  }
+
+  // 计算每轮的 productive 数量
+  let totalProductive = 0;
+  let totalDead = 0;
+  let emptyRounds = 0;
+  let consecutiveLowProductiveRounds = 0; // 连续 productive ≤ 1 的轮数
+
+  for (const log of recentLogs) {
+    const tagsGen = JSON.parse(log.tags_generated || "[]");
+    if (tagsGen.length === 0) {
+      emptyRounds++;
+      consecutiveLowProductiveRounds++;
+      continue;
+    }
+
+    const placeholders = tagsGen.map(() => "?").join(",");
+    const tagResults = db
+      .prepare(`SELECT status FROM tags WHERE tag IN (${placeholders})`)
+      .all(...tagsGen);
+
+    const productive = tagResults.filter(
+      (t) => t.status === "productive",
+    ).length;
+    const dead = tagResults.filter((t) => t.status === "dead").length;
+    totalProductive += productive;
+    totalDead += dead;
+
+    // 连续低效轮计数：productive ≤ 1 则累加，否则重置
+    if (productive <= 1) {
+      consecutiveLowProductiveRounds++;
+    } else {
+      consecutiveLowProductiveRounds = 0;
+    }
+  }
+
+  const avgProductive = totalProductive / lookback;
+  // 触发条件：平均 productive ≤ threshold，或连续 ≥2 轮 productive ≤ 1
+  const shouldReflect =
+    avgProductive <= threshold || consecutiveLowProductiveRounds >= 2;
+
+  return {
+    shouldReflect,
+    reason: shouldReflect
+      ? consecutiveLowProductiveRounds >= 2
+        ? `连续${consecutiveLowProductiveRounds}轮productive≤1，需要反思换方向`
+        : `最近${lookback}轮平均productive=${avgProductive.toFixed(1)} ≤ ${threshold}`
+      : `最近${lookback}轮平均productive=${avgProductive.toFixed(1)} > ${threshold}`,
+    recentRounds: recentLogs,
+    totalProductive,
+    totalDead,
+    emptyRounds,
+    avgProductive,
+    consecutiveLowProductiveRounds,
+  };
+}
+
+/**
+ * 获取反思所需的数据摘要（供 LLM 分析）
+ */
+export function getReflectionData(country) {
+  const db = getDb();
+  if (!db) return null;
+
+  // 1. 最近 10 轮 discover_log
+  const recentLogs = db
+    .prepare(
+      `SELECT round, strategy, tags_generated, tags_added,
+              avg_productive_score, avg_dead_score, created_at
+       FROM discover_log
+       WHERE country = ?
+       ORDER BY round DESC
+       LIMIT 10`,
+    )
+    .all(country);
+
+  // 2. productive top 20（成功模式）
+  const productiveTop = db
+    .prepare(
+      `SELECT tag, score, matched_authors, pushed_users
+       FROM tags
+       WHERE status = 'productive' AND countries LIKE ?
+       ORDER BY pushed_users DESC
+       LIMIT 20`,
+    )
+    .all(`%"${country}"%`);
+
+  // 3. dead 标签频次统计（哪些标签被反复生成且 dead）
+  const deadFrequency = db
+    .prepare(
+      `SELECT tag, score, last_error, created_at
+       FROM tags
+       WHERE status = 'dead' AND countries LIKE ?
+       ORDER BY created_at DESC
+       LIMIT 50`,
+    )
+    .all(`%"${country}"%`);
+
+  // 4. dead 标签的 error 类型统计
+  const errorStats = db
+    .prepare(
+      `SELECT
+         COALESCE(last_error, 'no_match') as error_type,
+         COUNT(*) as count
+       FROM tags
+       WHERE status = 'dead' AND countries LIKE ?
+       GROUP BY COALESCE(last_error, 'no_match')
+       ORDER BY count DESC`,
+    )
+    .all(`%"${country}"%`);
+
+  // 5. 语言分布：dead 标签中英语 vs 本地语言的比例
+  const lang = getLang(country);
+  const langNames = {
+    cs: "Czech",
+    el: "Greek",
+    hu: "Hungarian",
+    pt: "Portuguese",
+    es: "Spanish",
+    pl: "Polish",
+    nl: "Dutch",
+    de: "German",
+    fr: "French",
+    it: "Italian",
+    en: "English",
+  };
+  const langName = langNames[lang] || lang;
+
+  // 6. 当前活跃的硬规则
+  const activeRules = getActiveRules(country);
+
+  // 7. 健康指标
+  const health = getCountryHealth(country);
+
+  return {
+    country,
+    lang,
+    langName,
+    recentLogs,
+    productiveTop,
+    deadFrequency,
+    errorStats,
+    activeRules,
+    health,
+  };
+}
+
+/**
+ * 获取某语言判断函数（用于分析 dead 标签的语言）
+ * 简单判断：是否包含该国本地语言的特殊字符
+ */
+function getLang(country) {
+  const COUNTRY_LANG = {
+    CZ: "cs",
+    GR: "el",
+    HU: "hu",
+    PT: "pt",
+    ES: "es",
+    PL: "pl",
+    NL: "nl",
+    DE: "de",
+    FR: "fr",
+    IT: "it",
+    IE: "en",
+    AT: "de",
+    BE: "nl",
+  };
+  return COUNTRY_LANG[country] || "en";
+}