tt-help-cli-ycl 1.4.17 → 1.4.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.17",
+  "version": "1.4.18",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/comments.js

@@ -646,11 +646,13 @@ async function runAutoMode(options) {
         } catch (err) {
           console.error(`  × 获取任务失败: ${err.message}`);
           consecutiveErrors++;
-          if (consecutiveErrors > 10) {
-            console.error("  × 连续获取失败 >10 → 退出");
-            await shutdown("too many errors");
+          if (consecutiveErrors > 0 && consecutiveErrors % 30 === 0) {
+            console.error(
+              `  ⚠ 已连续获取失败 ${consecutiveErrors} 次，继续等待服务恢复...`,
+            );
           }
-          await delay(1000, actualInterval * 1000);
+          const waitMs = Math.min(consecutiveErrors * 1000, 60000);
+          await delay(waitMs, waitMs + 1000);
           continue;
         }
         consecutiveErrors = 0;

package/src/cli/tag.js

@@ -532,7 +532,9 @@ async function scoreSingleTag(
         return result;
       }
       // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径，避免误标 dead
-      throw new Error("no videos returned — possible network/captcha/slow-load, will retry");
+      throw new Error(
+        "no videos returned — possible network/captcha/slow-load, will retry",
+      );
     }
 
     // Step 2: 补充国家信息
@@ -1063,9 +1065,7 @@ export async function handleScoreAll(parsed) {
         if (!videos || videos.length === 0) {
           const deadSec = ((Date.now() - fetchStart) / 1000).toFixed(1);
           if (tagResult.error === "tag_not_found") {
-            log(
-              `  ⚠️ 标签不存在 (${deadSec}s)，标记 dead`,
-            );
+            log(`  ⚠️ 标签不存在 (${deadSec}s)，标记 dead`);
             result.status = "dead";
             result.error = "tag_not_found";
             await reportToServer(baseUrl, result, clientId, clientMeta);
@@ -1076,9 +1076,7 @@ export async function handleScoreAll(parsed) {
             continue;
           }
           // 非 tag_not_found 的 0 视频 → 抛异常让其走 error 重试路径，避免误标 dead
-          log(
-            `  ⚠️ 无视频 (${deadSec}s)，可能网络/风控/加载慢 → 抛异常重试`,
-          );
+          log(`  ⚠️ 无视频 (${deadSec}s)，可能网络/风控/加载慢 → 抛异常重试`);
           throw new Error("no videos returned (network/captcha/slow-load)");
         }
 

package/src/cli/watch.js

@@ -5,16 +5,26 @@ import { startWatchServer, openBrowser } from "../watch/server.js";
 
 export async function handleWatch(options) {
   const dataAnchor = options.dataAnchor || options.outputFile;
-  const { watchPort, llmRefill, llmRefillMinScore, llmRefillSampleSize } =
-    options;
+  const {
+    watchPort,
+    llmRefill,
+    llmRefillMinScore,
+    llmRefillSampleSize,
+    llmDiscover,
+  } = options;
 
   if (!dataAnchor) {
-    console.error("用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill]");
+    console.error(
+      "用法: tt-help watch -o <db路径> [-p 端口] [--llm-refill] [--no-llm-discover]",
+    );
     console.error("      tt-help watch -o data/result.db");
     console.error("      tt-help watch -o data/result.db -p 8080");
     console.error(
       "      tt-help watch -o data/result.db --llm-refill --llm-refill-min 60",
     );
+    console.error(
+      "      tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
+    );
     process.exit(1);
   }
 
@@ -37,7 +47,8 @@ export async function handleWatch(options) {
     dataAnchor,
     watchPort,
     store,
-    storeOptions,
+    { ...storeOptions, llmDiscover: llmDiscover ?? true },
+    options.watchPassword,
   );
   openBrowser(port);
 

package/src/lib/args.js

@@ -347,9 +347,11 @@ function parseInfoArgs(args) {
 function parseWatchArgs(args) {
   let dataAnchor = "./result.db";
   let watchPort = 3001;
+  let watchPassword = null;
   let llmRefill = true;
   let llmRefillMinScore = null;
   let llmRefillSampleSize = null;
+  let llmDiscover = true;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -357,6 +359,8 @@ function parseWatchArgs(args) {
       dataAnchor = args[++i];
     } else if (arg === "-p") {
       watchPort = parseInt(args[++i]) || 3001;
+    } else if (arg === "-P" || arg === "--password") {
+      watchPassword = args[++i] || null;
     } else if (arg === "--llm-refill") {
       llmRefill = true;
     } else if (arg === "--no-llm-refill") {
@@ -365,6 +369,10 @@ function parseWatchArgs(args) {
       llmRefillMinScore = parseInt(args[++i]) || 60;
     } else if (arg === "--llm-refill-sample") {
       llmRefillSampleSize = parseInt(args[++i]) || 100;
+    } else if (arg === "--llm-discover") {
+      llmDiscover = true;
+    } else if (arg === "--no-llm-discover") {
+      llmDiscover = false;
     }
   }
 
@@ -373,9 +381,11 @@ function parseWatchArgs(args) {
     outputFile: dataAnchor,
     dataAnchor,
     watchPort,
+    watchPassword,
     llmRefill,
     llmRefillMinScore,
     llmRefillSampleSize,
+    llmDiscover,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,

package/src/lib/constants.js

@@ -177,10 +177,20 @@ const HELP_TEXT = [
   "    示例: tt-help attach -p 5 -i 10",
   "          tt-help attach -c PL,DE,FR -p 5",
   "",
-  "  tt-help watch -o <db路径> [-p 端口]",
+  "  tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
   "    启动监控服务；运行期主数据仅来自 SQLite",
+  "    选项:",
+  "      -p, --port <端口>          服务端口，默认 3000",
+  "      -P, --password <密码>      Web 面板登录密码",
+  "      --llm-refill               启用 LLM 自动筛选毛料任务（默认开启）",
+  "      --no-llm-refill            关闭 LLM 任务打分",
+  "      --llm-discover             启用 LLM 自动发现标签（默认开启）",
+  "      --no-llm-discover          关闭 LLM tag 发现",
+  "      --llm-refill-min <分数>    最低入围分，默认 60",
+  "      --llm-refill-sample <数量> 采样量，默认 100",
   "    示例: tt-help watch -o data/result.db",
   "          tt-help watch -o data/result.db -p 8080",
+  "          tt-help watch -o data/result.db --no-llm-discover --no-llm-refill",
   "",
   "  tt-help open <端口号>",
   "    打开指定端口的浏览器，用于配置 TikTok 登录账户",
@@ -316,7 +326,7 @@ const HELP_TEXT = [
 ];
 
 const PUBLIC_HELP_HIDDEN_HEADERS = new Set([
-  "  tt-help watch -o <db路径> [-p 端口]",
+  "  tt-help watch -o <db路径> [-p 端口] [--no-llm-refill] [--no-llm-discover]",
   "  videostats  <db路径> -p <并发数>",
   "  db-import  --db <db路径> [--users users.json] [--done done.json] [--videos videos.json]",
 ]);

package/src/watch/data-store.js

@@ -100,6 +100,7 @@ import {
   getProductiveRefreshTask,
   claimRefreshTag,
   reportRefreshResult,
+  getRefreshStats,
 } from "./db-tags.js";
 
 // LLM 打分
@@ -1095,6 +1096,9 @@ export function createStore(filePath, options = {}) {
 
         offsetStore.set("_tag", tagOffset);
 
+        // 让出事件循环，避免长时间阻塞监控页面及其他客户端请求
+        await new Promise(setImmediate);
+
         // ===== 第二步：一次性取 comment（全局，最多 llmCommentLimit 条，免打分）=====
         let commentOffset = offsetStore.get("_comment") || 0;
         const commentGlobalCount =
@@ -1146,6 +1150,9 @@ export function createStore(filePath, options = {}) {
 
         offsetStore.set("_comment", commentOffset);
 
+        // 让出事件循环，避免长时间阻塞监控页面及其他客户端请求
+        await new Promise(setImmediate);
+
         // ===== 第三步：按国家取非 tag 非 comment，走 LLM 打分，直到合格数达到 llmLLMTarget =====
         for (const location of normalizedLocations) {
           if (allLLMQualified.length >= llmLLMTarget) break;
@@ -1194,6 +1201,9 @@ export function createStore(filePath, options = {}) {
 
             if (!samples.length) break;
 
+            // 让出事件循环，LLM API 调用期间允许处理其他请求
+            await new Promise(setImmediate);
+
             const scores = await scoreJobsBatch(
               samples,
               DEFAULT_TARGET_LOCATIONS,
@@ -1234,6 +1244,9 @@ export function createStore(filePath, options = {}) {
           `[data-store] LLM 调度完成: tag ${allTagQualified.length} + comment ${allCommentQualified.length} + LLM ${allLLMQualified.length} = 共 ${qualified.length} 条`,
         );
 
+        // 让出事件循环，避免事务写入阻塞监控页面
+        await new Promise(setImmediate);
+
         // 移动符合条件的记录
         const placeholders = qualified.map(() => "?").join(", ");
         const moveTxn = getDb().transaction(() => {
@@ -3362,6 +3375,7 @@ export function createStore(filePath, options = {}) {
     getActiveClients,
     moveSellerJobsToBase, // 将 jobs/raw_jobs 中 tt_seller=1 且 video_count=0 的记录移动到 jobs_base
     moveRestrictedJobsToBase, // 将 jobs 中 status='restricted' 的记录移动到 jobs_base
+    moveSingleJobToBase, // 将单个 job 移动到 jobs_base 并重置（目标商家页面用）
     registerVideos,
     getVideo,
     getVideos,
@@ -3389,6 +3403,7 @@ export function createStore(filePath, options = {}) {
     getProductiveRefreshTask,
     claimRefreshTag,
     reportRefreshResult,
+    getRefreshStats,
     data,
   };
 
@@ -3556,4 +3571,78 @@ export function createStore(filePath, options = {}) {
     markStatsDirty();
     return { ok: true, moved: deleted.changes || 0 };
   }
+
+  // 将单个 job 移动到 jobs_base（用于目标商家页面"移到base重走流程"）
+  function moveSingleJobToBase(uniqueId) {
+    const db = getDb();
+    if (!db) return { ok: false, error: "db not ready" };
+
+    const safeId = String(uniqueId).trim();
+    if (!safeId) return { ok: false, error: "invalid uniqueId" };
+
+    const existing = db
+      .prepare("SELECT * FROM jobs WHERE unique_id = ?")
+      .get(safeId);
+    if (!existing) return { ok: false, error: "user not found in jobs" };
+
+    const COLUMNS = [
+      "unique_id",
+      "nickname",
+      "status",
+      "sources",
+      "claimed_by",
+      "claimed_at",
+      "error",
+      "pinned",
+      "no_video",
+      "restricted",
+      "user_update_count",
+      "tt_seller",
+      "verified",
+      "video_count",
+      "comment_count",
+      "guessed_location",
+      "location_created",
+      "confirmed_location",
+      "modified_at",
+      "follower_count",
+      "following_count",
+      "heart_count",
+      "refresh_time",
+      "processed",
+      "processed_at",
+      "created_at",
+      "updated_at",
+      "region",
+      "signature",
+      "bio_link",
+      "sec_uid",
+      "status_code",
+      "latest_video_time",
+    ];
+    const cols = COLUMNS.join(",");
+
+    try {
+      db.prepare(
+        `INSERT OR REPLACE INTO jobs_base (${cols})
+         SELECT ${cols} FROM jobs WHERE unique_id = ?`,
+      ).run(safeId);
+
+      // 重置 user_update_count，保留 tt_seller，状态改为 pending
+      db.prepare(
+        `UPDATE jobs_base SET user_update_count = 0, status = 'pending',
+         claimed_by = NULL, claimed_at = NULL, processed = 0, processed_at = NULL,
+         error = NULL, restricted = 0
+         WHERE unique_id = ?`,
+      ).run(safeId);
+
+      db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(safeId);
+
+      markStatsDirty();
+      console.error(`[DB] moveSingleJobToBase: ${safeId} → jobs_base`);
+      return { ok: true };
+    } catch (e) {
+      return { ok: false, error: e.message };
+    }
+  }
 }

package/src/watch/db-crud.js

@@ -479,8 +479,23 @@ export function getPendingJobsUserUpdateCount() {
     .get().c;
 }
 
+// raw_jobs 计数缓存：raw_jobs 表数据量大（千万级），COUNT(*) 全表扫描耗时长，
+// 且该值变化频率低（仅在 batchUpdateUserInfo 移入非商家时变化），缓存 60 秒。
+let _rawJobsCountCache = null;
+let _rawJobsCountCacheTime = 0;
+const RAW_JOBS_COUNT_CACHE_TTL = 60000; // 60 秒
+
 export function getRawJobsCount() {
+  const now = Date.now();
+  if (
+    _rawJobsCountCache !== null &&
+    now - _rawJobsCountCacheTime < RAW_JOBS_COUNT_CACHE_TTL
+  ) {
+    return _rawJobsCountCache;
+  }
   const db = getDb();
   if (!db) return 0;
-  return db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
+  _rawJobsCountCache = db.prepare("SELECT COUNT(*) as c FROM raw_jobs").get().c;
+  _rawJobsCountCacheTime = now;
+  return _rawJobsCountCache;
 }

package/src/watch/db-schema.js

@@ -367,6 +367,7 @@ export function initDb(filePath) {
       round INTEGER NOT NULL DEFAULT 1,
       strategy TEXT,
       tags_generated TEXT NOT NULL DEFAULT '[]',
+      tags_raw TEXT,
       tags_added INTEGER DEFAULT 0,
       created_at TEXT NOT NULL DEFAULT (datetime('now')),
       productive_sample TEXT,
@@ -375,6 +376,10 @@ export function initDb(filePath) {
       avg_dead_score REAL DEFAULT 0
     )
   `);
+  // 迁移：tags_raw 列（用于记录 LLM 原始生成的 tag，去重前）
+  try {
+    db.exec("ALTER TABLE discover_log ADD COLUMN tags_raw TEXT");
+  } catch {}
 
   // ===== reflection_rules 表（反思闭环：LLM 自我学习规则） =====
   db.exec(`

package/src/watch/db-stats.js

@@ -61,11 +61,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
         COALESCE(location_created, '未知') as country,
         COUNT(*) as count,
         SUM(CASE
-          WHEN tt_seller = 1 AND verified = 0 ${
-            targetLocations.length
-              ? `AND location_created IN (${targetPlaceholders})`
-              : "AND 1 = 0"
-          }
+          WHEN tt_seller = 1 AND verified = 0
           THEN 1 ELSE 0 END) as targetCount
       FROM jobs
       WHERE status = 'done'
@@ -73,7 +69,7 @@ export function getDashboardStatsFromDb(targetLocations = []) {
       ORDER BY count DESC
     `,
     )
-    .all(...targetParams);
+    .all();
 
   const targetCountryStats = targetLocations.length
     ? db

package/src/watch/db-tags.js

@@ -22,7 +22,13 @@ export function insertTag(tag, countries, source = "llm") {
     return { inserted: false, error: "invalid tag" };
   }
   try {
-    // ON CONFLICT: 合并 countries 数组，避免跨国家共享 tag 时信息丢失
+    // 先查是否已存在且为 dead 状态，如果是则重置为 new（给二次机会）
+    const existing = db
+      .prepare("SELECT status FROM tags WHERE tag = ?")
+      .get(normalized);
+    const resetDead = existing && existing.status === "dead";
+
+    // ON CONFLICT: 合并 countries 数组；如果原状态是 dead 则重置为 new
     const result = db
       .prepare(
         `INSERT INTO tags (tag, countries, source) VALUES (@tag, @countries, @source)
@@ -32,9 +38,12 @@ export function insertTag(tag, countries, source = "llm") {
              UNION
              SELECT value FROM json_each(@countries)
            )
-         )`,
+         )${resetDead ? ", status = 'new', score = NULL, last_error = NULL, dead_reason = NULL, scored_at = NULL, score_count = 0" : ""}`,
       )
       .run({ tag: normalized, countries: JSON.stringify(countries), source });
+    if (resetDead) {
+      console.error(`[tags] 🔄 dead tag #${normalized} 重新生成，重置为 new`);
+    }
     return { inserted: result.changes > 0, tag: normalized };
   } catch (e) {
     return { inserted: false, error: e.message };
@@ -55,7 +64,8 @@ export function getTagsByStatus(
     sql += " AND countries LIKE ?";
     params.push(`%"${country}"%`);
   }
-  sql += " ORDER BY score ASC, created_at ASC LIMIT ? OFFSET ?";
+  sql +=
+    " ORDER BY refreshed_at DESC NULLS LAST, score ASC, created_at ASC LIMIT ? OFFSET ?";
   params.push(limit, offset);
   const rows = db.prepare(sql).all(...params);
   return rows.map(parseTagRow);
@@ -96,6 +106,48 @@ export function getTagStats(country = null) {
   };
 }
 
+export function getRefreshStats(hours = 24) {
+  const db = getDb();
+  if (!db) return { countries: [], total: 0, refreshed: 0 };
+
+  const rows = db
+    .prepare(
+      "SELECT countries FROM tags WHERE status = 'productive' AND score > 70",
+    )
+    .all();
+  const countrySet = new Set();
+  for (const r of rows) {
+    try {
+      for (const c of JSON.parse(r.countries || "[]")) countrySet.add(c);
+    } catch {}
+  }
+
+  const results = [];
+  let totalAll = 0;
+  let refreshedAll = 0;
+  const countryList = [...countrySet].sort();
+
+  const stmtTotal = db.prepare(
+    "SELECT COUNT(*) as cnt FROM tags WHERE status = 'productive' AND score > 70 AND countries LIKE ?",
+  );
+  const stmtRefresh = db.prepare(
+    `SELECT COUNT(*) as cnt FROM tags WHERE status = 'productive' AND score > 70 AND countries LIKE ? AND refreshed_at >= datetime('now', ?)`,
+  );
+
+  for (const c of countryList) {
+    const like = `%"${c}"%`;
+    const total = stmtTotal.get(like).cnt;
+    const refreshed = stmtRefresh.get(like, `-${hours} hours`).cnt;
+    if (total > 0) {
+      totalAll += total;
+      refreshedAll += refreshed;
+      results.push({ country: c, total, refreshed });
+    }
+  }
+
+  return { countries: results, total: totalAll, refreshed: refreshedAll };
+}
+
 export function getTagsByCountry(country, minScore = 0) {
   const db = getDb();
   if (!db) return [];
@@ -233,7 +285,8 @@ export function getAllTags(limit = 200, offset = 0, country = null) {
     sql += " WHERE countries LIKE ?";
     params.push(`%"${country}"%`);
   }
-  sql += " ORDER BY score DESC, created_at DESC LIMIT ? OFFSET ?";
+  sql +=
+    " ORDER BY refreshed_at DESC NULLS LAST, score DESC, created_at DESC LIMIT ? OFFSET ?";
   params.push(limit, offset);
   const rows = db.prepare(sql).all(...params);
   return rows.map(parseTagRow);
@@ -319,6 +372,7 @@ export function insertDiscoverLog({
   round,
   strategy,
   tagsGenerated,
+  tagsRaw,
   tagsAdded,
   productiveSample,
   deadSample,
@@ -332,15 +386,16 @@ export function insertDiscoverLog({
     const result = db
       .prepare(
         `INSERT INTO discover_log
-         (country, round, strategy, tags_generated, tags_added,
+         (country, round, strategy, tags_generated, tags_raw, tags_added,
           productive_sample, dead_sample, avg_productive_score, avg_dead_score)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       )
       .run(
         country,
         round,
         strategy || null,
         JSON.stringify(tagsGenerated || []),
+        tagsRaw ? JSON.stringify(tagsRaw) : null,
         tagsAdded || 0,
         JSON.stringify(productiveSample || []),
         JSON.stringify(deadSample || []),
@@ -386,6 +441,51 @@ export function getAllExistingTagNames() {
   return new Set(rows.map((r) => r.tag));
 }
 
+/**
+ * 获取活跃 tag 名集合（仅 productive + scored），用于全局去重
+ * 排除 dead tag，因为 insertTag 会在冲突时将 dead 重置为 new。
+ */
+export function getActiveTagNames() {
+  const db = getDb();
+  if (!db) return new Set();
+  const rows = db
+    .prepare("SELECT tag FROM tags WHERE status IN ('productive', 'scored')")
+    .all();
+  return new Set(rows.map((r) => r.tag));
+}
+
+/**
+ * 获取某国家最近一轮 discover 中 LLM 原始生成的全部 tag（去重前）
+ * 用于反馈给 LLM："你上轮生成了这些，全部已被数据库覆盖，换方向"
+ */
+export function getLastRoundAllGeneratedTags(country) {
+  const db = getDb();
+  if (!db) return null;
+
+  const row = db
+    .prepare(
+      `SELECT tags_raw, tags_generated, tags_added FROM discover_log
+       WHERE country = ?
+       ORDER BY round DESC
+       LIMIT 1`,
+    )
+    .get(country);
+
+  if (!row) return null;
+
+  const raw = JSON.parse(row.tags_raw || "[]");
+  const survived = JSON.parse(row.tags_generated || "[]");
+  const rejected = raw.filter((t) => !survived.includes(t));
+
+  return {
+    raw,
+    survived,
+    rejected,
+    tagsAdded: row.tags_added,
+    allRejected: row.tags_added === 0 && raw.length > 0,
+  };
+}
+
 export function getRecentGeneratedTags(country, lookback = 10) {
   const db = getDb();
   if (!db) return [];