tt-help-cli-ycl 1.4.12 → 1.4.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.12",
+  "version": "1.4.13",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/comments.js

@@ -528,6 +528,7 @@ async function runAutoMode(options) {
     maxComments,
     basePort,
     proxy: proxyOption,
+    locations,
   } = options;
 
   const actualParallel = Math.max(1, parallel || CFG.parallel);
@@ -535,6 +536,7 @@ async function runAutoMode(options) {
   const actualMaxComments = maxComments || CFG.maxComments;
   const effectiveProxy = proxyOption || configuredProxy;
   const portPoolStart = basePort || CFG.portPoolStart;
+  const jobLocations = locations || null;
 
   loadEnvConfig();
 
@@ -597,8 +599,9 @@ async function runAutoMode(options) {
   console.error(`\n╔══════════════════════════════════════════╗`);
   console.error(`║     自动评论采集模式（CDP 连接已登录浏览器）`);
   console.error(
-    `║  目标国家: ${DEFAULT_TARGET_LOCATIONS.slice(0, 6).join(",")}...`,
+    `║  目标国家: ${jobLocations || DEFAULT_TARGET_LOCATIONS.slice(0, 6).join(",") + "..."}`,
   );
+  if (jobLocations) console.error(`║  任务过滤: ${jobLocations}`);
   console.error(`║  服务端:   ${serverUrl}`);
   console.error(
     `║  CDP 端口池: ${portPoolStart}-${portPoolStart + CFG.portPoolSize - 1}（当前: ${session.port}）`,
@@ -635,9 +638,10 @@ async function runAutoMode(options) {
         // ── 领取任务 ──
         let tasks;
         try {
-          const resp = await apiGet(
-            `${serverUrl}/api/comment-tasks?limit=${actualParallel}`,
-          );
+          const apiUrl = jobLocations
+            ? `${serverUrl}/api/comment-tasks?limit=${actualParallel}&locations=${encodeURIComponent(jobLocations)}`
+            : `${serverUrl}/api/comment-tasks?limit=${actualParallel}`;
+          const resp = await apiGet(apiUrl);
           tasks = resp.tasks || [];
         } catch (err) {
           console.error(`  × 获取任务失败: ${err.message}`);
@@ -1031,6 +1035,7 @@ export async function handleComments(options) {
     commentsServer,
     commentsBasePort,
     commentsProxy,
+    commentsLocations,
   } = options;
 
   // 自动模式：无 URL
@@ -1042,6 +1047,7 @@ export async function handleComments(options) {
       maxComments: commentsMax,
       basePort: commentsBasePort,
       proxy: commentsProxy,
+      locations: commentsLocations,
     });
   }
 

package/src/lib/args.js

@@ -670,6 +670,7 @@ function parseCommentsArgs(args) {
   let commentsServer = defaultServer;
   let commentsBasePort = 6222;
   let commentsProxy = null;
+  let commentsLocations = null;
 
   const positional = [];
 
@@ -689,6 +690,12 @@ function parseCommentsArgs(args) {
       commentsBasePort = parseInt(args[++i]) || 6222;
     } else if (arg === "--proxy") {
       commentsProxy = args[++i];
+    } else if (
+      arg === "-l" ||
+      arg === "--location" ||
+      arg === "--job-locations"
+    ) {
+      commentsLocations = args[++i];
     } else {
       positional.push(arg);
     }
@@ -709,6 +716,7 @@ function parseCommentsArgs(args) {
     commentsServer,
     commentsBasePort,
     commentsProxy,
+    commentsLocations,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,

package/src/lib/constants.js

@@ -217,10 +217,13 @@ const HELP_TEXT = [
   "      -i, --interval       空闲间隔上限秒（默认: 10，实际 1~N 随机）",
   "      -s, --server         服务端地址（默认: http://127.0.0.1:3001）",
   "      -m, --max-comments   每视频最大评论数（默认: 200）",
-  "      --base-port          浏览器CDP端口（默认: 9222）",
+  "      -l, --location       按国家领取任务，逗号分隔（如: DE,FR,IT）",
+  "                           未指定时领取所有国家的任务",
+  "      --base-port          浏览器CDP端口（默认: 6222）",
   "      --save               去重后保存到服务端，来源标记为 comment",
   "    示例: tt-help comments",
-  "          tt-help comments -p 2 -i 15 --base-port 9223",
+  "          tt-help comments -p 2 -i 15 --base-port 6223",
+  "          tt-help comments -l DE,FR,IT -p 5",
   "          tt-help comments <视频URL> 50 --save",
   "",
   "  tag  <标签名> [...] [选项]",

package/src/watch/data-store.js

@@ -3130,20 +3130,32 @@ export function createStore(filePath, options = {}) {
     return videos.length;
   }
 
-  function getPendingCommentTasks(limit) {
+  function getPendingCommentTasks(limit, locations = null) {
     if (getDb()) {
       const l = Math.max(1, parseInt(limit) || 1);
+      const normalizedLocations =
+        Array.isArray(locations) && locations.length
+          ? locations
+              .map((loc) => String(loc).trim().toUpperCase())
+              .filter(Boolean)
+          : null;
+
+      const locationFilter = normalizedLocations
+        ? `AND UPPER(COALESCE(location_created, '')) IN (${normalizedLocations.map(() => "?").join(", ")})`
+        : "";
+
       const rows = getDb()
         .prepare(
           `
           SELECT *
           FROM videos
-          WHERE user_update_count IS NULL OR user_update_count <= 0
+          WHERE (user_update_count IS NULL OR user_update_count <= 0)
+          ${locationFilter}
           ORDER BY tt_seller DESC, registered_at ASC
           LIMIT ?
         `,
         )
-        .all(l);
+        .all(...(normalizedLocations || []), l);
       if (rows.length === 0) return [];
       const bumpStmt = getDb().prepare(
         `
@@ -3164,7 +3176,19 @@ export function createStore(filePath, options = {}) {
     }
 
     // 筛选待处理视频（userUpdateCount <= 0 或 null/undefined）
-    const pending = videos.filter((v) => (v.userUpdateCount || 0) <= 0);
+    let pending = videos.filter((v) => (v.userUpdateCount || 0) <= 0);
+    // 按 locations 过滤
+    if (Array.isArray(locations) && locations.length > 0) {
+      const upperLocs = locations.map((loc) =>
+        String(loc).trim().toUpperCase(),
+      );
+      pending = pending.filter((v) => {
+        const loc = String(v.locationCreated || "")
+          .trim()
+          .toUpperCase();
+        return upperLocs.includes(loc);
+      });
+    }
     // ttSeller=true 优先
     pending.sort((a, b) => {
       if (a.ttSeller && !b.ttSeller) return -1;

package/src/watch/db-schema.js

@@ -278,8 +278,84 @@ export function initDb(filePath) {
   migrateColumns("tags", {
     discover_round: "INTEGER DEFAULT 0",
     strategy: "TEXT",
+    dead_reason: "TEXT",
+    direction_id: "INTEGER",
   });
 
+  // ===== tag_directions 表（分层探索：通用词/分类/产品） =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tag_directions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      name TEXT NOT NULL,
+      level INTEGER NOT NULL DEFAULT 1,
+      parent_id INTEGER,
+      country TEXT,
+      lang_group TEXT,
+      status TEXT NOT NULL DEFAULT 'active',
+      total_experiments INTEGER DEFAULT 0,
+      productive_experiments INTEGER DEFAULT 0,
+      best_score REAL DEFAULT 0,
+      notes TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at TEXT
+    )
+  `);
+  migrateColumns("tag_directions", {
+    lang_group: "TEXT",
+  });
+
+  // ===== tag_experiments 表（每次探索的实验记录） =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tag_experiments (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      direction_id INTEGER,
+      country TEXT NOT NULL,
+      hypothesis TEXT,
+      strategy TEXT,
+      tags_tried TEXT NOT NULL DEFAULT '[]',
+      tags_productive TEXT NOT NULL DEFAULT '[]',
+      tags_dead_not_exist TEXT NOT NULL DEFAULT '[]',
+      tags_dead_no_match TEXT NOT NULL DEFAULT '[]',
+      conclusion TEXT,
+      productive_count INTEGER DEFAULT 0,
+      dead_count INTEGER DEFAULT 0,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+
+  // ===== tag_analyst_reports 表（LLM 分析师定期体检报告） =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS tag_analyst_reports (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      report_type TEXT NOT NULL DEFAULT 'daily',
+      countries TEXT NOT NULL DEFAULT '[]',
+      summary TEXT,
+      top_directions TEXT NOT NULL DEFAULT '[]',
+      worst_directions TEXT NOT NULL DEFAULT '[]',
+      cross_country_suggestions TEXT NOT NULL DEFAULT '[]',
+      cleanup_suggestions TEXT NOT NULL DEFAULT '[]',
+      next_strategy TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+
+  // ===== cross_country_learnings 表（跨国家经验复用记录） =====
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS cross_country_learnings (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      source_country TEXT NOT NULL,
+      source_tag TEXT NOT NULL,
+      source_direction_id INTEGER,
+      target_country TEXT NOT NULL,
+      target_tag TEXT,
+      transferred_at TEXT,
+      result_status TEXT,
+      score REAL,
+      notes TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+
   // ===== discover_log 表（记录每次 LLM discover 的策略和思考） =====
   db.exec(`
     CREATE TABLE IF NOT EXISTS discover_log (
@@ -334,6 +410,12 @@ export function initDb(filePath) {
     `CREATE INDEX IF NOT EXISTS idx_tags_score ON tags(score DESC)`,
     `CREATE INDEX IF NOT EXISTS idx_discover_log_country ON discover_log(country, round DESC)`,
     `CREATE INDEX IF NOT EXISTS idx_reflection_rules_active ON reflection_rules(country, is_active)`,
+    `CREATE INDEX IF NOT EXISTS idx_tag_directions_country ON tag_directions(country, level)`,
+    `CREATE INDEX IF NOT EXISTS idx_tag_experiments_country ON tag_experiments(country, created_at DESC)`,
+    `CREATE INDEX IF NOT EXISTS idx_tag_experiments_direction ON tag_experiments(direction_id)`,
+    `CREATE INDEX IF NOT EXISTS idx_cross_country_source ON cross_country_learnings(source_country, result_status)`,
+    `CREATE INDEX IF NOT EXISTS idx_cross_country_target ON cross_country_learnings(target_country)`,
+    `CREATE INDEX IF NOT EXISTS idx_tags_dead_reason ON tags(dead_reason)`,
   ];
   for (const sql of indexes) {
     db.exec(sql);

package/src/watch/db-tags.js

@@ -171,6 +171,23 @@ export function reportTagScore(tag, fields) {
     : null;
   const now = new Date().toISOString();
 
+  // 自动分类 dead_reason
+  let deadReason = null;
+  if (status === "dead" && error) {
+    const errLower = error.toLowerCase();
+    if (
+      errLower.includes("no videos found") ||
+      errLower.includes("tag_not_found") ||
+      errLower.includes("找不到")
+    ) {
+      deadReason = "not_exist";
+    } else if ((matchedAuthors ?? 0) === 0) {
+      deadReason = "no_match";
+    } else {
+      deadReason = "wrong_country";
+    }
+  }
+
   try {
     const result = db
       .prepare(
@@ -183,6 +200,7 @@ export function reportTagScore(tag, fields) {
           matched_countries = COALESCE(?, matched_countries),
           pushed_users = COALESCE(?, pushed_users),
           last_error = COALESCE(?, last_error),
+          dead_reason = COALESCE(?, dead_reason),
           scored_at = ?,
           score_count = score_count + 1
         WHERE tag = ?`,
@@ -196,6 +214,7 @@ export function reportTagScore(tag, fields) {
         matchedCountriesJson,
         pushedUsers ?? null,
         error ?? null,
+        deadReason,
         now,
         tag,
       );

package/src/watch/server.js

@@ -683,9 +683,18 @@ export function startWatchServer(
 
       if (req.method === "GET" && routePath === "/api/comment-tasks") {
         const limit = parseInt(params.limit) || 1;
-        const tasks = store.getPendingCommentTasks(limit);
+        const locationsParam = params.locations || "";
+        const locations = locationsParam
+          ? locationsParam
+              .split(",")
+              .map((s) => s.trim().toUpperCase())
+              .filter(Boolean)
+          : null;
+        const tasks = store.getPendingCommentTasks(limit, locations);
         const ts = new Date().toISOString().slice(11, 19);
-        console.error(`[JOB ${ts}] COMMENT-TASKS: ${tasks.length} tasks`);
+        console.error(
+          `[JOB ${ts}] COMMENT-TASKS: ${tasks.length} tasks${locations ? " (locations: " + locations.join(",") + ")" : ""}`,
+        );
         sendJSON(res, 200, { total: tasks.length, tasks });
         return;
       }
@@ -1491,6 +1500,327 @@ export function startWatchServer(
         return;
       }
 
+      // ====== 自学习系统 API ======
+
+      // GET /api/tags/selflearn/status — 自学习系统状态总览
+      if (req.method === "GET" && routePath === "/api/tags/selflearn/status") {
+        try {
+          const { getSelfLearnStatus } = await import("./tag-selflearn.js");
+          const status = getSelfLearnStatus();
+          sendJSON(res, 200, status);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/classify-dead — 分类所有 dead tag
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/classify-dead"
+      ) {
+        try {
+          const { classifyDeadTags } = await import("./tag-selflearn.js");
+          const body = await readBody(req);
+          const result = classifyDeadTags(body?.country || null);
+          sendJSON(res, 200, result);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/cleanup-truly-dead — 删除真正不存在的 dead tag
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/cleanup-truly-dead"
+      ) {
+        try {
+          const { cleanupTrulyDeadTags } = await import("./tag-selflearn.js");
+          const result = cleanupTrulyDeadTags();
+          sendJSON(res, 200, result);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // GET /api/tags/selflearn/dead-stats?country=DE — dead tag 分类统计
+      if (
+        req.method === "GET" &&
+        routePath === "/api/tags/selflearn/dead-stats"
+      ) {
+        try {
+          const { getDeadTagStats } = await import("./tag-selflearn.js");
+          const country = params.country || null;
+          const stats = getDeadTagStats(country);
+          sendJSON(res, 200, stats || {});
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/init-directions — 初始化某国的探索方向
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/init-directions"
+      ) {
+        try {
+          const { initDefaultDirections } = await import("./tag-selflearn.js");
+          const body = await readBody(req);
+          const country = body?.country;
+          if (!country) {
+            sendJSON(res, 400, { error: "country 参数必填" });
+            return;
+          }
+          const result = initDefaultDirections(country);
+          sendJSON(res, 200, {
+            ok: true,
+            count: result.length,
+            directions: result,
+          });
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // GET /api/tags/selflearn/directions?country=DE&level=1 — 获取方向列表
+      if (
+        req.method === "GET" &&
+        routePath === "/api/tags/selflearn/directions"
+      ) {
+        try {
+          const { getDirections } = await import("./tag-selflearn.js");
+          const country = params.country;
+          const level = parseInt(params.level) || 1;
+          if (!country) {
+            sendJSON(res, 400, { error: "country 参数必填" });
+            return;
+          }
+          const directions = getDirections(country, level);
+          sendJSON(res, 200, directions);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // GET /api/tags/selflearn/cross-country?country=ES — 跨国家推荐
+      if (
+        req.method === "GET" &&
+        routePath === "/api/tags/selflearn/cross-country"
+      ) {
+        try {
+          const { getCrossCountryRecommendations } =
+            await import("./tag-selflearn.js");
+          const country = params.country;
+          if (!country) {
+            sendJSON(res, 400, { error: "country 参数必填" });
+            return;
+          }
+          const limit = parseInt(params.limit) || 20;
+          const recommendations = getCrossCountryRecommendations(
+            country,
+            limit,
+          );
+          sendJSON(res, 200, recommendations);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/record-cross-country — 记录跨国家学习结果
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/record-cross-country"
+      ) {
+        try {
+          const { recordCrossCountryLearning } =
+            await import("./tag-selflearn.js");
+          const body = await readBody(req);
+          const {
+            sourceCountry,
+            sourceTag,
+            targetCountry,
+            targetTag,
+            resultStatus,
+            score,
+          } = body || {};
+          recordCrossCountryLearning(
+            sourceCountry,
+            sourceTag,
+            targetCountry,
+            targetTag,
+            resultStatus,
+            score,
+          );
+          sendJSON(res, 200, { ok: true });
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/record-experiment — 记录实验
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/record-experiment"
+      ) {
+        try {
+          const { recordExperiment } = await import("./tag-selflearn.js");
+          const body = await readBody(req);
+          const result = recordExperiment(body);
+          sendJSON(res, 200, result);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // GET /api/tags/selflearn/experiments?country=DE&limit=10 — 获取实验历史
+      if (
+        req.method === "GET" &&
+        routePath === "/api/tags/selflearn/experiments"
+      ) {
+        try {
+          const { getExperimentHistory } = await import("./tag-selflearn.js");
+          const country = params.country;
+          if (!country) {
+            sendJSON(res, 400, { error: "country 参数必填" });
+            return;
+          }
+          const limit = parseInt(params.limit) || 10;
+          const experiments = getExperimentHistory(country, limit);
+          sendJSON(res, 200, experiments);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/analyst — 运行 LLM 分析师
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/analyst"
+      ) {
+        try {
+          const { runLLMAnalyst } = await import("./tag-selflearn.js");
+          const body = await readBody(req);
+          const countries = body?.countries || null;
+          const report = await runLLMAnalyst(countries);
+          sendJSON(res, 200, report);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // GET /api/tags/selflearn/analyst/latest — 获取最新分析师报告
+      if (
+        req.method === "GET" &&
+        routePath === "/api/tags/selflearn/analyst/latest"
+      ) {
+        try {
+          const { getLatestAnalystReport } = await import("./tag-selflearn.js");
+          const report = getLatestAnalystReport();
+          sendJSON(res, 200, report || {});
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/analyst/execute-cleanup — 执行分析师建议的清理
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/analyst/execute-cleanup"
+      ) {
+        try {
+          const { executeAnalystCleanup } = await import("./tag-selflearn.js");
+          const result = executeAnalystCleanup();
+          sendJSON(res, 200, result);
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // POST /api/tags/selflearn/discover-layered — 分层探索生成 tag
+      if (
+        req.method === "POST" &&
+        routePath === "/api/tags/selflearn/discover-layered"
+      ) {
+        try {
+          const {
+            buildLayeredDiscoverPrompt,
+            getCrossCountryRecommendations,
+            getExperimentHistory,
+          } = await import("./tag-selflearn.js");
+          const { callLLM, parseTagsFromResponse, normalizeTag } =
+            await import("./tag-service.js");
+          const body = await readBody(req);
+          const { country, level = 1, directionName, count = 10 } = body || {};
+
+          if (!country) {
+            sendJSON(res, 400, { error: "country 参数必填" });
+            return;
+          }
+
+          // 收集分层探索所需数据
+          const storeTags = store.getTagsByCountry(country, 50);
+          const storeDead = store.getDeadTags(country);
+          const crossCountryTips = getCrossCountryRecommendations(country, 5);
+          const experimentHistory = getExperimentHistory(country, 5);
+
+          const prompt = buildLayeredDiscoverPrompt(country, {
+            level,
+            existingProductive: storeTags,
+            existingDead: storeDead,
+            directionName,
+            experimentHistory,
+            crossCountryTips,
+          });
+
+          console.error(
+            `[selflearn] ${country} L${level} discover (direction: ${directionName || "auto"})`,
+          );
+          const content = await callLLM(prompt, 0.7);
+          const tags = parseTagsFromResponse(content);
+
+          // 全局去重
+          const { getAllExistingTagNames } = await import("./db-tags.js");
+          const allExisting = getAllExistingTagNames();
+          const unique = [...new Set(tags)]
+            .filter((t) => !allExisting.has(t))
+            .slice(0, count);
+
+          // 入库
+          const inserted = [];
+          for (const tag of unique) {
+            const result = store.insertTag(tag, [country], "llm");
+            if (result.inserted) inserted.push(result.tag);
+          }
+
+          sendJSON(res, 200, {
+            country,
+            level,
+            directionName,
+            generated: tags.length,
+            afterDedup: unique.length,
+            inserted: inserted.length,
+            tags: inserted,
+          });
+        } catch (e) {
+          sendJSON(res, 500, { error: e.message });
+        }
+        return;
+      }
+
+      // ====== 原有静态资源 ======
+
       if (
         req.method === "GET" &&
         (routePath === "/" || routePath === "/index.html")