tt-help-cli-ycl 1.4.21 → 1.4.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.4.21",
+  "version": "1.4.23",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/comments.js

@@ -35,6 +35,7 @@ import {
 } from "../scraper/modules/captcha-handler.js";
 import { detectModals, closeAllModals } from "../lib/modal-handler.js";
 import { getMacOrUuid } from "../lib/mac-or-uuid.js";
+import { parseVideoInfo } from "../lib/parse-ssr.mjs";
 
 // ────────────────────────────────────────
 // Section 1: 常量 & 配置默认值
@@ -492,6 +493,49 @@ function extractVideoInfo() {
   return result;
 }
 
+/**
+ * 从视频页 SSR 数据中提取 challenges（标签/hashtags）
+ * @param {Page} page - Playwright page
+ * @returns {Promise<{ tagNames: string[], challenges: Array }>}
+ */
+async function extractVideoTags(page) {
+  try {
+    const html = await page.evaluate(() => document.documentElement.outerHTML);
+    const videoInfo = parseVideoInfo(html);
+    if (
+      !videoInfo ||
+      !videoInfo.challenges ||
+      videoInfo.challenges.length === 0
+    ) {
+      return { tagNames: [], challenges: [] };
+    }
+    const tagNames = videoInfo.challenges.map((c) => c.title).filter(Boolean);
+    return { tagNames, challenges: videoInfo.challenges };
+  } catch (e) {
+    console.error(`        [tags] 提取失败: ${e.message}`);
+    return { tagNames: [], challenges: [] };
+  }
+}
+
+/**
+ * 将视频 tag 保存到服务端（只插入不存在的，不覆盖已有）
+ * @returns {Promise<{ inserted: number }>}
+ */
+async function saveVideoTags(apiPost, serverUrl, tagNames, country) {
+  if (!tagNames || tagNames.length === 0) return { inserted: 0 };
+  try {
+    const result = await apiPost(`${serverUrl}/api/tags/batch-add-if-new`, {
+      tags: tagNames,
+      countries: [country],
+      source: "video",
+    });
+    return { inserted: result.inserted || 0 };
+  } catch (e) {
+    console.error(`        [tags] 保存失败: ${e.message}`);
+    return { inserted: 0 };
+  }
+}
+
 function extractUniqueAuthors(comments) {
   return [
     ...new Set(
@@ -668,7 +712,7 @@ async function runAutoMode(options) {
           if (shuttingDown) break;
 
           try {
-            const { id, href, locationCreated } = task;
+            const { id, href, locationCreated, ttSeller } = task;
             const loc = normalizeLocation(locationCreated);
             const taskNum = totalTasks() + 1;
 
@@ -691,6 +735,24 @@ async function runAutoMode(options) {
             // 导航到视频页
             await prepareVideoPage(session.page, href);
 
+            // ── 提取视频 tag（仅 ttSeller 视频）──
+            if (ttSeller && loc) {
+              const { tagNames } = await extractVideoTags(session.page);
+              if (tagNames.length > 0) {
+                const { inserted } = await saveVideoTags(
+                  apiPost,
+                  serverUrl,
+                  tagNames,
+                  loc,
+                );
+                if (inserted > 0) {
+                  console.error(
+                    `        [tags] 提取到 ${tagNames.length} 个: ${tagNames.join(", ")} → 新增 ${inserted} 个`,
+                  );
+                }
+              }
+            }
+
             // 获取评论
             const result = await fetchUserCommentsAPI(session.page, {
               maxComments: actualMaxComments,

package/src/cli/refresh.js

@@ -541,7 +541,7 @@ export async function handleRefresh(options) {
 
       const payload = {
         userInfo: refreshUserInfo,
-        topRecentVideo: result.topRecentVideo || null,
+        recentEcVideos: result.recentEcVideos || null,
         discoveredFollowing: (result.discoveredFollowing || []).map((f) => ({
           handle: Array.isArray(f) ? f[0] : f,
           displayName: Array.isArray(f) ? f[1] : null,

package/src/lib/parse-ssr.mjs

@@ -138,11 +138,17 @@ export function parseVideoInfo(rawHtml) {
   const item = vd.itemInfo.itemStruct;
   const author = item.author || {};
   const stats = item.stats || {};
+  const challenges = (item.challenges || []).map((c) => ({
+    id: c.id,
+    title: c.title,
+    desc: c.desc || "",
+  }));
   return {
     id: item.id,
     desc: item.desc || "",
     createTime: item.createTime || null,
     locationCreated: item.locationCreated || null,
+    challenges,
     author: {
       uniqueId: author.uniqueId,
       nickname: author.nickname,

package/src/lib/tag-discover.js

@@ -158,11 +158,15 @@ async function discoverTagsForCountryCli(
       `  🏥 ${country} health: ${health.healthStatus} | deadRatio=${health.deadRatio}% | productiveRatio=${health.productiveRatio}% | saturation=${health.saturation}%\n`,
     );
 
-    // 动态温度调整
-    if (health.healthStatus === "critical") {
-      temperature = 0.3;
-    } else if (health.healthStatus === "warning") {
+    // 动态温度调整：卡死时升温打破 LLM 确定性重复
+    if (health.consecutiveLowProductiveRounds >= 3) {
+      temperature = 1.0;
+    } else if (health.consecutiveLowProductiveRounds >= 2) {
+      temperature = 0.9;
+    } else if (health.healthStatus === "critical") {
       temperature = 0.5;
+    } else if (health.healthStatus === "warning") {
+      temperature = 0.6;
     }
 
     // 自动 niche 切换

package/src/lib/tag-fetcher.js

@@ -166,6 +166,11 @@ export async function fetchTagData(tag, options = {}) {
       throw new Error("标签页加载失败，TikTok 返回了错误页面");
     }
 
+    // 检测 CDN/网络拦截：15 秒后仍无任何 API 数据到达，说明被拦截而非真没数据
+    if (seenVideoIds.size === 0 && challengeInfo === null) {
+      throw new Error("无法获取标签页 API 数据（可能被 CDN 拦截或网络异常）");
+    }
+
     let lastCount = 0;
     let staleRounds = 0;
     const startTime = Date.now();

package/src/scraper/explore-core.js

@@ -42,7 +42,7 @@ async function processExplore(page, username, options, log) {
     keepFollow: false,
     locationCreated: null,
     latestVideoTime: null,
-    topRecentVideo: null,
+    recentEcVideos: null,
     noVideo: false,
     restricted: false,
     error: null,
@@ -97,6 +97,19 @@ async function processExplore(page, username, options, log) {
         nowSeconds - v.createTime <= SEVEN_DAYS_SECONDS,
     );
     if (recentVideos.length > 0) {
+      result.recentEcVideos = recentVideos.map((v) => ({
+        id: v.id,
+        href: v.href,
+        playCount: v.playCount,
+        createTime: v.createTime,
+        isAd: v.isAd,
+      }));
+      const maxPlay = recentVideos.reduce(
+        (max, v) => (v.playCount > max ? v.playCount : max),
+        0,
+      );
+      log(`  7天内 EC视频: ${recentVideos.length} 个, 最高播放 ${maxPlay}`);
+      // 兼容旧字段 topRecentVideo
       const topVideo = recentVideos.reduce((max, v) =>
         v.playCount > max.playCount ? v : max,
       );
@@ -107,9 +120,6 @@ async function processExplore(page, username, options, log) {
         createTime: topVideo.createTime,
         isAd: topVideo.isAd,
       };
-      log(
-        `  7天内 EC视频最高播放: ${topVideo.playCount} 次播放 (${recentVideos.length} 个EC候选)`,
-      );
     }
 
     if (videoArray.length <= 0) {

package/src/watch/data-store.js

@@ -86,6 +86,7 @@ import {
 // Tag CRUD
 import {
   insertTag,
+  insertTagsIfNew,
   getTagsByStatus,
   getTagsByCountry,
   getDeadTags,
@@ -319,7 +320,9 @@ function getTargetUsersFromDb(targetLocations = []) {
           tt_seller, verified, follower_count, following_count,
           location_created, latest_video_time, refresh_time,
           guessed_location, pinned, processed_at, video_count,
-          no_video, claimed_by, claimed_at, created_at, updated_at
+          no_video, claimed_by, claimed_at, created_at, updated_at,
+          top_video_play_count, top_video_href, top_video_is_ad,
+          recent_ec_videos
         FROM jobs
         WHERE tt_seller = 1
         AND verified = 0
@@ -348,6 +351,8 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
     search,
     limit,
     offset,
+    sort,
+    sortOrder,
   } = options;
   const placeholders = targetLocations.map(() => "?").join(", ");
   const baseParams = [...targetLocations];
@@ -400,6 +405,7 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
         top_video_play_count,
         top_video_href,
         top_video_is_ad,
+        recent_ec_videos,
         status,
         sources
       FROM jobs
@@ -440,6 +446,50 @@ function getTargetUsersByCountryFromDb(targetLocations = [], options = {}) {
         .prepare(countSql)
         .get(...params)?.cnt || 0;
 
+    // 服务端排序：当 sort=topVideoPlayCount 时，全量取出后按 EC 最大播放量排序再分页
+    if (sort === "topVideoPlayCount") {
+      const allRows = getDb()
+        .prepare(sql)
+        .all(...params)
+        .map(mapJobRow);
+
+      const getMaxPlay = (u) => {
+        try {
+          const ec =
+            typeof u.recentEcVideos === "string"
+              ? JSON.parse(u.recentEcVideos)
+              : Array.isArray(u.recentEcVideos)
+                ? u.recentEcVideos
+                : [];
+          if (ec.length > 0) {
+            return ec.reduce(
+              (max, v) => (v.playCount > max ? v.playCount : max),
+              0,
+            );
+          }
+        } catch (e) {}
+        return 0;
+      };
+
+      const isAsc = sortOrder === "asc";
+      allRows.sort((a, b) => {
+        const va = getMaxPlay(a);
+        const vb = getMaxPlay(b);
+        return isAsc ? va - vb : vb - va;
+      });
+
+      const safeLimit = Math.min(Math.floor(limit), 10000);
+      const safeOffset = Math.max(Math.floor(offset), 0);
+      const rows = allRows.slice(safeOffset, safeOffset + safeLimit);
+
+      return {
+        total,
+        limit: safeLimit,
+        offset: safeOffset,
+        users: rows,
+      };
+    }
+
     sql += ` LIMIT ? OFFSET ?`;
     const safeLimit = Math.min(Math.floor(limit), 10000);
     const safeOffset = Math.max(Math.floor(offset), 0);
@@ -2413,6 +2463,7 @@ export function createStore(filePath, options = {}) {
         "uniqueId",
         "sources",
         "topRecentVideo", // 单独处理，不进入通用循环
+        "recentEcVideos", // 单独处理，不进入通用循环
       ];
       for (const key of Object.keys(result)) {
         if (extraFields.includes(key)) continue;
@@ -2430,6 +2481,15 @@ export function createStore(filePath, options = {}) {
         user.topVideoHref = result.topRecentVideo.href || null;
         user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
       }
+      // 将 recentEcVideos 数组序列化为 JSON 字符串
+      if (
+        Array.isArray(result.recentEcVideos) &&
+        result.recentEcVideos.length > 0
+      ) {
+        user.recentEcVideos = JSON.stringify(result.recentEcVideos);
+      } else if (result.recentEcVideos === null) {
+        user.recentEcVideos = null;
+      }
       user.sources = [...new Set([...(user.sources || []), "processed"])];
     }
     if (user.status !== oldStatus) markStatsDirty();
@@ -2657,6 +2717,15 @@ export function createStore(filePath, options = {}) {
         user.topVideoHref = result.topRecentVideo.href || null;
         user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
       }
+      // 将 recentEcVideos 数组序列化为 JSON 字符串
+      if (
+        Array.isArray(result.recentEcVideos) &&
+        result.recentEcVideos.length > 0
+      ) {
+        user.recentEcVideos = JSON.stringify(result.recentEcVideos);
+      } else if (result.recentEcVideos === null) {
+        user.recentEcVideos = null;
+      }
       const newUsers = processDiscoveredUsers(result);
       const ret = updateJobInfo(uniqueId, user, false);
       if (ret.error) return { saved: false, error: ret.error };
@@ -2683,6 +2752,15 @@ export function createStore(filePath, options = {}) {
       user.topVideoHref = result.topRecentVideo.href || null;
       user.topVideoIsAd = result.topRecentVideo.isAd ?? null;
     }
+    // 将 recentEcVideos 数组序列化为 JSON 字符串
+    if (
+      Array.isArray(result.recentEcVideos) &&
+      result.recentEcVideos.length > 0
+    ) {
+      user.recentEcVideos = JSON.stringify(result.recentEcVideos);
+    } else if (result.recentEcVideos === null) {
+      user.recentEcVideos = null;
+    }
     const newUsers = processDiscoveredUsers(result);
     return { saved: true, newUsers };
   }
@@ -3601,6 +3679,7 @@ export function createStore(filePath, options = {}) {
     getLlmSampleOffsets, // 获取 LLM 采样偏移量状态
     // Tag 发现与打分
     insertTag,
+    insertTagsIfNew,
     getTagsByStatus,
     getTagsByCountry,
     getDeadTags,

package/src/watch/db-columns.js

@@ -49,6 +49,7 @@ export const JOB_EXT_COLUMNS = [
   "top_video_play_count",
   "top_video_href",
   "top_video_is_ad",
+  "recent_ec_videos",
 ];
 
 // 完整的 jobs 列 = 核心列 + 扩展列
@@ -91,6 +92,7 @@ export const WRITABLE_JOB_COLUMNS = new Set([
   "top_video_href",
   "top_video_is_ad",
   "user_create_time",
+  "recent_ec_videos",
 ]);
 
 // ===== Boolean 列（存储时 0/1，读取时转为 true/false/null） =====

package/src/watch/db-schema.js

@@ -129,6 +129,7 @@ export function initDb(filePath) {
     top_video_href: "TEXT",
     top_video_is_ad: "INTEGER",
     user_create_time: "INTEGER",
+    recent_ec_videos: "TEXT",
   });
 
   // ===== jobs_base 表 =====