tt-help-cli-ycl 1.3.13 → 1.3.15

package/src/scraper/explore-core.js

@@ -1,8 +1,6 @@
 import {
   delay,
   ensureBrowserReady,
-  setDelayConfig,
-  closeCommentPanel,
   retryWithBackoff,
   detectPageError,
   isLoggedIn,
@@ -10,19 +8,14 @@ import {
 } from './modules/page-helpers.js';
 import { detectCaptcha } from './modules/captcha-handler.js';
 export { ensureBrowserReady };
-import {
-  getUserInfo,
-  collectVideos,
-} from '../videos/core.js';
-import { scrapeSingleVideo } from './core.js';
+import { getUserInfo, collectVideos } from '../videos/core.js';
 import { extractFollowAndFollowers } from './modules/follow-extractor.js';
-import { extractCommentAuthors } from './modules/comment-extractor.js';
-import { extractGuessVideos } from './modules/guess-extractor.js';
+import { extractVideoLocation } from '../lib/scrape.js';
+import { maxFollowing as globalMaxFollowing, maxFollowers as globalMaxFollowers, maxVideos as globalMaxVideos } from '../lib/constants.js';
 
 async function processExplore(page, username, options, log) {
   const {
-    maxComments = 0,
-    maxGuess = 0,
+    maxVideos = 16,
     enableFollow = true,
     maxFollowing = 5,
     maxFollowers = 5,
@@ -70,7 +63,10 @@ async function processExplore(page, username, options, log) {
       result.captchaMessage = result.captchaMessage || '视频页出现验证码';
     }
 
-    const videoList = await collectVideos(page, username, 1, log);
+    const isSeller = result.userInfo?.ttSeller === true;
+    const effectiveMaxVideos = isSeller ? globalMaxVideos : maxVideos;
+    if (isSeller) log(`  商家用户，视频采集数: ${effectiveMaxVideos}`);
+    const videoList = await collectVideos(page, username, effectiveMaxVideos, log);
     const videoArray = videoList ? [...videoList.values()] : [];
     result.collectedVideos = videoArray.length;
 
@@ -87,84 +83,70 @@ async function processExplore(page, username, options, log) {
       return result;
     }
 
-    if (enableFollow) {
-      const loggedIn = await isLoggedIn(page);
-      if (!loggedIn) {
-        log('  [跳过] 获取关注/粉丝：未登录，请先登录 TikTok');
-        result.hasFollowData = false;
-        result.discoveredFollowing = [];
-        result.discoveredFollowers = [];
-      } else {
-        try {
-          log('  获取关注/粉丝...');
-          const { following, followers } = await extractFollowAndFollowers(
-            page, { maxFollowing, maxFollowers, log }
-          );
-          result.discoveredFollowing = following || [];
-          result.discoveredFollowers = followers || [];
-          result.hasFollowData = true;
-          log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
-        } catch (e) {
-          log(`  关注/粉丝提取失败: ${e.message}`);
-          result.hasFollowData = false;
-          result.discoveredFollowing = [];
-          result.discoveredFollowers = [];
-        }
+    // 从第一个视频获取 locationCreated
+    let locationCreated = null;
+    if (videoArray.length > 0) {
+      const firstVideo = videoArray[0];
+      const firstVideoUrl = firstVideo.href.startsWith('http')
+        ? firstVideo.href
+        : `https://www.tiktok.com${firstVideo.href}`;
+
+      try {
+        locationCreated = await extractVideoLocation(firstVideoUrl);
+      } catch (e) {
+        log(`  获取视频国家失败: ${e.message}`);
       }
     }
 
-    const firstVideo = videoArray[0];
-    const videoUrl = firstVideo.href.startsWith('http')
-      ? firstVideo.href
-      : `https://www.tiktok.com${firstVideo.href}`;
-
-    log(`  进入第一个视频: ${videoUrl}`);
-    await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
-    assertPageUrl(page, videoUrl.split('/video/')[0]);
-    await delay(1500, 2500);
-
-    const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');
-    result.locationCreated = videoData.locationCreated || null;
-    log(`  视频作者: ${videoData.videoAuthor} | 国家: ${result.locationCreated || '未知'}`);
+    result.locationCreated = locationCreated || null;
+    log(`  国家: ${result.locationCreated || '未知'}`);
 
+    // 国家筛选
     const locationList = (location || 'ES').split(',').map(s => s.trim().toUpperCase());
     const isTargetLocation = locationList.includes(result.locationCreated?.toUpperCase?.() || result.locationCreated);
 
     if (isTargetLocation) {
       result.keepFollow = true;
-      log(`  国家匹配 (${result.locationCreated} in [${location}])，获取评论和猜你喜欢...`);
-
-      if (maxComments > 0) {
-        const commentResult = await extractCommentAuthors(page, maxComments);
-        result.discoveredCommentAuthors = commentResult.authors || [];
-        if (commentResult.captchaDetected) {
-          result.captchaDetected = true;
-          result.captchaStage = 'comment';
-          result.captchaMessage = '评论阶段出现验证码';
-        }
-        await closeCommentPanel(page);
-        await delay(500, 1000);
-        log(`  评论用户: ${result.discoveredCommentAuthors.length}`);
-      }
+      log(`  国家匹配，获取关注/粉丝...`);
 
-      if (maxGuess > 0) {
-        const guessResult = await extractGuessVideos(page, maxGuess);
-        result.discoveredGuessAuthors = (guessResult || []).map(v => v.author).filter(Boolean);
-        await closeCommentPanel(page);
-        await delay(500, 1000);
-        log(`  猜你喜欢作者: ${result.discoveredGuessAuthors.length}`);
+      // 提取关注/粉丝
+      if (enableFollow) {
+        const loggedIn = await isLoggedIn(page);
+        if (!loggedIn) {
+          log('  [跳过] 获取关注/粉丝：未登录，请先登录 TikTok');
+          result.hasFollowData = false;
+          result.discoveredFollowing = [];
+          result.discoveredFollowers = [];
+        } else {
+          try {
+            const effectiveMaxFollowing = isSeller ? globalMaxFollowing : maxFollowing;
+            const effectiveMaxFollowers = isSeller ? globalMaxFollowers : maxFollowers;
+            if (isSeller) log(`  商家用户，关注采集: ${effectiveMaxFollowing}, 粉丝采集: ${effectiveMaxFollowers}`);
+            const { following, followers } = await extractFollowAndFollowers(
+              page, { maxFollowing: effectiveMaxFollowing, maxFollowers: effectiveMaxFollowers, log }
+            );
+            result.discoveredFollowing = following || [];
+            result.discoveredFollowers = followers || [];
+            result.hasFollowData = true;
+            log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
+          } catch (e) {
+            log(`  关注/粉丝提取失败: ${e.message}`);
+            result.hasFollowData = false;
+            result.discoveredFollowing = [];
+            result.discoveredFollowers = [];
+          }
+        }
       }
 
-      result.discoveredVideoAuthors = [{
-        uniqueId: videoData.uniqueId,
-        nickname: videoData.nickname,
-        locationCreated: videoData.locationCreated,
-      }];
+      // 携带视频列表供登记
+      result.videoList = videoArray;
     } else {
+      // 国家不匹配
       result.keepFollow = false;
-      log(`  国家不匹配 (${result.locationCreated} not in [${location}])，跳过评论/猜你喜欢，丢弃关注/粉丝`);
       result.discoveredFollowing = [];
       result.discoveredFollowers = [];
+      result.hasFollowData = false;
+      log(`  国家不匹配，跳过`);
     }
 
     result.processed = true;

package/src/videos/core.js

@@ -1,5 +1,5 @@
 import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.js';
-import { scrollAndCollect } from '../scraper/modules/scroll-collector.js';
+import { fetchUserVideosAPI } from '../lib/api-interceptor.js';
 
 async function getUserInfo(page) {
   return await page.evaluate(() => {
@@ -41,41 +41,12 @@ async function getUserInfo(page) {
 }
 
 async function collectVideos(page, username, maxVideos, log) {
-  const allLinks = await scrollAndCollect(page, {
-    container: '[class*="ColumnListContainer"]',
-    extraArgs: { handle: username },
-    collectFn: (container, args) => {
-      const pattern = '/@' + args.handle + '/video/';
-      return {
-        items: Array.from(document.querySelectorAll('a'))
-          .filter(el => (el.getAttribute('href') || '').includes(pattern))
-          .map(el => {
-            const href = el.getAttribute('href') || '';
-            const idMatch = href.match(/\/video\/(\d+)/);
-            return { id: idMatch ? idMatch[1] : null, href };
-          })
-          .filter(v => v.id),
-      };
-    },
-    maxItems: maxVideos,
-    delayRange: [2000, 3000],
-    staleThreshold: 5,
-    maxRounds: 500,
-    onRound: (round, items, allItems) => {
-      const uniqueCount = new Set(allItems.map(v => v.id)).size;
-      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
-        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
-      }
-    },
-  });
-
-  const uniqueVideos = new Map();
-  allLinks.forEach(v => {
-    if (!uniqueVideos.has(v.id)) uniqueVideos.set(v.id, v);
-  });
-
-  log(`收集完成: ${uniqueVideos.size} 个视频`);
-  return uniqueVideos;
+  const apiResult = await fetchUserVideosAPI(page, username, maxVideos, log);
+  if (apiResult && apiResult.size > 0) {
+    log(`收集完成: ${apiResult.size} 个视频`);
+    return apiResult;
+  }
+  return new Map();
 }
 
 async function runGetUserVideos(options) {

package/src/watch/data-store.js

@@ -12,6 +12,25 @@ export function createStore(filePath) {
   let data = [];
   let clientErrors = new Map();
 
+  // 视频存储（独立 JSON 文件）
+  let videos = [];
+  let videoFilePath = null;
+  if (filePath) {
+    const resolved = path.resolve(filePath);
+    videoFilePath = resolved.replace(/\.json$/, '-videos.json');
+    if (fs.existsSync(videoFilePath)) {
+      try {
+        const content = fs.readFileSync(videoFilePath, 'utf-8');
+        const parsed = JSON.parse(content);
+        if (Array.isArray(parsed)) {
+          videos = parsed;
+        }
+      } catch (e) {
+        console.error(`[data-store] 读取视频文件失败: ${e.message}`);
+      }
+    }
+  }
+
   let backupTimer = null;
 
   if (filePath) {
@@ -66,6 +85,12 @@ export function createStore(filePath) {
     fs.writeFileSync(resolved, json, 'utf-8');
   }
 
+  function saveVideos() {
+    if (!videoFilePath) return;
+    const json = JSON.stringify(videos, null, 2);
+    fs.writeFileSync(videoFilePath, json, 'utf-8');
+  }
+
   function stopBackup() {
     if (backupTimer) {
       clearInterval(backupTimer);
@@ -159,6 +184,12 @@ export function createStore(filePath) {
       next = pickFirst(seed);
     }
 
+    if (!next) {
+      const ttSeller = data.filter(u => u.status === 'pending' && u.ttSeller === true && u.verified === false);
+      ttSeller.sort((a, b) => locationTier(a) - locationTier(b));
+      next = pickFirst(ttSeller);
+    }
+
     if (!next) {
       const follow = data.filter(u => u.status === 'pending' && u.sources && (u.sources.includes('following') || u.sources.includes('follower')));
       follow.sort((a, b) => locationTier(a) - locationTier(b));
@@ -188,25 +219,25 @@ export function createStore(filePath) {
         nickname: typeof v === 'string' ? null : v.nickname || null,
         locationCreated: typeof v === 'string' ? null : v.locationCreated || null,
         guessedLocation: typeof v === 'string' ? guessedLocation : (v.guessedLocation || guessedLocation),
-        source: 'video',
+        sources: ['video'],
       })),
       ...(result.discoveredCommentAuthors || []).map(c => {
-        if (typeof c === 'string') return { uniqueId: c.replace(/^@/, ''), source: 'comment', guessedLocation };
-        return { uniqueId: (c.author || c.uniqueId || '').replace(/^@/, ''), nickname: c.nickname || null, source: 'comment', guessedLocation: c.guessedLocation || guessedLocation };
+        if (typeof c === 'string') return { uniqueId: c.replace(/^@/, ''), sources: ['comment'], guessedLocation };
+        return { uniqueId: (c.author || c.uniqueId || '').replace(/^@/, ''), nickname: c.nickname || null, sources: ['comment'], guessedLocation: c.guessedLocation || guessedLocation };
       }),
       ...(result.discoveredGuessAuthors || []).map(g => {
-        if (typeof g === 'string') return { uniqueId: g.replace(/^@/, ''), source: 'guess', guessedLocation };
-        return { uniqueId: (g.author || g.uniqueId || '').replace(/^@/, ''), nickname: g.nickname || null, source: 'guess', guessedLocation: g.guessedLocation || guessedLocation };
+        if (typeof g === 'string') return { uniqueId: g.replace(/^@/, ''), sources: ['guess'], guessedLocation };
+        return { uniqueId: (g.author || g.uniqueId || '').replace(/^@/, ''), nickname: g.nickname || null, sources: ['guess'], guessedLocation: g.guessedLocation || guessedLocation };
       }),
       ...(result.discoveredFollowing || []).map(f => {
         const handle = Array.isArray(f) ? f[0] : (f.handle || '');
         const name = Array.isArray(f) ? f[1] : (f.displayName || null);
-        return { uniqueId: handle.replace(/^@/, ''), nickname: name, source: 'following', guessedLocation: (typeof f === 'object' && f.guessedLocation) || guessedLocation };
+        return { uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following'], guessedLocation: (typeof f === 'object' && f.guessedLocation) || guessedLocation };
       }),
       ...(result.discoveredFollowers || []).map(f => {
         const handle = Array.isArray(f) ? f[0] : (f.handle || '');
         const name = Array.isArray(f) ? f[1] : (f.displayName || null);
-        return { uniqueId: handle.replace(/^@/, ''), nickname: name, source: 'follower', guessedLocation: (typeof f === 'object' && f.guessedLocation) || guessedLocation };
+        return { uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower'], guessedLocation: (typeof f === 'object' && f.guessedLocation) || guessedLocation };
       }),
     ].filter(u => u.uniqueId);
 
@@ -278,6 +309,7 @@ export function createStore(filePath) {
           user[key] = result[key];
         }
       }
+      user.sources = [...new Set([...(user.sources || []), 'processed'])];
     }
   }
 
@@ -419,12 +451,85 @@ export function createStore(filePath) {
     return Array.from(clientErrors.values());
   }
 
+  function getPendingUserUpdateTasks(limit) {
+    const l = Math.max(1, parseInt(limit) || 5);
+    const pending = data.filter(u => {
+      const updateCount = u.userUpdateCount;
+      const ttSellerEmpty = u.ttSeller === null || u.ttSeller === undefined || u.ttSeller === '';
+      if (!ttSellerEmpty) return false;
+      return updateCount === null || updateCount === undefined || updateCount <= 0;
+    }).slice(0, l);
+    // 接受任务时 userUpdateCount + 1
+    pending.forEach(u => {
+      u.userUpdateCount = (u.userUpdateCount || 0) + 1;
+      u.updatedAt = Date.now();
+    });
+    save();
+    return pending;
+  }
+
+  function updateUserInfo(uniqueId, info) {
+    const user = getUser(uniqueId);
+    if (!user) return { error: 'user not found' };
+    for (const key of Object.keys(info)) {
+      if (key === 'uniqueId' || key === 'sources') continue;
+      if (info[key] !== undefined && info[key] !== null && info[key] !== '') {
+        user[key] = info[key];
+      }
+    }
+    user.userUpdateCount = (user.userUpdateCount || 0) + 1;
+    user.updatedAt = Date.now();
+    save();
+    return { ok: true, userUpdateCount: user.userUpdateCount };
+  }
+
+  // 视频登记
+  function registerVideos(sourceUser, videoList, locationCreated, ttSeller) {
+    if (!videoList || !Array.isArray(videoList) || videoList.length === 0) {
+      return { registered: 0, skipped: 0 };
+    }
+
+    const existingIds = new Set(videos.map(v => v.id));
+    let registered = 0;
+    let skipped = 0;
+
+    for (const item of videoList) {
+      if (existingIds.has(item.id)) {
+        skipped++;
+        continue;
+      }
+      videos.push({
+        id: item.id,
+        href: item.href,
+        authorUniqueId: sourceUser,
+        locationCreated: locationCreated || null,
+        ttSeller: ttSeller || false,
+        registeredAt: Date.now(),
+      });
+      existingIds.add(item.id);
+      registered++;
+    }
+
+    saveVideos();
+    return { registered, skipped };
+  }
+
+  function getVideos() {
+    return videos;
+  }
+
+  function getVideoCount() {
+    return videos.length;
+  }
+
   return {
     save, getUser, hasUser, userExists, addUser,
     getPendingUsers, getProcessedUsers, getAllUsers,
     claimNextJob, commitJob, commitNewExplore, resetJob, togglePin,
     getNextRedoJob, commitRedoJob,
+    getPendingUserUpdateTasks, updateUserInfo,
     reportClientError, deleteClientError, getClientErrors,
+    registerVideos, getVideos, getVideoCount,
     stopBackup,
     data,
   };

package/src/watch/server.js

@@ -257,6 +257,19 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
         return;
       }
 
+      // 视频登记
+      if (req.method === 'POST' && routePath === '/api/videos') {
+        const body = await readBody(req);
+        const { sourceUser, videoList, locationCreated, ttSeller } = body;
+        if (!sourceUser) {
+          sendJSON(res, 400, { error: 'sourceUser 不能为空' });
+          return;
+        }
+        const ret = store.registerVideos(sourceUser, videoList || [], locationCreated, ttSeller);
+        sendJSON(res, 200, ret);
+        return;
+      }
+
       const jobPinMatch = routePath.match(/^\/api\/job\/([^/]+)\/pin$/);
       if (req.method === 'POST' && jobPinMatch) {
         const uniqueId = jobPinMatch[1];
@@ -285,6 +298,34 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
         return;
       }
 
+      if (req.method === 'GET' && routePath === '/api/user-update-tasks') {
+        const limit = params.limit;
+        const tasks = store.getPendingUserUpdateTasks(limit);
+        const ts = new Date().toISOString().slice(11, 19);
+        console.error(`[JOB ${ts}] USER-UPDATE-TASKS: ${tasks.length} tasks`);
+        sendJSON(res, 200, { total: tasks.length, tasks });
+        return;
+      }
+
+      const userInfoCommitMatch = routePath.match(/^\/api\/user-info\/([^/]+)$/);
+      if (req.method === 'PUT' && userInfoCommitMatch) {
+        const uniqueId = userInfoCommitMatch[1];
+        try {
+          const body = await readBody(req);
+          const ret = store.updateUserInfo(uniqueId, body);
+          if (ret.error) {
+            sendJSON(res, 404, { error: ret.error });
+            return;
+          }
+          const ts = new Date().toISOString().slice(11, 19);
+          console.error(`[JOB ${ts}] USER-INFO-UPDATE: ${uniqueId} (userUpdateCount=${ret.userUpdateCount})`);
+          sendJSON(res, 200, ret);
+        } catch (e) {
+          sendJSON(res, 400, { error: e.message });
+        }
+        return;
+      }
+
       const userExistsMatch = routePath.match(/^\/api\/user-exists\/([^/]+)$/);
       if (req.method === 'GET' && userExistsMatch) {
         const uniqueId = userExistsMatch[1];
@@ -415,6 +456,9 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
           if (sa !== sb) return sa - sb;
           if (a.status === 'done' && b.status === 'done') return (b.processedAt || 0) - (a.processedAt || 0);
           if (a.status === 'pending' && b.status === 'pending') {
+            const aSeller = a.ttSeller === true && a.verified === false ? 0 : 1;
+            const bSeller = b.ttSeller === true && b.verified === false ? 0 : 1;
+            if (aSeller !== bSeller) return aSeller - bSeller;
             const la = locationTier(a), lb = locationTier(b);
             if (la !== lb) return la - lb;
           }