tt-help-cli-ycl 1.2.0 → 1.3.1

package/src/data-store.mjs

@@ -0,0 +1,213 @@
+import fs from 'fs';
+import path from 'path';
+
+function inferStatus(u) {
+  if (u.restricted) return 'restricted';
+  if (u.error) return 'error';
+  if (u.processed) return 'done';
+  return 'pending';
+}
+
+export function createStore(filePath) {
+  let data = [];
+
+  if (filePath) {
+    const resolved = path.resolve(filePath);
+    if (fs.existsSync(resolved)) {
+      try {
+        const raw = fs.readFileSync(resolved, 'utf-8');
+        data = JSON.parse(raw);
+        if (!Array.isArray(data)) data = [];
+      } catch (e) {
+        console.error(`[data-store] 读取文件失败: ${e.message}`);
+        data = [];
+      }
+    }
+  }
+
+  for (const u of data) {
+    if (!u.status) u.status = inferStatus(u);
+  }
+
+  function save() {
+    if (!filePath) return;
+    const resolved = path.resolve(filePath);
+    try {
+      if (fs.existsSync(resolved)) {
+        const raw = fs.readFileSync(resolved, 'utf-8');
+        const diskData = JSON.parse(raw);
+        if (Array.isArray(diskData)) {
+          const memIds = new Set(data.map(u => u.uniqueId));
+          for (const diskUser of diskData) {
+            if (!memIds.has(diskUser.uniqueId)) {
+              if (!diskUser.status) diskUser.status = inferStatus(diskUser);
+              data.push(diskUser);
+            }
+          }
+        }
+      }
+    } catch (e) { console.error(`[data-store] 合并磁盘数据失败: ${e.message}`); }
+    const json = JSON.stringify(data, null, 2);
+    fs.writeFileSync(resolved, json, 'utf-8');
+  }
+
+  function getUser(uid) {
+    return data.find(u => u.uniqueId === uid);
+  }
+
+  function hasUser(uid) {
+    return getUser(uid) !== undefined;
+  }
+
+  function addUser(user) {
+    const existing = getUser(user.uniqueId);
+    if (existing) {
+      for (const key of Object.keys(user)) {
+        if (key === 'uniqueId' || key === 'sources') continue;
+        if (user[key] !== undefined && user[key] !== null && user[key] !== '') {
+          existing[key] = user[key];
+        }
+      }
+      if (user.status && existing.status !== 'done') {
+        existing.status = user.status;
+      }
+      if (user.processed && !existing.processedAt) {
+        existing.processedAt = Date.now();
+      }
+      if (user.sources && Array.isArray(user.sources)) {
+        existing.sources = [...new Set([...(existing.sources || []), ...user.sources])];
+      }
+    } else {
+      if (!user.status) user.status = inferStatus(user);
+      if (user.processed) user.processedAt = user.processedAt || Date.now();
+      data.unshift(user);
+    }
+  }
+
+  function getPendingUsers() {
+    return data.filter(u => u.status === 'pending');
+  }
+
+  function getProcessedUsers() {
+    return data.filter(u => u.status === 'done');
+  }
+
+  function getAllUsers() {
+    return data;
+  }
+
+  function claimNextJob(expireMs = 5 * 60 * 1000) {
+    let next = data.find(u => u.status === 'pending');
+
+    if (!next) {
+      const now = Date.now();
+      const expired = data.find(u =>
+        u.status === 'processing' && u.claimedAt && (now - u.claimedAt) > expireMs
+      );
+      if (expired) {
+        expired.status = 'pending';
+        delete expired.claimedAt;
+        next = expired;
+      }
+    }
+
+    if (next) {
+      next.status = 'processing';
+      next.claimedAt = Date.now();
+      return { uniqueId: next.uniqueId, nickname: next.nickname };
+    }
+    return null;
+  }
+
+  function commitJob(uniqueId, result) {
+    const user = getUser(uniqueId);
+    if (!user) return { saved: false, error: 'user not found' };
+
+    if (result.restricted) {
+      user.status = 'restricted';
+      if (result.userInfo) {
+        const info = result.userInfo;
+        for (const key of Object.keys(info)) {
+          if (key === 'uniqueId' || key === 'sources') continue;
+          if (info[key] !== undefined && info[key] !== null && info[key] !== '') {
+            user[key] = info[key];
+          }
+        }
+      }
+      user.processed = true;
+      user.processedAt = Date.now();
+      user.noVideo = true;
+      user.sources = [...new Set([...(user.sources || []), 'restricted'])];
+    } else if (result.error) {
+      user.status = 'error';
+      user.error = result.error;
+      user.sources = [...new Set([...(user.sources || []), 'error'])];
+    } else {
+      user.status = 'done';
+      user.processed = true;
+      user.processedAt = Date.now();
+      user.followerCount = result.userInfo?.followerCount ?? user.followerCount;
+      user.videoCount = result.userInfo?.videoCount ?? user.videoCount;
+      user.nickname = result.userInfo?.nickname || user.nickname;
+      user.locationCreated = result.userInfo?.locationCreated || user.locationCreated;
+      user.ttSeller = result.userInfo?.ttSeller ?? user.ttSeller;
+      user.verified = result.userInfo?.verified ?? user.verified;
+      user.region = result.userInfo?.region || user.region;
+      user.signature = result.userInfo?.signature ?? user.signature;
+      user.followingCount = result.userInfo?.followingCount ?? user.followingCount;
+      user.heartCount = result.userInfo?.heartCount ?? user.heartCount;
+      if (result.userInfo?.secUid) user.secUid = result.userInfo.secUid;
+      const extraFields = ['restricted', 'error', 'userInfo', 'discoveredVideoAuthors',
+        'discoveredCommentAuthors', 'discoveredGuessAuthors', 'discoveredFollowing',
+        'discoveredFollowers', 'uniqueId', 'sources'];
+      for (const key of Object.keys(result)) {
+        if (extraFields.includes(key)) continue;
+        if (result[key] !== undefined && result[key] !== null && result[key] !== '') {
+          user[key] = result[key];
+        }
+      }
+      user.sources = [...new Set([...(user.sources || []), 'processed'])];
+
+      const discovered = [
+        ...(result.discoveredVideoAuthors || []).map(v => ({
+          uniqueId: v.uniqueId, nickname: v.nickname, locationCreated: v.locationCreated,
+          sources: ['video']
+        })),
+        ...(result.discoveredCommentAuthors || []).map(c => {
+          const id = typeof c === 'string' ? c.replace(/^@/, '') : c.uniqueId;
+          return typeof c === 'string'
+            ? { uniqueId: id, sources: ['comment'] }
+            : { uniqueId: id, ...c, sources: [...new Set([...(c.sources || []), 'comment'])] };
+        }),
+        ...(result.discoveredGuessAuthors || []).map(g => {
+          const id = typeof g === 'string' ? g.replace(/^@/, '') : g.uniqueId;
+          return typeof g === 'string'
+            ? { uniqueId: id, sources: ['guess'] }
+            : { uniqueId: id, ...g, sources: [...new Set([...(g.sources || []), 'guess'])] };
+        }),
+        ...(result.discoveredFollowing || []).map(([handle, name]) => ({
+          uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following']
+        })),
+        ...(result.discoveredFollowers || []).map(([handle, name]) => ({
+          uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower']
+        })),
+      ];
+
+      for (const du of discovered) {
+        if (!du.uniqueId) continue;
+        addUser(du);
+      }
+    }
+
+    delete user.claimedAt;
+    save();
+    return { saved: true };
+  }
+
+  return {
+    save, getUser, hasUser, addUser,
+    getPendingUsers, getProcessedUsers, getAllUsers,
+    claimNextJob, commitJob,
+    data,
+  };
+}

package/src/{explore-core.cjs → explore-core.mjs}

@@ -1,157 +1,148 @@
-const {
-  delay,
-  ensureBrowserReady,
-  setDelayConfig,
-  closeCommentPanel,
-  retryWithBackoff,
-} = require('./scraper/modules/page-helpers.cjs');
-const {
-  getUserInfo,
-  collectVideos,
-  isPageRestricted,
-} = require('./get-user-videos-core.cjs');
-const { scrapeSingleVideo } = require('./scraper/core.cjs');
-const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
-const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
-const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
-
-async function processExplore(page, username, options, log) {
-  const {
-    maxComments = 0,
-    maxGuess = 0,
-    enableFollow = true,
-    maxFollowing = 200,
-    maxFollowers = 200,
-    location = 'ES',
-  } = options;
-
-  const result = {
-    userInfo: null,
-    discoveredVideoAuthors: [],
-    discoveredCommentAuthors: [],
-    discoveredGuessAuthors: [],
-    discoveredFollowing: [],
-    discoveredFollowers: [],
-    collectedVideos: 0,
-    processed: false,
-    hasFollowData: false,
-    keepFollow: false,
-    locationCreated: null,
-    noVideo: false,
-    error: null,
-  };
-
-  try {
-    log(`  访问 @${username} 主页...`);
-    const homeUrl = `https://www.tiktok.com/@${username}`;
-    await retryWithBackoff(() => page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
-    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-    await delay(1000, 2000);
-
-    // 1. 获取用户信息
-    log(`  获取用户信息...`);
-    const info = await getUserInfo(page);
-    if (info) {
-      result.userInfo = info;
-      log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
-    }
-
-    // 2. 获取关注+粉丝（在滚动前执行，避免按钮被滚出视口）
-    if (enableFollow) {
-      try {
-        log(`  获取关注/粉丝...`);
-        const { following, followers } = await extractFollowAndFollowers(
-          page,
-          { maxFollowing, maxFollowers, log }
-        );
-        result.discoveredFollowing = following || [];
-        result.discoveredFollowers = followers || [];
-        result.hasFollowData = true;
-        log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
-      } catch (e) {
-        log(`  关注/粉丝提取失败: ${e.message}`);
-        result.hasFollowData = false;
-        result.discoveredFollowing = [];
-        result.discoveredFollowers = [];
-      }
-    }
-
-    // 3. 获取视频列表
-    const videoList = await collectVideos(page, username, 1, log);
-    const videoArray = videoList ? [...videoList.values()] : [];
-    result.collectedVideos = videoArray.length;
-
-    if (videoArray.length <= 0) {
-      result.processed = true;
-      result.noVideo = true;
-      const restricted = await isPageRestricted(page);
-      if (restricted) {
-        result.restricted = true;
-        log(`  @${username} 页面受限（需登录），标记跳过`);
-      } else {
-        log(`  @${username} 没有视频，标记已处理`);
-      }
-      return result;
-    }
-
-    // 4. 进入第一个视频
-    const firstVideo = videoArray[0];
-    const videoUrl = firstVideo.href.startsWith('http')
-      ? firstVideo.href
-      : `https://www.tiktok.com${firstVideo.href}`;
-
-    log(`  进入第一个视频: ${videoUrl}`);
-    await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
-    await delay(1500, 2500);
-
-    // 5. 获取视频信息（含 locationCreated）
-    const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');
-    result.locationCreated = videoData.locationCreated || null;
-    log(`  视频作者: ${videoData.videoAuthor} | 国家: ${result.locationCreated || '未知'}`);
-
-    // 6. 判断是否为目标国家
-    const isTargetLocation = result.locationCreated === location;
-
-    if (isTargetLocation) {
-      result.keepFollow = true;
-      log(`  国家匹配 (${location})，获取评论和猜你喜欢...`);
-
-      if (maxComments > 0) {
-        const commentResult = await extractCommentAuthors(page, maxComments);
-        result.discoveredCommentAuthors = commentResult || [];
-        await closeCommentPanel(page);
-        await delay(500, 1000);
-        log(`  评论用户: ${result.discoveredCommentAuthors.length}`);
-      }
-
-      if (maxGuess > 0) {
-        const guessResult = await extractGuessVideos(page, maxGuess);
-        result.discoveredGuessAuthors = (guessResult || []).map(v => v.author).filter(Boolean);
-        await closeCommentPanel(page);
-        await delay(500, 1000);
-        log(`  猜你喜欢作者: ${result.discoveredGuessAuthors.length}`);
-      }
-
-      result.discoveredVideoAuthors = [{
-        uniqueId: videoData.uniqueId,
-        nickname: videoData.nickname,
-        locationCreated: videoData.locationCreated,
-      }];
-    } else {
-      result.keepFollow = false;
-      log(`  国家不匹配 (${result.locationCreated} !== ${location})，跳过评论/猜你喜欢，丢弃关注/粉丝`);
-      result.discoveredFollowing = [];
-      result.discoveredFollowers = [];
-    }
-
-    result.processed = true;
-
-  } catch (e) {
-    result.error = e.message;
-    log(`  [错误] ${e.message}`);
-  }
-
-  return result;
-}
-
-module.exports = { processExplore };
+import {
+  delay,
+  setDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+  detectPageError,
+} from './scraper/modules/page-helpers.mjs';
+import {
+  getUserInfo,
+  collectVideos,
+} from './get-user-videos-core.mjs';
+import { scrapeSingleVideo } from './scraper/core.mjs';
+import { extractFollowAndFollowers } from './scraper/modules/follow-extractor.mjs';
+import { extractCommentAuthors } from './scraper/modules/comment-extractor.mjs';
+import { extractGuessVideos } from './scraper/modules/guess-extractor.mjs';
+
+async function processExplore(page, username, options, log) {
+  const {
+    maxComments = 0,
+    maxGuess = 0,
+    enableFollow = true,
+    maxFollowing = 5,
+    maxFollowers = 5,
+    location = 'ES',
+  } = options;
+
+  const result = {
+    userInfo: null,
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    discoveredGuessAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    collectedVideos: 0,
+    processed: false,
+    hasFollowData: false,
+    keepFollow: false,
+    locationCreated: null,
+    noVideo: false,
+    error: null,
+  };
+
+  try {
+    log(`  访问 @${username} 主页...`);
+    const homeUrl = `https://www.tiktok.com/@${username}`;
+    await retryWithBackoff(() => page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    log('  获取用户信息...');
+    const info = await getUserInfo(page);
+    if (info) {
+      result.userInfo = info;
+      log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
+    }
+
+    const videoList = await collectVideos(page, username, 1, log);
+    const videoArray = videoList ? [...videoList.values()] : [];
+    result.collectedVideos = videoArray.length;
+
+    if (videoArray.length <= 0) {
+      result.processed = true;
+      result.noVideo = true;
+      const pageError = await detectPageError(page);
+      if (pageError) {
+        result.restricted = true;
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，标记已处理`);
+      }
+      return result;
+    }
+
+    if (enableFollow) {
+      try {
+        log('  获取关注/粉丝...');
+        const { following, followers } = await extractFollowAndFollowers(
+          page, { maxFollowing, maxFollowers, log }
+        );
+        result.discoveredFollowing = following || [];
+        result.discoveredFollowers = followers || [];
+        result.hasFollowData = true;
+        log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
+      } catch (e) {
+        log(`  关注/粉丝提取失败: ${e.message}`);
+        result.hasFollowData = false;
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      }
+    }
+
+    const firstVideo = videoArray[0];
+    const videoUrl = firstVideo.href.startsWith('http')
+      ? firstVideo.href
+      : `https://www.tiktok.com${firstVideo.href}`;
+
+    log(`  进入第一个视频: ${videoUrl}`);
+    await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    await delay(1500, 2500);
+
+    const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');
+    result.locationCreated = videoData.locationCreated || null;
+    log(`  视频作者: ${videoData.videoAuthor} | 国家: ${result.locationCreated || '未知'}`);
+
+    const isTargetLocation = result.locationCreated === location;
+
+    if (isTargetLocation) {
+      result.keepFollow = true;
+      log(`  国家匹配 (${location})，获取评论和猜你喜欢...`);
+
+      if (maxComments > 0) {
+        const commentResult = await extractCommentAuthors(page, maxComments);
+        result.discoveredCommentAuthors = commentResult || [];
+        await closeCommentPanel(page);
+        await delay(500, 1000);
+        log(`  评论用户: ${result.discoveredCommentAuthors.length}`);
+      }
+
+      if (maxGuess > 0) {
+        const guessResult = await extractGuessVideos(page, maxGuess);
+        result.discoveredGuessAuthors = (guessResult || []).map(v => v.author).filter(Boolean);
+        await closeCommentPanel(page);
+        await delay(500, 1000);
+        log(`  猜你喜欢作者: ${result.discoveredGuessAuthors.length}`);
+      }
+
+      result.discoveredVideoAuthors = [{
+        uniqueId: videoData.uniqueId,
+        nickname: videoData.nickname,
+        locationCreated: videoData.locationCreated,
+      }];
+    } else {
+      result.keepFollow = false;
+      log(`  国家不匹配 (${result.locationCreated} !== ${location})，跳过评论/猜你喜欢，丢弃关注/粉丝`);
+      result.discoveredFollowing = [];
+      result.discoveredFollowers = [];
+    }
+
+    result.processed = true;
+  } catch (e) {
+    result.error = e.message;
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+export { processExplore };

package/src/{get-user-videos-core.cjs → get-user-videos-core.mjs}

@@ -1,12 +1,12 @@
-const { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } = require('./scraper/modules/page-helpers.cjs');
-const { scrollAndCollect } = require('./scraper/modules/scroll-collector.cjs');
+import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from './scraper/modules/page-helpers.mjs';
+import { scrollAndCollect } from './scraper/modules/scroll-collector.mjs';
 
 async function getUserInfo(page) {
   return await page.evaluate(() => {
     const html = document.documentElement.outerHTML;
     const result = {};
 
-    const m = window.location.href.match(/\/@([^\/]+)/);
+    const m = window.location.href.match(/\/@([^/]+)/);
     if (m) result.uniqueId = m[1];
 
     const patterns = {
@@ -69,12 +69,9 @@ async function collectVideos(page, username, maxVideos, log) {
     },
   });
 
-  // 去重
   const uniqueVideos = new Map();
   allLinks.forEach(v => {
-    if (!uniqueVideos.has(v.id)) {
-      uniqueVideos.set(v.id, v);
-    }
+    if (!uniqueVideos.has(v.id)) uniqueVideos.set(v.id, v);
   });
 
   log(`收集完成: ${uniqueVideos.size} 个视频`);
@@ -82,12 +79,7 @@ async function collectVideos(page, username, maxVideos, log) {
 }
 
 async function runGetUserVideos(options) {
-  const {
-    username,
-    maxVideos = 5,
-    log = console.error,
-  } = options;
-
+  const { username, maxVideos = 5, log = console.error } = options;
   const url = `https://www.tiktok.com/@${username}`;
 
   log(`用户: @${username}`);
@@ -131,13 +123,4 @@ async function runGetUserVideos(options) {
   return { output, browser };
 }
 
-async function isPageRestricted(page) {
-  return await page.evaluate(() => {
-    const bodyText = document.body.innerText;
-    return !!(bodyText.includes('登录 TikTok') ||
-             bodyText.includes('观众管理功能') ||
-             bodyText.includes('Login to TikTok'));
-  });
-}
-
-module.exports = { getUserInfo, collectVideos, runGetUserVideos, isPageRestricted };
+export { getUserInfo, collectVideos, runGetUserVideos };