tt-help-cli-ycl 1.0.8 → 1.1.0

package/src/data-store.cjs

@@ -1,65 +1,69 @@
-const fs = require('fs');
-const path = require('path');
-
-function createStore(filePath) {
-  let data = [];
-
-  if (filePath) {
-    const resolved = path.resolve(filePath);
-    if (fs.existsSync(resolved)) {
-      try {
-        const raw = fs.readFileSync(resolved, 'utf-8');
-        data = JSON.parse(raw);
-        if (!Array.isArray(data)) data = [];
-      } catch (e) {
-        console.error(`[data-store] 读取文件失败: ${e.message}`);
-        data = [];
-      }
-    }
-  }
-
-  function save() {
-    if (!filePath) return;
-    const resolved = path.resolve(filePath);
-    const json = JSON.stringify(data, null, 2);
-    fs.writeFileSync(resolved, json, 'utf-8');
-  }
-
-  function getUser(uid) {
-    return data.find(u => u.uniqueId === uid);
-  }
-
-  function addUser(user) {
-    const existing = getUser(user.uniqueId);
-    if (existing) {
-      for (const key of Object.keys(user)) {
-        if (key === 'uniqueId') continue;
-        if (key === 'sources') continue;
-        if (user[key] !== undefined && user[key] !== null && user[key] !== '') {
-          existing[key] = user[key];
-        }
-      }
-      if (user.sources && Array.isArray(user.sources)) {
-        existing.sources = [...new Set([...(existing.sources || []), ...user.sources])];
-      }
-    } else {
-      data.push(user);
-    }
-  }
-
-  function getPendingUsers() {
-    return data.filter(u => u.followerCount === undefined);
-  }
-
-  function getAllUsers() {
-    return data;
-  }
-
-  function getProcessedUsers() {
-    return data.filter(u => u.followerCount !== undefined);
-  }
-
-  return { save, getUser, addUser, getPendingUsers, getAllUsers, getProcessedUsers, data };
-}
-
-module.exports = { createStore };
+const fs = require('fs');
+const path = require('path');
+
+function createStore(filePath) {
+  let data = [];
+
+  if (filePath) {
+    const resolved = path.resolve(filePath);
+    if (fs.existsSync(resolved)) {
+      try {
+        const raw = fs.readFileSync(resolved, 'utf-8');
+        data = JSON.parse(raw);
+        if (!Array.isArray(data)) data = [];
+      } catch (e) {
+        console.error(`[data-store] 读取文件失败: ${e.message}`);
+        data = [];
+      }
+    }
+  }
+
+  function save() {
+    if (!filePath) return;
+    const resolved = path.resolve(filePath);
+    const json = JSON.stringify(data, null, 2);
+    fs.writeFileSync(resolved, json, 'utf-8');
+  }
+
+  function getUser(uid) {
+    return data.find(u => u.uniqueId === uid);
+  }
+
+  function hasUser(uid) {
+    return getUser(uid) !== undefined;
+  }
+
+  function addUser(user) {
+    const existing = getUser(user.uniqueId);
+    if (existing) {
+      for (const key of Object.keys(user)) {
+        if (key === 'uniqueId') continue;
+        if (key === 'sources') continue;
+        if (user[key] !== undefined && user[key] !== null && user[key] !== '') {
+          existing[key] = user[key];
+        }
+      }
+      if (user.sources && Array.isArray(user.sources)) {
+        existing.sources = [...new Set([...(existing.sources || []), ...user.sources])];
+      }
+    } else {
+      data.push(user);
+    }
+  }
+
+  function getPendingUsers() {
+    return data.filter(u => u.followerCount === undefined);
+  }
+
+  function getAllUsers() {
+    return data;
+  }
+
+  function getProcessedUsers() {
+    return data.filter(u => u.processed === true || u.followerCount !== undefined);
+  }
+
+  return { save, getUser, hasUser, addUser, getPendingUsers, getAllUsers, getProcessedUsers, data };
+}
+
+module.exports = { createStore };

package/src/explore-core.cjs

@@ -0,0 +1,157 @@
+const {
+  delay,
+  ensureBrowserReady,
+  setDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+} = require('./scraper/modules/page-helpers.cjs');
+const {
+  getUserInfo,
+  collectVideos,
+  isPageRestricted,
+} = require('./get-user-videos-core.cjs');
+const { scrapeSingleVideo } = require('./scraper/core.cjs');
+const { extractFollowAndFollowers } = require('./scraper/modules/follow-extractor.cjs');
+const { extractCommentAuthors } = require('./scraper/modules/comment-extractor.cjs');
+const { extractGuessVideos } = require('./scraper/modules/guess-extractor.cjs');
+
+async function processExplore(page, username, options, log) {
+  const {
+    maxComments = 0,
+    maxGuess = 0,
+    enableFollow = true,
+    maxFollowing = 200,
+    maxFollowers = 200,
+    location = 'ES',
+  } = options;
+
+  const result = {
+    userInfo: null,
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    discoveredGuessAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    collectedVideos: 0,
+    processed: false,
+    hasFollowData: false,
+    keepFollow: false,
+    locationCreated: null,
+    noVideo: false,
+    error: null,
+  };
+
+  try {
+    log(`  访问 @${username} 主页...`);
+    const homeUrl = `https://www.tiktok.com/@${username}`;
+    await retryWithBackoff(() => page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    // 1. 获取用户信息
+    log(`  获取用户信息...`);
+    const info = await getUserInfo(page);
+    if (info) {
+      result.userInfo = info;
+      log(`  用户: ${info.nickname || username} | 粉丝: ${info.followerCount || '-'} | 视频: ${info.videoCount || '-'}`);
+    }
+
+    // 2. 获取关注+粉丝（在滚动前执行，避免按钮被滚出视口）
+    if (enableFollow) {
+      try {
+        log(`  获取关注/粉丝...`);
+        const { following, followers } = await extractFollowAndFollowers(
+          page,
+          { maxFollowing, maxFollowers, log }
+        );
+        result.discoveredFollowing = following || [];
+        result.discoveredFollowers = followers || [];
+        result.hasFollowData = true;
+        log(`  关注: ${result.discoveredFollowing.length}, 粉丝: ${result.discoveredFollowers.length}`);
+      } catch (e) {
+        log(`  关注/粉丝提取失败: ${e.message}`);
+        result.hasFollowData = false;
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      }
+    }
+
+    // 3. 获取视频列表
+    const videoList = await collectVideos(page, username, 1, log);
+    const videoArray = videoList ? [...videoList.values()] : [];
+    result.collectedVideos = videoArray.length;
+
+    if (videoArray.length <= 0) {
+      result.processed = true;
+      result.noVideo = true;
+      const restricted = await isPageRestricted(page);
+      if (restricted) {
+        result.restricted = true;
+        log(`  @${username} 页面受限（需登录），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，标记已处理`);
+      }
+      return result;
+    }
+
+    // 4. 进入第一个视频
+    const firstVideo = videoArray[0];
+    const videoUrl = firstVideo.href.startsWith('http')
+      ? firstVideo.href
+      : `https://www.tiktok.com${firstVideo.href}`;
+
+    log(`  进入第一个视频: ${videoUrl}`);
+    await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    await delay(1500, 2500);
+
+    // 5. 获取视频信息（含 locationCreated）
+    const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');
+    result.locationCreated = videoData.locationCreated || null;
+    log(`  视频作者: ${videoData.videoAuthor} | 国家: ${result.locationCreated || '未知'}`);
+
+    // 6. 判断是否为目标国家
+    const isTargetLocation = result.locationCreated === location;
+
+    if (isTargetLocation) {
+      result.keepFollow = true;
+      log(`  国家匹配 (${location})，获取评论和猜你喜欢...`);
+
+      if (maxComments > 0) {
+        const commentResult = await extractCommentAuthors(page, maxComments);
+        result.discoveredCommentAuthors = commentResult || [];
+        await closeCommentPanel(page);
+        await delay(500, 1000);
+        log(`  评论用户: ${result.discoveredCommentAuthors.length}`);
+      }
+
+      if (maxGuess > 0) {
+        const guessResult = await extractGuessVideos(page, maxGuess);
+        result.discoveredGuessAuthors = (guessResult || []).map(v => v.author).filter(Boolean);
+        await closeCommentPanel(page);
+        await delay(500, 1000);
+        log(`  猜你喜欢作者: ${result.discoveredGuessAuthors.length}`);
+      }
+
+      result.discoveredVideoAuthors = [{
+        uniqueId: videoData.uniqueId,
+        nickname: videoData.nickname,
+        locationCreated: videoData.locationCreated,
+      }];
+    } else {
+      result.keepFollow = false;
+      log(`  国家不匹配 (${result.locationCreated} !== ${location})，跳过评论/猜你喜欢，丢弃关注/粉丝`);
+      result.discoveredFollowing = [];
+      result.discoveredFollowers = [];
+    }
+
+    result.processed = true;
+
+  } catch (e) {
+    result.error = e.message;
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+module.exports = { processExplore };

package/src/get-user-videos-core.cjs

@@ -1,165 +1,143 @@
-const { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } = require('./scraper/modules/page-helpers.cjs');
-
-async function getUserInfo(page) {
-  return await page.evaluate(() => {
-    const html = document.documentElement.outerHTML;
-    const result = {};
-
-    const m = window.location.href.match(/\/@([^\/]+)/);
-    if (m) result.uniqueId = m[1];
-
-    const patterns = {
-      secUid: /"secUid":"([^"]+)"/,
-      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
-      ttSeller: /"ttSeller":\s*(true|false)/,
-      verified: /"verified":\s*(true|false)/,
-      followerCount: /"followerCount":(\d+)/,
-      videoCount: /"videoCount":(\d+)/,
-      followingCount: /"followingCount":(\d+)/,
-      heartCount: /"heartCount":(\d+)/,
-      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
-      locationCreated: /"locationCreated":"([^"]*)/,
-    };
-
-    const boolKeys = ['ttSeller', 'verified'];
-    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
-
-    for (const [key, pat] of Object.entries(patterns)) {
-      const match = html.match(pat);
-      if (match) {
-        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
-        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
-        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
-        else result[key] = match[1];
-      }
-    }
-
-    return result;
-  });
-}
-
-async function collectVideos(page, username, maxVideos, log) {
-  const uniqueVideos = new Map();
-
-  const collect = async () => {
-    const links = await page.evaluate((handle) => {
-      const pattern = '/@' + handle + '/video/';
-      return Array.from(document.querySelectorAll('a'))
-        .filter(el => (el.getAttribute('href') || '').includes(pattern))
-        .map(el => {
-          const href = el.getAttribute('href') || '';
-          const idMatch = href.match(/\/video\/(\d+)/);
-          return { id: idMatch ? idMatch[1] : null, href };
-        })
-        .filter(v => v.id);
-    }, username);
-
-    let newCount = 0;
-    for (const v of links) {
-      if (!uniqueVideos.has(v.id)) {
-        uniqueVideos.set(v.id, v);
-        newCount++;
-      }
-    }
-    return newCount;
-  };
-
-  let prevTotal = 0;
-  let staleCount = 0;
-
-  for (let i = 0; i < 500; i++) {
-    const newCount = await collect();
-    if (newCount > 0) {
-      staleCount = 0;
-      if (uniqueVideos.size > prevTotal + 10 || i % 10 === 0) {
-        log(`滚动 ${i}: ${uniqueVideos.size} 个视频 (新增 ${newCount})`);
-        prevTotal = uniqueVideos.size;
-      }
-    } else {
-      staleCount++;
-    }
-
-    if (uniqueVideos.size >= maxVideos) {
-      log(`达到上限 ${maxVideos}，实际收集 ${uniqueVideos.size} 个（页面一次性加载）`);
-      break;
-    }
-
-    if (staleCount >= 5) {
-      log(`连续 ${staleCount} 次无新增，停止`);
-      break;
-    }
-
-    await page.evaluate(() => {
-      const container = document.querySelector('[class*="ColumnListContainer"]');
-      if (container) {
-        container.scrollTop += 700;
-      } else {
-        window.scrollBy(0, 700);
-      }
-    });
-    await delay(2000, 3000);
-  }
-
-  return uniqueVideos;
-}
-
-async function runGetUserVideos(options) {
-  const {
-    username,
-    maxVideos = 5,
-    log = console.error,
-  } = options;
-
-  const url = `https://www.tiktok.com/@${username}`;
-
-  log(`用户: @${username}`);
-  log(`URL: ${url}`);
-  log(`最大视频数: ${maxVideos}\n`);
-
-  log('连接浏览器...');
-  const browser = await ensureBrowserReady();
-
-  let page;
-  try {
-    page = await ensureTikTokPage(browser, url);
-  } catch (e) {
-    await browser.close().catch(() => {});
-    throw e;
-  }
-
-  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
-  await delay(3000, 5000);
-  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
-
-  log('获取用户信息...');
-  const userInfo = await getUserInfo(page);
-  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
-
-  log('\n开始滚动收集视频...');
-  const videos = await collectVideos(page, username, maxVideos, log);
-  const allVideos = Array.from(videos.values());
-
-  log(`\n总计: ${allVideos.length} 个视频`);
-
-  const output = {
-    user: userInfo,
-    totalVideos: Math.min(allVideos.length, maxVideos),
-    videos: allVideos.slice(0, maxVideos).map(v => ({
-      id: v.id,
-      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
-    })),
-  };
-
-  return { output, browser };
-}
-
-async function isPageRestricted(page) {
-  return await page.evaluate(() => {
-    const bodyText = document.body.innerText;
-    return !!(bodyText.includes('登录 TikTok') ||
-             bodyText.includes('观众管理功能') ||
-             bodyText.includes('Login to TikTok'));
-  });
-}
-
+const { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } = require('./scraper/modules/page-helpers.cjs');
+const { scrollAndCollect } = require('./scraper/modules/scroll-collector.cjs');
+
+async function getUserInfo(page) {
+  return await page.evaluate(() => {
+    const html = document.documentElement.outerHTML;
+    const result = {};
+
+    const m = window.location.href.match(/\/@([^\/]+)/);
+    if (m) result.uniqueId = m[1];
+
+    const patterns = {
+      secUid: /"secUid":"([^"]+)"/,
+      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
+      ttSeller: /"ttSeller":\s*(true|false)/,
+      verified: /"verified":\s*(true|false)/,
+      followerCount: /"followerCount":(\d+)/,
+      videoCount: /"videoCount":(\d+)/,
+      followingCount: /"followingCount":(\d+)/,
+      heartCount: /"heartCount":(\d+)/,
+      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
+      locationCreated: /"locationCreated":"([^"]*)/,
+      region: /"region":"([^"]*)/,
+    };
+
+    const boolKeys = ['ttSeller', 'verified'];
+    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
+
+    for (const [key, pat] of Object.entries(patterns)) {
+      const match = html.match(pat);
+      if (match) {
+        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
+        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
+        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
+        else result[key] = match[1];
+      }
+    }
+
+    return result;
+  });
+}
+
+async function collectVideos(page, username, maxVideos, log) {
+  const allLinks = await scrollAndCollect(page, {
+    container: '[class*="ColumnListContainer"]',
+    extraArgs: { handle: username },
+    collectFn: (container, args) => {
+      const pattern = '/@' + args.handle + '/video/';
+      return {
+        items: Array.from(document.querySelectorAll('a'))
+          .filter(el => (el.getAttribute('href') || '').includes(pattern))
+          .map(el => {
+            const href = el.getAttribute('href') || '';
+            const idMatch = href.match(/\/video\/(\d+)/);
+            return { id: idMatch ? idMatch[1] : null, href };
+          })
+          .filter(v => v.id),
+      };
+    },
+    maxItems: maxVideos,
+    delayRange: [2000, 3000],
+    staleThreshold: 5,
+    maxRounds: 500,
+    onRound: (round, items, allItems) => {
+      const uniqueCount = new Set(allItems.map(v => v.id)).size;
+      if (uniqueCount > 0 && (uniqueCount % 10 === 0 || items.length > 0)) {
+        log(`滚动 ${round + 1}: ${uniqueCount} 个视频 (本轮 ${items.length} 条)`);
+      }
+    },
+  });
+
+  // 去重
+  const uniqueVideos = new Map();
+  allLinks.forEach(v => {
+    if (!uniqueVideos.has(v.id)) {
+      uniqueVideos.set(v.id, v);
+    }
+  });
+
+  log(`收集完成: ${uniqueVideos.size} 个视频`);
+  return uniqueVideos;
+}
+
+async function runGetUserVideos(options) {
+  const {
+    username,
+    maxVideos = 5,
+    log = console.error,
+  } = options;
+
+  const url = `https://www.tiktok.com/@${username}`;
+
+  log(`用户: @${username}`);
+  log(`URL: ${url}`);
+  log(`最大视频数: ${maxVideos}\n`);
+
+  log('连接浏览器...');
+  const browser = await ensureBrowserReady();
+
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, url);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
+  await delay(3000, 5000);
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+
+  log('获取用户信息...');
+  const userInfo = await getUserInfo(page);
+  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
+
+  log('\n开始滚动收集视频...');
+  const videos = await collectVideos(page, username, maxVideos, log);
+  const allVideos = Array.from(videos.values());
+
+  log(`\n总计: ${allVideos.length} 个视频`);
+
+  const output = {
+    user: userInfo,
+    totalVideos: Math.min(allVideos.length, maxVideos),
+    videos: allVideos.slice(0, maxVideos).map(v => ({
+      id: v.id,
+      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
+    })),
+  };
+
+  return { output, browser };
+}
+
+async function isPageRestricted(page) {
+  return await page.evaluate(() => {
+    const bodyText = document.body.innerText;
+    return !!(bodyText.includes('登录 TikTok') ||
+             bodyText.includes('观众管理功能') ||
+             bodyText.includes('Login to TikTok'));
+  });
+}
+
 module.exports = { getUserInfo, collectVideos, runGetUserVideos, isPageRestricted };