tt-help-cli-ycl 1.0.5 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/auto-core.cjs

@@ -0,0 +1,288 @@
+const {
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+} = require('./scraper/modules/page-helpers.cjs');
+const {
+  getUserInfo,
+  collectVideos,
+  isPageRestricted,
+} = require('./get-user-videos-core.cjs');
+const { runScrape } = require('./scraper/core.cjs');
+
+function mergeUserInfo(existing, incoming, source) {
+  const merged = { ...existing };
+  for (const [key, value] of Object.entries(incoming)) {
+    if (key === '_sources') continue;
+    if (value === undefined || value === null || value === '') continue;
+    if (typeof value === 'number' && typeof merged[key] === 'number') {
+      merged[key] = Math.max(merged[key], value);
+    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
+      merged[key] = value;
+    }
+  }
+  if (source) {
+    if (!merged._sources) merged._sources = [];
+    if (!merged._sources.includes(source)) merged._sources.push(source);
+  }
+  return merged;
+}
+
+async function runAuto(options) {
+  const {
+    username,
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    preset = null,
+    switchMax = null,
+    commentMax = null,
+    log = console.error,
+  } = options;
+
+  if (preset) {
+    setDelayConfig(preset);
+  } else if (switchMax || commentMax) {
+    setDelayConfig({
+      switchMax: switchMax || 5000,
+      commentMax: commentMax || 3000,
+    });
+  }
+
+  const config = getDelayConfig();
+  const cleanUsername = username.replace('@', '');
+
+  log(`auto 模式: @${cleanUsername}`);
+  log(`收集视频数: ${collectMax}, 每个滑动: ${scrapeDepth}次, 每视频评论数: ${maxComments}`);
+
+  const browser = await ensureBrowserReady();
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, `https://www.tiktok.com/@${cleanUsername}`);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  // [1/3] 获取种子用户信息
+  const profileUrl = `https://www.tiktok.com/@${cleanUsername}`;
+  log(`\n[1/3] 获取 @${cleanUsername} 的用户信息和视频列表...`);
+  await retryWithBackoff(() => page.goto(profileUrl, { waitUntil: 'load', timeout: 30000 }), { log });
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+  await delay(1000, 2000);
+
+  const seedUserInfo = await getUserInfo(page);
+  if (!seedUserInfo.uniqueId) {
+    seedUserInfo.uniqueId = cleanUsername;
+  }
+  log(`种子用户: ${seedUserInfo.nickname || seedUserInfo.uniqueId} (粉丝: ${seedUserInfo.followerCount || '-'})`);
+
+  // [2/3] 收集视频列表
+  const videos = await collectVideos(page, cleanUsername, collectMax, log);
+  const videoList = Array.from(videos.values()).slice(0, collectMax);
+  log(`获取到 ${videoList.length} 个视频`);
+
+  if (videoList.length === 0) {
+    const restricted = await isPageRestricted(page);
+    if (restricted) {
+      log('种子用户页面受限（需登录），结束');
+    } else {
+      log('没有获取到视频，结束');
+    }
+    const output = {
+      seedUser: { ...seedUserInfo, sources: ['seed'], restricted },
+      users: [{ ...seedUserInfo, sources: ['seed'], restricted }],
+      stats: {
+        totalVideos: 0,
+        totalUsers: 1,
+        fromSeed: 1,
+        fromVideo: 0,
+        fromComment: 0,
+      },
+    };
+    return { output, browser };
+  }
+
+  // [3/3] 循环每个视频，执行 runScrape
+  log(`\n[3/3] 开始循环抓取（${videoList.length} 个视频，每个滑动 ${scrapeDepth} 次）...`);
+
+  const users = new Map();
+  users.set(seedUserInfo.uniqueId, mergeUserInfo({}, seedUserInfo, 'seed'));
+
+  const restrictedUsers = new Set();
+  let totalVideosScraped = 0;
+
+  for (let i = 0; i < videoList.length; i++) {
+    const videoUrl = videoList[i].href.startsWith('http')
+      ? videoList[i].href
+      : `https://www.tiktok.com${videoList[i].href}`;
+
+    log(`\n[${i + 1}/${videoList.length}] ${videoUrl}`);
+
+    const { output: scrapeOutput } = await runScrape({
+      videoUrl,
+      maxVideos: scrapeDepth,
+      maxComments,
+      preset,
+      switchMax,
+      commentMax,
+      log,
+      browser,
+      page,
+    });
+
+    totalVideosScraped += (scrapeOutput && scrapeOutput.stats) ? scrapeOutput.stats.totalVideos : 0;
+
+    // 合并视频作者信息
+    for (const vd of scrapeOutput.videoDetails) {
+      if (restrictedUsers.has(vd.uniqueId)) continue;
+      const existing = users.get(vd.uniqueId);
+      users.set(vd.uniqueId, mergeUserInfo(existing || {}, vd, 'video'));
+    }
+
+    // 添加评论者
+    for (const cu of scrapeOutput.commentUsers) {
+      if (restrictedUsers.has(cu)) continue;
+      if (!users.has(cu)) {
+        users.set(cu, mergeUserInfo({}, { uniqueId: cu }, 'comment'));
+      }
+    }
+  }
+
+  // 构建输出
+  const usersList = [...users.values()].map(u => {
+    const { _sources, ...rest } = u;
+    return { ...rest, sources: _sources || [] };
+  });
+
+  usersList.sort((a, b) => {
+    const aIsSeed = a._sources && a._sources.includes('seed');
+    const bIsSeed = b._sources && b._sources.includes('seed');
+    if (aIsSeed && !bIsSeed) return -1;
+    if (!aIsSeed && bIsSeed) return 1;
+    const aHasInfo = a.nickname || a.followerCount;
+    const bHasInfo = b.nickname || b.followerCount;
+    if (aHasInfo && !bHasInfo) return -1;
+    if (!aHasInfo && bHasInfo) return 1;
+    return 0;
+  });
+
+  const output = usersList;
+
+  log(`\n结果: ${usersList.length} 个用户`);
+
+  return { output, browser };
+}
+
+async function processUser(page, username, options, log) {
+  const {
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    preset = 'fast',
+    switchMax = null,
+    commentMax = null,
+    browser = null,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    collectedVideos: [],
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    error: null,
+  };
+
+  try {
+    log(`\n[processUser] 访问 @${username}...`);
+    await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
+      waitUntil: 'load',
+      timeout: 30000,
+    }), { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    const info = await getUserInfo(page);
+    result.userInfo = info;
+    if (!info.uniqueId) {
+      info.uniqueId = username;
+    }
+    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
+
+    const videos = await collectVideos(page, username, collectMax, log);
+    const videoList = Array.from(videos.values()).slice(0, collectMax);
+    result.collectedVideos = videoList.map(v => ({
+      videoId: v.id,
+      videoUrl: v.href,
+    }));
+
+    if (videoList.length > 0) {
+      const allVideoAuthors = new Map();
+      const allCommentAuthors = new Set();
+
+      for (let i = 0; i < videoList.length; i++) {
+        const video = videoList[i];
+        const videoUrl = video.href.startsWith('http')
+          ? video.href
+          : `https://www.tiktok.com${video.href}`;
+        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
+
+        const scrapeResult = await runScrape({
+          videoUrl,
+          maxVideos: scrapeDepth,
+          maxComments,
+          preset,
+          switchMax,
+          commentMax,
+          browser,
+          page,
+          log,
+        });
+
+        const scrapeOutput = scrapeResult.output;
+
+        if (scrapeOutput && scrapeOutput.videoDetails) {
+          for (const vd of scrapeOutput.videoDetails) {
+            if (!allVideoAuthors.has(vd.uniqueId)) {
+              allVideoAuthors.set(vd.uniqueId, {
+                uniqueId: vd.uniqueId,
+                nickname: vd.nickname,
+                locationCreated: vd.locationCreated,
+              });
+            }
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.commentUsers) {
+          for (const cu of scrapeOutput.commentUsers) {
+            allCommentAuthors.add(cu);
+          }
+        }
+      }
+
+      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
+      result.discoveredCommentAuthors = [...allCommentAuthors];
+
+      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者`);
+    } else {
+      const restricted = await isPageRestricted(page);
+      result.restricted = restricted;
+      if (restricted) {
+        log(`  @${username} 页面受限（需登录），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，跳过 scrape`);
+      }
+    }
+  } catch (e) {
+    result.error = e.message;
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+module.exports = { runAuto, processUser };

package/src/data-store.cjs

@@ -0,0 +1,65 @@
+const fs = require('fs');
+const path = require('path');
+
+function createStore(filePath) {
+  let data = [];
+
+  if (filePath) {
+    const resolved = path.resolve(filePath);
+    if (fs.existsSync(resolved)) {
+      try {
+        const raw = fs.readFileSync(resolved, 'utf-8');
+        data = JSON.parse(raw);
+        if (!Array.isArray(data)) data = [];
+      } catch (e) {
+        console.error(`[data-store] 读取文件失败: ${e.message}`);
+        data = [];
+      }
+    }
+  }
+
+  function save() {
+    if (!filePath) return;
+    const resolved = path.resolve(filePath);
+    const json = JSON.stringify(data, null, 2);
+    fs.writeFileSync(resolved, json, 'utf-8');
+  }
+
+  function getUser(uid) {
+    return data.find(u => u.uniqueId === uid);
+  }
+
+  function addUser(user) {
+    const existing = getUser(user.uniqueId);
+    if (existing) {
+      for (const key of Object.keys(user)) {
+        if (key === 'uniqueId') continue;
+        if (key === 'sources') continue;
+        if (user[key] !== undefined && user[key] !== null && user[key] !== '') {
+          existing[key] = user[key];
+        }
+      }
+      if (user.sources && Array.isArray(user.sources)) {
+        existing.sources = [...new Set([...(existing.sources || []), ...user.sources])];
+      }
+    } else {
+      data.push(user);
+    }
+  }
+
+  function getPendingUsers() {
+    return data.filter(u => u.followerCount === undefined);
+  }
+
+  function getAllUsers() {
+    return data;
+  }
+
+  function getProcessedUsers() {
+    return data.filter(u => u.followerCount !== undefined);
+  }
+
+  return { save, getUser, addUser, getPendingUsers, getAllUsers, getProcessedUsers, data };
+}
+
+module.exports = { createStore };

package/src/get-user-videos-core.cjs

@@ -0,0 +1,165 @@
+const { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } = require('./scraper/modules/page-helpers.cjs');
+
+async function getUserInfo(page) {
+  return await page.evaluate(() => {
+    const html = document.documentElement.outerHTML;
+    const result = {};
+
+    const m = window.location.href.match(/\/@([^\/]+)/);
+    if (m) result.uniqueId = m[1];
+
+    const patterns = {
+      secUid: /"secUid":"([^"]+)"/,
+      nickname: /"nickname":"((?:[^"\\]|\\.)*)"/,
+      ttSeller: /"ttSeller":\s*(true|false)/,
+      verified: /"verified":\s*(true|false)/,
+      followerCount: /"followerCount":(\d+)/,
+      videoCount: /"videoCount":(\d+)/,
+      followingCount: /"followingCount":(\d+)/,
+      heartCount: /"heartCount":(\d+)/,
+      signature: /"signature":"((?:[^"\\]|\\.)*)"/,
+      locationCreated: /"locationCreated":"([^"]*)/,
+    };
+
+    const boolKeys = ['ttSeller', 'verified'];
+    const numKeys = ['followerCount', 'videoCount', 'followingCount', 'heartCount'];
+
+    for (const [key, pat] of Object.entries(patterns)) {
+      const match = html.match(pat);
+      if (match) {
+        if (boolKeys.includes(key)) result[key] = match[1] === 'true';
+        else if (numKeys.includes(key)) result[key] = parseInt(match[1], 10);
+        else if (key === 'signature') result[key] = match[1].replace(/\\n/g, '\n').replace(/\\\\/g, '\\');
+        else result[key] = match[1];
+      }
+    }
+
+    return result;
+  });
+}
+
+async function collectVideos(page, username, maxVideos, log) {
+  const uniqueVideos = new Map();
+
+  const collect = async () => {
+    const links = await page.evaluate((handle) => {
+      const pattern = '/@' + handle + '/video/';
+      return Array.from(document.querySelectorAll('a'))
+        .filter(el => (el.getAttribute('href') || '').includes(pattern))
+        .map(el => {
+          const href = el.getAttribute('href') || '';
+          const idMatch = href.match(/\/video\/(\d+)/);
+          return { id: idMatch ? idMatch[1] : null, href };
+        })
+        .filter(v => v.id);
+    }, username);
+
+    let newCount = 0;
+    for (const v of links) {
+      if (!uniqueVideos.has(v.id)) {
+        uniqueVideos.set(v.id, v);
+        newCount++;
+      }
+    }
+    return newCount;
+  };
+
+  let prevTotal = 0;
+  let staleCount = 0;
+
+  for (let i = 0; i < 500; i++) {
+    const newCount = await collect();
+    if (newCount > 0) {
+      staleCount = 0;
+      if (uniqueVideos.size > prevTotal + 10 || i % 10 === 0) {
+        log(`滚动 ${i}: ${uniqueVideos.size} 个视频 (新增 ${newCount})`);
+        prevTotal = uniqueVideos.size;
+      }
+    } else {
+      staleCount++;
+    }
+
+    if (uniqueVideos.size >= maxVideos) {
+      log(`达到上限 ${maxVideos}，实际收集 ${uniqueVideos.size} 个（页面一次性加载）`);
+      break;
+    }
+
+    if (staleCount >= 5) {
+      log(`连续 ${staleCount} 次无新增，停止`);
+      break;
+    }
+
+    await page.evaluate(() => {
+      const container = document.querySelector('[class*="ColumnListContainer"]');
+      if (container) {
+        container.scrollTop += 700;
+      } else {
+        window.scrollBy(0, 700);
+      }
+    });
+    await delay(2000, 3000);
+  }
+
+  return uniqueVideos;
+}
+
+async function runGetUserVideos(options) {
+  const {
+    username,
+    maxVideos = 5,
+    log = console.error,
+  } = options;
+
+  const url = `https://www.tiktok.com/@${username}`;
+
+  log(`用户: @${username}`);
+  log(`URL: ${url}`);
+  log(`最大视频数: ${maxVideos}\n`);
+
+  log('连接浏览器...');
+  const browser = await ensureBrowserReady();
+
+  let page;
+  try {
+    page = await ensureTikTokPage(browser, url);
+  } catch (e) {
+    await browser.close().catch(() => {});
+    throw e;
+  }
+
+  await retryWithBackoff(() => page.goto(url, { waitUntil: 'load', timeout: 30000 }), { log });
+  await delay(3000, 5000);
+  await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+
+  log('获取用户信息...');
+  const userInfo = await getUserInfo(page);
+  log('用户信息: ' + JSON.stringify(userInfo, null, 2));
+
+  log('\n开始滚动收集视频...');
+  const videos = await collectVideos(page, username, maxVideos, log);
+  const allVideos = Array.from(videos.values());
+
+  log(`\n总计: ${allVideos.length} 个视频`);
+
+  const output = {
+    user: userInfo,
+    totalVideos: Math.min(allVideos.length, maxVideos),
+    videos: allVideos.slice(0, maxVideos).map(v => ({
+      id: v.id,
+      url: v.href.startsWith('http') ? v.href : `https://www.tiktok.com${v.href}`,
+    })),
+  };
+
+  return { output, browser };
+}
+
+async function isPageRestricted(page) {
+  return await page.evaluate(() => {
+    const bodyText = document.body.innerText;
+    return !!(bodyText.includes('登录 TikTok') ||
+             bodyText.includes('观众管理功能') ||
+             bodyText.includes('Login to TikTok'));
+  });
+}
+
+module.exports = { getUserInfo, collectVideos, runGetUserVideos, isPageRestricted };

package/src/get-user-videos.cjs

@@ -0,0 +1,59 @@
+const { runGetUserVideos } = require('./get-user-videos-core.cjs');
+const fs = require('fs');
+const path = require('path');
+
+async function main() {
+  const rawArgs = process.argv.slice(2);
+
+  let outputPath = null;
+  const args = [];
+  for (let i = 0; i < rawArgs.length; i++) {
+    if (rawArgs[i] === '-o' || rawArgs[i] === '--output') {
+      outputPath = rawArgs[++i];
+    } else {
+      args.push(rawArgs[i]);
+    }
+  }
+
+  if (args.length < 1) {
+    console.error('用法: node get-user-videos.cjs <用户名> [最大视频数] [-o 输出路径]');
+    console.error('示例: node get-user-videos.cjs bar.lar.lar.moeta 1000');
+    console.error('      node get-user-videos.cjs username 50 -o videos.json');
+    console.error('');
+    console.error('选项: -o, --output <路径>  输出到文件; 不指定则输出到 stdout');
+    process.exit(1);
+  }
+
+  const username = args[0].replace('@', '');
+  const maxVideos = parseInt(args[1]) || 5;
+
+  let browser;
+  try {
+    const { output, browser: b } = await runGetUserVideos({
+      username,
+      maxVideos,
+      log: console.error,
+    });
+    browser = b;
+
+    const json = JSON.stringify(output, null, 2);
+    if (outputPath) {
+      const resultFile = path.isAbsolute(outputPath) ? outputPath : path.resolve(outputPath);
+      fs.mkdirSync(path.dirname(resultFile), { recursive: true });
+      fs.writeFileSync(resultFile, json, 'utf-8');
+      console.error(`已保存到 ${resultFile}`);
+    } else {
+      process.stdout.write(json + '\n');
+    }
+  } catch (err) {
+    console.error(err.message);
+    process.exit(1);
+  } finally {
+    if (browser) await browser.close().catch(() => {});
+  }
+}
+
+main().catch((err) => {
+  console.error(err.message);
+  process.exit(1);
+});