tt-help-cli-ycl 1.2.0 → 1.3.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.2.0",
+  "version": "1.3.1",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/auto-core.mjs

@@ -0,0 +1,174 @@
+import {
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  closeCommentPanel,
+  retryWithBackoff,
+  detectPageError,
+} from './scraper/modules/page-helpers.mjs';
+import {
+  getUserInfo,
+  collectVideos,
+} from './get-user-videos-core.mjs';
+import { runScrape } from './scraper/core.mjs';
+import { extractFollowAndFollowers } from './scraper/modules/follow-extractor.mjs';
+
+function mergeUserInfo(existing, incoming, source) {
+  const merged = { ...existing };
+  for (const [key, value] of Object.entries(incoming)) {
+    if (key === '_sources') continue;
+    if (value === undefined || value === null || value === '') continue;
+    if (typeof value === 'number' && typeof merged[key] === 'number') {
+      merged[key] = Math.max(merged[key], value);
+    } else if (merged[key] === undefined || merged[key] === null || merged[key] === '') {
+      merged[key] = value;
+    }
+  }
+  if (source) {
+    if (!merged._sources) merged._sources = [];
+    if (!merged._sources.includes(source)) merged._sources.push(source);
+  }
+  return merged;
+}
+
+async function processUser(page, username, options, log) {
+  const {
+    collectMax = 1,
+    scrapeDepth = 50,
+    maxComments = 200,
+    maxGuess = 10,
+    preset = 'fast',
+    switchMax = null,
+    commentMax = null,
+    enableFollow = false,
+    maxFollowing = 200,
+    maxFollowers = 200,
+    browser = null,
+  } = options;
+
+  const result = {
+    userInfo: null,
+    collectedVideos: [],
+    discoveredVideoAuthors: [],
+    discoveredCommentAuthors: [],
+    discoveredGuessAuthors: [],
+    discoveredFollowing: [],
+    discoveredFollowers: [],
+    error: null,
+  };
+
+  try {
+    log(`\n[processUser] 访问 @${username}...`);
+    await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
+      waitUntil: 'load', timeout: 30000,
+    }), { log });
+    await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
+    await delay(1000, 2000);
+
+    const info = await getUserInfo(page);
+    result.userInfo = info;
+    if (!info.uniqueId) info.uniqueId = username;
+    log(`  昵称: ${info.nickname || '-'} | 粉丝: ${info.followerCount || 0}`);
+
+    if (options.enableFollow) {
+      try {
+        log('  提取关注/粉丝列表...');
+        const { following, followers } = await extractFollowAndFollowers(page, {
+          maxFollowing: options.maxFollowing || 200,
+          maxFollowers: options.maxFollowers || 200,
+          log,
+        });
+        result.discoveredFollowing = following;
+        result.discoveredFollowers = followers;
+        log(`  关注: ${following.length} | 粉丝: ${followers.length}`);
+      } catch (e) {
+        log(`  关注/粉丝提取失败: ${e.message}`);
+        result.discoveredFollowing = [];
+        result.discoveredFollowers = [];
+      }
+    }
+
+    const videos = await collectVideos(page, username, collectMax, log);
+    const videoList = Array.from(videos.values()).slice(0, collectMax);
+    result.collectedVideos = videoList.map(v => ({
+      videoId: v.id,
+      videoUrl: v.href,
+    }));
+
+    if (videoList.length > 0) {
+      const allVideoAuthors = new Map();
+      const allCommentAuthors = new Set();
+      const allGuessAuthors = new Set();
+
+      for (let i = 0; i < videoList.length; i++) {
+        const video = videoList[i];
+        const videoUrl = video.href.startsWith('http')
+          ? video.href
+          : `https://www.tiktok.com${video.href}`;
+        log(`  [${i + 1}/${videoList.length}] 开始 scrape: ${videoUrl} (深度 ${scrapeDepth})`);
+
+        const scrapeResult = await runScrape({
+          videoUrl,
+          maxVideos: scrapeDepth,
+          maxComments,
+          maxGuess,
+          preset,
+          switchMax,
+          commentMax,
+          browser,
+          page,
+          log,
+        });
+
+        const scrapeOutput = scrapeResult.output;
+
+        if (scrapeOutput && scrapeOutput.videoDetails) {
+          for (const vd of scrapeOutput.videoDetails) {
+            if (!allVideoAuthors.has(vd.uniqueId)) {
+              allVideoAuthors.set(vd.uniqueId, {
+                uniqueId: vd.uniqueId,
+                nickname: vd.nickname,
+                locationCreated: vd.locationCreated,
+              });
+            }
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.commentUsers) {
+          for (const cu of scrapeOutput.commentUsers) {
+            allCommentAuthors.add(cu);
+          }
+        }
+
+        if (scrapeOutput && scrapeOutput.guessAuthors) {
+          for (const ga of scrapeOutput.guessAuthors) {
+            allGuessAuthors.add(ga);
+          }
+        }
+      }
+
+      result.discoveredVideoAuthors = [...allVideoAuthors.values()];
+      result.discoveredCommentAuthors = [...allCommentAuthors];
+      result.discoveredGuessAuthors = [...allGuessAuthors];
+
+      log(`  发现: ${result.discoveredVideoAuthors.length} 个视频作者, ${result.discoveredCommentAuthors.length} 个评论作者, ${result.discoveredGuessAuthors.length} 个猜你喜欢作者`);
+    } else {
+      const pageError = await detectPageError(page);
+      result.restricted = !!pageError;
+      if (pageError) {
+        log(`  @${username} 页面受限（${pageError}），标记跳过`);
+      } else {
+        log(`  @${username} 没有视频，跳过 scrape`);
+      }
+    }
+  } catch (e) {
+    result.error = e.message;
+    log(`  [错误] ${e.message}`);
+  }
+
+  return result;
+}
+
+export { processUser, mergeUserInfo };

package/src/cli/auto.js

@@ -0,0 +1,94 @@
+import { getOrCreatePage } from '../lib/browser/page.js';
+
+async function apiPost(url, body) {
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  });
+  return res.json();
+}
+
+async function apiGet(url) {
+  const res = await fetch(url);
+  return res.json();
+}
+
+export async function handleAuto(options) {
+  const { autoUsernames, autoCollectMax, autoScrapeDepth, autoMaxComments, autoMaxGuess,
+    autoPreset, autoSwitchDelay, autoCommentDelay, serverUrl,
+    autoEnableFollow, autoMaxFollowing, autoMaxFollowers } = options;
+
+  const runOptions = {
+    collectMax: autoCollectMax,
+    scrapeDepth: autoScrapeDepth,
+    maxComments: autoMaxComments,
+    maxGuess: autoMaxGuess,
+    preset: autoPreset,
+    switchMax: autoSwitchDelay,
+    commentMax: autoCommentDelay,
+    enableFollow: autoEnableFollow,
+    maxFollowing: autoMaxFollowing,
+    maxFollowers: autoMaxFollowers,
+  };
+
+  try {
+    await apiGet(`${serverUrl}/api/stats`);
+  } catch {
+    console.error(`无法连接服务端: ${serverUrl}`);
+    console.error('请先启动服务端: tt-help watch -o <数据文件>');
+    process.exit(1);
+  }
+
+  if (autoUsernames.length > 0) {
+    const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: autoUsernames });
+    console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
+  }
+
+  const { ensureBrowserReady, processUser } = await import('../lib/auto-browser.mjs');
+  const browser = await ensureBrowserReady();
+
+  try {
+    const page = await getOrCreatePage(browser);
+
+    let processedCount = 0;
+    let errorCount = 0;
+
+    while (true) {
+      const job = await apiGet(`${serverUrl}/api/job`);
+      if (!job.hasJob) break;
+
+      const username = job.user.uniqueId;
+      processedCount++;
+      console.error(`\n[${processedCount}] 处理 @${username}...`);
+
+      const result = await processUser(page, username, { ...runOptions, browser }, console.error);
+
+      if (result.restricted || result.error) {
+        if (result.error) errorCount++;
+        await apiPost(`${serverUrl}/api/job/${username}`, result);
+        continue;
+      }
+
+      const payload = {
+        userInfo: result.userInfo || {},
+        discoveredVideoAuthors: result.discoveredVideoAuthors || [],
+        discoveredCommentAuthors: result.discoveredCommentAuthors || [],
+        discoveredGuessAuthors: result.discoveredGuessAuthors || [],
+        discoveredFollowing: result.discoveredFollowing || [],
+        discoveredFollowers: result.discoveredFollowers || [],
+      };
+      await apiPost(`${serverUrl}/api/job/${username}`, payload);
+      console.error('  已提交');
+    }
+
+    const stats = await apiGet(`${serverUrl}/api/stats`);
+    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+  } catch (err) {
+    console.error(`自动抓取失败: ${err.message}`);
+    process.exit(1);
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}

package/src/cli/explore.js

@@ -0,0 +1,117 @@
+import { getOrCreatePage } from '../lib/browser/page.js';
+import { delay, getDelayConfig, setDelayConfig } from '../scraper/modules/page-helpers.mjs';
+
+async function apiPost(url, body) {
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  });
+  return res.json();
+}
+
+async function apiGet(url) {
+  const res = await fetch(url);
+  return res.json();
+}
+
+export async function handleExplore(options) {
+  const {
+    exploreUsernames, explorePreset, exploreMaxComments, exploreMaxGuess,
+    exploreEnableFollow, exploreMaxFollowing, exploreMaxFollowers,
+    exploreLocation, exploreMaxUsers, serverUrl,
+  } = options;
+
+  setDelayConfig(explorePreset);
+
+  try {
+    await apiGet(`${serverUrl}/api/stats`);
+  } catch {
+    console.error(`无法连接服务端: ${serverUrl}`);
+    console.error('请先启动服务端: tt-help watch -o <数据文件>');
+    process.exit(1);
+  }
+
+  if (exploreUsernames && exploreUsernames.length > 0) {
+    const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: exploreUsernames });
+    console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
+  }
+
+  console.error(`\n国家筛选: ${exploreLocation}`);
+  console.error(`评论: ${exploreMaxComments}, 猜你喜欢: ${exploreMaxGuess}`);
+  console.error(`关注/粉丝: ${exploreEnableFollow ? '启用' : '禁用'}`);
+  if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
+
+  const { ensureBrowserReady, processExplore } = await import('../lib/auto-browser.mjs');
+  const browser = await ensureBrowserReady();
+
+  try {
+    const page = await getOrCreatePage(browser);
+
+    let processedCount = 0;
+    let errorCount = 0;
+
+    while (true) {
+      const job = await apiGet(`${serverUrl}/api/job`);
+      if (!job.hasJob) break;
+
+      const username = job.user.uniqueId;
+      processedCount++;
+      console.error(`\n[${processedCount}] 探索 @${username}...`);
+
+      const { switchMax } = getDelayConfig();
+      await delay(switchMax, switchMax * 3);
+
+      const result = await processExplore(page, username, {
+        maxComments: exploreMaxComments,
+        maxGuess: exploreMaxGuess,
+        enableFollow: exploreEnableFollow,
+        maxFollowing: exploreMaxFollowing,
+        maxFollowers: exploreMaxFollowers,
+        location: exploreLocation,
+        browser,
+      }, console.error);
+
+      if (result.restricted) {
+        await apiPost(`${serverUrl}/api/job/${username}`, { restricted: true, userInfo: result.userInfo || {} });
+        continue;
+      }
+
+      if (result.error) {
+        errorCount++;
+        await apiPost(`${serverUrl}/api/job/${username}`, { error: result.error });
+        continue;
+      }
+
+      const payload = {
+        userInfo: result.userInfo || {},
+        discoveredVideoAuthors: result.discoveredVideoAuthors || [],
+        discoveredCommentAuthors: result.discoveredCommentAuthors || [],
+        discoveredGuessAuthors: result.discoveredGuessAuthors || [],
+        discoveredFollowing: result.discoveredFollowing || [],
+        discoveredFollowers: result.discoveredFollowers || [],
+        processed: result.processed,
+        hasFollowData: result.hasFollowData,
+        keepFollow: result.keepFollow,
+        locationCreated: result.locationCreated,
+        noVideo: result.noVideo,
+      };
+      await apiPost(`${serverUrl}/api/job/${username}`, payload);
+      console.error('  已提交');
+
+      if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
+        console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
+        break;
+      }
+    }
+
+    const stats = await apiGet(`${serverUrl}/api/stats`);
+    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+  } catch (err) {
+    console.error(`探索失败: ${err.message}`);
+    process.exit(1);
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}

package/src/cli/progress.js

@@ -0,0 +1,111 @@
+import { writeFileSync } from 'fs';
+import { formatOutput } from '../lib/output.js';
+import { deduplicate } from '../lib/output.js';
+import { applyFilter, formatFilterDescription } from '../lib/filter.js';
+import { calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from '../lib/io.js';
+import { randomDelay } from '../lib/delay.js';
+
+export async function processUrlsWithProgress({
+  urls,
+  proxyUrl,
+  outputFile,
+  outputFormat,
+  filter,
+  processFn,
+  label = '数据',
+}) {
+  const allResults = [];
+  const errors = [];
+
+  if (urls.length === 0) {
+    console.error('\n未获取到数据');
+    if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
+    return;
+  }
+
+  const concurrency = calculateConcurrency(urls.length);
+  const bars = createMultiProgressBars(concurrency);
+
+  const slots = Array.from({ length: concurrency }, () => []);
+  urls.forEach((url, i) => slots[i % concurrency].push(url));
+
+  bars.forEach((bar, i) => {
+    bar.total = slots[i].length;
+    bar.status = slots[i].length > 0 ? 'running' : 'done';
+  });
+
+  renderMultiProgressBars(bars);
+
+  const workers = slots.map(async (slotUrls, slotIndex) => {
+    for (const url of slotUrls) {
+      bars[slotIndex].url = url;
+      renderMultiProgressBars(bars);
+
+      await randomDelay();
+
+      try {
+        const results = await processFn(url, proxyUrl);
+        allResults.push(...results);
+        bars[slotIndex].current++;
+        bars[slotIndex].status = 'running';
+      } catch (err) {
+        errors.push({ url, message: err.message });
+        bars[slotIndex].current++;
+        bars[slotIndex].status = 'error';
+      }
+
+      renderMultiProgressBars(bars);
+    }
+    bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
+    renderMultiProgressBars(bars);
+  });
+
+  await Promise.all(workers);
+  clearProgressBars();
+
+  const uniqueResults = deduplicate(allResults);
+  const filteredResults = applyFilter(uniqueResults, filter);
+
+  if (errors.length > 0) {
+    const firstMsg = errors[0].message;
+    const isProxyError = ['不可用', '连接被拒绝', '连接中断', '超时', '无法解析']
+      .some(kw => firstMsg.includes(kw));
+
+    if (filteredResults.length === 0) {
+      if (isProxyError) {
+        console.error(`  所有请求失败，请检查代理: ${proxyUrl}\n`);
+      } else {
+        const show = errors.slice(0, 5);
+        for (const e of show) console.error(`  ✗ ${e.url}: ${e.message}\n`);
+        if (errors.length > 5) console.error(`  ... 还有 ${errors.length - 5} 个失败\n`);
+      }
+      console.error('未获取到数据');
+      if (outputFile) writeFileSync(outputFile, '[]', 'utf-8');
+      return;
+    } else {
+      if (isProxyError) {
+        console.error(`  ${errors.length} 个请求失败，请检查代理: ${proxyUrl}\n`);
+      } else {
+        console.error(`  ${errors.length} 个失败:`);
+        const show = errors.slice(0, 5);
+        for (const e of show) console.error(`    ✗ ${e.url}: ${e.message}`);
+        if (errors.length > 5) console.error(`    ... 还有 ${errors.length - 5} 个`);
+      }
+    }
+  }
+
+  const output = formatOutput(filteredResults, outputFormat);
+
+  if (outputFile) {
+    writeFileSync(outputFile, output, 'utf-8');
+    console.log(`\n结果已写入: ${outputFile}`);
+  } else {
+    process.stdout.write(output + '\n');
+  }
+
+  if (filter) {
+    console.log(`\n共 ${uniqueResults.length} 个${label}，过滤后 ${filteredResults.length} 个（过滤条件: ${formatFilterDescription(filter)}）`);
+  } else {
+    console.log(`\n共 ${filteredResults.length} 个${label}`);
+  }
+}

package/src/cli/scrape.js

@@ -0,0 +1,47 @@
+import { writeFileSync } from 'fs';
+
+export async function handleScrape(options) {
+  const { scrapeUrl, scrapePreset, scrapeMaxVideos, scrapeMaxComments, scrapeMaxGuess, scrapeSwitchDelay, scrapeCommentDelay, outputFile } = options;
+
+  if (!scrapeUrl) {
+    console.error('用法: tt-help scrape <视频URL> [preset] [最大视频数] [最大评论数] [-o 输出路径]');
+    console.error('预设: fast, normal, slow, stealth');
+    console.error('选项: -o, --output <路径>     输出到文件（默认输出到 stdout）');
+    console.error('      --switch-delay <ms>     视频切换延迟（毫秒）');
+    console.error('      --comment-delay <ms>    评论滚动延迟（毫秒）');
+    process.exit(1);
+  }
+
+  const { runScrape } = await import('../lib/scrape-browser.mjs');
+
+  let browser;
+  try {
+    const { output, browser: b } = await runScrape({
+      videoUrl: scrapeUrl,
+      maxVideos: scrapeMaxVideos,
+      maxComments: scrapeMaxComments,
+      maxGuess: scrapeMaxGuess,
+      preset: scrapePreset,
+      switchMax: scrapeSwitchDelay,
+      commentMax: scrapeCommentDelay,
+      log: console.error,
+    });
+    browser = b;
+
+    const json = JSON.stringify(output, null, 2);
+    if (outputFile) {
+      writeFileSync(outputFile, json, 'utf-8');
+      console.error(`结果已写入: ${outputFile}`);
+    } else {
+      process.stdout.write(json + '\n');
+    }
+
+    const stats = output.stats;
+    console.error(`\n共 ${stats.totalVideos} 个视频, ${stats.uniqueVideoAuthors} 个视频作者, ${stats.uniqueCommentAuthors} 个评论作者, ${stats.uniqueGuessAuthors} 个猜你喜欢作者`);
+  } catch (err) {
+    console.error(`浏览器抓取失败: ${err.message}`);
+    process.exit(1);
+  } finally {
+    if (browser) await browser.close().catch(() => {});
+  }
+}

package/src/cli/utils.js

@@ -0,0 +1,18 @@
+import { writeFileSync } from 'fs';
+
+export function cleanError(msg) {
+  return msg
+    .replace(/\x1b\[[0-9;]*m/g, '')
+    .replace(/\s*- navigating to.*/s, '')
+    .replace(/\s*Call log:/s, '')
+    .trim();
+}
+
+export function writeJson(data, outputFile) {
+  const json = JSON.stringify(data, null, 2);
+  if (outputFile) {
+    writeFileSync(outputFile, json, 'utf-8');
+  } else {
+    process.stdout.write(json + '\n');
+  }
+}

package/src/cli/videos.js

@@ -0,0 +1,41 @@
+import { writeFileSync } from 'fs';
+
+export async function handleVideos(options) {
+  const { videosUsername, videosMax, outputFile } = options;
+
+  if (!videosUsername) {
+    console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
+    console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
+    console.error('      tt-help videos username 50 -o videos.json');
+    console.error('');
+    console.error('选项: -o, --output <路径>  输出到文件（默认输出到 stdout）');
+    process.exit(1);
+  }
+
+  const { runGetUserVideos } = await import('../lib/get-user-videos-browser.mjs');
+
+  let browser;
+  try {
+    const { output, browser: b } = await runGetUserVideos({
+      username: videosUsername,
+      maxVideos: videosMax,
+      log: console.error,
+    });
+    browser = b;
+
+    const json = JSON.stringify(output, null, 2);
+    if (outputFile) {
+      writeFileSync(outputFile, json, 'utf-8');
+      console.error(`结果已写入: ${outputFile}`);
+    } else {
+      process.stdout.write(json + '\n');
+    }
+
+    console.error(`\n共 ${output.videos.length} 个视频, 用户: @${videosUsername}`);
+  } catch (err) {
+    console.error(`获取用户视频失败: ${err.message}`);
+    process.exit(1);
+  } finally {
+    if (browser) await browser.close().catch(() => {});
+  }
+}

package/src/cli/watch.js

@@ -0,0 +1,28 @@
+import { writeFileSync, existsSync } from 'fs';
+import { startWatchServer, openBrowser } from '../watch/server.mjs';
+
+export async function handleWatch(options) {
+  const { outputFile, watchPort } = options;
+
+  if (!outputFile) {
+    console.error('用法: tt-help watch -o <数据文件> [-p 端口]');
+    console.error('示例: tt-help watch -o data.json');
+    console.error('      tt-help watch -o data.json -p 8080');
+    process.exit(1);
+  }
+
+  if (!existsSync(outputFile)) {
+    console.error(`文件不存在: ${outputFile}`);
+    process.exit(1);
+  }
+
+  const { server, port } = await startWatchServer(outputFile, watchPort);
+  openBrowser(port);
+
+  process.once('SIGINT', () => {
+    server.close();
+    process.exit(0);
+  });
+
+  console.error('按 Ctrl+C 停止监控服务');
+}