tt-help-cli-ycl 1.0.6 → 1.0.7

package/src/lib/args.js

@@ -1,8 +1,234 @@
 import { readFileSync } from 'fs';
 import { proxy } from './constants.js';
 
+const PRESETS = ['fast', 'normal', 'slow', 'stealth'];
+
+function parseScrapeArgs(args) {
+  let scrapeUrl = null;
+  let scrapePreset = null;
+  let scrapeMaxVideos = 20;
+  let scrapeMaxComments = 999;
+  let scrapeSwitchDelay = null;
+  let scrapeCommentDelay = null;
+  let outputFile = null;
+
+  const positional = [];
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '-o' || arg === '--output') {
+      outputFile = args[++i];
+    } else if (arg === '--switch-delay') {
+      scrapeSwitchDelay = parseInt(args[++i]) || null;
+    } else if (arg === '--comment-delay') {
+      scrapeCommentDelay = parseInt(args[++i]) || null;
+    } else {
+      positional.push(arg);
+    }
+  }
+
+  scrapeUrl = positional[0] || null;
+
+  if (positional[1]) {
+    if (PRESETS.includes(positional[1].toLowerCase())) {
+      scrapePreset = positional[1].toLowerCase();
+      scrapeMaxVideos = parseInt(positional[2]) || 20;
+      scrapeMaxComments = parseInt(positional[3]) || 999;
+    } else {
+      scrapeMaxVideos = parseInt(positional[1]) || 20;
+      scrapeMaxComments = parseInt(positional[2]) || 999;
+    }
+  }
+
+  return {
+    subcommand: 'scrape',
+    scrapeUrl,
+    scrapePreset,
+    scrapeMaxVideos,
+    scrapeMaxComments,
+    scrapeSwitchDelay,
+    scrapeCommentDelay,
+    outputFile,
+    urls: [],
+    outputFormat: 'json',
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
+function parseAutoArgs(args) {
+  let autoCollectMax = 1;
+  let autoScrapeDepth = 50;
+  let autoMaxComments = 200;
+  let autoPreset = 'fast';
+  let autoSwitchDelay = null;
+  let autoCommentDelay = null;
+  let outputFile = null;
+  let autoWatch = false;
+  let autoWatchPort = 3000;
+
+  const positional = [];
+  const PRESETS = ['fast', 'normal', 'slow', 'stealth'];
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '-o' || arg === '--output') {
+      outputFile = args[++i];
+    } else if (arg === '--switch-delay') {
+      autoSwitchDelay = parseInt(args[++i]) || null;
+    } else if (arg === '--comment-delay') {
+      autoCommentDelay = parseInt(args[++i]) || null;
+    } else if (arg === '--watch') {
+      autoWatch = true;
+      if (args[i + 1] === '-p') {
+        autoWatchPort = parseInt(args[i + 2]) || 3000;
+        i += 2;
+      }
+    } else if (arg === '-p' && autoWatch) {
+      autoWatchPort = parseInt(args[++i]) || 3000;
+    } else {
+      positional.push(arg);
+    }
+  }
+
+  // 收集用户名（非 preset、非数字的都是用户名）
+  const usernames = [];
+  let j = 0;
+  while (j < positional.length && !PRESETS.includes(positional[j]?.toLowerCase()) && isNaN(positional[j])) {
+    usernames.push(positional[j].replace('@', ''));
+    j++;
+  }
+
+  // preset
+  if (j < positional.length && PRESETS.includes(positional[j].toLowerCase())) {
+    autoPreset = positional[j].toLowerCase();
+    j++;
+    autoCollectMax = parseInt(positional[j]) || 1; j++;
+    autoScrapeDepth = parseInt(positional[j]) || 50; j++;
+    autoMaxComments = parseInt(positional[j]) || 200; j++;
+  } else if (usernames.length > 0) {
+    autoCollectMax = parseInt(positional[j]) || 1; j++;
+    autoScrapeDepth = parseInt(positional[j]) || 50; j++;
+    autoMaxComments = parseInt(positional[j]) || 200;
+  }
+
+  return {
+    subcommand: 'auto',
+    autoUsernames: usernames,
+    autoCollectMax,
+    autoScrapeDepth,
+    autoMaxComments,
+    autoPreset,
+    autoSwitchDelay,
+    autoCommentDelay,
+    outputFile,
+    autoWatch,
+    autoWatchPort,
+    urls: [],
+    outputFormat: 'json',
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
+function parseVideosArgs(args) {
+  let videosUsername = null;
+  let videosMax = 5;
+  let outputFile = null;
+
+  const positional = [];
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '-o' || arg === '--output') {
+      outputFile = args[++i];
+    } else {
+      positional.push(arg);
+    }
+  }
+
+  videosUsername = positional[0] ? positional[0].replace('@', '') : null;
+  videosMax = parseInt(positional[1]) || 5;
+
+  return {
+    subcommand: 'videos',
+    videosUsername,
+    videosMax,
+    outputFile,
+    urls: [],
+    outputFormat: 'json',
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
+function parseWatchArgs(args) {
+  let outputFile = null;
+  let watchPort = 3000;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '-o' || arg === '--output') {
+      outputFile = args[++i];
+    } else if (arg === '-p') {
+      watchPort = parseInt(args[++i]) || 3000;
+    }
+  }
+
+  return {
+    subcommand: 'watch',
+    outputFile,
+    watchPort,
+    urls: [],
+    outputFormat: 'json',
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 export function parseArgs() {
   const args = process.argv.slice(2);
+
+  if (args.length > 0 && args[0] === 'scrape') {
+    return parseScrapeArgs(args.slice(1));
+  }
+
+  if (args.length > 0 && args[0] === 'videos') {
+    return parseVideosArgs(args.slice(1));
+  }
+
+  if (args.length > 0 && args[0] === 'auto') {
+    return parseAutoArgs(args.slice(1));
+  }
+
+  if (args.length > 0 && args[0] === 'watch') {
+    return parseWatchArgs(args.slice(1));
+  }
+
   const urls = [];
   let inputFile = null;
   let outputFile = null;
@@ -59,4 +285,4 @@ export function parseArgs() {
   }
 
   return { urls, outputFile, outputFormat, exploreCount, showConfig, showHelp, customProxy, configAction, configValue, pipeMode, filterStr };
-}
+}

package/src/lib/auto-browser.mjs

@@ -0,0 +1,11 @@
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const core = require('../auto-core.cjs');
+const helpers = require('../scraper/modules/page-helpers.cjs');
+
+export const runAuto = core.runAuto;
+export const processUser = core.processUser;
+export const ensureBrowserReady = helpers.ensureBrowserReady;
+export const ensureTikTokPage = helpers.ensureTikTokPage;
+export const closeCommentPanel = helpers.closeCommentPanel;

package/src/lib/constants.js

@@ -40,6 +40,9 @@ function saveBrowser(path) {
 
 const HELP_TEXT = [
   '用法: tt-help [选项]',
+  '       tt-help scrape <URL> [预设] [选项]',
+  '       tt-help videos <用户名> [最大视频数] [选项]',
+  '       tt-help auto <用户名> [preset] [收集数] [切换数] [每视频评论数] [选项]',
   '',
   '参数:',
   '  --explore [count]      从 Explore 页面获取视频列表（默认: 100）',
@@ -52,6 +55,42 @@ const HELP_TEXT = [
   '  -c, --config           显示当前配置',
   '  -h, --help             显示帮助',
   '',
+  '浏览器抓取模式 (scrape):',
+  '  tt-help scrape <URL> [preset] [最大视频数] [最大评论数] [-o 输出路径]',
+  '  tt-help scrape <URL> [最大视频数] [最大评论数] [--switch-delay ms] [--comment-delay ms]',
+  '  预设: fast(600ms/400ms), normal(1.5s/800ms), slow(3s/2s), stealth(5s/3.5s)',
+  '  不指定 -o 则输出到 stdout（可 pipe）',
+  '',
+  '用户视频模式 (videos):',
+  '  tt-help videos <用户名> [最大视频数] [-o 输出路径]',
+  '  获取用户信息和视频列表（浏览器模式）',
+  '  不指定 -o 则输出到 stdout（可 pipe）',
+  '',
+  '自动探索模式 (auto):',
+  '  tt-help auto [用户名...] [preset] [收集数] [切换数] [每视频评论数] [选项]',
+  '  支持多个用户名: tt-help auto @user1 @user2 -o data.json',
+  '  不指定用户名: 从 -o 数据源中读取未处理的用户继续探索',
+  '  收集数: 从种子用户收集的种子视频数（默认 1）',
+  '  切换数: 每个种子作者主页下切换视频的次数（默认 50）',
+  '  每视频评论数: 每个视频抓取评论数（默认 200）',
+  '  预设: fast（默认）, normal, slow, stealth',
+  '  选项: -o <路径>             数据源文件（不指定则输出到 stdout）',
+  '        --switch-delay <ms>   视频切换延迟（毫秒）',
+  '        --comment-delay <ms>  评论滚动延迟（毫秒）',
+  '        --watch [-p 3000]     启动 Web 监控页面实时查看数据',
+  '  返回: 纯用户数组，按 uniqueId 去重',
+  '  示例: tt-help auto username -o result.json',
+  '        tt-help auto @a @b fast 1 30 100 -o data.json',
+  '        tt-help auto -o data.json                  # 续跑',
+  '        tt-help auto username --watch -o data.json  # 带监控页面',
+  '',
+  '实时监控模式 (watch):',
+  '  tt-help watch -o <数据文件> [-p 端口]',
+  '  启动 Web 监控页面，实时查看采集数据',
+  '  支持在 auto 模式中通过 --watch 参数同时启动',
+  '  示例: tt-help watch -o data.json',
+  '        tt-help watch -o data.json -p 8080',
+  '',
   '过滤示例:',
   '  --filter "ttSeller=true&verified=false"           过滤卖家且未认证',
   '  --filter "locationCreated=DE,ES"                  过滤指定地区',
@@ -72,6 +111,13 @@ const HELP_TEXT = [
   '  tt-help --explore -f raw                 # 仅输出 URL 列表',
   '  tt-help --explore -f raw -o urls.txt     # 保存 URL 到文件',
   '  tt-help -i urls.txt -o result.json       # 再爬取这些 URL',
+  '  tt-help scrape <URL> fast 50 999        # 浏览器抓取（快速预设）',
+  '  tt-help scrape <URL> slow -o out.json    # 慢速预设，输出到文件',
+  '  tt-help scrape <URL> 20 999 | jq .       # 输出到 stdout 可 pipe',
+  '  tt-help videos username 100              # 获取用户 100 个视频',
+  '  tt-help videos username 50 -o out.json   # 输出到文件',
+  '  tt-help auto username -o out.json  # 自动探索（默认 fast/1/50/200）',
+  '  tt-help auto username slow 1 30 100 -o out.json  # 慢速探索',
   '  tt-help config set http://127.0.0.1:7890',
   '  tt-help https://www.tiktok.com/@username',
   '  tt-help https://... --filter "ttSeller=true&locationCreated=DE"',

package/src/lib/explore.js

@@ -8,6 +8,32 @@ function sleep(ms) {
   return new Promise(r => setTimeout(r, ms));
 }
 
+function isRetryableError(error) {
+  if (!error) return false;
+  const msg = (error.message || error.toString() || '').toLowerCase();
+  const patterns = ['interrupted', 'net::', 'econn', 'etimedout', 'enotfound', 'eai_again', 'esocketreset', 'connection.*refused', 'connection.*reset', 'failed.*navigate', 'target.*closed', 'crash'];
+  return patterns.some(p => new RegExp(p, 'i').test(msg));
+}
+
+async function retryGoto(page, url, options, { maxRetries = 3, baseDelay = 3000 } = {}) {
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await page.goto(url, options);
+    } catch (error) {
+      lastError = error;
+      if (attempt >= maxRetries || !isRetryableError(error)) {
+        throw error;
+      }
+      const jitter = Math.random() * 500;
+      const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
+      console.log(`    [重试] ${attempt + 1}/${maxRetries}，${Math.round(waitTime)}ms 后重试...`);
+      await sleep(waitTime);
+    }
+  }
+  throw lastError;
+}
+
 function detectBrowser() {
   const isMac = process.platform === 'darwin';
   const isWin = process.platform === 'win32';
@@ -146,7 +172,7 @@ export async function fetchExplore(count = 100) {
     });
 
     const page = await context.newPage();
-    await page.goto(EXPLORE_URL, { waitUntil: 'domcontentloaded', timeout: 30000 });
+    await retryGoto(page, EXPLORE_URL, { waitUntil: 'load', timeout: 30000 });
     console.log(`  [1/6] 页面已加载`);
 
     await sleep(5000);

package/src/lib/fetcher.js

@@ -18,17 +18,27 @@ const HEADERS = {
 export async function fetchHtml(url, proxyUrl) {
   const p = proxyUrl || DEFAULT_PROXY;
   const agent = new ProxyAgent(p);
-  try {
-    const res = await fetch(url, { 
-      headers: HEADERS, 
-      dispatcher: agent,
-      redirect: 'follow',
-    });
-    const html = await res.text();
-    return html;
-  } catch (err) {
-    throw new Error(`请求 ${url} 失败，代理 ${p} 不可用`);
+  let lastError;
+
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    try {
+      const res = await fetch(url, {
+        headers: HEADERS,
+        dispatcher: agent,
+        redirect: 'follow',
+      });
+      const html = await res.text();
+      return html;
+    } catch (err) {
+      lastError = err;
+      if (attempt < 3) {
+        const waitMs = Math.pow(2, attempt - 1) * 3000 + Math.random() * 2000;
+        await new Promise(r => setTimeout(r, waitMs));
+      }
+    }
   }
+
+  throw new Error(`请求 ${url} 失败（已重试 3 次），代理 ${p} 不可用`);
 }
 
 export function makeProfileUrl(handle) {

package/src/lib/get-user-videos-browser.mjs

@@ -0,0 +1,6 @@
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const core = require('../get-user-videos-core.cjs');
+
+export const runGetUserVideos = core.runGetUserVideos;

package/src/lib/scrape-browser.mjs

@@ -0,0 +1,6 @@
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const core = require('../scraper/core.cjs');
+
+export const runScrape = core.runScrape;