tt-help-cli-ycl 1.3.13 → 1.3.14

package/src/lib/constants.js

@@ -15,6 +15,10 @@ let server = 'http://127.0.0.1:3001';
 let configFile = null;
 let browser = null;
 let userId = null;
+let maxFollowing = 5;
+let maxFollowers = 5;
+let maxVideos = 1;
+let maxComments = 10;
 
 try {
   if (existsSync(configPath)) {
@@ -31,6 +35,18 @@ try {
     if (cfg.userId) {
       userId = cfg.userId;
     }
+    if (cfg.maxFollowing !== undefined) {
+      maxFollowing = cfg.maxFollowing;
+    }
+    if (cfg.maxFollowers !== undefined) {
+      maxFollowers = cfg.maxFollowers;
+    }
+    if (cfg.maxVideos !== undefined) {
+      maxVideos = cfg.maxVideos;
+    }
+    if (cfg.maxComments !== undefined) {
+      maxComments = cfg.maxComments;
+    }
     configFile = configPath;
   }
 } catch {
@@ -53,34 +69,86 @@ function saveUserId(id) {
   configFile = configPath;
 }
 
+function saveMaxFollowing(val) {
+  const cfg = existsSync(configPath) ? JSON.parse(readFileSync(configPath, 'utf-8')) : {};
+  cfg.maxFollowing = parseInt(val) || 5;
+  writeFileSync(configPath, JSON.stringify(cfg, null, 2), 'utf-8');
+  maxFollowing = cfg.maxFollowing;
+  configFile = configPath;
+}
+
+function saveMaxFollowers(val) {
+  const cfg = existsSync(configPath) ? JSON.parse(readFileSync(configPath, 'utf-8')) : {};
+  cfg.maxFollowers = parseInt(val) || 5;
+  writeFileSync(configPath, JSON.stringify(cfg, null, 2), 'utf-8');
+  maxFollowers = cfg.maxFollowers;
+  configFile = configPath;
+}
+
+function saveMaxVideos(val) {
+  const cfg = existsSync(configPath) ? JSON.parse(readFileSync(configPath, 'utf-8')) : {};
+  cfg.maxVideos = parseInt(val) || 1;
+  writeFileSync(configPath, JSON.stringify(cfg, null, 2), 'utf-8');
+  maxVideos = cfg.maxVideos;
+  configFile = configPath;
+}
+
+function saveMaxComments(val) {
+  const cfg = existsSync(configPath) ? JSON.parse(readFileSync(configPath, 'utf-8')) : {};
+  cfg.maxComments = parseInt(val) || 10;
+  writeFileSync(configPath, JSON.stringify(cfg, null, 2), 'utf-8');
+  maxComments = cfg.maxComments;
+  configFile = configPath;
+}
+
 const HELP_TEXT = [
-  '用法: tt-help explore <用户名> [preset] [选项]',
+  '用法:',
   '',
-  '  支持多个用户名: tt-help explore @user1 @user2 --server http://127.0.0.1:3001',
-  '  预设: fast, normal（默认）, slow, stealth',
-  '  选项:',
-  '    --server <URL>            服务端地址，默认 http://127.0.0.1:3001',
-  '    --location <国家代码>     国家筛选，逗号分隔，默认 PL,NL,BE,DE,FR,IT,ES,IE',
-  '    --max-comments <数量>     每视频最大评论数，默认 10',
-  '    --max-guess <数量>        每视频最大猜你喜欢数，默认 0',
-  '    --enable-follow           启用关注/粉丝提取（默认启用）',
-  '    --disable-follow          禁用关注/粉丝提取',
-  '    --max-following <数量>    最大获取关注数，默认 5',
-  '    --max-followers <数量>    最大获取粉丝数，默认 5',
-  '    --max-users <数量>        最大处理用户数，默认无限制',
-  '    --port <端口号>           CDP 端口，默认 9222',
-  '    --profile <名称>          浏览器 Profile 名称',
-  '    --user-id <编号>          客户端编号（设备ID），默认自动生成',
-  '  全局选项:',
+  '  tt-help explore <用户名> [preset] [选项]',
+  '    支持多个用户名: tt-help explore @user1 @user2 --server http://127.0.0.1:3001',
+  '    预设: fast, normal（默认）, slow, stealth',
+  '    选项:',
+  '      --server <URL>            服务端地址，默认 http://127.0.0.1:3001',
+  '      --location <国家代码>     国家筛选，逗号分隔，默认 PL,NL,BE,DE,FR,IT,ES,IE',
+  '      --max-comments <数量>     每视频最大评论数，默认 10',
+  '      --max-guess <数量>        每视频最大猜你喜欢数，默认 0',
+  '      --enable-follow           启用关注/粉丝提取（默认启用）',
+  '      --disable-follow          禁用关注/粉丝提取',
+  '      --max-following <数量>    最大获取关注数，默认 5',
+  '      --max-followers <数量>    最大获取粉丝数，默认 5',
+  '      --max-users <数量>        最大处理用户数，默认无限制',
+  '      --port <端口号>           CDP 端口，默认 9222',
+  '      --profile <名称>          浏览器 Profile 名称',
+  '      --user-id <编号>          客户端编号（设备ID），默认自动生成',
+  '',
+  '  tt-help info <URL> [URL2 ...] [--onlyvideo]',
+  '    获取用户/视频信息，支持多个 URL',
+  '    主页 URL      → 返回用户信息',
+  '    视频 URL      → 返回用户信息 + 视频信息',
+  '    视频 URL + --onlyvideo → 只返回视频信息',
+  '    示例: tt-help info https://www.tiktok.com/@nike',
+  '          tt-help info https://www.tiktok.com/@nike/video/7234567890 --onlyvideo',
+  '',
+  '  tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]',
+  '    后台轮询服务端任务接口，自动抓取 TikTok 用户信息',
+  '    -p, --parallel <N>     并行抓取数（默认: 1）',
+  '    -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）',
+  '    -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）',
+  '    示例: tt-help attach -p 5 -i 10',
+  '',
+  '  config  [show|set|reset]',
   '    config                    查看当前配置',
-  '    config set <key> <value>  设置配置（key: proxy, server, browser, userId）',
+    '    config set <key> <value>  设置配置（key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments）',
   '    config reset              重置所有配置为默认',
+  '',
+  '  全局选项:',
   '    -h, --help                显示帮助',
   '    --version                 显示版本号',
   '',
-  '  示例: tt-help explore qiqi23280 fast --location ES --max-comments 50',
-  '        tt-help explore @user1 --user-id my-device-001 --port 9333',
+  '  示例: tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@adidas',
+  '        tt-help explore qiqi23280 fast --location ES --max-comments 50',
   '        tt-help config set server http://127.0.0.1:3001',
+  '        tt-help attach -p 5 -i 10',
 ];
 
 function getConfigText() {
@@ -99,6 +167,10 @@ function getConfigText() {
     `  服务端: ${server}`,
     `  浏览器: ${browser || '未配置（将自动探测或回退）'}`,
     `  用户号: ${currentUserId || '未设置（首次运行 auto 自动创建）'}`,
+    `  商家关注采集数: ${maxFollowing}`,
+    `  粉丝采集数: ${maxFollowers}`,
+    `  视频采集数: ${maxVideos}`,
+    `  评论采集数: ${maxComments}`,
     `  输出格式: json`,
     `  默认输出: ${DEFAULT_OUTPUT}`,
     `  配置文件: ${configFile || '无（使用默认值）'}`,
@@ -113,7 +185,15 @@ export {
   HELP_TEXT,
   browser,
   userId,
+  maxFollowing,
+  maxFollowers,
+  maxVideos,
+  maxComments,
   saveBrowser,
   saveUserId,
+  saveMaxFollowing,
+  saveMaxFollowers,
+  saveMaxVideos,
+  saveMaxComments,
   getConfigText,
 };

package/src/lib/parse-ssr.mjs

@@ -0,0 +1,69 @@
+function parseSSR(rawHtml) {
+  if (!rawHtml.includes('__UNIVERSAL_DATA_FOR_REHYDRATION__')) return null;
+  const dataStart = rawHtml.indexOf('__UNIVERSAL_DATA_FOR_REHYDRATION__');
+  // 从该字符串后面找 <script 标签的 >，确保找到的是正确行的 >
+  const scriptStart = rawHtml.lastIndexOf('<script', dataStart);
+  const sIdx = (scriptStart >= 0 ? rawHtml.indexOf('>', scriptStart) : rawHtml.indexOf('>', dataStart)) + 1;
+  const eIdx = rawHtml.indexOf('</script>', sIdx);
+  if (sIdx < 0 || eIdx < 0) return null;
+  const jsonStr = rawHtml.substring(sIdx, eIdx);
+  return JSON.parse(jsonStr);
+}
+
+export function parseUserInfo(rawHtml) {
+  const data = parseSSR(rawHtml);
+  if (!data) return null;
+  const ud = data.__DEFAULT_SCOPE__['webapp.user-detail'];
+  if (!ud || !ud.userInfo) return null;
+  const u = ud.userInfo.user;
+  const s = ud.userInfo.stats;
+  return {
+    uniqueId: u.uniqueId,
+    nickname: u.nickname,
+    id: u.id,
+    verified: u.verified,
+    privateAccount: u.privateAccount,
+    language: u.language,
+    bio: u.signature || '',
+    avatar: u.avatarLarger || u.avatarMedium || u.avatarThumb || '',
+    followerCount: s.followerCount,
+    followingCount: s.followingCount,
+    heartCount: s.heartCount,
+    videoCount: s.videoCount,
+    diggCount: s.diggCount,
+    friendCount: s.friendCount,
+    secUid: u.secUid,
+    ttSeller: u.ttSeller || false,
+    locationCreated: u.locationCreated || null,
+  };
+}
+
+export function parseVideoInfo(rawHtml) {
+  const data = parseSSR(rawHtml);
+  if (!data) return null;
+  const vd = data.__DEFAULT_SCOPE__['webapp.video-detail'];
+  if (!vd || !vd.itemInfo || !vd.itemInfo.itemStruct) return null;
+  const item = vd.itemInfo.itemStruct;
+  const author = item.author || {};
+  const stats = item.stats || {};
+  return {
+    id: item.id,
+    desc: item.desc || '',
+    createTime: item.createTime || null,
+    locationCreated: item.locationCreated || null,
+    author: {
+      uniqueId: author.uniqueId,
+      nickname: author.nickname,
+      id: author.id,
+      verified: author.verified,
+      secUid: author.secUid,
+    },
+    stats: {
+      playCount: stats.playCount,
+      diggCount: stats.diggCount,
+      commentCount: stats.commentCount,
+      shareCount: stats.shareCount,
+      collectCount: stats.collectCount,
+    },
+  };
+}

package/src/lib/scrape.js

@@ -1,25 +1,75 @@
-import { extractUserSection, parseUserSection, extractLocationCreated } from './parser.js';
-import { fetchHtml, isProfileUrl } from './fetcher.js';
-import { toProfileUrl, isVideoUrl, extractUniqueId } from './url.js';
-
-export async function extractUserData(profileUrl, proxyUrl) {
-  const profileHtml = await fetchHtml(profileUrl, proxyUrl);
-  const section = extractUserSection(profileHtml);
-  if (!section) throw new Error('无法解析用户信息');
-  const data = parseUserSection(section);
-  data.locationCreated = extractLocationCreated(profileHtml);
-  return data;
+import { TikTokScraper } from './tiktok-scraper.mjs';
+import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from './url.js';
+
+// Lazy singleton for TikTokScraper
+let scraperInstance = null;
+let scraperInitPromise = null;
+
+async function getScraper() {
+  if (scraperInstance) return scraperInstance;
+  if (scraperInitPromise) return scraperInitPromise;
+  scraperInitPromise = (async () => {
+    const scraper = new TikTokScraper();
+    await scraper.init();
+    scraperInstance = scraper;
+    scraperInitPromise = null;
+    return scraper;
+  })();
+  return scraperInitPromise;
+}
+
+export async function closeScraper() {
+  if (scraperInstance) {
+    await scraperInstance.close();
+    scraperInstance = null;
+  }
+}
+
+// Map parseUserInfo output to legacy parser.js format
+function mapUserInfo(user) {
+  if (!user) return null;
+  return {
+    uniqueId: user.uniqueId,
+    uid: user.id,
+    secUid: user.secUid,
+    nickname: user.nickname,
+    signature: user.bio,
+    ttSeller: user.ttSeller,
+    verified: user.verified,
+    followerCount: user.followerCount,
+    followingCount: user.followingCount,
+    heartCount: user.heartCount,
+    videoCount: user.videoCount,
+    diggCount: user.diggCount,
+    avatarLarger: user.avatar,
+    locationCreated: user.locationCreated,
+  };
+}
+
+// Map parseVideoInfo output to legacy format
+function mapVideoLocation(video) {
+  if (!video) return null;
+  return video.locationCreated;
+}
+
+export async function extractUserData(url) {
+  const scraper = await getScraper();
+  const uniqueId = extractUniqueId(url);
+  if (!uniqueId) throw new Error(`无法从URL提取用户名: ${url}`);
+  const user = await scraper.getUserInfo(normalizeUsername(uniqueId));
+  if (!user) throw new Error('无法解析用户信息');
+  return mapUserInfo(user);
 }
 
-export async function extractVideoLocation(videoUrl, proxyUrl) {
-  const videoHtml = await fetchHtml(videoUrl, proxyUrl);
-  return extractLocationCreated(videoHtml);
+export async function extractVideoLocation(videoUrl) {
+  const scraper = await getScraper();
+  const video = await scraper.getVideoInfo(videoUrl);
+  return mapVideoLocation(video);
 }
 
-export async function processUrl(url, proxyUrl) {
+export async function processUrl(url) {
   if (isProfileUrl(url)) {
-    const profileUrl = toProfileUrl(url);
-    const profileData = await extractUserData(profileUrl, proxyUrl);
+    const profileData = await extractUserData(url);
     return [profileData];
   }
 
@@ -27,10 +77,9 @@ export async function processUrl(url, proxyUrl) {
     const profileHandle = extractUniqueId(url);
     if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
 
-    const profileUrl = toProfileUrl(profileHandle);
     const [profileData, locationCreated] = await Promise.all([
-      extractUserData(profileUrl, proxyUrl),
-      extractVideoLocation(url, proxyUrl),
+      extractUserData(url),
+      extractVideoLocation(url),
     ]);
 
     return [{ ...profileData, locationCreated }];

package/src/lib/tiktok-scraper.mjs

@@ -0,0 +1,176 @@
+import { chromium } from 'playwright';
+import { detectBrowser } from './browser/launch.js';
+import { parseUserInfo, parseVideoInfo } from './parse-ssr.mjs';
+
+const DEFAULT_POOL_SIZE = 3;
+const DEFAULT_WAF_TTL = 120000;
+const DEFAULT_WARM_URL = 'https://www.tiktok.com/@nike';
+
+function delay(ms) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+class PageSlot {
+  constructor(page) {
+    this.page = page;
+    this.lock = new PromiseQueue();
+  }
+}
+
+class PromiseQueue {
+  constructor() {
+    this._queue = [];
+    this._processing = false;
+  }
+  async run(task) {
+    return new Promise((resolve, reject) => {
+      this._queue.push({ task, resolve, reject });
+      this._process();
+    });
+  }
+  async _process() {
+    if (this._processing) return;
+    this._processing = true;
+    while (this._queue.length > 0) {
+      const { task, resolve, reject } = this._queue.shift();
+      try {
+        const result = await task();
+        resolve(result);
+      } catch (e) {
+        reject(e);
+      }
+    }
+    this._processing = false;
+  }
+}
+
+export class TikTokScraper {
+  constructor({ poolSize = DEFAULT_POOL_SIZE, wafTtl = DEFAULT_WAF_TTL, warmUrl = DEFAULT_WARM_URL } = {}) {
+    this.poolSize = poolSize;
+    this.wafTtl = wafTtl;
+    this.warmUrl = warmUrl;
+    this.browser = null;
+    this.context = null;
+    this.slots = [];
+    this.slotIdx = 0;
+    this.lastWarmTime = 0;
+    this.warmPromise = null;
+  }
+
+  async init() {
+    const executablePath = detectBrowser();
+    if (!executablePath) {
+      throw new Error('未找到本地浏览器（Chrome/Edge），请先安装浏览器或执行 npx playwright install');
+    }
+    this.browser = await chromium.launch({
+      headless: true,
+      executablePath,
+      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
+    });
+    this.context = await this.browser.newContext();
+    for (let i = 0; i < this.poolSize; i++) {
+      this.slots.push(new PageSlot(await this.context.newPage()));
+    }
+    await this.warmWaf();
+  }
+
+  async close() {
+    if (this.browser) {
+      await this.browser.close();
+      this.browser = null;
+      this.context = null;
+      this.slots = [];
+    }
+  }
+
+  async restart() {
+    await this.close();
+    await this.init();
+  }
+
+  isAlive() {
+    try {
+      return !!(this.browser && this.browser.isConnected());
+    } catch {
+      return false;
+    }
+  }
+
+  async warmWaf() {
+    if (this.warmPromise) return this.warmPromise;
+    this.warmPromise = (async () => {
+      const page = this.slots[0].page;
+      await page.goto(this.warmUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
+      await delay(1500);
+      this.lastWarmTime = Date.now();
+      this.warmPromise = null;
+    })();
+    return this.warmPromise;
+  }
+
+  _needWarm() {
+    return Date.now() - this.lastWarmTime > this.wafTtl;
+  }
+
+  _pickSlot() {
+    const slot = this.slots[this.slotIdx % this.poolSize];
+    this.slotIdx++;
+    return slot;
+  }
+
+  async _ensurePage(slot) {
+    try {
+      if (!slot.page.isClosed()) return slot.page;
+    } catch {}
+    slot.page = await this.context.newPage();
+    return slot.page;
+  }
+
+  async _fetchViewSource(url, slot) {
+    const page = await this._ensurePage(slot);
+
+    await page.goto('view-source:' + url, {
+      waitUntil: 'domcontentloaded',
+      timeout: 15000,
+    });
+
+    return await page.evaluate(() => {
+      const rows = document.querySelectorAll('tr');
+      let content = '';
+      rows.forEach(r => {
+        const lc = r.querySelector('.line-content');
+        if (lc) content += lc.textContent + '\n';
+      });
+      return content;
+    });
+  }
+
+  async getUserInfo(uniqueId) {
+    if (this._needWarm()) await this.warmWaf();
+    const slot = this._pickSlot();
+    return slot.lock.run(async () => {
+      const rawHtml = await this._fetchViewSource(
+        `https://www.tiktok.com/@${uniqueId}`,
+        slot
+      );
+      return parseUserInfo(rawHtml);
+    });
+  }
+
+  async getVideoInfo(videoUrl) {
+    if (this._needWarm()) await this.warmWaf();
+    const slot = this._pickSlot();
+    return slot.lock.run(async () => {
+      const rawHtml = await this._fetchViewSource(videoUrl, slot);
+      return parseVideoInfo(rawHtml);
+    });
+  }
+
+  async getUserAndVideo(videoUrl) {
+    if (this._needWarm()) await this.warmWaf();
+    const video = await this.getVideoInfo(videoUrl);
+    if (!video) return null;
+    const user = await this.getUserInfo(video.author.uniqueId);
+    return { user, video };
+  }
+}

package/src/lib/url.js

@@ -1,7 +1,7 @@
 const BASE_URL = 'https://www.tiktok.com';
 
 export function extractUniqueId(url) {
-  const m = url.match(/\/@([^/]+)/);
+  const m = url.match(/\/@([\w.-]+)/);
   return m ? m[1] : null;
 }
 
@@ -30,7 +30,7 @@ export function ensureAbsoluteUrl(href) {
 }
 
 export function isProfileUrl(url) {
-  return /\/@[\w-]+(?:$|[?#])/.test(url);
+  return /\/@[\w.-]+(?:$|[?#])/.test(url);
 }
 
 export function isVideoUrl(url) {

package/src/main.js

@@ -1,30 +1,22 @@
 import { parseArgs } from './lib/args.js';
 import { proxy, HELP_TEXT, getConfigText } from './lib/constants.js';
-import { parseFilter } from './lib/filter.js';
-import { handleScrape } from './cli/scrape.js';
-import { handleVideos } from './cli/videos.js';
-import { handleAuto } from './cli/auto.js';
+import { handleInfo } from './cli/info.js';
 import { handleExplore } from './cli/explore.js';
-import { handleRefresh } from './cli/refresh.js';
+import { handleAttach } from './cli/attach.js';
 import { handleWatch } from './cli/watch.js';
 import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
-import { runExploreDefault, runScrapeDefault } from './cli/explore-default.js';
 
 async function main() {
   const parsed = parseArgs();
 
   switch (parsed.subcommand) {
-    case 'scrape': return handleScrape(parsed);
-    case 'videos': return handleVideos(parsed);
-    case 'auto':   return handleAuto(parsed);
-    case 'explore':return handleExplore(parsed);
-    case 'refresh':return handleRefresh(parsed);
-    case 'watch':  return handleWatch(parsed);
+    case 'explore': return handleExplore(parsed);
+    case 'info':    return handleInfo(parsed);
+    case 'attach':  return handleAttach(parsed);
+    case 'watch':   return handleWatch(parsed);
   }
 
-  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue, pipeMode, filterStr } = parsed;
-  const proxyUrl = customProxy || proxy;
-  const filter = parseFilter(filterStr);
+  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue } = parsed;
 
   if (showVersion) {
     console.log(version);
@@ -35,11 +27,13 @@ async function main() {
   if (showCfg) return showConfig(urls, outputFile);
   if (urls.length === 0 && exploreCount === 0) return showUsage();
 
+  // 默认行为：URL 走 info，--explore 走 explore
   if (exploreCount > 0) {
-    await runExploreDefault(exploreCount, urls, proxyUrl, outputFile, outputFormat, pipeMode, filter);
-  } else {
-    await runScrapeDefault(urls, proxyUrl, outputFile, outputFormat, filter);
+    return handleExplore({ ...parsed, subcommand: 'explore' });
   }
+
+  // 有 URL 默认走 info
+  return handleInfo(parsed);
 }
 
 main().catch(err => {