tt-help-cli-ycl 1.0.5 → 1.0.7

package/src/main.mjs

@@ -4,7 +4,9 @@ import { fetchExplore } from './lib/explore.js';
 import { processUrl } from './lib/scrape.js';
 import { deduplicate, formatOutput } from './lib/output.js';
 import { parseFilter, applyFilter, formatFilterDescription } from './lib/filter.js';
+import { createProgressBar, calculateConcurrency, createMultiProgressBars, renderMultiProgressBars, clearProgressBars } from './lib/io.js';
 import { writeFileSync, readFileSync, existsSync } from 'fs';
+import { startWatchServer, openBrowser } from './watch/server.mjs';
 
 function showConfig(urls, outputFile) {
   const lines = [...CONFIG_TEXT];
@@ -72,6 +74,10 @@ function handleConfig(action, value) {
   process.exit(1);
 }
 
+function randomDelay() {
+  return new Promise(r => setTimeout(r, Math.random() * 600 + 200));
+}
+
 function cleanError(msg) {
   return msg
     .replace(/\x1b\[[0-9;]*m/g, '')
@@ -112,16 +118,46 @@ async function runExplore(exploreCount, urls, proxyUrl, outputFile, outputFormat
   if (urls.length > 0) {
     const errors = [];
 
-    for (let i = 0; i < urls.length; i++) {
-      const bar = '█'.repeat(i + 1).padEnd(urls.length);
-      process.stdout.write(`\r  [${bar}] ${i + 1}/${urls.length}`);
-      try {
-        const results = await processUrl(urls[i], proxyUrl);
-        allResults.push(...results);
-      } catch (err) {
-        errors.push({ url: urls[i], message: err.message });
+    const concurrency = calculateConcurrency(urls.length);
+    const bars = createMultiProgressBars(concurrency);
+
+    const slots = Array.from({ length: concurrency }, () => []);
+    urls.forEach((url, i) => slots[i % concurrency].push(url));
+
+    bars.forEach((bar, i) => {
+      bar.total = slots[i].length;
+      bar.status = slots[i].length > 0 ? 'running' : 'done';
+    });
+
+    renderMultiProgressBars(bars);
+
+    const workers = slots.map(async (slotUrls, slotIndex) => {
+      for (const url of slotUrls) {
+        bars[slotIndex].url = url;
+        renderMultiProgressBars(bars);
+
+        await randomDelay();
+
+        try {
+          const results = await processUrl(url, proxyUrl);
+          allResults.push(...results);
+          bars[slotIndex].current++;
+          bars[slotIndex].status = 'running';
+        } catch (err) {
+          errors.push({ url, message: err.message });
+          bars[slotIndex].current++;
+          bars[slotIndex].status = 'error';
+        }
+
+        renderMultiProgressBars(bars);
       }
-    }
+      bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
+      renderMultiProgressBars(bars);
+    });
+
+    await Promise.all(workers);
+
+    clearProgressBars();
     console.log();
 
     if (errors.length > 0) {
@@ -173,16 +209,52 @@ async function runScrape(urls, proxyUrl, outputFile, outputFormat, filter) {
   const allResults = [];
   const errors = [];
 
-  for (let i = 0; i < urls.length; i++) {
-    const bar = '█'.repeat(i + 1).padEnd(urls.length);
-    process.stdout.write(`\r  [${bar}] ${i + 1}/${urls.length}`);
-    try {
-      const results = await processUrl(urls[i], proxyUrl);
-      allResults.push(...results);
-    } catch (err) {
-      errors.push({ url: urls[i], message: err.message });
+  if (urls.length === 0) {
+    console.log('\n未获取到数据');
+    if (outputFile) {
+      writeFileSync(outputFile, '[]', 'utf-8');
     }
+    return;
   }
+
+  const concurrency = calculateConcurrency(urls.length);
+  const bars = createMultiProgressBars(concurrency);
+
+  const slots = Array.from({ length: concurrency }, () => []);
+  urls.forEach((url, i) => slots[i % concurrency].push(url));
+
+  bars.forEach((bar, i) => {
+    bar.total = slots[i].length;
+    bar.status = slots[i].length > 0 ? 'running' : 'done';
+  });
+
+  renderMultiProgressBars(bars);
+
+  const workers = slots.map(async (slotUrls, slotIndex) => {
+    for (const url of slotUrls) {
+      bars[slotIndex].url = url;
+      renderMultiProgressBars(bars);
+
+      try {
+        const results = await processUrl(url, proxyUrl);
+        allResults.push(...results);
+        bars[slotIndex].current++;
+        bars[slotIndex].status = 'running';
+      } catch (err) {
+        errors.push({ url, message: err.message });
+        bars[slotIndex].current++;
+        bars[slotIndex].status = 'error';
+      }
+
+      renderMultiProgressBars(bars);
+    }
+    bars[slotIndex].status = bars[slotIndex].current === bars[slotIndex].total ? 'done' : 'error';
+    renderMultiProgressBars(bars);
+  });
+
+  await Promise.all(workers);
+
+  clearProgressBars();
   console.log();
 
   const uniqueResults = deduplicate(allResults);
@@ -242,8 +314,309 @@ async function runScrape(urls, proxyUrl, outputFile, outputFormat, filter) {
   }
 }
 
+async function handleScrape(options) {
+  const { scrapeUrl, scrapePreset, scrapeMaxVideos, scrapeMaxComments, scrapeSwitchDelay, scrapeCommentDelay, outputFile } = options;
+
+  if (!scrapeUrl) {
+    console.error('用法: tt-help scrape <视频URL> [preset] [最大视频数] [最大评论数] [-o 输出路径]');
+    console.error('预设: fast, normal, slow, stealth');
+    console.error('选项: -o, --output <路径>     输出到文件（默认输出到 stdout）');
+    console.error('      --switch-delay <ms>     视频切换延迟（毫秒）');
+    console.error('      --comment-delay <ms>    评论滚动延迟（毫秒）');
+    process.exit(1);
+  }
+
+  const { runScrape } = await import('./lib/scrape-browser.mjs');
+
+  let browser;
+  try {
+    const { output, browser: b } = await runScrape({
+      videoUrl: scrapeUrl,
+      maxVideos: scrapeMaxVideos,
+      maxComments: scrapeMaxComments,
+      preset: scrapePreset,
+      switchMax: scrapeSwitchDelay,
+      commentMax: scrapeCommentDelay,
+      log: console.error,
+    });
+    browser = b;
+
+    const json = JSON.stringify(output, null, 2);
+    if (outputFile) {
+      writeFileSync(outputFile, json, 'utf-8');
+      console.error(`结果已写入: ${outputFile}`);
+    } else {
+      process.stdout.write(json + '\n');
+    }
+
+    const stats = output.stats;
+    console.error(`\n共 ${stats.totalVideos} 个视频, ${stats.uniqueVideoAuthors} 个视频作者, ${stats.uniqueCommentAuthors} 个评论作者`);
+  } catch (err) {
+    console.error(`浏览器抓取失败: ${err.message}`);
+    process.exit(1);
+  } finally {
+    if (browser) await browser.close().catch(() => {});
+  }
+}
+
+async function handleWatch(options) {
+  const { outputFile, watchPort } = options;
+
+  if (!outputFile) {
+    console.error('用法: tt-help watch -o <数据文件> [-p 端口]');
+    console.error('示例: tt-help watch -o data.json');
+    console.error('      tt-help watch -o data.json -p 8080');
+    process.exit(1);
+  }
+
+  if (!existsSync(outputFile)) {
+    console.error(`文件不存在: ${outputFile}`);
+    process.exit(1);
+  }
+
+  const { server, port } = await startWatchServer(outputFile, watchPort);
+  openBrowser(port);
+
+  process.once('SIGINT', () => {
+    server.close();
+    process.exit(0);
+  });
+
+  console.error(`按 Ctrl+C 停止监控服务`);
+}
+
+async function handleAuto(options) {
+  const { autoUsernames, autoCollectMax, autoScrapeDepth, autoMaxComments, autoPreset, autoSwitchDelay, autoCommentDelay, outputFile, autoWatch, autoWatchPort } = options;
+
+  const runOptions = {
+    collectMax: autoCollectMax,
+    scrapeDepth: autoScrapeDepth,
+    maxComments: autoMaxComments,
+    preset: autoPreset,
+    switchMax: autoSwitchDelay,
+    commentMax: autoCommentDelay,
+  };
+
+  // 数据源
+  const { createRequire } = await import('module');
+  const require = createRequire(import.meta.url);
+  const { createStore } = require('./data-store.cjs');
+  const store = createStore(outputFile);
+
+  // 构建队列：命令行用户名插队到前面，文件中的未处理用户追加到后面
+  const queue = [...new Set(autoUsernames)];
+  const pendingFromStore = store.getPendingUsers().filter(u => !u.restricted);
+  pendingFromStore.forEach(u => {
+    if (!queue.includes(u.uniqueId)) {
+      queue.push(u.uniqueId);
+    }
+  });
+
+  if (queue.length === 0) {
+    console.error('没有待处理的用户');
+    return;
+  }
+
+  console.error(`队列: ${queue.length} 个用户待处理`);
+  if (autoUsernames.length > 0) {
+    console.error(`  命令行: @${autoUsernames.join(', @')}`);
+  }
+  if (pendingFromStore.length > 0) {
+    console.error(`  数据源: ${pendingFromStore.length} 个未处理用户`);
+  }
+
+  // Watch server
+  let watchServer = null;
+  let watchPort = null;
+  if (autoWatch) {
+    if (!outputFile) {
+      console.error('--watch 需要指定 -o 输出文件');
+      process.exit(1);
+    }
+    ({ server: watchServer, port: watchPort } = await startWatchServer(outputFile, autoWatchPort || 3000));
+    openBrowser(watchPort);
+  }
+
+  // 启动浏览器
+  const { ensureBrowserReady, processUser } = await import('./lib/auto-browser.mjs');
+
+  const browser = await ensureBrowserReady();
+
+  try {
+    const contexts = browser.contexts();
+    let page = null;
+    for (const ctx of contexts) {
+      for (const p of ctx.pages()) {
+        if (p.url().includes('tiktok.com')) {
+          page = p;
+          break;
+        }
+      }
+      if (page) break;
+    }
+    if (!page) {
+      const defaultCtx = contexts[0] || await browser.newContext();
+      page = await defaultCtx.newPage();
+    }
+
+    let processedCount = 0;
+    let errorCount = 0;
+
+    for (let i = 0; i < queue.length; i++) {
+      const username = queue[i];
+      console.error(`\n[${i + 1}/${queue.length}] 处理 @${username}...`);
+
+      const result = await processUser(page, username, { ...runOptions, browser }, console.error);
+
+      if (result.restricted) {
+        store.addUser({
+          uniqueId: username,
+          restricted: true,
+          sources: ['restricted'],
+        });
+        store.save();
+        continue;
+      }
+
+      if (result.error) {
+        errorCount++;
+        store.addUser({
+          uniqueId: username,
+          error: result.error,
+          sources: ['error'],
+        });
+        store.save();
+        continue;
+      }
+
+      // 写入用户信息（持续合并更新，不管是否已存在）
+      const userEntry = {
+        uniqueId: username,
+        ...result.userInfo,
+        sources: ['processed'],
+      };
+      store.addUser(userEntry);
+
+      // 发现的视频作者（持续合并更新，不管是否已存在）
+      for (const va of result.discoveredVideoAuthors) {
+        store.addUser({
+          uniqueId: va.uniqueId,
+          nickname: va.nickname,
+          locationCreated: va.locationCreated,
+          sources: ['video'],
+        });
+        if (!store.getUser(va.uniqueId) || !store.getUser(va.uniqueId).followerCount) {
+          if (!queue.includes(va.uniqueId)) {
+            queue.push(va.uniqueId);
+          }
+        }
+      }
+
+      // 发现的评论作者
+      for (const ca of result.discoveredCommentAuthors) {
+        const caId = ca.replace(/^@/, '');
+        store.addUser({
+          uniqueId: caId,
+          sources: ['comment'],
+        });
+        if (!store.getUser(caId) || !store.getUser(caId).followerCount) {
+          if (!queue.includes(caId)) {
+            queue.push(caId);
+          }
+        }
+      }
+
+      processedCount++;
+      store.save();
+      console.error(`  已保存，当前共 ${store.getAllUsers().length} 个用户`);
+    }
+
+    const output = store.getAllUsers();
+    if (outputFile) {
+      console.error(`\n完成: ${processedCount} 个用户已处理, ${errorCount} 个出错, 共 ${output.length} 个用户`);
+      console.error(`数据已保存到: ${outputFile}`);
+    } else {
+      const json = JSON.stringify(output, null, 2);
+      process.stdout.write(json + '\n');
+    }
+  } catch (err) {
+    console.error(`自动抓取失败: ${err.message}`);
+    if (watchServer) watchServer.close();
+    process.exit(1);
+  } finally {
+    await browser.close().catch(() => {});
+    if (watchServer) {
+      watchServer.close();
+      console.error(`Watch 监控服务已停止: http://127.0.0.1:${watchPort}`);
+    }
+  }
+}
+
+async function handleVideos(options) {
+  const { videosUsername, videosMax, outputFile } = options;
+
+  if (!videosUsername) {
+    console.error('用法: tt-help videos <用户名> [最大视频数] [-o 输出路径]');
+    console.error('示例: tt-help videos bar.lar.lar.moeta 1000');
+    console.error('      tt-help videos username 50 -o videos.json');
+    console.error('');
+    console.error('选项: -o, --output <路径>  输出到文件（默认输出到 stdout）');
+    process.exit(1);
+  }
+
+  const { runGetUserVideos } = await import('./lib/get-user-videos-browser.mjs');
+
+  let browser;
+  try {
+    const { output, browser: b } = await runGetUserVideos({
+      username: videosUsername,
+      maxVideos: videosMax,
+      log: console.error,
+    });
+    browser = b;
+
+    const json = JSON.stringify(output, null, 2);
+    if (outputFile) {
+      writeFileSync(outputFile, json, 'utf-8');
+      console.error(`结果已写入: ${outputFile}`);
+    } else {
+      process.stdout.write(json + '\n');
+    }
+
+    const stats = output.videos.length;
+    console.error(`\n共 ${stats} 个视频, 用户: @${videosUsername}`);
+  } catch (err) {
+    console.error(`获取用户视频失败: ${err.message}`);
+    process.exit(1);
+  } finally {
+    if (browser) await browser.close().catch(() => {});
+  }
+}
+
 async function main() {
-  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, customProxy, configAction, configValue, pipeMode, filterStr } = parseArgs();
+  const parsed = parseArgs();
+
+  if (parsed.subcommand === 'scrape') {
+    await handleScrape(parsed);
+    return;
+  }
+
+  if (parsed.subcommand === 'videos') {
+    await handleVideos(parsed);
+    return;
+  }
+
+  if (parsed.subcommand === 'auto') {
+    await handleAuto(parsed);
+    return;
+  }
+
+  if (parsed.subcommand === 'watch') {
+    await handleWatch(parsed);
+    return;
+  }
+
+  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, customProxy, configAction, configValue, pipeMode, filterStr } = parsed;
   const proxyUrl = customProxy || proxy;
   const filter = parseFilter(filterStr);
 

package/src/results/user-videos-bar.lar.lar.moeta.json

@@ -0,0 +1,37 @@
+{
+  "user": {
+    "uniqueId": "bar.lar.lar.moeta",
+    "secUid": "MS4wLjABAAAA3cgKTWvKfga0JAWeakAzx3zQ-aFAC8RuQvxD4HQFraKKsc_TbOIyMo3_ofVlXofV",
+    "nickname": "Bar Lar Lar Moetain",
+    "ttSeller": false,
+    "verified": false,
+    "followerCount": 24000,
+    "videoCount": 749,
+    "followingCount": 4293,
+    "heartCount": 254300,
+    "signature": ""
+  },
+  "totalVideos": 5,
+  "videos": [
+    {
+      "id": "7638231799084158228",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638231799084158228"
+    },
+    {
+      "id": "7638162444698914068",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638162444698914068"
+    },
+    {
+      "id": "7638116251767819541",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638116251767819541"
+    },
+    {
+      "id": "7638069637321690388",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638069637321690388"
+    },
+    {
+      "id": "7637927171025112341",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7637927171025112341"
+    }
+  ]
+}

package/src/scraper/core.cjs

@@ -0,0 +1,192 @@
+const {
+  closeCommentPanel,
+  delay,
+  ensureBrowserReady,
+  ensureTikTokPage,
+  setDelayConfig,
+  getDelayConfig,
+  retryWithBackoff,
+} = require("./modules/page-helpers.cjs");
+const { extractCommentAuthors } = require("./modules/comment-extractor.cjs");
+
+async function scrapeSingleVideo(page, maxComments, log) {
+  const config = getDelayConfig();
+
+  await page
+    .waitForSelector('[class*="VideoMeta"]', { timeout: 10000 })
+    .catch(() => {});
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  const userData = await page.evaluate(() => {
+    const result = {};
+
+    const m = window.location.href.match(/\/@([^\/]+)\/video/);
+    if (m) result.uniqueId = m[1];
+
+    const authorEls = document.querySelectorAll('[class*="Author"]');
+    for (const el of authorEls) {
+      const text = (el.textContent || "").trim();
+      if (text && !text.includes("TikTok") && !text.includes("Share")) {
+        result.nickname = text;
+        break;
+      }
+    }
+
+    const html = document.documentElement.outerHTML;
+    const locMatch = html.match(/"locationCreated":"([^"]*)/);
+    if (locMatch) result.locationCreated = locMatch[1];
+
+    return result;
+  });
+
+  const videoAuthor = userData.uniqueId ? "@" + userData.uniqueId : null;
+  if (!videoAuthor) {
+    throw new Error("无法获取视频作者");
+  }
+
+  const commentUsers = await extractCommentAuthors(page, maxComments);
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  const uniqueUsers = [...new Set(commentUsers)];
+
+  return {
+    videoAuthor,
+    uniqueId: userData.uniqueId,
+    nickname: userData.nickname,
+    locationCreated: userData.locationCreated,
+    commentUsers: uniqueUsers,
+  };
+}
+
+async function runScrape(options) {
+  const {
+    videoUrl,
+    maxVideos = 20,
+    maxComments = 999,
+    preset = null,
+    switchMax = null,
+    commentMax = null,
+    log = console.error,
+    browser: externalBrowser = null,
+    page: externalPage = null,
+  } = options;
+
+  if (preset) {
+    setDelayConfig(preset);
+  } else if (switchMax || commentMax) {
+    setDelayConfig({
+      switchMax: switchMax || 5000,
+      commentMax: commentMax || 3000,
+    });
+  }
+
+  const config = getDelayConfig();
+
+  let browser, page;
+  const isExternal = !!(externalBrowser && externalPage);
+
+  if (!isExternal) {
+    log(`视频地址: ${videoUrl}`);
+    log(
+      `视频数: ${maxVideos}, 评论数: ${maxComments}, 切换延迟: ${config.switchMax}ms, 评论延迟: ${config.commentMax}ms`,
+    );
+  }
+
+  if (isExternal) {
+    browser = externalBrowser;
+    page = externalPage;
+  } else {
+    browser = await ensureBrowserReady();
+    try {
+      page = await ensureTikTokPage(browser, videoUrl);
+    } catch (e) {
+      await browser.close().catch(() => {});
+      throw e;
+    }
+  }
+
+  await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: "load", timeout: 30000 }), { log });
+  await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+
+  const allResults = [];
+  const videoAuthors = new Set();
+  const commentUsers = new Set();
+  const allCommentAuthorsList = [];
+
+  for (let i = 0; i < maxVideos; i++) {
+    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+    let result;
+    try {
+      result = await scrapeSingleVideo(page, maxComments, log);
+    } catch (e) {
+      log(`[${i + 1}/${maxVideos}] 跳过: ${e.message}`);
+      if (i < maxVideos - 1) {
+        await page.evaluate(() => {
+          const container = document.querySelector(
+            '[class*="ColumnListContainer"]',
+          );
+          if (container) container.scrollTop += 700;
+          else window.scrollBy(0, 700);
+        });
+        await delay(Math.round(config.switchMax * 0.5), config.switchMax);
+      }
+      continue;
+    }
+    allResults.push(result);
+    videoAuthors.add(result.videoAuthor);
+    result.commentUsers.forEach((u) => commentUsers.add(u));
+    allCommentAuthorsList.push(...result.commentUsers);
+
+    if ((i + 1) % 5 === 0 || i === 0) {
+      log(
+        `[${i + 1}/${maxVideos}] ${result.videoAuthor} | 昵称: ${result.nickname || "-"} | 评论用户: ${result.commentUsers.length}`,
+      );
+    }
+
+    if (i < maxVideos - 1) {
+      await page.evaluate(() => {
+        const container = document.querySelector(
+          '[class*="ColumnListContainer"]',
+        );
+        if (container) container.scrollTop += 700;
+      });
+      await delay(2000, config.switchMax);
+    }
+  }
+
+  log(
+    `\n结果: 视频作者 ${videoAuthors.size} | 评论用户 ${commentUsers.size} | 总评论 ${allCommentAuthorsList.length}`,
+  );
+
+  const videoDetails = {};
+  for (const r of allResults) {
+    const key = r.videoAuthor;
+    if (!videoDetails[key]) {
+      videoDetails[key] = {
+        videoAuthor: r.videoAuthor,
+        uniqueId: r.uniqueId,
+        nickname: r.nickname,
+        locationCreated: r.locationCreated,
+      };
+    }
+  }
+
+  const output = {
+    videoDetails: Object.values(videoDetails),
+    commentUsers: [...commentUsers].sort(),
+    allCommentAuthorsList,
+    stats: {
+      totalVideos: allResults.length,
+      uniqueVideoAuthors: videoAuthors.size,
+      uniqueCommentAuthors: commentUsers.size,
+    },
+  };
+
+  return { output, browser, isExternal };
+}
+
+module.exports = { scrapeSingleVideo, runScrape };