tt-help-cli-ycl 1.0.8 → 1.1.0

package/src/lib/explore.js

@@ -1,244 +1,225 @@
-import { chromium } from 'playwright';
-import { existsSync, accessSync } from 'fs';
-import { browser, saveBrowser, configPath } from './constants.js';
-
-const EXPLORE_URL = 'https://www.tiktok.com/explore';
-
-function sleep(ms) {
-  return new Promise(r => setTimeout(r, ms));
-}
-
-function isRetryableError(error) {
-  if (!error) return false;
-  const msg = (error.message || error.toString() || '').toLowerCase();
-  const patterns = ['interrupted', 'net::', 'econn', 'etimedout', 'enotfound', 'eai_again', 'esocketreset', 'connection.*refused', 'connection.*reset', 'failed.*navigate', 'target.*closed', 'crash'];
-  return patterns.some(p => new RegExp(p, 'i').test(msg));
-}
-
-async function retryGoto(page, url, options, { maxRetries = 3, baseDelay = 3000 } = {}) {
-  let lastError;
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    try {
-      return await page.goto(url, options);
-    } catch (error) {
-      lastError = error;
-      if (attempt >= maxRetries || !isRetryableError(error)) {
-        throw error;
-      }
-      const jitter = Math.random() * 500;
-      const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
-      console.log(`    [重试] ${attempt + 1}/${maxRetries}，${Math.round(waitTime)}ms 后重试...`);
-      await sleep(waitTime);
-    }
-  }
-  throw lastError;
-}
-
-function detectBrowser() {
-  const isMac = process.platform === 'darwin';
-  const isWin = process.platform === 'win32';
-  const isLinux = process.platform === 'linux';
-
-  const paths = [];
-
-  if (isMac) {
-    paths.push(
-      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
-      '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
-      '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
-      '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
-    );
-  } else if (isWin) {
-    const localAppData = process.env.LOCALAPPDATA || '';
-    const programFiles = process.env.PROGRAMFILES || '';
-    const programFilesX86 = process.env['PROGRAMFILES(X86)'] || '';
-    paths.push(
-      `${programFiles}\\Google\\Chrome\\Application\\chrome.exe`,
-      `${programFilesX86}\\Google\\Chrome\\Application\\chrome.exe`,
-      `${localAppData}\\Google\\Chrome\\Application\\chrome.exe`,
-      `${programFiles}\\Microsoft\\Edge\\Application\\msedge.exe`,
-      `${programFilesX86}\\Microsoft\\Edge\\Application\\msedge.exe`,
-    );
-  } else if (isLinux) {
-    paths.push(
-      '/usr/bin/google-chrome',
-      '/usr/bin/google-chrome-stable',
-      '/usr/bin/chromium-browser',
-      '/usr/bin/chromium',
-      '/snap/bin/chromium',
-      '/usr/bin/microsoft-edge',
-    );
-  }
-
-  for (const p of paths) {
-    try {
-      accessSync(p);
-      return p;
-    } catch {
-      // 文件不存在或无权限
-    }
-  }
-  return null;
-}
-
-export async function fetchExplore(count = 100) {
-  let browserPath = browser;
-  let browserSource = '配置';
-
-  if (!browserPath) {
-    console.log(`  [0/6] 未配置浏览器，正在自动探测...`);
-    const detected = detectBrowser();
-    if (detected) {
-      browserPath = detected;
-      browserSource = '自动探测';
-      try {
-        saveBrowser(browserPath);
-        console.log(`  [0/6] 已保存浏览器路径到配置: ${configPath}`);
-      } catch (err) {
-        console.log(`  [0/6] 保存配置失败: ${err.message}`);
-      }
-    }
-  }
-
-  let browserLaunched = false;
-  let instance;
-
-  const launchOptions = {
-    headless: true,
-    args: [
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-      '--disable-blink-features=AutomationControlled',
-      '--disable-dev-shm-usage',
-    ],
-  };
-
-  if (browserPath) {
-    console.log(`  [0/6] 使用${browserSource}浏览器: ${browserPath}`);
-    launchOptions.executablePath = browserPath;
-  }
-
-  try {
-    instance = await chromium.launch(launchOptions);
-    browserLaunched = true;
-  } catch (err) {
-    if (browserPath) {
-      console.log(`  [0/6] 浏览器启动失败 (${err.message})，回退到 Playwright Chromium...`);
-    }
-    instance = await chromium.launch({
-      headless: true,
-      args: launchOptions.args,
-    });
-    browserLaunched = true;
-  }
-
-  if (!browserLaunched) {
-    throw new Error('无法启动浏览器，请确保已安装 Chrome/Edge 或运行 "npx playwright install chromium"');
-  }
-
-  try {
-    const context = await instance.newContext({
-      viewport: { width: 1280, height: 900 },
-      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-      locale: 'en-US',
-    });
-
-    // 注入反检测脚本
-    await context.addInitScript(() => {
-      // 重写 navigator.webdriver
-      Object.defineProperty(navigator, 'webdriver', { get: () => false });
-
-      // 伪造 window.chrome
-      if (!window.chrome) {
-        window.chrome = { runtime: {} };
-      }
-
-      // 覆写 permissions query
-      const originalQuery = window.navigator.permissions.query;
-      window.navigator.permissions.query = (params) =>
-        params.name === 'notifications'
-          ? Promise.resolve({ state: Notification.permission })
-          : originalQuery(params);
-
-      // 覆写 languages
-      Object.defineProperty(navigator, 'languages', {
-        get: () => ['en-US', 'en'],
-      });
-
-      // 覆写 plugins
-      Object.defineProperty(navigator, 'plugins', {
-        get: () => [1, 2, 3, 4, 5],
-      });
-    });
-
-    const page = await context.newPage();
-    await retryGoto(page, EXPLORE_URL, { waitUntil: 'load', timeout: 30000 });
-    console.log(`  [1/6] 页面已加载`);
-
-    await sleep(5000);
-
-    let lastCount = 0;
-    let noNewCount = 0;
-
-    for (let i = 0; i < 50; i++) {
-      await page.evaluate(() => {
-        window.scrollBy({ top: window.innerHeight * 0.8, behavior: 'smooth' });
-      });
-
-      await sleep(1500 + Math.random() * 1000);
-
-      const urls = await page.$$eval('a', els =>
-        els.map(a => a.href).filter(u => /\/video\/\d{16,20}/.test(u))
-      );
-      const uniqueCount = [...new Set(urls)].length;
-
-      if (uniqueCount > lastCount) {
-        noNewCount = 0;
-      } else {
-        noNewCount++;
-      }
-      lastCount = uniqueCount;
-
-      if ((i + 1) % 10 === 0) {
-        console.log(`  [2/6] 滚动 ${i + 1}/50，当前 ${uniqueCount} 个视频`);
-      }
-
-      if (noNewCount >= 5) {
-        console.log(`  [3/6] 内容加载完成（${uniqueCount} 个视频）`);
-        break;
-      }
-
-      if (uniqueCount >= count * 2) {
-        console.log(`  [3/6] 视频数量已充足（${uniqueCount} 个）`);
-        break;
-      }
-    }
-
-    await sleep(3000);
-
-    const urls = await page.$$eval('a', els =>
-      els.map(a => a.href).filter(u => /\/video\/\d{16,20}/.test(u))
-    );
-    const unique = [...new Set(urls)];
-    console.log(`  [4/6] 共检测到 ${unique.length} 个不重复视频`);
-
-    const results = [];
-    const seen = new Set();
-    for (const url of unique) {
-      if (results.length >= count) break;
-      const videoId = url.match(/video\/(\d{16,20})$/)?.[1];
-      if (videoId && !seen.has(videoId)) {
-        seen.add(videoId);
-        const user = url.match(/\/@([^\/]+)/)?.[1];
-        if (user) {
-          results.push({ user, id: videoId, url });
-        }
-      }
-    }
-
-    console.log(`  [5/6] 去重后 ${results.length} 个`);
-    return results;
-  } finally {
-    await instance.close();
-  }
-}
+import { chromium } from 'playwright';
+import { existsSync, accessSync } from 'fs';
+import { browser, saveBrowser, configPath } from './constants.js';
+import scrollCollector from '../scraper/modules/scroll-collector.cjs';
+
+const EXPLORE_URL = 'https://www.tiktok.com/explore';
+
+function sleep(ms) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+function isRetryableError(error) {
+  if (!error) return false;
+  const msg = (error.message || error.toString() || '').toLowerCase();
+  const patterns = ['interrupted', 'net::', 'econn', 'etimedout', 'enotfound', 'eai_again', 'esocketreset', 'connection.*refused', 'connection.*reset', 'failed.*navigate', 'target.*closed', 'crash'];
+  return patterns.some(p => new RegExp(p, 'i').test(msg));
+}
+
+async function retryGoto(page, url, options, { maxRetries = 3, baseDelay = 3000 } = {}) {
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await page.goto(url, options);
+    } catch (error) {
+      lastError = error;
+      if (attempt >= maxRetries || !isRetryableError(error)) {
+        throw error;
+      }
+      const jitter = Math.random() * 500;
+      const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
+      console.log(`    [重试] ${attempt + 1}/${maxRetries}，${Math.round(waitTime)}ms 后重试...`);
+      await sleep(waitTime);
+    }
+  }
+  throw lastError;
+}
+
+function detectBrowser() {
+  const isMac = process.platform === 'darwin';
+  const isWin = process.platform === 'win32';
+  const isLinux = process.platform === 'linux';
+
+  const paths = [];
+
+  if (isMac) {
+    paths.push(
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+      '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
+      '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
+    );
+  } else if (isWin) {
+    const localAppData = process.env.LOCALAPPDATA || '';
+    const programFiles = process.env.PROGRAMFILES || '';
+    const programFilesX86 = process.env['PROGRAMFILES(X86)'] || '';
+    paths.push(
+      `${programFiles}\\Google\\Chrome\\Application\\chrome.exe`,
+      `${programFilesX86}\\Google\\Chrome\\Application\\chrome.exe`,
+      `${localAppData}\\Google\\Chrome\\Application\\chrome.exe`,
+      `${programFiles}\\Microsoft\\Edge\\Application\\msedge.exe`,
+      `${programFilesX86}\\Microsoft\\Edge\\Application\\msedge.exe`,
+    );
+  } else if (isLinux) {
+    paths.push(
+      '/usr/bin/google-chrome',
+      '/usr/bin/google-chrome-stable',
+      '/usr/bin/chromium-browser',
+      '/usr/bin/chromium',
+      '/snap/bin/chromium',
+      '/usr/bin/microsoft-edge',
+    );
+  }
+
+  for (const p of paths) {
+    try {
+      accessSync(p);
+      return p;
+    } catch {
+      // 文件不存在或无权限
+    }
+  }
+  return null;
+}
+
+export async function fetchExplore(count = 100) {
+  let browserPath = browser;
+  let browserSource = '配置';
+
+  if (!browserPath) {
+    console.log(`  [0/6] 未配置浏览器，正在自动探测...`);
+    const detected = detectBrowser();
+    if (detected) {
+      browserPath = detected;
+      browserSource = '自动探测';
+      try {
+        saveBrowser(browserPath);
+        console.log(`  [0/6] 已保存浏览器路径到配置: ${configPath}`);
+      } catch (err) {
+        console.log(`  [0/6] 保存配置失败: ${err.message}`);
+      }
+    }
+  }
+
+  let browserLaunched = false;
+  let instance;
+
+  const launchOptions = {
+    headless: true,
+    args: [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-blink-features=AutomationControlled',
+      '--disable-dev-shm-usage',
+    ],
+  };
+
+  if (browserPath) {
+    console.log(`  [0/6] 使用${browserSource}浏览器: ${browserPath}`);
+    launchOptions.executablePath = browserPath;
+  }
+
+  try {
+    instance = await chromium.launch(launchOptions);
+    browserLaunched = true;
+  } catch (err) {
+    if (browserPath) {
+      console.log(`  [0/6] 浏览器启动失败 (${err.message})，回退到 Playwright Chromium...`);
+    }
+    instance = await chromium.launch({
+      headless: true,
+      args: launchOptions.args,
+    });
+    browserLaunched = true;
+  }
+
+  if (!browserLaunched) {
+    throw new Error('无法启动浏览器，请确保已安装 Chrome/Edge 或运行 "npx playwright install chromium"');
+  }
+
+  try {
+    const context = await instance.newContext({
+      viewport: { width: 1280, height: 900 },
+      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+      locale: 'en-US',
+    });
+
+    // 注入反检测脚本
+    await context.addInitScript(() => {
+      // 重写 navigator.webdriver
+      Object.defineProperty(navigator, 'webdriver', { get: () => false });
+
+      // 伪造 window.chrome
+      if (!window.chrome) {
+        window.chrome = { runtime: {} };
+      }
+
+      // 覆写 permissions query
+      const originalQuery = window.navigator.permissions.query;
+      window.navigator.permissions.query = (params) =>
+        params.name === 'notifications'
+          ? Promise.resolve({ state: Notification.permission })
+          : originalQuery(params);
+
+      // 覆写 languages
+      Object.defineProperty(navigator, 'languages', {
+        get: () => ['en-US', 'en'],
+      });
+
+      // 覆写 plugins
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [1, 2, 3, 4, 5],
+      });
+    });
+
+    const page = await context.newPage();
+    await retryGoto(page, EXPLORE_URL, { waitUntil: 'load', timeout: 30000 });
+    console.log(`  [1/6] 页面已加载`);
+
+    await sleep(5000);
+
+    const allUrls = await scrollCollector.scrollAndCollect(page, {
+      container: null,
+      collectFn: () => {
+        return {
+          items: Array.from(document.querySelectorAll('a'))
+            .filter(a => /\/video\/\d{16,20}/.test(a.href))
+            .map(a => a.href),
+        };
+      },
+      maxItems: count * 2,
+      delayRange: [1500, 2500],
+      staleThreshold: 5,
+      onRound: (round, items, allItems) => {
+        if ((round + 1) % 10 === 0) {
+          const uniqueCount = [...new Set(allItems)].length;
+          console.log(`  [2/6] 滚动 ${round + 1}，当前 ${uniqueCount} 个视频`);
+        }
+      },
+    });
+
+    await sleep(3000);
+
+    const unique = [...new Set(allUrls)];
+    console.log(`  [4/6] 共检测到 ${unique.length} 个不重复视频`);
+
+    const results = [];
+    const seen = new Set();
+    for (const url of unique) {
+      if (results.length >= count) break;
+      const videoId = url.match(/video\/(\d{16,20})$/)?.[1];
+      if (videoId && !seen.has(videoId)) {
+        seen.add(videoId);
+        const user = url.match(/\/@([^\/]+)/)?.[1];
+        if (user) {
+          results.push({ user, id: videoId, url });
+        }
+      }
+    }
+
+    console.log(`  [5/6] 去重后 ${results.length} 个`);
+    return results;
+  } finally {
+    await instance.close();
+  }
+}

package/src/lib/fetcher.js

@@ -1,60 +1,60 @@
-import { fetch, ProxyAgent } from 'undici';
-import { DEFAULT_PROXY } from './constants.js';
-
-const HEADERS = {
-  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-  'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
-  'Accept-Encoding': 'gzip, deflate, br',
-  'Connection': 'keep-alive',
-  'Upgrade-Insecure-Requests': '1',
-  'Sec-Fetch-Dest': 'document',
-  'Sec-Fetch-Mode': 'navigate',
-  'Sec-Fetch-Site': 'none',
-  'Sec-Fetch-User': '?1',
-  'Cache-Control': 'max-age=0',
-};
-
-export async function fetchHtml(url, proxyUrl) {
-  const p = proxyUrl || DEFAULT_PROXY;
-  const agent = new ProxyAgent(p);
-  let lastError;
-
-  for (let attempt = 1; attempt <= 3; attempt++) {
-    try {
-      const res = await fetch(url, {
-        headers: HEADERS,
-        dispatcher: agent,
-        redirect: 'follow',
-      });
-      const html = await res.text();
-      return html;
-    } catch (err) {
-      lastError = err;
-      if (attempt < 3) {
-        const waitMs = Math.pow(2, attempt - 1) * 3000 + Math.random() * 2000;
-        await new Promise(r => setTimeout(r, waitMs));
-      }
-    }
-  }
-
-  throw new Error(`请求 ${url} 失败（已重试 3 次），代理 ${p} 不可用`);
-}
-
-export function makeProfileUrl(handle) {
-  if (handle.startsWith('http')) return handle;
-  return `https://www.tiktok.com/${handle}`;
-}
-
-export function isProfileUrl(url) {
-  return /\/@[\w-]+(?:$|[?#])/.test(url);
-}
-
-export function isVideoUrl(url) {
-  return /\/video\/\d+/.test(url);
-}
-
-export function extractProfileHandle(url) {
-  const m = url.match(/https:\/\/www\.tiktok\.com\/(@[\w-]+)/);
-  return m ? m[1] : null;
-}
+import { fetch, ProxyAgent } from 'undici';
+import { DEFAULT_PROXY } from './constants.js';
+
+const HEADERS = {
+  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+  'Accept-Encoding': 'gzip, deflate, br',
+  'Connection': 'keep-alive',
+  'Upgrade-Insecure-Requests': '1',
+  'Sec-Fetch-Dest': 'document',
+  'Sec-Fetch-Mode': 'navigate',
+  'Sec-Fetch-Site': 'none',
+  'Sec-Fetch-User': '?1',
+  'Cache-Control': 'max-age=0',
+};
+
+export async function fetchHtml(url, proxyUrl) {
+  const p = proxyUrl || DEFAULT_PROXY;
+  const agent = new ProxyAgent(p);
+  let lastError;
+
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    try {
+      const res = await fetch(url, {
+        headers: HEADERS,
+        dispatcher: agent,
+        redirect: 'follow',
+      });
+      const html = await res.text();
+      return html;
+    } catch (err) {
+      lastError = err;
+      if (attempt < 3) {
+        const waitMs = Math.pow(2, attempt - 1) * 3000 + Math.random() * 2000;
+        await new Promise(r => setTimeout(r, waitMs));
+      }
+    }
+  }
+
+  throw new Error(`请求 ${url} 失败（已重试 3 次），代理 ${p} 不可用`);
+}
+
+export function makeProfileUrl(handle) {
+  if (handle.startsWith('http')) return handle;
+  return `https://www.tiktok.com/${handle}`;
+}
+
+export function isProfileUrl(url) {
+  return /\/@[\w-]+(?:$|[?#])/.test(url);
+}
+
+export function isVideoUrl(url) {
+  return /\/video\/\d+/.test(url);
+}
+
+export function extractProfileHandle(url) {
+  const m = url.match(/https:\/\/www\.tiktok\.com\/(@[\w-]+)/);
+  return m ? m[1] : null;
+}