tt-help-cli-ycl 1.3.0 → 1.3.1

package/src/lib/explore.js

@@ -1,7 +1,9 @@
 import { chromium } from 'playwright';
-import { existsSync, accessSync } from 'fs';
 import { browser, saveBrowser, configPath } from './constants.js';
-import scrollCollector from '../scraper/modules/scroll-collector.cjs';
+import { detectBrowser } from './browser/launch.js';
+import { getAntiDetectScript } from './browser/anti-detect.js';
+import { retryWithBackoff } from './retry.js';
+import { scrollAndCollect } from '../scraper/modules/scroll-collector.mjs';
 
 const EXPLORE_URL = 'https://www.tiktok.com/explore';
 
@@ -9,85 +11,12 @@ function sleep(ms) {
   return new Promise(r => setTimeout(r, ms));
 }
 
-function isRetryableError(error) {
-  if (!error) return false;
-  const msg = (error.message || error.toString() || '').toLowerCase();
-  const patterns = ['interrupted', 'net::', 'econn', 'etimedout', 'enotfound', 'eai_again', 'esocketreset', 'connection.*refused', 'connection.*reset', 'failed.*navigate', 'target.*closed', 'crash'];
-  return patterns.some(p => new RegExp(p, 'i').test(msg));
-}
-
-async function retryGoto(page, url, options, { maxRetries = 3, baseDelay = 3000 } = {}) {
-  let lastError;
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    try {
-      return await page.goto(url, options);
-    } catch (error) {
-      lastError = error;
-      if (attempt >= maxRetries || !isRetryableError(error)) {
-        throw error;
-      }
-      const jitter = Math.random() * 500;
-      const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
-      console.log(`    [重试] ${attempt + 1}/${maxRetries}，${Math.round(waitTime)}ms 后重试...`);
-      await sleep(waitTime);
-    }
-  }
-  throw lastError;
-}
-
-function detectBrowser() {
-  const isMac = process.platform === 'darwin';
-  const isWin = process.platform === 'win32';
-  const isLinux = process.platform === 'linux';
-
-  const paths = [];
-
-  if (isMac) {
-    paths.push(
-      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
-      '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
-      '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
-      '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
-    );
-  } else if (isWin) {
-    const localAppData = process.env.LOCALAPPDATA || '';
-    const programFiles = process.env.PROGRAMFILES || '';
-    const programFilesX86 = process.env['PROGRAMFILES(X86)'] || '';
-    paths.push(
-      `${programFiles}\\Google\\Chrome\\Application\\chrome.exe`,
-      `${programFilesX86}\\Google\\Chrome\\Application\\chrome.exe`,
-      `${localAppData}\\Google\\Chrome\\Application\\chrome.exe`,
-      `${programFiles}\\Microsoft\\Edge\\Application\\msedge.exe`,
-      `${programFilesX86}\\Microsoft\\Edge\\Application\\msedge.exe`,
-    );
-  } else if (isLinux) {
-    paths.push(
-      '/usr/bin/google-chrome',
-      '/usr/bin/google-chrome-stable',
-      '/usr/bin/chromium-browser',
-      '/usr/bin/chromium',
-      '/snap/bin/chromium',
-      '/usr/bin/microsoft-edge',
-    );
-  }
-
-  for (const p of paths) {
-    try {
-      accessSync(p);
-      return p;
-    } catch {
-      // 文件不存在或无权限
-    }
-  }
-  return null;
-}
-
 export async function fetchExplore(count = 100) {
   let browserPath = browser;
   let browserSource = '配置';
 
   if (!browserPath) {
-    console.log(`  [0/6] 未配置浏览器，正在自动探测...`);
+    console.log('  [0/6] 未配置浏览器，正在自动探测...');
     const detected = detectBrowser();
     if (detected) {
       browserPath = detected;
@@ -101,9 +30,6 @@ export async function fetchExplore(count = 100) {
     }
   }
 
-  let browserLaunched = false;
-  let instance;
-
   const launchOptions = {
     headless: true,
     args: [
@@ -119,9 +45,9 @@ export async function fetchExplore(count = 100) {
     launchOptions.executablePath = browserPath;
   }
 
+  let instance;
   try {
     instance = await chromium.launch(launchOptions);
-    browserLaunched = true;
   } catch (err) {
     if (browserPath) {
       console.log(`  [0/6] 浏览器启动失败 (${err.message})，回退到 Playwright Chromium...`);
@@ -130,11 +56,6 @@ export async function fetchExplore(count = 100) {
       headless: true,
       args: launchOptions.args,
     });
-    browserLaunched = true;
-  }
-
-  if (!browserLaunched) {
-    throw new Error('无法启动浏览器，请确保已安装 Chrome/Edge 或运行 "npx playwright install chromium"');
   }
 
   try {
@@ -144,49 +65,21 @@ export async function fetchExplore(count = 100) {
       locale: 'en-US',
     });
 
-    // 注入反检测脚本
-    await context.addInitScript(() => {
-      // 重写 navigator.webdriver
-      Object.defineProperty(navigator, 'webdriver', { get: () => false });
-
-      // 伪造 window.chrome
-      if (!window.chrome) {
-        window.chrome = { runtime: {} };
-      }
-
-      // 覆写 permissions query
-      const originalQuery = window.navigator.permissions.query;
-      window.navigator.permissions.query = (params) =>
-        params.name === 'notifications'
-          ? Promise.resolve({ state: Notification.permission })
-          : originalQuery(params);
-
-      // 覆写 languages
-      Object.defineProperty(navigator, 'languages', {
-        get: () => ['en-US', 'en'],
-      });
-
-      // 覆写 plugins
-      Object.defineProperty(navigator, 'plugins', {
-        get: () => [1, 2, 3, 4, 5],
-      });
-    });
+    await context.addInitScript(getAntiDetectScript());
 
     const page = await context.newPage();
-    await retryGoto(page, EXPLORE_URL, { waitUntil: 'load', timeout: 30000 });
-    console.log(`  [1/6] 页面已加载`);
+    await retryWithBackoff(() => page.goto(EXPLORE_URL, { waitUntil: 'load', timeout: 30000 }));
+    console.log('  [1/6] 页面已加载');
 
     await sleep(5000);
 
-    const allUrls = await scrollCollector.scrollAndCollect(page, {
+    const allUrls = await scrollAndCollect(page, {
       container: null,
-      collectFn: () => {
-        return {
-          items: Array.from(document.querySelectorAll('a'))
-            .filter(a => /\/video\/\d{16,20}/.test(a.href))
-            .map(a => a.href),
-        };
-      },
+      collectFn: () => ({
+        items: Array.from(document.querySelectorAll('a'))
+          .filter(a => /\/video\/\d{16,20}/.test(a.href))
+          .map(a => a.href),
+      }),
       maxItems: count * 2,
       delayRange: [1500, 2500],
       staleThreshold: 5,
@@ -210,7 +103,7 @@ export async function fetchExplore(count = 100) {
       const videoId = url.match(/video\/(\d{16,20})$/)?.[1];
       if (videoId && !seen.has(videoId)) {
         seen.add(videoId);
-        const user = url.match(/\/@([^\/]+)/)?.[1];
+        const user = url.match(/\/@([^/]+)/)?.[1];
         if (user) {
           results.push({ user, id: videoId, url });
         }

package/src/lib/fetcher.js

@@ -1,5 +1,6 @@
 import { fetch, ProxyAgent } from 'undici';
 import { DEFAULT_PROXY } from './constants.js';
+import { isProfileUrl } from './url.js';
 
 const HEADERS = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
@@ -15,6 +16,8 @@ const HEADERS = {
   'Cache-Control': 'max-age=0',
 };
 
+export { isProfileUrl } from './url.js';
+
 export async function fetchHtml(url, proxyUrl) {
   const p = proxyUrl || DEFAULT_PROXY;
   const agent = new ProxyAgent(p);
@@ -40,21 +43,3 @@ export async function fetchHtml(url, proxyUrl) {
 
   throw new Error(`请求 ${url} 失败（已重试 3 次），代理 ${p} 不可用`);
 }
-
-export function makeProfileUrl(handle) {
-  if (handle.startsWith('http')) return handle;
-  return `https://www.tiktok.com/${handle}`;
-}
-
-export function isProfileUrl(url) {
-  return /\/@[\w-]+(?:$|[?#])/.test(url);
-}
-
-export function isVideoUrl(url) {
-  return /\/video\/\d+/.test(url);
-}
-
-export function extractProfileHandle(url) {
-  const m = url.match(/https:\/\/www\.tiktok\.com\/(@[\w-]+)/);
-  return m ? m[1] : null;
-}

package/src/lib/get-user-videos-browser.mjs

@@ -1,6 +1 @@
-import { createRequire } from 'module';
-
-const require = createRequire(import.meta.url);
-const core = require('../get-user-videos-core.cjs');
-
-export const runGetUserVideos = core.runGetUserVideos;
+export { runGetUserVideos } from '../get-user-videos-core.mjs';

package/src/lib/io.js

@@ -1,19 +1,7 @@
-import { writeFileSync, readFileSync } from 'fs';
+import { extractDisplayPath } from './url.js';
 
 let lastBarCount = 0;
 
-export function writeOutput(data, outputFile) {
-  const output = JSON.stringify(data, null, 2);
-  const target = outputFile || 'tiktok_data.json';
-  writeFileSync(target, output, 'utf-8');
-  console.log(`结果已写入: ${target}`);
-}
-
-export function readUrlFile(filePath) {
-  const content = readFileSync(filePath, 'utf-8');
-  return content.split(/\r?\n/).map(l => l.trim()).filter(l => l.startsWith('http'));
-}
-
 export function createProgressBar(current, total, maxWidth = 30) {
   const filled = Math.round((current / total) * maxWidth);
   return '█'.repeat(filled).padEnd(maxWidth);
@@ -23,16 +11,6 @@ export function calculateConcurrency(total) {
   return Math.min(5, Math.max(1, Math.floor(total / 10)), total);
 }
 
-export function extractUrlDisplay(url) {
-  try {
-    const pathname = new URL(url).pathname;
-    const parts = pathname.split('/').filter(Boolean);
-    return parts.slice(-2).join('/');
-  } catch {
-    return url;
-  }
-}
-
 export function createMultiProgressBars(count) {
   return Array.from({ length: count }, () => ({
     current: 0,
@@ -44,26 +22,26 @@ export function createMultiProgressBars(count) {
 
 export function renderMultiProgressBars(bars, maxWidth = 30) {
   const activeBars = bars.filter(bar => bar.total > 0);
-  
+
   if (activeBars.length === 0) return;
-  
+
   const lines = activeBars.map((bar) => {
     const prog = createProgressBar(bar.current, bar.total, maxWidth);
     const icon = bar.status === 'done' ? '✓' :
                  bar.status === 'error' ? '' : '⟳';
-    const urlDisplay = bar.url ? extractUrlDisplay(bar.url) : '';
+    const urlDisplay = bar.url ? extractDisplayPath(bar.url) : '';
     return `  [${prog}] ${bar.current}/${bar.total} ${icon} ${urlDisplay}`;
   });
-  
+
   const output = lines.join('\n');
-  
+
   if (lastBarCount > 0) {
     process.stdout.write(`\x1b[${lastBarCount}A`);
   }
-  
+
   process.stdout.write('\x1b[0J');
   process.stdout.write(output + '\n');
-  
+
   lastBarCount = activeBars.length;
 }
 

package/src/lib/parser.js

@@ -1,4 +1,4 @@
-import { USER_SECTION_SIZE } from './constants.js';
+export const USER_SECTION_SIZE = 12000;
 
 export function extractUserSection(html) {
   const idx = html.indexOf('"uniqueId"');

package/src/lib/retry.js

@@ -0,0 +1,44 @@
+import { delay } from './delay.js';
+
+const RETRYABLE_PATTERNS = [
+  'interrupted',
+  'Navigation.*interrupted',
+  'net::',
+  'ECONN',
+  'ETIMEDOUT',
+  'ENOTFOUND',
+  'EAI_AGAIN',
+  'ESOCKETRESET',
+  'connection.*refused',
+  'connection.*reset',
+  'failed.*navigate',
+  'target.*closed',
+  'crash',
+];
+
+export function isRetryableError(error) {
+  if (!error) return false;
+  const msg = (error.message || error.toString() || '').toLowerCase();
+  return RETRYABLE_PATTERNS.some(p => new RegExp(p, 'i').test(msg));
+}
+
+export async function retryWithBackoff(fn, { maxRetries = 3, baseDelay = 3000, log } = {}) {
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt >= maxRetries || !isRetryableError(error)) {
+        throw error;
+      }
+      const jitter = Math.random() * 2000;
+      const waitTime = baseDelay * Math.pow(2, attempt) + jitter;
+      if (log) {
+        log(`  [重试] ${attempt + 1}/${maxRetries}，${Math.round(waitTime / 1000)}s 后重试...`);
+      }
+      await delay(Math.round(waitTime), Math.round(waitTime));
+    }
+  }
+  throw lastError;
+}

package/src/lib/scrape-browser.mjs

@@ -1,6 +1 @@
-import { createRequire } from 'module';
-
-const require = createRequire(import.meta.url);
-const core = require('../scraper/core.cjs');
-
-export const runScrape = core.runScrape;
+export { runScrape } from '../scraper/core.mjs';

package/src/lib/scrape.js

@@ -1,5 +1,6 @@
 import { extractUserSection, parseUserSection, extractLocationCreated } from './parser.js';
-import { fetchHtml, makeProfileUrl, isProfileUrl, isVideoUrl, extractProfileHandle } from './fetcher.js';
+import { fetchHtml, isProfileUrl } from './fetcher.js';
+import { toProfileUrl, isVideoUrl, extractUniqueId } from './url.js';
 
 export async function extractUserData(profileUrl, proxyUrl) {
   const profileHtml = await fetchHtml(profileUrl, proxyUrl);
@@ -17,16 +18,16 @@ export async function extractVideoLocation(videoUrl, proxyUrl) {
 
 export async function processUrl(url, proxyUrl) {
   if (isProfileUrl(url)) {
-    const profileUrl = makeProfileUrl(url);
+    const profileUrl = toProfileUrl(url);
     const profileData = await extractUserData(profileUrl, proxyUrl);
     return [profileData];
   }
 
   if (isVideoUrl(url)) {
-    const profileHandle = extractProfileHandle(url);
+    const profileHandle = extractUniqueId(url);
     if (!profileHandle) throw new Error(`无法从视频URL提取用户主页: ${url}`);
 
-    const profileUrl = makeProfileUrl(profileHandle);
+    const profileUrl = toProfileUrl(profileHandle);
     const [profileData, locationCreated] = await Promise.all([
       extractUserData(profileUrl, proxyUrl),
       extractVideoLocation(url, proxyUrl),

package/src/lib/url.js

@@ -0,0 +1,52 @@
+const BASE_URL = 'https://www.tiktok.com';
+
+export function extractUniqueId(url) {
+  const m = url.match(/\/@([^/]+)/);
+  return m ? m[1] : null;
+}
+
+export function extractVideoId(url) {
+  const m = url.match(/\/video\/(\d+)/);
+  return m ? m[1] : null;
+}
+
+export function normalizeUsername(input) {
+  return (input || '').replace(/^@/, '');
+}
+
+export function toProfileUrl(handle) {
+  const clean = normalizeUsername(handle);
+  return `${BASE_URL}/@${clean}`;
+}
+
+export function toVideoUrl(handle, videoId) {
+  const clean = normalizeUsername(handle);
+  return `${BASE_URL}/@${clean}/video/${videoId}`;
+}
+
+export function ensureAbsoluteUrl(href) {
+  if (href.startsWith('http')) return href;
+  return `${BASE_URL}${href}`;
+}
+
+export function isProfileUrl(url) {
+  return /\/@[\w-]+(?:$|[?#])/.test(url);
+}
+
+export function isVideoUrl(url) {
+  return /\/video\/\d+/.test(url);
+}
+
+export function extractDisplayPath(url) {
+  try {
+    const parts = new URL(url).pathname.split('/').filter(Boolean);
+    return parts.slice(-2).join('/');
+  } catch {
+    return url;
+  }
+}
+
+export function extractAuthorFromVideoUrl(url) {
+  const m = url.match(/@([^/]+)\/video/);
+  return m ? '@' + m[1] : null;
+}