tt-help-cli-ycl 1.3.0 → 1.3.2

package/src/lib/explore-fetch.js

@@ -0,0 +1,118 @@
+import { chromium } from 'playwright';
+import { browser, saveBrowser, configPath } from './constants.js';
+import { detectBrowser } from './browser/launch.js';
+import { getAntiDetectScript } from './browser/anti-detect.js';
+import { retryWithBackoff } from './retry.js';
+import { scrollAndCollect } from '../scraper/modules/scroll-collector.mjs';
+
+const EXPLORE_URL = 'https://www.tiktok.com/explore';
+
+function sleep(ms) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+export async function fetchExplore(count = 100) {
+  let browserPath = browser;
+  let browserSource = '配置';
+
+  if (!browserPath) {
+    console.log('  [0/6] 未配置浏览器，正在自动探测...');
+    const detected = detectBrowser();
+    if (detected) {
+      browserPath = detected;
+      browserSource = '自动探测';
+      try {
+        saveBrowser(browserPath);
+        console.log(`  [0/6] 已保存浏览器路径到配置: ${configPath}`);
+      } catch (err) {
+        console.log(`  [0/6] 保存配置失败: ${err.message}`);
+      }
+    }
+  }
+
+  const launchOptions = {
+    headless: true,
+    args: [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-blink-features=AutomationControlled',
+      '--disable-dev-shm-usage',
+    ],
+  };
+
+  if (browserPath) {
+    console.log(`  [0/6] 使用${browserSource}浏览器: ${browserPath}`);
+    launchOptions.executablePath = browserPath;
+  }
+
+  let instance;
+  try {
+    instance = await chromium.launch(launchOptions);
+  } catch (err) {
+    if (browserPath) {
+      console.log(`  [0/6] 浏览器启动失败 (${err.message})，回退到 Playwright Chromium...`);
+    }
+    instance = await chromium.launch({
+      headless: true,
+      args: launchOptions.args,
+    });
+  }
+
+  try {
+    const context = await instance.newContext({
+      viewport: { width: 1280, height: 900 },
+      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+      locale: 'en-US',
+    });
+
+    await context.addInitScript(getAntiDetectScript());
+
+    const page = await context.newPage();
+    await retryWithBackoff(() => page.goto(EXPLORE_URL, { waitUntil: 'load', timeout: 30000 }));
+    console.log('  [1/6] 页面已加载');
+
+    await sleep(5000);
+
+    const allUrls = await scrollAndCollect(page, {
+      container: null,
+      collectFn: () => ({
+        items: Array.from(document.querySelectorAll('a'))
+          .filter(a => /\/video\/\d{16,20}/.test(a.href))
+          .map(a => a.href),
+      }),
+      maxItems: count * 2,
+      delayRange: [1500, 2500],
+      staleThreshold: 5,
+      onRound: (round, items, allItems) => {
+        if ((round + 1) % 10 === 0) {
+          const uniqueCount = [...new Set(allItems)].length;
+          console.log(`  [2/6] 滚动 ${round + 1}，当前 ${uniqueCount} 个视频`);
+        }
+      },
+    });
+
+    await sleep(3000);
+
+    const unique = [...new Set(allUrls)];
+    console.log(`  [4/6] 共检测到 ${unique.length} 个不重复视频`);
+
+    const results = [];
+    const seen = new Set();
+    for (const url of unique) {
+      if (results.length >= count) break;
+      const videoId = url.match(/video\/(\d{16,20})$/)?.[1];
+      if (videoId && !seen.has(videoId)) {
+        seen.add(videoId);
+        const user = url.match(/\/@([^/]+)/)?.[1];
+        if (user) {
+          results.push({ user, id: videoId, url });
+        }
+      }
+    }
+
+    console.log(`  [5/6] 去重后 ${results.length} 个`);
+    return results;
+  } finally {
+    await instance.close();
+  }
+}

package/src/lib/fetcher.js

@@ -1,60 +1,45 @@
-import { fetch, ProxyAgent } from 'undici';
-import { DEFAULT_PROXY } from './constants.js';
-
-const HEADERS = {
-  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-  'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
-  'Accept-Encoding': 'gzip, deflate, br',
-  'Connection': 'keep-alive',
-  'Upgrade-Insecure-Requests': '1',
-  'Sec-Fetch-Dest': 'document',
-  'Sec-Fetch-Mode': 'navigate',
-  'Sec-Fetch-Site': 'none',
-  'Sec-Fetch-User': '?1',
-  'Cache-Control': 'max-age=0',
-};
-
-export async function fetchHtml(url, proxyUrl) {
-  const p = proxyUrl || DEFAULT_PROXY;
-  const agent = new ProxyAgent(p);
-  let lastError;
-
-  for (let attempt = 1; attempt <= 3; attempt++) {
-    try {
-      const res = await fetch(url, {
-        headers: HEADERS,
-        dispatcher: agent,
-        redirect: 'follow',
-      });
-      const html = await res.text();
-      return html;
-    } catch (err) {
-      lastError = err;
-      if (attempt < 3) {
-        const waitMs = Math.pow(2, attempt - 1) * 3000 + Math.random() * 2000;
-        await new Promise(r => setTimeout(r, waitMs));
-      }
-    }
-  }
-
-  throw new Error(`请求 ${url} 失败（已重试 3 次），代理 ${p} 不可用`);
-}
-
-export function makeProfileUrl(handle) {
-  if (handle.startsWith('http')) return handle;
-  return `https://www.tiktok.com/${handle}`;
-}
-
-export function isProfileUrl(url) {
-  return /\/@[\w-]+(?:$|[?#])/.test(url);
-}
-
-export function isVideoUrl(url) {
-  return /\/video\/\d+/.test(url);
-}
-
-export function extractProfileHandle(url) {
-  const m = url.match(/https:\/\/www\.tiktok\.com\/(@[\w-]+)/);
-  return m ? m[1] : null;
-}
+import { fetch, ProxyAgent } from 'undici';
+import { DEFAULT_PROXY } from './constants.js';
+import { isProfileUrl } from './url.js';
+
+const HEADERS = {
+  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+  'Accept-Encoding': 'gzip, deflate, br',
+  'Connection': 'keep-alive',
+  'Upgrade-Insecure-Requests': '1',
+  'Sec-Fetch-Dest': 'document',
+  'Sec-Fetch-Mode': 'navigate',
+  'Sec-Fetch-Site': 'none',
+  'Sec-Fetch-User': '?1',
+  'Cache-Control': 'max-age=0',
+};
+
+export { isProfileUrl } from './url.js';
+
+export async function fetchHtml(url, proxyUrl) {
+  const p = proxyUrl || DEFAULT_PROXY;
+  const agent = new ProxyAgent(p);
+  let lastError;
+
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    try {
+      const res = await fetch(url, {
+        headers: HEADERS,
+        dispatcher: agent,
+        redirect: 'follow',
+      });
+      const html = await res.text();
+      return html;
+    } catch (err) {
+      lastError = err;
+      if (attempt < 3) {
+        const waitMs = Math.pow(2, attempt - 1) * 3000 + Math.random() * 2000;
+        await new Promise(r => setTimeout(r, waitMs));
+      }
+    }
+  }
+
+  throw new Error(`请求 ${url} 失败（已重试 3 次），代理 ${p} 不可用`);
+}

package/src/lib/filter.js

@@ -1,66 +1,66 @@
-export function parseFilter(filterStr) {
-  if (!filterStr) return null;
-
-  const filter = {};
-  const pairs = filterStr.split('&');
-
-  for (const pair of pairs) {
-    const [key, value] = pair.split('=');
-    if (!key || value === undefined) continue;
-
-    const trimmedKey = key.trim();
-    const trimmedValue = value.trim();
-
-    // 处理布尔值
-    if (trimmedValue === 'true') {
-      filter[trimmedKey] = true;
-    } else if (trimmedValue === 'false') {
-      filter[trimmedKey] = false;
-    } else {
-      // 支持逗号分隔的多个值（如 locationCreated=DE,ES）
-      filter[trimmedKey] = trimmedValue.split(',').map(v => v.trim());
-    }
-  }
-
-  return Object.keys(filter).length > 0 ? filter : null;
-}
-
-export function applyFilter(results, filter) {
-  if (!filter || results.length === 0) return results;
-
-  return results.filter(item => {
-    for (const [key, expectedValue] of Object.entries(filter)) {
-      const actualValue = item[key];
-
-      // 如果字段不存在，过滤掉
-      if (actualValue === undefined || actualValue === null) {
-        return false;
-      }
-
-      // 数组值匹配（如 locationCreated=DE,ES）
-      if (Array.isArray(expectedValue)) {
-        if (!expectedValue.includes(String(actualValue))) {
-          return false;
-        }
-      }
-      // 布尔值或精确匹配
-      else if (actualValue !== expectedValue) {
-        return false;
-      }
-    }
-    return true;
-  });
-}
-
-export function formatFilterDescription(filter) {
-  if (!filter) return '';
-
-  const parts = Object.entries(filter).map(([key, value]) => {
-    if (Array.isArray(value)) {
-      return `${key}=${value.join(',')}`;
-    }
-    return `${key}=${value}`;
-  });
-
-  return parts.join(' & ');
-}
+export function parseFilter(filterStr) {
+  if (!filterStr) return null;
+
+  const filter = {};
+  const pairs = filterStr.split('&');
+
+  for (const pair of pairs) {
+    const [key, value] = pair.split('=');
+    if (!key || value === undefined) continue;
+
+    const trimmedKey = key.trim();
+    const trimmedValue = value.trim();
+
+    // 处理布尔值
+    if (trimmedValue === 'true') {
+      filter[trimmedKey] = true;
+    } else if (trimmedValue === 'false') {
+      filter[trimmedKey] = false;
+    } else {
+      // 支持逗号分隔的多个值（如 locationCreated=DE,ES）
+      filter[trimmedKey] = trimmedValue.split(',').map(v => v.trim());
+    }
+  }
+
+  return Object.keys(filter).length > 0 ? filter : null;
+}
+
+export function applyFilter(results, filter) {
+  if (!filter || results.length === 0) return results;
+
+  return results.filter(item => {
+    for (const [key, expectedValue] of Object.entries(filter)) {
+      const actualValue = item[key];
+
+      // 如果字段不存在，过滤掉
+      if (actualValue === undefined || actualValue === null) {
+        return false;
+      }
+
+      // 数组值匹配（如 locationCreated=DE,ES）
+      if (Array.isArray(expectedValue)) {
+        if (!expectedValue.includes(String(actualValue))) {
+          return false;
+        }
+      }
+      // 布尔值或精确匹配
+      else if (actualValue !== expectedValue) {
+        return false;
+      }
+    }
+    return true;
+  });
+}
+
+export function formatFilterDescription(filter) {
+  if (!filter) return '';
+
+  const parts = Object.entries(filter).map(([key, value]) => {
+    if (Array.isArray(value)) {
+      return `${key}=${value.join(',')}`;
+    }
+    return `${key}=${value}`;
+  });
+
+  return parts.join(' & ');
+}

package/src/lib/io.js

@@ -1,76 +1,54 @@
-import { writeFileSync, readFileSync } from 'fs';
-
-let lastBarCount = 0;
-
-export function writeOutput(data, outputFile) {
-  const output = JSON.stringify(data, null, 2);
-  const target = outputFile || 'tiktok_data.json';
-  writeFileSync(target, output, 'utf-8');
-  console.log(`结果已写入: ${target}`);
-}
-
-export function readUrlFile(filePath) {
-  const content = readFileSync(filePath, 'utf-8');
-  return content.split(/\r?\n/).map(l => l.trim()).filter(l => l.startsWith('http'));
-}
-
-export function createProgressBar(current, total, maxWidth = 30) {
-  const filled = Math.round((current / total) * maxWidth);
-  return '█'.repeat(filled).padEnd(maxWidth);
-}
-
-export function calculateConcurrency(total) {
-  return Math.min(5, Math.max(1, Math.floor(total / 10)), total);
-}
-
-export function extractUrlDisplay(url) {
-  try {
-    const pathname = new URL(url).pathname;
-    const parts = pathname.split('/').filter(Boolean);
-    return parts.slice(-2).join('/');
-  } catch {
-    return url;
-  }
-}
-
-export function createMultiProgressBars(count) {
-  return Array.from({ length: count }, () => ({
-    current: 0,
-    total: 0,
-    status: 'pending',
-    url: '',
-  }));
-}
-
-export function renderMultiProgressBars(bars, maxWidth = 30) {
-  const activeBars = bars.filter(bar => bar.total > 0);
-  
-  if (activeBars.length === 0) return;
-  
-  const lines = activeBars.map((bar) => {
-    const prog = createProgressBar(bar.current, bar.total, maxWidth);
-    const icon = bar.status === 'done' ? '✓' :
-                 bar.status === 'error' ? '' : '⟳';
-    const urlDisplay = bar.url ? extractUrlDisplay(bar.url) : '';
-    return `  [${prog}] ${bar.current}/${bar.total} ${icon} ${urlDisplay}`;
-  });
-  
-  const output = lines.join('\n');
-  
-  if (lastBarCount > 0) {
-    process.stdout.write(`\x1b[${lastBarCount}A`);
-  }
-  
-  process.stdout.write('\x1b[0J');
-  process.stdout.write(output + '\n');
-  
-  lastBarCount = activeBars.length;
-}
-
-export function clearProgressBars() {
-  if (lastBarCount > 0) {
-    process.stdout.write(`\x1b[${lastBarCount}A`);
-    process.stdout.write('\x1b[0J');
-    lastBarCount = 0;
-  }
-}
+import { extractDisplayPath } from './url.js';
+
+let lastBarCount = 0;
+
+export function createProgressBar(current, total, maxWidth = 30) {
+  const filled = Math.round((current / total) * maxWidth);
+  return '█'.repeat(filled).padEnd(maxWidth);
+}
+
+export function calculateConcurrency(total) {
+  return Math.min(5, Math.max(1, Math.floor(total / 10)), total);
+}
+
+export function createMultiProgressBars(count) {
+  return Array.from({ length: count }, () => ({
+    current: 0,
+    total: 0,
+    status: 'pending',
+    url: '',
+  }));
+}
+
+export function renderMultiProgressBars(bars, maxWidth = 30) {
+  const activeBars = bars.filter(bar => bar.total > 0);
+
+  if (activeBars.length === 0) return;
+
+  const lines = activeBars.map((bar) => {
+    const prog = createProgressBar(bar.current, bar.total, maxWidth);
+    const icon = bar.status === 'done' ? '✓' :
+                 bar.status === 'error' ? '' : '⟳';
+    const urlDisplay = bar.url ? extractDisplayPath(bar.url) : '';
+    return `  [${prog}] ${bar.current}/${bar.total} ${icon} ${urlDisplay}`;
+  });
+
+  const output = lines.join('\n');
+
+  if (lastBarCount > 0) {
+    process.stdout.write(`\x1b[${lastBarCount}A`);
+  }
+
+  process.stdout.write('\x1b[0J');
+  process.stdout.write(output + '\n');
+
+  lastBarCount = activeBars.length;
+}
+
+export function clearProgressBars() {
+  if (lastBarCount > 0) {
+    process.stdout.write(`\x1b[${lastBarCount}A`);
+    process.stdout.write('\x1b[0J');
+    lastBarCount = 0;
+  }
+}

package/src/lib/output.js

@@ -1,80 +1,80 @@
-export function deduplicate(results) {
-  const seen = new Set();
-  return results.filter(r => {
-    if (r.id) {
-      const key = r.id;
-      if (seen.has(key)) return false;
-      seen.add(key);
-      return true;
-    }
-    const key = r.secUid || r.uniqueId;
-    if (seen.has(key)) return false;
-    seen.add(key);
-    return true;
-  });
-}
-
-export function formatTable(data) {
-  if (data.length === 0) return '';
-
-  if (data.length === 1) {
-    const lines = [];
-    for (const [key, val] of Object.entries(data[0])) {
-      if (typeof val === 'string' && val.length > 80) {
-        lines.push(`  ${key}: ${val.substring(0, 80)}...`);
-      } else {
-        lines.push(`  ${key}: ${val}`);
-      }
-    }
-    return lines.join('\n');
-  }
-
-  const cols = [
-    { key: 'uniqueId', label: '用户名', width: 20 },
-    { key: 'locationCreated', label: '地区', width: 6 },
-    { key: 'nickname', label: '昵称', width: 20 },
-    { key: 'ttSeller', label: 'TT卖家', width: 8 },
-    { key: 'verified', label: '已认证', width: 8 },
-    { key: 'followerCount', label: '粉丝', width: 10 },
-    { key: 'videoCount', label: '视频', width: 8 },
-  ];
-
-  for (const row of data) {
-    for (const col of cols) {
-      const val = String(row[col.key] ?? '-');
-      col.width = Math.max(col.width, val.length, col.label.length);
-    }
-  }
-
-  const sep = (w) => '-'.repeat(w);
-  const pad = (s, w) => s.padEnd(w);
-
-  const header = cols.map(c => pad(c.label, c.width)).join(' │ ');
-  const divider = cols.map(c => sep(c.width)).join('-+-');
-  const rows = data.map(r =>
-    cols.map(c => pad(String(r[c.key] ?? '-'), c.width)).join(' │ ')
-  );
-
-  return [header, divider, ...rows].join('\n');
-}
-
-export function formatOutput(data, format) {
-  if (format === 'table') return formatTable(data);
-
-  if (format === 'raw') {
-    if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
-      return data.map(d => d.url).join('\n');
-    }
-    if (Array.isArray(data) && data.length > 0 && 'uniqueId' in data[0]) {
-      return data.map(d => `https://www.tiktok.com/@${d.uniqueId}`).join('\n');
-    }
-    return JSON.stringify(data, null, 2);
-  }
-
-  // Default JSON output, but for explore results (url-only) output pure text
-  if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
-    return data.map(d => d.url).join('\n');
-  }
-
-  return JSON.stringify(data, null, 2);
-}
+export function deduplicate(results) {
+  const seen = new Set();
+  return results.filter(r => {
+    if (r.id) {
+      const key = r.id;
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    }
+    const key = r.secUid || r.uniqueId;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+export function formatTable(data) {
+  if (data.length === 0) return '';
+
+  if (data.length === 1) {
+    const lines = [];
+    for (const [key, val] of Object.entries(data[0])) {
+      if (typeof val === 'string' && val.length > 80) {
+        lines.push(`  ${key}: ${val.substring(0, 80)}...`);
+      } else {
+        lines.push(`  ${key}: ${val}`);
+      }
+    }
+    return lines.join('\n');
+  }
+
+  const cols = [
+    { key: 'uniqueId', label: '用户名', width: 20 },
+    { key: 'locationCreated', label: '地区', width: 6 },
+    { key: 'nickname', label: '昵称', width: 20 },
+    { key: 'ttSeller', label: 'TT卖家', width: 8 },
+    { key: 'verified', label: '已认证', width: 8 },
+    { key: 'followerCount', label: '粉丝', width: 10 },
+    { key: 'videoCount', label: '视频', width: 8 },
+  ];
+
+  for (const row of data) {
+    for (const col of cols) {
+      const val = String(row[col.key] ?? '-');
+      col.width = Math.max(col.width, val.length, col.label.length);
+    }
+  }
+
+  const sep = (w) => '-'.repeat(w);
+  const pad = (s, w) => s.padEnd(w);
+
+  const header = cols.map(c => pad(c.label, c.width)).join(' │ ');
+  const divider = cols.map(c => sep(c.width)).join('-+-');
+  const rows = data.map(r =>
+    cols.map(c => pad(String(r[c.key] ?? '-'), c.width)).join(' │ ')
+  );
+
+  return [header, divider, ...rows].join('\n');
+}
+
+export function formatOutput(data, format) {
+  if (format === 'table') return formatTable(data);
+
+  if (format === 'raw') {
+    if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
+      return data.map(d => d.url).join('\n');
+    }
+    if (Array.isArray(data) && data.length > 0 && 'uniqueId' in data[0]) {
+      return data.map(d => `https://www.tiktok.com/@${d.uniqueId}`).join('\n');
+    }
+    return JSON.stringify(data, null, 2);
+  }
+
+  // Default JSON output, but for explore results (url-only) output pure text
+  if (Array.isArray(data) && data.length > 0 && 'url' in data[0]) {
+    return data.map(d => d.url).join('\n');
+  }
+
+  return JSON.stringify(data, null, 2);
+}