tt-help-cli-ycl 1.3.13 → 1.3.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.13",
+  "version": "1.3.14",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {
@@ -39,6 +39,8 @@
   },
   "homepage": "https://github.com/jsjhycl/tt-help-cli#readme",
   "dependencies": {
+    "axios": "^1.16.1",
+    "https-proxy-agent": "^9.0.0",
     "playwright": "^1.59.1",
     "undici": "^8.1.0"
   }

package/src/cli/attach.js

@@ -0,0 +1,160 @@
+import { TikTokScraper } from '../lib/tiktok-scraper.mjs';
+
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
+async function apiGet(url) {
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
+}
+
+async function apiPut(url, body) {
+  return withRetry(`PUT ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'PUT',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
+  });
+}
+
+function isBrowserClosedError(err) {
+  if (!err) return false;
+  const msg = err.message || err.toString() || '';
+  return msg.includes('Target page, context or browser has been closed') ||
+         msg.includes('browser has been closed') ||
+         msg.includes('browserContext.newPage') ||
+         msg.includes('Protocol error');
+}
+
+export async function handleAttach(options) {
+  const { attachParallel, attachInterval, serverUrl, showHelp } = options;
+
+  if (showHelp) {
+    console.error('用法: tt-help attach [-p 并行数] [-i 间隔秒数] [-s 服务端地址]');
+    console.error('');
+    console.error('参数:');
+    console.error('  -p, --parallel <N>     并行抓取数（默认: 1）');
+    console.error('  -i, --interval <N>     无任务时轮询间隔，单位秒（默认: 10）');
+    console.error('  -s, --server <URL>     服务端地址（默认: http://127.0.0.1:3001）');
+    console.error('');
+    console.error('说明:');
+    console.error('  后台轮询服务端 /api/user-update-tasks 接口，自动抓取 TikTok 用户信息');
+    console.error('  抓取完成后通过 PUT /api/user-update-result/{uniqueId} 回传结果');
+    console.error('  浏览器崩溃时自动重启，支持长时间无人值守运行');
+    console.error('');
+    console.error('示例:');
+    console.error('  tt-help attach');
+    console.error('  tt-help attach -p 5 -i 10');
+    console.error('  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001');
+    return;
+  }
+
+  console.error(`[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}`);
+
+  const scraper = new TikTokScraper();
+
+  try {
+    await scraper.init();
+    console.error('[Attach] 浏览器初始化完成，开始循环接收任务...');
+
+    let loopCount = 0;
+    let browserRestartCount = 0;
+
+    while (true) {
+      loopCount++;
+
+      // 检查浏览器是否存活，不存活则重启
+      if (!scraper.isAlive()) {
+        console.error(`[Attach] 浏览器已关闭，正在重启 (${++browserRestartCount})...`);
+        await scraper.restart();
+        console.error('[Attach] 浏览器重启完成');
+      }
+
+      const { total, tasks } = await apiGet(`${serverUrl}/api/user-update-tasks?limit=${attachParallel}`);
+
+      if (!tasks || tasks.length === 0) {
+        if (loopCount === 1) {
+          console.error(`[Attach] 当前无待更新任务，${attachInterval}秒后重试...`);
+        }
+        await new Promise(r => setTimeout(r, attachInterval * 1000));
+        continue;
+      }
+
+      console.error(`\n[Attach] 获取到 ${tasks.length} 个待更新任务...`);
+
+      const results = await Promise.allSettled(
+        tasks.map(async (task) => {
+          const uniqueId = task.uniqueId;
+          console.error(`  → 获取 @${uniqueId} 的用户信息...`);
+          try {
+            const info = await scraper.getUserInfo(uniqueId);
+            return { uniqueId, info, error: null };
+          } catch (err) {
+            return { uniqueId, info: null, error: err };
+          }
+        })
+      );
+
+      let successCount = 0;
+      let failCount = 0;
+      let needRestart = false;
+
+      for (const result of results) {
+        if (result.status === 'fulfilled') {
+          const { uniqueId, info, error } = result.value;
+          if (error) {
+            if (isBrowserClosedError(error)) {
+              needRestart = true;
+            }
+            console.error(`  ✗ @${uniqueId} 获取失败: ${error.message || '未知错误'}`);
+            failCount++;
+          } else if (info) {
+            try {
+              await apiPut(`${serverUrl}/api/user-info/${encodeURIComponent(uniqueId)}`, info);
+              console.error(`  ✓ @${uniqueId} 已提交更新`);
+              successCount++;
+            } catch (err) {
+              console.error(`  ✗ @${uniqueId} 提交失败: ${err.message}`);
+              failCount++;
+            }
+          } else {
+            console.error(`  ✗ @${uniqueId} 未获取到用户信息`);
+            failCount++;
+          }
+        } else {
+          console.error(`  ✗ 任务执行异常: ${result.reason?.message || '未知错误'}`);
+          failCount++;
+        }
+      }
+
+      console.error(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
+
+      if (needRestart) {
+        console.error('[Attach] 检测到浏览器异常，将在下一轮重启...');
+      }
+
+      await new Promise(r => setTimeout(r, 500));
+    }
+  } catch (err) {
+    console.error(`[Attach] 运行异常: ${err.message}`);
+    throw err;
+  } finally {
+    await scraper.close();
+  }
+}

package/src/cli/config.js

@@ -1,4 +1,4 @@
-import { HELP_TEXT, configPath, saveBrowser, saveUserId, getConfigText } from '../lib/constants.js';
+import { HELP_TEXT, configPath, saveBrowser, saveUserId, saveMaxFollowing, saveMaxFollowers, saveMaxVideos, saveMaxComments, getConfigText } from '../lib/constants.js';
 import { readFileSync, writeFileSync, existsSync } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
@@ -34,7 +34,7 @@ function handleConfig(action, key, value) {
     case 'set': {
       if (!key) {
         console.error('用法: tt-help config set <key> <value>');
-        console.error('  可用 key: proxy, server, browser, userId');
+        console.error('  可用 key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments');
         return;
       }
 
@@ -78,9 +78,45 @@ function handleConfig(action, key, value) {
           console.error(`用户号已更新: ${value}`);
           break;
 
+        case 'maxFollowing':
+          if (!value) {
+            console.error('请提供 maxFollowing 的值');
+            return;
+          }
+          saveMaxFollowing(value);
+          console.error(`商家关注采集数已更新: ${value}`);
+          break;
+
+        case 'maxFollowers':
+          if (!value) {
+            console.error('请提供 maxFollowers 的值');
+            return;
+          }
+          saveMaxFollowers(value);
+          console.error(`粉丝采集数已更新: ${value}`);
+          break;
+
+        case 'maxVideos':
+          if (!value) {
+            console.error('请提供 maxVideos 的值');
+            return;
+          }
+          saveMaxVideos(value);
+          console.error(`视频采集数已更新: ${value}`);
+          break;
+
+        case 'maxComments':
+          if (!value) {
+            console.error('请提供 maxComments 的值');
+            return;
+          }
+          saveMaxComments(value);
+          console.error(`评论采集数已更新: ${value}`);
+          break;
+
         default:
           console.error(`未知配置项: ${key}`);
-          console.error('  可用 key: proxy, server, browser, userId');
+          console.error('  可用 key: proxy, server, browser, userId, maxFollowing, maxFollowers, maxVideos, maxComments');
       }
       break;
     }

package/src/cli/explore.js

@@ -45,6 +45,7 @@ export async function handleExplore(options) {
     exploreEnableFollow, exploreMaxFollowing, exploreMaxFollowers,
     exploreLocation, exploreMaxUsers, serverUrl,
     explorePort, exploreProfile, exploreUserId,
+    exploreMaxVideos,
   } = options;
 
   let userId = exploreUserId || configuredUserId;
@@ -64,7 +65,7 @@ export async function handleExplore(options) {
   }
 
   console.error(`\n国家筛选: ${exploreLocation}`);
-  console.error(`评论: ${exploreMaxComments}, 猜你喜欢: ${exploreMaxGuess}`);
+  console.error(`视频采集: ${exploreMaxVideos || 1}`);
   console.error(`关注/粉丝: ${exploreEnableFollow ? '启用' : '禁用'}`);
   console.error(`服务器: ${serverUrl}（断开会自动重连）`);
   if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
@@ -111,8 +112,7 @@ export async function handleExplore(options) {
     await delay(switchMax, switchMax * 3);
 
     let result = await processExplore(page, username, {
-      maxComments: exploreMaxComments,
-      maxGuess: exploreMaxGuess,
+      maxVideos: exploreMaxVideos,
       enableFollow: exploreEnableFollow,
       maxFollowing: exploreMaxFollowing,
       maxFollowers: exploreMaxFollowers,
@@ -128,8 +128,7 @@ export async function handleExplore(options) {
       Object.assign(page, newPage);
       // 重试当前用户
       result = await processExplore(page, username, {
-        maxComments: exploreMaxComments,
-        maxGuess: exploreMaxGuess,
+        maxVideos: exploreMaxVideos,
         enableFollow: exploreEnableFollow,
         maxFollowing: exploreMaxFollowing,
         maxFollowers: exploreMaxFollowers,
@@ -189,11 +188,6 @@ export async function handleExplore(options) {
 
     const payload = {
       userInfo: result.userInfo || {},
-      discoveredVideoAuthors: (result.discoveredVideoAuthors || []).map(item =>
-        typeof item === 'object' ? { ...item, guessedLocation } : item
-      ),
-      discoveredCommentAuthors: (result.discoveredCommentAuthors || []).map(author => ({ author, guessedLocation })),
-      discoveredGuessAuthors: (result.discoveredGuessAuthors || []).map(author => ({ author, guessedLocation })),
       discoveredFollowing: (result.discoveredFollowing || []).map(f => ({
         handle: Array.isArray(f) ? f[0] : f,
         displayName: Array.isArray(f) ? f[1] : null,
@@ -209,8 +203,21 @@ export async function handleExplore(options) {
       keepFollow: result.keepFollow,
       locationCreated: result.locationCreated,
       noVideo: result.noVideo,
+      collectedVideos: result.collectedVideos,
     };
     await apiPost(`${serverUrl}/api/job/${username}`, payload);
+
+    // 视频登记
+    if (result.videoList && result.videoList.length > 0) {
+      const { registered, skipped } = await apiPost(`${serverUrl}/api/videos`, {
+        sourceUser: username,
+        videoList: result.videoList,
+        locationCreated: result.locationCreated,
+        ttSeller: result.userInfo?.ttSeller || false,
+      });
+      console.error(`  视频登记: ${registered} 条新增, ${skipped} 条已存在`);
+    }
+
     console.error('  已提交');
 
     if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {

package/src/cli/info.js

@@ -0,0 +1,88 @@
+import { TikTokScraper } from '../lib/tiktok-scraper.mjs';
+import { isProfileUrl, isVideoUrl, extractUniqueId, normalizeUsername } from '../lib/url.js';
+
+async function handleInfo(options) {
+  const { infoUrls, infoOnlyVideo } = options;
+
+  if (!infoUrls || infoUrls.length === 0) {
+    console.error('用法: tt-help info <URL> [URL2 URL3...] [--onlyvideo]');
+    console.error('');
+    console.error('参数:');
+    console.error('  <URL>              TikTok 主页或视频 URL，支持多个 URL 同时查询');
+    console.error('  --onlyvideo        只返回视频信息（不返回用户信息）');
+    console.error('');
+    console.error('默认行为:');
+    console.error('  主页 URL           → 返回用户信息（bio、region、粉丝数等）');
+    console.error('  视频 URL           → 返回用户信息 + 视频信息');
+    console.error('  视频 URL + --onlyvideo → 只返回视频信息');
+    console.error('');
+    console.error('示例:');
+    console.error('  tt-help info https://www.tiktok.com/@nike');
+    console.error('  tt-help info https://www.tiktok.com/@nike/video/7234567890');
+    console.error('  tt-help info https://www.tiktok.com/@nike https://www.tiktok.com/@apple');
+    process.exit(1);
+  }
+
+  const scraper = new TikTokScraper({ poolSize: 1 });
+
+  try {
+    await scraper.init();
+    const result = {};
+
+    for (const url of infoUrls) {
+      if (isProfileUrl(url)) {
+        const uniqueId = extractUniqueId(url);
+        const normalized = normalizeUsername(uniqueId);
+        const user = await scraper.getUserInfo(normalized);
+        if (!user) {
+          console.error(`无法获取用户 @${uniqueId} 的信息`);
+          continue;
+        }
+        result[normalized] = { user };
+        console.error(`用户: @${user.uniqueId} (${user.nickname})`);
+      } else if (isVideoUrl(url)) {
+        const uniqueId = extractUniqueId(url);
+        const normalized = normalizeUsername(uniqueId);
+
+        if (infoOnlyVideo) {
+          const video = await scraper.getVideoInfo(url);
+          if (!video) {
+            console.error(`无法获取视频信息: ${url}`);
+            continue;
+          }
+          const key = normalized + '/video/' + video.id;
+          result[key] = { video };
+          console.error(`视频: ${video.id}`);
+        } else {
+          const [user, video] = await Promise.all([
+            scraper.getUserInfo(normalized),
+            scraper.getVideoInfo(url),
+          ]);
+          const entry = {};
+          if (user) {
+            entry.user = user;
+            console.error(`用户: @${user.uniqueId} (${user.nickname})`);
+          }
+          if (video) {
+            entry.video = video;
+            console.error(`视频: ${video.id}`);
+          }
+          if (!user && !video) {
+            console.error(`无法获取信息: ${url}`);
+            continue;
+          }
+          const key = normalized + '/video/' + (video ? video.id : 'unknown');
+          result[key] = entry;
+        }
+      } else {
+        console.error(`无法识别 URL: ${url}`);
+      }
+    }
+
+    console.log(JSON.stringify(result, null, 2));
+  } finally {
+    await scraper.close();
+  }
+}
+
+export { handleInfo };

package/src/lib/args.js

@@ -164,6 +164,7 @@ function parseExploreArgs(args) {
   let explorePort = null;
   let exploreProfile = null;
   let exploreUserId = null;
+  let exploreMaxVideos = 1;
 
   const positional = [];
   const PRESETS = ['fast', 'normal', 'slow', 'stealth'];
@@ -194,6 +195,8 @@ function parseExploreArgs(args) {
       exploreProfile = args[++i];
     } else if (arg === '--user-id') {
       exploreUserId = args[++i];
+    } else if (arg === '--max-videos') {
+      exploreMaxVideos = parseInt(args[++i]) || 1;
     } else {
       positional.push(arg);
     }
@@ -226,6 +229,7 @@ function parseExploreArgs(args) {
     explorePort,
     exploreProfile,
     exploreUserId,
+    exploreMaxVideos,
     urls: [],
     outputFormat: 'json',
     exploreCount: 0,
@@ -276,6 +280,38 @@ function parseVideosArgs(args) {
   };
 }
 
+function parseInfoArgs(args) {
+  let onlyVideo = false;
+  const urls = [];
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '--onlyvideo') {
+      onlyVideo = true;
+    } else if (!arg.startsWith('--')) {
+      urls.push(arg);
+    }
+  }
+
+  return {
+    subcommand: 'info',
+    infoUrls: urls,
+    infoOnlyVideo: onlyVideo,
+    outputFile: null,
+    outputFormat: 'json',
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    showVersion: false,
+    customProxy: null,
+    configAction: null,
+    configKey: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 function parseWatchArgs(args) {
   let outputFile = './result.json';
   let watchPort = 3001;
@@ -356,10 +392,43 @@ function parseRefreshArgs(args) {
   };
 }
 
+function parseAttachArgs(args) {
+  let parallel = 1;
+  let interval = 10;
+  let serverUrl = defaultServer;
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === '-p' || arg === '--parallel') {
+      parallel = parseInt(args[++i], 10) || 1;
+    } else if (arg === '-i' || arg === '--interval') {
+      interval = parseInt(args[++i], 10) || 10;
+    } else if (arg === '-s' || arg === '--server') {
+      serverUrl = args[++i];
+    }
+  }
+
+  return {
+    subcommand: 'attach',
+    attachParallel: parallel,
+    attachInterval: interval,
+    serverUrl,
+    urls: [],
+    outputFormat: 'json',
+    exploreCount: 0,
+    showConfig: false,
+    showHelp: false,
+    customProxy: null,
+    configAction: null,
+    configValue: null,
+    pipeMode: false,
+    filterStr: null,
+  };
+}
+
 export function parseArgs() {
   const args = process.argv.slice(2);
 
-  // Global flags take precedence over subcommands
   if (args.includes('-h') || args.includes('--help')) {
     return { showHelp: true, showVersion: false, subcommand: null, urls: [], outputFile: null, outputFormat: 'json', exploreCount: 0, showConfig: false, customProxy: null, configAction: null, configValue: null, pipeMode: false, filterStr: null };
   }
@@ -367,30 +436,22 @@ export function parseArgs() {
     return { showHelp: false, showVersion: true, subcommand: null, urls: [], outputFile: null, outputFormat: 'json', exploreCount: 0, showConfig: false, customProxy: null, configAction: null, configValue: null, pipeMode: false, filterStr: null };
   }
 
-  if (args.length > 0 && args[0] === 'scrape') {
-    return parseScrapeArgs(args.slice(1));
-  }
-
-  if (args.length > 0 && args[0] === 'videos') {
-    return parseVideosArgs(args.slice(1));
-  }
-
-  if (args.length > 0 && args[0] === 'auto') {
-    return parseAutoArgs(args.slice(1));
-  }
-
   if (args.length > 0 && args[0] === 'explore') {
     return parseExploreArgs(args.slice(1));
   }
 
-  if (args.length > 0 && args[0] === 'refresh') {
-    return parseRefreshArgs(args.slice(1));
+  if (args.length > 0 && args[0] === 'info') {
+    return parseInfoArgs(args.slice(1));
   }
 
   if (args.length > 0 && args[0] === 'watch') {
     return parseWatchArgs(args.slice(1));
   }
 
+  if (args.length > 0 && args[0] === 'attach') {
+    return parseAttachArgs(args.slice(1));
+  }
+
   const urls = [];
   let inputFile = null;
   let outputFile = null;