tt-help-cli-ycl 1.3.5 → 1.3.7

package/README.md

@@ -1,17 +1,17 @@
-# tt-help
-
-TikTok user & video data scraper — extract ttSeller, verified, locationCreated from HTML source.
-
-## Install
-
-```bash
-npm i -g tt-help
-```
-
-## Usage
-
-```bash
-tt-help [options] <urls...>
-```
-
-示例、代理配置、和更多用法请查看仓库或 `src/main.mjs` 中的帮助文本。
+# tt-help
+
+TikTok user & video data scraper — extract ttSeller, verified, locationCreated from HTML source.
+
+## Install
+
+```bash
+npm i -g tt-help
+```
+
+## Usage
+
+```bash
+tt-help [options] <urls...>
+```
+
+示例、代理配置、和更多用法请查看仓库或 `src/main.mjs` 中的帮助文本。

package/cli.js

@@ -1,9 +1,9 @@
-#!/usr/bin/env node
-import { fileURLToPath } from 'url';
-import { dirname, resolve } from 'path';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-
-const mainPath = resolve(__dirname, 'src', 'main.mjs');
-await import(`file://${mainPath}`);
+#!/usr/bin/env node
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const mainPath = resolve(__dirname, 'src', 'main.mjs');
+await import(`file://${mainPath}`);

package/package.json

@@ -1,45 +1,45 @@
-{
-  "name": "tt-help-cli-ycl",
-  "version": "1.3.5",
-  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
-  "type": "module",
-  "bin": {
-    "tt-help": "cli.js"
-  },
-  "main": "src/main.mjs",
-  "files": [
-    "cli.js",
-    "src/"
-  ],
-  "scripts": {
-    "start": "node src/main.mjs"
-  },
-  "keywords": [
-    "tiktok",
-    "scraper",
-    "cli",
-    "seller",
-    "ttSeller"
-  ],
-  "author": "jsjhycl",
-  "license": "ISC",
-  "engines": {
-    "node": ">=18"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/jsjhycl/tt-help-cli.git"
-  },
-  "bugs": {
-    "url": "https://github.com/jsjhycl/tt-help-cli/issues"
-  },
-  "homepage": "https://github.com/jsjhycl/tt-help-cli#readme",
-  "dependencies": {
-    "playwright": "^1.59.1",
-    "tt-help-cli-ycl": "^1.3.2",
-    "undici": "^8.1.0"
-  }
-}
+{
+  "name": "tt-help-cli-ycl",
+  "version": "1.3.7",
+  "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
+  "type": "module",
+  "bin": {
+    "tt-help": "cli.js"
+  },
+  "main": "src/main.mjs",
+  "files": [
+    "cli.js",
+    "src/"
+  ],
+  "scripts": {
+    "start": "node src/main.mjs"
+  },
+  "keywords": [
+    "tiktok",
+    "scraper",
+    "cli",
+    "seller",
+    "ttSeller"
+  ],
+  "author": "jsjhycl",
+  "license": "ISC",
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jsjhycl/tt-help-cli.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jsjhycl/tt-help-cli/issues"
+  },
+  "homepage": "https://github.com/jsjhycl/tt-help-cli#readme",
+  "dependencies": {
+    "playwright": "^1.59.1",
+    "tt-help-cli-ycl": "^1.3.2",
+    "undici": "^8.1.0"
+  }
+}

package/src/cli/auto.js

@@ -1,94 +1,131 @@
-import { getOrCreatePage } from '../lib/browser/page.js';
-
-async function apiPost(url, body) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-  });
-  return res.json();
-}
-
-async function apiGet(url) {
-  const res = await fetch(url);
-  return res.json();
-}
-
-export async function handleAuto(options) {
-  const { autoUsernames, autoCollectMax, autoScrapeDepth, autoMaxComments, autoMaxGuess,
-    autoPreset, autoSwitchDelay, autoCommentDelay, serverUrl,
-    autoEnableFollow, autoMaxFollowing, autoMaxFollowers } = options;
-
-  const runOptions = {
-    collectMax: autoCollectMax,
-    scrapeDepth: autoScrapeDepth,
-    maxComments: autoMaxComments,
-    maxGuess: autoMaxGuess,
-    preset: autoPreset,
-    switchMax: autoSwitchDelay,
-    commentMax: autoCommentDelay,
-    enableFollow: autoEnableFollow,
-    maxFollowing: autoMaxFollowing,
-    maxFollowers: autoMaxFollowers,
-  };
-
-  try {
-    await apiGet(`${serverUrl}/api/stats`);
-  } catch {
-    console.error(`无法连接服务端: ${serverUrl}`);
-    console.error('请先启动服务端: tt-help watch -o <数据文件>');
-    process.exit(1);
-  }
-
-  if (autoUsernames.length > 0) {
-    const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: autoUsernames });
-    console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
-  }
-
-  const { ensureBrowserReady, processUser } = await import('../scraper/auto-core.mjs');
-  const browser = await ensureBrowserReady();
-
-  try {
-    const page = await getOrCreatePage(browser);
-
-    let processedCount = 0;
-    let errorCount = 0;
-
-    while (true) {
-      const job = await apiGet(`${serverUrl}/api/job`);
-      if (!job.hasJob) break;
-
-      const username = job.user.uniqueId;
-      processedCount++;
-      console.error(`\n[${processedCount}] 处理 @${username}...`);
-
-      const result = await processUser(page, username, { ...runOptions, browser }, console.error);
-
-      if (result.restricted || result.error) {
-        if (result.error) errorCount++;
-        await apiPost(`${serverUrl}/api/job/${username}`, result);
-        continue;
-      }
-
-      const payload = {
-        userInfo: result.userInfo || {},
-        discoveredVideoAuthors: result.discoveredVideoAuthors || [],
-        discoveredCommentAuthors: result.discoveredCommentAuthors || [],
-        discoveredGuessAuthors: result.discoveredGuessAuthors || [],
-        discoveredFollowing: result.discoveredFollowing || [],
-        discoveredFollowers: result.discoveredFollowers || [],
-      };
-      await apiPost(`${serverUrl}/api/job/${username}`, payload);
-      console.error('  已提交');
-    }
-
-    const stats = await apiGet(`${serverUrl}/api/stats`);
-    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
-    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
-  } catch (err) {
-    console.error(`自动抓取失败: ${err.message}`);
-    process.exit(1);
-  } finally {
-    await browser.close().catch(() => {});
-  }
-}
+import { getOrCreatePage } from '../lib/browser/page.js';
+import { userId as configuredUserId, saveUserId } from '../lib/constants.js';
+import { getMacOrUuid } from '../lib/mac-or-uuid.js';
+
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
+async function apiPost(url, body) {
+  return withRetry(`POST ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
+  });
+}
+
+async function apiGet(url) {
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
+}
+
+export async function handleAuto(options) {
+  const { autoUsernames, autoCollectMax, autoScrapeDepth, autoMaxComments, autoMaxGuess,
+    autoPreset, autoSwitchDelay, autoCommentDelay, serverUrl,
+    autoEnableFollow, autoMaxFollowing, autoMaxFollowers } = options;
+
+  let userId = configuredUserId;
+  if (!userId) {
+    userId = await getMacOrUuid();
+    saveUserId(userId);
+    console.error(`[初始化] 未检测到本地用户编号，已生成并使用: ${userId}`);
+  }
+
+  const runOptions = {
+    collectMax: autoCollectMax,
+    scrapeDepth: autoScrapeDepth,
+    maxComments: autoMaxComments,
+    maxGuess: autoMaxGuess,
+    preset: autoPreset,
+    switchMax: autoSwitchDelay,
+    commentMax: autoCommentDelay,
+    enableFollow: autoEnableFollow,
+    maxFollowing: autoMaxFollowing,
+    maxFollowers: autoMaxFollowers,
+    userId,
+  };
+
+  await apiGet(`${serverUrl}/api/stats`);
+
+  if (autoUsernames.length > 0) {
+    const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: autoUsernames });
+    console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
+  }
+
+  console.error(`服务器: ${serverUrl}（断开会自动重连）`);
+
+  const { ensureBrowserReady, processUser } = await import('../scraper/auto-core.mjs');
+  const browser = await ensureBrowserReady();
+
+  const page = await getOrCreatePage(browser);
+
+  let processedCount = 0;
+  let errorCount = 0;
+
+  while (true) {
+    const job = await apiGet(`${serverUrl}/api/job?userId=${encodeURIComponent(userId)}`);
+    if (!job.hasJob) break;
+
+    const username = job.user.uniqueId;
+    processedCount++;
+    let proxyRetry = 0;
+
+    while (true) {
+      console.error(`\n[${processedCount}] 处理 @${username}...${proxyRetry > 0 ? ` (代理重试 ${proxyRetry})` : ''}`);
+
+      const result = await processUser(page, username, { ...runOptions, browser }, console.error);
+
+      if (result.restricted) {
+        await apiPost(`${serverUrl}/api/job/${username}`, result);
+        break;
+      }
+
+      if (result.error && result.error.includes('代理错误')) {
+        proxyRetry++;
+        console.error(`  [代理错误] ${result.error}，等待 10s 后重试...`);
+        await new Promise(r => setTimeout(r, 10000));
+        continue;
+      }
+
+      if (result.error) {
+        errorCount++;
+        await apiPost(`${serverUrl}/api/job/${username}`, result);
+        break;
+      }
+
+      const payload = {
+        userInfo: result.userInfo || {},
+        discoveredVideoAuthors: result.discoveredVideoAuthors || [],
+        discoveredCommentAuthors: result.discoveredCommentAuthors || [],
+        discoveredGuessAuthors: result.discoveredGuessAuthors || [],
+        discoveredFollowing: result.discoveredFollowing || [],
+        discoveredFollowers: result.discoveredFollowers || [],
+      };
+      await apiPost(`${serverUrl}/api/job/${username}`, payload);
+      console.error('  已提交');
+      break;
+    }
+  }
+
+  const stats = await apiGet(`${serverUrl}/api/stats`);
+  console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+  console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+
+  await browser.close().catch(() => {});
+}

package/src/cli/explore.js

@@ -1,116 +1,147 @@
-import { getOrCreatePage } from '../lib/browser/page.js';
-import { delay, getDelayConfig, setDelayConfig } from '../scraper/modules/page-helpers.mjs';
-
-async function apiPost(url, body) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-  });
-  return res.json();
-}
-
-async function apiGet(url) {
-  const res = await fetch(url);
-  return res.json();
-}
-
-export async function handleExplore(options) {
-  const {
-    exploreUsernames, explorePreset, exploreMaxComments, exploreMaxGuess,
-    exploreEnableFollow, exploreMaxFollowing, exploreMaxFollowers,
-    exploreLocation, exploreMaxUsers, serverUrl,
-  } = options;
-
-  setDelayConfig(explorePreset);
-
-  try {
-    await apiGet(`${serverUrl}/api/stats`);
-  } catch {
-    console.error(`无法连接服务端: ${serverUrl}，退出`);
-    process.exit(1);
-  }
-
-  if (exploreUsernames && exploreUsernames.length > 0) {
-    const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: exploreUsernames });
-    console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
-  }
-
-  console.error(`\n国家筛选: ${exploreLocation}`);
-  console.error(`评论: ${exploreMaxComments}, 猜你喜欢: ${exploreMaxGuess}`);
-  console.error(`关注/粉丝: ${exploreEnableFollow ? '启用' : '禁用'}`);
-  if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
-
-  const { ensureBrowserReady, processExplore } = await import('../scraper/explore-core.mjs');
-  const browser = await ensureBrowserReady();
-
-  try {
-    const page = await getOrCreatePage(browser);
-
-    let processedCount = 0;
-    let errorCount = 0;
-
-    while (true) {
-      const job = await apiGet(`${serverUrl}/api/job`);
-      if (!job.hasJob) break;
-
-      const username = job.user.uniqueId;
-      processedCount++;
-      console.error(`\n[${processedCount}] 探索 @${username}...`);
-
-      const { switchMax } = getDelayConfig();
-      await delay(switchMax, switchMax * 3);
-
-      const result = await processExplore(page, username, {
-        maxComments: exploreMaxComments,
-        maxGuess: exploreMaxGuess,
-        enableFollow: exploreEnableFollow,
-        maxFollowing: exploreMaxFollowing,
-        maxFollowers: exploreMaxFollowers,
-        location: exploreLocation,
-        browser,
-      }, console.error);
-
-      if (result.restricted) {
-        await apiPost(`${serverUrl}/api/job/${username}`, { restricted: true, userInfo: result.userInfo || {} });
-        continue;
-      }
-
-      if (result.error) {
-        errorCount++;
-        await apiPost(`${serverUrl}/api/job/${username}`, { error: result.error });
-        continue;
-      }
-
-      const payload = {
-        userInfo: result.userInfo || {},
-        discoveredVideoAuthors: result.discoveredVideoAuthors || [],
-        discoveredCommentAuthors: result.discoveredCommentAuthors || [],
-        discoveredGuessAuthors: result.discoveredGuessAuthors || [],
-        discoveredFollowing: result.discoveredFollowing || [],
-        discoveredFollowers: result.discoveredFollowers || [],
-        processed: result.processed,
-        hasFollowData: result.hasFollowData,
-        keepFollow: result.keepFollow,
-        locationCreated: result.locationCreated,
-        noVideo: result.noVideo,
-      };
-      await apiPost(`${serverUrl}/api/job/${username}`, payload);
-      console.error('  已提交');
-
-      if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
-        console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
-        break;
-      }
-    }
-
-    const stats = await apiGet(`${serverUrl}/api/stats`);
-    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
-    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
-  } catch (err) {
-    console.error(`探索失败: ${err.message}`);
-    process.exit(1);
-  } finally {
-    await browser.close().catch(() => {});
-  }
-}
+import { getOrCreatePage } from '../lib/browser/page.js';
+import { delay, getDelayConfig, setDelayConfig } from '../scraper/modules/page-helpers.mjs';
+import { userId as configuredUserId, saveUserId } from '../lib/constants.js';
+import { getMacOrUuid } from '../lib/mac-or-uuid.js';
+
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
+async function apiPost(url, body) {
+  return withRetry(`POST ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
+  });
+}
+
+async function apiGet(url) {
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
+}
+
+export async function handleExplore(options) {
+  const {
+    exploreUsernames, explorePreset, exploreMaxComments, exploreMaxGuess,
+    exploreEnableFollow, exploreMaxFollowing, exploreMaxFollowers,
+    exploreLocation, exploreMaxUsers, serverUrl,
+  } = options;
+
+  let userId = configuredUserId;
+  if (!userId) {
+    userId = await getMacOrUuid();
+    saveUserId(userId);
+    console.error(`[初始化] 未检测到本地用户编号，已生成并使用: ${userId}`);
+  }
+
+  setDelayConfig(explorePreset);
+
+  await apiGet(`${serverUrl}/api/stats`);
+
+  if (exploreUsernames && exploreUsernames.length > 0) {
+    const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: exploreUsernames });
+    console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
+  }
+
+  console.error(`\n国家筛选: ${exploreLocation}`);
+  console.error(`评论: ${exploreMaxComments}, 猜你喜欢: ${exploreMaxGuess}`);
+  console.error(`关注/粉丝: ${exploreEnableFollow ? '启用' : '禁用'}`);
+  console.error(`服务器: ${serverUrl}（断开会自动重连）`);
+  if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
+
+  const { ensureBrowserReady, processExplore } = await import('../scraper/explore-core.mjs');
+  const browser = await ensureBrowserReady();
+
+  const page = await getOrCreatePage(browser);
+
+  let processedCount = 0;
+  let errorCount = 0;
+
+  while (true) {
+    const job = await apiGet(`${serverUrl}/api/job?userId=${encodeURIComponent(userId)}`);
+    if (!job.hasJob) break;
+
+    const username = job.user.uniqueId;
+    processedCount++;
+    let proxyRetry = 0;
+
+    while (true) {
+      console.error(`\n[${processedCount}] 探索 @${username}...${proxyRetry > 0 ? ` (代理重试 ${proxyRetry})` : ''}`);
+
+      const { switchMax } = getDelayConfig();
+      await delay(switchMax, switchMax * 3);
+
+      const result = await processExplore(page, username, {
+        maxComments: exploreMaxComments,
+        maxGuess: exploreMaxGuess,
+        enableFollow: exploreEnableFollow,
+        maxFollowing: exploreMaxFollowing,
+        maxFollowers: exploreMaxFollowers,
+        location: exploreLocation,
+        browser,
+      }, console.error);
+
+      if (result.restricted) {
+        await apiPost(`${serverUrl}/api/job/${username}`, { restricted: true, userInfo: result.userInfo || {} });
+        break;
+      }
+
+      if (result.error && result.error.includes('代理错误')) {
+        proxyRetry++;
+        console.error(`  [代理错误] ${result.error}，等待 10s 后重试...`);
+        await new Promise(r => setTimeout(r, 10000));
+        continue;
+      }
+
+      if (result.error) {
+        errorCount++;
+        await apiPost(`${serverUrl}/api/job/${username}`, { error: result.error });
+        break;
+      }
+
+      const payload = {
+        userInfo: result.userInfo || {},
+        discoveredVideoAuthors: result.discoveredVideoAuthors || [],
+        discoveredCommentAuthors: result.discoveredCommentAuthors || [],
+        discoveredGuessAuthors: result.discoveredGuessAuthors || [],
+        discoveredFollowing: result.discoveredFollowing || [],
+        discoveredFollowers: result.discoveredFollowers || [],
+        processed: result.processed,
+        hasFollowData: result.hasFollowData,
+        keepFollow: result.keepFollow,
+        locationCreated: result.locationCreated,
+        noVideo: result.noVideo,
+      };
+      await apiPost(`${serverUrl}/api/job/${username}`, payload);
+      console.error('  已提交');
+      break;
+    }
+
+    if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
+      console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
+      break;
+    }
+  }
+
+  const stats = await apiGet(`${serverUrl}/api/stats`);
+  console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+  console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+
+  await browser.close().catch(() => {});
+}