tt-help-cli-ycl 1.3.5 → 1.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.5",
+  "version": "1.3.6",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/auto.js

@@ -1,17 +1,36 @@
 import { getOrCreatePage } from '../lib/browser/page.js';
 
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
 async function apiPost(url, body) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
+  return withRetry(`POST ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
   });
-  return res.json();
 }
 
 async function apiGet(url) {
-  const res = await fetch(url);
-  return res.json();
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
 }
 
 export async function handleAuto(options) {
@@ -32,44 +51,54 @@ export async function handleAuto(options) {
     maxFollowers: autoMaxFollowers,
   };
 
-  try {
-    await apiGet(`${serverUrl}/api/stats`);
-  } catch {
-    console.error(`无法连接服务端: ${serverUrl}`);
-    console.error('请先启动服务端: tt-help watch -o <数据文件>');
-    process.exit(1);
-  }
+  await apiGet(`${serverUrl}/api/stats`);
 
   if (autoUsernames.length > 0) {
     const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: autoUsernames });
     console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
   }
 
+  console.error(`服务器: ${serverUrl}（断开会自动重连）`);
+
   const { ensureBrowserReady, processUser } = await import('../scraper/auto-core.mjs');
   const browser = await ensureBrowserReady();
 
-  try {
-    const page = await getOrCreatePage(browser);
+  const page = await getOrCreatePage(browser);
 
-    let processedCount = 0;
-    let errorCount = 0;
+  let processedCount = 0;
+  let errorCount = 0;
 
-    while (true) {
-      const job = await apiGet(`${serverUrl}/api/job`);
-      if (!job.hasJob) break;
+  while (true) {
+    const job = await apiGet(`${serverUrl}/api/job`);
+    if (!job.hasJob) break;
 
-      const username = job.user.uniqueId;
-      processedCount++;
-      console.error(`\n[${processedCount}] 处理 @${username}...`);
+    const username = job.user.uniqueId;
+    processedCount++;
+    let proxyRetry = 0;
+
+    while (true) {
+      console.error(`\n[${processedCount}] 处理 @${username}...${proxyRetry > 0 ? ` (代理重试 ${proxyRetry})` : ''}`);
 
       const result = await processUser(page, username, { ...runOptions, browser }, console.error);
 
-      if (result.restricted || result.error) {
-        if (result.error) errorCount++;
+      if (result.restricted) {
         await apiPost(`${serverUrl}/api/job/${username}`, result);
+        break;
+      }
+
+      if (result.error && result.error.includes('代理错误')) {
+        proxyRetry++;
+        console.error(`  [代理错误] ${result.error}，等待 10s 后重试...`);
+        await new Promise(r => setTimeout(r, 10000));
         continue;
       }
 
+      if (result.error) {
+        errorCount++;
+        await apiPost(`${serverUrl}/api/job/${username}`, result);
+        break;
+      }
+
       const payload = {
         userInfo: result.userInfo || {},
         discoveredVideoAuthors: result.discoveredVideoAuthors || [],
@@ -80,15 +109,13 @@ export async function handleAuto(options) {
       };
       await apiPost(`${serverUrl}/api/job/${username}`, payload);
       console.error('  已提交');
+      break;
     }
-
-    const stats = await apiGet(`${serverUrl}/api/stats`);
-    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
-    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
-  } catch (err) {
-    console.error(`自动抓取失败: ${err.message}`);
-    process.exit(1);
-  } finally {
-    await browser.close().catch(() => {});
   }
+
+  const stats = await apiGet(`${serverUrl}/api/stats`);
+  console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+  console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+
+  await browser.close().catch(() => {});
 }

package/src/cli/explore.js

@@ -1,18 +1,37 @@
 import { getOrCreatePage } from '../lib/browser/page.js';
 import { delay, getDelayConfig, setDelayConfig } from '../scraper/modules/page-helpers.mjs';
 
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
 async function apiPost(url, body) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
+  return withRetry(`POST ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
   });
-  return res.json();
 }
 
 async function apiGet(url) {
-  const res = await fetch(url);
-  return res.json();
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
 }
 
 export async function handleExplore(options) {
@@ -24,12 +43,7 @@ export async function handleExplore(options) {
 
   setDelayConfig(explorePreset);
 
-  try {
-    await apiGet(`${serverUrl}/api/stats`);
-  } catch {
-    console.error(`无法连接服务端: ${serverUrl}，退出`);
-    process.exit(1);
-  }
+  await apiGet(`${serverUrl}/api/stats`);
 
   if (exploreUsernames && exploreUsernames.length > 0) {
     const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: exploreUsernames });
@@ -39,24 +53,27 @@ export async function handleExplore(options) {
   console.error(`\n国家筛选: ${exploreLocation}`);
   console.error(`评论: ${exploreMaxComments}, 猜你喜欢: ${exploreMaxGuess}`);
   console.error(`关注/粉丝: ${exploreEnableFollow ? '启用' : '禁用'}`);
+  console.error(`服务器: ${serverUrl}（断开会自动重连）`);
   if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
 
   const { ensureBrowserReady, processExplore } = await import('../scraper/explore-core.mjs');
   const browser = await ensureBrowserReady();
 
-  try {
-    const page = await getOrCreatePage(browser);
+  const page = await getOrCreatePage(browser);
 
-    let processedCount = 0;
-    let errorCount = 0;
+  let processedCount = 0;
+  let errorCount = 0;
 
-    while (true) {
-      const job = await apiGet(`${serverUrl}/api/job`);
-      if (!job.hasJob) break;
+  while (true) {
+    const job = await apiGet(`${serverUrl}/api/job`);
+    if (!job.hasJob) break;
 
-      const username = job.user.uniqueId;
-      processedCount++;
-      console.error(`\n[${processedCount}] 探索 @${username}...`);
+    const username = job.user.uniqueId;
+    processedCount++;
+    let proxyRetry = 0;
+
+    while (true) {
+      console.error(`\n[${processedCount}] 探索 @${username}...${proxyRetry > 0 ? ` (代理重试 ${proxyRetry})` : ''}`);
 
       const { switchMax } = getDelayConfig();
       await delay(switchMax, switchMax * 3);
@@ -73,13 +90,20 @@ export async function handleExplore(options) {
 
       if (result.restricted) {
         await apiPost(`${serverUrl}/api/job/${username}`, { restricted: true, userInfo: result.userInfo || {} });
+        break;
+      }
+
+      if (result.error && result.error.includes('代理错误')) {
+        proxyRetry++;
+        console.error(`  [代理错误] ${result.error}，等待 10s 后重试...`);
+        await new Promise(r => setTimeout(r, 10000));
         continue;
       }
 
       if (result.error) {
         errorCount++;
         await apiPost(`${serverUrl}/api/job/${username}`, { error: result.error });
-        continue;
+        break;
       }
 
       const payload = {
@@ -97,20 +121,18 @@ export async function handleExplore(options) {
       };
       await apiPost(`${serverUrl}/api/job/${username}`, payload);
       console.error('  已提交');
-
-      if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
-        console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
-        break;
-      }
+      break;
     }
 
-    const stats = await apiGet(`${serverUrl}/api/stats`);
-    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
-    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
-  } catch (err) {
-    console.error(`探索失败: ${err.message}`);
-    process.exit(1);
-  } finally {
-    await browser.close().catch(() => {});
+    if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
+      console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
+      break;
+    }
   }
+
+  const stats = await apiGet(`${serverUrl}/api/stats`);
+  console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+  console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+
+  await browser.close().catch(() => {});
 }

package/src/cli/watch.js

@@ -1,4 +1,5 @@
-import { writeFileSync, existsSync } from 'fs';
+import { existsSync } from 'fs';
+import { createStore } from '../watch/data-store.mjs';
 import { startWatchServer, openBrowser } from '../watch/server.mjs';
 
 export async function handleWatch(options) {
@@ -16,10 +17,12 @@ export async function handleWatch(options) {
     process.exit(1);
   }
 
-  const { server, port } = await startWatchServer(outputFile, watchPort);
+  const store = createStore(outputFile);
+  const { server, port } = await startWatchServer(outputFile, watchPort, store);
   openBrowser(port);
 
   process.once('SIGINT', () => {
+    store.stopBackup();
     server.close();
     process.exit(0);
   });

package/src/lib/browser/page.js

@@ -78,3 +78,10 @@ export async function getOrCreatePage(browser) {
   }
   return page;
 }
+
+export function assertPageUrl(page, expectedPath) {
+  const actual = page.url();
+  if (!actual.includes(expectedPath)) {
+    throw new Error(`[代理错误] 预期访问 ${expectedPath}，实际跳转到了 ${actual}`);
+  }
+}

package/src/scraper/auto-core.mjs

@@ -8,6 +8,7 @@ import {
   retryWithBackoff,
   detectPageError,
   isLoggedIn,
+  assertPageUrl,
 } from './modules/page-helpers.mjs';
 export { ensureBrowserReady };
 import {
@@ -66,6 +67,7 @@ async function processUser(page, username, options, log) {
     await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
       waitUntil: 'load', timeout: 30000,
     }), { log });
+    assertPageUrl(page, `@${username}`);
     await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
     await delay(1000, 2000);
 

package/src/scraper/core.mjs

@@ -6,6 +6,7 @@ import {
   setDelayConfig,
   getDelayConfig,
   retryWithBackoff,
+  assertPageUrl,
 } from './modules/page-helpers.mjs';
 import { extractCommentAuthors } from './modules/comment-extractor.mjs';
 import { extractGuessVideos } from './modules/guess-extractor.mjs';
@@ -100,6 +101,7 @@ async function runScrape(options) {
   }
 
   await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'load', timeout: 30000 }), { log });
+  assertPageUrl(page, videoUrl.split('/video/')[0]);
   await delay(Math.round(config.switchMax * 0.5), config.switchMax);
   await closeCommentPanel(page);
   await delay(Math.round(config.commentMax * 0.5), config.commentMax);

package/src/scraper/explore-core.mjs

@@ -6,6 +6,7 @@ import {
   retryWithBackoff,
   detectPageError,
   isLoggedIn,
+  assertPageUrl,
 } from './modules/page-helpers.mjs';
 export { ensureBrowserReady };
 import {
@@ -47,6 +48,7 @@ async function processExplore(page, username, options, log) {
     log(`  访问 @${username} 主页...`);
     const homeUrl = `https://www.tiktok.com/@${username}`;
     await retryWithBackoff(() => page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    assertPageUrl(page, `@${username}`);
     await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
     await delay(1000, 2000);
 
@@ -107,6 +109,7 @@ async function processExplore(page, username, options, log) {
 
     log(`  进入第一个视频: ${videoUrl}`);
     await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    assertPageUrl(page, videoUrl.split('/video/')[0]);
     await delay(1500, 2500);
 
     const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');

package/src/scraper/modules/page-helpers.mjs

@@ -12,6 +12,7 @@ import {
   findTikTokPage,
   getOrCreatePage,
   isLoggedIn,
+  assertPageUrl,
 } from '../../lib/browser/page.js';
 import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
 import {
@@ -34,6 +35,7 @@ export {
   findTikTokPage,
   getOrCreatePage,
   isLoggedIn,
+  assertPageUrl,
   retryWithBackoff,
   isRetryableError,
   extractUserSection,

package/src/watch/data-store.mjs

@@ -11,18 +11,47 @@ function inferStatus(u) {
 export function createStore(filePath) {
   let data = [];
 
+  let backupTimer = null;
+
   if (filePath) {
     const resolved = path.resolve(filePath);
+    const backupDir = path.join(path.dirname(resolved), '.backup');
+    const maxBackups = 3;
+
     if (fs.existsSync(resolved)) {
       try {
-        const raw = fs.readFileSync(resolved, 'utf-8');
-        data = JSON.parse(raw);
-        if (!Array.isArray(data)) data = [];
+        const content = fs.readFileSync(resolved, 'utf-8');
+        data = JSON.parse(content);
+        if (!Array.isArray(data)) {
+          data = [];
+        }
       } catch (e) {
         console.error(`[data-store] 读取文件失败: ${e.message}`);
         data = [];
       }
     }
+
+    function runBackup() {
+      if (!fs.existsSync(resolved)) return;
+      if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true });
+      const now = new Date();
+      const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, 13);
+      const backupFile = path.join(backupDir, `data-${timestamp}.json`);
+      try {
+        fs.copyFileSync(resolved, backupFile);
+        const files = fs.readdirSync(backupDir)
+          .filter(f => f.startsWith('data-') && f.endsWith('.json'))
+          .sort()
+          .map(f => path.join(backupDir, f));
+        while (files.length > maxBackups) {
+          fs.unlinkSync(files.shift());
+        }
+      } catch (e) {
+        console.error(`[data-store] 备份失败: ${e.message}`);
+      }
+    }
+
+    backupTimer = setInterval(runBackup, 60 * 60 * 1000);
   }
 
   for (const u of data) {
@@ -32,25 +61,17 @@ export function createStore(filePath) {
   function save() {
     if (!filePath) return;
     const resolved = path.resolve(filePath);
-    try {
-      if (fs.existsSync(resolved)) {
-        const raw = fs.readFileSync(resolved, 'utf-8');
-        const diskData = JSON.parse(raw);
-        if (Array.isArray(diskData)) {
-          const memIds = new Set(data.map(u => u.uniqueId));
-          for (const diskUser of diskData) {
-            if (!memIds.has(diskUser.uniqueId)) {
-              if (!diskUser.status) diskUser.status = inferStatus(diskUser);
-              data.push(diskUser);
-            }
-          }
-        }
-      }
-    } catch (e) { console.error(`[data-store] 合并磁盘数据失败: ${e.message}`); }
     const json = JSON.stringify(data, null, 2);
     fs.writeFileSync(resolved, json, 'utf-8');
   }
 
+  function stopBackup() {
+    if (backupTimer) {
+      clearInterval(backupTimer);
+      backupTimer = null;
+    }
+  }
+
   function getUser(uid) {
     return data.find(u => u.uniqueId === uid);
   }
@@ -234,6 +255,7 @@ export function createStore(filePath) {
     save, getUser, hasUser, addUser,
     getPendingUsers, getProcessedUsers, getAllUsers,
     claimNextJob, commitJob, resetJob, togglePin,
+    stopBackup,
     data,
   };
 }

package/src/watch/server.mjs

@@ -104,9 +104,9 @@ function sendJSON(res, code, data) {
   res.end(JSON.stringify(data));
 }
 
-export function startWatchServer(outputFile, port = 3000) {
+export function startWatchServer(outputFile, port = 3000, existingStore) {
   return new Promise((_resolve, reject) => {
-    const store = createStore(outputFile);
+    const store = existingStore || createStore(outputFile);
 
     const server = http.createServer(async (req, res) => {
       const { path: routePath, params } = parseQuery(req.url);