tt-help-cli-ycl 1.3.4 → 1.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.4",
+  "version": "1.3.6",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/auto.js

@@ -1,17 +1,36 @@
 import { getOrCreatePage } from '../lib/browser/page.js';
 
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
 async function apiPost(url, body) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
+  return withRetry(`POST ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
   });
-  return res.json();
 }
 
 async function apiGet(url) {
-  const res = await fetch(url);
-  return res.json();
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
 }
 
 export async function handleAuto(options) {
@@ -32,44 +51,54 @@ export async function handleAuto(options) {
     maxFollowers: autoMaxFollowers,
   };
 
-  try {
-    await apiGet(`${serverUrl}/api/stats`);
-  } catch {
-    console.error(`无法连接服务端: ${serverUrl}`);
-    console.error('请先启动服务端: tt-help watch -o <数据文件>');
-    process.exit(1);
-  }
+  await apiGet(`${serverUrl}/api/stats`);
 
   if (autoUsernames.length > 0) {
     const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: autoUsernames });
     console.error(`种子用户: ${added} 个新增, ${skipped} 个已存在`);
   }
 
+  console.error(`服务器: ${serverUrl}（断开会自动重连）`);
+
   const { ensureBrowserReady, processUser } = await import('../scraper/auto-core.mjs');
   const browser = await ensureBrowserReady();
 
-  try {
-    const page = await getOrCreatePage(browser);
+  const page = await getOrCreatePage(browser);
 
-    let processedCount = 0;
-    let errorCount = 0;
+  let processedCount = 0;
+  let errorCount = 0;
 
-    while (true) {
-      const job = await apiGet(`${serverUrl}/api/job`);
-      if (!job.hasJob) break;
+  while (true) {
+    const job = await apiGet(`${serverUrl}/api/job`);
+    if (!job.hasJob) break;
 
-      const username = job.user.uniqueId;
-      processedCount++;
-      console.error(`\n[${processedCount}] 处理 @${username}...`);
+    const username = job.user.uniqueId;
+    processedCount++;
+    let proxyRetry = 0;
+
+    while (true) {
+      console.error(`\n[${processedCount}] 处理 @${username}...${proxyRetry > 0 ? ` (代理重试 ${proxyRetry})` : ''}`);
 
       const result = await processUser(page, username, { ...runOptions, browser }, console.error);
 
-      if (result.restricted || result.error) {
-        if (result.error) errorCount++;
+      if (result.restricted) {
         await apiPost(`${serverUrl}/api/job/${username}`, result);
+        break;
+      }
+
+      if (result.error && result.error.includes('代理错误')) {
+        proxyRetry++;
+        console.error(`  [代理错误] ${result.error}，等待 10s 后重试...`);
+        await new Promise(r => setTimeout(r, 10000));
         continue;
       }
 
+      if (result.error) {
+        errorCount++;
+        await apiPost(`${serverUrl}/api/job/${username}`, result);
+        break;
+      }
+
       const payload = {
         userInfo: result.userInfo || {},
         discoveredVideoAuthors: result.discoveredVideoAuthors || [],
@@ -80,15 +109,13 @@ export async function handleAuto(options) {
       };
       await apiPost(`${serverUrl}/api/job/${username}`, payload);
       console.error('  已提交');
+      break;
     }
-
-    const stats = await apiGet(`${serverUrl}/api/stats`);
-    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
-    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
-  } catch (err) {
-    console.error(`自动抓取失败: ${err.message}`);
-    process.exit(1);
-  } finally {
-    await browser.close().catch(() => {});
   }
+
+  const stats = await apiGet(`${serverUrl}/api/stats`);
+  console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+  console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+
+  await browser.close().catch(() => {});
 }

package/src/cli/explore.js

@@ -1,18 +1,37 @@
 import { getOrCreatePage } from '../lib/browser/page.js';
 import { delay, getDelayConfig, setDelayConfig } from '../scraper/modules/page-helpers.mjs';
 
+const MAX_RETRY_WAIT = 5 * 60 * 1000;
+
+async function withRetry(label, fn) {
+  let backoff = 1000;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      console.error(`[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`);
+      await new Promise(r => setTimeout(r, backoff));
+      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
+    }
+  }
+}
+
 async function apiPost(url, body) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
+  return withRetry(`POST ${url}`, async () => {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    return res.json();
   });
-  return res.json();
 }
 
 async function apiGet(url) {
-  const res = await fetch(url);
-  return res.json();
+  return withRetry(`GET ${url}`, async () => {
+    const res = await fetch(url);
+    return res.json();
+  });
 }
 
 export async function handleExplore(options) {
@@ -24,12 +43,7 @@ export async function handleExplore(options) {
 
   setDelayConfig(explorePreset);
 
-  try {
-    await apiGet(`${serverUrl}/api/stats`);
-  } catch {
-    console.error(`无法连接服务端: ${serverUrl}，退出`);
-    process.exit(1);
-  }
+  await apiGet(`${serverUrl}/api/stats`);
 
   if (exploreUsernames && exploreUsernames.length > 0) {
     const { added, skipped } = await apiPost(`${serverUrl}/api/users`, { usernames: exploreUsernames });
@@ -39,24 +53,27 @@ export async function handleExplore(options) {
   console.error(`\n国家筛选: ${exploreLocation}`);
   console.error(`评论: ${exploreMaxComments}, 猜你喜欢: ${exploreMaxGuess}`);
   console.error(`关注/粉丝: ${exploreEnableFollow ? '启用' : '禁用'}`);
+  console.error(`服务器: ${serverUrl}（断开会自动重连）`);
   if (exploreMaxUsers > 0) console.error(`上限: ${exploreMaxUsers} 个用户`);
 
   const { ensureBrowserReady, processExplore } = await import('../scraper/explore-core.mjs');
   const browser = await ensureBrowserReady();
 
-  try {
-    const page = await getOrCreatePage(browser);
+  const page = await getOrCreatePage(browser);
 
-    let processedCount = 0;
-    let errorCount = 0;
+  let processedCount = 0;
+  let errorCount = 0;
 
-    while (true) {
-      const job = await apiGet(`${serverUrl}/api/job`);
-      if (!job.hasJob) break;
+  while (true) {
+    const job = await apiGet(`${serverUrl}/api/job`);
+    if (!job.hasJob) break;
 
-      const username = job.user.uniqueId;
-      processedCount++;
-      console.error(`\n[${processedCount}] 探索 @${username}...`);
+    const username = job.user.uniqueId;
+    processedCount++;
+    let proxyRetry = 0;
+
+    while (true) {
+      console.error(`\n[${processedCount}] 探索 @${username}...${proxyRetry > 0 ? ` (代理重试 ${proxyRetry})` : ''}`);
 
       const { switchMax } = getDelayConfig();
       await delay(switchMax, switchMax * 3);
@@ -73,13 +90,20 @@ export async function handleExplore(options) {
 
       if (result.restricted) {
         await apiPost(`${serverUrl}/api/job/${username}`, { restricted: true, userInfo: result.userInfo || {} });
+        break;
+      }
+
+      if (result.error && result.error.includes('代理错误')) {
+        proxyRetry++;
+        console.error(`  [代理错误] ${result.error}，等待 10s 后重试...`);
+        await new Promise(r => setTimeout(r, 10000));
         continue;
       }
 
       if (result.error) {
         errorCount++;
         await apiPost(`${serverUrl}/api/job/${username}`, { error: result.error });
-        continue;
+        break;
       }
 
       const payload = {
@@ -97,20 +121,18 @@ export async function handleExplore(options) {
       };
       await apiPost(`${serverUrl}/api/job/${username}`, payload);
       console.error('  已提交');
-
-      if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
-        console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
-        break;
-      }
+      break;
     }
 
-    const stats = await apiGet(`${serverUrl}/api/stats`);
-    console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
-    console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
-  } catch (err) {
-    console.error(`探索失败: ${err.message}`);
-    process.exit(1);
-  } finally {
-    await browser.close().catch(() => {});
+    if (exploreMaxUsers > 0 && processedCount >= exploreMaxUsers) {
+      console.error(`\n已达上限 ${exploreMaxUsers} 个用户，停止处理`);
+      break;
+    }
   }
+
+  const stats = await apiGet(`${serverUrl}/api/stats`);
+  console.error(`\n完成: ${processedCount} 个用户处理, ${errorCount} 个出错`);
+  console.error(`  总用户: ${stats.totalUsers}, 已完成: ${stats.processedUsers}, 待处理: ${stats.pendingUsers}, 错误: ${stats.errorUsers}`);
+
+  await browser.close().catch(() => {});
 }

package/src/cli/watch.js

@@ -1,4 +1,5 @@
-import { writeFileSync, existsSync } from 'fs';
+import { existsSync } from 'fs';
+import { createStore } from '../watch/data-store.mjs';
 import { startWatchServer, openBrowser } from '../watch/server.mjs';
 
 export async function handleWatch(options) {
@@ -16,10 +17,12 @@ export async function handleWatch(options) {
     process.exit(1);
   }
 
-  const { server, port } = await startWatchServer(outputFile, watchPort);
+  const store = createStore(outputFile);
+  const { server, port } = await startWatchServer(outputFile, watchPort, store);
   openBrowser(port);
 
   process.once('SIGINT', () => {
+    store.stopBackup();
     server.close();
     process.exit(0);
   });

package/src/lib/browser/page.js

@@ -78,3 +78,10 @@ export async function getOrCreatePage(browser) {
   }
   return page;
 }
+
+export function assertPageUrl(page, expectedPath) {
+  const actual = page.url();
+  if (!actual.includes(expectedPath)) {
+    throw new Error(`[代理错误] 预期访问 ${expectedPath}，实际跳转到了 ${actual}`);
+  }
+}

package/src/scraper/auto-core.mjs

@@ -8,6 +8,7 @@ import {
   retryWithBackoff,
   detectPageError,
   isLoggedIn,
+  assertPageUrl,
 } from './modules/page-helpers.mjs';
 export { ensureBrowserReady };
 import {
@@ -66,6 +67,7 @@ async function processUser(page, username, options, log) {
     await retryWithBackoff(() => page.goto(`https://www.tiktok.com/@${username}`, {
       waitUntil: 'load', timeout: 30000,
     }), { log });
+    assertPageUrl(page, `@${username}`);
     await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
     await delay(1000, 2000);
 

package/src/scraper/core.mjs

@@ -6,6 +6,7 @@ import {
   setDelayConfig,
   getDelayConfig,
   retryWithBackoff,
+  assertPageUrl,
 } from './modules/page-helpers.mjs';
 import { extractCommentAuthors } from './modules/comment-extractor.mjs';
 import { extractGuessVideos } from './modules/guess-extractor.mjs';
@@ -100,6 +101,7 @@ async function runScrape(options) {
   }
 
   await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'load', timeout: 30000 }), { log });
+  assertPageUrl(page, videoUrl.split('/video/')[0]);
   await delay(Math.round(config.switchMax * 0.5), config.switchMax);
   await closeCommentPanel(page);
   await delay(Math.round(config.commentMax * 0.5), config.commentMax);

package/src/scraper/explore-core.mjs

@@ -6,6 +6,7 @@ import {
   retryWithBackoff,
   detectPageError,
   isLoggedIn,
+  assertPageUrl,
 } from './modules/page-helpers.mjs';
 export { ensureBrowserReady };
 import {
@@ -47,6 +48,7 @@ async function processExplore(page, username, options, log) {
     log(`  访问 @${username} 主页...`);
     const homeUrl = `https://www.tiktok.com/@${username}`;
     await retryWithBackoff(() => page.goto(homeUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    assertPageUrl(page, `@${username}`);
     await page.waitForSelector('[class*="DivVideoList"]', { timeout: 10000 }).catch(() => {});
     await delay(1000, 2000);
 
@@ -107,6 +109,7 @@ async function processExplore(page, username, options, log) {
 
     log(`  进入第一个视频: ${videoUrl}`);
     await retryWithBackoff(() => page.goto(videoUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }), { log });
+    assertPageUrl(page, videoUrl.split('/video/')[0]);
     await delay(1500, 2500);
 
     const videoData = await scrapeSingleVideo(page, 0, 0, log, 'NEVER_MATCH');

package/src/scraper/modules/page-helpers.mjs

@@ -12,6 +12,7 @@ import {
   findTikTokPage,
   getOrCreatePage,
   isLoggedIn,
+  assertPageUrl,
 } from '../../lib/browser/page.js';
 import { retryWithBackoff, isRetryableError } from '../../lib/retry.js';
 import {
@@ -34,6 +35,7 @@ export {
   findTikTokPage,
   getOrCreatePage,
   isLoggedIn,
+  assertPageUrl,
   retryWithBackoff,
   isRetryableError,
   extractUserSection,

package/src/watch/data-store.mjs

@@ -11,18 +11,47 @@ function inferStatus(u) {
 export function createStore(filePath) {
   let data = [];
 
+  let backupTimer = null;
+
   if (filePath) {
     const resolved = path.resolve(filePath);
+    const backupDir = path.join(path.dirname(resolved), '.backup');
+    const maxBackups = 3;
+
     if (fs.existsSync(resolved)) {
       try {
-        const raw = fs.readFileSync(resolved, 'utf-8');
-        data = JSON.parse(raw);
-        if (!Array.isArray(data)) data = [];
+        const content = fs.readFileSync(resolved, 'utf-8');
+        data = JSON.parse(content);
+        if (!Array.isArray(data)) {
+          data = [];
+        }
       } catch (e) {
         console.error(`[data-store] 读取文件失败: ${e.message}`);
         data = [];
       }
     }
+
+    function runBackup() {
+      if (!fs.existsSync(resolved)) return;
+      if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true });
+      const now = new Date();
+      const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, 13);
+      const backupFile = path.join(backupDir, `data-${timestamp}.json`);
+      try {
+        fs.copyFileSync(resolved, backupFile);
+        const files = fs.readdirSync(backupDir)
+          .filter(f => f.startsWith('data-') && f.endsWith('.json'))
+          .sort()
+          .map(f => path.join(backupDir, f));
+        while (files.length > maxBackups) {
+          fs.unlinkSync(files.shift());
+        }
+      } catch (e) {
+        console.error(`[data-store] 备份失败: ${e.message}`);
+      }
+    }
+
+    backupTimer = setInterval(runBackup, 60 * 60 * 1000);
   }
 
   for (const u of data) {
@@ -32,25 +61,17 @@ export function createStore(filePath) {
   function save() {
     if (!filePath) return;
     const resolved = path.resolve(filePath);
-    try {
-      if (fs.existsSync(resolved)) {
-        const raw = fs.readFileSync(resolved, 'utf-8');
-        const diskData = JSON.parse(raw);
-        if (Array.isArray(diskData)) {
-          const memIds = new Set(data.map(u => u.uniqueId));
-          for (const diskUser of diskData) {
-            if (!memIds.has(diskUser.uniqueId)) {
-              if (!diskUser.status) diskUser.status = inferStatus(diskUser);
-              data.push(diskUser);
-            }
-          }
-        }
-      }
-    } catch (e) { console.error(`[data-store] 合并磁盘数据失败: ${e.message}`); }
     const json = JSON.stringify(data, null, 2);
     fs.writeFileSync(resolved, json, 'utf-8');
   }
 
+  function stopBackup() {
+    if (backupTimer) {
+      clearInterval(backupTimer);
+      backupTimer = null;
+    }
+  }
+
   function getUser(uid) {
     return data.find(u => u.uniqueId === uid);
   }
@@ -234,6 +255,7 @@ export function createStore(filePath) {
     save, getUser, hasUser, addUser,
     getPendingUsers, getProcessedUsers, getAllUsers,
     claimNextJob, commitJob, resetJob, togglePin,
+    stopBackup,
     data,
   };
 }

package/src/watch/public/index.html

@@ -11,7 +11,7 @@
   .header h1 { font-size: 18px; color: #fe2c55; }
   .header .meta { font-size: 12px; color: #888; }
   .header .status { font-size: 12px; color: #4ade80; }
-  .stats { display: grid; grid-template-columns: repeat(6, 1fr); gap: 12px; margin-bottom: 16px; }
+  .stats { display: grid; grid-template-columns: repeat(7, 1fr); gap: 12px; margin-bottom: 16px; }
   .stat-card { background: #1a1a24; border-radius: 8px; padding: 16px; text-align: center; }
   .stat-card .label { font-size: 12px; color: #888; margin-bottom: 8px; }
   .stat-card .value { font-size: 28px; font-weight: 700; }
@@ -96,7 +96,8 @@
   <div class="stat-card"><div class="label">处理中</div><div class="value total" id="statProcessing">0</div></div>
   <div class="stat-card"><div class="label">已完成</div><div class="value done" id="statDone">0</div></div>
   <div class="stat-card"><div class="label">待处理</div><div class="value pending" id="statPending">0</div></div>
-  <div class="stat-card"><div class="label">错误/受限</div><div class="value error" id="statError">0</div></div>
+  <div class="stat-card"><div class="label">错误</div><div class="value error" id="statError">0</div></div>
+  <div class="stat-card"><div class="label">受限</div><div class="value error" id="statRestricted">0</div></div>
   <div class="stat-card clickable" id="statTargetCard"><div class="label">目标用户(ES商家)</div><div class="value target" id="statTarget">0</div></div>
 </div>
 <div class="charts">
@@ -125,6 +126,7 @@
     <button data-filter="done" onclick="setFilter('done')">已处理</button>
     <button data-filter="error" onclick="setFilter('error')">错误</button>
     <button data-filter="restricted" onclick="setFilter('restricted')">受限</button>
+    <button data-filter="target" onclick="setFilter('target')" style="background:#7c3aed;color:#fff">目标用户</button>
   </div>
   <div class="table-scroll">
     <table>
@@ -156,7 +158,11 @@ async function fetchStats() {
 async function fetchUsers() {
   try {
     const params = new URLSearchParams();
-    if (currentFilter !== 'all') params.set('status', currentFilter);
+    if (currentFilter === 'target') {
+      params.set('target', '1');
+    } else if (currentFilter !== 'all') {
+      params.set('status', currentFilter);
+    }
     const search = document.getElementById('searchInput').value.trim();
     if (search) params.set('search', search);
     params.set('limit', '200');
@@ -187,7 +193,8 @@ function renderStats() {
   flashEl('statProcessing', d.processingUsers || 0);
   flashEl('statDone', d.processedUsers);
   flashEl('statPending', d.pendingUsers);
-  flashEl('statError', d.restrictedUsers + d.errorUsers);
+  flashEl('statError', d.errorUsers);
+  flashEl('statRestricted', d.restrictedUsers);
   flashEl('statTarget', d.targetUsers);
   document.getElementById('lastUpdate').textContent = '\u66f4\u65b0\u4e8e ' + new Date().toLocaleTimeString();
   document.getElementById('fileMeta').textContent = (d.processingUsers || 0) + ' \u5904\u7406\u4e2d, ' + d.totalUsers + ' \u4e2a\u7528\u6237';
@@ -431,8 +438,20 @@ document.getElementById('statTargetCard').addEventListener('click', async () =>
     const data = await res.json();
     if (!data.users.length) { showToast('暂无目标用户', true); return; }
     const text = data.users.map(u => '@' + u.uniqueId).join(', ');
-    await navigator.clipboard.writeText(text);
-    showToast(data.users.length + ' 个目标用户 ID 已复制到剪贴板');
+    if (navigator.clipboard && navigator.clipboard.writeText) {
+      await navigator.clipboard.writeText(text);
+      showToast(data.users.length + ' 个目标用户 ID 已复制到剪贴板');
+    } else {
+      const ta = document.createElement('textarea');
+      ta.value = text;
+      ta.style.position = 'fixed';
+      ta.style.left = '-9999px';
+      document.body.appendChild(ta);
+      ta.select();
+      document.execCommand('copy');
+      document.body.removeChild(ta);
+      showToast(data.users.length + ' 个目标用户 ID 已复制到剪贴板');
+    }
   } catch (e) {
     showToast('获取失败: ' + e.message, true);
   }

package/src/watch/server.mjs

@@ -104,9 +104,9 @@ function sendJSON(res, code, data) {
   res.end(JSON.stringify(data));
 }
 
-export function startWatchServer(outputFile, port = 3000) {
+export function startWatchServer(outputFile, port = 3000, existingStore) {
   return new Promise((_resolve, reject) => {
-    const store = createStore(outputFile);
+    const store = existingStore || createStore(outputFile);
 
     const server = http.createServer(async (req, res) => {
       const { path: routePath, params } = parseQuery(req.url);
@@ -211,6 +211,11 @@ export function startWatchServer(outputFile, port = 3000) {
         if (params.status && params.status !== 'all') {
           filtered = filtered.filter(u => u.status === params.status);
         }
+        if (params.target === '1') {
+          filtered = filtered.filter(u =>
+            u.ttSeller && u.verified === false && u.locationCreated === 'ES'
+          );
+        }
         if (params.search) {
           const s = params.search.toLowerCase();
           filtered = filtered.filter(u =>