tt-help-cli-ycl 1.3.16 → 1.3.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.16",
+  "version": "1.3.18",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/attach.js

@@ -22,10 +22,11 @@ async function apiGet(url) {
   });
 }
 
-async function apiPut(url, body) {
-  return withRetry(`PUT ${url}`, async () => {
+
+async function apiPost(url, body) {
+  return withRetry(`POST ${url}`, async () => {
     const res = await fetch(url, {
-      method: 'PUT',
+      method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(body),
     });
@@ -111,37 +112,56 @@ export async function handleAttach(options) {
         })
       );
 
-      let successCount = 0;
-      let failCount = 0;
-      let needRestart = false;
-
-      for (const result of results) {
-        if (result.status === 'fulfilled') {
-          const { uniqueId, info, error } = result.value;
-          if (error) {
-            if (isBrowserClosedError(error)) {
-              needRestart = true;
-            }
-            console.error(`  ✗ @${uniqueId} 获取失败: ${error.message || '未知错误'}`);
-            failCount++;
-          } else if (info) {
-            try {
-              await apiPut(`${serverUrl}/api/user-info/${encodeURIComponent(uniqueId)}`, info);
-              console.error(`  ✓ @${uniqueId} 已提交更新`);
-              successCount++;
-            } catch (err) {
-              console.error(`  ✗ @${uniqueId} 提交失败: ${err.message}`);
-              failCount++;
-            }
-          } else {
-            console.error(`  ✗ @${uniqueId} 未获取到用户信息`);
-            failCount++;
-          }
-        } else {
-          console.error(`  ✗ 任务执行异常: ${result.reason?.message || '未知错误'}`);
-          failCount++;
-        }
-      }
+       let successCount = 0;
+       let failCount = 0;
+       let needRestart = false;
+
+       // 收集抓取成功的任务，记录抓取失败的
+       const successTasks = [];
+       for (const result of results) {
+         if (result.status === 'fulfilled') {
+           const { uniqueId, info, error } = result.value;
+           if (error) {
+             if (isBrowserClosedError(error)) {
+               needRestart = true;
+             }
+             console.error(`  ✗ @${uniqueId} 获取失败: ${error.message || '未知错误'}`);
+             failCount++;
+           } else if (info) {
+             successTasks.push({ uniqueId, info });
+           } else {
+             console.error(`  ✗ @${uniqueId} 未获取到用户信息`);
+             failCount++;
+           }
+         } else {
+           console.error(`  ✗ 任务执行异常: ${result.reason?.message || '未知错误'}`);
+           failCount++;
+         }
+       }
+
+       // 批量提交成功的结果
+       if (successTasks.length > 0) {
+         try {
+           const batchRet = await apiPost(`${serverUrl}/api/user-info-batch`, { updates: successTasks });
+           if (batchRet && batchRet.results) {
+             for (const r of batchRet.results) {
+               if (r.ok) {
+                 successCount++;
+                 console.error(`  ✓ @${r.uniqueId} 已提交更新`);
+               } else {
+                 failCount++;
+                 console.error(`  ✗ @${r.uniqueId} 提交失败: ${r.error}`);
+               }
+             }
+           } else {
+             successCount = successTasks.length;
+             console.error(`  ✓ 批量提交完成 (${successTasks.length} 条)`);
+           }
+         } catch (err) {
+           failCount += successTasks.length;
+           console.error(`  ✗ 批量提交失败: ${err.message}`);
+         }
+       }
 
       console.error(`  本批结果: ${successCount} 成功, ${failCount} 失败\n`);
 

package/src/lib/tiktok-scraper.mjs

@@ -99,14 +99,15 @@ export class TikTokScraper {
   async warmWaf() {
     if (this.warmPromise) return this.warmPromise;
     this.warmPromise = (async () => {
+      const page = await this.context.newPage();
       try {
-        const page = this.slots[0].page;
         await page.goto(this.warmUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
         await delay(1500);
         this.lastWarmTime = Date.now();
       } catch (e) {
         console.error(`[warmWaf] failed: ${e.message}`);
       } finally {
+        await page.close();
         this.warmPromise = null;
       }
     })();

package/src/watch/data-store.js

@@ -10,8 +10,53 @@ function inferStatus(u) {
 
 export function createStore(filePath) {
   let data = [];
+  // uniqueId → index 内存索引，O(1) 查找
+  let uidIndex = new Map();
   let clientErrors = new Map();
 
+  // done 用户归档（独立文件，主文件只保留活跃用户）
+  let doneArchive = [];
+  let doneArchivePath = null;
+  let doneUidIndex = new Map();
+  if (filePath) {
+    doneArchivePath = path.resolve(filePath).replace(/\.json$/, '-done.json');
+    if (fs.existsSync(doneArchivePath)) {
+      try {
+        const content = fs.readFileSync(doneArchivePath, 'utf-8');
+        const parsed = JSON.parse(content);
+        if (Array.isArray(parsed)) doneArchive = parsed;
+      } catch (e) {
+        console.error(`[data-store] 读取 done 归档失败: ${e.message}`);
+      }
+    }
+  }
+
+  // stats 缓存
+  let statsCache = null;
+  let statsDirty = true;
+
+  function markStatsDirty() {
+    statsDirty = true;
+  }
+
+  function computeStatsInternal() {
+    const total = data.length;
+    const statusCounts = { pending: 0, processing: 0, done: 0, error: 0, restricted: 0 };
+    for (const u of data) {
+      statusCounts[u.status] = (statusCounts[u.status] || 0) + 1;
+    }
+    statsCache = { total, statusCounts };
+    statsDirty = false;
+    return statsCache;
+  }
+
+  function getStats() {
+    if (statsDirty) {
+      return computeStatsInternal();
+    }
+    return statsCache;
+  }
+
   // 视频存储（独立 JSON 文件）
   let videos = [];
   let videoFilePath = null;
@@ -74,15 +119,64 @@ export function createStore(filePath) {
     backupTimer = setInterval(runBackup, 60 * 60 * 1000);
   }
 
-  for (const u of data) {
+  // 构建索引 + 推断 status
+  for (let i = 0; i < data.length; i++) {
+    const u = data[i];
     if (!u.status) u.status = inferStatus(u);
+    uidIndex.set(u.uniqueId, i);
+  }
+
+  // 构建 done 归档索引
+  for (let i = 0; i < doneArchive.length; i++) {
+    doneUidIndex.set(doneArchive[i].uniqueId, i);
+  }
+
+  function rebuildIndex() {
+    uidIndex = new Map();
+    for (let i = 0; i < data.length; i++) {
+      uidIndex.set(data[i].uniqueId, i);
+    }
   }
 
   function save() {
     if (!filePath) return;
     const resolved = path.resolve(filePath);
-    const json = JSON.stringify(data, null, 2);
-    fs.writeFileSync(resolved, json, 'utf-8');
+
+    // 将 done 用户归档到独立文件
+    if (doneArchivePath && data.length > 1000) {
+      const active = [];
+      const toArchive = [];
+      for (const u of data) {
+        if (u.status === 'done') {
+          toArchive.push(u);
+        } else {
+          active.push(u);
+        }
+      }
+      if (toArchive.length > 0) {
+        doneArchive = doneArchive.concat(toArchive);
+        data = active;
+        rebuildIndex();
+        // 重建 done 归档索引
+        doneUidIndex = new Map();
+        for (let i = 0; i < doneArchive.length; i++) {
+          doneUidIndex.set(doneArchive[i].uniqueId, i);
+        }
+      }
+    }
+
+    const json = JSON.stringify(data);
+    const tmpPath = resolved + '.tmp';
+    fs.writeFileSync(tmpPath, json, 'utf-8');
+    fs.renameSync(tmpPath, resolved);
+
+    // 写 done 归档
+    if (doneArchivePath && doneArchive.length > 0) {
+      const doneJson = JSON.stringify(doneArchive);
+      const doneTmp = doneArchivePath + '.tmp';
+      fs.writeFileSync(doneTmp, doneJson, 'utf-8');
+      fs.renameSync(doneTmp, doneArchivePath);
+    }
   }
 
   function saveVideos() {
@@ -99,15 +193,19 @@ export function createStore(filePath) {
   }
 
   function getUser(uid) {
-    return data.find(u => u.uniqueId === uid);
+    const idx = uidIndex.get(uid);
+    if (idx !== undefined) return data[idx];
+    const doneIdx = doneUidIndex.get(uid);
+    if (doneIdx !== undefined) return doneArchive[doneIdx];
+    return undefined;
   }
 
   function hasUser(uid) {
-    return getUser(uid) !== undefined;
+    return uidIndex.has(uid) || doneUidIndex.has(uid);
   }
 
   function userExists(uid) {
-    return hasUser(uid);
+    return uidIndex.has(uid) || doneUidIndex.has(uid);
   }
 
   function addUser(user, append) {
@@ -122,8 +220,15 @@ export function createStore(filePath) {
     } else {
       if (!user.status) user.status = inferStatus(user);
       if (user.processed) user.processedAt = user.processedAt || Date.now();
-      if (append) data.push(user);
-      else data.unshift(user);
+      if (append) {
+        const idx = data.length;
+        data.push(user);
+        uidIndex.set(user.uniqueId, idx);
+      } else {
+        data.unshift(user);
+        uidIndex.set(user.uniqueId, 0);
+      }
+      markStatsDirty();
     }
   }
 
@@ -132,10 +237,13 @@ export function createStore(filePath) {
   }
 
   function getProcessedUsers() {
-    return data.filter(u => u.status === 'done');
+    return doneArchive.length > 0 ? data.filter(u => u.status === 'done').concat(doneArchive) : data.filter(u => u.status === 'done');
   }
 
   function getAllUsers() {
+    if (doneArchive.length > 0) {
+      return data.concat(doneArchive);
+    }
     return data;
   }
 
@@ -173,6 +281,7 @@ export function createStore(filePath) {
       );
       if (expired) {
         expired.status = 'pending';
+        markStatsDirty();
         delete expired.claimedAt;
         next = expired;
       }
@@ -204,6 +313,7 @@ export function createStore(filePath) {
 
     if (next) {
       next.status = 'processing';
+      markStatsDirty();
       next.claimedAt = now;
       next.claimedBy = userId;
       return { uniqueId: next.uniqueId, nickname: next.nickname, claimedAt: next.claimedAt, claimedBy: userId };
@@ -241,10 +351,19 @@ export function createStore(filePath) {
       }),
     ].filter(u => u.uniqueId);
 
-    const newUsers = [];
+    // 先对 discovered 内部去重，再用 uidIndex 批量判断
+    const seen = new Set();
+    const unique = [];
     for (const d of discovered) {
-      const existing = getUser(d.uniqueId);
-      if (!existing) {
+      if (!seen.has(d.uniqueId)) {
+        seen.add(d.uniqueId);
+        unique.push(d);
+      }
+    }
+
+    const newUsers = [];
+    for (const d of unique) {
+      if (!hasUser(d.uniqueId)) {
         addUser(d, true);
         newUsers.push(d.uniqueId);
       }
@@ -253,6 +372,7 @@ export function createStore(filePath) {
   }
 
   function updateUserFromResult(user, result) {
+    const oldStatus = user.status;
     if (result.restricted) {
       user.status = 'restricted';
       if (result.userInfo) {
@@ -311,6 +431,7 @@ export function createStore(filePath) {
       }
       user.sources = [...new Set([...(user.sources || []), 'processed'])];
     }
+    if (user.status !== oldStatus) markStatsDirty();
   }
 
   function commitJob(uniqueId, result) {
@@ -351,6 +472,7 @@ export function createStore(filePath) {
     const user = getUser(uniqueId);
     if (!user) return { saved: false, error: 'user not found' };
     user.status = 'pending';
+    markStatsDirty();
     delete user.claimedAt;
     delete user.processedAt;
     delete user.processed;
@@ -483,6 +605,28 @@ export function createStore(filePath) {
     return { ok: true, userUpdateCount: user.userUpdateCount };
   }
 
+  function batchUpdateUserInfo(updates) {
+    const results = [];
+    for (const item of updates) {
+      const user = getUser(item.uniqueId);
+      if (!user) {
+        results.push({ uniqueId: item.uniqueId, error: 'user not found' });
+        continue;
+      }
+      for (const key of Object.keys(item.info)) {
+        if (key === 'uniqueId' || key === 'sources') continue;
+        if (item.info[key] !== undefined && item.info[key] !== null && item.info[key] !== '') {
+          user[key] = item.info[key];
+        }
+      }
+      user.userUpdateCount = (user.userUpdateCount || 0) + 1;
+      user.updatedAt = Date.now();
+      results.push({ uniqueId: item.uniqueId, ok: true, userUpdateCount: user.userUpdateCount });
+    }
+    save();
+    return results;
+  }
+
   // 视频登记
   function registerVideos(sourceUser, videoList, locationCreated, ttSeller) {
     if (!videoList || !Array.isArray(videoList) || videoList.length === 0) {
@@ -524,10 +668,10 @@ export function createStore(filePath) {
 
   return {
     save, getUser, hasUser, userExists, addUser,
-    getPendingUsers, getProcessedUsers, getAllUsers,
+    getPendingUsers, getProcessedUsers, getAllUsers, getStats,
     claimNextJob, commitJob, commitNewExplore, resetJob, togglePin,
     getNextRedoJob, commitRedoJob,
-    getPendingUserUpdateTasks, updateUserInfo,
+    getPendingUserUpdateTasks, updateUserInfo, batchUpdateUserInfo,
     reportClientError, deleteClientError, getClientErrors,
     registerVideos, getVideos, getVideoCount,
     stopBackup,

package/src/watch/server.js

@@ -280,6 +280,8 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
 
       if (req.method === 'GET' && routePath === '/api/stats') {
         const stats = computeStats(store.getAllUsers());
+        const quick = store.getStats();
+        if (quick) stats.quickStats = quick;
         sendJSON(res, 200, stats);
         return;
       }
@@ -307,7 +309,23 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
         return;
       }
 
-      const userInfoCommitMatch = routePath.match(/^\/api\/user-info\/([^/]+)$/);
+       if (req.method === 'POST' && routePath === '/api/user-info-batch') {
+          try {
+            const body = await readBody(req);
+            const updates = body.updates || [];
+            const results = store.batchUpdateUserInfo(updates);
+            const okCount = results.filter(r => r.ok).length;
+            const errCount = results.filter(r => r.error).length;
+            const ts = new Date().toISOString().slice(11, 19);
+            console.error(`[JOB ${ts}] USER-INFO-BATCH: ${okCount} ok, ${errCount} error (total=${updates.length})`);
+            sendJSON(res, 200, { results, total: updates.length, ok: okCount, error: errCount });
+          } catch (e) {
+            sendJSON(res, 400, { error: e.message });
+          }
+          return;
+        }
+
+       const userInfoCommitMatch = routePath.match(/^\/api\/user-info\/([^/]+)$/);
       if (req.method === 'PUT' && userInfoCommitMatch) {
         const uniqueId = userInfoCommitMatch[1];
         try {
@@ -467,7 +485,24 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
 
         const limit = parseInt(params.limit) || 50;
         const offset = parseInt(params.offset) || 0;
-        const paged = filtered.slice(offset, offset + limit);
+        let paged = filtered.slice(offset, offset + limit);
+
+        // 字段裁剪：轻量视图只返回必要字段
+        if (params.view === 'light') {
+          paged = paged.map(u => ({
+            uniqueId: u.uniqueId,
+            nickname: u.nickname,
+            status: u.status,
+            sources: u.sources,
+            ttSeller: u.ttSeller,
+            verified: u.verified,
+            followerCount: u.followerCount,
+            locationCreated: u.locationCreated,
+            guessedLocation: u.guessedLocation,
+            pinned: u.pinned,
+            processedAt: u.processedAt,
+          }));
+        }
 
         sendJSON(res, 200, { total: filtered.length, users: paged });
         return;