tt-help-cli-ycl 1.3.16 → 1.3.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.16",
+  "version": "1.3.17",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/lib/tiktok-scraper.mjs

@@ -99,14 +99,15 @@ export class TikTokScraper {
   async warmWaf() {
     if (this.warmPromise) return this.warmPromise;
     this.warmPromise = (async () => {
+      const page = await this.context.newPage();
       try {
-        const page = this.slots[0].page;
         await page.goto(this.warmUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
         await delay(1500);
         this.lastWarmTime = Date.now();
       } catch (e) {
         console.error(`[warmWaf] failed: ${e.message}`);
       } finally {
+        await page.close();
         this.warmPromise = null;
       }
     })();

package/src/watch/data-store.js

@@ -10,8 +10,53 @@ function inferStatus(u) {
 
 export function createStore(filePath) {
   let data = [];
+  // uniqueId → index 内存索引，O(1) 查找
+  let uidIndex = new Map();
   let clientErrors = new Map();
 
+  // done 用户归档（独立文件，主文件只保留活跃用户）
+  let doneArchive = [];
+  let doneArchivePath = null;
+  let doneUidIndex = new Map();
+  if (filePath) {
+    doneArchivePath = path.resolve(filePath).replace(/\.json$/, '-done.json');
+    if (fs.existsSync(doneArchivePath)) {
+      try {
+        const content = fs.readFileSync(doneArchivePath, 'utf-8');
+        const parsed = JSON.parse(content);
+        if (Array.isArray(parsed)) doneArchive = parsed;
+      } catch (e) {
+        console.error(`[data-store] 读取 done 归档失败: ${e.message}`);
+      }
+    }
+  }
+
+  // stats 缓存
+  let statsCache = null;
+  let statsDirty = true;
+
+  function markStatsDirty() {
+    statsDirty = true;
+  }
+
+  function computeStatsInternal() {
+    const total = data.length;
+    const statusCounts = { pending: 0, processing: 0, done: 0, error: 0, restricted: 0 };
+    for (const u of data) {
+      statusCounts[u.status] = (statusCounts[u.status] || 0) + 1;
+    }
+    statsCache = { total, statusCounts };
+    statsDirty = false;
+    return statsCache;
+  }
+
+  function getStats() {
+    if (statsDirty) {
+      return computeStatsInternal();
+    }
+    return statsCache;
+  }
+
   // 视频存储（独立 JSON 文件）
   let videos = [];
   let videoFilePath = null;
@@ -74,15 +119,64 @@ export function createStore(filePath) {
     backupTimer = setInterval(runBackup, 60 * 60 * 1000);
   }
 
-  for (const u of data) {
+  // 构建索引 + 推断 status
+  for (let i = 0; i < data.length; i++) {
+    const u = data[i];
     if (!u.status) u.status = inferStatus(u);
+    uidIndex.set(u.uniqueId, i);
+  }
+
+  // 构建 done 归档索引
+  for (let i = 0; i < doneArchive.length; i++) {
+    doneUidIndex.set(doneArchive[i].uniqueId, i);
+  }
+
+  function rebuildIndex() {
+    uidIndex = new Map();
+    for (let i = 0; i < data.length; i++) {
+      uidIndex.set(data[i].uniqueId, i);
+    }
   }
 
   function save() {
     if (!filePath) return;
     const resolved = path.resolve(filePath);
-    const json = JSON.stringify(data, null, 2);
-    fs.writeFileSync(resolved, json, 'utf-8');
+
+    // 将 done 用户归档到独立文件
+    if (doneArchivePath && data.length > 1000) {
+      const active = [];
+      const toArchive = [];
+      for (const u of data) {
+        if (u.status === 'done') {
+          toArchive.push(u);
+        } else {
+          active.push(u);
+        }
+      }
+      if (toArchive.length > 0) {
+        doneArchive = doneArchive.concat(toArchive);
+        data = active;
+        rebuildIndex();
+        // 重建 done 归档索引
+        doneUidIndex = new Map();
+        for (let i = 0; i < doneArchive.length; i++) {
+          doneUidIndex.set(doneArchive[i].uniqueId, i);
+        }
+      }
+    }
+
+    const json = JSON.stringify(data);
+    const tmpPath = resolved + '.tmp';
+    fs.writeFileSync(tmpPath, json, 'utf-8');
+    fs.renameSync(tmpPath, resolved);
+
+    // 写 done 归档
+    if (doneArchivePath && doneArchive.length > 0) {
+      const doneJson = JSON.stringify(doneArchive);
+      const doneTmp = doneArchivePath + '.tmp';
+      fs.writeFileSync(doneTmp, doneJson, 'utf-8');
+      fs.renameSync(doneTmp, doneArchivePath);
+    }
   }
 
   function saveVideos() {
@@ -99,15 +193,19 @@ export function createStore(filePath) {
   }
 
   function getUser(uid) {
-    return data.find(u => u.uniqueId === uid);
+    const idx = uidIndex.get(uid);
+    if (idx !== undefined) return data[idx];
+    const doneIdx = doneUidIndex.get(uid);
+    if (doneIdx !== undefined) return doneArchive[doneIdx];
+    return undefined;
   }
 
   function hasUser(uid) {
-    return getUser(uid) !== undefined;
+    return uidIndex.has(uid) || doneUidIndex.has(uid);
   }
 
   function userExists(uid) {
-    return hasUser(uid);
+    return uidIndex.has(uid) || doneUidIndex.has(uid);
   }
 
   function addUser(user, append) {
@@ -122,8 +220,15 @@ export function createStore(filePath) {
     } else {
       if (!user.status) user.status = inferStatus(user);
       if (user.processed) user.processedAt = user.processedAt || Date.now();
-      if (append) data.push(user);
-      else data.unshift(user);
+      if (append) {
+        const idx = data.length;
+        data.push(user);
+        uidIndex.set(user.uniqueId, idx);
+      } else {
+        data.unshift(user);
+        uidIndex.set(user.uniqueId, 0);
+      }
+      markStatsDirty();
     }
   }
 
@@ -132,10 +237,13 @@ export function createStore(filePath) {
   }
 
   function getProcessedUsers() {
-    return data.filter(u => u.status === 'done');
+    return doneArchive.length > 0 ? data.filter(u => u.status === 'done').concat(doneArchive) : data.filter(u => u.status === 'done');
   }
 
   function getAllUsers() {
+    if (doneArchive.length > 0) {
+      return data.concat(doneArchive);
+    }
     return data;
   }
 
@@ -173,6 +281,7 @@ export function createStore(filePath) {
       );
       if (expired) {
         expired.status = 'pending';
+        markStatsDirty();
         delete expired.claimedAt;
         next = expired;
       }
@@ -204,6 +313,7 @@ export function createStore(filePath) {
 
     if (next) {
       next.status = 'processing';
+      markStatsDirty();
       next.claimedAt = now;
       next.claimedBy = userId;
       return { uniqueId: next.uniqueId, nickname: next.nickname, claimedAt: next.claimedAt, claimedBy: userId };
@@ -241,10 +351,19 @@ export function createStore(filePath) {
       }),
     ].filter(u => u.uniqueId);
 
-    const newUsers = [];
+    // 先对 discovered 内部去重，再用 uidIndex 批量判断
+    const seen = new Set();
+    const unique = [];
     for (const d of discovered) {
-      const existing = getUser(d.uniqueId);
-      if (!existing) {
+      if (!seen.has(d.uniqueId)) {
+        seen.add(d.uniqueId);
+        unique.push(d);
+      }
+    }
+
+    const newUsers = [];
+    for (const d of unique) {
+      if (!hasUser(d.uniqueId)) {
         addUser(d, true);
         newUsers.push(d.uniqueId);
       }
@@ -253,6 +372,7 @@ export function createStore(filePath) {
   }
 
   function updateUserFromResult(user, result) {
+    const oldStatus = user.status;
     if (result.restricted) {
       user.status = 'restricted';
       if (result.userInfo) {
@@ -311,6 +431,7 @@ export function createStore(filePath) {
       }
       user.sources = [...new Set([...(user.sources || []), 'processed'])];
     }
+    if (user.status !== oldStatus) markStatsDirty();
   }
 
   function commitJob(uniqueId, result) {
@@ -351,6 +472,7 @@ export function createStore(filePath) {
     const user = getUser(uniqueId);
     if (!user) return { saved: false, error: 'user not found' };
     user.status = 'pending';
+    markStatsDirty();
     delete user.claimedAt;
     delete user.processedAt;
     delete user.processed;
@@ -524,7 +646,7 @@ export function createStore(filePath) {
 
   return {
     save, getUser, hasUser, userExists, addUser,
-    getPendingUsers, getProcessedUsers, getAllUsers,
+    getPendingUsers, getProcessedUsers, getAllUsers, getStats,
     claimNextJob, commitJob, commitNewExplore, resetJob, togglePin,
     getNextRedoJob, commitRedoJob,
     getPendingUserUpdateTasks, updateUserInfo,

package/src/watch/server.js

@@ -280,6 +280,8 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
 
       if (req.method === 'GET' && routePath === '/api/stats') {
         const stats = computeStats(store.getAllUsers());
+        const quick = store.getStats();
+        if (quick) stats.quickStats = quick;
         sendJSON(res, 200, stats);
         return;
       }
@@ -467,7 +469,24 @@ export function startWatchServer(outputFile, port = 3000, existingStore) {
 
         const limit = parseInt(params.limit) || 50;
         const offset = parseInt(params.offset) || 0;
-        const paged = filtered.slice(offset, offset + limit);
+        let paged = filtered.slice(offset, offset + limit);
+
+        // 字段裁剪：轻量视图只返回必要字段
+        if (params.view === 'light') {
+          paged = paged.map(u => ({
+            uniqueId: u.uniqueId,
+            nickname: u.nickname,
+            status: u.status,
+            sources: u.sources,
+            ttSeller: u.ttSeller,
+            verified: u.verified,
+            followerCount: u.followerCount,
+            locationCreated: u.locationCreated,
+            guessedLocation: u.guessedLocation,
+            pinned: u.pinned,
+            processedAt: u.processedAt,
+          }));
+        }
 
         sendJSON(res, 200, { total: filtered.length, users: paged });
         return;