tt-help-cli-ycl 1.3.78 → 1.3.79

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.78",
+  "version": "1.3.79",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/src/cli/attach.js

@@ -83,6 +83,7 @@ export async function handleAttach(options) {
     serverUrl,
     attachCountries,
     customProxy,
+    attachPoolSize,
     showHelp,
   } = options;
   const effectiveProxy = customProxy || configuredProxy;
@@ -110,6 +111,9 @@ export async function handleAttach(options) {
     attachLog(
       "  --proxy <代理地址>    HTTP 代理地址（如 http://127.0.0.1:7890），不配置则从 ~/.tt-help.json 读取",
     );
+    attachLog(
+      "  --size <N>            浏览器页面池大小（默认: 3，不建议超过 3）",
+    );
     attachLog("");
     attachLog("说明:");
     attachLog(
@@ -117,12 +121,16 @@ export async function handleAttach(options) {
     );
     attachLog("  抓取完成后通过 POST /api/user-info-batch 批量回传结果");
     attachLog("  浏览器崩溃时自动重启，支持长时间无人值守运行");
+    attachLog(
+      "  --size 控制单个浏览器内并发的 page 数，增大可提升吞吐但可能触发 TikTok 限流",
+    );
     attachLog("");
     attachLog("示例:");
     attachLog("  tt-help attach");
     attachLog("  tt-help attach -p 5 -i 10");
     attachLog("  tt-help attach -p 3 -i 5 -s http://127.0.0.1:3001");
     attachLog("  tt-help attach -c PL,DE,FR -p 5");
+    attachLog("  tt-help attach -p 5 --size 5");
     return;
   }
 
@@ -135,6 +143,7 @@ export async function handleAttach(options) {
   );
 
   const scraper = new TikTokScraper({
+    poolSize: attachPoolSize || 3,
     proxyServer: effectiveProxy || null,
   });
   const shutdown = async (signal) => {

package/src/lib/args.js

@@ -534,6 +534,7 @@ function parseAttachArgs(args) {
   let serverUrl = defaultServer;
   let countries = [];
   let customProxy = null;
+  let poolSize = 3;
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -550,6 +551,8 @@ function parseAttachArgs(args) {
         .filter(Boolean);
     } else if (arg === "--proxy") {
       customProxy = args[++i];
+    } else if (arg === "--size") {
+      poolSize = parseInt(args[++i], 10) || 3;
     }
   }
 
@@ -560,6 +563,7 @@ function parseAttachArgs(args) {
     serverUrl,
     attachCountries: countries,
     customProxy,
+    attachPoolSize: poolSize,
     urls: [],
     outputFormat: "json",
     exploreCount: 0,

package/src/watch/data-store.js

@@ -143,6 +143,66 @@ function initUserDb(filePath) {
   if (!existingJobColumns.has("bio_link")) {
     db.exec(`ALTER TABLE jobs ADD COLUMN bio_link TEXT`);
   }
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS jobs_base (
+      unique_id TEXT PRIMARY KEY,
+      nickname TEXT,
+      status TEXT DEFAULT 'pending',
+      sources TEXT,
+      claimed_by TEXT,
+      claimed_at INTEGER,
+      error TEXT,
+      pinned INTEGER DEFAULT 0,
+      no_video INTEGER DEFAULT 0,
+      restricted INTEGER DEFAULT 0,
+      user_update_count INTEGER DEFAULT 0,
+      tt_seller INTEGER,
+      verified INTEGER,
+      video_count INTEGER DEFAULT 0,
+      comment_count INTEGER DEFAULT 0,
+      guessed_location TEXT,
+      location_created TEXT,
+      confirmed_location TEXT,
+      modified_at INTEGER,
+      follower_count INTEGER DEFAULT 0,
+      following_count INTEGER DEFAULT 0,
+      heart_count INTEGER DEFAULT 0,
+      refresh_time INTEGER,
+      processed INTEGER DEFAULT 0,
+      processed_at INTEGER,
+      created_at INTEGER,
+      updated_at INTEGER,
+      region TEXT,
+      signature TEXT,
+      sec_uid TEXT,
+      status_code INTEGER,
+      latest_video_time INTEGER,
+      bio_link TEXT
+    )
+  `);
+
+  // 迁移：为已存在的 jobs_base 表补全列
+  const existingJobBaseColumns = new Set(
+    db
+      .prepare("PRAGMA table_info(jobs_base)")
+      .all()
+      .map((c) => c.name),
+  );
+  if (!existingJobBaseColumns.has("status_code")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN status_code INTEGER`);
+  }
+  if (!existingJobBaseColumns.has("latest_video_time")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN latest_video_time INTEGER`);
+  }
+  if (!existingJobBaseColumns.has("confirmed_location")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN confirmed_location TEXT`);
+  }
+  if (!existingJobBaseColumns.has("modified_at")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN modified_at INTEGER`);
+  }
+  if (!existingJobBaseColumns.has("bio_link")) {
+    db.exec(`ALTER TABLE jobs_base ADD COLUMN bio_link TEXT`);
+  }
   db.exec(`
     CREATE TABLE IF NOT EXISTS raw_jobs (
       unique_id TEXT PRIMARY KEY,
@@ -635,11 +695,16 @@ function getDashboardStatsFromDb(targetLocations = []) {
         .all(...targetLocations)
     : [];
 
+  const jobsBaseCount = db
+    .prepare("SELECT COUNT(*) as total FROM jobs_base")
+    .get().total;
+
   return {
     totalUsers: aggregateRow.total,
     rawJobs: getRawJobsCount(),
     dbTotalUsers: getUserDbCount(),
     jobsTotal: aggregateRow.total,
+    jobsBaseTotal: jobsBaseCount,
     jobsPending: aggregateRow.pending,
     processedUsers: aggregateRow.done,
     pendingUsers: aggregateRow.pending,
@@ -1730,6 +1795,91 @@ function inferStatus(u) {
   return "pending";
 }
 
+function addJobBaseToDb(user) {
+  if (!db) return;
+  const now = Date.now();
+  db.prepare(
+    `
+    INSERT OR IGNORE INTO jobs_base (
+      unique_id,
+      nickname,
+      status,
+      sources,
+      claimed_by,
+      claimed_at,
+      error,
+      pinned,
+      no_video,
+      restricted,
+      user_update_count,
+      tt_seller,
+      verified,
+      video_count,
+      comment_count,
+      guessed_location,
+      location_created,
+      follower_count,
+      following_count,
+      heart_count,
+      refresh_time,
+      processed,
+      processed_at,
+      created_at,
+      updated_at,
+      region,
+      signature,
+      bio_link,
+      sec_uid
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `,
+  ).run(
+    user.uniqueId,
+    user.nickname || null,
+    user.status || inferStatus(user),
+    JSON.stringify(
+      Array.isArray(user.sources) ? [...new Set(user.sources)] : [],
+    ),
+    user.claimedBy || null,
+    user.claimedAt || null,
+    user.error || null,
+    user.pinned ? 1 : 0,
+    user.noVideo ? 1 : 0,
+    user.restricted ? 1 : 0,
+    user.userUpdateCount || 0,
+    user.ttSeller === undefined ||
+      user.ttSeller === null ||
+      user.ttSeller === ""
+      ? null
+      : user.ttSeller
+        ? 1
+        : 0,
+    user.verified === undefined ||
+      user.verified === null ||
+      user.verified === ""
+      ? null
+      : user.verified
+        ? 1
+        : 0,
+    user.videoCount || 0,
+    user.commentCount || 0,
+    user.guessedLocation || null,
+    user.locationCreated || null,
+    user.followerCount || 0,
+    user.followingCount || 0,
+    user.heartCount || 0,
+    user.refreshTime || null,
+    user.processed ? 1 : 0,
+    user.processedAt || null,
+    user.createdAt || now,
+    user.updatedAt || now,
+    user.region || null,
+    user.signature || null,
+    user.bioLink?.link || user.bioLink?.url || user.bioLink || null,
+    user.secUid || null,
+  );
+}
+
 function addJob(user) {
   if (!db) {
     addUserToDb(user);
@@ -1951,9 +2101,9 @@ export function createStore(filePath) {
   function addUser(user, append) {
     const memoryIdx = uidIndex.get(user.uniqueId);
     if (db && memoryIdx === undefined) {
-      const existingJob = getJobRow(user.uniqueId);
-      if (existingJob) {
-        return updateJobInfo(user.uniqueId, user, false);
+      // 用 users 表判重（所有发现过的用户合集），而不是 jobs 表
+      if (hasUserInDb(user.uniqueId)) {
+        return;
       }
       addJob(user);
       return;
@@ -2802,7 +2952,8 @@ export function createStore(filePath) {
     const newUsers = [];
     for (const d of unique) {
       if (!hasUser(d.uniqueId)) {
-        addJob(d);
+        addUserToDb(d);
+        addJobBaseToDb(d);
         newUsers.push(d.uniqueId);
       }
     }
@@ -3423,37 +3574,37 @@ export function createStore(filePath) {
       });
       txn(updates);
 
-      // 在事务外执行移动操作（避免嵌套 transaction 问题）
+      // 批量移动非商家用户到 raw_jobs（优化：一次 SQL 搞定）
       if (moveList.length > 0) {
-        const moveTxn = db.transaction((ids) => {
-          ids.forEach((uid) => {
-            db.prepare(
-              `
-              INSERT OR REPLACE INTO raw_jobs (
-                unique_id, nickname, status, sources, claimed_by, claimed_at,
-                error, pinned, no_video, restricted, user_update_count,
-                tt_seller, verified, video_count, comment_count,
-                guessed_location, location_created, confirmed_location, modified_at,
-                follower_count, following_count, heart_count, refresh_time,
-                processed, processed_at, created_at, updated_at,
-                region, signature, bio_link, sec_uid, status_code, latest_video_time
-              )
-              SELECT
-                unique_id, nickname, status, sources, claimed_by, claimed_at,
-                error, pinned, no_video, restricted, user_update_count,
-                tt_seller, verified, video_count, comment_count,
-                guessed_location, location_created, confirmed_location, modified_at,
-                follower_count, following_count, heart_count, refresh_time,
-                processed, processed_at, created_at, updated_at,
-                region, signature, bio_link, sec_uid, status_code, latest_video_time
-              FROM jobs WHERE unique_id = ?
-            `,
-            ).run(uid);
-
-            db.prepare("DELETE FROM jobs WHERE unique_id = ?").run(uid);
-          });
-        });
-        moveTxn(moveList);
+        const placeholders = moveList.map(() => "?").join(",");
+        // 批量 INSERT 到 raw_jobs
+        db.prepare(
+          `
+          INSERT OR REPLACE INTO raw_jobs (
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          )
+          SELECT
+            unique_id, nickname, status, sources, claimed_by, claimed_at,
+            error, pinned, no_video, restricted, user_update_count,
+            tt_seller, verified, video_count, comment_count,
+            guessed_location, location_created, confirmed_location, modified_at,
+            follower_count, following_count, heart_count, refresh_time,
+            processed, processed_at, created_at, updated_at,
+            region, signature, bio_link, sec_uid, status_code, latest_video_time
+          FROM jobs WHERE unique_id IN (${placeholders})
+        `,
+        ).run(...moveList);
+
+        // 批量 DELETE 从 jobs
+        db.prepare(`DELETE FROM jobs WHERE unique_id IN (${placeholders})`).run(
+          ...moveList,
+        );
       }
 
       // 清理内部标记

package/src/watch/public/app.js

@@ -159,6 +159,9 @@ function renderStats() {
   flashEl("statProcessing", d.processingUsers || 0);
   flashEl("statDone", d.processedUsers);
   flashEl("statPending", d.pendingUsers);
+  const statJobsBase = document.getElementById("statJobsBase");
+  if (statJobsBase)
+    statJobsBase.textContent = `jobs_base: ${formatStatNum(d.jobsBaseTotal || 0)}`;
   flashEl("statError", d.errorUsers);
   flashEl("statRestricted", d.restrictedUsers);
   flashEl("statTarget", d.targetUsers, { full: true });

package/src/watch/public/index.html

@@ -35,6 +35,7 @@
       <div class="stat-card clickable pending-card" id="statPendingCard" onclick="navigateToPending()">
         <div class="label">待处理</div>
         <div class="value pending" id="statPending">0</div>
+        <div class="value-sub" id="statJobsBase">jobs_base: 0</div>
       </div>
       <div class="stat-card">
         <div class="label">错误</div>

package/src/watch/server.js

@@ -217,6 +217,17 @@ export function startWatchServer(dataAnchor, port = 3000, existingStore) {
       }
 
       // 调试接口：直接查询数据库原始数据
+      // 临时 API：执行 VACUUM 优化数据库
+      if (req.method === "POST" && routePath === "/api/db-vacuum") {
+        try {
+          store.rawQuery("VACUUM");
+          sendJSON(res, 200, { message: "VACUUM completed" });
+        } catch (e) {
+          sendJSON(res, 400, { error: e.message });
+        }
+        return;
+      }
+
       if (req.method === "GET" && routePath === "/api/db-query") {
         const sql = params.sql || "SELECT * FROM jobs LIMIT 10";
         const limit = Math.min(parseInt(params.limit) || 100, 1000);