tt-help-cli-ycl 1.3.83 → 1.3.85

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tt-help-cli-ycl",
-  "version": "1.3.83",
+  "version": "1.3.85",
   "description": "TikTok user & video data scraper - extract ttSeller, verified, locationCreated from HTML source",
   "type": "module",
   "bin": {

package/scripts/test-refill-order.mjs

@@ -0,0 +1,218 @@
+#!/usr/bin/env node
+/**
+ * 冒烟测试：验证 refillJobsFromRaw 按 video_count ASC 排序
+ *
+ * 测试场景：
+ * 1. raw_jobs 中插入不同 video_count 的测试数据
+ * 2. 调用 refillJobsFromRaw 移动数据
+ * 3. 验证 jobs 中数据的顺序是 video_count 从小到大
+ */
+
+import Database from "better-sqlite3";
+import path from "path";
+import fs from "fs";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const dbPath = path.join(__dirname, "../test-refill-order.db");
+
+// 清理旧数据库
+if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
+
+const db = new Database(dbPath);
+
+// 建表
+db.exec(`
+  CREATE TABLE raw_jobs (
+    unique_id TEXT PRIMARY KEY,
+    nickname TEXT,
+    video_count INTEGER,
+    follower_count INTEGER,
+    following_count INTEGER,
+    guessed_location TEXT,
+    sources TEXT,
+    pinned INTEGER DEFAULT 0,
+    tt_seller INTEGER DEFAULT 0,
+    verified INTEGER DEFAULT 0,
+    comment_count INTEGER DEFAULT 0,
+    location_created TEXT,
+    confirmed_location TEXT,
+    heart_count INTEGER DEFAULT 0,
+    created_at INTEGER,
+    updated_at INTEGER,
+    region TEXT,
+    signature TEXT,
+    bio_link TEXT,
+    sec_uid TEXT,
+    status_code INTEGER,
+    latest_video_time INTEGER
+  );
+
+  CREATE TABLE jobs (
+    unique_id TEXT PRIMARY KEY,
+    nickname TEXT,
+    status TEXT DEFAULT 'pending',
+    video_count INTEGER,
+    follower_count INTEGER,
+    following_count INTEGER,
+    guessed_location TEXT,
+    sources TEXT,
+    pinned INTEGER DEFAULT 0,
+    tt_seller INTEGER DEFAULT 0,
+    verified INTEGER DEFAULT 0,
+    comment_count INTEGER DEFAULT 0,
+    location_created TEXT,
+    confirmed_location TEXT,
+    heart_count INTEGER DEFAULT 0,
+    created_at INTEGER,
+    updated_at INTEGER,
+    region TEXT,
+    signature TEXT,
+    bio_link TEXT,
+    sec_uid TEXT,
+    status_code INTEGER,
+    latest_video_time INTEGER,
+    claimed_at INTEGER,
+    claimed_by TEXT
+  );
+`);
+
+// 插入测试数据：video_count 分别为 100, 5, 50, 1, 200
+const testData = [
+  {
+    unique_id: "user_a",
+    video_count: 100,
+    follower_count: 500,
+    following_count: 100,
+    guessed_location: "DE",
+  },
+  {
+    unique_id: "user_b",
+    video_count: 5,
+    follower_count: 200,
+    following_count: 50,
+    guessed_location: "DE",
+  },
+  {
+    unique_id: "user_c",
+    video_count: 50,
+    follower_count: 300,
+    following_count: 80,
+    guessed_location: "DE",
+  },
+  {
+    unique_id: "user_d",
+    video_count: 1,
+    follower_count: 100,
+    following_count: 30,
+    guessed_location: "DE",
+  },
+  {
+    unique_id: "user_e",
+    video_count: 200,
+    follower_count: 1000,
+    following_count: 200,
+    guessed_location: "DE",
+  },
+  {
+    unique_id: "user_f",
+    video_count: null,
+    follower_count: 150,
+    following_count: 40,
+    guessed_location: "DE",
+  },
+];
+
+const insertStmt = db.prepare(`
+  INSERT INTO raw_jobs (unique_id, nickname, video_count, follower_count, following_count, guessed_location, created_at, updated_at)
+  VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+`);
+
+const now = Date.now();
+testData.forEach((row, i) => {
+  insertStmt.run(
+    row.unique_id,
+    `Nickname ${row.unique_id}`,
+    row.video_count,
+    row.follower_count,
+    row.following_count,
+    row.guessed_location,
+    now - i * 1000,
+    now,
+  );
+});
+
+console.error("=== 插入测试数据 ===");
+testData.forEach((row) => {
+  console.error(`  ${row.unique_id}: video_count=${row.video_count}`);
+});
+
+// 动态加载 createStore
+const { createStore } = await import("../src/watch/data-store.js");
+
+const store = createStore(dbPath);
+
+// 调用 refillJobsFromRaw（不启用 LLM）
+console.error("\n=== 调用 refillJobsFromRaw ===");
+const result = store.refillJobsFromRaw(["DE"], 500, { llmScore: false });
+console.error(`  移动了 ${result.moved} 条记录`);
+
+// 验证 jobs 中的数据顺序
+const jobs = db
+  .prepare(`SELECT unique_id, video_count FROM jobs ORDER BY rowid ASC`)
+  .all();
+
+console.error("\n=== jobs 表中的数据顺序（按插入顺序 rowid）===");
+jobs.forEach((row, i) => {
+  console.error(`  ${i + 1}. ${row.unique_id}: video_count=${row.video_count}`);
+});
+
+// 验证排序：video_count 应该是从小到大
+let orderCorrect = true;
+for (let i = 1; i < jobs.length; i++) {
+  const prev = jobs[i - 1].video_count ?? 0;
+  const curr = jobs[i].video_count ?? 0;
+  if (prev > curr) {
+    orderCorrect = false;
+    console.error(
+      `\n  ❌ 顺序错误: 第 ${i} 条 (${prev}) > 第 ${i + 1} 条 (${curr})`,
+    );
+  }
+}
+
+// 预期顺序：user_d(1), user_b(5), user_f(null→0), user_c(50), user_a(100), user_e(200)
+// 注意：null 会被 COALESCE 为 0，所以 user_f 应该在 user_d 和 user_b 之间
+const expectedOrder = [
+  "user_d",
+  "user_f",
+  "user_b",
+  "user_c",
+  "user_a",
+  "user_e",
+];
+const actualOrder = jobs.map((j) => j.unique_id);
+
+console.error("\n=== 顺序验证 ===");
+console.error(`  预期: ${expectedOrder.join(", ")}`);
+console.error(`  实际: ${actualOrder.join(", ")}`);
+
+if (JSON.stringify(expectedOrder) === JSON.stringify(actualOrder)) {
+  console.error(
+    "\n✅ 冒烟测试通过：refillJobsFromRaw 按 video_count ASC 排序正确",
+  );
+} else {
+  console.error("\n⚠️  顺序与预期不完全一致，但检查是否为升序...");
+  if (orderCorrect) {
+    console.error(
+      "✅ 数据是升序排列的（null 和 0 的排序可能因 SQLite 版本略有差异）",
+    );
+  } else {
+    console.error("❌ 冒烟测试失败：数据未按 video_count 升序排列");
+    process.exitCode = 1;
+  }
+}
+
+// 清理
+db.close();
+fs.unlinkSync(dbPath);
+console.error(`\n已清理测试数据库: ${dbPath}`);

package/src/cli/attach.js

@@ -1,9 +1,9 @@
 import { TikTokScraper } from "../lib/tiktok-scraper.mjs";
 import { CDNBlockedError } from "../lib/parse-ssr.mjs";
 import { proxy as configuredProxy } from "../lib/constants.js";
+import { createApiClient } from "../lib/api-client.js";
 import v8 from "node:v8";
 
-const MAX_RETRY_WAIT = 5 * 60 * 1000;
 const HEAP_RESTART_RATIO = 0.72;
 const MAX_TASK_BATCHES_BEFORE_RESTART = 200;
 
@@ -17,39 +17,6 @@ function attachLog(message = "") {
   console.error(`[${formatNow()}] ${message}`);
 }
 
-async function withRetry(label, fn) {
-  let backoff = 1000;
-  while (true) {
-    try {
-      return await fn();
-    } catch (err) {
-      attachLog(
-        `[连接] ${label} 失败: ${err.message}，${backoff / 1000}秒后重试...`,
-      );
-      await new Promise((r) => setTimeout(r, backoff));
-      if (backoff < MAX_RETRY_WAIT) backoff *= 2;
-    }
-  }
-}
-
-async function apiGet(url) {
-  return withRetry(`GET ${url}`, async () => {
-    const res = await fetch(url);
-    return res.json();
-  });
-}
-
-async function apiPost(url, body) {
-  return withRetry(`POST ${url}`, async () => {
-    const res = await fetch(url, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify(body),
-    });
-    return res.json();
-  });
-}
-
 function isBrowserClosedError(err) {
   if (!err) return false;
   const msg = err.message || err.toString() || "";
@@ -143,6 +110,8 @@ export async function handleAttach(options) {
     `[Attach] 并行数: ${attachParallel}, 空闲间隔: ${attachInterval}秒, 服务端: ${serverUrl}${countryStr}`,
   );
 
+  const { apiGet, apiPost } = createApiClient({ log: false });
+
   const scraper = new TikTokScraper({
     poolSize: attachPoolSize || 3,
     proxyServer: effectiveProxy || null,

package/src/cli/auto.js

@@ -6,6 +6,7 @@ import {
 import { userId as configuredUserId, saveUserId } from "../lib/constants.js";
 import { getMacOrUuid } from "../lib/mac-or-uuid.js";
 import { ensureBrowserReady as ensureBrowserReadyCDP } from "../lib/browser/cdp.js";
+import { createApiClient } from "../lib/api-client.js";
 
 const MAX_RETRY_WAIT = 5 * 60 * 1000;
 
@@ -24,24 +25,6 @@ async function withRetry(label, fn) {
   }
 }
 
-async function apiPost(url, body) {
-  return withRetry(`POST ${url}`, async () => {
-    const res = await fetch(url, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify(body),
-    });
-    return res.json();
-  });
-}
-
-async function apiGet(url) {
-  return withRetry(`GET ${url}`, async () => {
-    const res = await fetch(url);
-    return res.json();
-  });
-}
-
 export async function handleAuto(options) {
   const {
     autoUsernames,
@@ -87,6 +70,8 @@ export async function handleAuto(options) {
       console.error(`[初始化] 未检测到本地用户编号，已生成并使用: ${userId}`);
     }
 
+    const { apiGet, apiPost } = createApiClient();
+
     const runOptions = {
       collectMax: autoCollectMax,
       scrapeDepth: autoScrapeDepth,

package/src/cli/comments.js

@@ -7,6 +7,7 @@ import {
   isLocationInList,
   normalizeLocation,
 } from "../lib/target-locations.js";
+import { createApiClient } from "../lib/api-client.js";
 
 async function waitForPageReady(page, timeout = 30000) {
   const startTime = Date.now();
@@ -32,64 +33,7 @@ async function safeEvaluate(page, fn) {
   }
 }
 
-async function withRetry(label, fn, maxRetries = 3) {
-  let backoff = 2000;
-  for (let i = 0; i < maxRetries; i++) {
-    try {
-      return await fn();
-    } catch (err) {
-      if (i < maxRetries - 1) {
-        console.error(
-          `  [${label}] 失败: ${err.message}，${backoff / 1000}s 后重试...`,
-        );
-        await new Promise((r) => setTimeout(r, backoff));
-        backoff *= 2;
-      } else {
-        throw err;
-      }
-    }
-  }
-}
-
-async function apiPost(url, body) {
-  return withRetry(`POST ${url}`, async () => {
-    const res = await fetch(url, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify(body),
-    });
-    if (!res.ok) {
-      const errText = await res.text();
-      throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
-    }
-    return res.json();
-  });
-}
 
-async function apiPut(url) {
-  return withRetry(`PUT ${url}`, async () => {
-    const res = await fetch(url, {
-      method: "PUT",
-      headers: { "Content-Type": "application/json" },
-    });
-    if (!res.ok) {
-      const errText = await res.text();
-      throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
-    }
-    return res.json();
-  });
-}
-
-async function apiGet(url) {
-  return withRetry(`GET ${url}`, async () => {
-    const res = await fetch(url);
-    if (!res.ok) {
-      const errText = await res.text();
-      throw new Error(`HTTP ${res.status}: ${errText.substring(0, 200)}`);
-    }
-    return res.json();
-  });
-}
 
 function isBrowserClosedError(err) {
   if (!err) return false;
@@ -110,6 +54,12 @@ function isBrowserClosedError(err) {
  */
 async function runAutoMode(options) {
   const { serverUrl, parallel, interval, maxComments } = options;
+  const { apiGet, apiPost, apiPut } = createApiClient({
+    checkStatus: true,
+    maxRetries: 2,
+    backoff: 2000,
+    log: true,
+  });
   const actualParallel = Math.max(1, parallel || 1);
   const actualInterval = interval || 10;
   const actualMaxComments = maxComments || 200;
@@ -478,6 +428,12 @@ export async function handleComments(options) {
 
       const guessedLocation = normalizeLocation(videoInfo?.locationCreated);
       const serverUrl = commentsServer || defaultServer;
+      const { apiPost } = createApiClient({
+        checkStatus: true,
+        maxRetries: 2,
+        backoff: 2000,
+        log: true,
+      });
 
       if (
         guessedLocation &&