tt-help-cli-ycl 1.3.52 → 1.3.57

package/src/lib/tiktok-scraper.mjs

@@ -1,31 +1,21 @@
-import os from "os";
-import path from "path";
-import fs from "fs";
 import { chromium } from "playwright";
 import { detectBrowser } from "./browser/launch.js";
-import { parseUserInfo, parseVideoInfo } from "./parse-ssr.mjs";
+import {
+  parseUserInfo,
+  parseVideoInfo,
+  isRetryableFailure,
+} from "./parse-ssr.mjs";
 
 const DEFAULT_POOL_SIZE = 3;
 const DEFAULT_WAF_TTL = 120000;
 const DEFAULT_WARM_URL = "https://www.tiktok.com/@nike";
 const BROWSER_CLOSE_TIMEOUT = 5000;
 const DEFAULT_MAX_REQUESTS_PER_PAGE = 50;
-const FALLBACK_PROFILE_PORT = 9999;
 
 function delay(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
-function getFallbackProfileDir() {
-  const profile = `p${FALLBACK_PROFILE_PORT}`;
-  return path.join(
-    os.homedir(),
-    "Library",
-    "Application Support",
-    `Microsoft Edge For Testing_${profile}`,
-  );
-}
-
 class PageSlot {
   constructor(page) {
     this.page = page;
@@ -61,49 +51,6 @@ class PromiseQueue {
   }
 }
 
-function createLaunchOptions(executablePath) {
-  return {
-    headless: true,
-    executablePath,
-    handleSIGINT: false,
-    handleSIGTERM: false,
-    handleSIGHUP: false,
-    args: [
-      "--no-sandbox",
-      "--disable-setuid-sandbox",
-      "--disable-dev-shm-usage",
-    ],
-  };
-}
-
-async function initContext(executablePath, poolSize, userDataDir) {
-  let context;
-  let browser = null;
-  const slots = [];
-
-  if (userDataDir) {
-    context = await chromium.launchPersistentContext(
-      userDataDir,
-      createLaunchOptions(executablePath),
-    );
-    const existing = context.pages();
-    if (existing.length > 0) {
-      slots.push(new PageSlot(existing[0]));
-    }
-    for (let i = slots.length; i < poolSize; i++) {
-      slots.push(new PageSlot(await context.newPage()));
-    }
-  } else {
-    browser = await chromium.launch(createLaunchOptions(executablePath));
-    context = await browser.newContext();
-    for (let i = 0; i < poolSize; i++) {
-      slots.push(new PageSlot(await context.newPage()));
-    }
-  }
-
-  return { browser, context, slots };
-}
-
 export class TikTokScraper {
   constructor({
     poolSize = DEFAULT_POOL_SIZE,
@@ -121,11 +68,6 @@ export class TikTokScraper {
     this.slotIdx = 0;
     this.lastWarmTime = 0;
     this.warmPromise = null;
-    // 登录态 pool（init 时直接启动）
-    this.authBrowser = null;
-    this.authContext = null;
-    this.authSlots = [];
-    this.authSlotIdx = 0;
   }
 
   async init() {
@@ -135,73 +77,49 @@ export class TikTokScraper {
         "未找到本地浏览器（Chrome/Edge），请先安装浏览器或执行 npx playwright install",
       );
     }
-    const { browser, context, slots } = await initContext(
+    this.browser = await chromium.launch({
+      headless: true,
       executablePath,
-      this.poolSize,
-      null,
-    );
-    this.browser = browser;
-    this.context = context;
-    this.slots = slots;
-
-    // 启动登录态 pool（1 个 slot）
-    // profile 不存在则跳过，不影响主流程
-    const fallbackDir = getFallbackProfileDir();
-    if (fs.existsSync(fallbackDir)) {
-      const {
-        browser: authBrowser,
-        context: authContext,
-        slots: authSlots,
-      } = await initContext(executablePath, 1, fallbackDir);
-      this.authBrowser = authBrowser;
-      this.authContext = authContext;
-      this.authSlots = authSlots;
-    } else {
-      console.error(
-        `[TikTokScraper] 登录态 profile 不存在 (${fallbackDir})，跳过登录态 pool。请先运行 tt-help open 9999 登录 TikTok`,
-      );
+      handleSIGINT: false,
+      handleSIGTERM: false,
+      handleSIGHUP: false,
+      args: [
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-dev-shm-usage",
+      ],
+    });
+    this.context = await this.browser.newContext();
+    for (let i = 0; i < this.poolSize; i++) {
+      this.slots.push(new PageSlot(await this.context.newPage()));
     }
-
     await this.warmWaf();
   }
 
   async close() {
-    const closeAll = async (browser) => {
-      if (browser) {
-        let closeTimedOut = false;
-        const closePromise = browser.close().catch((error) => {
-          console.error(
-            `[TikTokScraper] browser.close() failed: ${error.message}`,
-          );
-        });
-        await Promise.race([
-          closePromise,
-          delay(BROWSER_CLOSE_TIMEOUT).then(() => {
-            closeTimedOut = true;
-          }),
-        ]);
-        if (closeTimedOut) {
-          console.error(
-            `[TikTokScraper] browser.close() 超时 ${BROWSER_CLOSE_TIMEOUT}ms，跳过等待并继续退出`,
-          );
-        }
+    if (this.browser) {
+      const browser = this.browser;
+      let closeTimedOut = false;
+      const closePromise = browser.close().catch((error) => {
+        console.error(
+          `[TikTokScraper] browser.close() failed: ${error.message}`,
+        );
+      });
+      await Promise.race([
+        closePromise,
+        delay(BROWSER_CLOSE_TIMEOUT).then(() => {
+          closeTimedOut = true;
+        }),
+      ]);
+      if (closeTimedOut) {
+        console.error(
+          `[TikTokScraper] browser.close() 超时 ${BROWSER_CLOSE_TIMEOUT}ms，跳过等待并继续退出`,
+        );
       }
-    };
-
-    // 无登录态的 browser
-    await closeAll(this.browser);
-    // 登录态的 context（launchPersistentContext 返回的是 context 当 browser）
-    if (this.authContext) {
-      try {
-        await this.authContext.close();
-      } catch {}
+      this.browser = null;
+      this.context = null;
+      this.slots = [];
     }
-    this.browser = null;
-    this.context = null;
-    this.slots = [];
-    this.authBrowser = null;
-    this.authContext = null;
-    this.authSlots = [];
   }
 
   async restart() {
@@ -211,7 +129,7 @@ export class TikTokScraper {
 
   isAlive() {
     try {
-      return !!(this.context && (!this.browser || this.browser.isConnected()));
+      return !!(this.browser && this.browser.isConnected());
     } catch {
       return false;
     }
@@ -238,11 +156,17 @@ export class TikTokScraper {
     return this.warmPromise;
   }
 
-  isWarmExpired() {
+  _needWarm() {
     return Date.now() - this.lastWarmTime > this.wafTtl;
   }
 
-  async _ensurePage(slot, context) {
+  _pickSlot() {
+    const slot = this.slots[this.slotIdx % this.poolSize];
+    this.slotIdx++;
+    return slot;
+  }
+
+  async _ensurePage(slot) {
     try {
       if (
         !slot.page.isClosed() &&
@@ -252,111 +176,88 @@ export class TikTokScraper {
       }
     } catch {}
     await slot.page?.close().catch(() => {});
-    slot.page = await context.newPage();
+    slot.page = await this.context.newPage();
     slot.requestCount = 0;
     return slot.page;
   }
 
-  async _fetchViewSource(url, ctx) {
-    const slots = ctx === this.authContext ? this.authSlots : this.slots;
-    const slotIdx = ctx === this.authContext ? this.authSlotIdx : this.slotIdx;
-    const slot = slots[slotIdx % slots.length];
-    if (ctx === this.authContext) {
-      this.authSlotIdx++;
-    } else {
-      this.slotIdx++;
-    }
-
-    try {
-      return await slot.lock.run(async () => {
-        const page = await this._ensurePage(slot, ctx);
-
-        if (ctx === this.context && this.isWarmExpired()) {
-          await this.warmWaf();
-        }
+  async _fetchViewSource(url, slot) {
+    const page = await this._ensurePage(slot);
 
-        await page.goto("view-source:" + url, {
-          waitUntil: "domcontentloaded",
-          timeout: 15000,
-        });
-
-        const content = await page.evaluate(() => {
-          const rows = document.querySelectorAll("tr");
-          let content = "";
-          rows.forEach((r) => {
-            const lc = r.querySelector(".line-content");
-            if (lc) content += lc.textContent + "\n";
-          });
-          return content;
-        });
-
-        // 导航到 about:blank 释放当前页面的 DOM 和 JS 堆
-        await page
-          .goto("about:blank", {
-            waitUntil: "domcontentloaded",
-            timeout: 5000,
-          })
-          .catch(() => {});
-
-        slot.requestCount += 1;
+    await page.goto("view-source:" + url, {
+      waitUntil: "domcontentloaded",
+      timeout: 15000,
+    });
 
-        return content;
+    const content = await page.evaluate(() => {
+      const rows = document.querySelectorAll("tr");
+      let content = "";
+      rows.forEach((r) => {
+        const lc = r.querySelector(".line-content");
+        if (lc) content += lc.textContent + "\n";
       });
-    } catch (e) {
-      console.error(`[TikTokScraper] _fetchViewSource failed: ${e.message}`);
-      return null;
-    }
-  }
-
-  async getUserInfo(username) {
-    const normalizedUsername = username.startsWith("@")
-      ? username.slice(1)
-      : username;
-    const url = `https://www.tiktok.com/@${normalizedUsername}`;
-
-    // 第一趟：无登录态
-    let rawHtml = await this._fetchViewSource(url, this.context);
-    let result = rawHtml ? parseUserInfo(rawHtml) : null;
+      return content;
+    });
 
-    // 解析失败：先尝试 warmWaf 后重试
-    if (!result) {
-      try {
-        await this.warmWaf();
-      } catch {}
-      rawHtml = await this._fetchViewSource(url, this.context);
-      result = rawHtml ? parseUserInfo(rawHtml) : null;
-    }
+    // 导航到 about:blank 释放当前页面的 DOM 和 JS 堆
+    await page
+      .goto("about:blank", {
+        waitUntil: "domcontentloaded",
+        timeout: 5000,
+      })
+      .catch(() => {});
 
-    // 仍然失败：使用登录态 pool
-    if (!result && this.authContext) {
-      rawHtml = await this._fetchViewSource(url, this.authContext);
-      result = rawHtml ? parseUserInfo(rawHtml) : null;
-    }
+    slot.requestCount += 1;
 
-    return result || null;
+    return content;
   }
 
-  async getVideoInfo(videoUrl) {
-    // 第一趟：无登录态
-    let rawHtml = await this._fetchViewSource(videoUrl, this.context);
-    let result = rawHtml ? parseVideoInfo(rawHtml) : null;
-
-    // 解析失败：先尝试 warmWaf 后重试
-    if (!result) {
-      try {
-        await this.warmWaf();
-      } catch {}
-      rawHtml = await this._fetchViewSource(videoUrl, this.context);
-      result = rawHtml ? parseVideoInfo(rawHtml) : null;
-    }
-
-    // 仍然失败：使用登录态 pool
-    if (!result && this.authContext) {
-      rawHtml = await this._fetchViewSource(videoUrl, this.authContext);
-      result = rawHtml ? parseVideoInfo(rawHtml) : null;
-    }
+  async getUserInfo(uniqueId, maxRetries = 3) {
+    const slot = this._pickSlot();
+    return slot.lock.run(async () => {
+      let rawHtml = await this._fetchViewSource(
+        `https://www.tiktok.com/@${uniqueId}`,
+        slot,
+      );
+      let result = parseUserInfo(rawHtml);
+      for (let attempt = 1; !result && attempt <= maxRetries; attempt++) {
+        // 检查是否值得重试：用户异常/不存在则跳过重试
+        if (!isRetryableFailure(rawHtml)) {
+          break;
+        }
+        try {
+          await this.warmWaf();
+        } catch {}
+        await delay(500 * attempt);
+        rawHtml = await this._fetchViewSource(
+          `https://www.tiktok.com/@${uniqueId}`,
+          slot,
+        );
+        result = parseUserInfo(rawHtml);
+      }
+      return result || null;
+    });
+  }
 
-    return result || null;
+  async getVideoInfo(videoUrl, maxRetries = 3) {
+    const slot = this._pickSlot();
+    return slot.lock.run(async () => {
+      let rawHtml = await this._fetchViewSource(videoUrl, slot);
+      let result = parseVideoInfo(rawHtml);
+      for (let attempt = 1; !result && attempt <= maxRetries; attempt++) {
+        // 检查是否值得重试
+        if (!isRetryableFailure(rawHtml)) {
+          break;
+        }
+        try {
+          await this.warmWaf();
+        } catch {}
+        await delay(500 * attempt);
+        rawHtml = await this._fetchViewSource(videoUrl, slot);
+        result = parseVideoInfo(rawHtml);
+      }
+      return result || null;
+    });
   }
 
   async getUserAndVideo(videoUrl) {

package/src/scraper/explore-core.js

@@ -38,6 +38,7 @@ async function processExplore(page, username, options, log) {
     keepFollow: false,
     locationCreated: null,
     noVideo: false,
+    restricted: false,
     error: null,
   };
 
@@ -172,6 +173,7 @@ async function processExplore(page, username, options, log) {
       result.videoList = videoArray;
     } else {
       // 国家不匹配
+      result.restricted = true;
       result.keepFollow = false;
       result.discoveredFollowing = [];
       result.discoveredFollowers = [];