tt-help-cli-ycl 1.3.34 → 1.3.36

package/src/lib/tiktok-scraper.mjs

@@ -1,194 +1,232 @@
-import { chromium } from 'playwright';
-import { detectBrowser } from './browser/launch.js';
-import { parseUserInfo, parseVideoInfo } from './parse-ssr.mjs';
-
-const DEFAULT_POOL_SIZE = 3;
-const DEFAULT_WAF_TTL = 120000;
-const DEFAULT_WARM_URL = 'https://www.tiktok.com/@nike';
-
-function delay(ms) {
-  return new Promise(r => setTimeout(r, ms));
-}
-
-class PageSlot {
-  constructor(page) {
-    this.page = page;
-    this.lock = new PromiseQueue();
-  }
-}
-
-class PromiseQueue {
-  constructor() {
-    this._queue = [];
-    this._processing = false;
-  }
-  async run(task) {
-    return new Promise((resolve, reject) => {
-      this._queue.push({ task, resolve, reject });
-      this._process();
-    });
-  }
-  async _process() {
-    if (this._processing) return;
-    this._processing = true;
-    while (this._queue.length > 0) {
-      const { task, resolve, reject } = this._queue.shift();
-      try {
-        const result = await task();
-        resolve(result);
-      } catch (e) {
-        reject(e);
-      }
-    }
-    this._processing = false;
-  }
-}
-
-export class TikTokScraper {
-  constructor({ poolSize = DEFAULT_POOL_SIZE, wafTtl = DEFAULT_WAF_TTL, warmUrl = DEFAULT_WARM_URL } = {}) {
-    this.poolSize = poolSize;
-    this.wafTtl = wafTtl;
-    this.warmUrl = warmUrl;
-    this.browser = null;
-    this.context = null;
-    this.slots = [];
-    this.slotIdx = 0;
-    this.lastWarmTime = 0;
-    this.warmPromise = null;
-  }
-
-  async init() {
-    const executablePath = detectBrowser();
-    if (!executablePath) {
-      throw new Error('未找到本地浏览器（Chrome/Edge），请先安装浏览器或执行 npx playwright install');
-    }
-    this.browser = await chromium.launch({
-      headless: true,
-      executablePath,
-      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
-    });
-    this.context = await this.browser.newContext();
-    for (let i = 0; i < this.poolSize; i++) {
-      this.slots.push(new PageSlot(await this.context.newPage()));
-    }
-    await this.warmWaf();
-  }
-
-  async close() {
-    if (this.browser) {
-      await this.browser.close();
-      this.browser = null;
-      this.context = null;
-      this.slots = [];
-    }
-  }
-
-  async restart() {
-    await this.close();
-    await this.init();
-  }
-
-  isAlive() {
-    try {
-      return !!(this.browser && this.browser.isConnected());
-    } catch {
-      return false;
-    }
-  }
-
-  async warmWaf() {
-    if (this.warmPromise) return this.warmPromise;
-    this.warmPromise = (async () => {
-      const page = await this.context.newPage();
-      try {
-        await page.goto(this.warmUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
-        await delay(1500);
-        this.lastWarmTime = Date.now();
-      } catch (e) {
-        console.error(`[warmWaf] failed: ${e.message}`);
-      } finally {
-        await page.close();
-        this.warmPromise = null;
-      }
-    })();
-    return this.warmPromise;
-  }
-
-  _needWarm() {
-    return Date.now() - this.lastWarmTime > this.wafTtl;
-  }
-
-  _pickSlot() {
-    const slot = this.slots[this.slotIdx % this.poolSize];
-    this.slotIdx++;
-    return slot;
-  }
-
-  async _ensurePage(slot) {
-    try {
-      if (!slot.page.isClosed()) return slot.page;
-    } catch {}
-    slot.page = await this.context.newPage();
-    return slot.page;
-  }
-
-  async _fetchViewSource(url, slot) {
-    const page = await this._ensurePage(slot);
-
-    await page.goto('view-source:' + url, {
-      waitUntil: 'domcontentloaded',
-      timeout: 15000,
-    });
-
-    return await page.evaluate(() => {
-      const rows = document.querySelectorAll('tr');
-      let content = '';
-      rows.forEach(r => {
-        const lc = r.querySelector('.line-content');
-        if (lc) content += lc.textContent + '\n';
-      });
-      return content;
-    });
-  }
-
-  async getUserInfo(uniqueId) {
-    const slot = this._pickSlot();
-    return slot.lock.run(async () => {
-      let rawHtml = await this._fetchViewSource(
-        `https://www.tiktok.com/@${uniqueId}`,
-        slot
-      );
-      let result = parseUserInfo(rawHtml);
-      if (!result) {
-        try { await this.warmWaf(); } catch {}
-        rawHtml = await this._fetchViewSource(
-          `https://www.tiktok.com/@${uniqueId}`,
-          slot
-        );
-        result = parseUserInfo(rawHtml);
-      }
-      return result || null;
-    });
-  }
-
-  async getVideoInfo(videoUrl) {
-    const slot = this._pickSlot();
-    return slot.lock.run(async () => {
-      let rawHtml = await this._fetchViewSource(videoUrl, slot);
-      let result = parseVideoInfo(rawHtml);
-      if (!result) {
-        try { await this.warmWaf(); } catch {}
-        rawHtml = await this._fetchViewSource(videoUrl, slot);
-        result = parseVideoInfo(rawHtml);
-      }
-      return result || null;
-    });
-  }
-
-  async getUserAndVideo(videoUrl) {
-    const video = await this.getVideoInfo(videoUrl);
-    if (!video) return null;
-    const user = await this.getUserInfo(video.author.uniqueId);
-    return { user, video };
-  }
-}
+import { chromium } from "playwright";
+import { detectBrowser } from "./browser/launch.js";
+import { parseUserInfo, parseVideoInfo } from "./parse-ssr.mjs";
+
+const DEFAULT_POOL_SIZE = 3;
+const DEFAULT_WAF_TTL = 120000;
+const DEFAULT_WARM_URL = "https://www.tiktok.com/@nike";
+const BROWSER_CLOSE_TIMEOUT = 5000;
+
+function delay(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+class PageSlot {
+  constructor(page) {
+    this.page = page;
+    this.lock = new PromiseQueue();
+  }
+}
+
+class PromiseQueue {
+  constructor() {
+    this._queue = [];
+    this._processing = false;
+  }
+  async run(task) {
+    return new Promise((resolve, reject) => {
+      this._queue.push({ task, resolve, reject });
+      this._process();
+    });
+  }
+  async _process() {
+    if (this._processing) return;
+    this._processing = true;
+    while (this._queue.length > 0) {
+      const { task, resolve, reject } = this._queue.shift();
+      try {
+        const result = await task();
+        resolve(result);
+      } catch (e) {
+        reject(e);
+      }
+    }
+    this._processing = false;
+  }
+}
+
+export class TikTokScraper {
+  constructor({
+    poolSize = DEFAULT_POOL_SIZE,
+    wafTtl = DEFAULT_WAF_TTL,
+    warmUrl = DEFAULT_WARM_URL,
+  } = {}) {
+    this.poolSize = poolSize;
+    this.wafTtl = wafTtl;
+    this.warmUrl = warmUrl;
+    this.browser = null;
+    this.context = null;
+    this.slots = [];
+    this.slotIdx = 0;
+    this.lastWarmTime = 0;
+    this.warmPromise = null;
+  }
+
+  async init() {
+    const executablePath = detectBrowser();
+    if (!executablePath) {
+      throw new Error(
+        "未找到本地浏览器（Chrome/Edge），请先安装浏览器或执行 npx playwright install",
+      );
+    }
+    this.browser = await chromium.launch({
+      headless: true,
+      executablePath,
+      handleSIGINT: false,
+      handleSIGTERM: false,
+      handleSIGHUP: false,
+      args: [
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-dev-shm-usage",
+      ],
+    });
+    this.context = await this.browser.newContext();
+    for (let i = 0; i < this.poolSize; i++) {
+      this.slots.push(new PageSlot(await this.context.newPage()));
+    }
+    await this.warmWaf();
+  }
+
+  async close() {
+    if (this.browser) {
+      const browser = this.browser;
+      let closeTimedOut = false;
+      const closePromise = browser.close().catch((error) => {
+        console.error(
+          `[TikTokScraper] browser.close() failed: ${error.message}`,
+        );
+      });
+      await Promise.race([
+        closePromise,
+        delay(BROWSER_CLOSE_TIMEOUT).then(() => {
+          closeTimedOut = true;
+        }),
+      ]);
+      if (closeTimedOut) {
+        console.error(
+          `[TikTokScraper] browser.close() 超时 ${BROWSER_CLOSE_TIMEOUT}ms，跳过等待并继续退出`,
+        );
+      }
+      this.browser = null;
+      this.context = null;
+      this.slots = [];
+    }
+  }
+
+  async restart() {
+    await this.close();
+    await this.init();
+  }
+
+  isAlive() {
+    try {
+      return !!(this.browser && this.browser.isConnected());
+    } catch {
+      return false;
+    }
+  }
+
+  async warmWaf() {
+    if (this.warmPromise) return this.warmPromise;
+    this.warmPromise = (async () => {
+      const page = await this.context.newPage();
+      try {
+        await page.goto(this.warmUrl, {
+          waitUntil: "domcontentloaded",
+          timeout: 15000,
+        });
+        await delay(1500);
+        this.lastWarmTime = Date.now();
+      } catch (e) {
+        console.error(`[warmWaf] failed: ${e.message}`);
+      } finally {
+        await page.close();
+        this.warmPromise = null;
+      }
+    })();
+    return this.warmPromise;
+  }
+
+  _needWarm() {
+    return Date.now() - this.lastWarmTime > this.wafTtl;
+  }
+
+  _pickSlot() {
+    const slot = this.slots[this.slotIdx % this.poolSize];
+    this.slotIdx++;
+    return slot;
+  }
+
+  async _ensurePage(slot) {
+    try {
+      if (!slot.page.isClosed()) return slot.page;
+    } catch {}
+    slot.page = await this.context.newPage();
+    return slot.page;
+  }
+
+  async _fetchViewSource(url, slot) {
+    const page = await this._ensurePage(slot);
+
+    await page.goto("view-source:" + url, {
+      waitUntil: "domcontentloaded",
+      timeout: 15000,
+    });
+
+    return await page.evaluate(() => {
+      const rows = document.querySelectorAll("tr");
+      let content = "";
+      rows.forEach((r) => {
+        const lc = r.querySelector(".line-content");
+        if (lc) content += lc.textContent + "\n";
+      });
+      return content;
+    });
+  }
+
+  async getUserInfo(uniqueId) {
+    const slot = this._pickSlot();
+    return slot.lock.run(async () => {
+      let rawHtml = await this._fetchViewSource(
+        `https://www.tiktok.com/@${uniqueId}`,
+        slot,
+      );
+      let result = parseUserInfo(rawHtml);
+      if (!result) {
+        try {
+          await this.warmWaf();
+        } catch {}
+        rawHtml = await this._fetchViewSource(
+          `https://www.tiktok.com/@${uniqueId}`,
+          slot,
+        );
+        result = parseUserInfo(rawHtml);
+      }
+      return result || null;
+    });
+  }
+
+  async getVideoInfo(videoUrl) {
+    const slot = this._pickSlot();
+    return slot.lock.run(async () => {
+      let rawHtml = await this._fetchViewSource(videoUrl, slot);
+      let result = parseVideoInfo(rawHtml);
+      if (!result) {
+        try {
+          await this.warmWaf();
+        } catch {}
+        rawHtml = await this._fetchViewSource(videoUrl, slot);
+        result = parseVideoInfo(rawHtml);
+      }
+      return result || null;
+    });
+  }
+
+  async getUserAndVideo(videoUrl) {
+    const video = await this.getVideoInfo(videoUrl);
+    if (!video) return null;
+    const user = await this.getUserInfo(video.author.uniqueId);
+    return { user, video };
+  }
+}

package/src/lib/url.js

@@ -1,52 +1,52 @@
-const BASE_URL = 'https://www.tiktok.com';
-
-export function extractUniqueId(url) {
-  const m = url.match(/\/@([\w.-]+)/);
-  return m ? m[1] : null;
-}
-
-export function extractVideoId(url) {
-  const m = url.match(/\/video\/(\d+)/);
-  return m ? m[1] : null;
-}
-
-export function normalizeUsername(input) {
-  return (input || '').replace(/^@/, '');
-}
-
-export function toProfileUrl(handle) {
-  const clean = normalizeUsername(handle);
-  return `${BASE_URL}/@${clean}`;
-}
-
-export function toVideoUrl(handle, videoId) {
-  const clean = normalizeUsername(handle);
-  return `${BASE_URL}/@${clean}/video/${videoId}`;
-}
-
-export function ensureAbsoluteUrl(href) {
-  if (href.startsWith('http')) return href;
-  return `${BASE_URL}${href}`;
-}
-
-export function isProfileUrl(url) {
-  return /\/@[\w.-]+(?:$|[?#])/.test(url);
-}
-
-export function isVideoUrl(url) {
-  return /\/video\/\d+/.test(url);
-}
-
-export function extractDisplayPath(url) {
-  try {
-    const parts = new URL(url).pathname.split('/').filter(Boolean);
-    return parts.slice(-2).join('/');
-  } catch {
-    return url;
-  }
-}
-
-export function extractAuthorFromVideoUrl(url) {
-  const m = url.match(/@([^/]+)\/video/);
-  return m ? '@' + m[1] : null;
-}
+const BASE_URL = 'https://www.tiktok.com';
+
+export function extractUniqueId(url) {
+  const m = url.match(/\/@([\w.-]+)/);
+  return m ? m[1] : null;
+}
+
+export function extractVideoId(url) {
+  const m = url.match(/\/video\/(\d+)/);
+  return m ? m[1] : null;
+}
+
+export function normalizeUsername(input) {
+  return (input || '').replace(/^@/, '');
+}
+
+export function toProfileUrl(handle) {
+  const clean = normalizeUsername(handle);
+  return `${BASE_URL}/@${clean}`;
+}
+
+export function toVideoUrl(handle, videoId) {
+  const clean = normalizeUsername(handle);
+  return `${BASE_URL}/@${clean}/video/${videoId}`;
+}
+
+export function ensureAbsoluteUrl(href) {
+  if (href.startsWith('http')) return href;
+  return `${BASE_URL}${href}`;
+}
+
+export function isProfileUrl(url) {
+  return /\/@[\w.-]+(?:$|[?#])/.test(url);
+}
+
+export function isVideoUrl(url) {
+  return /\/video\/\d+/.test(url);
+}
+
+export function extractDisplayPath(url) {
+  try {
+    const parts = new URL(url).pathname.split('/').filter(Boolean);
+    return parts.slice(-2).join('/');
+  } catch {
+    return url;
+  }
+}
+
+export function extractAuthorFromVideoUrl(url) {
+  const m = url.match(/@([^/]+)\/video/);
+  return m ? '@' + m[1] : null;
+}

package/src/main.js

@@ -1,48 +1,70 @@
-import { parseArgs } from './lib/args.js';
-import { proxy, HELP_TEXT, getConfigText } from './lib/constants.js';
-import { handleInfo } from './cli/info.js';
-import { handleExplore } from './cli/explore.js';
-import { handleAttach } from './cli/attach.js';
-import { handleWatch } from './cli/watch.js';
-import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
-import { handleOpen } from './cli/open.js';
-import { handleComments } from './cli/comments.js';
-import { handleVideoStats } from './cli/videostats.js';
-
-async function main() {
-  const parsed = parseArgs();
-
-  switch (parsed.subcommand) {
-    case 'explore':    return handleExplore(parsed);
-    case 'info':       return handleInfo(parsed);
-    case 'attach':     return handleAttach(parsed);
-    case 'watch':      return handleWatch(parsed);
-    case 'open':       return handleOpen(parsed);
-    case 'comments':   return handleComments(parsed);
-    case 'videostats': return handleVideoStats(parsed);
-  }
-
-  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue } = parsed;
-
-  if (showVersion) {
-    console.log(version);
-    process.exit(0);
-  }
-  if (showHelp) return showUsage();
-  if (configAction) return handleConfig(configAction, configKey, configValue);
-  if (showCfg) return showConfig(urls, outputFile);
-  if (urls.length === 0 && exploreCount === 0) return showUsage();
-
-  // 默认行为：URL 走 info，--explore 走 explore
-  if (exploreCount > 0) {
-    return handleExplore({ ...parsed, subcommand: 'explore' });
-  }
-
-  // 有 URL 默认走 info
-  return handleInfo(parsed);
-}
-
-main().catch(err => {
-  console.error(`错误: ${err.message}`);
-  process.exit(1);
-});
+import { parseArgs } from "./lib/args.js";
+import { proxy, HELP_TEXT, getConfigText } from "./lib/constants.js";
+import { handleInfo } from "./cli/info.js";
+import { handleExplore } from "./cli/explore.js";
+import { handleAttach } from "./cli/attach.js";
+import { handleWatch } from "./cli/watch.js";
+import { handleConfig, showConfig, showUsage, version } from "./cli/config.js";
+import { handleOpen } from "./cli/open.js";
+import { handleComments } from "./cli/comments.js";
+import { handleVideoStats } from "./cli/videostats.js";
+import { handleDbImport } from "./cli/db-import.js";
+
+async function main() {
+  const parsed = parseArgs();
+
+  switch (parsed.subcommand) {
+    case "explore":
+      return handleExplore(parsed);
+    case "info":
+      return handleInfo(parsed);
+    case "attach":
+      return handleAttach(parsed);
+    case "watch":
+      return handleWatch(parsed);
+    case "open":
+      return handleOpen(parsed);
+    case "comments":
+      return handleComments(parsed);
+    case "videostats":
+      return handleVideoStats(parsed);
+    case "db-import":
+      return handleDbImport(parsed);
+  }
+
+  const {
+    urls,
+    outputFile,
+    outputFormat,
+    exploreCount,
+    showConfig: showCfg,
+    showHelp,
+    showVersion,
+    customProxy,
+    configAction,
+    configKey,
+    configValue,
+  } = parsed;
+
+  if (showVersion) {
+    console.log(version);
+    process.exit(0);
+  }
+  if (showHelp) return showUsage();
+  if (configAction) return handleConfig(configAction, configKey, configValue);
+  if (showCfg) return showConfig(urls, outputFile);
+  if (urls.length === 0 && exploreCount === 0) return showUsage();
+
+  // 默认行为：URL 走 info，--explore 走 explore
+  if (exploreCount > 0) {
+    return handleExplore({ ...parsed, subcommand: "explore" });
+  }
+
+  // 有 URL 默认走 info
+  return handleInfo(parsed);
+}
+
+main().catch((err) => {
+  console.error(`错误: ${err.message}`);
+  process.exit(1);
+});