tt-help-cli-ycl 1.3.11 → 1.3.12

package/src/{scraper/modules/scroll-collector.mjs → lib/scroll-collector.js}

@@ -1,189 +1,231 @@
-import { delay } from "../../lib/delay.js";
-import { detectPageError } from "./page-error-detector.mjs";
-
-async function doCollect(
-  page,
-  { container, findScrollable, fnStr, extraArgs },
-) {
-  return page.evaluate(
-    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
-      let el;
-      if (!containerSelector) {
-        el = window;
-      } else {
-        el = document.querySelector(containerSelector);
-        if (!el) {
-          el = window;
-        } else if (findScrollableFlag) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) el = document.body;
-        }
-      }
-      const fn = eval("(" + fnStr + ")");
-      return fn(el, args);
-    },
-    {
-      fn: fnStr,
-      containerSelector: container,
-      findScrollableFlag: findScrollable,
-      args: extraArgs,
-    },
-  );
-}
-
-const LOADING_SELECTORS = [
-  '[class*="loading"]',
-  '[class*="Loading"]',
-  '[class*="spinner"]',
-  '[class*="Spinner"]',
-  '[class*="skeleton"]',
-  '[class*="Skeleton"]',
-  '[aria-busy="true"]',
-];
-
-async function waitForLoading(page) {
-  const maxWait = 5000;
-  const startTime = Date.now();
-  while (Date.now() - startTime < maxWait) {
-    const isLoading = await page.evaluate((sels) => {
-      if (document.readyState !== "complete") return true;
-      for (const sel of sels) {
-        const el = document.querySelector(sel);
-        if (el && el.offsetParent !== null) return true;
-      }
-      return false;
-    }, LOADING_SELECTORS);
-    if (!isLoading) return;
-    await delay(300, 600);
-  }
-}
-
-export async function scrollAndCollect(page, options) {
-  const {
-    container,
-    findScrollable = false,
-    collectFn,
-    extraArgs,
-    delayRange = [800, 1500],
-    maxItems,
-    maxRounds = 200,
-    staleThreshold = 3,
-    uniqueKey,
-    onRound,
-  } = options;
-
-  if (!collectFn) throw new Error("collectFn is required");
-
-  const fnStr =
-    typeof collectFn === "function" ? collectFn.toString() : collectFn;
-  const allItems = [];
-  const seenKeys = uniqueKey ? new Set() : null;
-  let staleCount = 0;
-
-  const processItems = (result) => {
-    const raw = result.items || [];
-    const newItems = uniqueKey
-      ? raw.filter((item) => {
-          const key = uniqueKey(item);
-          if (seenKeys.has(key)) return false;
-          seenKeys.add(key);
-          return true;
-        })
-      : raw;
-    allItems.push(...newItems);
-    return newItems;
-  };
-
-  const isDone = (newItems) => {
-    if (maxItems !== undefined && allItems.length >= maxItems) return true;
-    if (newItems.length === 0) {
-      staleCount++;
-      if (staleCount >= staleThreshold) return true;
-    } else {
-      staleCount = 0;
-    }
-    return false;
-  };
-
-  const collectCtx = { container, findScrollable, fnStr, extraArgs };
-
-  const pageError = await detectPageError(page);
-
-  if (pageError) return [];
-
-  await waitForLoading(page);
-  let result = await doCollect(page, collectCtx);
-  let newItems = processItems(result);
-  if (onRound) onRound(0, newItems, allItems);
-  if (isDone(newItems)) return allItems;
-
-  for (let round = 1; round < maxRounds; round++) {
-    await threePhaseScroll(page, { container, findScrollable });
-    await delay(delayRange[0], delayRange[1]);
-    await waitForLoading(page);
-
-    result = await doCollect(page, collectCtx);
-    newItems = processItems(result);
-
-    if (onRound) onRound(round, newItems, allItems);
-
-    if (isDone(newItems)) break;
-  }
-
-  return allItems;
-}
-
-async function threePhaseScroll(page, { container, findScrollable }) {
-  await page.evaluate(
-    async (opts) => {
-      let el;
-      if (!opts.container) {
-        el = window;
-      } else {
-        el = document.querySelector(opts.container);
-        if (!el) {
-          el = window;
-        } else if (opts.findScrollable) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) el = document.body;
-        }
-      }
-
-      const randDelay = (min, max) =>
-        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
-
-      if (el === window) {
-        window.scrollBy(0, window.innerHeight);
-        await randDelay(400, 800);
-        window.scrollBy(0, -200);
-        await randDelay(200, 400);
-        window.scrollBy(0, window.innerHeight);
-      } else {
-        el.scrollTop = el.scrollHeight;
-        await randDelay(400, 800);
-        el.scrollTop -= 100 + Math.random() * 100;
-        await randDelay(200, 400);
-        el.scrollTop = el.scrollHeight;
-      }
-    },
-    { container, findScrollable },
-  );
-}
+import { delay } from './delay.js';
+import { detectPageError } from './page-error-detector.js';
+
+async function doCollect(
+  page,
+  { container, findScrollable, fnStr, extraArgs },
+) {
+  return page.evaluate(
+    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
+      let el;
+      if (!containerSelector) {
+        el = window;
+      } else {
+        el = document.querySelector(containerSelector);
+        if (!el) {
+          el = window;
+        } else if (findScrollableFlag) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const fn = eval("(" + fnStr + ")");
+      return fn(el, args);
+    },
+    { fn: fnStr, containerSelector: container, findScrollableFlag: findScrollable, args: extraArgs },
+  );
+}
+
+const LOADING_SELECTORS = [
+  '[class*="loading"]',
+  '[class*="Loading"]',
+  '[class*="spinner"]',
+  '[class*="Spinner"]',
+  '[class*="skeleton"]',
+  '[class*="Skeleton"]',
+  '[aria-busy="true"]',
+];
+
+async function waitForLoading(page) {
+  const maxWait = 5000;
+  const startTime = Date.now();
+  while (Date.now() - startTime < maxWait) {
+    const isLoading = await page.evaluate((sels) => {
+      if (document.readyState !== "complete") return true;
+      for (const sel of sels) {
+        const el = document.querySelector(sel);
+        if (el && el.offsetParent !== null) return true;
+      }
+      return false;
+    }, LOADING_SELECTORS);
+    if (!isLoading) return;
+    await delay(300, 600);
+  }
+}
+
+async function collectWithoutScroll(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+
+  const result = await doCollect(page, {
+    container,
+    findScrollable,
+    fnStr,
+    extraArgs,
+  });
+
+  return result.items || [];
+}
+
+async function scrollToBottom(page, options = {}) {
+  const {
+    container,
+    findScrollable = false,
+    delayRange = [800, 1500],
+    maxScroll = 200,
+  } = options;
+
+  for (let i = 0; i < maxScroll; i++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+  }
+}
+
+async function scrollAndCollect(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+    delayRange = [800, 1500],
+    maxItems,
+    maxRounds = 200,
+    staleThreshold = 3,
+    uniqueKey,
+    onRound,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+  const allItems = [];
+  const seenKeys = uniqueKey ? new Set() : null;
+  let staleCount = 0;
+
+  const processItems = (result) => {
+    const raw = result.items || [];
+    const newItems = uniqueKey
+      ? raw.filter((item) => {
+          const key = uniqueKey(item);
+          if (seenKeys.has(key)) return false;
+          seenKeys.add(key);
+          return true;
+        })
+      : raw;
+    allItems.push(...newItems);
+    return newItems;
+  };
+
+  const isDone = (newItems) => {
+    if (maxItems !== undefined && allItems.length >= maxItems) return true;
+    if (newItems.length === 0) {
+      staleCount++;
+      if (staleCount >= staleThreshold) return true;
+    } else {
+      staleCount = 0;
+    }
+    return false;
+  };
+
+  const collectCtx = { container, findScrollable, fnStr, extraArgs };
+
+  const pageError = await detectPageError(page);
+
+  if (pageError) return [];
+
+  await waitForLoading(page);
+  let result = await doCollect(page, collectCtx);
+  let newItems = processItems(result);
+  if (onRound) onRound(0, newItems, allItems);
+  if (isDone(newItems)) return allItems;
+
+  for (let round = 1; round < maxRounds; round++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+    await waitForLoading(page);
+
+    result = await doCollect(page, collectCtx);
+    newItems = processItems(result);
+
+    if (onRound) onRound(round, newItems, allItems);
+
+    if (isDone(newItems)) break;
+  }
+
+  return allItems;
+}
+
+async function threePhaseScroll(page, { container, findScrollable }) {
+  await page.evaluate(
+    async (opts) => {
+      let el;
+      if (!opts.container) {
+        el = window;
+      } else {
+        el = document.querySelector(opts.container);
+        if (!el) {
+          el = window;
+        } else if (opts.findScrollable) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const randDelay = (min, max) =>
+        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
+
+      if (el === window) {
+        window.scrollBy(0, window.innerHeight);
+        await randDelay(400, 800);
+        window.scrollBy(0, -200);
+        await randDelay(200, 400);
+        window.scrollBy(0, window.innerHeight);
+      } else {
+        el.scrollTop = el.scrollHeight;
+        await randDelay(400, 800);
+        el.scrollTop -= 100 + Math.random() * 100;
+        await randDelay(200, 400);
+        el.scrollTop = el.scrollHeight;
+      }
+    },
+    { container, findScrollable },
+  );
+}
+
+export {
+  scrollAndCollect,
+  scrollToBottom,
+  collectWithoutScroll,
+  doCollect,
+  waitForLoading,
+  threePhaseScroll,
+};

package/src/main.js

@@ -0,0 +1,46 @@
+import { parseArgs } from './lib/args.js';
+import { proxy, HELP_TEXT, getConfigText } from './lib/constants.js';
+import { parseFilter } from './lib/filter.js';
+import { handleScrape } from './cli/scrape.js';
+import { handleVideos } from './cli/videos.js';
+import { handleAuto } from './cli/auto.js';
+import { handleExplore } from './cli/explore.js';
+import { handleWatch } from './cli/watch.js';
+import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
+import { runExploreDefault, runScrapeDefault } from './cli/explore-default.js';
+
+async function main() {
+  const parsed = parseArgs();
+
+  switch (parsed.subcommand) {
+    case 'scrape': return handleScrape(parsed);
+    case 'videos': return handleVideos(parsed);
+    case 'auto':   return handleAuto(parsed);
+    case 'explore':return handleExplore(parsed);
+    case 'watch':  return handleWatch(parsed);
+  }
+
+  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue, pipeMode, filterStr } = parsed;
+  const proxyUrl = customProxy || proxy;
+  const filter = parseFilter(filterStr);
+
+  if (showVersion) {
+    console.log(version);
+    process.exit(0);
+  }
+  if (showHelp) return showUsage();
+  if (configAction) return handleConfig(configAction, configKey, configValue);
+  if (showCfg) return showConfig(urls, outputFile);
+  if (urls.length === 0 && exploreCount === 0) return showUsage();
+
+  if (exploreCount > 0) {
+    await runExploreDefault(exploreCount, urls, proxyUrl, outputFile, outputFormat, pipeMode, filter);
+  } else {
+    await runScrapeDefault(urls, proxyUrl, outputFile, outputFormat, filter);
+  }
+}
+
+main().catch(err => {
+  console.error(`错误: ${err.message}`);
+  process.exit(1);
+});

package/src/scraper/{auto-core.mjs → auto-core.js}

@@ -9,15 +9,15 @@ import {
   detectPageError,
   isLoggedIn,
   assertPageUrl,
-} from './modules/page-helpers.mjs';
-import { detectCaptcha } from './modules/captcha-handler.mjs';
+} from './modules/page-helpers.js';
+import { detectCaptcha } from './modules/captcha-handler.js';
 export { ensureBrowserReady };
 import {
   getUserInfo,
   collectVideos,
-} from '../videos/core.mjs';
-import { runScrape } from './core.mjs';
-import { extractFollowAndFollowers } from './modules/follow-extractor.mjs';
+} from '../videos/core.js';
+import { runScrape } from './core.js';
+import { extractFollowAndFollowers } from './modules/follow-extractor.js';
 
 function mergeUserInfo(existing, incoming, source) {
   const merged = { ...existing };

package/src/scraper/{core.mjs → core.js}

@@ -7,9 +7,9 @@ import {
   getDelayConfig,
   retryWithBackoff,
   assertPageUrl,
-} from './modules/page-helpers.mjs';
-import { extractCommentAuthors } from './modules/comment-extractor.mjs';
-import { extractGuessVideos } from './modules/guess-extractor.mjs';
+} from './modules/page-helpers.js';
+import { extractCommentAuthors } from './modules/comment-extractor.js';
+import { extractGuessVideos } from './modules/guess-extractor.js';
 
 async function scrapeSingleVideo(page, maxComments, maxGuess, log, location = 'ES') {
   const config = getDelayConfig();

package/src/scraper/{explore-core.mjs → explore-core.js}

@@ -7,17 +7,17 @@ import {
   detectPageError,
   isLoggedIn,
   assertPageUrl,
-} from './modules/page-helpers.mjs';
-import { detectCaptcha } from './modules/captcha-handler.mjs';
+} from './modules/page-helpers.js';
+import { detectCaptcha } from './modules/captcha-handler.js';
 export { ensureBrowserReady };
 import {
   getUserInfo,
   collectVideos,
-} from '../videos/core.mjs';
-import { scrapeSingleVideo } from './core.mjs';
-import { extractFollowAndFollowers } from './modules/follow-extractor.mjs';
-import { extractCommentAuthors } from './modules/comment-extractor.mjs';
-import { extractGuessVideos } from './modules/guess-extractor.mjs';
+} from '../videos/core.js';
+import { scrapeSingleVideo } from './core.js';
+import { extractFollowAndFollowers } from './modules/follow-extractor.js';
+import { extractCommentAuthors } from './modules/comment-extractor.js';
+import { extractGuessVideos } from './modules/guess-extractor.js';
 
 async function processExplore(page, username, options, log) {
   const {

package/src/scraper/modules/{comment-extractor.mjs → comment-extractor.js}

@@ -1,6 +1,6 @@
-import { delay, getDelayConfig, closeCommentPanel } from "./page-helpers.mjs";
-import { scrollAndCollect } from "./scroll-collector.mjs";
-import { waitAndGetCaptcha } from "./captcha-handler.mjs";
+import { delay, getDelayConfig, closeCommentPanel } from "./page-helpers.js";
+import { scrollAndCollect } from "./scroll-collector.js";
+import { waitAndGetCaptcha } from "./captcha-handler.js";
 
 async function openCommentPanel(page) {
   const tabs = page.locator('[class*="tabbar-item"]');

package/src/scraper/modules/{follow-extractor.mjs → follow-extractor.js}

@@ -1,5 +1,5 @@
-import { delay, getDelayConfig } from "./page-helpers.mjs";
-import { scrollAndCollect } from "./scroll-collector.mjs";
+import { delay, getDelayConfig } from "./page-helpers.js";
+import { scrollAndCollect } from "./scroll-collector.js";
 
 const FILTER_WORDS = ["主页", "已关注", "粉丝", "推荐"];
 

package/src/scraper/modules/{guess-extractor.mjs → guess-extractor.js}

@@ -1,5 +1,5 @@
-import { delay, getDelayConfig, closeCommentPanel } from './page-helpers.mjs';
-import { scrollAndCollect } from './scroll-collector.mjs';
+import { delay, getDelayConfig, closeCommentPanel } from './page-helpers.js';
+import { scrollAndCollect } from './scroll-collector.js';
 
 async function openGuessTab(page) {
   const tabs = page.locator('[class*="tabbar-item"]');

package/src/scraper/modules/page-error-detector.js

@@ -0,0 +1 @@
+export { detectPageError } from '../../lib/page-error-detector.js';

package/src/scraper/modules/{page-helpers.mjs → page-helpers.js}

@@ -21,7 +21,7 @@ import {
   extractLocationCreated,
   USER_SECTION_SIZE,
 } from '../../lib/parser.js';
-import { detectPageError } from './page-error-detector.mjs';
+import { detectPageError } from './page-error-detector.js';
 
 export {
   delay,

package/src/scraper/modules/scroll-collector.js

@@ -0,0 +1,8 @@
+export {
+  scrollAndCollect,
+  scrollToBottom,
+  collectWithoutScroll,
+  doCollect,
+  waitForLoading,
+  threePhaseScroll,
+} from '../../lib/scroll-collector.js';

package/src/videos/{core.mjs → core.js}

@@ -1,5 +1,5 @@
-import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.mjs';
-import { scrollAndCollect } from '../scraper/modules/scroll-collector.mjs';
+import { delay, ensureBrowserReady, ensureTikTokPage, retryWithBackoff } from '../scraper/modules/page-helpers.js';
+import { scrollAndCollect } from '../scraper/modules/scroll-collector.js';
 
 async function getUserInfo(page) {
   return await page.evaluate(() => {