tt-help-cli-ycl 1.3.11 → 1.3.13

package/src/{scraper/modules/scroll-collector.mjs → lib/scroll-collector.js}

@@ -1,189 +1,231 @@
-import { delay } from "../../lib/delay.js";
-import { detectPageError } from "./page-error-detector.mjs";
-
-async function doCollect(
-  page,
-  { container, findScrollable, fnStr, extraArgs },
-) {
-  return page.evaluate(
-    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
-      let el;
-      if (!containerSelector) {
-        el = window;
-      } else {
-        el = document.querySelector(containerSelector);
-        if (!el) {
-          el = window;
-        } else if (findScrollableFlag) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) el = document.body;
-        }
-      }
-      const fn = eval("(" + fnStr + ")");
-      return fn(el, args);
-    },
-    {
-      fn: fnStr,
-      containerSelector: container,
-      findScrollableFlag: findScrollable,
-      args: extraArgs,
-    },
-  );
-}
-
-const LOADING_SELECTORS = [
-  '[class*="loading"]',
-  '[class*="Loading"]',
-  '[class*="spinner"]',
-  '[class*="Spinner"]',
-  '[class*="skeleton"]',
-  '[class*="Skeleton"]',
-  '[aria-busy="true"]',
-];
-
-async function waitForLoading(page) {
-  const maxWait = 5000;
-  const startTime = Date.now();
-  while (Date.now() - startTime < maxWait) {
-    const isLoading = await page.evaluate((sels) => {
-      if (document.readyState !== "complete") return true;
-      for (const sel of sels) {
-        const el = document.querySelector(sel);
-        if (el && el.offsetParent !== null) return true;
-      }
-      return false;
-    }, LOADING_SELECTORS);
-    if (!isLoading) return;
-    await delay(300, 600);
-  }
-}
-
-export async function scrollAndCollect(page, options) {
-  const {
-    container,
-    findScrollable = false,
-    collectFn,
-    extraArgs,
-    delayRange = [800, 1500],
-    maxItems,
-    maxRounds = 200,
-    staleThreshold = 3,
-    uniqueKey,
-    onRound,
-  } = options;
-
-  if (!collectFn) throw new Error("collectFn is required");
-
-  const fnStr =
-    typeof collectFn === "function" ? collectFn.toString() : collectFn;
-  const allItems = [];
-  const seenKeys = uniqueKey ? new Set() : null;
-  let staleCount = 0;
-
-  const processItems = (result) => {
-    const raw = result.items || [];
-    const newItems = uniqueKey
-      ? raw.filter((item) => {
-          const key = uniqueKey(item);
-          if (seenKeys.has(key)) return false;
-          seenKeys.add(key);
-          return true;
-        })
-      : raw;
-    allItems.push(...newItems);
-    return newItems;
-  };
-
-  const isDone = (newItems) => {
-    if (maxItems !== undefined && allItems.length >= maxItems) return true;
-    if (newItems.length === 0) {
-      staleCount++;
-      if (staleCount >= staleThreshold) return true;
-    } else {
-      staleCount = 0;
-    }
-    return false;
-  };
-
-  const collectCtx = { container, findScrollable, fnStr, extraArgs };
-
-  const pageError = await detectPageError(page);
-
-  if (pageError) return [];
-
-  await waitForLoading(page);
-  let result = await doCollect(page, collectCtx);
-  let newItems = processItems(result);
-  if (onRound) onRound(0, newItems, allItems);
-  if (isDone(newItems)) return allItems;
-
-  for (let round = 1; round < maxRounds; round++) {
-    await threePhaseScroll(page, { container, findScrollable });
-    await delay(delayRange[0], delayRange[1]);
-    await waitForLoading(page);
-
-    result = await doCollect(page, collectCtx);
-    newItems = processItems(result);
-
-    if (onRound) onRound(round, newItems, allItems);
-
-    if (isDone(newItems)) break;
-  }
-
-  return allItems;
-}
-
-async function threePhaseScroll(page, { container, findScrollable }) {
-  await page.evaluate(
-    async (opts) => {
-      let el;
-      if (!opts.container) {
-        el = window;
-      } else {
-        el = document.querySelector(opts.container);
-        if (!el) {
-          el = window;
-        } else if (opts.findScrollable) {
-          let current = el;
-          let found = false;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) {
-              el = current;
-              found = true;
-              break;
-            }
-            current = current.parentElement;
-          }
-          if (!found) el = document.body;
-        }
-      }
-
-      const randDelay = (min, max) =>
-        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
-
-      if (el === window) {
-        window.scrollBy(0, window.innerHeight);
-        await randDelay(400, 800);
-        window.scrollBy(0, -200);
-        await randDelay(200, 400);
-        window.scrollBy(0, window.innerHeight);
-      } else {
-        el.scrollTop = el.scrollHeight;
-        await randDelay(400, 800);
-        el.scrollTop -= 100 + Math.random() * 100;
-        await randDelay(200, 400);
-        el.scrollTop = el.scrollHeight;
-      }
-    },
-    { container, findScrollable },
-  );
-}
+import { delay } from './delay.js';
+import { detectPageError } from './page-error-detector.js';
+
+async function doCollect(
+  page,
+  { container, findScrollable, fnStr, extraArgs },
+) {
+  return page.evaluate(
+    ({ fn: fnStr, containerSelector, findScrollableFlag, args }) => {
+      let el;
+      if (!containerSelector) {
+        el = window;
+      } else {
+        el = document.querySelector(containerSelector);
+        if (!el) {
+          el = window;
+        } else if (findScrollableFlag) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const fn = eval("(" + fnStr + ")");
+      return fn(el, args);
+    },
+    { fn: fnStr, containerSelector: container, findScrollableFlag: findScrollable, args: extraArgs },
+  );
+}
+
+const LOADING_SELECTORS = [
+  '[class*="loading"]',
+  '[class*="Loading"]',
+  '[class*="spinner"]',
+  '[class*="Spinner"]',
+  '[class*="skeleton"]',
+  '[class*="Skeleton"]',
+  '[aria-busy="true"]',
+];
+
+async function waitForLoading(page) {
+  const maxWait = 5000;
+  const startTime = Date.now();
+  while (Date.now() - startTime < maxWait) {
+    const isLoading = await page.evaluate((sels) => {
+      if (document.readyState !== "complete") return true;
+      for (const sel of sels) {
+        const el = document.querySelector(sel);
+        if (el && el.offsetParent !== null) return true;
+      }
+      return false;
+    }, LOADING_SELECTORS);
+    if (!isLoading) return;
+    await delay(300, 600);
+  }
+}
+
+async function collectWithoutScroll(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+
+  const result = await doCollect(page, {
+    container,
+    findScrollable,
+    fnStr,
+    extraArgs,
+  });
+
+  return result.items || [];
+}
+
+async function scrollToBottom(page, options = {}) {
+  const {
+    container,
+    findScrollable = false,
+    delayRange = [800, 1500],
+    maxScroll = 200,
+  } = options;
+
+  for (let i = 0; i < maxScroll; i++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+  }
+}
+
+async function scrollAndCollect(page, options) {
+  const {
+    container,
+    findScrollable = false,
+    collectFn,
+    extraArgs,
+    delayRange = [800, 1500],
+    maxItems,
+    maxRounds = 200,
+    staleThreshold = 3,
+    uniqueKey,
+    onRound,
+  } = options;
+
+  if (!collectFn) throw new Error("collectFn is required");
+
+  const fnStr =
+    typeof collectFn === "function" ? collectFn.toString() : collectFn;
+  const allItems = [];
+  const seenKeys = uniqueKey ? new Set() : null;
+  let staleCount = 0;
+
+  const processItems = (result) => {
+    const raw = result.items || [];
+    const newItems = uniqueKey
+      ? raw.filter((item) => {
+          const key = uniqueKey(item);
+          if (seenKeys.has(key)) return false;
+          seenKeys.add(key);
+          return true;
+        })
+      : raw;
+    allItems.push(...newItems);
+    return newItems;
+  };
+
+  const isDone = (newItems) => {
+    if (maxItems !== undefined && allItems.length >= maxItems) return true;
+    if (newItems.length === 0) {
+      staleCount++;
+      if (staleCount >= staleThreshold) return true;
+    } else {
+      staleCount = 0;
+    }
+    return false;
+  };
+
+  const collectCtx = { container, findScrollable, fnStr, extraArgs };
+
+  const pageError = await detectPageError(page);
+
+  if (pageError) return [];
+
+  await waitForLoading(page);
+  let result = await doCollect(page, collectCtx);
+  let newItems = processItems(result);
+  if (onRound) onRound(0, newItems, allItems);
+  if (isDone(newItems)) return allItems;
+
+  for (let round = 1; round < maxRounds; round++) {
+    await threePhaseScroll(page, { container, findScrollable });
+    await delay(delayRange[0], delayRange[1]);
+    await waitForLoading(page);
+
+    result = await doCollect(page, collectCtx);
+    newItems = processItems(result);
+
+    if (onRound) onRound(round, newItems, allItems);
+
+    if (isDone(newItems)) break;
+  }
+
+  return allItems;
+}
+
+async function threePhaseScroll(page, { container, findScrollable }) {
+  await page.evaluate(
+    async (opts) => {
+      let el;
+      if (!opts.container) {
+        el = window;
+      } else {
+        el = document.querySelector(opts.container);
+        if (!el) {
+          el = window;
+        } else if (opts.findScrollable) {
+          let current = el;
+          let found = false;
+          while (current && current !== document.body) {
+            if (current.scrollHeight > current.clientHeight + 10) {
+              el = current;
+              found = true;
+              break;
+            }
+            current = current.parentElement;
+          }
+          if (!found) el = document.body;
+        }
+      }
+
+      const randDelay = (min, max) =>
+        new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
+
+      if (el === window) {
+        window.scrollBy(0, window.innerHeight);
+        await randDelay(400, 800);
+        window.scrollBy(0, -200);
+        await randDelay(200, 400);
+        window.scrollBy(0, window.innerHeight);
+      } else {
+        el.scrollTop = el.scrollHeight;
+        await randDelay(400, 800);
+        el.scrollTop -= 100 + Math.random() * 100;
+        await randDelay(200, 400);
+        el.scrollTop = el.scrollHeight;
+      }
+    },
+    { container, findScrollable },
+  );
+}
+
+export {
+  scrollAndCollect,
+  scrollToBottom,
+  collectWithoutScroll,
+  doCollect,
+  waitForLoading,
+  threePhaseScroll,
+};

package/src/lib/url.js

@@ -1,52 +1,52 @@
-const BASE_URL = 'https://www.tiktok.com';
-
-export function extractUniqueId(url) {
-  const m = url.match(/\/@([^/]+)/);
-  return m ? m[1] : null;
-}
-
-export function extractVideoId(url) {
-  const m = url.match(/\/video\/(\d+)/);
-  return m ? m[1] : null;
-}
-
-export function normalizeUsername(input) {
-  return (input || '').replace(/^@/, '');
-}
-
-export function toProfileUrl(handle) {
-  const clean = normalizeUsername(handle);
-  return `${BASE_URL}/@${clean}`;
-}
-
-export function toVideoUrl(handle, videoId) {
-  const clean = normalizeUsername(handle);
-  return `${BASE_URL}/@${clean}/video/${videoId}`;
-}
-
-export function ensureAbsoluteUrl(href) {
-  if (href.startsWith('http')) return href;
-  return `${BASE_URL}${href}`;
-}
-
-export function isProfileUrl(url) {
-  return /\/@[\w-]+(?:$|[?#])/.test(url);
-}
-
-export function isVideoUrl(url) {
-  return /\/video\/\d+/.test(url);
-}
-
-export function extractDisplayPath(url) {
-  try {
-    const parts = new URL(url).pathname.split('/').filter(Boolean);
-    return parts.slice(-2).join('/');
-  } catch {
-    return url;
-  }
-}
-
-export function extractAuthorFromVideoUrl(url) {
-  const m = url.match(/@([^/]+)\/video/);
-  return m ? '@' + m[1] : null;
-}
+const BASE_URL = 'https://www.tiktok.com';
+
+export function extractUniqueId(url) {
+  const m = url.match(/\/@([^/]+)/);
+  return m ? m[1] : null;
+}
+
+export function extractVideoId(url) {
+  const m = url.match(/\/video\/(\d+)/);
+  return m ? m[1] : null;
+}
+
+export function normalizeUsername(input) {
+  return (input || '').replace(/^@/, '');
+}
+
+export function toProfileUrl(handle) {
+  const clean = normalizeUsername(handle);
+  return `${BASE_URL}/@${clean}`;
+}
+
+export function toVideoUrl(handle, videoId) {
+  const clean = normalizeUsername(handle);
+  return `${BASE_URL}/@${clean}/video/${videoId}`;
+}
+
+export function ensureAbsoluteUrl(href) {
+  if (href.startsWith('http')) return href;
+  return `${BASE_URL}${href}`;
+}
+
+export function isProfileUrl(url) {
+  return /\/@[\w-]+(?:$|[?#])/.test(url);
+}
+
+export function isVideoUrl(url) {
+  return /\/video\/\d+/.test(url);
+}
+
+export function extractDisplayPath(url) {
+  try {
+    const parts = new URL(url).pathname.split('/').filter(Boolean);
+    return parts.slice(-2).join('/');
+  } catch {
+    return url;
+  }
+}
+
+export function extractAuthorFromVideoUrl(url) {
+  const m = url.match(/@([^/]+)\/video/);
+  return m ? '@' + m[1] : null;
+}

package/src/main.js

@@ -0,0 +1,48 @@
+import { parseArgs } from './lib/args.js';
+import { proxy, HELP_TEXT, getConfigText } from './lib/constants.js';
+import { parseFilter } from './lib/filter.js';
+import { handleScrape } from './cli/scrape.js';
+import { handleVideos } from './cli/videos.js';
+import { handleAuto } from './cli/auto.js';
+import { handleExplore } from './cli/explore.js';
+import { handleRefresh } from './cli/refresh.js';
+import { handleWatch } from './cli/watch.js';
+import { handleConfig, showConfig, showUsage, version } from './cli/config.js';
+import { runExploreDefault, runScrapeDefault } from './cli/explore-default.js';
+
+async function main() {
+  const parsed = parseArgs();
+
+  switch (parsed.subcommand) {
+    case 'scrape': return handleScrape(parsed);
+    case 'videos': return handleVideos(parsed);
+    case 'auto':   return handleAuto(parsed);
+    case 'explore':return handleExplore(parsed);
+    case 'refresh':return handleRefresh(parsed);
+    case 'watch':  return handleWatch(parsed);
+  }
+
+  const { urls, outputFile, outputFormat, exploreCount, showConfig: showCfg, showHelp, showVersion, customProxy, configAction, configKey, configValue, pipeMode, filterStr } = parsed;
+  const proxyUrl = customProxy || proxy;
+  const filter = parseFilter(filterStr);
+
+  if (showVersion) {
+    console.log(version);
+    process.exit(0);
+  }
+  if (showHelp) return showUsage();
+  if (configAction) return handleConfig(configAction, configKey, configValue);
+  if (showCfg) return showConfig(urls, outputFile);
+  if (urls.length === 0 && exploreCount === 0) return showUsage();
+
+  if (exploreCount > 0) {
+    await runExploreDefault(exploreCount, urls, proxyUrl, outputFile, outputFormat, pipeMode, filter);
+  } else {
+    await runScrapeDefault(urls, proxyUrl, outputFile, outputFormat, filter);
+  }
+}
+
+main().catch(err => {
+  console.error(`错误: ${err.message}`);
+  process.exit(1);
+});

package/src/results/user-videos-bar.lar.lar.moeta.json

@@ -0,0 +1,37 @@
+{
+  "user": {
+    "uniqueId": "bar.lar.lar.moeta",
+    "secUid": "MS4wLjABAAAA3cgKTWvKfga0JAWeakAzx3zQ-aFAC8RuQvxD4HQFraKKsc_TbOIyMo3_ofVlXofV",
+    "nickname": "Bar Lar Lar Moetain",
+    "ttSeller": false,
+    "verified": false,
+    "followerCount": 24000,
+    "videoCount": 749,
+    "followingCount": 4293,
+    "heartCount": 254300,
+    "signature": ""
+  },
+  "totalVideos": 5,
+  "videos": [
+    {
+      "id": "7638231799084158228",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638231799084158228"
+    },
+    {
+      "id": "7638162444698914068",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638162444698914068"
+    },
+    {
+      "id": "7638116251767819541",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638116251767819541"
+    },
+    {
+      "id": "7638069637321690388",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638069637321690388"
+    },
+    {
+      "id": "7637927171025112341",
+      "url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7637927171025112341"
+    }
+  ]
+}