tt-help-cli-ycl 1.0.7 → 1.1.0

package/src/scraper/modules/comment-extractor.cjs

@@ -1,122 +1,49 @@
-const { delay, getDelayConfig } = require('./page-helpers.cjs');
-
-async function openCommentPanel(page) {
-  const tabs = page.locator('[class*="tabbar-item"]');
-  const commentTab = tabs.filter({ hasText: '评论' }).first();
-  await commentTab.click();
-  const config = getDelayConfig();
-  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
-  await page.waitForSelector('[class*="CommentListContainer"]', { timeout: 5000 }).catch(() => {});
-}
-
-async function extractCommentAuthors(page, maxComments = 10) {
-  await openCommentPanel(page);
-
-  const config = getDelayConfig();
-  const allAuthors = [];
-  let maxScrollHeight = 0;
-  let noChangeCount = 0;
-  const maxNoChange = 3;
-  let scrollAttempts = 0;
-  const maxScrollAttempts = 30;
-
-  while (allAuthors.length < maxComments && scrollAttempts < maxScrollAttempts) {
-    await page.evaluate(() => {
-      try {
-        const selectors = ['.DivCommentMain', '[class*="CommentMain"]'];
-        let container = null;
-        for (const sel of selectors) {
-          container = document.querySelector(sel);
-          if (container) break;
-        }
-        if (!container) return;
-
-        function findScrollableAncestor(el) {
-          if (el.scrollHeight > el.clientHeight + 10) return el;
-          let current = el.parentElement;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) return current;
-            current = current.parentElement;
-          }
-          return el;
-        }
-
-        const scrollContainer = findScrollableAncestor(container);
-        scrollContainer.scrollTop = scrollContainer.scrollHeight;
-      } catch (e) {}
-    });
-
-    await new Promise(r => setTimeout(r, 500));
-
-    let evalResult;
-    try {
-      evalResult = await page.evaluate(() => {
-        const selectors = ['.DivCommentMain', '[class*="CommentMain"]'];
-        let container = null;
-        for (const sel of selectors) {
-          container = document.querySelector(sel);
-          if (container) break;
-        }
-        if (!container) return { authors: [], scrollHeight: 0, containerFound: false, listFound: false };
-
-        function findScrollableAncestor(el) {
-          if (el.scrollHeight > el.clientHeight + 10) return el;
-          let current = el.parentElement;
-          while (current && current !== document.body) {
-            if (current.scrollHeight > current.clientHeight + 10) return current;
-            current = current.parentElement;
-          }
-          return el;
-        }
-
-        const scrollContainer = findScrollableAncestor(container);
-        const currentHeight = scrollContainer.scrollHeight;
-
-        const lc = document.querySelector('[class*="CommentListContainer"]');
-        if (!lc) return { authors: [], scrollHeight: currentHeight, containerFound: true, listFound: false };
-
-        const result = [];
-        Array.from(lc.children).forEach(w => {
-          const authorEl = w.querySelector('[class*="UsernameContentWrapper"]');
-          if (!authorEl) return;
-          const link = authorEl.querySelector('a');
-          if (link && link.href) {
-            const m = link.href.match(/@([^/]+)/);
-            if (m) result.push('@' + m[1]);
-          }
-        });
-        return { authors: result, scrollHeight: currentHeight, containerFound: true, listFound: true };
-      });
-    } catch (e) {
-      break;
-    }
-
-    const { authors, scrollHeight, containerFound, listFound } = evalResult;
-
-    if (!containerFound || !listFound) {
-      break;
-    }
-
-    allAuthors.push(...authors);
-
-    if (scrollHeight === maxScrollHeight) {
-      noChangeCount++;
-      if (noChangeCount >= maxNoChange) {
-        break;
-      }
-    } else {
-      noChangeCount = 0;
-    }
-    maxScrollHeight = scrollHeight;
-
-    if (allAuthors.length >= maxComments) break;
-    scrollAttempts++;
-    await delay(Math.round(config.commentMax * 0.3), config.commentMax);
-  }
-
-  return [...new Set(allAuthors.slice(0, maxComments))];
-}
-
-module.exports = {
-  extractCommentAuthors,
-};
+const { delay, getDelayConfig, closeCommentPanel } = require('./page-helpers.cjs');
+const { scrollAndCollect } = require('./scroll-collector.cjs');
+
+async function openCommentPanel(page) {
+  const tabs = page.locator('[class*="tabbar-item"]');
+  const commentTab = tabs.filter({ hasText: '评论' }).first();
+  await commentTab.click();
+  const config = getDelayConfig();
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+  await page.waitForSelector('[class*="CommentListContainer"]', { timeout: 5000 }).catch(() => {});
+}
+
+async function extractCommentAuthors(page, maxComments = 10) {
+  await openCommentPanel(page);
+
+  const config = getDelayConfig();
+  const allAuthors = await scrollAndCollect(page, {
+    container: '[class*="CommentMain"]',
+    findScrollable: true,
+    collectFn: (container) => {
+      const list = document.querySelector('[class*="CommentListContainer"]');
+      if (!list) return { items: [] };
+
+      const authors = [];
+      Array.from(list.children).forEach(wrapper => {
+        const link = wrapper.querySelector('[class*="UsernameContentWrapper"] a');
+        if (link) {
+          const href = link.href || link.getAttribute('href');
+          const m = href && href.match(/@([^/]+)/);
+          if (m) authors.push('@' + m[1]);
+        }
+      });
+      return { items: authors };
+    },
+    uniqueKey: (a) => a,
+    maxItems: maxComments,
+    delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
+    staleThreshold: 3,
+  });
+
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  return allAuthors.slice(0, maxComments);
+}
+
+module.exports = {
+  extractCommentAuthors,
+};

package/src/scraper/modules/follow-extractor.cjs

@@ -0,0 +1,112 @@
+const { delay, getDelayConfig } = require('./page-helpers.cjs');
+const { scrollAndCollect } = require('./scroll-collector.cjs');
+
+const FILTER_WORDS = ['主页', '已关注', '粉丝', '推荐'];
+
+async function openFollowModal(page) {
+  const el = await page.$('[data-e2e=following]');
+  if (!el) {
+    throw new Error('未找到 [data-e2e=following] 元素，请确认当前页面为用户主页');
+  }
+  await el.evaluate(el => el.parentElement.click());
+  await delay(2000, 3000);
+
+  await page.waitForSelector('[class*=DivUserListContainer]', { timeout: 5000 }).catch(() => {
+    throw new Error('关注弹窗未出现 DivUserListContainer');
+  });
+}
+
+async function switchToFollowersTab(page) {
+  await page.evaluate(() => {
+    const tabs = document.querySelectorAll('[class*=DivTabItem]');
+    for (const tab of tabs) {
+      if (tab.textContent?.includes('粉丝')) {
+        tab.click();
+        return;
+      }
+    }
+    throw new Error('未找到粉丝 Tab');
+  });
+  await delay(2000, 3000);
+  await page.waitForSelector('[class*=DivUserListContainer]', { timeout: 5000 }).catch(() => {});
+}
+
+async function closeFollowModal(page) {
+  await page.evaluate(() => {
+    const closeBtn = document.querySelector('[data-e2e=follow-popup-close]');
+    if (closeBtn) closeBtn.click();
+  });
+  await delay(1000, 2000);
+}
+
+function createUserCollectFn() {
+  return (container) => {
+    const FILTER_WORDS = ['主页', '已关注', '粉丝', '推荐'];
+    const modal = document.querySelector('[class*=eyhy6180]');
+    const root = modal || document;
+    const users = [];
+    const seen = new Set();
+    const links = root.querySelectorAll('a[href*="/@"]');
+    for (const link of links) {
+      const match = link.href.match(/@([^/?]+)/);
+      if (!match) continue;
+      const handle = '@' + decodeURIComponent(match[1]);
+      const text = (link.textContent || '').trim();
+      if (text.length <= 2) continue;
+      if (FILTER_WORDS.includes(text)) continue;
+      if (seen.has(handle)) continue;
+      seen.add(handle);
+      users.push({ handle, displayName: text });
+    }
+    return { items: users };
+  };
+}
+
+async function extractUsersFromModal(page, maxUsers) {
+  const config = getDelayConfig();
+  const allUsers = await scrollAndCollect(page, {
+    container: '[class*=DivUserListContainer]',
+    findScrollable: false,
+    collectFn: createUserCollectFn(),
+    uniqueKey: (u) => u.handle,
+    maxItems: maxUsers,
+    delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
+    staleThreshold: 5,
+  });
+
+  return allUsers.slice(0, maxUsers);
+}
+
+async function extractFollowAndFollowers(page, options = {}) {
+  const {
+    maxFollowing = 999,
+    maxFollowers = 999,
+    log = () => {},
+  } = options;
+
+  log('打开关注弹窗...');
+  await openFollowModal(page);
+
+  log('提取关注列表...');
+  const following = await extractUsersFromModal(page, maxFollowing);
+  log(`  已关注: ${following.length}`);
+
+  log('切换到粉丝 Tab...');
+  await switchToFollowersTab(page);
+
+  log('提取粉丝列表...');
+  const followers = await extractUsersFromModal(page, maxFollowers);
+  log(`  粉丝: ${followers.length}`);
+
+  log('关闭弹窗...');
+  await closeFollowModal(page);
+
+  return {
+    following: following.map(u => [u.handle, u.displayName]),
+    followers: followers.map(u => [u.handle, u.displayName]),
+  };
+}
+
+module.exports = {
+  extractFollowAndFollowers,
+};

package/src/scraper/modules/guess-extractor.cjs

@@ -0,0 +1,53 @@
+const { delay, getDelayConfig, closeCommentPanel } = require('./page-helpers.cjs');
+const { scrollAndCollect } = require('./scroll-collector.cjs');
+
+async function openGuessTab(page) {
+  const tabs = page.locator('[class*="tabbar-item"]');
+  const guessTab = tabs.filter({ hasText: /猜你喜欢/i }).first();
+  await guessTab.click();
+  const config = getDelayConfig();
+  await delay(Math.round(config.commentMax * 0.5), config.commentMax);
+  await page.waitForSelector('[class*="Related"]', { timeout: 5000 }).catch(() => {});
+}
+
+async function extractGuessVideos(page, maxVideos = 10) {
+  await openGuessTab(page);
+
+  const config = getDelayConfig();
+  const allVideos = await scrollAndCollect(page, {
+    container: '[class*="Related"]',
+    findScrollable: true,
+    collectFn: (container) => {
+      const items = [];
+      Array.from(container.querySelectorAll('[class*="DivItemContainer"]')).forEach(item => {
+        const link = item.querySelector('a[href*="/video/"]');
+        if (link) {
+          const href = link.href || link.getAttribute('href');
+          const m = href && href.match(/@([^/]+)\/video\/(\d+)/);
+          if (m) {
+            items.push({
+              author: '@' + m[1],
+              videoId: m[2],
+              url: href,
+              title: '',
+            });
+          }
+        }
+      });
+      return { items };
+    },
+    uniqueKey: (v) => v.videoId,
+    maxItems: maxVideos,
+    delayRange: [Math.round(config.commentMax * 0.3), config.commentMax],
+    staleThreshold: 3,
+  });
+
+  await closeCommentPanel(page);
+  await delay(Math.round(config.commentMax * 0.3), config.commentMax);
+
+  return allVideos.slice(0, maxVideos);
+}
+
+module.exports = {
+  extractGuessVideos,
+};