smart-web-mcp 0.5.19 → 0.5.21

package/scripts/undetected_chromedriver_fetch.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+import json
+import re
+import subprocess
+import sys
+import time
+from typing import Any
+
+
+def fail(code: str, message: str, category: str = "unavailable"):
+    print(
+        json.dumps(
+            {
+                "ok": False,
+                "method": "undetected_chromedriver",
+                "error": {
+                    "category": category,
+                    "code": code,
+                    "message": message,
+                },
+            }
+        )
+    )
+    raise SystemExit(0)
+
+
+payload = {}
+try:
+    payload = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
+except Exception as exc:  # noqa: BLE001
+    fail("uc_invalid_payload", str(exc), "parse_error")
+
+uc: Any = None
+try:
+    import undetected_chromedriver as uc  # type: ignore[import-not-found]
+except Exception as exc:  # noqa: BLE001
+    fail("uc_import_failed", str(exc))
+
+
+def run_js(driver, script, *args):
+    try:
+        return driver.execute_script(script, *args)
+    except Exception:  # noqa: BLE001
+        return None
+
+
+url = str(payload.get("url") or "").strip()
+timeout_ms = int(payload.get("timeoutMs") or 25000)
+scroll_steps = max(0, min(12, int(payload.get("scrollSteps") or 0)))
+max_load_more_clicks = max(0, min(10, int(payload.get("maxLoadMoreClicks") or 0)))
+max_anchors = max(50, min(2000, int(payload.get("maxAnchors") or 400)))
+wait_until = str(payload.get("waitUntil") or "networkidle")
+user_agent = str(payload.get("userAgent") or "Mozilla/5.0")
+chrome_path = str(payload.get("chromePath") or "").strip()
+extra_headers = (
+    payload.get("extraHeaders") if isinstance(payload.get("extraHeaders"), dict) else {}
+)
+
+if not url:
+    fail("uc_missing_url", "URL is required", "parse_error")
+
+
+def chrome_major_version(binary_path: str):
+    commands = (
+        [binary_path]
+        if binary_path
+        else ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
+    )
+    for command in commands:
+        if not command:
+            continue
+        try:
+            completed = subprocess.run(
+                [command, "--version"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+                check=False,
+            )
+        except Exception:
+            continue
+        text = f"{completed.stdout} {completed.stderr}"
+        match = re.search(r"(\d+)\.\d+\.\d+\.\d+", text)
+        if match:
+            return int(match.group(1))
+    return None
+
+
+options = uc.ChromeOptions()
+options.add_argument("--headless=new")
+options.add_argument("--disable-blink-features=AutomationControlled")
+options.add_argument("--no-sandbox")
+options.add_argument("--disable-dev-shm-usage")
+options.add_argument("--window-size=1440,2400")
+options.add_argument("--lang=ko-KR")
+options.add_argument(f"--user-agent={user_agent}")
+if chrome_path:
+    options.binary_location = chrome_path
+
+driver = None
+
+try:
+    version_main = chrome_major_version(chrome_path)
+    driver = uc.Chrome(
+        options=options,
+        headless=True,
+        browser_executable_path=chrome_path or None,
+        use_subprocess=True,
+        version_main=version_main,
+    )
+    if extra_headers:
+        driver.execute_cdp_cmd("Network.enable", {})
+        driver.execute_cdp_cmd(
+            "Network.setExtraHTTPHeaders",
+            {"headers": {str(key): str(value) for key, value in extra_headers.items()}},
+        )
+    driver.set_page_load_timeout(max(5, timeout_ms // 1000))
+    driver.get(url)
+    time.sleep(1.0 if wait_until == "domcontentloaded" else 2.0)
+
+    load_more_script = """
+      const pattern = /^(load more|show more|more posts|more results|view more|see more|더보기|더 보기|더 불러오기)$/i;
+      const nodes = Array.from(document.querySelectorAll('button, [role="button"], input[type="button"], input[type="submit"], a'));
+      for (const node of nodes) {
+        const text = (node.innerText || node.getAttribute('aria-label') || node.getAttribute('title') || '').replace(/\s+/g, ' ').trim();
+        if (!text || !pattern.test(text)) continue;
+        if (node.tagName === 'A') {
+          const href = node.getAttribute('href') || '';
+          if (href && href !== '#' && !href.toLowerCase().startsWith('javascript:')) continue;
+        }
+        const rect = node.getBoundingClientRect();
+        if (rect.width <= 0 || rect.height <= 0) continue;
+        node.click();
+        return true;
+      }
+      return false;
+    """
+
+    previous_height = (
+        run_js(
+            driver,
+            "return document.body?.scrollHeight || document.documentElement?.scrollHeight || 0",
+        )
+        or 0
+    )
+    stagnant = 0
+    load_more_clicks = 0
+    completed_scrolls = 0
+    for _ in range(scroll_steps):
+        clicked = False
+        if load_more_clicks < max_load_more_clicks:
+            clicked = bool(run_js(driver, load_more_script))
+            if clicked:
+                load_more_clicks += 1
+                time.sleep(0.7)
+        run_js(
+            driver,
+            "window.scrollTo(0, document.body?.scrollHeight || document.documentElement?.scrollHeight || 0)",
+        )
+        time.sleep(0.8)
+        completed_scrolls += 1
+        current_height = (
+            run_js(
+                driver,
+                "return document.body?.scrollHeight || document.documentElement?.scrollHeight || 0",
+            )
+            or previous_height
+        )
+        if not clicked and current_height <= previous_height:
+            stagnant += 1
+        else:
+            stagnant = 0
+        previous_height = current_height
+        if stagnant >= 2:
+            break
+
+    extract_script = """
+      const maxItems = arguments[0];
+      const selectors = ['main','article','[role="main"]','.theme-doc-markdown','.theme-doc-markdown.markdown','.markdown','.md-content','.VPDoc','.vp-doc','.prose','.documentation','.docs-content','.content'];
+      const nodes = Array.from(document.querySelectorAll('a[href]')).slice(0, maxItems).map((element) => {
+        const closest = element.closest('article, li, tr, section, main, .topic-list-item, .topic-body, .topic-post, .athing, .titleline, [role="article"], .feed-item, .timeline-item, .post, .storylink');
+        const text = (element.innerText || element.getAttribute('aria-label') || element.getAttribute('title') || '').replace(/\s+/g, ' ').trim();
+        const contextSource = (closest && closest.innerText) || element.innerText || element.textContent || '';
+        return {
+          href: element.href || '',
+          text,
+          context: contextSource.replace(/\s+/g, ' ').trim().slice(0, 500),
+          rel: element.rel || '',
+          class_name: typeof element.className === 'string' ? element.className : '',
+          aria_label: element.getAttribute('aria-label') || '',
+        };
+      });
+      const candidates = selectors.flatMap((selector) => Array.from(document.querySelectorAll(selector)));
+      const unique = Array.from(new Set(candidates));
+      const best = unique.map((node) => ({ node, score: (node.innerText || '').trim().length })).sort((a, b) => b.score - a.score)[0]?.node || document.body;
+      const clone = best instanceof HTMLElement ? best.cloneNode(true) : document.body.cloneNode(true);
+      if (clone instanceof HTMLElement) {
+        clone.querySelectorAll('script, style, noscript, svg, canvas, iframe, form, button, input, textarea, select, nav, header, footer, aside').forEach((node) => node.remove());
+        clone.querySelectorAll('.sidebar, .toc, .table-of-contents, .breadcrumb, .breadcrumbs, .edit-this-page, .theme-edit-this-page, .theme-doc-toc-desktop, .theme-doc-toc-mobile, .pagination-nav').forEach((node) => node.remove());
+      }
+      const headings = clone instanceof HTMLElement ? Array.from(clone.querySelectorAll('h1, h2, h3, h4, h5, h6')).map((node) => ({ level: Number(node.tagName.slice(1)), text: (node.textContent || '').replace(/\s+/g, ' ').trim(), id: node.id || '' })).filter((item) => item.text) : [];
+      return {
+        title: document.title || '',
+        html: document.documentElement ? document.documentElement.outerHTML : '',
+        text: document.body ? document.body.innerText || '' : '',
+        final_url: window.location.href,
+        anchors: nodes,
+        content_html: clone instanceof HTMLElement ? clone.innerHTML : '',
+        content_text: clone instanceof HTMLElement ? (clone.innerText || '').replace(/\s+/g, ' ').trim() : '',
+        headings,
+      };
+    """
+
+    extracted = run_js(driver, extract_script, max_anchors) or {}
+    anchors = extracted.get("anchors") if isinstance(extracted, dict) else []
+    links = []
+    seen = set()
+    for anchor in anchors or []:
+        href = str((anchor or {}).get("href") or "").strip()
+        if href and href not in seen:
+            seen.add(href)
+            links.append(href)
+
+    print(
+        json.dumps(
+            {
+                "ok": True,
+                "method": "undetected_chromedriver",
+                "url": url,
+                "final_url": str(
+                    (extracted or {}).get("final_url") or driver.current_url or url
+                ),
+                "title": str((extracted or {}).get("title") or ""),
+                "html": str((extracted or {}).get("html") or ""),
+                "text": str((extracted or {}).get("text") or ""),
+                "content_html": str((extracted or {}).get("content_html") or ""),
+                "content_text": str((extracted or {}).get("content_text") or ""),
+                "headings": (extracted or {}).get("headings") or [],
+                "links": links,
+                "anchors": anchors or [],
+                "scroll_steps": completed_scrolls,
+                "load_more_clicks": load_more_clicks,
+            }
+        )
+    )
+except Exception as exc:  # noqa: BLE001
+    message = str(exc)
+    fail(
+        "uc_fetch_failed",
+        message,
+        "timeout" if "timeout" in message.lower() else "unavailable",
+    )
+finally:
+    if driver is not None:
+        try:
+            driver.quit()
+        except Exception:
+            pass