browser-agent-server 1.0.0__py3-none-any.whl

browser_agent/chrome.py

@@ -0,0 +1,591 @@
+"""Real headful Chrome controller with zero-leak CDP (no Runtime.enable) and X11 Turnstile solver."""
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import logging
+import os
+import re
+import shutil
+import socket
+import struct
+import subprocess
+import time
+import urllib.request
+from pathlib import Path
+from urllib.parse import urlparse
+
+from .x11_input import X11Display
+
+log = logging.getLogger("browser_agent.chrome")
+
+CF_CHALLENGE_MARKERS = (
+    "just a moment...",
+    "attention required! | cloudflare",
+    "verify you are human",
+    "checking your browser before accessing",
+    "cf-turnstile",
+    "challenges.cloudflare.com/turnstile",
+    "_cf_chl_opt",
+)
+
+# Forbidden CDP domains that trigger Cloudflare / DataDome / PerimeterX detection
+FORBIDDEN_CDP_METHODS = frozenset({"Runtime.enable", "Console.enable", "Debugger.enable"})
+
+
+class StealthCDPConnection:
+    """Minimal RFC-6455 WebSocket client for Chrome DevTools Protocol without Runtime.enable leaks."""
+
+    def __init__(self, ws_url: str, timeout: float = 20.0):
+        parsed = urlparse(ws_url)
+        host = parsed.hostname or "127.0.0.1"
+        port = parsed.port or 9222
+        path = parsed.path or "/"
+        if parsed.query:
+            path = f"{path}?{parsed.query}"
+
+        self.sock = socket.create_connection((host, port), timeout=timeout)
+        self._handshake(host, port, path)
+        self._msg_id = 0
+
+    def _handshake(self, host: str, port: int, path: str) -> None:
+        raw_key = os.urandom(16)
+        sec_key = base64.b64encode(raw_key).decode("ascii")
+        req = (
+            f"GET {path} HTTP/1.1\r\n"
+            f"Host: {host}:{port}\r\n"
+            "Upgrade: websocket\r\n"
+            "Connection: Upgrade\r\n"
+            f"Sec-WebSocket-Key: {sec_key}\r\n"
+            "Sec-WebSocket-Version: 13\r\n\r\n"
+        )
+        self.sock.sendall(req.encode("ascii"))
+        resp = b""
+        while b"\r\n\r\n" not in resp:
+            chunk = self.sock.recv(4096)
+            if not chunk:
+                raise ConnectionError("WebSocket handshake closed prematurely")
+            resp += chunk
+        header_block, self._rx_buf = resp.split(b"\r\n\r\n", 1)
+        if b"101" not in header_block.splitlines()[0]:
+            raise ConnectionError(f"WebSocket handshake failed: {header_block[:120]!r}")
+        expected = base64.b64encode(
+            hashlib.sha1((sec_key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode("ascii")).digest()
+        )
+        if expected.lower() not in header_block.lower():
+            raise ConnectionError("Invalid Sec-WebSocket-Accept header")
+
+    def send_frame(self, payload: bytes, opcode: int = 0x1) -> None:
+        mask_key = os.urandom(4)
+        length = len(payload)
+        header = bytearray([0x80 | opcode])
+        if length < 126:
+            header.append(0x80 | length)
+        elif length < 65536:
+            header.append(0x80 | 126)
+            header.extend(struct.pack("!H", length))
+        else:
+            header.append(0x80 | 127)
+            header.extend(struct.pack("!Q", length))
+        header.extend(mask_key)
+        masked = bytes(b ^ mask_key[i % 4] for i, b in enumerate(payload))
+        self.sock.sendall(bytes(header) + masked)
+
+    def _read_exact(self, n: int) -> bytes:
+        while len(self._rx_buf) < n:
+            chunk = self.sock.recv(65536)
+            if not chunk:
+                raise ConnectionError("WebSocket closed while reading frame")
+            self._rx_buf += chunk
+        out, self._rx_buf = self._rx_buf[:n], self._rx_buf[n:]
+        return out
+
+    def recv_frame(self) -> bytes:
+        while True:
+            b1, b2 = self._read_exact(2)
+            opcode = b1 & 0x0F
+            masked = bool(b2 & 0x80)
+            length = b2 & 0x7F
+            if length == 126:
+                length = struct.unpack("!H", self._read_exact(2))[0]
+            elif length == 127:
+                length = struct.unpack("!Q", self._read_exact(8))[0]
+            mask_key = self._read_exact(4) if masked else b""
+            data = self._read_exact(length)
+            if masked:
+                data = bytes(b ^ mask_key[i % 4] for i, b in enumerate(data))
+            if opcode == 0x8:  # Close
+                raise ConnectionError("WebSocket closed by peer")
+            if opcode == 0x9:  # Ping -> Pong
+                self.send_frame(data, opcode=0xA)
+                continue
+            if opcode in (0x1, 0x2):
+                return data
+
+    def call(self, method: str, params: dict | None = None, timeout: float = 15.0) -> dict:
+        if method in FORBIDDEN_CDP_METHODS:
+            raise ValueError(f"Forbidden CDP method {method} blocked to prevent bot detection")
+        self._msg_id += 1
+        req_id = self._msg_id
+        msg = json.dumps({"id": req_id, "method": method, "params": params or {}}).encode("utf-8")
+        self.sock.settimeout(timeout)
+        self.send_frame(msg)
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            raw = self.recv_frame()
+            data = json.loads(raw.decode("utf-8", errors="replace"))
+            if data.get("id") == req_id:
+                if "error" in data:
+                    raise RuntimeError(f"CDP error in {method}: {data['error']}")
+                return data.get("result") or {}
+        raise TimeoutError(f"CDP call {method} timed out after {timeout}s")
+
+    def close(self) -> None:
+        try:
+            self.sock.close()
+        except OSError:
+            pass
+
+
+def find_chrome_binary() -> str:
+    for candidate in (
+        os.environ.get("CHROME_BIN", ""),
+        "google-chrome-stable",
+        "google-chrome",
+        "chromium-browser",
+        "chromium",
+        "/usr/bin/google-chrome-stable",
+        "/usr/bin/chromium-browser",
+        "/usr/bin/chromium",
+    ):
+        if candidate and shutil.which(candidate):
+            return shutil.which(candidate) or candidate
+    raise FileNotFoundError(
+        "Chrome/Chromium binary not found. Run scripts/setup_browser_agent.sh to install google-chrome-stable."
+    )
+
+
+def is_warp_proxy_ready(host: str = "127.0.0.1", port: int = 40000) -> bool:
+    """Check if local Cloudflare WARP SOCKS5 proxy is listening on 127.0.0.1:40000."""
+    try:
+        with socket.create_connection((host, port), timeout=0.4):
+            return True
+    except OSError:
+        return False
+
+
+def is_cloudflare_challenge(html: str, title: str = "") -> bool:
+    hay = f"{title}\n{html[:12000]}".lower()
+    return any(m in hay for m in CF_CHALLENGE_MARKERS)
+
+
+def _default_state_dir() -> Path:
+    """XDG state directory for the agent (profile, logs). Override with BROWSER_AGENT_STATE_DIR."""
+    if os.environ.get("BROWSER_AGENT_STATE_DIR"):
+        return Path(os.environ["BROWSER_AGENT_STATE_DIR"])
+    xdg = os.environ.get("XDG_STATE_HOME") or str(Path.home() / ".local" / "state")
+    return Path(xdg) / "browser-agent"
+
+
+class RealChromeSession:
+    """Manages a real headful Chrome window on Xvfb with hardware-level X11 mouse/keyboard control."""
+
+    def __init__(
+        self,
+        display: str = ":99",
+        width: int = 1280,
+        height: int = 720,
+        cdp_port: int = 9222,
+        user_data_dir: str | None = None,
+        proxy_url: str | None = None,
+    ):
+        self.x11 = X11Display(display=display, width=width, height=height)
+        self.cdp_port = cdp_port
+        if user_data_dir is None:
+            # Chrome profile with persisted logins. Set BROWSER_AGENT_PROFILE_DIR to keep
+            # a pre-existing profile (e.g. /tmp/browser-agent-profile on legacy installs).
+            user_data_dir = os.environ.get("BROWSER_AGENT_PROFILE_DIR") or str(_default_state_dir() / "profile")
+        self.user_data_dir = Path(user_data_dir)
+        self.proxy_url = proxy_url
+        self._active_proxy: str | None = None
+        self._chrome_proc: subprocess.Popen | None = None
+        self._stderr_path = _default_state_dir() / "chrome.err"
+        self._stderr_file = None
+        self.last_used: float = time.time()
+        # Typical Chrome top UI bar height (tabs + address bar) in openbox
+        self.chrome_top_offset: int = 82
+
+    def is_running(self) -> bool:
+        return self._chrome_proc is not None and self._chrome_proc.poll() is None
+
+    def _read_chrome_stderr(self) -> str:
+        try:
+            if self._stderr_path.exists():
+                lines = self._stderr_path.read_text(encoding="utf-8", errors="replace").strip().splitlines()
+                return " | ".join(lines[-8:])[:500]
+        except OSError:
+            pass
+        return ""
+
+    def start(self, use_proxy: bool | None = None) -> None:
+        self.last_used = time.time()
+        self.x11.start()
+        want_proxy = self.proxy_url or ("socks5://127.0.0.1:40000" if (use_proxy and is_warp_proxy_ready()) else None)
+        if self.is_running():
+            if want_proxy == self._active_proxy:
+                return
+            log.info("Restarting Chrome to switch proxy (%s -> %s)", self._active_proxy or "direct", want_proxy or "direct")
+            self._stop_chrome_only()
+
+        self.user_data_dir.mkdir(parents=True, exist_ok=True)
+        for lock_name in ("SingletonLock", "SingletonCookie", "SingletonSocket"):
+            lock_path = self.user_data_dir / lock_name
+            if lock_path.exists() or lock_path.is_symlink():
+                try:
+                    lock_path.unlink()
+                except OSError:
+                    pass
+
+        chrome_bin = find_chrome_binary()
+        args = [
+            chrome_bin,
+            f"--user-data-dir={self.user_data_dir}",
+            f"--remote-debugging-port={self.cdp_port}",
+            "--remote-debugging-address=127.0.0.1",
+            "--remote-allow-origins=*",
+            f"--window-size={self.x11.width},{self.x11.height}",
+            "--window-position=0,0",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--disable-dev-shm-usage",
+            "--disable-gpu",
+            "--disable-software-rasterizer",
+            "--disable-background-networking",
+            "--disable-background-timer-throttling",
+            "--disable-backgrounding-occluded-windows",
+            "--disable-breakpad",
+            "--disable-component-update",
+            "--disable-default-apps",
+            "--disable-extensions",
+            "--disable-hang-monitor",
+            "--disable-popup-blocking",
+            "--disable-prompt-on-repost",
+            "--disable-sync",
+            "--disable-translate",
+            "--metrics-recording-only",
+            "--no-pings",
+            "--password-store=basic",
+            "--use-mock-keychain",
+            "--renderer-process-limit=1",
+            "--process-per-site",
+            "--js-flags=--max-old-space-size=256",
+            "--disable-blink-features=AutomationControlled",
+            "--lang=en-US",
+        ]
+        if os.geteuid() == 0:
+            args.extend(["--no-sandbox", "--disable-setuid-sandbox"])
+
+        self._active_proxy = want_proxy
+        if want_proxy:
+            args.append(f"--proxy-server={want_proxy}")
+
+        args.append("about:blank")
+        log.info("Launching Chrome: bin=%s display=%s proxy=%s", chrome_bin, self.x11.display, want_proxy or "direct")
+        self._stderr_file = self._stderr_path.open("w", encoding="utf-8")
+        self._chrome_proc = subprocess.Popen(
+            args,
+            env=self.x11.env,
+            stdout=subprocess.DEVNULL,
+            stderr=self._stderr_file,
+        )
+        t0 = time.time()
+        target = self._wait_for_cdp(timeout=12.0)
+        log.info(
+            "Chrome ready in %.2fs (PID=%s, proxy=%s, ws=%s)",
+            time.time() - t0,
+            self._chrome_proc.pid,
+            want_proxy or "direct",
+            target.get("webSocketDebuggerUrl", ""),
+        )
+
+    def _wait_for_cdp(self, timeout: float = 12.0) -> dict:
+        deadline = time.time() + timeout
+        list_url = f"http://127.0.0.1:{self.cdp_port}/json/list"
+        last_err = ""
+        while time.time() < deadline:
+            if self._chrome_proc is not None and self._chrome_proc.poll() is not None:
+                rc = self._chrome_proc.returncode
+                stderr_tail = self._read_chrome_stderr()
+                raise RuntimeError(f"Chrome exited prematurely (rc={rc}): {stderr_tail}")
+            try:
+                with urllib.request.urlopen(list_url, timeout=1.5) as resp:
+                    targets = json.loads(resp.read().decode("utf-8"))
+                    for t in targets:
+                        if t.get("type") == "page" and t.get("webSocketDebuggerUrl"):
+                            return t
+                    req = urllib.request.Request(
+                        f"http://127.0.0.1:{self.cdp_port}/json/new?about:blank", method="PUT"
+                    )
+                    with urllib.request.urlopen(req, timeout=1.5) as new_resp:
+                        t = json.loads(new_resp.read().decode("utf-8"))
+                        if t.get("webSocketDebuggerUrl"):
+                            return t
+            except Exception as e:  # noqa: BLE001
+                last_err = f"{type(e).__name__}: {e}"
+                time.sleep(0.25)
+        stderr_tail = self._read_chrome_stderr()
+        raise TimeoutError(
+            f"Chrome CDP port {self.cdp_port} not ready after {timeout}s (last_err={last_err}, stderr={stderr_tail})"
+        )
+
+    def _connect_page_cdp(self) -> StealthCDPConnection:
+        target = self._wait_for_cdp(timeout=6.0)
+        return StealthCDPConnection(target["webSocketDebuggerUrl"])
+
+    def _stop_chrome_only(self) -> None:
+        if self._chrome_proc and self._chrome_proc.poll() is None:
+            self._chrome_proc.terminate()
+            try:
+                self._chrome_proc.wait(timeout=3)
+            except subprocess.TimeoutExpired:
+                self._chrome_proc.kill()
+        self._chrome_proc = None
+        if self._stderr_file:
+            try:
+                self._stderr_file.close()
+            except OSError:
+                pass
+            self._stderr_file = None
+
+    def stop(self) -> None:
+        self._stop_chrome_only()
+        self.x11.stop()
+        for p in Path("/tmp").glob("Crashpad*"):
+            shutil.rmtree(p, ignore_errors=True)
+
+    def get_html_and_title(self) -> tuple[str, str, str]:
+        """Return (current_url, page_title, outer_html) using DOM.getOuterHTML (zero Runtime.enable)."""
+        cdp = self._connect_page_cdp()
+        try:
+            doc = cdp.call("DOM.getDocument", {"depth": 1}, timeout=12.0)
+            root = doc.get("root") or {}
+            node_id = root.get("nodeId", 1)
+            cur_url = root.get("documentURL") or ""
+            outer = cdp.call("DOM.getOuterHTML", {"nodeId": node_id}, timeout=20.0).get("outerHTML", "")
+            m = re.search(r"<title[^>]*>(.*?)</title>", outer, flags=re.IGNORECASE | re.DOTALL)
+            title = re.sub(r"\s+", " ", m.group(1)).strip() if m else ""
+            return cur_url, title, outer
+        finally:
+            cdp.close()
+
+    def get_cookies(self) -> list[dict]:
+        cdp = self._connect_page_cdp()
+        try:
+            res = cdp.call("Network.getAllCookies")
+            return res.get("cookies", [])
+        except Exception:  # noqa: BLE001
+            return []
+        finally:
+            cdp.close()
+
+    def _locate_turnstile_screen_coords(self) -> tuple[int, int]:
+        """Locate the Cloudflare Turnstile checkbox in X11 screen coordinates."""
+        cdp = self._connect_page_cdp()
+        try:
+            doc = cdp.call("DOM.getDocument", {"depth": -1, "pierce": True})
+            root_id = (doc.get("root") or {}).get("nodeId", 1)
+            for selector in (
+                "iframe[src*='challenges.cloudflare.com']",
+                "iframe[id^='cf-chl-widget']",
+                ".cf-turnstile",
+                "#turnstile-wrapper",
+                "#challenge-stage",
+            ):
+                try:
+                    q = cdp.call("DOM.querySelector", {"nodeId": root_id, "selector": selector})
+                    nid = q.get("nodeId") or 0
+                    if nid > 0:
+                        box = cdp.call("DOM.getBoxModel", {"nodeId": nid}).get("model", {})
+                        content = box.get("content") or []
+                        if len(content) >= 6:
+                            left, top = float(content[0]), float(content[1])
+                            # Turnstile checkbox is ~28px from the left edge and vertically centered (~32px down)
+                            sx = int(round(left + 28))
+                            sy = int(round(top + 32 + self.chrome_top_offset))
+                            if 10 < sx < self.x11.width - 10 and 40 < sy < self.x11.height - 10:
+                                return sx, sy
+                except Exception:  # noqa: BLE001
+                    continue
+        except Exception:  # noqa: BLE001
+            pass
+        finally:
+            cdp.close()
+
+        # Standard Cloudflare full-page interstitial Turnstile checkbox position on 1280x720 window
+        return (215, 290 + self.chrome_top_offset)
+
+    def solve_cloudflare_if_present(self, max_attempts: int = 3) -> bool:
+        """Detect Cloudflare challenge and solve it using human Bezier X11 mouse movement + click."""
+        _, title, html = self.get_html_and_title()
+        if not is_cloudflare_challenge(html, title):
+            return False
+
+        log.info("Cloudflare challenge detected (%r) — solving with real X11 mouse...", title)
+        for attempt in range(1, max_attempts + 1):
+            # Give Turnstile widget 2.5s to finish initializing its iframe
+            time.sleep(2.5)
+            _, title, html = self.get_html_and_title()
+            if not is_cloudflare_challenge(html, title):
+                log.info("Cloudflare challenge auto-resolved before click")
+                return True
+
+            sx, sy = self._locate_turnstile_screen_coords()
+            # Perform natural warm-up mouse movement across the viewport first
+            self.x11.move_mouse(
+                sx + 140,
+                max(120, sy - 60),
+                duration=0.25,
+            )
+            time.sleep(0.15)
+            log.info("Attempt %d/%d: clicking Turnstile checkbox at X11 (%d, %d)", attempt, max_attempts, sx, sy)
+            self.x11.click(sx, sy, jitter=3)
+            time.sleep(4.0)
+
+            _, title, html = self.get_html_and_title()
+            if not is_cloudflare_challenge(html, title):
+                log.info("Cloudflare Turnstile passed on attempt %d!", attempt)
+                return True
+
+        return False
+
+    def fetch_page(
+        self,
+        url: str,
+        wait_sec: float = 3.5,
+        solve_cloudflare: bool = True,
+        use_proxy: bool | None = None,
+        include_screenshot: bool = False,
+    ) -> dict:
+        """Open URL in real headful Chrome, solve Cloudflare if needed, and return rendered HTML & text."""
+        res = self._fetch_page_once(
+            url=url,
+            wait_sec=wait_sec,
+            solve_cloudflare=solve_cloudflare,
+            use_proxy=bool(use_proxy),
+            include_screenshot=include_screenshot,
+        )
+        # Auto-retry via local Cloudflare WARP proxy (127.0.0.1:40000) if direct datacenter IP was blocked
+        if (
+            use_proxy is None
+            and (res.get("blocked") or len(res.get("text") or "") < 160)
+            and is_warp_proxy_ready()
+        ):
+            log.info("Direct fetch blocked or empty for %s — retrying via WARP proxy (127.0.0.1:40000)", url)
+            res = self._fetch_page_once(
+                url=url,
+                wait_sec=wait_sec,
+                solve_cloudflare=solve_cloudflare,
+                use_proxy=True,
+                include_screenshot=include_screenshot,
+            )
+        return res
+
+    def _fetch_page_once(
+        self,
+        url: str,
+        wait_sec: float,
+        solve_cloudflare: bool,
+        use_proxy: bool,
+        include_screenshot: bool,
+    ) -> dict:
+        self.start(use_proxy=use_proxy)
+        self.last_used = time.time()
+
+        log.info("🌐 Navigating to %s (proxy=%s, wait=%.1fs)", url, self._active_proxy or "direct", wait_sec)
+        cdp = self._connect_page_cdp()
+        try:
+            try:
+                cdp.call("Page.navigate", {"url": url}, timeout=9.0)
+            except TimeoutError:
+                log.warning("⚠️ Page.navigate timed out on slow subresources for %s — stopping load & reading DOM", url)
+                try:
+                    cdp.call("Page.stopLoading", timeout=3.0)
+                except Exception:  # noqa: BLE001
+                    pass
+        finally:
+            cdp.close()
+
+        time.sleep(max(1.0, float(wait_sec)))
+        cur_url, title, html = self.get_html_and_title()
+        log.info("📄 Initial DOM: url=%s title=%r html_len=%d cf=%s",
+                 cur_url, title[:60], len(html), is_cloudflare_challenge(html, title))
+
+        cf_solved = False
+        if solve_cloudflare:
+            cf_solved = self.solve_cloudflare_if_present()
+            if cf_solved:
+                time.sleep(1.5)
+
+        self.x11.scroll(clicks=3, direction="down", x=self.x11.width // 2, y=self.x11.height // 2)
+        time.sleep(0.5)
+
+        final_url, title, html = self.get_html_and_title()
+        blocked = is_cloudflare_challenge(html, title)
+        text = _extract_text_from_html(html, final_url or url)
+        log.info(
+            "✅ Fetch finished: url=%s title=%r html_len=%d text_len=%d blocked=%s cf_solved=%s proxy=%s",
+            final_url or url, title[:60], len(html), len(text), blocked, cf_solved, self._active_proxy or "direct",
+        )
+        screenshot_b64 = ""
+        if include_screenshot:
+            try:
+                screenshot_b64 = base64.b64encode(self.x11.screenshot_bytes()).decode("ascii")
+            except Exception:  # noqa: BLE001
+                pass
+
+        try:
+            cdp = self._connect_page_cdp()
+            try:
+                cdp.call("Page.navigate", {"url": "about:blank"})
+            finally:
+                cdp.close()
+        except Exception:  # noqa: BLE001
+            pass
+
+        return {
+            "ok": not blocked and bool(html),
+            "blocked": blocked,
+            "cloudflare_solved": cf_solved,
+            "via_warp": bool(self._active_proxy),
+            "url": final_url or url,
+            "title": title,
+            "html": html,
+            "text": text,
+            "cookies": self.get_cookies(),
+            "screenshot_b64": screenshot_b64,
+        }
+
+
+def _extract_text_from_html(html: str, url: str = "") -> str:
+    if not html:
+        return ""
+    try:
+        import trafilatura
+
+        extracted = trafilatura.extract(
+            html,
+            url=url or None,
+            include_comments=False,
+            include_tables=False,
+            favor_recall=True,
+        )
+        if extracted and len(extracted.strip()) >= 120:
+            return extracted.strip()
+    except Exception:  # noqa: BLE001
+        pass
+
+    # Fallback plain-text strip
+    cleaned = re.sub(r"<(script|style|noscript|svg|header|footer|nav)[^>]*>.*?</\1>", " ", html, flags=re.I | re.S)
+    cleaned = re.sub(r"<[^>]+>", " ", cleaned)
+    return re.sub(r"\s+", " ", cleaned).strip()