google-browser-scraper 0.0.1__py3-none-any.whl

google_browser_scraper/__init__.py

@@ -0,0 +1,27 @@
+"""Google search results from a real browser through your own proxy.
+
+from google_browser_scraper import Scraper, Settings, ProxyTemplate
+
+proxy = ProxyTemplate("http://user-session-{session}:pass@gate.example.com:7000")
+for record in Scraper(proxy, Settings(pages=2)).run(["best running shoes"]):
+    print(record["organic_results"])
+"""
+
+__version__ = "0.0.1"
+
+from .classify import Verdict, classify  # noqa: E402
+from .parse import parse_serp  # noqa: E402
+from .proxy import NodeMavenSource, ProxyTemplate  # noqa: E402
+from .scraper import ExitsRefused, Scraper, Settings  # noqa: E402
+
+__all__ = [
+    "ExitsRefused",
+    "NodeMavenSource",
+    "ProxyTemplate",
+    "Scraper",
+    "Settings",
+    "Verdict",
+    "__version__",
+    "classify",
+    "parse_serp",
+]

google_browser_scraper/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

google_browser_scraper/browser.py

@@ -0,0 +1,203 @@
+"""One browser identity: one profile, one proxy session, one cookie jar.
+
+- Patchright, headful: headless builds announce `HeadlessChrome` in the UA.
+- A persistent context with `no_viewport=True`, so the page reports the real
+  screen rather than an emulated one.
+- `locale` is never set: setting it through the context makes the main thread
+  and Web Workers disagree. `timezone_id` goes through CDP emulation and agrees
+  in workers, and is off unless asked for.
+- The query is typed into the box, not sent as `/search?q=`.
+"""
+
+from __future__ import annotations
+
+import random
+import shutil
+import tempfile
+import time
+from dataclasses import dataclass
+
+HOME_URL = "https://www.google.com/?hl=en"
+SEARCH_BOX = "textarea[name='q'], input[name='q']"
+READY = "#rso, #search"
+# Reject first: it sets the smaller cookie. The middle selector is the
+# redirect form of the wall served to EU exits.
+CONSENT_BUTTONS = (
+    "button#W0wltc",
+    'form[action="https://consent.google.com/save"]'
+    ':has(input[name="set_eom"][value="true"]) button',
+    "button#L2AGLb",
+)
+NAV_TIMEOUT_MS = 60_000
+WARM_TIMEOUT_MS = 30_000
+
+
+@dataclass(frozen=True)
+class Fetched:
+    """What one navigation left on screen."""
+
+    url: str
+    status: int | None
+    html: str
+    consent_dismissed: bool = False
+
+
+class BrowserSession:
+    """A Patchright persistent context bound to one proxy session.
+
+    Use as a context manager. The profile directory is temporary and removed on
+    close, so an identity is never reused by accident after it is retired.
+
+    `browser_args` are extra Chromium switches. **Never pass a
+    `--disable-features` here**: Chromium honours only the last such switch, so
+    one from the caller silently discards Playwright's own list, which includes
+    `OptimizationHints`.
+    """
+
+    name = "patchright"
+
+    def __init__(
+        self,
+        proxy: dict[str, str] | None = None,
+        *,
+        headless: bool = False,
+        channel: str | None = None,
+        timezone_id: str | None = None,
+        browser_args: list[str] | tuple[str, ...] = (),
+        rng: random.Random | None = None,
+    ) -> None:
+        if any(a.startswith(("--disable-features", "--enable-features")) for a in browser_args):
+            raise ValueError(
+                "--disable-features/--enable-features would replace the browser's "
+                "own list rather than add to it; not accepted"
+            )
+        self._proxy = proxy
+        self._headless = headless
+        self._channel = channel
+        self._timezone_id = timezone_id
+        self._args = list(browser_args)
+        self._rng = rng or random.Random()
+        self._pw = None
+        self._context = None
+        self._profile = None
+        self.page = None
+
+    def __enter__(self) -> BrowserSession:
+        self._profile = tempfile.mkdtemp(prefix="gbs-profile-")
+        try:
+            self._context = self._launch(self._profile)
+        except Exception:
+            self.close()
+            raise
+        pages = self._context.pages
+        self.page = pages[0] if pages else self._context.new_page()
+        return self
+
+    def _launch(self, profile: str):
+        from patchright.sync_api import sync_playwright
+
+        self._pw = sync_playwright().start()
+        return self._pw.chromium.launch_persistent_context(
+            user_data_dir=profile,
+            headless=self._headless,
+            channel=self._channel,
+            proxy=self._proxy,
+            no_viewport=True,
+            timezone_id=self._timezone_id,
+            args=self._args or None,
+        )
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    def close(self) -> None:
+        for step in (
+            lambda: self._context and self._context.close(),
+            lambda: self._pw and self._pw.stop(),
+        ):
+            try:
+                step()
+            except Exception:
+                pass
+        if self._profile:
+            shutil.rmtree(self._profile, ignore_errors=True)
+        self._context = self._pw = self._profile = self.page = None
+
+    # -- warming -------------------------------------------------------------
+
+    def visit(self, url: str, dwell_seconds: float) -> bool:
+        """Open a warm-up page and stay on it. Returns whether it arrived."""
+        try:
+            self.page.goto(url, wait_until="domcontentloaded", timeout=WARM_TIMEOUT_MS)
+        except Exception:
+            return False
+        time.sleep(dwell_seconds)
+        return True
+
+    # -- searching -----------------------------------------------------------
+
+    def search(self, query: str) -> Fetched:
+        """Type `query` into Google's box and return the page it led to."""
+        page = self.page
+        if not any(h.is_visible() for h in page.query_selector_all(SEARCH_BOX)):
+            page.goto(HOME_URL, wait_until="domcontentloaded", timeout=NAV_TIMEOUT_MS)
+        dismissed = self._dismiss_consent()
+        # Act on the handle that was found, never on the selector again: the
+        # selector also matches a hidden field, and `page.click` would pick it.
+        box = page.wait_for_selector(SEARCH_BOX, state="visible", timeout=NAV_TIMEOUT_MS)
+        box.click()
+        # A results page keeps the previous query in the box. Typing after it
+        # appends, and the appended query returns a real page for the wrong
+        # question. Select it so the typing replaces it.
+        if box.input_value():
+            box.press("ControlOrMeta+a")
+        box.type(query, delay=self._rng.randint(45, 140))
+        typed = box.input_value()
+        if typed != query:
+            raise RuntimeError(f"search box holds {typed!r} after typing {query!r}; not submitting")
+        # `no_wait_after=True` keeps `press` from waiting for the navigation a
+        # second time with Playwright's own 30 s default under ours.
+        with page.expect_navigation(wait_until="domcontentloaded", timeout=NAV_TIMEOUT_MS) as nav:
+            box.press("Enter", no_wait_after=True, timeout=NAV_TIMEOUT_MS)
+        return self._snapshot(nav.value, dismissed)
+
+    def next_page(self) -> Fetched | None:
+        """Click "Next" on the current results page, if there is one."""
+        link = self.page.query_selector("a#pnnext")
+        if link is None or not link.is_visible():
+            return None
+        with self.page.expect_navigation(
+            wait_until="domcontentloaded", timeout=NAV_TIMEOUT_MS
+        ) as nav:
+            link.click()
+        return self._snapshot(nav.value, False)
+
+    def _snapshot(self, response, dismissed: bool) -> Fetched:
+        # Google builds results in the browser; reading at domcontentloaded
+        # catches the "enable JavaScript" scaffold. A refusal never grows the
+        # container, so the short timeout is expected to expire on those.
+        try:
+            self.page.wait_for_selector(READY, state="attached", timeout=8_000)
+        except Exception:
+            pass
+        return Fetched(
+            url=self.page.url,
+            status=response.status if response is not None else None,
+            html=self.page.content(),
+            consent_dismissed=dismissed,
+        )
+
+    def _dismiss_consent(self) -> bool:
+        """Clear the consent overlay if one is up. It is intermittent, so
+        finding none is normal and not an error."""
+        for selector in CONSENT_BUTTONS:
+            for handle in self.page.query_selector_all(selector):
+                if handle.is_visible():
+                    handle.click(timeout=10_000)
+                    try:
+                        self.page.wait_for_load_state("domcontentloaded", timeout=10_000)
+                    except Exception:
+                        pass
+                    self.page.wait_for_timeout(500)
+                    return True
+        return False

google_browser_scraper/classify.py

@@ -0,0 +1,105 @@
+"""Decide what Google served before anything tries to read results out of it.
+
+A refusal must never reach the parser, because a parser handed a refusal returns
+zero results, and "Google found nothing" and "Google refused this exit" need
+opposite responses: the first is an answer, the second means drop the exit and
+ask again from a fresh one.
+
+Only structural markers are used - ids, paths and endpoints - so a translated
+page classifies the same as an English one.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+OK = "ok"
+NO_RESULTS = "no_results"
+SORRY = "sorry"
+WALL = "wall"
+CONSENT = "consent"
+NO_JS = "no_js"
+BLOCK = "block"
+EMPTY = "empty"
+
+#: Google looked at this exit and refused it. Drop the exit.
+REFUSED = frozenset({SORRY, WALL, BLOCK})
+#: Google answered the query. Keep the exit.
+SERVED = frozenset({OK, NO_RESULTS})
+
+
+@dataclass(frozen=True)
+class Verdict:
+    kind: str
+    reason: str
+
+    @property
+    def served(self) -> bool:
+        return self.kind in SERVED
+
+    @property
+    def refused(self) -> bool:
+        return self.kind in REFUSED
+
+
+def classify(url: str | None, html: str | None, status: int | None = None) -> Verdict:
+    """Classify one Google response from its final URL, body and status.
+
+    Order matters. A served page can carry a dormant consent panel, so the
+    results test runs before the inline consent test; reversing them scores
+    working pages as interstitials.
+    """
+    if not html:
+        return Verdict(EMPTY, "no body")
+
+    parts = urlsplit(url or "")
+    host = parts.netloc.lower()
+    path = parts.path.lower()
+    low = html.lower()
+
+    # The refusal redirects to /sorry/ with a reCAPTCHA. It has arrived as 429
+    # and as 200 with the same body, so the status is a hint and not the rule.
+    if host.startswith("sorry.") or path.startswith("/sorry"):
+        return Verdict(SORRY, "redirected to /sorry/ with a captcha")
+    # A redirect stub that was not followed: a tiny body pointing at /sorry/
+    # is the refusal, not a page.
+    if len(html) < 2000 and "/sorry/" in low:
+        return Verdict(SORRY, "short body pointing at /sorry/")
+    if status == 429:
+        return Verdict(SORRY, "HTTP 429")
+    if "unusual traffic from your computer" in low:
+        return Verdict(SORRY, "unusual-traffic page")
+    if host.startswith("consent."):
+        return Verdict(CONSENT, "consent interstitial")
+
+    has_rso = 'id="rso"' in low
+    has_search = 'id="search"' in low
+    if has_rso:
+        return Verdict(OK, "results container present")
+
+    # A real empty SERP keeps `#search`. Retiring an exit on it would throw away
+    # a working identity for a query that simply has no results.
+    if has_search:
+        return Verdict(NO_RESULTS, "search container present, no results in it")
+
+    if "consent.google.com" in low:
+        return Verdict(CONSENT, "consent wall served inline, no results behind it")
+
+    # The "enable JavaScript" scaffold. It also carries `emsg=SG_REL`, so it is
+    # tested before the wall below. From a browser that ran scripts it means
+    # Google declined to render, so the scraper drops the exit.
+    if "enablejs" in low:
+        return Verdict(NO_JS, "served the no-JavaScript scaffold")
+
+    # A SERP-shaped page with an error marker and no results container: a
+    # refusal that does not redirect.
+    if "emsg=sg_rel" in low:
+        return Verdict(WALL, "SERP-shaped wall without results (emsg=SG_REL)")
+
+    if "<noscript" in low and "<h3" not in low:
+        return Verdict(NO_JS, "a scaffold with no results and a noscript block")
+
+    if "error 403" in low or status == 403:
+        return Verdict(BLOCK, "HTTP 403 page")
+    return Verdict(BLOCK, "no results and no known interstitial")

google_browser_scraper/cli.py

@@ -0,0 +1,281 @@
+"""Command line: `search`, `serve`, `mcp`, `parse` and `doctor`."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shlex
+import sys
+from pathlib import Path
+
+from . import __version__, engines, warmup
+from .classify import classify
+from .output import WRITERS, open_writer, use_utf8
+from .parse import parse_serp
+from .proxy import NodeMavenSource, ProxyTemplate
+from .scraper import ExitsRefused, Scraper, Settings
+
+
+def main(argv: list[str] | None = None) -> int:
+    use_utf8(sys.stdout)
+    use_utf8(sys.stderr, errors="backslashreplace")
+    parser = _parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 2
+    try:
+        return args.handler(args)
+    except (ValueError, RuntimeError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="google-browser-scraper",
+        description="Google results from a real browser through your own proxy.",
+    )
+    parser.add_argument("--version", action="version", version=__version__)
+    sub = parser.add_subparsers(dest="command")
+
+    search = sub.add_parser("search", help="run queries and write results")
+    search.add_argument("queries", nargs="*", help="queries; or use --file")
+    search.add_argument("-f", "--file", type=Path, help="one query per line")
+    _scraper_options(search)
+    search.add_argument("--pages", type=int, default=1, help="result pages per query (default 1)")
+    search.add_argument("--format", choices=list(WRITERS), default="jsonl")
+    search.add_argument("-o", "--output", type=Path, help="output file (default stdout)")
+    search.add_argument(
+        "--price-per-gb",
+        type=float,
+        help="your proxy price, to print the cost per 1000 pages at the end",
+    )
+    search.set_defaults(handler=_search)
+
+    serve = sub.add_parser("serve", help="SerpApi-style HTTP API: /search.json?q=...")
+    _scraper_options(serve)
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8000)
+    serve.add_argument("--max-pages", type=int, default=3)
+    serve.add_argument("--prewarm", action="store_true", help="warm an exit at startup")
+    serve.set_defaults(handler=_serve)
+
+    mcp = sub.add_parser("mcp", help="MCP server over stdio with a google_search tool")
+    _scraper_options(mcp)
+    mcp.add_argument("--prewarm", action="store_true", help="warm an exit at startup")
+    mcp.set_defaults(handler=_mcp)
+
+    parse = sub.add_parser("parse", help="parse a saved results page, offline")
+    parse.add_argument("file", type=Path)
+    parse.set_defaults(handler=_parse)
+
+    doctor = sub.add_parser("doctor", help="check this machine's browser; sends nothing")
+    doctor.add_argument("--engine", choices=list(engines.ENGINES), default=engines.DEFAULT_ENGINE)
+    doctor.add_argument("--headless", action="store_true")
+    doctor.add_argument("--channel")
+    doctor.add_argument("--browser-arg", action="append", default=[], dest="browser_args")
+    doctor.set_defaults(handler=_doctor)
+    return parser
+
+
+def _scraper_options(p: argparse.ArgumentParser) -> None:
+    source = p.add_mutually_exclusive_group()
+    source.add_argument(
+        "--proxy",
+        help="proxy URL with a {session} placeholder for sticky sessions; also read from GBS_PROXY",
+    )
+    source.add_argument(
+        "--nodemaven",
+        action="store_true",
+        help="NodeMaven gateway from NODEMAVEN_LOGIN / NODEMAVEN_PASSWORD",
+    )
+    source.add_argument("--no-proxy", action="store_true", help="use this machine's own connection")
+    p.add_argument("--country", help="exit country for --nodemaven, e.g. us")
+    p.add_argument(
+        "--engine",
+        choices=list(engines.ENGINES),
+        default=engines.DEFAULT_ENGINE,
+        help="browser engine: patchright (default) or the experimental cloak",
+    )
+    p.add_argument(
+        "--warmup",
+        choices=list(warmup.LADDERS),
+        default=warmup.DEFAULT,
+        help=f"pages opened before the first query on an exit (default {warmup.DEFAULT})",
+    )
+    p.add_argument(
+        "--max-per-exit",
+        type=int,
+        default=10,
+        help="queries before an exit is retired even if still served",
+    )
+    p.add_argument(
+        "--gap",
+        type=_pair,
+        default=(8.0, 20.0),
+        metavar="LOW,HIGH",
+        help="seconds between queries on one exit (default 8,20)",
+    )
+    p.add_argument(
+        "--headless",
+        action="store_true",
+        help="run without a window. Not recommended: see `doctor`",
+    )
+    p.add_argument("--channel", help="browser channel, e.g. chrome to use installed Chrome")
+    p.add_argument("--timezone", help="IANA timezone for the browser")
+    p.add_argument(
+        "--browser-arg",
+        action="append",
+        default=[],
+        dest="browser_args",
+        help="extra Chromium switch, repeatable; also GBS_BROWSER_ARGS",
+    )
+    p.add_argument(
+        "--no-relay",
+        action="store_true",
+        help="hand the proxy to the browser directly, without the traffic count",
+    )
+    p.add_argument(
+        "--save-html",
+        type=Path,
+        metavar="DIR",
+        help="keep every page's HTML here, gzipped (they carry the exit's location)",
+    )
+    p.add_argument(
+        "--no-resolve-links",
+        action="store_true",
+        help="leave result links as Google /goto redirects (saves one small request per result)",
+    )
+    p.add_argument("-v", "--verbose", action="store_true", help="log identities to stderr")
+
+
+def _build(args, *, pages: int = 1) -> tuple[Scraper, str]:
+    if args.nodemaven:
+        proxy = NodeMavenSource(country=args.country)
+    elif args.no_proxy:
+        proxy = None
+    else:
+        url = args.proxy or os.environ.get("GBS_PROXY")
+        if not url:
+            raise ValueError("give --proxy, --nodemaven or --no-proxy")
+        proxy = ProxyTemplate(url)
+    if args.country and not args.nodemaven:
+        raise ValueError(
+            "--country only applies to --nodemaven; put the country in your "
+            "provider's proxy URL instead"
+        )
+    if proxy is not None and not proxy.sticky:
+        print(
+            "warning: the proxy URL has no {session} placeholder, so the exit may change "
+            "between requests and warming it does nothing",
+            file=sys.stderr,
+        )
+    browser_args = tuple(args.browser_args) + tuple(
+        shlex.split(os.environ.get("GBS_BROWSER_ARGS", ""))
+    )
+    settings = Settings(
+        engine=args.engine,
+        warmup=args.warmup,
+        pages=pages,
+        max_queries_per_exit=args.max_per_exit,
+        gap_seconds=args.gap,
+        headless=args.headless,
+        channel=args.channel,
+        timezone_id=args.timezone,
+        browser_args=browser_args,
+        use_relay=not args.no_relay,
+        save_html=str(args.save_html) if args.save_html else None,
+        resolve_links=not args.no_resolve_links,
+    )
+    log = (lambda e: print(json.dumps(e), file=sys.stderr)) if args.verbose else None
+    described = proxy.describe() if proxy else "none (this machine's own address)"
+    if args.verbose:
+        print(f"proxy: {described}; engine: {args.engine}", file=sys.stderr)
+    return Scraper(proxy, settings, on_event=log), described
+
+
+def _search(args) -> int:
+    queries = list(args.queries)
+    if args.file:
+        queries += args.file.read_text(encoding="utf-8").splitlines()
+    if not any(q.strip() for q in queries):
+        raise ValueError("no queries given")
+    scraper, _ = _build(args, pages=args.pages)
+    stream = args.output.open("w", encoding="utf-8", newline="") if args.output else sys.stdout
+    writer = open_writer(args.format, stream)
+    failed, code = 0, 0
+    try:
+        for record in scraper.run(queries):
+            failed += record["search_metadata"]["status"] != "Success"
+            writer.write(record)
+    except ExitsRefused as exc:
+        print(f"stopped: {exc}", file=sys.stderr)
+        code = 3
+    finally:
+        writer.close()
+        if args.output:
+            stream.close()
+        print("summary: " + json.dumps(scraper.stats.summary(args.price_per_gb)), file=sys.stderr)
+    return code or (4 if failed else 0)
+
+
+def _serve(args) -> int:
+    from .server import Worker, serve
+
+    worker = Worker(lambda: _build(args)[0], prewarm=args.prewarm).start()
+    serve(worker, host=args.host, port=args.port, max_pages=args.max_pages)
+    return 0
+
+
+def _mcp(args) -> int:
+    from .mcp import McpServer
+    from .server import Worker
+
+    worker = Worker(lambda: _build(args)[0], prewarm=args.prewarm).start()
+    try:
+        McpServer(worker).serve()
+    finally:
+        worker.stop()
+    return 0
+
+
+def _parse(args) -> int:
+    page = args.file.read_text(encoding="utf-8", errors="replace")
+    verdict = classify("https://www.google.com/search", page)
+    out = {"page_verdict": verdict.kind, "verdict_reason": verdict.reason}
+    if verdict.served:
+        out.update(parse_serp(page))
+    json.dump(out, sys.stdout, ensure_ascii=False, indent=2)
+    sys.stdout.write("\n")
+    return 0 if verdict.served else 4
+
+
+def _doctor(args) -> int:
+    from .doctor import FAIL, assess, read_fingerprint
+
+    browser_args = tuple(args.browser_args) + tuple(
+        shlex.split(os.environ.get("GBS_BROWSER_ARGS", ""))
+    )
+    fp = read_fingerprint(
+        engine=args.engine, headless=args.headless, channel=args.channel, browser_args=browser_args
+    )
+    checks = assess(fp)
+    width = max(len(c.name) for c in checks)
+    print(f"engine: {args.engine}")
+    for c in checks:
+        print(f"[{c.status:4}] {c.name:<{width}}  {c.detail}")
+    return 1 if any(c.status == FAIL for c in checks) else 0
+
+
+def _pair(text: str) -> tuple[float, float]:
+    try:
+        low, high = (float(x) for x in text.split(","))
+    except ValueError:
+        raise argparse.ArgumentTypeError("expected LOW,HIGH, e.g. 8,20") from None
+    return low, high
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())