google-browser-scraper 0.0.1__tar.gz

google_browser_scraper-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 NodeMaven
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

google_browser_scraper-0.0.1/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: google-browser-scraper
+Version: 0.0.1
+Summary: Google Search results from a real browser through your own sticky proxy, with captchas and blocks reported explicitly.
+Author: NodeMaven
+License: MIT
+Project-URL: Source, https://github.com/nodemaven/google-browser-scraper
+Project-URL: Issues, https://github.com/nodemaven/google-browser-scraper/issues
+Project-URL: Benchmark, https://github.com/nodemaven/proxy-benchmark
+Keywords: google,serp,scraper,search,proxy,playwright,patchright
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: patchright==1.62.2
+Requires-Dist: lxml>=5
+Provides-Extra: nodemaven
+Requires-Dist: nodemaven>=0.1.5; extra == "nodemaven"
+Provides-Extra: cloak
+Requires-Dist: cloakbrowser>=0.5; extra == "cloak"
+Provides-Extra: all
+Requires-Dist: nodemaven>=0.1.5; extra == "all"
+Requires-Dist: cloakbrowser>=0.5; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Dynamic: license-file
+
+# google-browser-scraper
+
+Google Search results from a real browser, through your own sticky proxy.
+
+Plain HTTP is no longer a reliable way to scrape Google: it often expects a
+browser that runs JavaScript, and a fresh browser on a fresh proxy exit can be
+refused quickly. This tool takes a browser path instead. It warms a sticky proxy
+session with ordinary browsing, types the query into Google's search box, checks
+whether Google served results or refused, and writes one JSON record per results
+page. Captchas, blocks and failures are reported as such, never as "no results".
+
+## Install
+
+```
+pip install google-browser-scraper
+patchright install chromium
+```
+
+## Quickstart
+
+You need a residential proxy with **sticky sessions**. Put `{session}` where
+your provider expects a session id in the username:
+
+```
+google-browser-scraper search "best running shoes" \
+  --proxy "http://USER-session-{session}:PASS@gate.example.com:7000" \
+  -o results.jsonl
+```
+
+Success is a `results.jsonl` with one JSON object per results page:
+
+```json
+{
+  "search_metadata": {"status": "Success", "page_verdict": "ok"},
+  "search_parameters": {"engine": "google", "q": "best running shoes", "page": 1},
+  "organic_results": [
+    {"position": 1, "title": "...", "link": "https://...", "snippet": "..."}
+  ],
+  "related_questions": [{"question": "..."}],
+  "exit": {"session": "...", "queries_on_exit": 1, "bytes": "..."}
+}
+```
+
+The first query takes a few minutes, because a new exit is warmed up first, and
+the warm-up costs noticeably more traffic than a search. Later queries on the
+same exit take seconds. A summary of pages served and traffic used is printed at
+the end of every run.
+
+With NodeMaven, the session id is handled for you and the credentials come from
+the environment:
+
+```
+pip install "google-browser-scraper[nodemaven]"
+export NODEMAVEN_LOGIN=... NODEMAVEN_PASSWORD=...
+google-browser-scraper search -f queries.txt --nodemaven --country us -o results.jsonl
+```
+
+From Python:
+
+```python
+from google_browser_scraper import ProxyTemplate, Scraper, Settings
+
+proxy = ProxyTemplate("http://USER-session-{session}:PASS@gate.example.com:7000")
+for page in Scraper(proxy, Settings(pages=2)).run(["best running shoes"]):
+    for result in page.get("organic_results", []):
+        print(result["position"], result["title"], result["link"])
+```
+
+## What you get
+
+The schema is intentionally SerpApi-like: `organic_results`, `ads`,
+`related_questions`, `related_searches`, `pagination` and `search_metadata`, so
+simple consumers can often switch with minimal mapping. Knowledge panels,
+shopping, images and other special blocks are not parsed.
+
+Each record also has an `exit` block: which proxy session answered and how many
+bytes the page cost. A query that failed is a record too, with an `error`.
+`--format csv` writes the organic results as a table, and `--price-per-gb` adds
+the cost per 1000 pages to the summary.
+
+## How it works
+
+1. **New identity**: a sticky proxy session and a fresh browser profile.
+2. **Warm-up**: a few ordinary pages, then Google's front page.
+3. **Search**: the query is typed into the box and submitted.
+4. **Check before parsing**: the page is classified first, so a captcha or a
+   block is reported as one and never parsed as an empty result.
+5. **Keep or drop**: an exit Google answers is reused for the next queries; an
+   exit it refuses is dropped and the query is retried on a new one. If exit
+   after exit is refused, the run stops instead of spending more traffic.
+
+Google wraps result links in encrypted `/goto` redirects. Each one is resolved
+with a small request through a separate short-lived proxy session, so link
+resolution does not touch the warmed browser session. `--no-resolve-links`
+skips it.
+
+Traffic goes through a small local relay that counts bytes and blocks a large
+model download Chrome otherwise makes on every fresh profile.
+
+The defaults come from measurements in
+[proxy-benchmark](https://github.com/nodemaven/proxy-benchmark).
+
+## Proxies
+
+Any provider with sticky sessions works through `--proxy` and a `{session}`
+placeholder. A proxy without one is accepted with a warning: if the exit changes
+on every request, warming it does nothing. SOCKS5 needs an IP-whitelisted
+endpoint, because Chrome cannot send a SOCKS5 username and password.
+
+## Run it as an API
+
+```
+google-browser-scraper serve --nodemaven --country us --prewarm
+curl "http://127.0.0.1:8000/search.json?q=best+running+shoes"
+```
+
+Same record format as above. Requests run one at a time on one held identity.
+Set `GBS_API_KEY` to require a key; without one the server only listens on
+localhost.
+
+## Use it from an AI agent (MCP)
+
+```json
+{
+  "mcpServers": {
+    "google-search": {
+      "command": "google-browser-scraper",
+      "args": ["mcp", "--nodemaven", "--country", "us", "--prewarm"],
+      "env": {"NODEMAVEN_LOGIN": "...", "NODEMAVEN_PASSWORD": "..."}
+    }
+  }
+}
+```
+
+It exposes one tool, `google_search(query, pages)`.
+
+## Docker
+
+```
+docker build -t google-browser-scraper .
+docker run --rm google-browser-scraper doctor
+docker run --rm -p 8000:8000 -e GBS_API_KEY=change-me \
+  -e GBS_PROXY='http://USER-session-{session}:PASS@gate.example.com:7000' \
+  google-browser-scraper serve --host 0.0.0.0
+```
+
+## Check your machine
+
+```
+google-browser-scraper doctor
+```
+
+Starts the browser on a blank page, sends nothing, and reports what a website
+would see: whether the browser looks headless, whether WebGL works, the screen
+size. Google answers some machines much less than others, so run this first.
+
+## What this is not
+
+Not a free Google Search API, and not a guarantee that every proxy exit will
+work. It is a browser-based collector that makes failures explicit, drops exits
+Google refuses, and tells you what each page cost.
+
+## Status
+
+Early. Tested on Windows. **Linux and Docker are experimental**: on a machine
+without a GPU, Chrome has no WebGL, which a normal desktop always has; the
+Docker image turns on a software renderer for it. The `cloak` engine
+(`--engine cloak`) is experimental too; `patchright` is the default.
+
+Google changes its pages often. If results come back empty or wrong, run with
+`--save-html DIR` and open an issue with what you see.
+
+You are responsible for using this within Google's terms and the law where you
+run it.
+
+## Adding an engine
+
+An engine joins this tool only after it has been measured on
+[proxy-benchmark](https://github.com/nodemaven/proxy-benchmark): add it there,
+run the warm-up ladder against Google -
+
+```
+python scripts/run_ladder.py --engines your-engine
+```
+
+- and open a pull request here with the run file. The ladder runs identities
+with and without the warm-up in the same hours, so the result shows whether the
+engine actually gets served, not whether it got lucky once.
+
+## License
+
+MIT

google_browser_scraper-0.0.1/README.md

@@ -0,0 +1,192 @@
+# google-browser-scraper
+
+Google Search results from a real browser, through your own sticky proxy.
+
+Plain HTTP is no longer a reliable way to scrape Google: it often expects a
+browser that runs JavaScript, and a fresh browser on a fresh proxy exit can be
+refused quickly. This tool takes a browser path instead. It warms a sticky proxy
+session with ordinary browsing, types the query into Google's search box, checks
+whether Google served results or refused, and writes one JSON record per results
+page. Captchas, blocks and failures are reported as such, never as "no results".
+
+## Install
+
+```
+pip install google-browser-scraper
+patchright install chromium
+```
+
+## Quickstart
+
+You need a residential proxy with **sticky sessions**. Put `{session}` where
+your provider expects a session id in the username:
+
+```
+google-browser-scraper search "best running shoes" \
+  --proxy "http://USER-session-{session}:PASS@gate.example.com:7000" \
+  -o results.jsonl
+```
+
+Success is a `results.jsonl` with one JSON object per results page:
+
+```json
+{
+  "search_metadata": {"status": "Success", "page_verdict": "ok"},
+  "search_parameters": {"engine": "google", "q": "best running shoes", "page": 1},
+  "organic_results": [
+    {"position": 1, "title": "...", "link": "https://...", "snippet": "..."}
+  ],
+  "related_questions": [{"question": "..."}],
+  "exit": {"session": "...", "queries_on_exit": 1, "bytes": "..."}
+}
+```
+
+The first query takes a few minutes, because a new exit is warmed up first, and
+the warm-up costs noticeably more traffic than a search. Later queries on the
+same exit take seconds. A summary of pages served and traffic used is printed at
+the end of every run.
+
+With NodeMaven, the session id is handled for you and the credentials come from
+the environment:
+
+```
+pip install "google-browser-scraper[nodemaven]"
+export NODEMAVEN_LOGIN=... NODEMAVEN_PASSWORD=...
+google-browser-scraper search -f queries.txt --nodemaven --country us -o results.jsonl
+```
+
+From Python:
+
+```python
+from google_browser_scraper import ProxyTemplate, Scraper, Settings
+
+proxy = ProxyTemplate("http://USER-session-{session}:PASS@gate.example.com:7000")
+for page in Scraper(proxy, Settings(pages=2)).run(["best running shoes"]):
+    for result in page.get("organic_results", []):
+        print(result["position"], result["title"], result["link"])
+```
+
+## What you get
+
+The schema is intentionally SerpApi-like: `organic_results`, `ads`,
+`related_questions`, `related_searches`, `pagination` and `search_metadata`, so
+simple consumers can often switch with minimal mapping. Knowledge panels,
+shopping, images and other special blocks are not parsed.
+
+Each record also has an `exit` block: which proxy session answered and how many
+bytes the page cost. A query that failed is a record too, with an `error`.
+`--format csv` writes the organic results as a table, and `--price-per-gb` adds
+the cost per 1000 pages to the summary.
+
+## How it works
+
+1. **New identity**: a sticky proxy session and a fresh browser profile.
+2. **Warm-up**: a few ordinary pages, then Google's front page.
+3. **Search**: the query is typed into the box and submitted.
+4. **Check before parsing**: the page is classified first, so a captcha or a
+   block is reported as one and never parsed as an empty result.
+5. **Keep or drop**: an exit Google answers is reused for the next queries; an
+   exit it refuses is dropped and the query is retried on a new one. If exit
+   after exit is refused, the run stops instead of spending more traffic.
+
+Google wraps result links in encrypted `/goto` redirects. Each one is resolved
+with a small request through a separate short-lived proxy session, so link
+resolution does not touch the warmed browser session. `--no-resolve-links`
+skips it.
+
+Traffic goes through a small local relay that counts bytes and blocks a large
+model download Chrome otherwise makes on every fresh profile.
+
+The defaults come from measurements in
+[proxy-benchmark](https://github.com/nodemaven/proxy-benchmark).
+
+## Proxies
+
+Any provider with sticky sessions works through `--proxy` and a `{session}`
+placeholder. A proxy without one is accepted with a warning: if the exit changes
+on every request, warming it does nothing. SOCKS5 needs an IP-whitelisted
+endpoint, because Chrome cannot send a SOCKS5 username and password.
+
+## Run it as an API
+
+```
+google-browser-scraper serve --nodemaven --country us --prewarm
+curl "http://127.0.0.1:8000/search.json?q=best+running+shoes"
+```
+
+Same record format as above. Requests run one at a time on one held identity.
+Set `GBS_API_KEY` to require a key; without one the server only listens on
+localhost.
+
+## Use it from an AI agent (MCP)
+
+```json
+{
+  "mcpServers": {
+    "google-search": {
+      "command": "google-browser-scraper",
+      "args": ["mcp", "--nodemaven", "--country", "us", "--prewarm"],
+      "env": {"NODEMAVEN_LOGIN": "...", "NODEMAVEN_PASSWORD": "..."}
+    }
+  }
+}
+```
+
+It exposes one tool, `google_search(query, pages)`.
+
+## Docker
+
+```
+docker build -t google-browser-scraper .
+docker run --rm google-browser-scraper doctor
+docker run --rm -p 8000:8000 -e GBS_API_KEY=change-me \
+  -e GBS_PROXY='http://USER-session-{session}:PASS@gate.example.com:7000' \
+  google-browser-scraper serve --host 0.0.0.0
+```
+
+## Check your machine
+
+```
+google-browser-scraper doctor
+```
+
+Starts the browser on a blank page, sends nothing, and reports what a website
+would see: whether the browser looks headless, whether WebGL works, the screen
+size. Google answers some machines much less than others, so run this first.
+
+## What this is not
+
+Not a free Google Search API, and not a guarantee that every proxy exit will
+work. It is a browser-based collector that makes failures explicit, drops exits
+Google refuses, and tells you what each page cost.
+
+## Status
+
+Early. Tested on Windows. **Linux and Docker are experimental**: on a machine
+without a GPU, Chrome has no WebGL, which a normal desktop always has; the
+Docker image turns on a software renderer for it. The `cloak` engine
+(`--engine cloak`) is experimental too; `patchright` is the default.
+
+Google changes its pages often. If results come back empty or wrong, run with
+`--save-html DIR` and open an issue with what you see.
+
+You are responsible for using this within Google's terms and the law where you
+run it.
+
+## Adding an engine
+
+An engine joins this tool only after it has been measured on
+[proxy-benchmark](https://github.com/nodemaven/proxy-benchmark): add it there,
+run the warm-up ladder against Google -
+
+```
+python scripts/run_ladder.py --engines your-engine
+```
+
+- and open a pull request here with the run file. The ladder runs identities
+with and without the warm-up in the same hours, so the result shows whether the
+engine actually gets served, not whether it got lucky once.
+
+## License
+
+MIT

google_browser_scraper-0.0.1/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "google-browser-scraper"
+version = "0.0.1"
+description = "Google Search results from a real browser through your own sticky proxy, with captchas and blocks reported explicitly."
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.10"
+authors = [{ name = "NodeMaven" }]
+keywords = ["google", "serp", "scraper", "search", "proxy", "playwright", "patchright"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+]
+dependencies = [
+    # Pinned, because each patchright release ships a different Chromium and
+    # the browser build is part of what Google sees. 1.62.2 is Chromium 151,
+    # the build every live run of this tool was made with.
+    "patchright==1.62.2",
+    "lxml>=5",
+]
+
+[project.optional-dependencies]
+# The NodeMaven SDK builds sticky-session usernames for its gateway. Any other
+# provider works through a proxy URL with a `{session}` placeholder instead.
+nodemaven = ["nodemaven>=0.1.5"]
+# CloakBrowser's patched Chromium; downloads its own binary on first use.
+cloak = ["cloakbrowser>=0.5"]
+all = ["nodemaven>=0.1.5", "cloakbrowser>=0.5"]
+dev = ["pytest>=8", "ruff>=0.6"]
+
+[project.scripts]
+google-browser-scraper = "google_browser_scraper.cli:main"
+
+[project.urls]
+Source = "https://github.com/nodemaven/google-browser-scraper"
+Issues = "https://github.com/nodemaven/google-browser-scraper/issues"
+Benchmark = "https://github.com/nodemaven/proxy-benchmark"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "B", "UP"]

google_browser_scraper-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

google_browser_scraper-0.0.1/src/google_browser_scraper/__init__.py

@@ -0,0 +1,27 @@
+"""Google search results from a real browser through your own proxy.
+
+from google_browser_scraper import Scraper, Settings, ProxyTemplate
+
+proxy = ProxyTemplate("http://user-session-{session}:pass@gate.example.com:7000")
+for record in Scraper(proxy, Settings(pages=2)).run(["best running shoes"]):
+    print(record["organic_results"])
+"""
+
+__version__ = "0.0.1"
+
+from .classify import Verdict, classify  # noqa: E402
+from .parse import parse_serp  # noqa: E402
+from .proxy import NodeMavenSource, ProxyTemplate  # noqa: E402
+from .scraper import ExitsRefused, Scraper, Settings  # noqa: E402
+
+__all__ = [
+    "ExitsRefused",
+    "NodeMavenSource",
+    "ProxyTemplate",
+    "Scraper",
+    "Settings",
+    "Verdict",
+    "__version__",
+    "classify",
+    "parse_serp",
+]

google_browser_scraper-0.0.1/src/google_browser_scraper/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())