serpmaven 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. serpmaven-0.2.2/LICENSE +21 -0
  2. serpmaven-0.2.2/MANIFEST.in +1 -0
  3. serpmaven-0.2.2/PKG-INFO +547 -0
  4. serpmaven-0.2.2/README.md +515 -0
  5. serpmaven-0.2.2/pyproject.toml +74 -0
  6. serpmaven-0.2.2/setup.cfg +4 -0
  7. serpmaven-0.2.2/src/serpmaven/__init__.py +27 -0
  8. serpmaven-0.2.2/src/serpmaven/__main__.py +5 -0
  9. serpmaven-0.2.2/src/serpmaven/browser.py +485 -0
  10. serpmaven-0.2.2/src/serpmaven/classify.py +105 -0
  11. serpmaven-0.2.2/src/serpmaven/cli.py +535 -0
  12. serpmaven-0.2.2/src/serpmaven/doctor.py +144 -0
  13. serpmaven-0.2.2/src/serpmaven/engines.py +53 -0
  14. serpmaven-0.2.2/src/serpmaven/env.py +18 -0
  15. serpmaven-0.2.2/src/serpmaven/hints.py +45 -0
  16. serpmaven-0.2.2/src/serpmaven/hybrid.py +580 -0
  17. serpmaven-0.2.2/src/serpmaven/links.py +358 -0
  18. serpmaven-0.2.2/src/serpmaven/mcp.py +195 -0
  19. serpmaven-0.2.2/src/serpmaven/output.py +151 -0
  20. serpmaven-0.2.2/src/serpmaven/parse.py +560 -0
  21. serpmaven-0.2.2/src/serpmaven/pool.py +202 -0
  22. serpmaven-0.2.2/src/serpmaven/progress.py +179 -0
  23. serpmaven-0.2.2/src/serpmaven/proxy.py +194 -0
  24. serpmaven-0.2.2/src/serpmaven/quality.py +93 -0
  25. serpmaven-0.2.2/src/serpmaven/relay.py +340 -0
  26. serpmaven-0.2.2/src/serpmaven/scraper.py +1844 -0
  27. serpmaven-0.2.2/src/serpmaven/server.py +538 -0
  28. serpmaven-0.2.2/src/serpmaven/warmup.py +44 -0
  29. serpmaven-0.2.2/src/serpmaven.egg-info/PKG-INFO +547 -0
  30. serpmaven-0.2.2/src/serpmaven.egg-info/SOURCES.txt +51 -0
  31. serpmaven-0.2.2/src/serpmaven.egg-info/dependency_links.txt +1 -0
  32. serpmaven-0.2.2/src/serpmaven.egg-info/entry_points.txt +2 -0
  33. serpmaven-0.2.2/src/serpmaven.egg-info/requires.txt +17 -0
  34. serpmaven-0.2.2/src/serpmaven.egg-info/top_level.txt +1 -0
  35. serpmaven-0.2.2/tests/data/google_browser_scraper_021/google_browser_scraper/__init__.py +45 -0
  36. serpmaven-0.2.2/tests/fakegoogle.py +96 -0
  37. serpmaven-0.2.2/tests/pages.py +149 -0
  38. serpmaven-0.2.2/tests/stdio_mcp_child.py +39 -0
  39. serpmaven-0.2.2/tests/test_browser_local.py +179 -0
  40. serpmaven-0.2.2/tests/test_classify.py +58 -0
  41. serpmaven-0.2.2/tests/test_cli_and_output.py +330 -0
  42. serpmaven-0.2.2/tests/test_engines_local.py +162 -0
  43. serpmaven-0.2.2/tests/test_env_and_legacy.py +137 -0
  44. serpmaven-0.2.2/tests/test_first_run.py +364 -0
  45. serpmaven-0.2.2/tests/test_hybrid.py +516 -0
  46. serpmaven-0.2.2/tests/test_links.py +539 -0
  47. serpmaven-0.2.2/tests/test_parse.py +303 -0
  48. serpmaven-0.2.2/tests/test_pool.py +271 -0
  49. serpmaven-0.2.2/tests/test_proxy.py +178 -0
  50. serpmaven-0.2.2/tests/test_relay.py +247 -0
  51. serpmaven-0.2.2/tests/test_scraper.py +1845 -0
  52. serpmaven-0.2.2/tests/test_serve_and_mcp.py +741 -0
  53. serpmaven-0.2.2/tests/test_warmup.py +15 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 NodeMaven
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ recursive-include tests *.py
@@ -0,0 +1,547 @@
1
+ Metadata-Version: 2.4
2
+ Name: serpmaven
3
+ Version: 0.2.2
4
+ Summary: SERP Maven: self-hosted SERP API. Search results as JSON from a real Chrome browser through your own residential proxy. HTTP API, CLI and MCP server.
5
+ Author: NodeMaven
6
+ License: MIT
7
+ Project-URL: Source, https://github.com/nodemaven/serpmaven
8
+ Project-URL: Issues, https://github.com/nodemaven/serpmaven/issues
9
+ Project-URL: Benchmark, https://github.com/nodemaven/proxy-benchmark
10
+ Keywords: google,google-search,google-search-api,serp,serp-api,scraper,search,self-hosted,proxy,mcp,playwright,patchright
11
+ Classifier: Development Status :: 2 - Pre-Alpha
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
15
+ Requires-Python: >=3.10
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: patchright==1.64.3
19
+ Requires-Dist: lxml>=5
20
+ Requires-Dist: curl_cffi==0.17.0
21
+ Provides-Extra: nodemaven
22
+ Requires-Dist: nodemaven>=0.1.5; extra == "nodemaven"
23
+ Provides-Extra: cloak
24
+ Requires-Dist: cloakbrowser>=0.5; extra == "cloak"
25
+ Provides-Extra: all
26
+ Requires-Dist: nodemaven>=0.1.5; extra == "all"
27
+ Requires-Dist: cloakbrowser>=0.5; extra == "all"
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest>=8; extra == "dev"
30
+ Requires-Dist: ruff>=0.6; extra == "dev"
31
+ Dynamic: license-file
32
+
33
+ <div align="center">
34
+
35
+ <img src="https://raw.githubusercontent.com/nodemaven/serpmaven/main/assets/banner.png" alt="SERP Maven">
36
+
37
+ <h1>SERP Maven</h1>
38
+
39
+ <p><b>Self-hosted SERP API alternative: search results from a real Chrome browser and your own residential proxy.</b></p>
40
+
41
+ <p><code>pip install serpmaven</code>. Formerly <code>gserp</code> and <code>google-browser-scraper</code>: both now install serpmaven, and their old commands, imports and <code>GSERP_*</code> / <code>GBS_*</code> variables keep working.</p>
42
+
43
+ [![pypi](https://img.shields.io/pypi/v/serpmaven?style=flat-square&cacheSeconds=3600)](https://pypi.org/project/serpmaven/)
44
+ [![python](https://img.shields.io/badge/python-3.10%2B-blue?style=flat-square)](https://pypi.org/project/serpmaven/)
45
+ [![tests](https://img.shields.io/github/actions/workflow/status/nodemaven/serpmaven/ci.yml?branch=main&style=flat-square&label=tests)](https://github.com/nodemaven/serpmaven/actions/workflows/ci.yml)
46
+ [![license](https://img.shields.io/pypi/l/serpmaven?style=flat-square&cacheSeconds=3600)](https://opensource.org/licenses/MIT)
47
+ [![docker](https://img.shields.io/badge/docker-ghcr.io-2496ED?style=flat-square&logo=docker&logoColor=white)](https://github.com/nodemaven/serpmaven/pkgs/container/serpmaven)
48
+ [![2 GB free with code GOOGLENM](https://img.shields.io/badge/%F0%9F%8E%81%202%20GB%20free-code%20GOOGLENM-1153FC?style=flat-square)](https://dashboard.nodemaven.com/accounts/signup/?utm_source=github&utm_medium=readme&utm_campaign=google_browser_scraper&utm_content=promo_badge)
49
+
50
+ <a href="https://devhunt.org/tool/google-search-scraper"><img src="https://devhunt.org/badge/google-search-scraper.svg" alt="SERP Maven on DevHunt" height="44"></a>
51
+
52
+ [Quickstart](#quickstart) · [Output](#what-you-get) · [Poisoned results](#poisoned-results-protection) · [HTTP API](#run-it-as-an-api) · [MCP](#use-it-from-an-ai-agent-mcp) · [Docker](#docker) · [How it works](#how-it-works) · [Options](#options)
53
+
54
+ </div>
55
+
56
+ <br>
57
+
58
+ <p align="center">
59
+ <img src="https://raw.githubusercontent.com/nodemaven/serpmaven/main/assets/demo.gif" alt="A real Chrome window showing Google results next to the JSON the scraper produced for them" width="900">
60
+ </p>
61
+
62
+ <div align="center">
63
+
64
+ | ✅ **100%** | 📉 **0.4 MB** | ⚡ **1.4 s** |
65
+ |:---:|:---:|:---:|
66
+ | success rate | per search | per search on a warm proxy IP |
67
+
68
+ <sub>Latest version benchmarks (v0.1)</sub>
69
+
70
+ </div>
71
+
72
+ Scrape Google search results (SERP) with a real Chrome browser and your own
73
+ residential proxy, and get clean JSON back. Use it as a CLI, a Python library,
74
+ a local HTTP API, or an MCP server that gives AI agents live Google search.
75
+
76
+ Plain HTTP is no longer a reliable way to scrape Google, and a fresh browser on a
77
+ fresh proxy IP is refused quickly. So the scraper warms its first sticky proxy
78
+ session with ordinary browsing and starts later ones from the cookies Google
79
+ already accepts, lets the browser pass Google's checks, and then fetches results
80
+ pages directly over HTTPS with the browser's own cookies. Captchas and
81
+ blocks are reported as such, never as "no results".
82
+
83
+ ## Features
84
+
85
+ <table>
86
+ <tr>
87
+ <td width="50%" valign="top">
88
+
89
+ ⚡ **Hybrid mode**<br>
90
+ Results pages over HTTPS between Google's checks: a fraction of a browser's traffic and time. The browser takes over by itself when needed.
91
+
92
+ </td>
93
+ <td width="50%" valign="top">
94
+
95
+ 🛡️ **Poisoned results protection**<br>
96
+ Pages of results for some other query are caught: the proxy IP serving them is dropped within a few pages and the query is asked again on a fresh one.
97
+
98
+ </td>
99
+ </tr>
100
+ <tr>
101
+ <td valign="top">
102
+
103
+ 🌐 **Real browser**<br>
104
+ Real Chrome via Patchright, warmed up once and then carrying the cookies Google already trusts to every new identity, so Google's checks pass.
105
+
106
+ </td>
107
+ <td valign="top">
108
+
109
+ 🧩 **SERP API-style JSON**<br>
110
+ `organic_results`, `ads`, `related_questions`, `related_searches`, `pagination`, `ai_overview`, `local_results`.
111
+
112
+ </td>
113
+ </tr>
114
+ <tr>
115
+ <td valign="top">
116
+
117
+ 🔌 **Bring your own proxy**<br>
118
+ Any residential proxy with sticky sessions, built-in [NodeMaven](https://nodemaven.com/?utm_source=github&utm_medium=readme&utm_campaign=google_browser_scraper&utm_content=features) support.
119
+
120
+ </td>
121
+ <td valign="top">
122
+
123
+ 🚦 **Honest failures**<br>
124
+ Captchas and blocks come back labeled, not as empty results.
125
+
126
+ </td>
127
+ </tr>
128
+ <tr>
129
+ <td valign="top">
130
+
131
+ 🤖 **For AI agents**<br>
132
+ An MCP server that gives agents live Google search, with the AI Overview and map places.
133
+
134
+ </td>
135
+ <td valign="top">
136
+
137
+ 🐳 **Run it your way**<br>
138
+ CLI, Python library, HTTP API, MCP server or Docker image. Traffic and cost per 1000 pages in every run summary.
139
+
140
+ </td>
141
+ </tr>
142
+ </table>
143
+
144
+ ## Use cases
145
+
146
+ - SEO rank tracking and SERP monitoring for your own keywords
147
+ - Keyword and competitor research from live Google results
148
+ - Web search for AI agents and LLM apps through MCP
149
+ - A self-hosted alternative to paid Google search APIs when you already pay for proxies
150
+
151
+ ## Install
152
+
153
+ Python 3.10 or newer, in a virtual environment:
154
+
155
+ ```bash
156
+ python3 -m venv serpmaven-env
157
+ source serpmaven-env/bin/activate
158
+ pip install serpmaven
159
+ patchright install chromium
160
+ serpmaven doctor
161
+ ```
162
+
163
+ In PowerShell:
164
+
165
+ ```powershell
166
+ python -m venv serpmaven-env
167
+ Set-ExecutionPolicy -Scope Process Bypass -Force # lets this window run Activate.ps1
168
+ .\serpmaven-env\Scripts\Activate.ps1
169
+ pip install serpmaven
170
+ patchright install chromium
171
+ serpmaven doctor
172
+ ```
173
+
174
+ `doctor` ends with `ready` when the browser starts and looks like a desktop.
175
+
176
+ <details>
177
+ <summary><b>On a Linux server (no screen)</b></summary>
178
+
179
+ <br>
180
+
181
+ The browser runs with a window, so a server needs a virtual display, and
182
+ software WebGL when there is no GPU:
183
+
184
+ ```bash
185
+ sudo apt install xvfb
186
+ sudo serpmaven-env/bin/patchright install-deps chromium
187
+ export SERPMAVEN_BROWSER_ARGS=--enable-unsafe-swiftshader
188
+ xvfb-run -a serpmaven doctor
189
+ ```
190
+
191
+ Then start every command with `xvfb-run -a`. The [Docker image](#docker) has
192
+ all of this built in.
193
+
194
+ </details>
195
+
196
+ ## Quickstart
197
+
198
+ > [!NOTE]
199
+ > You need a residential proxy with **sticky sessions**. Put `{session}` where
200
+ > your provider expects a session id in the username.
201
+
202
+ ```bash
203
+ serpmaven search "best running shoes" --proxy "http://USER-session-{session}:PASS@gate.example.com:7000" -o results.jsonl
204
+ ```
205
+
206
+ The first query takes a few minutes while the first proxy IP warms up on a few
207
+ ordinary pages; the terminal shows each step. Later queries take seconds, and
208
+ new proxy IPs start from its cookies. Every run ends with the pages served and
209
+ the traffic used, warm-up apart from the pages.
210
+
211
+ ### With NodeMaven
212
+
213
+ The session id is handled for you and the credentials come from the environment:
214
+
215
+ ```bash
216
+ pip install "serpmaven[nodemaven]"
217
+ export NODEMAVEN_LOGIN=... NODEMAVEN_PASSWORD=...
218
+ printf "best running shoes\nhow to boil eggs\n" > queries.txt
219
+ serpmaven search -f queries.txt --nodemaven --country us -o results.jsonl
220
+ ```
221
+
222
+ In PowerShell:
223
+
224
+ ```powershell
225
+ pip install "serpmaven[nodemaven]"
226
+ $env:NODEMAVEN_LOGIN = "..."; $env:NODEMAVEN_PASSWORD = "..."
227
+ Set-Content queries.txt "best running shoes", "how to boil eggs"
228
+ serpmaven search -f queries.txt --nodemaven --country us -o results.jsonl
229
+ ```
230
+
231
+ > [!TIP]
232
+ > 🎁 **2 GB of free traffic.** No NodeMaven account yet? [Sign up](https://dashboard.nodemaven.com/accounts/signup/?utm_source=github&utm_medium=readme&utm_campaign=google_browser_scraper&utm_content=signup_promo) and enter the code **GOOGLENM**. Once per account.
233
+
234
+ <details>
235
+ <summary><b>Cap a run's traffic with a demo sub-user</b></summary>
236
+
237
+ <br>
238
+
239
+ `--nodemaven-demo` creates a sub-user with a traffic limit from your dashboard API
240
+ key, shares it across workers, and deletes it when the run ends (`--keep-subuser`
241
+ keeps it). The account needs sub-users enabled. A fresh warm-up alone uses about
242
+ 60 MB, so keep the cap well above that:
243
+
244
+ ```bash
245
+ export NODEMAVEN_API_KEY=...
246
+ serpmaven search -f queries.txt --nodemaven-demo --traffic-limit-mb 300 -o results.jsonl
247
+ ```
248
+
249
+ </details>
250
+
251
+ <details>
252
+ <summary><b>Try it without a proxy</b></summary>
253
+
254
+ <br>
255
+
256
+ ```bash
257
+ serpmaven search "best running shoes" --no-proxy
258
+ ```
259
+
260
+ Google then sees one address for every query, so it starts answering with
261
+ captchas sooner than it would across a pool of proxy IPs. Those pages come back as
262
+ `captcha` in `page_verdict`, not as empty results.
263
+
264
+ </details>
265
+
266
+ ### From Python
267
+
268
+ ```python
269
+ from serpmaven import ProxyTemplate, Scraper, Settings
270
+
271
+ proxy = ProxyTemplate("http://USER-session-{session}:PASS@gate.example.com:7000")
272
+ for page in Scraper(proxy, Settings(pages=2)).run(["best running shoes"]):
273
+ for result in page.get("organic_results", []):
274
+ print(result["position"], result["title"], result["link"])
275
+ ```
276
+
277
+ ## What you get
278
+
279
+ One JSON object per results page:
280
+
281
+ ```json
282
+ {
283
+ "search_metadata": {"status": "Success", "page_verdict": "ok", "fetched_with": "http"},
284
+ "search_parameters": {"engine": "google", "q": "best running shoes", "page": 1},
285
+ "organic_results": [
286
+ {"position": 1, "title": "...", "link": "https://...", "snippet": "..."}
287
+ ],
288
+ "related_questions": [{"question": "..."}],
289
+ "ai_overview": {"text": "..."},
290
+ "exit": {"session": "...", "queries_on_exit": 1, "bytes": "..."}
291
+ }
292
+ ```
293
+
294
+ The field names are the ones common to SERP APIs, so simple consumers can often
295
+ switch with minimal mapping.
296
+
297
+ <details>
298
+ <summary><b>All fields</b></summary>
299
+
300
+ <br>
301
+
302
+ - `organic_results`, `ads`, `related_questions`, `related_searches`, `pagination`, `search_metadata`
303
+ - `ai_overview`: the text of Google's AI Overview when the page shows one (pages in English, `--hl en`, the default)
304
+ - `local_results`: the places of the map pack on local queries (name, rating, reviews, type, address, phone, opening hours)
305
+ - `exit`: which proxy IP (sticky session) answered and how many bytes the page cost
306
+ - `search_metadata.fetched_with`: `http` or `browser`
307
+ - `search_metadata.timings`: seconds per step (`http` for a page fetched over HTTP; `type`, `load`, `ready` and `read` for one the browser rendered; `parse`; `links`, which overlaps the next query)
308
+ - A query that failed is a record too, with an `error`
309
+
310
+ Knowledge panels, shopping, images and other special blocks are not parsed.
311
+ `--format csv` writes the organic results as a table, and `--price-per-gb` adds
312
+ the cost per 1000 pages to the summary.
313
+
314
+ </details>
315
+
316
+ ## Poisoned results protection
317
+
318
+ Google does not always block a scraper. Sometimes it answers with a normal-looking
319
+ results page for a different query: no captcha, no error, just wrong data that
320
+ slips into your pipeline unnoticed.
321
+
322
+ The scraper checks results pages against the queries it asked. When a proxy IP
323
+ starts returning unrelated results, it is dropped within a few pages and the query
324
+ is asked again on a fresh one.
325
+
326
+ - No extra requests, no extra traffic: the check runs on the page already downloaded
327
+ - On by default
328
+
329
+ ## Run it as an API
330
+
331
+ ```bash
332
+ serpmaven serve --nodemaven --country us --prewarm
333
+ curl "http://127.0.0.1:8000/search.json?q=best+running+shoes"
334
+ ```
335
+
336
+ In PowerShell, call `curl.exe` (plain `curl` there is another command).
337
+
338
+ - Same record format as above
339
+ - `hl` (language) and `gl` (country) per request: `/search.json?q=...&hl=de&gl=at`
340
+ - `--prewarm` finds a proxy IP Google serves at startup, so the first request does not wait for a warm-up; `/health` answers 503 `warming` until then
341
+ - Results pages are read one at a time on one held proxy IP; a page's links resolve in the background, so the next request starts while the last one waits for its links
342
+ - `links=N` resolves the links of the first N organic results only (`0`: none)
343
+ - `links_wait=S` answers after S seconds with the links resolved by then; links still being resolved are `null` and counted in `search_metadata.links_pending`
344
+ - `async=true` answers at once with `search_metadata.id`; `GET /searches/<id>.json` answers `"status": "Processing"` until the search and its links are done, then the record. The newest 500 searches are kept there, each for up to an hour; up to 50 async searches wait for the browser at once, more are answered 429
345
+ - Set `SERPMAVEN_API_KEY` to require a key; without one the server only listens on localhost
346
+
347
+ ## Use it from an AI agent (MCP)
348
+
349
+ ```json
350
+ {
351
+ "mcpServers": {
352
+ "google-search": {
353
+ "command": "serpmaven",
354
+ "args": ["mcp", "--nodemaven", "--country", "us", "--prewarm"],
355
+ "env": {"NODEMAVEN_LOGIN": "...", "NODEMAVEN_PASSWORD": "..."}
356
+ }
357
+ }
358
+ }
359
+ ```
360
+
361
+ One tool, `google_search(query, pages, hl, gl)`. It returns the organic results,
362
+ "People also ask", related searches, and the AI Overview and map places when
363
+ Google shows them.
364
+
365
+ ## Docker
366
+
367
+ A ready image is published to GitHub Container Registry with every release:
368
+
369
+ ```bash
370
+ docker run --rm ghcr.io/nodemaven/serpmaven doctor
371
+ docker run --rm -p 8000:8000 -e SERPMAVEN_API_KEY=change-me \
372
+ -e SERPMAVEN_PROXY='http://USER-session-{session}:PASS@gate.example.com:7000' \
373
+ ghcr.io/nodemaven/serpmaven serve --host 0.0.0.0
374
+ ```
375
+
376
+ To build it yourself: `docker build -t serpmaven .`
377
+
378
+ ## How it works
379
+
380
+ 1. **New identity:** a sticky proxy session and a fresh browser profile.
381
+ 2. **Warm-up:** the first proxy IP opens a few ordinary pages, staying on each
382
+ for 20-45 seconds. Later ones start from the Google cookies of a served proxy
383
+ IP and are warmed up only when no such set is accepted.
384
+ 3. **Search:** the first query on a proxy IP goes through the browser. Once Google
385
+ serves it, the next pages come over HTTPS with the browser's cookies, until
386
+ Google asks for its check again.
387
+ 4. **Check before parsing:** a captcha or a block is reported as one, never
388
+ parsed as an empty result.
389
+ 5. **Keep or drop:** a proxy IP Google answers is reused; one it refuses is dropped
390
+ and the query retried on a new one. If one proxy IP after another is refused, the run
391
+ stops instead of spending more traffic.
392
+
393
+ <details>
394
+ <summary><b>More detail: HTTP between checks, holding proxy IPs, copied cookies, result links, traffic</b></summary>
395
+
396
+ <br>
397
+
398
+ **HTTP between checks.** After the browser passes Google's check, results pages
399
+ are fetched over HTTPS with the browser's own headers and cookies. If that stops
400
+ working, the scraper goes back to the browser on its own. `--no-hybrid` uses the
401
+ browser for every page.
402
+
403
+ **Unrelated results.** A proxy IP that starts returning results unrelated to the
404
+ queries is dropped, and the query is asked again on a new one.
405
+
406
+ **Holding proxy IPs.** A served proxy IP is kept for up to 300 queries. Every new one
407
+ pays for a browser check, and for a warm-up when no cookie set is accepted, so a
408
+ short cap mostly buys more of that traffic. Every 50
409
+ results pages it renders, the browser is restarted on the same profile and proxy IP, which keeps
410
+ its memory from growing over a long hold. A browser error on a search is retried
411
+ once on the same proxy IP before it is dropped; a network error, running out of
412
+ memory or a crashed browser drops it at once.
413
+
414
+ **Copying cookies.** With `--renew-cookies 0`, the command line copies the
415
+ cookies of the first proxy IP Google serves and starts new proxy IPs from that
416
+ copy, on new sticky sessions, instead of warming each one; all workers share
417
+ the copy. Google's acceptance follows the cookies more than the address, so a
418
+ copy usually starts out served and costs a few MB instead of a full warm-up. A
419
+ copy waits 40-60 seconds after its browser starts before the first search.
420
+
421
+ **Copying the cache.** Pages get cheaper as a proxy IP is held, because the
422
+ browser cache fills with Google's shared scripts and images. Copies carry that
423
+ cache too, so a new proxy IP starts with cheap pages, and the copy is refreshed
424
+ from the held one at each browser restart, when its cache is warm. If two
425
+ copies carrying the cache are refused on their first search, the rest of the
426
+ run copies cookies only (`--no-clone-cache` does that from the start).
427
+
428
+ **Starting from saved cookies.** `--cookies FILE` gives every new proxy IP the
429
+ Google cookies of a browser Google already serves, so it skips the warm-up; the
430
+ file can come from another machine. `--renew-cookies N` replaces that set with
431
+ the cookies of a served proxy IP every N queries, so it stays fresh, and without
432
+ `--cookies` it ends the warm-ups after the first renewal. It is on by default,
433
+ every 25 queries, so a run warms up its first proxy IP and rarely another;
434
+ `--renew-cookies 0` goes back to copies of the first served profile.
435
+ `--cookies-out DIR` saves each renewed set for the next run, readable by its
436
+ owner only. If every set is refused in turn, new proxy IPs are warmed up again,
437
+ and with `--renew-cookies` the next renewal brings the sets back. A file is a
438
+ JSON list of Playwright cookie records, or an object with that list under
439
+ `"cookies"`, and must hold Google's `NID` cookie.
440
+
441
+ **Result links.** Google wraps result links in encrypted `/goto` redirects.
442
+ Each one is resolved with a small request on the proxy IP the page came from,
443
+ with that page's cookies, in the background while the next query runs; a link
444
+ left over is tried through a separate short-lived proxy session. Google answers
445
+ only a few links of a page at once and holds the rest on a "Verifying your
446
+ request" page; a link held for up to 30 seconds is asked again when the wait is
447
+ over, twice at most and within 45 seconds a page, so a page's links can take up
448
+ to 45 seconds while the queries carry on.
449
+ Records still come out in the order each browser searched them.
450
+
451
+ **Traffic.** Everything goes through a small local relay that counts bytes per
452
+ host and refuses what no search needs: the models and data files Chrome
453
+ otherwise downloads into every fresh profile, and the videos of a warm-up page.
454
+
455
+ </details>
456
+
457
+ The defaults come from measurements in
458
+ [proxy-benchmark](https://github.com/nodemaven/proxy-benchmark).
459
+
460
+ ## Options
461
+
462
+ <details>
463
+ <summary><b>All search options</b></summary>
464
+
465
+ <br>
466
+
467
+ | Option | Default | What it does |
468
+ |---|---|---|
469
+ | `--workers N` | 1 | Search on N proxy IPs at once, one browser each, from one list of queries |
470
+ | `--max-per-exit N` | 300 | Queries before a served proxy IP is retired |
471
+ | `--restart-every N` | 50 | Results pages the browser rendered before it restarts on the same profile and proxy IP; `0` never |
472
+ | `--no-hybrid` | | Fetch every results page in the browser instead of over HTTP between Google's checks |
473
+ | `--hl`, `--gl` | `en`, unset | Language and country of the results |
474
+ | `--clone N` | 10 | With `--renew-cookies 0`: new proxy IPs started from copied cookies; `0` turns copying off |
475
+ | `--clone-max-age` | 180 | Minutes copied cookies stay in use |
476
+ | `--clone-refusals N` | 1 | Copies refused in a row on their first query that end the copying |
477
+ | `--no-clone-cache` | | Copies carry cookies only, not the browser cache |
478
+ | `--cookies FILE` | | Start every new proxy IP from the Google cookies in FILE instead of a warm-up; repeat for more files, the next one taking over after two proxy IPs in a row are refused on their first query, and warm-ups again once every file has been refused that way. Turns `--clone` off |
479
+ | `--renew-cookies N` | 25 | Every N served queries, a proxy IP's Google cookies become the set new proxy IPs start from; `0` turns it off and brings back `--clone` |
480
+ | `--cookies-out DIR` | | Write each renewed cookie set to DIR, for the next run's `--cookies` |
481
+ | `--warmup` | `L3` | Pages a fresh proxy IP opens before its first query: `L3`, `light` (without Google Trends: much less traffic, served on the first query less often), `L2`, `L1`, `off` |
482
+ | `--spare N` | 0 | Proxy IPs warmed in the background so a retired one is replaced at once; each costs a warm-up |
483
+ | `--direct` | `all` | Open results pages directly: `all`, `follow-ups` (all but the first query on each proxy IP), `none` |
484
+ | `--input` | `type` | How a query goes into the search box: `type`, `insert`, `words` |
485
+ | `--gap LOW,HIGH` | 2,5 | Seconds between queries on one proxy IP |
486
+ | `--pages N` | 1 | Result pages per query |
487
+ | `--format` | `jsonl` | `jsonl` or `csv` |
488
+ | `--price-per-gb` | | Your proxy price, to print the cost per 1000 pages |
489
+ | `--no-resolve-links` | | Keep result links as Google `/goto` redirects |
490
+ | `--save-html DIR` | | Keep every page's HTML, gzipped |
491
+
492
+ `-v` prints every step as JSON instead, and the summary as one JSON line.
493
+ `serpmaven search --help` lists the rest.
494
+
495
+ </details>
496
+
497
+ ## Proxies
498
+
499
+ Any provider with sticky sessions works through `--proxy` and a `{session}`
500
+ placeholder. A proxy without one is accepted with a warning: if the proxy IP changes
501
+ on every request, warming it does nothing. SOCKS5 needs an IP-whitelisted
502
+ endpoint, because Chrome cannot send a SOCKS5 username and password.
503
+
504
+ ## Check your machine
505
+
506
+ ```bash
507
+ serpmaven doctor
508
+ ```
509
+
510
+ Starts the browser on a blank page, sends nothing, and reports what a website
511
+ would see: whether the browser looks headless, whether WebGL works, the screen
512
+ size. Google answers some machines much less than others, so run this first.
513
+
514
+ ## What this is not
515
+
516
+ Not a free Google Search API, and not a guarantee that every proxy IP will
517
+ work. It is a browser-based collector that makes failures explicit, drops proxy IPs
518
+ Google refuses, and tells you what each page cost.
519
+
520
+ ## Status
521
+
522
+ Early. The `cloak` engine (`--engine cloak`) is experimental; `patchright` is
523
+ the default.
524
+
525
+ Google changes its pages often. If results come back empty or wrong, run with
526
+ `--save-html DIR` and open an issue with what you see.
527
+
528
+ You are responsible for using this within Google's terms and the law where you
529
+ run it.
530
+
531
+ ## Adding an engine
532
+
533
+ An engine joins this tool only after it has been measured on
534
+ [proxy-benchmark](https://github.com/nodemaven/proxy-benchmark). Add it there
535
+ and run the warm-up ladder against Google:
536
+
537
+ ```bash
538
+ python scripts/run_ladder.py --engines your-engine
539
+ ```
540
+
541
+ Then open a pull request here with the run file. The ladder runs identities
542
+ with and without the warm-up in the same hours, so the result shows whether the
543
+ engine actually gets served, not whether it got lucky once.
544
+
545
+ ## License
546
+
547
+ [MIT](https://github.com/nodemaven/serpmaven/blob/main/LICENSE)