serpscraper-dev 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- serpscraper_dev-1.2.0/PKG-INFO +244 -0
- serpscraper_dev-1.2.0/README.md +222 -0
- serpscraper_dev-1.2.0/pyproject.toml +37 -0
- serpscraper_dev-1.2.0/serpscraper/__init__.py +22 -0
- serpscraper_dev-1.2.0/serpscraper/client.py +221 -0
- serpscraper_dev-1.2.0/serpscraper/config.py +59 -0
- serpscraper_dev-1.2.0/serpscraper/exceptions.py +23 -0
- serpscraper_dev-1.2.0/serpscraper/py.typed +0 -0
- serpscraper_dev-1.2.0/serpscraper/response.py +84 -0
- serpscraper_dev-1.2.0/serpscraper_dev.egg-info/PKG-INFO +244 -0
- serpscraper_dev-1.2.0/serpscraper_dev.egg-info/SOURCES.txt +14 -0
- serpscraper_dev-1.2.0/serpscraper_dev.egg-info/dependency_links.txt +1 -0
- serpscraper_dev-1.2.0/serpscraper_dev.egg-info/top_level.txt +1 -0
- serpscraper_dev-1.2.0/setup.cfg +4 -0
- serpscraper_dev-1.2.0/setup.py +3 -0
- serpscraper_dev-1.2.0/tests/test_client.py +110 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: serpscraper-dev
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Python SERP API client: real-time Google search results, AI Overviews, Google Trends, news, images, maps and SEO data as JSON. 5,000 free credits on signup.
|
|
5
|
+
Author-email: "SerpScraper.Dev" <support@serpscraper.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://serpscraper.dev/serp-api
|
|
8
|
+
Project-URL: Documentation, https://serpscraper.dev/docs
|
|
9
|
+
Project-URL: Repository, https://github.com/tienphat/serpscraper.dev
|
|
10
|
+
Project-URL: Issues, https://serpscraper.dev/contact
|
|
11
|
+
Keywords: serp,serp-api,serpscraper,serp-scraper,google-search-api,google-serp,google-search,bing-search,search-api,rank-tracker,rank-checker,seo,seo-api,keyword-research,backlinks,google-ai-overview,google-trends,web-scraping,proxy-api
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# serpscraper: Official Python SDK for SerpScraper.Dev (Google SERP API, AI Overview & Trends)
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<a href="https://serpscraper.dev"><img src="https://serpscraper.dev/img/logo.png" width="100" alt="SerpScraper.Dev Logo"></a>
|
|
27
|
+
</p>
|
|
28
|
+
|
|
29
|
+
<p align="center">
|
|
30
|
+
Official <strong>Python</strong> client library for <a href="https://serpscraper.dev"><strong>SerpScraper.Dev</strong></a> — a fast <a href="https://serpscraper.dev/serp-api">SERP API</a> for real-time Google search rankings, <a href="https://serpscraper.dev/google-ai-overview-api">Google AI Overviews</a>, and <a href="https://serpscraper.dev/google-trends-api">Google Trends</a>.
|
|
31
|
+
</p>
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<a href="https://pypi.org/project/serpscraper-dev/"><img src="https://img.shields.io/pypi/v/serpscraper-dev.svg" alt="PyPI Version"></a>
|
|
35
|
+
<img src="https://img.shields.io/badge/Python-3.8%2B-3776AB?logo=python" alt="Python 3.8+">
|
|
36
|
+
<img src="https://img.shields.io/badge/Dependencies-Zero-success" alt="Zero Dependencies">
|
|
37
|
+
<a href="../LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="MIT License"></a>
|
|
38
|
+
<a href="https://serpscraper.dev"><img src="https://img.shields.io/badge/API%20Key-Free%20Tier-green" alt="Free API Key"></a>
|
|
39
|
+
</p>
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## ⚡ Why Use `serpscraper` for Python?
|
|
44
|
+
|
|
45
|
+
- 🐍 **Zero External Dependencies**: Built 100% on Python standard library (`urllib.request`, `json`). Zero conflict with requests, httpx, or aiohttp versions in your virtual environment.
|
|
46
|
+
- 🤖 **Native Google AI Overview Extraction**: Directly inspect synthesized AI answers, citation cards, and markdown content via the [Google AI Overview API](https://serpscraper.dev/google-ai-overview-api) with `results.summary` and `results.sources`.
|
|
47
|
+
- 📈 **Real-Time Google Trends**: Monitor live trending searches and historical search interest with the [Google Trends API](https://serpscraper.dev/google-trends-api).
|
|
48
|
+
- 🧠 **AI & LLM Ready**: Perfect for LangChain, LlamaIndex, AutoGen, and CrewAI web-search grounding and RAG pipelines.
|
|
49
|
+
- 🌍 **Global Geolocation Targeting**: Target searches from 190+ countries (`gl`) and all languages (`hl`).
|
|
50
|
+
- 🛡️ **Bypass Bot Protection**: Rotating residential IP pool completely bypasses Cloudflare, Akamai, and Google CAPTCHAs.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 📦 Installation
|
|
55
|
+
|
|
56
|
+
Install via pip:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install serpscraper-dev
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Or install from source in editable mode:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install -e .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## 🚀 Quick Start
|
|
71
|
+
|
|
72
|
+
Get your free API key at [serpscraper.dev](https://serpscraper.dev).
|
|
73
|
+
|
|
74
|
+
### 1. Google Web Search & AI Overview Extraction
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from serpscraper import SerpScraperClient
|
|
78
|
+
|
|
79
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
80
|
+
|
|
81
|
+
# Scrape Google organic results + AI Overview targeting US English
|
|
82
|
+
results = client.google_search("best vector database for rag", gl="US", hl="en")
|
|
83
|
+
|
|
84
|
+
print(f"Fetched {len(results)} results in {results.in_seconds}s:\n")
|
|
85
|
+
|
|
86
|
+
# 1. Iterate through Organic Search Rankings
|
|
87
|
+
for item in results:
|
|
88
|
+
print(f"#{item['position']} {item['title']}")
|
|
89
|
+
print(f" URL: {item['link']}")
|
|
90
|
+
print(f" Snippet: {item.get('snippet', '')}\n")
|
|
91
|
+
|
|
92
|
+
# 2. Extract Google AI Overview (SGE) & Sources
|
|
93
|
+
if results.summary:
|
|
94
|
+
print("──────────────────────────────────────────")
|
|
95
|
+
print(f"🤖 [Google AI Overview]:\n{results.summary}")
|
|
96
|
+
print("──────────────────────────────────────────")
|
|
97
|
+
print("Cited Sources:")
|
|
98
|
+
for src in results.sources:
|
|
99
|
+
print(f" - {src['title']} ({src['link']})")
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
### 2. Google Trends Real-Time & Interest Over Time
|
|
105
|
+
|
|
106
|
+
Track market trends and trending searches in real time via the [Google Trends API](https://serpscraper.dev/google-trends-api):
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from serpscraper import SerpScraperClient
|
|
110
|
+
|
|
111
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
112
|
+
|
|
113
|
+
# 1. Fetch live trending queries in the US
|
|
114
|
+
trends = client.trends_now(gl="US")
|
|
115
|
+
|
|
116
|
+
print("🔥 Top Trending Searches:")
|
|
117
|
+
for t in trends:
|
|
118
|
+
traffic = t.get("traffic", "N/A")
|
|
119
|
+
print(f"#{t['position']} {t['keyword']} (Search volume: {traffic})")
|
|
120
|
+
|
|
121
|
+
# 2. Fetch Google Trends Interest Over Time
|
|
122
|
+
interest = client.trends_interest("deepseek", gl="US", time="today 12-m")
|
|
123
|
+
|
|
124
|
+
print("\n📊 Interest Timeline Points:")
|
|
125
|
+
for point in interest.timeline_data:
|
|
126
|
+
print(f"{point['date']}: {point['value']}")
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 🤖 Use Case: LangChain & LLM Tool Integration
|
|
132
|
+
|
|
133
|
+
Turn SerpScraper into a live web search tool for OpenAI, Anthropic, or local LLMs:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from serpscraper import SerpScraperClient
|
|
137
|
+
|
|
138
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
139
|
+
|
|
140
|
+
def serp_search_tool(query: str) -> str:
|
|
141
|
+
"""Useful for searching the live web and getting real-time information."""
|
|
142
|
+
results = client.google_search(query, gl="US", hl="en")
|
|
143
|
+
|
|
144
|
+
output = []
|
|
145
|
+
if results.summary:
|
|
146
|
+
output.append(f"AI Overview Summary:\n{results.summary}\n")
|
|
147
|
+
|
|
148
|
+
output.append("Top Web Results:")
|
|
149
|
+
for item in results[:5]:
|
|
150
|
+
output.append(f"- {item['title']}: {item['snippet']} ({item['link']})")
|
|
151
|
+
|
|
152
|
+
return "\n".join(output)
|
|
153
|
+
|
|
154
|
+
# Ready to pass as a tool into LangChain, LlamaIndex, or AutoGen!
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## 🌐 Supported API Methods
|
|
160
|
+
|
|
161
|
+
All methods return a clean `SerpScraperResponse` object with intuitive helpers:
|
|
162
|
+
|
|
163
|
+
| Python Method | API Endpoint | Documentation & Landing Page |
|
|
164
|
+
|---|---|---|
|
|
165
|
+
| `client.google_search(kw, **opts)` | `webs-search` (engine=google) | [Google Search API](https://serpscraper.dev/web-search-api) |
|
|
166
|
+
| `client.web_search(kw, **opts)` | `webs-search` | [Web Search API](https://serpscraper.dev/web-search-api) |
|
|
167
|
+
| `client.aio_search(kw, **opts)` | `aio` | [Google AI Overview API](https://serpscraper.dev/google-ai-overview-api) |
|
|
168
|
+
| `client.trends_now(**opts)` | `trends-now-search` | [Google Trends API (Real-Time)](https://serpscraper.dev/google-trends-api) |
|
|
169
|
+
| `client.trends_interest(kw, **opts)` | `trends-interest-search` | [Google Trends Interest Over Time](https://serpscraper.dev/google-trends-api) |
|
|
170
|
+
| `client.image_search(kw, **opts)` | `images-search` | [Image Search API](https://serpscraper.dev/image-search-api) |
|
|
171
|
+
| `client.video_search(kw, **opts)` | `videos-search` | [Video Search API](https://serpscraper.dev/video-search-api) |
|
|
172
|
+
| `client.news_search(kw, **opts)` | `news-search` | [News Search API](https://serpscraper.dev/news-search-api) |
|
|
173
|
+
| `client.shopping_search(kw, **opts)` | `shopping-search` | [Shopping Search API](https://serpscraper.dev/shopping-search-api) |
|
|
174
|
+
| `client.scholar_search(kw, **opts)` | `scholar-search` | [Google Scholar API](https://serpscraper.dev/scholar-search-api) |
|
|
175
|
+
| `client.maps_search(kw, **opts)` | `maps-search` | [Google Maps API](https://serpscraper.dev/maps-search-api) |
|
|
176
|
+
| `client.reviews_search(kw, **opts)` | `reviews-search` | [Reviews Search API](https://serpscraper.dev/reviews-search-api) |
|
|
177
|
+
| `client.autocomplete(kw, **opts)` | `autocomplete` | [SERP Autocomplete Suggestions](https://serpscraper.dev/serp-api) |
|
|
178
|
+
| `client.extract_webpage(url)` | `webpage` | [Webpage Content Extractor API](https://serpscraper.dev/webpage-api) |
|
|
179
|
+
| `client.ip_lookup(ip)` | `ip-lookup` | [IP Geolocation & Threat API](https://serpscraper.dev/ip-lookup-api) |
|
|
180
|
+
| `client.exchange_rate(from_c, to_c)` | `exchange-rate` | [Exchange Rate API](https://serpscraper.dev/exchange-rate-api) |
|
|
181
|
+
| `client.crypto_price(symbols, vs)` | `crypto-price` | [Cryptocurrency Price API](https://serpscraper.dev/crypto-price-api) |
|
|
182
|
+
| `client.domain_info(domain)` | `domain-info` | [Domain Intelligence & WHOIS API](https://serpscraper.dev/domain-intelligence-api) |
|
|
183
|
+
| `client.seo_domain_overview(domain)` | `seo/domain-overview` | [SEO Domain Overview API](https://serpscraper.dev/docs) |
|
|
184
|
+
| `client.seo_backlinks(domain)` | `seo/backlinks` | [SEO Backlinks API](https://serpscraper.dev/docs) |
|
|
185
|
+
| `client.seo_keyword_metrics(keywords)` | `seo/keywords/metrics` | [Keyword Metrics API](https://serpscraper.dev/docs) |
|
|
186
|
+
| `client.seo_keyword_suggestions(keyword)` | `seo/keywords/suggestions` | [Keyword Suggestions API](https://serpscraper.dev/docs) |
|
|
187
|
+
| `client.proxy_session(country)` | `proxy/session` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
188
|
+
| `client.proxy_rotate(session_id)` | `proxy/rotate` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
189
|
+
| `client.proxy_locations()` | `proxy/locations` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## 🛡️ Error Handling
|
|
194
|
+
|
|
195
|
+
Catch specific exceptions from `serpscraper.exceptions`:
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from serpscraper import SerpScraperClient
|
|
199
|
+
from serpscraper.exceptions import (
|
|
200
|
+
AuthenticationError,
|
|
201
|
+
RateLimitError,
|
|
202
|
+
NetworkError,
|
|
203
|
+
SerpScraperError,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
207
|
+
|
|
208
|
+
try:
|
|
209
|
+
results = client.google_search("query")
|
|
210
|
+
except AuthenticationError as e:
|
|
211
|
+
print("Invalid API Key:", e)
|
|
212
|
+
except RateLimitError as e:
|
|
213
|
+
print("Credit limit exceeded:", e)
|
|
214
|
+
except NetworkError as e:
|
|
215
|
+
print("Connection failed:", e)
|
|
216
|
+
except SerpScraperError as e:
|
|
217
|
+
print("API Error:", e)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## 🧪 Testing
|
|
223
|
+
|
|
224
|
+
Run standard unit tests:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
python -m unittest discover -s tests
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
|
|
232
|
+
## 📚 Resources & Documentation
|
|
233
|
+
|
|
234
|
+
- 🌐 **Website**: [https://serpscraper.dev](https://serpscraper.dev)
|
|
235
|
+
- 📖 **Official API Documentation**: [https://serpscraper.dev/docs](https://serpscraper.dev/docs)
|
|
236
|
+
- 🔑 **Get Free API Key**: [https://serpscraper.dev](https://serpscraper.dev)
|
|
237
|
+
- 🐛 **Issue Tracker**: [GitHub Issues](https://github.com/tienphat/serpscraper.dev/issues)
|
|
238
|
+
- 💬 **Telegram Support**: [@peterpanpro](https://t.me/peterpanpro)
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## 📄 License
|
|
243
|
+
|
|
244
|
+
The SerpScraper Python SDK is open-sourced software licensed under the [MIT License](../LICENSE).
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# serpscraper: Official Python SDK for SerpScraper.Dev (Google SERP API, AI Overview & Trends)
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<a href="https://serpscraper.dev"><img src="https://serpscraper.dev/img/logo.png" width="100" alt="SerpScraper.Dev Logo"></a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
Official <strong>Python</strong> client library for <a href="https://serpscraper.dev"><strong>SerpScraper.Dev</strong></a> — a fast <a href="https://serpscraper.dev/serp-api">SERP API</a> for real-time Google search rankings, <a href="https://serpscraper.dev/google-ai-overview-api">Google AI Overviews</a>, and <a href="https://serpscraper.dev/google-trends-api">Google Trends</a>.
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<a href="https://pypi.org/project/serpscraper-dev/"><img src="https://img.shields.io/pypi/v/serpscraper-dev.svg" alt="PyPI Version"></a>
|
|
13
|
+
<img src="https://img.shields.io/badge/Python-3.8%2B-3776AB?logo=python" alt="Python 3.8+">
|
|
14
|
+
<img src="https://img.shields.io/badge/Dependencies-Zero-success" alt="Zero Dependencies">
|
|
15
|
+
<a href="../LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="MIT License"></a>
|
|
16
|
+
<a href="https://serpscraper.dev"><img src="https://img.shields.io/badge/API%20Key-Free%20Tier-green" alt="Free API Key"></a>
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## ⚡ Why Use `serpscraper` for Python?
|
|
22
|
+
|
|
23
|
+
- 🐍 **Zero External Dependencies**: Built 100% on Python standard library (`urllib.request`, `json`). Zero conflict with requests, httpx, or aiohttp versions in your virtual environment.
|
|
24
|
+
- 🤖 **Native Google AI Overview Extraction**: Directly inspect synthesized AI answers, citation cards, and markdown content via the [Google AI Overview API](https://serpscraper.dev/google-ai-overview-api) with `results.summary` and `results.sources`.
|
|
25
|
+
- 📈 **Real-Time Google Trends**: Monitor live trending searches and historical search interest with the [Google Trends API](https://serpscraper.dev/google-trends-api).
|
|
26
|
+
- 🧠 **AI & LLM Ready**: Perfect for LangChain, LlamaIndex, AutoGen, and CrewAI web-search grounding and RAG pipelines.
|
|
27
|
+
- 🌍 **Global Geolocation Targeting**: Target searches from 190+ countries (`gl`) and all languages (`hl`).
|
|
28
|
+
- 🛡️ **Bypass Bot Protection**: Rotating residential IP pool completely bypasses Cloudflare, Akamai, and Google CAPTCHAs.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## 📦 Installation
|
|
33
|
+
|
|
34
|
+
Install via pip:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install serpscraper-dev
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Or install from source in editable mode:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install -e .
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## 🚀 Quick Start
|
|
49
|
+
|
|
50
|
+
Get your free API key at [serpscraper.dev](https://serpscraper.dev).
|
|
51
|
+
|
|
52
|
+
### 1. Google Web Search & AI Overview Extraction
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from serpscraper import SerpScraperClient
|
|
56
|
+
|
|
57
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
58
|
+
|
|
59
|
+
# Scrape Google organic results + AI Overview targeting US English
|
|
60
|
+
results = client.google_search("best vector database for rag", gl="US", hl="en")
|
|
61
|
+
|
|
62
|
+
print(f"Fetched {len(results)} results in {results.in_seconds}s:\n")
|
|
63
|
+
|
|
64
|
+
# 1. Iterate through Organic Search Rankings
|
|
65
|
+
for item in results:
|
|
66
|
+
print(f"#{item['position']} {item['title']}")
|
|
67
|
+
print(f" URL: {item['link']}")
|
|
68
|
+
print(f" Snippet: {item.get('snippet', '')}\n")
|
|
69
|
+
|
|
70
|
+
# 2. Extract Google AI Overview (SGE) & Sources
|
|
71
|
+
if results.summary:
|
|
72
|
+
print("──────────────────────────────────────────")
|
|
73
|
+
print(f"🤖 [Google AI Overview]:\n{results.summary}")
|
|
74
|
+
print("──────────────────────────────────────────")
|
|
75
|
+
print("Cited Sources:")
|
|
76
|
+
for src in results.sources:
|
|
77
|
+
print(f" - {src['title']} ({src['link']})")
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
### 2. Google Trends Real-Time & Interest Over Time
|
|
83
|
+
|
|
84
|
+
Track market trends and trending searches in real time via the [Google Trends API](https://serpscraper.dev/google-trends-api):
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from serpscraper import SerpScraperClient
|
|
88
|
+
|
|
89
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
90
|
+
|
|
91
|
+
# 1. Fetch live trending queries in the US
|
|
92
|
+
trends = client.trends_now(gl="US")
|
|
93
|
+
|
|
94
|
+
print("🔥 Top Trending Searches:")
|
|
95
|
+
for t in trends:
|
|
96
|
+
traffic = t.get("traffic", "N/A")
|
|
97
|
+
print(f"#{t['position']} {t['keyword']} (Search volume: {traffic})")
|
|
98
|
+
|
|
99
|
+
# 2. Fetch Google Trends Interest Over Time
|
|
100
|
+
interest = client.trends_interest("deepseek", gl="US", time="today 12-m")
|
|
101
|
+
|
|
102
|
+
print("\n📊 Interest Timeline Points:")
|
|
103
|
+
for point in interest.timeline_data:
|
|
104
|
+
print(f"{point['date']}: {point['value']}")
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## 🤖 Use Case: LangChain & LLM Tool Integration
|
|
110
|
+
|
|
111
|
+
Turn SerpScraper into a live web search tool for OpenAI, Anthropic, or local LLMs:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from serpscraper import SerpScraperClient
|
|
115
|
+
|
|
116
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
117
|
+
|
|
118
|
+
def serp_search_tool(query: str) -> str:
|
|
119
|
+
"""Useful for searching the live web and getting real-time information."""
|
|
120
|
+
results = client.google_search(query, gl="US", hl="en")
|
|
121
|
+
|
|
122
|
+
output = []
|
|
123
|
+
if results.summary:
|
|
124
|
+
output.append(f"AI Overview Summary:\n{results.summary}\n")
|
|
125
|
+
|
|
126
|
+
output.append("Top Web Results:")
|
|
127
|
+
for item in results[:5]:
|
|
128
|
+
output.append(f"- {item['title']}: {item['snippet']} ({item['link']})")
|
|
129
|
+
|
|
130
|
+
return "\n".join(output)
|
|
131
|
+
|
|
132
|
+
# Ready to pass as a tool into LangChain, LlamaIndex, or AutoGen!
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## 🌐 Supported API Methods
|
|
138
|
+
|
|
139
|
+
All methods return a clean `SerpScraperResponse` object with intuitive helpers:
|
|
140
|
+
|
|
141
|
+
| Python Method | API Endpoint | Documentation & Landing Page |
|
|
142
|
+
|---|---|---|
|
|
143
|
+
| `client.google_search(kw, **opts)` | `webs-search` (engine=google) | [Google Search API](https://serpscraper.dev/web-search-api) |
|
|
144
|
+
| `client.web_search(kw, **opts)` | `webs-search` | [Web Search API](https://serpscraper.dev/web-search-api) |
|
|
145
|
+
| `client.aio_search(kw, **opts)` | `aio` | [Google AI Overview API](https://serpscraper.dev/google-ai-overview-api) |
|
|
146
|
+
| `client.trends_now(**opts)` | `trends-now-search` | [Google Trends API (Real-Time)](https://serpscraper.dev/google-trends-api) |
|
|
147
|
+
| `client.trends_interest(kw, **opts)` | `trends-interest-search` | [Google Trends Interest Over Time](https://serpscraper.dev/google-trends-api) |
|
|
148
|
+
| `client.image_search(kw, **opts)` | `images-search` | [Image Search API](https://serpscraper.dev/image-search-api) |
|
|
149
|
+
| `client.video_search(kw, **opts)` | `videos-search` | [Video Search API](https://serpscraper.dev/video-search-api) |
|
|
150
|
+
| `client.news_search(kw, **opts)` | `news-search` | [News Search API](https://serpscraper.dev/news-search-api) |
|
|
151
|
+
| `client.shopping_search(kw, **opts)` | `shopping-search` | [Shopping Search API](https://serpscraper.dev/shopping-search-api) |
|
|
152
|
+
| `client.scholar_search(kw, **opts)` | `scholar-search` | [Google Scholar API](https://serpscraper.dev/scholar-search-api) |
|
|
153
|
+
| `client.maps_search(kw, **opts)` | `maps-search` | [Google Maps API](https://serpscraper.dev/maps-search-api) |
|
|
154
|
+
| `client.reviews_search(kw, **opts)` | `reviews-search` | [Reviews Search API](https://serpscraper.dev/reviews-search-api) |
|
|
155
|
+
| `client.autocomplete(kw, **opts)` | `autocomplete` | [SERP Autocomplete Suggestions](https://serpscraper.dev/serp-api) |
|
|
156
|
+
| `client.extract_webpage(url)` | `webpage` | [Webpage Content Extractor API](https://serpscraper.dev/webpage-api) |
|
|
157
|
+
| `client.ip_lookup(ip)` | `ip-lookup` | [IP Geolocation & Threat API](https://serpscraper.dev/ip-lookup-api) |
|
|
158
|
+
| `client.exchange_rate(from_c, to_c)` | `exchange-rate` | [Exchange Rate API](https://serpscraper.dev/exchange-rate-api) |
|
|
159
|
+
| `client.crypto_price(symbols, vs)` | `crypto-price` | [Cryptocurrency Price API](https://serpscraper.dev/crypto-price-api) |
|
|
160
|
+
| `client.domain_info(domain)` | `domain-info` | [Domain Intelligence & WHOIS API](https://serpscraper.dev/domain-intelligence-api) |
|
|
161
|
+
| `client.seo_domain_overview(domain)` | `seo/domain-overview` | [SEO Domain Overview API](https://serpscraper.dev/docs) |
|
|
162
|
+
| `client.seo_backlinks(domain)` | `seo/backlinks` | [SEO Backlinks API](https://serpscraper.dev/docs) |
|
|
163
|
+
| `client.seo_keyword_metrics(keywords)` | `seo/keywords/metrics` | [Keyword Metrics API](https://serpscraper.dev/docs) |
|
|
164
|
+
| `client.seo_keyword_suggestions(keyword)` | `seo/keywords/suggestions` | [Keyword Suggestions API](https://serpscraper.dev/docs) |
|
|
165
|
+
| `client.proxy_session(country)` | `proxy/session` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
166
|
+
| `client.proxy_rotate(session_id)` | `proxy/rotate` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
167
|
+
| `client.proxy_locations()` | `proxy/locations` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## 🛡️ Error Handling
|
|
172
|
+
|
|
173
|
+
Catch specific exceptions from `serpscraper.exceptions`:
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
from serpscraper import SerpScraperClient
|
|
177
|
+
from serpscraper.exceptions import (
|
|
178
|
+
AuthenticationError,
|
|
179
|
+
RateLimitError,
|
|
180
|
+
NetworkError,
|
|
181
|
+
SerpScraperError,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
results = client.google_search("query")
|
|
188
|
+
except AuthenticationError as e:
|
|
189
|
+
print("Invalid API Key:", e)
|
|
190
|
+
except RateLimitError as e:
|
|
191
|
+
print("Credit limit exceeded:", e)
|
|
192
|
+
except NetworkError as e:
|
|
193
|
+
print("Connection failed:", e)
|
|
194
|
+
except SerpScraperError as e:
|
|
195
|
+
print("API Error:", e)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## 🧪 Testing
|
|
201
|
+
|
|
202
|
+
Run standard unit tests:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
python -m unittest discover -s tests
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## 📚 Resources & Documentation
|
|
211
|
+
|
|
212
|
+
- 🌐 **Website**: [https://serpscraper.dev](https://serpscraper.dev)
|
|
213
|
+
- 📖 **Official API Documentation**: [https://serpscraper.dev/docs](https://serpscraper.dev/docs)
|
|
214
|
+
- 🔑 **Get Free API Key**: [https://serpscraper.dev](https://serpscraper.dev)
|
|
215
|
+
- 🐛 **Issue Tracker**: [GitHub Issues](https://github.com/tienphat/serpscraper.dev/issues)
|
|
216
|
+
- 💬 **Telegram Support**: [@peterpanpro](https://t.me/peterpanpro)
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## 📄 License
|
|
221
|
+
|
|
222
|
+
The SerpScraper Python SDK is open-sourced software licensed under the [MIT License](../LICENSE).
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "serpscraper-dev"
|
|
7
|
+
version = "1.2.0"
|
|
8
|
+
description = "Python SERP API client: real-time Google search results, AI Overviews, Google Trends, news, images, maps and SEO data as JSON. 5,000 free credits on signup."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "SerpScraper.Dev", email = "support@serpscraper.dev" }
|
|
14
|
+
]
|
|
15
|
+
keywords = ["serp", "serp-api", "serpscraper", "serp-scraper", "google-search-api", "google-serp", "google-search", "bing-search", "search-api", "rank-tracker", "rank-checker", "seo", "seo-api", "keyword-research", "backlinks", "google-ai-overview", "google-trends", "web-scraping", "proxy-api"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 5 - Production/Stable",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
23
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://serpscraper.dev/serp-api"
|
|
29
|
+
Documentation = "https://serpscraper.dev/docs"
|
|
30
|
+
Repository = "https://github.com/tienphat/serpscraper.dev"
|
|
31
|
+
Issues = "https://serpscraper.dev/contact"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
include = ["serpscraper*"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
serpscraper = ["py.typed"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""SerpScraper Python SDK — official library for SerpScraper.Dev API."""
|
|
2
|
+
|
|
3
|
+
from .client import SerpScraperClient
|
|
4
|
+
from .config import SerpScraperConfig
|
|
5
|
+
from .exceptions import (
|
|
6
|
+
AuthException,
|
|
7
|
+
NetworkException,
|
|
8
|
+
RateLimitException,
|
|
9
|
+
SerpScraperException,
|
|
10
|
+
)
|
|
11
|
+
from .response import SerpScraperResponse
|
|
12
|
+
|
|
13
|
+
__version__ = "1.1.0"
|
|
14
|
+
__all__ = [
|
|
15
|
+
"SerpScraperClient",
|
|
16
|
+
"SerpScraperConfig",
|
|
17
|
+
"SerpScraperResponse",
|
|
18
|
+
"SerpScraperException",
|
|
19
|
+
"AuthException",
|
|
20
|
+
"RateLimitException",
|
|
21
|
+
"NetworkException",
|
|
22
|
+
]
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""SerpScraper client implementation in pure Python (zero external dependencies)."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import urllib.error
|
|
5
|
+
import urllib.parse
|
|
6
|
+
import urllib.request
|
|
7
|
+
from typing import Any, Dict, Optional, Union
|
|
8
|
+
|
|
9
|
+
from .config import SerpScraperConfig
|
|
10
|
+
from .exceptions import (
|
|
11
|
+
AuthException,
|
|
12
|
+
NetworkException,
|
|
13
|
+
RateLimitException,
|
|
14
|
+
SerpScraperException,
|
|
15
|
+
)
|
|
16
|
+
from .response import SerpScraperResponse
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SerpScraperClient:
|
|
20
|
+
"""Official Python Client for SerpScraper.Dev.
|
|
21
|
+
|
|
22
|
+
Usage:
|
|
23
|
+
client = SerpScraperClient("your-api-key")
|
|
24
|
+
results = client.google_search("openai o3", gl="US", hl="en")
|
|
25
|
+
for item in results:
|
|
26
|
+
print(item["title"], item["link"])
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, config: Union[SerpScraperConfig, str]):
|
|
30
|
+
if isinstance(config, str):
|
|
31
|
+
self.config = SerpScraperConfig(config)
|
|
32
|
+
else:
|
|
33
|
+
self.config = config
|
|
34
|
+
|
|
35
|
+
# ── Search Endpoints ──────────────────────────────────────────────────────
|
|
36
|
+
|
|
37
|
+
def google_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
38
|
+
"""Search Google Organic rankings + extract Google AI Overview."""
|
|
39
|
+
opts = {"engine": "google", **options}
|
|
40
|
+
return self.web_search(keyword, **opts)
|
|
41
|
+
|
|
42
|
+
def web_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
43
|
+
"""Web search results (Bing default or engine='google')."""
|
|
44
|
+
return self._call("webs-search", {"keyword": keyword, **options})
|
|
45
|
+
|
|
46
|
+
def aio_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
47
|
+
"""Standalone AI Overview synthesis (Gemini / Google Grounding or Bing AIO)."""
|
|
48
|
+
return self._call("aio", {"keyword": keyword, **options})
|
|
49
|
+
|
|
50
|
+
def trends_now(self, **options: Any) -> SerpScraperResponse:
|
|
51
|
+
"""Google Trends Real-Time trending searches by country (gl='US')."""
|
|
52
|
+
return self._call("trends-now-search", options)
|
|
53
|
+
|
|
54
|
+
def trends_interest(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
55
|
+
"""Google Trends Interest Over Time (time='today 12-m', gl='US')."""
|
|
56
|
+
return self._call("trends-interest-search", {"keyword": keyword, **options})
|
|
57
|
+
|
|
58
|
+
def image_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
59
|
+
"""Image results with thumbnails and source URLs."""
|
|
60
|
+
return self._call("images-search", {"keyword": keyword, **options})
|
|
61
|
+
|
|
62
|
+
def video_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
63
|
+
"""Video results with duration, views, and provider."""
|
|
64
|
+
return self._call("videos-search", {"keyword": keyword, **options})
|
|
65
|
+
|
|
66
|
+
def news_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
67
|
+
"""News articles with source and publication time."""
|
|
68
|
+
return self._call("news-search", {"keyword": keyword, **options})
|
|
69
|
+
|
|
70
|
+
def shopping_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
71
|
+
"""E-commerce product results with price and merchant."""
|
|
72
|
+
return self._call("shopping-search", {"keyword": keyword, **options})
|
|
73
|
+
|
|
74
|
+
def autocomplete(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
75
|
+
"""Search query auto-suggestions."""
|
|
76
|
+
return self._call("autocomplete", {"keyword": keyword, **options})
|
|
77
|
+
|
|
78
|
+
def scholar_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
79
|
+
"""Google Scholar scientific publications and citations."""
|
|
80
|
+
return self._call("scholar-search", {"keyword": keyword, **options})
|
|
81
|
+
|
|
82
|
+
def maps_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
83
|
+
"""Local maps and places with addresses and phone numbers."""
|
|
84
|
+
return self._call("maps-search", {"keyword": keyword, **options})
|
|
85
|
+
|
|
86
|
+
def reviews_search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
87
|
+
"""Review results with rating signals."""
|
|
88
|
+
return self._call("reviews-search", {"keyword": keyword, **options})
|
|
89
|
+
|
|
90
|
+
def search(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
91
|
+
"""Unified endpoint — auto-detects type or SerpApi format."""
|
|
92
|
+
return self._call("search", {"keyword": keyword, **options})
|
|
93
|
+
|
|
94
|
+
def extract_webpage(self, url: str, include_html: bool = False) -> SerpScraperResponse:
|
|
95
|
+
"""Extract content, metadata, and links from any target URL."""
|
|
96
|
+
params: Dict[str, Any] = {"url": url}
|
|
97
|
+
if include_html:
|
|
98
|
+
params["include_html"] = "true"
|
|
99
|
+
return self._call("webpage", params)
|
|
100
|
+
|
|
101
|
+
# ── Utility Endpoints ─────────────────────────────────────────────────────
|
|
102
|
+
|
|
103
|
+
def ip_lookup(self, ip: Optional[str] = None, **options: Any) -> SerpScraperResponse:
|
|
104
|
+
"""IP Geolocation & proxy/hosting/mobile detection."""
|
|
105
|
+
params = {**options}
|
|
106
|
+
if ip:
|
|
107
|
+
params["ip"] = ip
|
|
108
|
+
return self._call("ip-lookup", params)
|
|
109
|
+
|
|
110
|
+
def exchange_rate(
|
|
111
|
+
self, from_currency: str = "USD", to_currency: Optional[str] = None, **options: Any
|
|
112
|
+
) -> SerpScraperResponse:
|
|
113
|
+
"""Fiat currency exchange rates (ECB)."""
|
|
114
|
+
params = {"from": from_currency, **options}
|
|
115
|
+
if to_currency:
|
|
116
|
+
params["to"] = to_currency
|
|
117
|
+
return self._call("exchange-rate", params)
|
|
118
|
+
|
|
119
|
+
def crypto_price(
|
|
120
|
+
self, symbols: str = "btc,eth", vs_currencies: str = "usd", **options: Any
|
|
121
|
+
) -> SerpScraperResponse:
|
|
122
|
+
"""Real-time crypto prices & 24h market stats."""
|
|
123
|
+
params = {"symbols": symbols, "vs_currencies": vs_currencies, **options}
|
|
124
|
+
return self._call("crypto-price", params)
|
|
125
|
+
|
|
126
|
+
def domain_info(self, domain: str, **options: Any) -> SerpScraperResponse:
|
|
127
|
+
"""WHOIS, SSL certificate, DNS, and domain intelligence."""
|
|
128
|
+
return self._call("domain-info", {"domain": domain, **options})
|
|
129
|
+
|
|
130
|
+
# ── SEO Endpoints ─────────────────────────────────────────────────────────
|
|
131
|
+
|
|
132
|
+
def seo_domain_overview(self, domain: str, **options: Any) -> SerpScraperResponse:
|
|
133
|
+
"""Domain SEO overview: rank, organic traffic, keyword count, technologies."""
|
|
134
|
+
return self._call("seo/domain-overview", {"domain": domain, **options})
|
|
135
|
+
|
|
136
|
+
def seo_backlinks(self, domain: str, **options: Any) -> SerpScraperResponse:
|
|
137
|
+
"""Backlink profile: domain rating, backlink count, referring domains."""
|
|
138
|
+
return self._call("seo/backlinks", {"domain": domain, **options})
|
|
139
|
+
|
|
140
|
+
def seo_keyword_metrics(self, keywords: Any, **options: Any) -> SerpScraperResponse:
|
|
141
|
+
"""Search volume, CPC and competition for up to 50 keywords (1 credit per 10)."""
|
|
142
|
+
joined = ",".join(keywords) if isinstance(keywords, (list, tuple)) else keywords
|
|
143
|
+
return self._call("seo/keywords/metrics", {"keywords": joined, **options})
|
|
144
|
+
|
|
145
|
+
def seo_keyword_suggestions(self, keyword: str, **options: Any) -> SerpScraperResponse:
|
|
146
|
+
"""Autocomplete-based keyword suggestions."""
|
|
147
|
+
return self._call("seo/keywords/suggestions", {"keyword": keyword, **options})
|
|
148
|
+
|
|
149
|
+
# ── Proxy Endpoints ───────────────────────────────────────────────────────
|
|
150
|
+
|
|
151
|
+
def proxy_session(self, country: Optional[str] = None, **options: Any) -> SerpScraperResponse:
|
|
152
|
+
"""Allocate a proxy session, optionally pinned to a country code."""
|
|
153
|
+
params = dict(options)
|
|
154
|
+
if country:
|
|
155
|
+
params["country"] = country
|
|
156
|
+
return self._call("proxy/session", params)
|
|
157
|
+
|
|
158
|
+
def proxy_rotate(self, session_id: str, **options: Any) -> SerpScraperResponse:
|
|
159
|
+
"""Rotate the exit IP of an active proxy session."""
|
|
160
|
+
return self._call("proxy/rotate", {"session_id": session_id, **options})
|
|
161
|
+
|
|
162
|
+
def proxy_locations(self, **options: Any) -> SerpScraperResponse:
|
|
163
|
+
"""List supported proxy countries and regions."""
|
|
164
|
+
return self._call("proxy/locations", dict(options))
|
|
165
|
+
|
|
166
|
+
# ── Internal Network Handling ─────────────────────────────────────────────
|
|
167
|
+
|
|
168
|
+
def _call(self, endpoint: str, params: Dict[str, Any]) -> SerpScraperResponse:
|
|
169
|
+
call_params = {k: v for k, v in params.items() if v is not None}
|
|
170
|
+
|
|
171
|
+
if self.config.country and "gl" not in call_params:
|
|
172
|
+
call_params["gl"] = self.config.country
|
|
173
|
+
if self.config.language and "hl" not in call_params:
|
|
174
|
+
call_params["hl"] = self.config.language
|
|
175
|
+
|
|
176
|
+
query_string = urllib.parse.urlencode(call_params)
|
|
177
|
+
url = f"{self.config.base_url}/{endpoint.lstrip('/')}?{query_string}"
|
|
178
|
+
|
|
179
|
+
req = urllib.request.Request(
|
|
180
|
+
url,
|
|
181
|
+
headers={
|
|
182
|
+
"Accept": "application/json",
|
|
183
|
+
"X-Api-Key": self.config.token,
|
|
184
|
+
"User-Agent": self.config.user_agent,
|
|
185
|
+
},
|
|
186
|
+
method="GET",
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
try:
|
|
190
|
+
with urllib.request.urlopen(req, timeout=self.config.timeout) as resp:
|
|
191
|
+
status_code = resp.status
|
|
192
|
+
body = resp.read().decode("utf-8")
|
|
193
|
+
except urllib.error.HTTPError as e:
|
|
194
|
+
status_code = e.code
|
|
195
|
+
body = e.read().decode("utf-8")
|
|
196
|
+
except urllib.error.URLError as e:
|
|
197
|
+
raise NetworkException(f"Network connection error: {e.reason}") from e
|
|
198
|
+
except Exception as e:
|
|
199
|
+
raise NetworkException(f"Request failed: {str(e)}") from e
|
|
200
|
+
|
|
201
|
+
return self._parse_response(body, status_code)
|
|
202
|
+
|
|
203
|
+
def _parse_response(self, body: str, status_code: int) -> SerpScraperResponse:
|
|
204
|
+
try:
|
|
205
|
+
data = json.loads(body)
|
|
206
|
+
except json.JSONDecodeError as e:
|
|
207
|
+
raise SerpScraperException(f"Invalid JSON from API (HTTP {status_code}): {body[:100]}", status_code) from e
|
|
208
|
+
|
|
209
|
+
if not isinstance(data, dict):
|
|
210
|
+
data = {"data": data}
|
|
211
|
+
|
|
212
|
+
message = str(data.get("message") or data.get("error") or f"HTTP {status_code}")
|
|
213
|
+
|
|
214
|
+
if status_code in (401, 403):
|
|
215
|
+
raise AuthException(message, status_code)
|
|
216
|
+
if status_code == 429:
|
|
217
|
+
raise RateLimitException(message, status_code)
|
|
218
|
+
if status_code >= 400:
|
|
219
|
+
raise SerpScraperException(message, status_code)
|
|
220
|
+
|
|
221
|
+
return SerpScraperResponse(data)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""SerpScraper client configuration."""
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class SerpScraperConfig:
|
|
7
|
+
DEFAULT_BASE_URL = "https://serpscraper.dev/api/v1"
|
|
8
|
+
DEFAULT_TIMEOUT = 30
|
|
9
|
+
DEFAULT_USER_AGENT = "SerpScraperDev-Python/1.1 (+https://github.com/tienphat/serpscraper-php)"
|
|
10
|
+
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
token: str,
|
|
14
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
15
|
+
timeout: int = DEFAULT_TIMEOUT,
|
|
16
|
+
country: Optional[str] = None,
|
|
17
|
+
language: Optional[str] = None,
|
|
18
|
+
user_agent: str = DEFAULT_USER_AGENT,
|
|
19
|
+
):
|
|
20
|
+
token = token.strip()
|
|
21
|
+
if not token:
|
|
22
|
+
raise ValueError("SerpScraper token must not be empty.")
|
|
23
|
+
|
|
24
|
+
self.token = token
|
|
25
|
+
self.base_url = base_url.rstrip("/")
|
|
26
|
+
self.timeout = max(1, int(timeout))
|
|
27
|
+
self.country = country.upper() if country else None
|
|
28
|
+
self.language = language.lower() if language else None
|
|
29
|
+
self.user_agent = user_agent
|
|
30
|
+
|
|
31
|
+
def with_country(self, country: str) -> "SerpScraperConfig":
|
|
32
|
+
return SerpScraperConfig(
|
|
33
|
+
token=self.token,
|
|
34
|
+
base_url=self.base_url,
|
|
35
|
+
timeout=self.timeout,
|
|
36
|
+
country=country,
|
|
37
|
+
language=self.language,
|
|
38
|
+
user_agent=self.user_agent,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def with_language(self, language: str) -> "SerpScraperConfig":
|
|
42
|
+
return SerpScraperConfig(
|
|
43
|
+
token=self.token,
|
|
44
|
+
base_url=self.base_url,
|
|
45
|
+
timeout=self.timeout,
|
|
46
|
+
country=self.country,
|
|
47
|
+
language=language,
|
|
48
|
+
user_agent=self.user_agent,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def with_timeout(self, timeout: int) -> "SerpScraperConfig":
|
|
52
|
+
return SerpScraperConfig(
|
|
53
|
+
token=self.token,
|
|
54
|
+
base_url=self.base_url,
|
|
55
|
+
timeout=timeout,
|
|
56
|
+
country=self.country,
|
|
57
|
+
language=self.language,
|
|
58
|
+
user_agent=self.user_agent,
|
|
59
|
+
)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""SerpScraper exceptions hierarchy."""
|
|
2
|
+
|
|
3
|
+
class SerpScraperException(Exception):
|
|
4
|
+
"""Base exception for all SerpScraper API errors."""
|
|
5
|
+
def __init__(self, message: str, status_code: int = 0):
|
|
6
|
+
super().__init__(message)
|
|
7
|
+
self.message = message
|
|
8
|
+
self.status_code = status_code
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AuthException(SerpScraperException):
|
|
12
|
+
"""Raised when authentication fails (HTTP 401 or 403)."""
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RateLimitException(SerpScraperException):
|
|
17
|
+
"""Raised when credit quota is exceeded or rate limit hit (HTTP 429)."""
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class NetworkException(SerpScraperException):
|
|
22
|
+
"""Raised when a network/connection error occurs."""
|
|
23
|
+
pass
|
|
File without changes
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""SerpScraper response wrapper."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Any, Dict, Iterator, List, Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SerpScraperResponse:
|
|
8
|
+
def __init__(self, raw: Dict[str, Any]):
|
|
9
|
+
self.raw = raw or {}
|
|
10
|
+
|
|
11
|
+
@property
|
|
12
|
+
def data(self) -> List[Dict[str, Any]]:
|
|
13
|
+
val = self.raw.get("data", [])
|
|
14
|
+
if isinstance(val, dict):
|
|
15
|
+
for key in [
|
|
16
|
+
"items", "products", "images", "videos", "news", "scholar",
|
|
17
|
+
"places", "reviews", "suggestions", "interest_over_time", "trending_searches"
|
|
18
|
+
]:
|
|
19
|
+
if key in val and isinstance(val[key], list):
|
|
20
|
+
return val[key]
|
|
21
|
+
return []
|
|
22
|
+
return val if isinstance(val, list) else []
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def first(self) -> Optional[Dict[str, Any]]:
|
|
26
|
+
items = self.data
|
|
27
|
+
return items[0] if items else None
|
|
28
|
+
|
|
29
|
+
def pluck(self, field: str) -> List[Any]:
|
|
30
|
+
return [item[field] for item in self.data if isinstance(item, dict) and field in item]
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def ai_overview(self) -> Optional[Dict[str, Any]]:
|
|
34
|
+
return self.raw.get("ai_overview")
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def summary(self) -> Optional[str]:
|
|
38
|
+
aio = self.ai_overview
|
|
39
|
+
return aio.get("summary") if isinstance(aio, dict) else None
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def sources(self) -> List[Dict[str, Any]]:
|
|
43
|
+
aio = self.ai_overview
|
|
44
|
+
return aio.get("sources", []) if isinstance(aio, dict) else []
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def timeline_data(self) -> List[Dict[str, Any]]:
|
|
48
|
+
data = self.raw.get("data", {})
|
|
49
|
+
if isinstance(data, dict):
|
|
50
|
+
if "interest_over_time" in data and isinstance(data["interest_over_time"], list):
|
|
51
|
+
return data["interest_over_time"]
|
|
52
|
+
if "timeline_data" in data and isinstance(data["timeline_data"], list):
|
|
53
|
+
return data["timeline_data"]
|
|
54
|
+
return []
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def request(self) -> Dict[str, Any]:
|
|
58
|
+
return self.raw.get("request", {})
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def in_seconds(self) -> float:
|
|
62
|
+
return float(self.raw.get("in_seconds", 0.0))
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def is_empty(self) -> bool:
|
|
66
|
+
return len(self.data) == 0
|
|
67
|
+
|
|
68
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
69
|
+
return self.raw
|
|
70
|
+
|
|
71
|
+
def to_json(self, indent: int = 2) -> str:
|
|
72
|
+
return json.dumps(self.raw, indent=indent, ensure_ascii=False)
|
|
73
|
+
|
|
74
|
+
def __len__(self) -> int:
|
|
75
|
+
return len(self.data)
|
|
76
|
+
|
|
77
|
+
def __iter__(self) -> Iterator[Dict[str, Any]]:
|
|
78
|
+
return iter(self.data)
|
|
79
|
+
|
|
80
|
+
def __getitem__(self, index: int) -> Dict[str, Any]:
|
|
81
|
+
return self.data[index]
|
|
82
|
+
|
|
83
|
+
def __repr__(self) -> str:
|
|
84
|
+
return f"<SerpScraperResponse items={len(self.data)} latency={self.in_seconds:.3f}s>"
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: serpscraper-dev
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Python SERP API client: real-time Google search results, AI Overviews, Google Trends, news, images, maps and SEO data as JSON. 5,000 free credits on signup.
|
|
5
|
+
Author-email: "SerpScraper.Dev" <support@serpscraper.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://serpscraper.dev/serp-api
|
|
8
|
+
Project-URL: Documentation, https://serpscraper.dev/docs
|
|
9
|
+
Project-URL: Repository, https://github.com/tienphat/serpscraper.dev
|
|
10
|
+
Project-URL: Issues, https://serpscraper.dev/contact
|
|
11
|
+
Keywords: serp,serp-api,serpscraper,serp-scraper,google-search-api,google-serp,google-search,bing-search,search-api,rank-tracker,rank-checker,seo,seo-api,keyword-research,backlinks,google-ai-overview,google-trends,web-scraping,proxy-api
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# serpscraper: Official Python SDK for SerpScraper.Dev (Google SERP API, AI Overview & Trends)
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<a href="https://serpscraper.dev"><img src="https://serpscraper.dev/img/logo.png" width="100" alt="SerpScraper.Dev Logo"></a>
|
|
27
|
+
</p>
|
|
28
|
+
|
|
29
|
+
<p align="center">
|
|
30
|
+
Official <strong>Python</strong> client library for <a href="https://serpscraper.dev"><strong>SerpScraper.Dev</strong></a> — a fast <a href="https://serpscraper.dev/serp-api">SERP API</a> for real-time Google search rankings, <a href="https://serpscraper.dev/google-ai-overview-api">Google AI Overviews</a>, and <a href="https://serpscraper.dev/google-trends-api">Google Trends</a>.
|
|
31
|
+
</p>
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<a href="https://pypi.org/project/serpscraper-dev/"><img src="https://img.shields.io/pypi/v/serpscraper-dev.svg" alt="PyPI Version"></a>
|
|
35
|
+
<img src="https://img.shields.io/badge/Python-3.8%2B-3776AB?logo=python" alt="Python 3.8+">
|
|
36
|
+
<img src="https://img.shields.io/badge/Dependencies-Zero-success" alt="Zero Dependencies">
|
|
37
|
+
<a href="../LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="MIT License"></a>
|
|
38
|
+
<a href="https://serpscraper.dev"><img src="https://img.shields.io/badge/API%20Key-Free%20Tier-green" alt="Free API Key"></a>
|
|
39
|
+
</p>
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## ⚡ Why Use `serpscraper` for Python?
|
|
44
|
+
|
|
45
|
+
- 🐍 **Zero External Dependencies**: Built 100% on Python standard library (`urllib.request`, `json`). Zero conflict with requests, httpx, or aiohttp versions in your virtual environment.
|
|
46
|
+
- 🤖 **Native Google AI Overview Extraction**: Directly inspect synthesized AI answers, citation cards, and markdown content via the [Google AI Overview API](https://serpscraper.dev/google-ai-overview-api) with `results.summary` and `results.sources`.
|
|
47
|
+
- 📈 **Real-Time Google Trends**: Monitor live trending searches and historical search interest with the [Google Trends API](https://serpscraper.dev/google-trends-api).
|
|
48
|
+
- 🧠 **AI & LLM Ready**: Perfect for LangChain, LlamaIndex, AutoGen, and CrewAI web-search grounding and RAG pipelines.
|
|
49
|
+
- 🌍 **Global Geolocation Targeting**: Target searches from 190+ countries (`gl`) and all languages (`hl`).
|
|
50
|
+
- 🛡️ **Bypass Bot Protection**: Rotating residential IP pool completely bypasses Cloudflare, Akamai, and Google CAPTCHAs.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 📦 Installation
|
|
55
|
+
|
|
56
|
+
Install via pip:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install serpscraper-dev
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Or install from source in editable mode:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install -e .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## 🚀 Quick Start
|
|
71
|
+
|
|
72
|
+
Get your free API key at [serpscraper.dev](https://serpscraper.dev).
|
|
73
|
+
|
|
74
|
+
### 1. Google Web Search & AI Overview Extraction
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from serpscraper import SerpScraperClient
|
|
78
|
+
|
|
79
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
80
|
+
|
|
81
|
+
# Scrape Google organic results + AI Overview targeting US English
|
|
82
|
+
results = client.google_search("best vector database for rag", gl="US", hl="en")
|
|
83
|
+
|
|
84
|
+
print(f"Fetched {len(results)} results in {results.in_seconds}s:\n")
|
|
85
|
+
|
|
86
|
+
# 1. Iterate through Organic Search Rankings
|
|
87
|
+
for item in results:
|
|
88
|
+
print(f"#{item['position']} {item['title']}")
|
|
89
|
+
print(f" URL: {item['link']}")
|
|
90
|
+
print(f" Snippet: {item.get('snippet', '')}\n")
|
|
91
|
+
|
|
92
|
+
# 2. Extract Google AI Overview (SGE) & Sources
|
|
93
|
+
if results.summary:
|
|
94
|
+
print("──────────────────────────────────────────")
|
|
95
|
+
print(f"🤖 [Google AI Overview]:\n{results.summary}")
|
|
96
|
+
print("──────────────────────────────────────────")
|
|
97
|
+
print("Cited Sources:")
|
|
98
|
+
for src in results.sources:
|
|
99
|
+
print(f" - {src['title']} ({src['link']})")
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
### 2. Google Trends Real-Time & Interest Over Time
|
|
105
|
+
|
|
106
|
+
Track market trends and trending searches in real time via the [Google Trends API](https://serpscraper.dev/google-trends-api):
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from serpscraper import SerpScraperClient
|
|
110
|
+
|
|
111
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
112
|
+
|
|
113
|
+
# 1. Fetch live trending queries in the US
|
|
114
|
+
trends = client.trends_now(gl="US")
|
|
115
|
+
|
|
116
|
+
print("🔥 Top Trending Searches:")
|
|
117
|
+
for t in trends:
|
|
118
|
+
traffic = t.get("traffic", "N/A")
|
|
119
|
+
print(f"#{t['position']} {t['keyword']} (Search volume: {traffic})")
|
|
120
|
+
|
|
121
|
+
# 2. Fetch Google Trends Interest Over Time
|
|
122
|
+
interest = client.trends_interest("deepseek", gl="US", time="today 12-m")
|
|
123
|
+
|
|
124
|
+
print("\n📊 Interest Timeline Points:")
|
|
125
|
+
for point in interest.timeline_data:
|
|
126
|
+
print(f"{point['date']}: {point['value']}")
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 🤖 Use Case: LangChain & LLM Tool Integration
|
|
132
|
+
|
|
133
|
+
Turn SerpScraper into a live web search tool for OpenAI, Anthropic, or local LLMs:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from serpscraper import SerpScraperClient
|
|
137
|
+
|
|
138
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
139
|
+
|
|
140
|
+
def serp_search_tool(query: str) -> str:
|
|
141
|
+
"""Useful for searching the live web and getting real-time information."""
|
|
142
|
+
results = client.google_search(query, gl="US", hl="en")
|
|
143
|
+
|
|
144
|
+
output = []
|
|
145
|
+
if results.summary:
|
|
146
|
+
output.append(f"AI Overview Summary:\n{results.summary}\n")
|
|
147
|
+
|
|
148
|
+
output.append("Top Web Results:")
|
|
149
|
+
for item in results[:5]:
|
|
150
|
+
output.append(f"- {item['title']}: {item['snippet']} ({item['link']})")
|
|
151
|
+
|
|
152
|
+
return "\n".join(output)
|
|
153
|
+
|
|
154
|
+
# Ready to pass as a tool into LangChain, LlamaIndex, or AutoGen!
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## 🌐 Supported API Methods
|
|
160
|
+
|
|
161
|
+
All methods return a clean `SerpScraperResponse` object with intuitive helpers:
|
|
162
|
+
|
|
163
|
+
| Python Method | API Endpoint | Documentation & Landing Page |
|
|
164
|
+
|---|---|---|
|
|
165
|
+
| `client.google_search(kw, **opts)` | `webs-search` (engine=google) | [Google Search API](https://serpscraper.dev/web-search-api) |
|
|
166
|
+
| `client.web_search(kw, **opts)` | `webs-search` | [Web Search API](https://serpscraper.dev/web-search-api) |
|
|
167
|
+
| `client.aio_search(kw, **opts)` | `aio` | [Google AI Overview API](https://serpscraper.dev/google-ai-overview-api) |
|
|
168
|
+
| `client.trends_now(**opts)` | `trends-now-search` | [Google Trends API (Real-Time)](https://serpscraper.dev/google-trends-api) |
|
|
169
|
+
| `client.trends_interest(kw, **opts)` | `trends-interest-search` | [Google Trends Interest Over Time](https://serpscraper.dev/google-trends-api) |
|
|
170
|
+
| `client.image_search(kw, **opts)` | `images-search` | [Image Search API](https://serpscraper.dev/image-search-api) |
|
|
171
|
+
| `client.video_search(kw, **opts)` | `videos-search` | [Video Search API](https://serpscraper.dev/video-search-api) |
|
|
172
|
+
| `client.news_search(kw, **opts)` | `news-search` | [News Search API](https://serpscraper.dev/news-search-api) |
|
|
173
|
+
| `client.shopping_search(kw, **opts)` | `shopping-search` | [Shopping Search API](https://serpscraper.dev/shopping-search-api) |
|
|
174
|
+
| `client.scholar_search(kw, **opts)` | `scholar-search` | [Google Scholar API](https://serpscraper.dev/scholar-search-api) |
|
|
175
|
+
| `client.maps_search(kw, **opts)` | `maps-search` | [Google Maps API](https://serpscraper.dev/maps-search-api) |
|
|
176
|
+
| `client.reviews_search(kw, **opts)` | `reviews-search` | [Reviews Search API](https://serpscraper.dev/reviews-search-api) |
|
|
177
|
+
| `client.autocomplete(kw, **opts)` | `autocomplete` | [SERP Autocomplete Suggestions](https://serpscraper.dev/serp-api) |
|
|
178
|
+
| `client.extract_webpage(url)` | `webpage` | [Webpage Content Extractor API](https://serpscraper.dev/webpage-api) |
|
|
179
|
+
| `client.ip_lookup(ip)` | `ip-lookup` | [IP Geolocation & Threat API](https://serpscraper.dev/ip-lookup-api) |
|
|
180
|
+
| `client.exchange_rate(from_c, to_c)` | `exchange-rate` | [Exchange Rate API](https://serpscraper.dev/exchange-rate-api) |
|
|
181
|
+
| `client.crypto_price(symbols, vs)` | `crypto-price` | [Cryptocurrency Price API](https://serpscraper.dev/crypto-price-api) |
|
|
182
|
+
| `client.domain_info(domain)` | `domain-info` | [Domain Intelligence & WHOIS API](https://serpscraper.dev/domain-intelligence-api) |
|
|
183
|
+
| `client.seo_domain_overview(domain)` | `seo/domain-overview` | [SEO Domain Overview API](https://serpscraper.dev/docs) |
|
|
184
|
+
| `client.seo_backlinks(domain)` | `seo/backlinks` | [SEO Backlinks API](https://serpscraper.dev/docs) |
|
|
185
|
+
| `client.seo_keyword_metrics(keywords)` | `seo/keywords/metrics` | [Keyword Metrics API](https://serpscraper.dev/docs) |
|
|
186
|
+
| `client.seo_keyword_suggestions(keyword)` | `seo/keywords/suggestions` | [Keyword Suggestions API](https://serpscraper.dev/docs) |
|
|
187
|
+
| `client.proxy_session(country)` | `proxy/session` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
188
|
+
| `client.proxy_rotate(session_id)` | `proxy/rotate` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
189
|
+
| `client.proxy_locations()` | `proxy/locations` | [Proxy API](https://serpscraper.dev/proxy-api) |
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## 🛡️ Error Handling
|
|
194
|
+
|
|
195
|
+
Catch specific exceptions from `serpscraper.exceptions`:
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from serpscraper import SerpScraperClient
|
|
199
|
+
from serpscraper.exceptions import (
|
|
200
|
+
AuthenticationError,
|
|
201
|
+
RateLimitError,
|
|
202
|
+
NetworkError,
|
|
203
|
+
SerpScraperError,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
client = SerpScraperClient("YOUR_API_KEY")
|
|
207
|
+
|
|
208
|
+
try:
|
|
209
|
+
results = client.google_search("query")
|
|
210
|
+
except AuthenticationError as e:
|
|
211
|
+
print("Invalid API Key:", e)
|
|
212
|
+
except RateLimitError as e:
|
|
213
|
+
print("Credit limit exceeded:", e)
|
|
214
|
+
except NetworkError as e:
|
|
215
|
+
print("Connection failed:", e)
|
|
216
|
+
except SerpScraperError as e:
|
|
217
|
+
print("API Error:", e)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## 🧪 Testing
|
|
223
|
+
|
|
224
|
+
Run standard unit tests:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
python -m unittest discover -s tests
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
|
|
232
|
+
## 📚 Resources & Documentation
|
|
233
|
+
|
|
234
|
+
- 🌐 **Website**: [https://serpscraper.dev](https://serpscraper.dev)
|
|
235
|
+
- 📖 **Official API Documentation**: [https://serpscraper.dev/docs](https://serpscraper.dev/docs)
|
|
236
|
+
- 🔑 **Get Free API Key**: [https://serpscraper.dev](https://serpscraper.dev)
|
|
237
|
+
- 🐛 **Issue Tracker**: [GitHub Issues](https://github.com/tienphat/serpscraper.dev/issues)
|
|
238
|
+
- 💬 **Telegram Support**: [@peterpanpro](https://t.me/peterpanpro)
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## 📄 License
|
|
243
|
+
|
|
244
|
+
The SerpScraper Python SDK is open-sourced software licensed under the [MIT License](../LICENSE).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
serpscraper/__init__.py
|
|
5
|
+
serpscraper/client.py
|
|
6
|
+
serpscraper/config.py
|
|
7
|
+
serpscraper/exceptions.py
|
|
8
|
+
serpscraper/py.typed
|
|
9
|
+
serpscraper/response.py
|
|
10
|
+
serpscraper_dev.egg-info/PKG-INFO
|
|
11
|
+
serpscraper_dev.egg-info/SOURCES.txt
|
|
12
|
+
serpscraper_dev.egg-info/dependency_links.txt
|
|
13
|
+
serpscraper_dev.egg-info/top_level.txt
|
|
14
|
+
tests/test_client.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
serpscraper
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import patch, MagicMock
|
|
3
|
+
from io import BytesIO
|
|
4
|
+
import json
|
|
5
|
+
import urllib.error
|
|
6
|
+
|
|
7
|
+
from serpscraper import SerpScraperClient, SerpScraperConfig, SerpScraperResponse
|
|
8
|
+
from serpscraper.exceptions import AuthException, RateLimitException, SerpScraperException
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestSerpScraperPython(unittest.TestCase):
|
|
12
|
+
def test_config_initialization(self):
|
|
13
|
+
config = SerpScraperConfig("test-token", country="US", language="en")
|
|
14
|
+
self.assertEqual(config.token, "test-token")
|
|
15
|
+
self.assertEqual(config.country, "US")
|
|
16
|
+
self.assertEqual(config.language, "en")
|
|
17
|
+
self.assertEqual(config.base_url, "https://serpscraper.dev/api/v1")
|
|
18
|
+
|
|
19
|
+
def test_config_empty_token_raises(self):
|
|
20
|
+
with self.assertRaises(ValueError):
|
|
21
|
+
SerpScraperConfig("")
|
|
22
|
+
|
|
23
|
+
def test_response_accessors(self):
|
|
24
|
+
raw = {
|
|
25
|
+
"data": [
|
|
26
|
+
{"position": 1, "title": "Example", "link": "https://example.com"}
|
|
27
|
+
],
|
|
28
|
+
"ai_overview": {
|
|
29
|
+
"summary": "This is an AI summary.",
|
|
30
|
+
"sources": [{"title": "Wiki", "link": "https://wiki.org"}]
|
|
31
|
+
},
|
|
32
|
+
"in_seconds": 0.45
|
|
33
|
+
}
|
|
34
|
+
res = SerpScraperResponse(raw)
|
|
35
|
+
self.assertEqual(len(res), 1)
|
|
36
|
+
self.assertEqual(res.first["title"], "Example")
|
|
37
|
+
self.assertEqual(res.pluck("link"), ["https://example.com"])
|
|
38
|
+
self.assertEqual(res.summary, "This is an AI summary.")
|
|
39
|
+
self.assertEqual(len(res.sources), 1)
|
|
40
|
+
self.assertEqual(res.in_seconds, 0.45)
|
|
41
|
+
self.assertFalse(res.is_empty)
|
|
42
|
+
|
|
43
|
+
@patch("urllib.request.urlopen")
|
|
44
|
+
def test_client_google_search(self, mock_urlopen):
|
|
45
|
+
mock_response = MagicMock()
|
|
46
|
+
mock_response.status = 200
|
|
47
|
+
mock_response.read.return_value = json.dumps({
|
|
48
|
+
"request": {"keyword": "test", "engine": "google"},
|
|
49
|
+
"data": [{"title": "Google Result", "link": "https://google.com"}],
|
|
50
|
+
"in_seconds": 1.2
|
|
51
|
+
}).encode("utf-8")
|
|
52
|
+
mock_response.__enter__.return_value = mock_response
|
|
53
|
+
mock_urlopen.return_value = mock_response
|
|
54
|
+
|
|
55
|
+
client = SerpScraperClient("my-secret-key")
|
|
56
|
+
res = client.google_search("test", gl="US")
|
|
57
|
+
|
|
58
|
+
self.assertEqual(len(res), 1)
|
|
59
|
+
self.assertEqual(res.first["title"], "Google Result")
|
|
60
|
+
|
|
61
|
+
req = mock_urlopen.call_args[0][0]
|
|
62
|
+
self.assertIn("engine=google", req.full_url)
|
|
63
|
+
self.assertIn("keyword=test", req.full_url)
|
|
64
|
+
self.assertNotIn("my-secret-key", req.full_url)
|
|
65
|
+
self.assertEqual(req.get_header("X-api-key"), "my-secret-key")
|
|
66
|
+
|
|
67
|
+
@patch("urllib.request.urlopen")
|
|
68
|
+
def test_seo_and_proxy_endpoints_build_expected_urls(self, mock_urlopen):
|
|
69
|
+
mock_response = MagicMock()
|
|
70
|
+
mock_response.status = 200
|
|
71
|
+
mock_response.read.return_value = json.dumps({"data": [], "in_seconds": 0}).encode("utf-8")
|
|
72
|
+
mock_response.__enter__.return_value = mock_response
|
|
73
|
+
mock_urlopen.return_value = mock_response
|
|
74
|
+
|
|
75
|
+
client = SerpScraperClient("tok")
|
|
76
|
+
calls = [
|
|
77
|
+
(client.seo_domain_overview("apple.com"), "/seo/domain-overview?", "domain=apple.com"),
|
|
78
|
+
(client.seo_backlinks("github.com"), "/seo/backlinks?", "domain=github.com"),
|
|
79
|
+
(client.seo_keyword_metrics(["serp api", "rank tracker"]), "/seo/keywords/metrics?", "keywords=serp+api%2Crank+tracker"),
|
|
80
|
+
(client.seo_keyword_suggestions("laravel"), "/seo/keywords/suggestions?", "keyword=laravel"),
|
|
81
|
+
(client.proxy_session("US"), "/proxy/session?", "country=US"),
|
|
82
|
+
(client.proxy_rotate("abc123"), "/proxy/rotate?", "session_id=abc123"),
|
|
83
|
+
(client.proxy_locations(), "/proxy/locations", "/proxy/locations"),
|
|
84
|
+
]
|
|
85
|
+
urls = [c.args[0].full_url for c in mock_urlopen.call_args_list]
|
|
86
|
+
self.assertEqual(len(urls), len(calls))
|
|
87
|
+
for url, (_, path, param) in zip(urls, calls):
|
|
88
|
+
self.assertIn(path, url)
|
|
89
|
+
self.assertIn(param, url)
|
|
90
|
+
|
|
91
|
+
client.proxy_session()
|
|
92
|
+
self.assertNotIn("country=", mock_urlopen.call_args[0][0].full_url)
|
|
93
|
+
|
|
94
|
+
@patch("urllib.request.urlopen")
|
|
95
|
+
def test_client_auth_error_raises_exception(self, mock_urlopen):
|
|
96
|
+
mock_urlopen.side_effect = urllib.error.HTTPError(
|
|
97
|
+
url="http://test",
|
|
98
|
+
code=401,
|
|
99
|
+
msg="Unauthorized",
|
|
100
|
+
hdrs={},
|
|
101
|
+
fp=BytesIO(b'{"message": "Invalid token key!"}')
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
client = SerpScraperClient("bad-token")
|
|
105
|
+
with self.assertRaises(AuthException):
|
|
106
|
+
client.web_search("test")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
unittest.main()
|