gmapsscraper-sdk 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gmapsscraper_sdk-0.1.0/.github/workflows/ci.yml +24 -0
- gmapsscraper_sdk-0.1.0/.gitignore +7 -0
- gmapsscraper_sdk-0.1.0/LICENSE +21 -0
- gmapsscraper_sdk-0.1.0/PKG-INFO +159 -0
- gmapsscraper_sdk-0.1.0/README.md +131 -0
- gmapsscraper_sdk-0.1.0/examples/basic.py +19 -0
- gmapsscraper_sdk-0.1.0/pyproject.toml +51 -0
- gmapsscraper_sdk-0.1.0/src/gmapsscraper/__init__.py +231 -0
- gmapsscraper_sdk-0.1.0/src/gmapsscraper/py.typed +0 -0
- gmapsscraper_sdk-0.1.0/tests/test_client.py +179 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python: ['3.9', '3.11', '3.13']
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: ${{ matrix.python }}
|
|
19
|
+
- run: pip install build twine
|
|
20
|
+
- run: python -m build
|
|
21
|
+
- run: twine check dist/*
|
|
22
|
+
- run: pip install dist/*.whl
|
|
23
|
+
# Run tests against the installed wheel, not the source tree
|
|
24
|
+
- run: mv src src_moved && python -m unittest discover -s tests -v
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 gmapsscraper.io
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gmapsscraper-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Official Python SDK for the gmapsscraper.io API β scrape Google Maps business data: names, emails, phones, websites, ratings, reviews.
|
|
5
|
+
Project-URL: Homepage, https://gmapsscraper.io
|
|
6
|
+
Project-URL: Repository, https://github.com/gmapsscraper/gmapsscraper-python
|
|
7
|
+
Project-URL: Documentation, https://gmapsscraper.io/llms.txt
|
|
8
|
+
Project-URL: Issues, https://github.com/gmapsscraper/gmapsscraper-python/issues
|
|
9
|
+
Author: gmapsscraper.io
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: api client,b2b,business data,email extractor,google maps,google maps scraper,lead generation,leads,local business,places,reviews,scraper,sdk
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# gmapsscraper
|
|
30
|
+
|
|
31
|
+
[](https://pypi.org/project/gmapsscraper-sdk/)
|
|
32
|
+
[](https://pypi.org/project/gmapsscraper-sdk/)
|
|
33
|
+
[](./LICENSE)
|
|
34
|
+
|
|
35
|
+
Official Python SDK for the [gmapsscraper.io](https://gmapsscraper.io) API β a **Google Maps scraper** for lead generation. Extract business names, addresses, phone numbers, **emails**, websites, ratings, review counts, categories and coordinates from Google Maps in a few lines of Python.
|
|
36
|
+
|
|
37
|
+
- πͺΆ **Zero dependencies** β pure standard library (`urllib` + `csv`)
|
|
38
|
+
- π **Python 3.9+**, fully type-hinted (`py.typed`)
|
|
39
|
+
- π§ **Email extraction** β crawls business websites for contact emails
|
|
40
|
+
- πΊοΈ **Auto-geocoding** β just write `"dentist in Chicago IL"`, no coordinates needed
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install gmapsscraper-sdk
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The import name is simply `gmapsscraper`:
|
|
49
|
+
|
|
50
|
+
Get a free API key (10 credits = 5 searches, no credit card) at **[gmapsscraper.io/dashboard](https://gmapsscraper.io/dashboard)**.
|
|
51
|
+
|
|
52
|
+
## Quick start
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from gmapsscraper import GMapsScraper
|
|
56
|
+
|
|
57
|
+
client = GMapsScraper("YOUR_API_KEY")
|
|
58
|
+
|
|
59
|
+
# One call: submit β poll β download parsed results
|
|
60
|
+
leads = client.scrape("coffee shop in Austin TX", email=True)
|
|
61
|
+
|
|
62
|
+
print(len(leads), "businesses found")
|
|
63
|
+
print(leads[0])
|
|
64
|
+
# {
|
|
65
|
+
# "title": "Houndstooth Coffee",
|
|
66
|
+
# "address": "401 Congress Ave ...",
|
|
67
|
+
# "phone": "+1 512-...",
|
|
68
|
+
# "email": "hello@...",
|
|
69
|
+
# "website": "https://...",
|
|
70
|
+
# "rating": "4.7",
|
|
71
|
+
# "reviews_count": "1912",
|
|
72
|
+
# "category": "Coffee shop",
|
|
73
|
+
# "latitude": "30.2672", "longitude": "-97.7431",
|
|
74
|
+
# "google_maps_url": "https://www.google.com/maps/place/...",
|
|
75
|
+
# "opening_hours": "..."
|
|
76
|
+
# }
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Step-by-step API
|
|
80
|
+
|
|
81
|
+
If you want control over each stage (e.g. queue jobs and collect later):
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
# 1. Submit a job (costs 2 credits, multiple keywords = same cost)
|
|
85
|
+
job = client.create_job(
|
|
86
|
+
["plumber in Miami FL", "plumbing service in Miami FL"],
|
|
87
|
+
email=True, depth=2,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# 2. Wait for completion (polls every 10s)
|
|
91
|
+
client.wait_for_job(job["id"], on_progress=lambda j: print("status:", j["status"]))
|
|
92
|
+
|
|
93
|
+
# 3a. Parsed dictsβ¦
|
|
94
|
+
records = client.download_records(job["id"])
|
|
95
|
+
|
|
96
|
+
# 3b. β¦or the raw CSV
|
|
97
|
+
csv_text = client.download_csv(job["id"])
|
|
98
|
+
|
|
99
|
+
# Check your balance
|
|
100
|
+
balance = client.credits() # {"credits": 8}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Options
|
|
104
|
+
|
|
105
|
+
All options for `scrape()` / `create_job()` (names match the REST API wire format and are stable):
|
|
106
|
+
|
|
107
|
+
| Option | Type | Default | Description |
|
|
108
|
+
| ----------- | -------------- | ------- | ---------------------------------------------------- |
|
|
109
|
+
| `email` | bool | `False` | Extract business emails from websites |
|
|
110
|
+
| `depth` | int (1β2) | `2` | Higher = more results, **same credit cost** |
|
|
111
|
+
| `zoom` | int (1β21) | `15` | Map zoom level |
|
|
112
|
+
| `radius` | int (meters) | `20000` | Search radius |
|
|
113
|
+
| `lang` | str | `"en"` | ISO 639-1 result language |
|
|
114
|
+
| `fast_mode` | bool | `True` | Skip deep website crawling |
|
|
115
|
+
| `max_time` | int (s) | `3600` | Job timeout on the backend |
|
|
116
|
+
| `lat`/`lon` | str \| float | β | Coordinates (auto-geocoded from keywords if omitted) |
|
|
117
|
+
|
|
118
|
+
`scrape()` and `wait_for_job()` also accept `poll_interval` (seconds, default `10` β the API minimum; keep it there to avoid rate limits), `timeout` (seconds, default `3600`) and `on_progress(job)`.
|
|
119
|
+
|
|
120
|
+
## Error handling
|
|
121
|
+
|
|
122
|
+
All failures β HTTP errors, network failures, failed jobs and timeouts β raise `GMapsScraperError` with `status` and `body`. Invalid arguments raise `TypeError`:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from gmapsscraper import GMapsScraper, GMapsScraperError
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
client.scrape("dentist in Chicago IL")
|
|
129
|
+
except GMapsScraperError as err:
|
|
130
|
+
# 401 invalid key Β· 402 out of credits Β· 422 bad params Β· 429 rate limited
|
|
131
|
+
print(err.status, err)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
| Status | Meaning |
|
|
135
|
+
| ------ | -------------------------------------------------------------------------- |
|
|
136
|
+
| 401 | Invalid API key |
|
|
137
|
+
| 402 | Insufficient credits β top up at [gmapsscraper.io](https://gmapsscraper.io/#pricing) |
|
|
138
|
+
| 422 | Invalid parameters |
|
|
139
|
+
| 429 | Rate limited (1000 req/day) or too many concurrent jobs (max 10) |
|
|
140
|
+
| 502 | Backend temporarily unavailable |
|
|
141
|
+
|
|
142
|
+
## Tips for better results
|
|
143
|
+
|
|
144
|
+
- Be specific: `"vegan restaurant in Brooklyn NY"` beats `"restaurant in New York"`.
|
|
145
|
+
- Pass several related keywords in one job β broader coverage, same 2 credits.
|
|
146
|
+
- Set `email=True` whenever you need contact info for cold outreach.
|
|
147
|
+
- Jobs typically finish in 30β120 seconds.
|
|
148
|
+
|
|
149
|
+
## Related resources
|
|
150
|
+
|
|
151
|
+
- π [API documentation](https://gmapsscraper.io/llms.txt) (also great as LLM context)
|
|
152
|
+
- π¦ [Node.js SDK](https://github.com/gmapsscraper/gmapsscraper-js) β `npm install @gmapsscraper/sdk`
|
|
153
|
+
- π€ [Claude / AI agent skills](https://github.com/gmapsscraper/google-maps-agent-skills) β use gmapsscraper from Claude Code and other agents
|
|
154
|
+
- 𧩠[Chrome extension](https://github.com/gmapsscraper/google-maps-scraper) β scrape Google Maps without code
|
|
155
|
+
- βοΈ [Blog: Google Maps scraping guides](https://gmapsscraper.io/blog)
|
|
156
|
+
|
|
157
|
+
## License
|
|
158
|
+
|
|
159
|
+
[MIT](./LICENSE) Β© [gmapsscraper.io](https://gmapsscraper.io)
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# gmapsscraper
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/gmapsscraper-sdk/)
|
|
4
|
+
[](https://pypi.org/project/gmapsscraper-sdk/)
|
|
5
|
+
[](./LICENSE)
|
|
6
|
+
|
|
7
|
+
Official Python SDK for the [gmapsscraper.io](https://gmapsscraper.io) API β a **Google Maps scraper** for lead generation. Extract business names, addresses, phone numbers, **emails**, websites, ratings, review counts, categories and coordinates from Google Maps in a few lines of Python.
|
|
8
|
+
|
|
9
|
+
- πͺΆ **Zero dependencies** β pure standard library (`urllib` + `csv`)
|
|
10
|
+
- π **Python 3.9+**, fully type-hinted (`py.typed`)
|
|
11
|
+
- π§ **Email extraction** β crawls business websites for contact emails
|
|
12
|
+
- πΊοΈ **Auto-geocoding** β just write `"dentist in Chicago IL"`, no coordinates needed
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install gmapsscraper-sdk
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The import name is simply `gmapsscraper`:
|
|
21
|
+
|
|
22
|
+
Get a free API key (10 credits = 5 searches, no credit card) at **[gmapsscraper.io/dashboard](https://gmapsscraper.io/dashboard)**.
|
|
23
|
+
|
|
24
|
+
## Quick start
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
from gmapsscraper import GMapsScraper
|
|
28
|
+
|
|
29
|
+
client = GMapsScraper("YOUR_API_KEY")
|
|
30
|
+
|
|
31
|
+
# One call: submit β poll β download parsed results
|
|
32
|
+
leads = client.scrape("coffee shop in Austin TX", email=True)
|
|
33
|
+
|
|
34
|
+
print(len(leads), "businesses found")
|
|
35
|
+
print(leads[0])
|
|
36
|
+
# {
|
|
37
|
+
# "title": "Houndstooth Coffee",
|
|
38
|
+
# "address": "401 Congress Ave ...",
|
|
39
|
+
# "phone": "+1 512-...",
|
|
40
|
+
# "email": "hello@...",
|
|
41
|
+
# "website": "https://...",
|
|
42
|
+
# "rating": "4.7",
|
|
43
|
+
# "reviews_count": "1912",
|
|
44
|
+
# "category": "Coffee shop",
|
|
45
|
+
# "latitude": "30.2672", "longitude": "-97.7431",
|
|
46
|
+
# "google_maps_url": "https://www.google.com/maps/place/...",
|
|
47
|
+
# "opening_hours": "..."
|
|
48
|
+
# }
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Step-by-step API
|
|
52
|
+
|
|
53
|
+
If you want control over each stage (e.g. queue jobs and collect later):
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
# 1. Submit a job (costs 2 credits, multiple keywords = same cost)
|
|
57
|
+
job = client.create_job(
|
|
58
|
+
["plumber in Miami FL", "plumbing service in Miami FL"],
|
|
59
|
+
email=True, depth=2,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# 2. Wait for completion (polls every 10s)
|
|
63
|
+
client.wait_for_job(job["id"], on_progress=lambda j: print("status:", j["status"]))
|
|
64
|
+
|
|
65
|
+
# 3a. Parsed dictsβ¦
|
|
66
|
+
records = client.download_records(job["id"])
|
|
67
|
+
|
|
68
|
+
# 3b. β¦or the raw CSV
|
|
69
|
+
csv_text = client.download_csv(job["id"])
|
|
70
|
+
|
|
71
|
+
# Check your balance
|
|
72
|
+
balance = client.credits() # {"credits": 8}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Options
|
|
76
|
+
|
|
77
|
+
All options for `scrape()` / `create_job()` (names match the REST API wire format and are stable):
|
|
78
|
+
|
|
79
|
+
| Option | Type | Default | Description |
|
|
80
|
+
| ----------- | -------------- | ------- | ---------------------------------------------------- |
|
|
81
|
+
| `email` | bool | `False` | Extract business emails from websites |
|
|
82
|
+
| `depth` | int (1β2) | `2` | Higher = more results, **same credit cost** |
|
|
83
|
+
| `zoom` | int (1β21) | `15` | Map zoom level |
|
|
84
|
+
| `radius` | int (meters) | `20000` | Search radius |
|
|
85
|
+
| `lang` | str | `"en"` | ISO 639-1 result language |
|
|
86
|
+
| `fast_mode` | bool | `True` | Skip deep website crawling |
|
|
87
|
+
| `max_time` | int (s) | `3600` | Job timeout on the backend |
|
|
88
|
+
| `lat`/`lon` | str \| float | β | Coordinates (auto-geocoded from keywords if omitted) |
|
|
89
|
+
|
|
90
|
+
`scrape()` and `wait_for_job()` also accept `poll_interval` (seconds, default `10` β the API minimum; keep it there to avoid rate limits), `timeout` (seconds, default `3600`) and `on_progress(job)`.
|
|
91
|
+
|
|
92
|
+
## Error handling
|
|
93
|
+
|
|
94
|
+
All failures β HTTP errors, network failures, failed jobs and timeouts β raise `GMapsScraperError` with `status` and `body`. Invalid arguments raise `TypeError`:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from gmapsscraper import GMapsScraper, GMapsScraperError
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
client.scrape("dentist in Chicago IL")
|
|
101
|
+
except GMapsScraperError as err:
|
|
102
|
+
# 401 invalid key Β· 402 out of credits Β· 422 bad params Β· 429 rate limited
|
|
103
|
+
print(err.status, err)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
| Status | Meaning |
|
|
107
|
+
| ------ | -------------------------------------------------------------------------- |
|
|
108
|
+
| 401 | Invalid API key |
|
|
109
|
+
| 402 | Insufficient credits β top up at [gmapsscraper.io](https://gmapsscraper.io/#pricing) |
|
|
110
|
+
| 422 | Invalid parameters |
|
|
111
|
+
| 429 | Rate limited (1000 req/day) or too many concurrent jobs (max 10) |
|
|
112
|
+
| 502 | Backend temporarily unavailable |
|
|
113
|
+
|
|
114
|
+
## Tips for better results
|
|
115
|
+
|
|
116
|
+
- Be specific: `"vegan restaurant in Brooklyn NY"` beats `"restaurant in New York"`.
|
|
117
|
+
- Pass several related keywords in one job β broader coverage, same 2 credits.
|
|
118
|
+
- Set `email=True` whenever you need contact info for cold outreach.
|
|
119
|
+
- Jobs typically finish in 30β120 seconds.
|
|
120
|
+
|
|
121
|
+
## Related resources
|
|
122
|
+
|
|
123
|
+
- π [API documentation](https://gmapsscraper.io/llms.txt) (also great as LLM context)
|
|
124
|
+
- π¦ [Node.js SDK](https://github.com/gmapsscraper/gmapsscraper-js) β `npm install @gmapsscraper/sdk`
|
|
125
|
+
- π€ [Claude / AI agent skills](https://github.com/gmapsscraper/google-maps-agent-skills) β use gmapsscraper from Claude Code and other agents
|
|
126
|
+
- 𧩠[Chrome extension](https://github.com/gmapsscraper/google-maps-scraper) β scrape Google Maps without code
|
|
127
|
+
- βοΈ [Blog: Google Maps scraping guides](https://gmapsscraper.io/blog)
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
[MIT](./LICENSE) Β© [gmapsscraper.io](https://gmapsscraper.io)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Usage: GMAPSSCRAPER_API_KEY=your_key python examples/basic.py
|
|
2
|
+
# Get a free key (10 credits): https://gmapsscraper.io/dashboard
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
from gmapsscraper import GMapsScraper
|
|
6
|
+
|
|
7
|
+
client = GMapsScraper(os.environ["GMAPSSCRAPER_API_KEY"])
|
|
8
|
+
|
|
9
|
+
print("Credits remaining:", client.credits()["credits"])
|
|
10
|
+
|
|
11
|
+
leads = client.scrape(
|
|
12
|
+
"coffee shop in Austin TX",
|
|
13
|
+
email=True,
|
|
14
|
+
on_progress=lambda job: print(f"Job {job['id']}: {job['status']}"),
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
print(f"Found {len(leads)} businesses")
|
|
18
|
+
for lead in leads[:5]:
|
|
19
|
+
print(f"- {lead['title']} | {lead['phone']} | {lead['email']} | {lead['website']}")
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "gmapsscraper-sdk"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Official Python SDK for the gmapsscraper.io API β scrape Google Maps business data: names, emails, phones, websites, ratings, reviews."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{ name = "gmapsscraper.io" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"google maps scraper",
|
|
15
|
+
"google maps",
|
|
16
|
+
"scraper",
|
|
17
|
+
"lead generation",
|
|
18
|
+
"leads",
|
|
19
|
+
"email extractor",
|
|
20
|
+
"local business",
|
|
21
|
+
"business data",
|
|
22
|
+
"b2b",
|
|
23
|
+
"places",
|
|
24
|
+
"reviews",
|
|
25
|
+
"sdk",
|
|
26
|
+
"api client",
|
|
27
|
+
]
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Development Status :: 4 - Beta",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"License :: OSI Approved :: MIT License",
|
|
32
|
+
"Operating System :: OS Independent",
|
|
33
|
+
"Programming Language :: Python :: 3",
|
|
34
|
+
"Programming Language :: Python :: 3.9",
|
|
35
|
+
"Programming Language :: Python :: 3.10",
|
|
36
|
+
"Programming Language :: Python :: 3.11",
|
|
37
|
+
"Programming Language :: Python :: 3.12",
|
|
38
|
+
"Programming Language :: Python :: 3.13",
|
|
39
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
40
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
41
|
+
"Typing :: Typed",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://gmapsscraper.io"
|
|
46
|
+
Repository = "https://github.com/gmapsscraper/gmapsscraper-python"
|
|
47
|
+
Documentation = "https://gmapsscraper.io/llms.txt"
|
|
48
|
+
Issues = "https://github.com/gmapsscraper/gmapsscraper-python/issues"
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
packages = ["src/gmapsscraper"]
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Official Python SDK for the gmapsscraper.io API.
|
|
2
|
+
|
|
3
|
+
Scrape Google Maps business data β names, addresses, phones, emails,
|
|
4
|
+
websites, ratings, reviews. Get a free API key (10 credits) at
|
|
5
|
+
https://gmapsscraper.io/dashboard
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv as _csv
|
|
11
|
+
import io
|
|
12
|
+
import json
|
|
13
|
+
import time
|
|
14
|
+
import urllib.error
|
|
15
|
+
import urllib.request
|
|
16
|
+
from urllib.parse import quote
|
|
17
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
|
|
18
|
+
|
|
19
|
+
__version__ = "0.1.0"
|
|
20
|
+
__all__ = ["GMapsScraper", "GMapsScraperError", "parse_csv"]
|
|
21
|
+
|
|
22
|
+
DEFAULT_BASE_URL = "https://gmapsscraper.io/api/v1"
|
|
23
|
+
|
|
24
|
+
# (method, url, headers, body) -> (status_code, response_text)
|
|
25
|
+
HttpCallable = Callable[[str, str, Dict[str, str], Optional[str]], Tuple[int, str]]
|
|
26
|
+
|
|
27
|
+
BusinessRecord = Dict[str, str]
|
|
28
|
+
Job = Dict[str, Any]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class GMapsScraperError(Exception):
|
|
32
|
+
"""Raised for HTTP errors, network failures, failed jobs and timeouts.
|
|
33
|
+
|
|
34
|
+
Attributes:
|
|
35
|
+
status: HTTP status code, if the error came from an HTTP response.
|
|
36
|
+
body: Parsed JSON error body or failed job dict, when available.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, message: str, status: Optional[int] = None, body: Any = None):
|
|
40
|
+
super().__init__(message)
|
|
41
|
+
self.status = status
|
|
42
|
+
self.body = body
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _default_http(
|
|
46
|
+
method: str, url: str, headers: Dict[str, str], body: Optional[str], timeout: float = 120.0
|
|
47
|
+
) -> Tuple[int, str]:
|
|
48
|
+
request = urllib.request.Request(
|
|
49
|
+
url,
|
|
50
|
+
data=body.encode("utf-8") if body is not None else None,
|
|
51
|
+
headers=headers,
|
|
52
|
+
method=method,
|
|
53
|
+
)
|
|
54
|
+
try:
|
|
55
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
56
|
+
return response.getcode(), response.read().decode("utf-8")
|
|
57
|
+
except urllib.error.HTTPError as exc:
|
|
58
|
+
return exc.code, exc.read().decode("utf-8", "replace")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class GMapsScraper:
|
|
62
|
+
"""Client for the gmapsscraper.io REST API.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
api_key: Your API key β get one at https://gmapsscraper.io/dashboard
|
|
66
|
+
base_url: API base URL (default: https://gmapsscraper.io/api/v1).
|
|
67
|
+
http_timeout: Per-request HTTP timeout in seconds for the default
|
|
68
|
+
transport (default: 120). Independent of the job-polling ``timeout``.
|
|
69
|
+
http: Optional transport override for testing:
|
|
70
|
+
``(method, url, headers, body) -> (status_code, response_text)``.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
def __init__(
|
|
74
|
+
self,
|
|
75
|
+
api_key: str,
|
|
76
|
+
*,
|
|
77
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
78
|
+
http_timeout: float = 120.0,
|
|
79
|
+
http: Optional[HttpCallable] = None,
|
|
80
|
+
):
|
|
81
|
+
if not isinstance(api_key, str) or not api_key.strip():
|
|
82
|
+
raise TypeError("api_key is required β get one at https://gmapsscraper.io/dashboard")
|
|
83
|
+
self._api_key = api_key
|
|
84
|
+
self.base_url = base_url.rstrip("/")
|
|
85
|
+
self._http = http or (
|
|
86
|
+
lambda method, url, headers, body: _default_http(method, url, headers, body, timeout=http_timeout)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def __repr__(self) -> str: # never expose the API key in logs/repr
|
|
90
|
+
return f"GMapsScraper(base_url={self.base_url!r})"
|
|
91
|
+
|
|
92
|
+
def _request(self, path: str, *, method: str = "GET", body: Any = None, raw: bool = False) -> Any:
|
|
93
|
+
headers = {
|
|
94
|
+
"Authorization": f"Bearer {self._api_key}",
|
|
95
|
+
"User-Agent": f"gmapsscraper-python/{__version__}",
|
|
96
|
+
}
|
|
97
|
+
payload = None
|
|
98
|
+
if body is not None:
|
|
99
|
+
headers["Content-Type"] = "application/json"
|
|
100
|
+
payload = json.dumps(body)
|
|
101
|
+
try:
|
|
102
|
+
status, text = self._http(method, f"{self.base_url}{path}", headers, payload)
|
|
103
|
+
except GMapsScraperError:
|
|
104
|
+
raise
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
raise GMapsScraperError(f"Network error: {exc}") from exc
|
|
107
|
+
if not 200 <= status < 300:
|
|
108
|
+
err_body: Any = None
|
|
109
|
+
try:
|
|
110
|
+
err_body = json.loads(text)
|
|
111
|
+
except ValueError:
|
|
112
|
+
pass
|
|
113
|
+
api_error = err_body.get("error") if isinstance(err_body, dict) else None
|
|
114
|
+
if isinstance(api_error, str) and api_error:
|
|
115
|
+
message = api_error
|
|
116
|
+
elif api_error is not None:
|
|
117
|
+
message = json.dumps(api_error)
|
|
118
|
+
else:
|
|
119
|
+
message = f"Request failed with HTTP {status}"
|
|
120
|
+
raise GMapsScraperError(message, status=status, body=err_body)
|
|
121
|
+
if raw:
|
|
122
|
+
return text
|
|
123
|
+
try:
|
|
124
|
+
return json.loads(text)
|
|
125
|
+
except ValueError as exc:
|
|
126
|
+
raise GMapsScraperError(f"Failed to parse response: {exc}", status=status) from exc
|
|
127
|
+
|
|
128
|
+
def create_job(self, keywords: Union[str, Iterable[str]], **options: Any) -> Dict[str, Any]:
|
|
129
|
+
"""Submit a scrape job. Costs 2 credits.
|
|
130
|
+
|
|
131
|
+
Option names match the REST API wire format and are stable:
|
|
132
|
+
``email``, ``depth``, ``zoom``, ``radius``, ``lang``, ``fast_mode``,
|
|
133
|
+
``max_time``, ``lat``, ``lon``, ``name``.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
``{"id": "job_xxx", "credits_remaining": 8}``
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
kw = [keywords] if isinstance(keywords, str) else list(keywords)
|
|
140
|
+
except TypeError:
|
|
141
|
+
raise TypeError(
|
|
142
|
+
'keywords must be a non-empty string or an iterable of non-empty strings, '
|
|
143
|
+
'e.g. "coffee shop in Austin TX"'
|
|
144
|
+
) from None
|
|
145
|
+
if not kw or any(not isinstance(k, str) or not k.strip() for k in kw):
|
|
146
|
+
raise TypeError(
|
|
147
|
+
'keywords must be a non-empty string or an iterable of non-empty strings, '
|
|
148
|
+
'e.g. "coffee shop in Austin TX"'
|
|
149
|
+
)
|
|
150
|
+
return self._request("/scrape", method="POST", body={**options, "keywords": kw})
|
|
151
|
+
|
|
152
|
+
def get_job(self, job_id: str) -> Job:
|
|
153
|
+
"""Get job status: ``{"id", "status": "running"|"complete"|"failed", "name"}``."""
|
|
154
|
+
return self._request(f"/jobs/{quote(job_id, safe='')}")
|
|
155
|
+
|
|
156
|
+
def wait_for_job(
|
|
157
|
+
self,
|
|
158
|
+
job_id: str,
|
|
159
|
+
*,
|
|
160
|
+
poll_interval: float = 10.0,
|
|
161
|
+
timeout: float = 3600.0,
|
|
162
|
+
on_progress: Optional[Callable[[Job], None]] = None,
|
|
163
|
+
) -> Job:
|
|
164
|
+
"""Poll a job until it completes; raises GMapsScraperError on failure or timeout."""
|
|
165
|
+
start = time.monotonic()
|
|
166
|
+
while True:
|
|
167
|
+
job = self.get_job(job_id)
|
|
168
|
+
if on_progress is not None:
|
|
169
|
+
on_progress(job)
|
|
170
|
+
status = job.get("status")
|
|
171
|
+
if status == "complete":
|
|
172
|
+
return job
|
|
173
|
+
if status == "failed":
|
|
174
|
+
reason = job.get("error")
|
|
175
|
+
suffix = f": {reason}" if isinstance(reason, str) and reason else ""
|
|
176
|
+
raise GMapsScraperError(f"Job {job_id} failed{suffix}", body=job)
|
|
177
|
+
remaining = timeout - (time.monotonic() - start)
|
|
178
|
+
if remaining <= 0:
|
|
179
|
+
raise GMapsScraperError(f"Timed out after {timeout}s waiting for job {job_id}", body=job)
|
|
180
|
+
time.sleep(min(poll_interval, remaining))
|
|
181
|
+
|
|
182
|
+
def download_csv(self, job_id: str) -> str:
|
|
183
|
+
"""Download job results as a raw CSV string.
|
|
184
|
+
|
|
185
|
+
Columns: title, address, phone, email, website, rating, reviews_count,
|
|
186
|
+
category, latitude, longitude, google_maps_url, opening_hours.
|
|
187
|
+
|
|
188
|
+
Note: the full CSV is buffered in memory; typical result sets are a few
|
|
189
|
+
hundred KB at most.
|
|
190
|
+
"""
|
|
191
|
+
return self._request(f"/jobs/{quote(job_id, safe='')}/download", raw=True)
|
|
192
|
+
|
|
193
|
+
def download_records(self, job_id: str) -> List[BusinessRecord]:
|
|
194
|
+
"""Download job results parsed into a list of dicts (one per business)."""
|
|
195
|
+
return parse_csv(self.download_csv(job_id))
|
|
196
|
+
|
|
197
|
+
def scrape(
|
|
198
|
+
self,
|
|
199
|
+
keywords: Union[str, Iterable[str]],
|
|
200
|
+
*,
|
|
201
|
+
poll_interval: float = 10.0,
|
|
202
|
+
timeout: float = 3600.0,
|
|
203
|
+
on_progress: Optional[Callable[[Job], None]] = None,
|
|
204
|
+
**options: Any,
|
|
205
|
+
) -> List[BusinessRecord]:
|
|
206
|
+
"""One call: create a job, wait for completion, return parsed records."""
|
|
207
|
+
job = self.create_job(keywords, **options)
|
|
208
|
+
job_id = job.get("id") if isinstance(job, dict) else None
|
|
209
|
+
if not job_id:
|
|
210
|
+
raise GMapsScraperError("Malformed response from /scrape: missing job id", body=job)
|
|
211
|
+
self.wait_for_job(job_id, poll_interval=poll_interval, timeout=timeout, on_progress=on_progress)
|
|
212
|
+
return self.download_records(job_id)
|
|
213
|
+
|
|
214
|
+
def credits(self) -> Dict[str, Any]:
|
|
215
|
+
"""Get remaining credit balance: ``{"credits": 8}``."""
|
|
216
|
+
return self._request("/credits")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def parse_csv(text: str) -> List[BusinessRecord]:
|
|
220
|
+
"""Parse RFC 4180 CSV text (strips a UTF-8 BOM) into a list of dicts."""
|
|
221
|
+
if text.startswith("\ufeff"):
|
|
222
|
+
text = text[1:]
|
|
223
|
+
reader = _csv.DictReader(io.StringIO(text))
|
|
224
|
+
records: List[BusinessRecord] = []
|
|
225
|
+
for row in reader:
|
|
226
|
+
records.append({
|
|
227
|
+
key: (value if isinstance(value, str) else "")
|
|
228
|
+
for key, value in row.items()
|
|
229
|
+
if key is not None
|
|
230
|
+
})
|
|
231
|
+
return records
|
|
File without changes
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import sys
|
|
3
|
+
import time
|
|
4
|
+
import unittest
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
|
8
|
+
|
|
9
|
+
from gmapsscraper import GMapsScraper, GMapsScraperError, parse_csv # noqa: E402
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def make_http(responses):
|
|
13
|
+
"""Build a fake transport from a list of (status, body) tuples (last one repeats)."""
|
|
14
|
+
calls = []
|
|
15
|
+
|
|
16
|
+
def http(method, url, headers, body):
|
|
17
|
+
index = min(len(calls), len(responses) - 1)
|
|
18
|
+
calls.append({"method": method, "url": url, "headers": headers, "body": body})
|
|
19
|
+
return responses[index]
|
|
20
|
+
|
|
21
|
+
http.calls = calls
|
|
22
|
+
return http
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ClientTest(unittest.TestCase):
|
|
26
|
+
def test_requires_api_key(self):
|
|
27
|
+
with self.assertRaises(TypeError):
|
|
28
|
+
GMapsScraper("")
|
|
29
|
+
with self.assertRaises(TypeError):
|
|
30
|
+
GMapsScraper(" ")
|
|
31
|
+
with self.assertRaises(TypeError):
|
|
32
|
+
GMapsScraper(None)
|
|
33
|
+
|
|
34
|
+
def test_repr_does_not_leak_api_key(self):
|
|
35
|
+
client = GMapsScraper("secret_key")
|
|
36
|
+
self.assertNotIn("secret_key", repr(client))
|
|
37
|
+
|
|
38
|
+
def test_create_job_sends_auth_and_wraps_keyword(self):
|
|
39
|
+
http = make_http([(201, json.dumps({"id": "job_1", "credits_remaining": 8}))])
|
|
40
|
+
client = GMapsScraper("key_123", http=http)
|
|
41
|
+
result = client.create_job("coffee shop in Austin TX", email=True)
|
|
42
|
+
self.assertEqual(result["id"], "job_1")
|
|
43
|
+
call = http.calls[0]
|
|
44
|
+
self.assertEqual(call["url"], "https://gmapsscraper.io/api/v1/scrape")
|
|
45
|
+
self.assertEqual(call["headers"]["Authorization"], "Bearer key_123")
|
|
46
|
+
self.assertEqual(
|
|
47
|
+
json.loads(call["body"]),
|
|
48
|
+
{"email": True, "keywords": ["coffee shop in Austin TX"]},
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def test_create_job_keywords_cannot_be_clobbered(self):
|
|
52
|
+
http = make_http([(201, json.dumps({"id": "job_2", "credits_remaining": 6}))])
|
|
53
|
+
client = GMapsScraper("key_123", http=http)
|
|
54
|
+
# Python itself rejects double-passing keywords; nothing reaches the API
|
|
55
|
+
with self.assertRaises(TypeError):
|
|
56
|
+
client.create_job(["a in NYC"], keywords=["evil"], depth=1)
|
|
57
|
+
self.assertEqual(http.calls, [])
|
|
58
|
+
# ...but passing keywords purely as a kwarg is fine
|
|
59
|
+
client.create_job(keywords=["a in NYC", "b in NYC"], depth=1)
|
|
60
|
+
self.assertEqual(
|
|
61
|
+
json.loads(http.calls[0]["body"]),
|
|
62
|
+
{"depth": 1, "keywords": ["a in NYC", "b in NYC"]},
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def test_create_job_rejects_invalid_keywords(self):
|
|
66
|
+
client = GMapsScraper("key_123", http=make_http([(201, "{}")]))
|
|
67
|
+
for bad in ([], "", ["ok", " "], [None], 42):
|
|
68
|
+
with self.assertRaises(TypeError):
|
|
69
|
+
client.create_job(bad)
|
|
70
|
+
|
|
71
|
+
def test_http_error_with_json_body(self):
|
|
72
|
+
http = make_http([(402, json.dumps({"error": "Insufficient credits"}))])
|
|
73
|
+
client = GMapsScraper("key_123", http=http)
|
|
74
|
+
with self.assertRaises(GMapsScraperError) as ctx:
|
|
75
|
+
client.credits()
|
|
76
|
+
self.assertEqual(ctx.exception.status, 402)
|
|
77
|
+
self.assertEqual(str(ctx.exception), "Insufficient credits")
|
|
78
|
+
|
|
79
|
+
def test_http_error_with_non_json_body(self):
|
|
80
|
+
http = make_http([(502, "<html>gateway error</html>")])
|
|
81
|
+
client = GMapsScraper("key_123", http=http)
|
|
82
|
+
with self.assertRaises(GMapsScraperError) as ctx:
|
|
83
|
+
client.credits()
|
|
84
|
+
self.assertEqual(ctx.exception.status, 502)
|
|
85
|
+
self.assertIn("HTTP 502", str(ctx.exception))
|
|
86
|
+
|
|
87
|
+
def test_network_error_is_wrapped_with_cause(self):
|
|
88
|
+
boom = OSError("connection refused")
|
|
89
|
+
|
|
90
|
+
def http(method, url, headers, body):
|
|
91
|
+
raise boom
|
|
92
|
+
|
|
93
|
+
client = GMapsScraper("key_123", http=http)
|
|
94
|
+
with self.assertRaises(GMapsScraperError) as ctx:
|
|
95
|
+
client.credits()
|
|
96
|
+
self.assertIn("Network error", str(ctx.exception))
|
|
97
|
+
self.assertIs(ctx.exception.__cause__, boom)
|
|
98
|
+
|
|
99
|
+
def test_wait_for_job_polls_until_complete(self):
|
|
100
|
+
http = make_http([
|
|
101
|
+
(200, json.dumps({"id": "job_1", "status": "running"})),
|
|
102
|
+
(200, json.dumps({"id": "job_1", "status": "running"})),
|
|
103
|
+
(200, json.dumps({"id": "job_1", "status": "complete"})),
|
|
104
|
+
])
|
|
105
|
+
client = GMapsScraper("key_123", http=http)
|
|
106
|
+
seen = []
|
|
107
|
+
job = client.wait_for_job("job_1", poll_interval=0.001, on_progress=lambda j: seen.append(j["status"]))
|
|
108
|
+
self.assertEqual(job["status"], "complete")
|
|
109
|
+
self.assertEqual(seen, ["running", "running", "complete"])
|
|
110
|
+
|
|
111
|
+
def test_wait_for_job_raises_on_failed(self):
|
|
112
|
+
http = make_http([(200, json.dumps({"id": "job_1", "status": "failed", "error": "backend exploded"}))])
|
|
113
|
+
client = GMapsScraper("key_123", http=http)
|
|
114
|
+
with self.assertRaises(GMapsScraperError) as ctx:
|
|
115
|
+
client.wait_for_job("job_1", poll_interval=0.001)
|
|
116
|
+
self.assertIn("backend exploded", str(ctx.exception))
|
|
117
|
+
|
|
118
|
+
def test_wait_for_job_times_out_but_not_early(self):
|
|
119
|
+
http = make_http([(200, json.dumps({"id": "job_1", "status": "running"}))])
|
|
120
|
+
client = GMapsScraper("key_123", http=http)
|
|
121
|
+
start = time.monotonic()
|
|
122
|
+
with self.assertRaises(GMapsScraperError) as ctx:
|
|
123
|
+
client.wait_for_job("job_1", poll_interval=0.005, timeout=0.03)
|
|
124
|
+
self.assertIn("Timed out", str(ctx.exception))
|
|
125
|
+
self.assertGreaterEqual(time.monotonic() - start, 0.03)
|
|
126
|
+
|
|
127
|
+
def test_scrape_full_flow(self):
|
|
128
|
+
csv_text = 'title,phone,email\n"Joe\'s ""Best"" Pizza, Inc",+1 555,joe@example.com\nBar,+2 555,\n'
|
|
129
|
+
|
|
130
|
+
def http(method, url, headers, body):
|
|
131
|
+
if url.endswith("/scrape"):
|
|
132
|
+
return 201, json.dumps({"id": "job_9", "credits_remaining": 6})
|
|
133
|
+
if url.endswith("/download"):
|
|
134
|
+
return 200, csv_text
|
|
135
|
+
return 200, json.dumps({"id": "job_9", "status": "complete"})
|
|
136
|
+
|
|
137
|
+
bodies = []
|
|
138
|
+
|
|
139
|
+
def recording_http(method, url, headers, body):
|
|
140
|
+
if body is not None:
|
|
141
|
+
bodies.append(json.loads(body))
|
|
142
|
+
return http(method, url, headers, body)
|
|
143
|
+
|
|
144
|
+
client = GMapsScraper("key_123", http=recording_http)
|
|
145
|
+
records = client.scrape(["pizza in NYC"], poll_interval=0.001, email=True)
|
|
146
|
+
# Client-side polling options must never leak into the API request body
|
|
147
|
+
self.assertEqual(bodies, [{"email": True, "keywords": ["pizza in NYC"]}])
|
|
148
|
+
self.assertEqual(len(records), 2)
|
|
149
|
+
self.assertEqual(records[0]["title"], 'Joe\'s "Best" Pizza, Inc')
|
|
150
|
+
self.assertEqual(records[0]["email"], "joe@example.com")
|
|
151
|
+
self.assertEqual(records[1]["email"], "")
|
|
152
|
+
|
|
153
|
+
def test_job_id_is_url_quoted(self):
|
|
154
|
+
http = make_http([(200, json.dumps({"id": "x", "status": "complete"}))])
|
|
155
|
+
client = GMapsScraper("key_123", http=http)
|
|
156
|
+
client.get_job("job/../weird id")
|
|
157
|
+
self.assertIn("/jobs/job%2F..%2Fweird%20id", http.calls[0]["url"])
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class ParseCsvTest(unittest.TestCase):
|
|
161
|
+
def test_quoted_newlines_and_crlf(self):
|
|
162
|
+
rows = parse_csv('a,b\r\n"line1\nline2",x\r\n')
|
|
163
|
+
self.assertEqual(rows, [{"a": "line1\nline2", "b": "x"}])
|
|
164
|
+
|
|
165
|
+
def test_empty_and_header_only(self):
|
|
166
|
+
self.assertEqual(parse_csv(""), [])
|
|
167
|
+
self.assertEqual(parse_csv("a,b\n"), [])
|
|
168
|
+
|
|
169
|
+
def test_strips_utf8_bom(self):
|
|
170
|
+
rows = parse_csv("\ufefftitle,phone\nJoe,+1 555\n")
|
|
171
|
+
self.assertEqual(rows, [{"title": "Joe", "phone": "+1 555"}])
|
|
172
|
+
|
|
173
|
+
def test_short_rows_fill_empty_strings(self):
|
|
174
|
+
rows = parse_csv("a,b,c\n1,2\n")
|
|
175
|
+
self.assertEqual(rows, [{"a": "1", "b": "2", "c": ""}])
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
if __name__ == "__main__":
|
|
179
|
+
unittest.main()
|