google-maps-leads-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- google_maps_leads_scraper-0.1.0/LICENSE +21 -0
- google_maps_leads_scraper-0.1.0/MANIFEST.in +3 -0
- google_maps_leads_scraper-0.1.0/PKG-INFO +131 -0
- google_maps_leads_scraper-0.1.0/README.md +109 -0
- google_maps_leads_scraper-0.1.0/examples/leads_grid.py +31 -0
- google_maps_leads_scraper-0.1.0/pyproject.toml +34 -0
- google_maps_leads_scraper-0.1.0/setup.cfg +4 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper/__init__.py +21 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper/cli.py +141 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper/client.py +203 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper/models.py +275 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper.egg-info/PKG-INFO +131 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper.egg-info/SOURCES.txt +21 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper.egg-info/dependency_links.txt +1 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper.egg-info/entry_points.txt +2 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper.egg-info/requires.txt +1 -0
- google_maps_leads_scraper-0.1.0/src/google_maps_leads_scraper.egg-info/top_level.txt +1 -0
- google_maps_leads_scraper-0.1.0/tests/fixtures/place.json +57 -0
- google_maps_leads_scraper-0.1.0/tests/fixtures/reviews_page1.json +77 -0
- google_maps_leads_scraper-0.1.0/tests/fixtures/reviews_page2.json +17 -0
- google_maps_leads_scraper-0.1.0/tests/fixtures/search_page1.json +710 -0
- google_maps_leads_scraper-0.1.0/tests/fixtures/search_page2.json +255 -0
- google_maps_leads_scraper-0.1.0/tests/test_scraper.py +217 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Buğrahan Saka
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: google-maps-leads-scraper
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Export Google Maps businesses (name, phone, website, address, rating, hours, coordinates), place details and reviews to CSV with Python and the Scrape.do Google Maps API.
|
|
5
|
+
Author: Buğrahan Saka
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/bugrahansss/google-maps-leads-scraper
|
|
8
|
+
Project-URL: Source, https://github.com/bugrahansss/google-maps-leads-scraper
|
|
9
|
+
Project-URL: Issues, https://github.com/bugrahansss/google-maps-leads-scraper/issues
|
|
10
|
+
Project-URL: Guide, https://scrape.do/blog/google-maps-scraping/
|
|
11
|
+
Keywords: google maps,google maps scraper,lead generation,web scraping,places,reviews,scrapedo
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
17
|
+
Requires-Python: >=3.8
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: requests>=2.25
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# Google Maps Leads Scraper
|
|
24
|
+
|
|
25
|
+
[](https://pypi.org/project/google-maps-leads-scraper/)
|
|
26
|
+
[](https://pypi.org/project/google-maps-leads-scraper/)
|
|
27
|
+
[](LICENSE)
|
|
28
|
+
|
|
29
|
+
Turn a Google Maps search into a lead list. Give it a query and a point on the map; get back every business with its **name, phone, website, address, rating, review count, opening hours, coordinates and place ID**, as JSON or CSV. It can also pull a single place's full details (rating breakdown, plus code, attributes) and its **reviews** with owner replies.
|
|
30
|
+
|
|
31
|
+
It runs on the [Scrape.do Google Maps API](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=google-maps-leads-scraper), which returns parsed JSON, so there is no HTML, protobuf or browser automation to maintain.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install google-maps-leads-scraper
|
|
35
|
+
export SCRAPEDO_TOKEN=your_token
|
|
36
|
+
google-maps-leads-scraper search "coffee shop" --ll 30.2672,-97.7431 --pages 6 -f csv -o austin-coffee.csv
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
ok 'coffee shop' @ 30.2672,-97.7431: 104 place(s)
|
|
41
|
+
104 record(s), 60 credits used, saved to austin-coffee.csv
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Why not just request google.com/maps?
|
|
45
|
+
|
|
46
|
+
Google Maps is a JavaScript app. The page you get from `requests` has no business data in it, and the internal endpoints the app calls (`/search?tbm=map`, `/maps/preview/place`) answer in a nested protobuf-over-JSON array with an anti-XSSI prefix and no stable field names. Scripts that send many of them meet the consent wall, CAPTCHAs and rate limits, and the reviews endpoint most old tutorials used (`listentitiesreviews`) has been retired.
|
|
47
|
+
|
|
48
|
+
The Scrape.do Google Maps API makes the request from a residential IP and returns clean JSON:
|
|
49
|
+
|
|
50
|
+
| Endpoint | Input | Returns |
|
|
51
|
+
| --- | --- | --- |
|
|
52
|
+
| `/plugin/google/maps/search` | `q`, `ll`, `start` | 20 places per page, about 120 per query |
|
|
53
|
+
| `/plugin/google/maps/place` | `place_id` or `data_cid` | one place with hours, rating breakdown, attributes |
|
|
54
|
+
| `/plugin/google/maps/reviews` | `data_id` or `place_id`, `sort_by` | about 10 reviews per page, owner replies, topics |
|
|
55
|
+
|
|
56
|
+
Every successful call costs 10 credits, so a full page of 20 leads costs 10 credits and 1,000 leads cost about 500. The free plan has 1,000 credits a month.
|
|
57
|
+
|
|
58
|
+
## Command line
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
# One query around a point, 6 pages (the ~120 place maximum)
|
|
62
|
+
google-maps-leads-scraper search "plumber" --ll 40.7128,-74.0060 --pages 6 -f csv -o nyc-plumbers.csv
|
|
63
|
+
|
|
64
|
+
# The location can be in the query instead
|
|
65
|
+
google-maps-leads-scraper search "vegan restaurant in Lisbon" --gl pt
|
|
66
|
+
|
|
67
|
+
# Several queries across several points, merged without duplicates
|
|
68
|
+
google-maps-leads-scraper search "dentist" "orthodontist" \
|
|
69
|
+
--ll 30.2672,-97.7431 --ll 30.3072,-97.7559 --pages 2 -f csv -o dentists.csv
|
|
70
|
+
|
|
71
|
+
# Full details for places
|
|
72
|
+
google-maps-leads-scraper place ChIJ_UbNVbC1RIYRTavD-VMmvZA
|
|
73
|
+
|
|
74
|
+
# Reviews, newest first, two pages
|
|
75
|
+
google-maps-leads-scraper reviews --data-id 0x8644b5b055cd46fd:0x90bd2653f9c3ab4d --sort newest --pages 2 -f csv -o reviews.csv
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`--ll` takes `lat,lng`, `lat,lng,zoom` or the `@lat,lng,14z` form from a Maps URL. `search --details` also fetches every place's detail page (10 more credits per place).
|
|
79
|
+
|
|
80
|
+
## Python
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from google_maps_leads_scraper import GoogleMapsScraper
|
|
84
|
+
|
|
85
|
+
maps = GoogleMapsScraper(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
|
|
86
|
+
|
|
87
|
+
places = maps.search("coffee shop", ll="30.2672,-97.7431", pages=3)
|
|
88
|
+
for p in places[:3]:
|
|
89
|
+
print(p.name, p.rating, p.reviews, p.phone, p.website)
|
|
90
|
+
|
|
91
|
+
details = maps.place(places[0].place_id)
|
|
92
|
+
print(details.hours) # Mon: 7 AM–2 PM; Tue: 7 AM–2 PM; ...
|
|
93
|
+
print(details.rating_breakdown) # {'1': 16, '2': 6, '3': 14, '4': 41, '5': 604}
|
|
94
|
+
|
|
95
|
+
for review in maps.reviews(data_id=places[0].data_id, sort_by="lowest_rating", pages=2):
|
|
96
|
+
print(review.rating, review.date, review.text[:80], "| reply:", bool(review.owner_reply))
|
|
97
|
+
|
|
98
|
+
print(maps.credits_used)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
To cover a whole city, run the query around several points; `search_grid` does that and drops duplicates by place ID. See [`examples/leads_grid.py`](examples/leads_grid.py).
|
|
102
|
+
|
|
103
|
+
## Output fields
|
|
104
|
+
|
|
105
|
+
**Places:** `name`, `category`, `categories`, `rating`, `reviews`, `phone`, `website`, `address`, `latitude`, `longitude`, `hours`, `open_state`, `place_id`, `data_id`, `data_cid`, `plus_code`, `thumbnail`, `maps_url`. JSON output adds `rating_breakdown` and `attributes` (accessibility, service options and so on).
|
|
106
|
+
|
|
107
|
+
**Reviews:** `place_id`, `data_id`, `review_id`, `rating`, `date`, `iso_date`, `text`, `likes`, `author`, `author_reviews`, `local_guide`, `owner_reply`, `owner_reply_date`, `link`.
|
|
108
|
+
|
|
109
|
+
`place_id` feeds `place`; `data_id` feeds `reviews`.
|
|
110
|
+
|
|
111
|
+
## Use it responsibly
|
|
112
|
+
|
|
113
|
+
Business listings are public, but reviews carry the names of the people who wrote them. Store only what you need, follow the privacy rules that apply to you (GDPR, CCPA), respect opt-outs before cold outreach, and read Google's terms for your use case.
|
|
114
|
+
|
|
115
|
+
## Also available
|
|
116
|
+
|
|
117
|
+
- **No-code:** [Google Maps Leads Scraper on Apify](https://apify.com/scrapedo/google-maps-leads-scraper)
|
|
118
|
+
- **Browser demo:** [Hugging Face Space](https://huggingface.co/spaces/bugrahans/google-maps-leads-scraper)
|
|
119
|
+
- **Full guide:** [Google Maps Scraper: How to Scrape Google Maps for Leads](https://scrape.do/blog/google-maps-scraping/?utm_source=github&utm_medium=readme&utm_campaign=google-maps-leads-scraper), including the raw Python approach without the API
|
|
120
|
+
|
|
121
|
+
## Tests
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
python -m unittest discover -s tests
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The fixtures are made-up businesses on example.com; the tests never hit the network.
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
MIT
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Google Maps Leads Scraper
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/google-maps-leads-scraper/)
|
|
4
|
+
[](https://pypi.org/project/google-maps-leads-scraper/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Turn a Google Maps search into a lead list. Give it a query and a point on the map; get back every business with its **name, phone, website, address, rating, review count, opening hours, coordinates and place ID**, as JSON or CSV. It can also pull a single place's full details (rating breakdown, plus code, attributes) and its **reviews** with owner replies.
|
|
8
|
+
|
|
9
|
+
It runs on the [Scrape.do Google Maps API](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=google-maps-leads-scraper), which returns parsed JSON, so there is no HTML, protobuf or browser automation to maintain.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install google-maps-leads-scraper
|
|
13
|
+
export SCRAPEDO_TOKEN=your_token
|
|
14
|
+
google-maps-leads-scraper search "coffee shop" --ll 30.2672,-97.7431 --pages 6 -f csv -o austin-coffee.csv
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
ok 'coffee shop' @ 30.2672,-97.7431: 104 place(s)
|
|
19
|
+
104 record(s), 60 credits used, saved to austin-coffee.csv
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Why not just request google.com/maps?
|
|
23
|
+
|
|
24
|
+
Google Maps is a JavaScript app. The page you get from `requests` has no business data in it, and the internal endpoints the app calls (`/search?tbm=map`, `/maps/preview/place`) answer in a nested protobuf-over-JSON array with an anti-XSSI prefix and no stable field names. Scripts that send many of them meet the consent wall, CAPTCHAs and rate limits, and the reviews endpoint most old tutorials used (`listentitiesreviews`) has been retired.
|
|
25
|
+
|
|
26
|
+
The Scrape.do Google Maps API makes the request from a residential IP and returns clean JSON:
|
|
27
|
+
|
|
28
|
+
| Endpoint | Input | Returns |
|
|
29
|
+
| --- | --- | --- |
|
|
30
|
+
| `/plugin/google/maps/search` | `q`, `ll`, `start` | 20 places per page, about 120 per query |
|
|
31
|
+
| `/plugin/google/maps/place` | `place_id` or `data_cid` | one place with hours, rating breakdown, attributes |
|
|
32
|
+
| `/plugin/google/maps/reviews` | `data_id` or `place_id`, `sort_by` | about 10 reviews per page, owner replies, topics |
|
|
33
|
+
|
|
34
|
+
Every successful call costs 10 credits, so a full page of 20 leads costs 10 credits and 1,000 leads cost about 500. The free plan has 1,000 credits a month.
|
|
35
|
+
|
|
36
|
+
## Command line
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# One query around a point, 6 pages (the ~120 place maximum)
|
|
40
|
+
google-maps-leads-scraper search "plumber" --ll 40.7128,-74.0060 --pages 6 -f csv -o nyc-plumbers.csv
|
|
41
|
+
|
|
42
|
+
# The location can be in the query instead
|
|
43
|
+
google-maps-leads-scraper search "vegan restaurant in Lisbon" --gl pt
|
|
44
|
+
|
|
45
|
+
# Several queries across several points, merged without duplicates
|
|
46
|
+
google-maps-leads-scraper search "dentist" "orthodontist" \
|
|
47
|
+
--ll 30.2672,-97.7431 --ll 30.3072,-97.7559 --pages 2 -f csv -o dentists.csv
|
|
48
|
+
|
|
49
|
+
# Full details for places
|
|
50
|
+
google-maps-leads-scraper place ChIJ_UbNVbC1RIYRTavD-VMmvZA
|
|
51
|
+
|
|
52
|
+
# Reviews, newest first, two pages
|
|
53
|
+
google-maps-leads-scraper reviews --data-id 0x8644b5b055cd46fd:0x90bd2653f9c3ab4d --sort newest --pages 2 -f csv -o reviews.csv
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`--ll` takes `lat,lng`, `lat,lng,zoom` or the `@lat,lng,14z` form from a Maps URL. `search --details` also fetches every place's detail page (10 more credits per place).
|
|
57
|
+
|
|
58
|
+
## Python
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from google_maps_leads_scraper import GoogleMapsScraper
|
|
62
|
+
|
|
63
|
+
maps = GoogleMapsScraper(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
|
|
64
|
+
|
|
65
|
+
places = maps.search("coffee shop", ll="30.2672,-97.7431", pages=3)
|
|
66
|
+
for p in places[:3]:
|
|
67
|
+
print(p.name, p.rating, p.reviews, p.phone, p.website)
|
|
68
|
+
|
|
69
|
+
details = maps.place(places[0].place_id)
|
|
70
|
+
print(details.hours) # Mon: 7 AM–2 PM; Tue: 7 AM–2 PM; ...
|
|
71
|
+
print(details.rating_breakdown) # {'1': 16, '2': 6, '3': 14, '4': 41, '5': 604}
|
|
72
|
+
|
|
73
|
+
for review in maps.reviews(data_id=places[0].data_id, sort_by="lowest_rating", pages=2):
|
|
74
|
+
print(review.rating, review.date, review.text[:80], "| reply:", bool(review.owner_reply))
|
|
75
|
+
|
|
76
|
+
print(maps.credits_used)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
To cover a whole city, run the query around several points; `search_grid` does that and drops duplicates by place ID. See [`examples/leads_grid.py`](examples/leads_grid.py).
|
|
80
|
+
|
|
81
|
+
## Output fields
|
|
82
|
+
|
|
83
|
+
**Places:** `name`, `category`, `categories`, `rating`, `reviews`, `phone`, `website`, `address`, `latitude`, `longitude`, `hours`, `open_state`, `place_id`, `data_id`, `data_cid`, `plus_code`, `thumbnail`, `maps_url`. JSON output adds `rating_breakdown` and `attributes` (accessibility, service options and so on).
|
|
84
|
+
|
|
85
|
+
**Reviews:** `place_id`, `data_id`, `review_id`, `rating`, `date`, `iso_date`, `text`, `likes`, `author`, `author_reviews`, `local_guide`, `owner_reply`, `owner_reply_date`, `link`.
|
|
86
|
+
|
|
87
|
+
`place_id` feeds `place`; `data_id` feeds `reviews`.
|
|
88
|
+
|
|
89
|
+
## Use it responsibly
|
|
90
|
+
|
|
91
|
+
Business listings are public, but reviews carry the names of the people who wrote them. Store only what you need, follow the privacy rules that apply to you (GDPR, CCPA), respect opt-outs before cold outreach, and read Google's terms for your use case.
|
|
92
|
+
|
|
93
|
+
## Also available
|
|
94
|
+
|
|
95
|
+
- **No-code:** [Google Maps Leads Scraper on Apify](https://apify.com/scrapedo/google-maps-leads-scraper)
|
|
96
|
+
- **Browser demo:** [Hugging Face Space](https://huggingface.co/spaces/bugrahans/google-maps-leads-scraper)
|
|
97
|
+
- **Full guide:** [Google Maps Scraper: How to Scrape Google Maps for Leads](https://scrape.do/blog/google-maps-scraping/?utm_source=github&utm_medium=readme&utm_campaign=google-maps-leads-scraper), including the raw Python approach without the API
|
|
98
|
+
|
|
99
|
+
## Tests
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
python -m unittest discover -s tests
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The fixtures are made-up businesses on example.com; the tests never hit the network.
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Build a lead list for one category across a small grid of points in Austin, TX.
|
|
2
|
+
|
|
3
|
+
One Google Maps query stops at about 120 places, so a city-wide list repeats
|
|
4
|
+
the query around several centre points and drops the duplicates.
|
|
5
|
+
|
|
6
|
+
export SCRAPEDO_TOKEN=...
|
|
7
|
+
python examples/leads_grid.py
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
|
|
12
|
+
from google_maps_leads_scraper import PLACE_FIELDS, GoogleMapsScraper
|
|
13
|
+
|
|
14
|
+
POINTS = [
|
|
15
|
+
"30.2672,-97.7431", # downtown
|
|
16
|
+
"30.3072,-97.7559", # north central
|
|
17
|
+
"30.2240,-97.7690", # south
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
maps = GoogleMapsScraper() # reads SCRAPEDO_TOKEN
|
|
21
|
+
places = maps.search_grid(["dentist"], POINTS, pages=2)
|
|
22
|
+
|
|
23
|
+
# Keep the ones a sales team can actually contact
|
|
24
|
+
leads = [p for p in places if p.phone or p.website]
|
|
25
|
+
|
|
26
|
+
with open("austin-dentists.csv", "w", newline="", encoding="utf-8") as handle:
|
|
27
|
+
writer = csv.DictWriter(handle, fieldnames=PLACE_FIELDS)
|
|
28
|
+
writer.writeheader()
|
|
29
|
+
writer.writerows(p.to_row() for p in leads)
|
|
30
|
+
|
|
31
|
+
print(f"{len(leads)} leads from {len(places)} places, {maps.credits_used} credits")
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "google-maps-leads-scraper"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Export Google Maps businesses (name, phone, website, address, rating, hours, coordinates), place details and reviews to CSV with Python and the Scrape.do Google Maps API."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Buğrahan Saka" }]
|
|
14
|
+
keywords = ["google maps", "google maps scraper", "lead generation", "web scraping", "places", "reviews", "scrapedo"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["requests>=2.25"]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/bugrahansss/google-maps-leads-scraper"
|
|
26
|
+
Source = "https://github.com/bugrahansss/google-maps-leads-scraper"
|
|
27
|
+
Issues = "https://github.com/bugrahansss/google-maps-leads-scraper/issues"
|
|
28
|
+
Guide = "https://scrape.do/blog/google-maps-scraping/"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
google-maps-leads-scraper = "google_maps_leads_scraper.cli:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Export Google Maps businesses, place details and reviews with Python and the Scrape.do Google Maps API."""
|
|
2
|
+
|
|
3
|
+
from .client import GoogleMapsScraper, ScrapeError
|
|
4
|
+
from .models import PLACE_FIELDS, REVIEW_FIELDS, Place, Review, dedupe, format_hours, format_ll, parse_place, parse_reviews, parse_search
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"GoogleMapsScraper",
|
|
8
|
+
"PLACE_FIELDS",
|
|
9
|
+
"Place",
|
|
10
|
+
"REVIEW_FIELDS",
|
|
11
|
+
"Review",
|
|
12
|
+
"ScrapeError",
|
|
13
|
+
"dedupe",
|
|
14
|
+
"format_hours",
|
|
15
|
+
"format_ll",
|
|
16
|
+
"parse_place",
|
|
17
|
+
"parse_reviews",
|
|
18
|
+
"parse_search",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Command line entry point.
|
|
2
|
+
|
|
3
|
+
google-maps-leads-scraper search "coffee shop" --ll 30.2672,-97.7431 --pages 3 -f csv -o leads.csv
|
|
4
|
+
google-maps-leads-scraper place ChIJ...
|
|
5
|
+
google-maps-leads-scraper reviews --data-id 0x...:0x... --sort newest --pages 2
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import csv
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
from typing import Any, Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .client import SORT_OPTIONS, TOKEN_ENV, GoogleMapsScraper, ScrapeError
|
|
18
|
+
from .models import PLACE_FIELDS, REVIEW_FIELDS, dedupe
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
22
|
+
parser = argparse.ArgumentParser(
|
|
23
|
+
prog="google-maps-leads-scraper",
|
|
24
|
+
description="Export Google Maps businesses and reviews to JSON or CSV through the Scrape.do Google Maps API.",
|
|
25
|
+
)
|
|
26
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
27
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
28
|
+
common.add_argument("-t", "--token", help=f"Scrape.do API token (default: ${TOKEN_ENV})")
|
|
29
|
+
common.add_argument("--gl", default="us", help="Country code for results (default: us)")
|
|
30
|
+
common.add_argument("--hl", default="en", help="Language code (default: en)")
|
|
31
|
+
common.add_argument("-f", "--format", choices=("json", "csv"), default="json", help="Output format (default: json)")
|
|
32
|
+
common.add_argument("-o", "--output", help="Write to this file instead of stdout")
|
|
33
|
+
|
|
34
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
35
|
+
|
|
36
|
+
search = sub.add_parser("search", parents=[common], help="Businesses for one or more queries")
|
|
37
|
+
search.add_argument("queries", nargs="+", help='Search terms, e.g. "dentist" or "coffee shop in Austin, TX"')
|
|
38
|
+
search.add_argument("--ll", action="append", default=[], help="Centre point lat,lng (repeatable for a grid)")
|
|
39
|
+
search.add_argument("--zoom", type=float, default=14, help="Map zoom for --ll (default: 14)")
|
|
40
|
+
search.add_argument("--pages", type=int, default=1, help="Pages per query and point, 20 places each, max 6 (default: 1)")
|
|
41
|
+
search.add_argument("--details", action="store_true", help="Also fetch each place's details (10 credits per place)")
|
|
42
|
+
|
|
43
|
+
place = sub.add_parser("place", parents=[common], help="Details for places by place_id")
|
|
44
|
+
place.add_argument("place_ids", nargs="*", help="Google place IDs (ChIJ...)")
|
|
45
|
+
place.add_argument("--cid", action="append", default=[], help="Look up by data_cid instead (repeatable)")
|
|
46
|
+
|
|
47
|
+
reviews = sub.add_parser("reviews", parents=[common], help="Reviews for one place")
|
|
48
|
+
target = reviews.add_mutually_exclusive_group(required=True)
|
|
49
|
+
target.add_argument("--data-id", help="Feature ID, e.g. 0x8644b5b055cd46fd:0x90bd2653f9c3ab4d")
|
|
50
|
+
target.add_argument("--place-id", help="Google place ID (ChIJ...)")
|
|
51
|
+
reviews.add_argument("--sort", choices=SORT_OPTIONS, default="newest", help="Review order (default: newest)")
|
|
52
|
+
reviews.add_argument("--pages", type=int, default=1, help="Review pages, about 10 each (default: 1)")
|
|
53
|
+
reviews.add_argument("--query", default="", help="Only reviews that mention this word")
|
|
54
|
+
return parser
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _write(rows: List[Dict[str, Any]], records: List[Dict[str, Any]], fields: List[str], fmt: str, output: Optional[str]) -> None:
|
|
58
|
+
stream = open(output, "w", encoding="utf-8", newline="" if fmt == "csv" else None) if output else sys.stdout
|
|
59
|
+
try:
|
|
60
|
+
if fmt == "csv":
|
|
61
|
+
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore")
|
|
62
|
+
writer.writeheader()
|
|
63
|
+
writer.writerows(rows)
|
|
64
|
+
else:
|
|
65
|
+
json.dump(records, stream, indent=2, ensure_ascii=False)
|
|
66
|
+
stream.write("\n")
|
|
67
|
+
finally:
|
|
68
|
+
if output:
|
|
69
|
+
stream.close()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
73
|
+
args = build_parser().parse_args(argv)
|
|
74
|
+
try:
|
|
75
|
+
maps = GoogleMapsScraper(token=args.token, gl=args.gl, hl=args.hl)
|
|
76
|
+
except ValueError as exc:
|
|
77
|
+
print(exc, file=sys.stderr)
|
|
78
|
+
return 2
|
|
79
|
+
|
|
80
|
+
failures = 0
|
|
81
|
+
if args.command == "search":
|
|
82
|
+
places = []
|
|
83
|
+
for query in args.queries:
|
|
84
|
+
for point in args.ll or [None]:
|
|
85
|
+
label = f"{query!r}" + (f" @ {point}" if point else "")
|
|
86
|
+
try:
|
|
87
|
+
found = maps.search(query, ll=point, pages=args.pages, zoom=args.zoom)
|
|
88
|
+
except (ScrapeError, ValueError) as exc:
|
|
89
|
+
failures += 1
|
|
90
|
+
print(f"fail {label}: {exc}", file=sys.stderr)
|
|
91
|
+
continue
|
|
92
|
+
print(f"ok {label}: {len(found)} place(s)", file=sys.stderr)
|
|
93
|
+
places.extend(found)
|
|
94
|
+
places = dedupe(places)
|
|
95
|
+
if args.details:
|
|
96
|
+
detailed = []
|
|
97
|
+
for p in places:
|
|
98
|
+
try:
|
|
99
|
+
detailed.append(maps.place(p.place_id, p.data_cid) if (p.place_id or p.data_cid) else p)
|
|
100
|
+
except (ScrapeError, ValueError) as exc:
|
|
101
|
+
print(f"fail details {p.name}: {exc}", file=sys.stderr)
|
|
102
|
+
detailed.append(p)
|
|
103
|
+
places = detailed
|
|
104
|
+
_write([p.to_row() for p in places], [p.to_dict() for p in places], PLACE_FIELDS, args.format, args.output)
|
|
105
|
+
count = len(places)
|
|
106
|
+
elif args.command == "place":
|
|
107
|
+
if not args.place_ids and not args.cid:
|
|
108
|
+
print("Pass at least one place_id or --cid.", file=sys.stderr)
|
|
109
|
+
return 2
|
|
110
|
+
places = []
|
|
111
|
+
for pid in args.place_ids:
|
|
112
|
+
try:
|
|
113
|
+
places.append(maps.place(place_id=pid))
|
|
114
|
+
except (ScrapeError, ValueError) as exc:
|
|
115
|
+
failures += 1
|
|
116
|
+
print(f"fail {pid}: {exc}", file=sys.stderr)
|
|
117
|
+
for cid in args.cid:
|
|
118
|
+
try:
|
|
119
|
+
places.append(maps.place(data_cid=cid))
|
|
120
|
+
except (ScrapeError, ValueError) as exc:
|
|
121
|
+
failures += 1
|
|
122
|
+
print(f"fail cid {cid}: {exc}", file=sys.stderr)
|
|
123
|
+
_write([p.to_row() for p in places], [p.to_dict() for p in places], PLACE_FIELDS, args.format, args.output)
|
|
124
|
+
count = len(places)
|
|
125
|
+
else:
|
|
126
|
+
try:
|
|
127
|
+
found = maps.reviews(
|
|
128
|
+
data_id=args.data_id or "", place_id=args.place_id or "", sort_by=args.sort, pages=args.pages, query=args.query
|
|
129
|
+
)
|
|
130
|
+
except (ScrapeError, ValueError) as exc:
|
|
131
|
+
print(f"fail reviews: {exc}", file=sys.stderr)
|
|
132
|
+
return 1
|
|
133
|
+
_write([r.to_row() for r in found], [r.to_dict() for r in found], REVIEW_FIELDS, args.format, args.output)
|
|
134
|
+
count = len(found)
|
|
135
|
+
|
|
136
|
+
print(f"{count} record(s), {maps.credits_used} credits used" + (f", saved to {args.output}" if args.output else ""), file=sys.stderr)
|
|
137
|
+
return 1 if failures and not count else 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
if __name__ == "__main__":
|
|
141
|
+
raise SystemExit(main())
|