boltscraper 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- boltscraper/__init__.py +27 -0
- boltscraper/client.py +166 -0
- boltscraper/exceptions.py +38 -0
- boltscraper/maps.py +106 -0
- boltscraper/reviews.py +80 -0
- boltscraper/webhooks.py +79 -0
- boltscraper/website.py +67 -0
- boltscraper-1.0.0.dist-info/METADATA +231 -0
- boltscraper-1.0.0.dist-info/RECORD +12 -0
- boltscraper-1.0.0.dist-info/WHEEL +5 -0
- boltscraper-1.0.0.dist-info/licenses/LICENSE +21 -0
- boltscraper-1.0.0.dist-info/top_level.txt +1 -0
boltscraper/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bolt Scraper Official Python SDK
|
|
3
|
+
Fast, reliable, and proxy-free scraping for Google Maps, Google Reviews, and Websites.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .client import BoltScraper, VERSION
|
|
7
|
+
from .exceptions import (
|
|
8
|
+
BoltScraperError,
|
|
9
|
+
AuthenticationError,
|
|
10
|
+
QuotaExceededError,
|
|
11
|
+
NotFoundError,
|
|
12
|
+
RateLimitError,
|
|
13
|
+
UpstreamError
|
|
14
|
+
)
|
|
15
|
+
from .webhooks import WebhooksResource
|
|
16
|
+
|
|
17
|
+
__version__ = VERSION
|
|
18
|
+
__all__ = [
|
|
19
|
+
"BoltScraper",
|
|
20
|
+
"WebhooksResource",
|
|
21
|
+
"BoltScraperError",
|
|
22
|
+
"AuthenticationError",
|
|
23
|
+
"QuotaExceededError",
|
|
24
|
+
"NotFoundError",
|
|
25
|
+
"RateLimitError",
|
|
26
|
+
"UpstreamError"
|
|
27
|
+
]
|
boltscraper/client.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bolt Scraper Official Python SDK Client
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
from typing import Optional, Dict, Any
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from .exceptions import (
|
|
11
|
+
BoltScraperError,
|
|
12
|
+
AuthenticationError,
|
|
13
|
+
QuotaExceededError,
|
|
14
|
+
NotFoundError,
|
|
15
|
+
RateLimitError,
|
|
16
|
+
UpstreamError
|
|
17
|
+
)
|
|
18
|
+
from .maps import MapsResource
|
|
19
|
+
from .reviews import ReviewsResource
|
|
20
|
+
from .website import WebsiteResource
|
|
21
|
+
from .webhooks import WebhooksResource
|
|
22
|
+
|
|
23
|
+
DEFAULT_BASE_URL = "https://boltscraper.com/api/v1"
|
|
24
|
+
VERSION = "1.0.0"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class BoltScraper:
|
|
28
|
+
"""
|
|
29
|
+
Official Python SDK client for Bolt Scraper.
|
|
30
|
+
|
|
31
|
+
Usage:
|
|
32
|
+
from boltscraper import BoltScraper
|
|
33
|
+
|
|
34
|
+
# Initialize with your API key
|
|
35
|
+
client = BoltScraper(api_key="bs_live_your_key_here")
|
|
36
|
+
|
|
37
|
+
# 1. Search Google Maps in real time
|
|
38
|
+
results = client.maps.search("Plumbers in Austin, TX", page=1)
|
|
39
|
+
|
|
40
|
+
# 2. Extract customer reviews without OAuth
|
|
41
|
+
reviews = client.reviews.search("ChIJb8Jg9q-1RIYRAxvq_mZ0gXY", max_reviews=100)
|
|
42
|
+
|
|
43
|
+
# 3. Extract direct business emails and socials from a domain
|
|
44
|
+
company = client.website.scrape("https://example.com")
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
api_key: Optional[str] = None,
|
|
50
|
+
base_url: str = DEFAULT_BASE_URL,
|
|
51
|
+
timeout: int = 60,
|
|
52
|
+
max_retries: int = 3,
|
|
53
|
+
auto_retry_rate_limit: bool = True
|
|
54
|
+
):
|
|
55
|
+
self.api_key = api_key or os.getenv("BOLTSCRAPER_API_KEY")
|
|
56
|
+
if not self.api_key:
|
|
57
|
+
raise AuthenticationError(
|
|
58
|
+
"Missing Bolt Scraper API key. Pass 'api_key' to BoltScraper(api_key='bs_...') "
|
|
59
|
+
"or set the 'BOLTSCRAPER_API_KEY' environment variable. "
|
|
60
|
+
"Get your key at https://app.boltscraper.com"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
self.base_url = base_url.rstrip("/")
|
|
64
|
+
self.timeout = timeout
|
|
65
|
+
self.max_retries = max_retries
|
|
66
|
+
self.auto_retry_rate_limit = auto_retry_rate_limit
|
|
67
|
+
|
|
68
|
+
self.session = requests.Session()
|
|
69
|
+
self.session.headers.update({
|
|
70
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
71
|
+
"Content-Type": "application/json",
|
|
72
|
+
"User-Agent": f"boltscraper-python/{VERSION}"
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
# Initialize resources
|
|
76
|
+
self.maps = MapsResource(self)
|
|
77
|
+
self.reviews = ReviewsResource(self)
|
|
78
|
+
self.website = WebsiteResource(self)
|
|
79
|
+
self.webhooks = WebhooksResource(self)
|
|
80
|
+
|
|
81
|
+
def _request(
|
|
82
|
+
self,
|
|
83
|
+
method: str,
|
|
84
|
+
path: str,
|
|
85
|
+
params: Optional[Dict[str, Any]] = None,
|
|
86
|
+
json: Optional[Dict[str, Any]] = None,
|
|
87
|
+
raw_bytes: bool = False
|
|
88
|
+
) -> Any:
|
|
89
|
+
url = f"{self.base_url}{path}"
|
|
90
|
+
retries = 0
|
|
91
|
+
|
|
92
|
+
while True:
|
|
93
|
+
try:
|
|
94
|
+
response = self.session.request(
|
|
95
|
+
method=method,
|
|
96
|
+
url=url,
|
|
97
|
+
params=params,
|
|
98
|
+
json=json,
|
|
99
|
+
timeout=self.timeout
|
|
100
|
+
)
|
|
101
|
+
except requests.exceptions.RequestException as e:
|
|
102
|
+
raise BoltScraperError(f"Network request to Bolt Scraper failed: {str(e)}")
|
|
103
|
+
|
|
104
|
+
if response.status_code == 200:
|
|
105
|
+
if raw_bytes:
|
|
106
|
+
return response.content
|
|
107
|
+
try:
|
|
108
|
+
return response.json()
|
|
109
|
+
except ValueError:
|
|
110
|
+
return response.text
|
|
111
|
+
|
|
112
|
+
# Handle 429 with automatic exponential backoff
|
|
113
|
+
if response.status_code == 429:
|
|
114
|
+
retry_after = int(response.headers.get("Retry-After", 2 ** retries))
|
|
115
|
+
if self.auto_retry_rate_limit and retries < self.max_retries:
|
|
116
|
+
retries += 1
|
|
117
|
+
time.sleep(retry_after)
|
|
118
|
+
continue
|
|
119
|
+
try:
|
|
120
|
+
err_json = response.json()
|
|
121
|
+
except ValueError:
|
|
122
|
+
err_json = {}
|
|
123
|
+
raise RateLimitError(
|
|
124
|
+
f"Bolt Scraper rate limit exceeded (60 req/min). Retry after {retry_after}s.",
|
|
125
|
+
retry_after=retry_after,
|
|
126
|
+
response_body=err_json
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Parse error payload
|
|
130
|
+
try:
|
|
131
|
+
err_body = response.json()
|
|
132
|
+
err_msg = err_body.get("error") or err_body.get("message") or response.text
|
|
133
|
+
except ValueError:
|
|
134
|
+
err_body = {}
|
|
135
|
+
err_msg = response.text or f"HTTP {response.status_code}"
|
|
136
|
+
|
|
137
|
+
if response.status_code == 401:
|
|
138
|
+
raise AuthenticationError(
|
|
139
|
+
f"Authentication failed: {err_msg}. Verify your 'bs_...' API key at https://app.boltscraper.com",
|
|
140
|
+
status_code=401,
|
|
141
|
+
response_body=err_body
|
|
142
|
+
)
|
|
143
|
+
elif response.status_code == 403:
|
|
144
|
+
raise QuotaExceededError(
|
|
145
|
+
f"Quota exceeded: {err_msg}. Upgrade your plan at https://boltscraper.com/pricing",
|
|
146
|
+
status_code=403,
|
|
147
|
+
response_body=err_body
|
|
148
|
+
)
|
|
149
|
+
elif response.status_code == 404:
|
|
150
|
+
raise NotFoundError(
|
|
151
|
+
f"Resource not found: {err_msg}",
|
|
152
|
+
status_code=404,
|
|
153
|
+
response_body=err_body
|
|
154
|
+
)
|
|
155
|
+
elif response.status_code in (502, 503, 504):
|
|
156
|
+
raise UpstreamError(
|
|
157
|
+
f"Target upstream error (0 credits billed): {err_msg}",
|
|
158
|
+
status_code=response.status_code,
|
|
159
|
+
response_body=err_body
|
|
160
|
+
)
|
|
161
|
+
else:
|
|
162
|
+
raise BoltScraperError(
|
|
163
|
+
f"Bolt Scraper API error ({response.status_code}): {err_msg}",
|
|
164
|
+
status_code=response.status_code,
|
|
165
|
+
response_body=err_body
|
|
166
|
+
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bolt Scraper Python SDK - Exception Classes
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
class BoltScraperError(Exception):
|
|
6
|
+
"""Base exception for all Bolt Scraper SDK errors."""
|
|
7
|
+
def __init__(self, message: str, status_code: int = None, response_body: dict = None):
|
|
8
|
+
super().__init__(message)
|
|
9
|
+
self.message = message
|
|
10
|
+
self.status_code = status_code
|
|
11
|
+
self.response_body = response_body or {}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class AuthenticationError(BoltScraperError):
|
|
15
|
+
"""Raised on 401 Unauthorized - invalid or missing API key."""
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class QuotaExceededError(BoltScraperError):
|
|
20
|
+
"""Raised on 403 Forbidden - monthly lead or crawl credits exhausted."""
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class NotFoundError(BoltScraperError):
|
|
25
|
+
"""Raised on 404 Not Found - resource or job ID not found."""
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class RateLimitError(BoltScraperError):
|
|
30
|
+
"""Raised on 429 Too Many Requests - hit 60 req/min limit."""
|
|
31
|
+
def __init__(self, message: str, retry_after: int = 1, status_code: int = 429, response_body: dict = None):
|
|
32
|
+
super().__init__(message, status_code, response_body)
|
|
33
|
+
self.retry_after = retry_after
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class UpstreamError(BoltScraperError):
|
|
37
|
+
"""Raised on 502/504 Bad Gateway - target domain unreachable or upstream failure."""
|
|
38
|
+
pass
|
boltscraper/maps.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Google Maps Scraper Resource
|
|
3
|
+
Handles real-time search, asynchronous scraping jobs, and lead exports.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import List, Union, Optional, Dict, Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MapsResource:
|
|
10
|
+
def __init__(self, client):
|
|
11
|
+
self._client = client
|
|
12
|
+
|
|
13
|
+
def search(
|
|
14
|
+
self,
|
|
15
|
+
keyword: str,
|
|
16
|
+
page: int = 1,
|
|
17
|
+
hl: str = "en"
|
|
18
|
+
) -> Dict[str, Any]:
|
|
19
|
+
"""
|
|
20
|
+
Synchronous real-time Google Maps search (~2-4s response).
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
keyword: Search query (e.g. 'Coffee shops in Seattle' or 'Dentists Paris')
|
|
24
|
+
page: Page number (1 to 15, yielding up to 20 results per page)
|
|
25
|
+
hl: Two-letter interface language code (e.g. 'en', 'es', 'de', 'fr')
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Dictionary containing pagination metadata and extracted leads with 60+ attributes.
|
|
29
|
+
"""
|
|
30
|
+
payload = {
|
|
31
|
+
"keyword": keyword,
|
|
32
|
+
"page": page,
|
|
33
|
+
"hl": hl
|
|
34
|
+
}
|
|
35
|
+
return self._client._request("POST", "/maps/search", json=payload)
|
|
36
|
+
|
|
37
|
+
def create_job(
|
|
38
|
+
self,
|
|
39
|
+
keywords: Union[str, List[str]],
|
|
40
|
+
title: Optional[str] = None,
|
|
41
|
+
scrape_websites: bool = True,
|
|
42
|
+
ultra_mode: bool = True,
|
|
43
|
+
max_results: Optional[int] = None,
|
|
44
|
+
webhook_url: Optional[str] = None
|
|
45
|
+
) -> Dict[str, Any]:
|
|
46
|
+
"""
|
|
47
|
+
Create an asynchronous Google Maps background scraping job.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
keywords: Single search string or list of query strings
|
|
51
|
+
title: Custom label for the scraping task
|
|
52
|
+
scrape_websites: Automatically crawl discovered websites for emails and social links
|
|
53
|
+
ultra_mode: Enable multi-point grid scanning for up to 3x more listings
|
|
54
|
+
max_results: Optional cap on listings per query
|
|
55
|
+
webhook_url: Optional webhook URL to notify when completed
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
Dictionary containing job_id and initial job status.
|
|
59
|
+
"""
|
|
60
|
+
if isinstance(keywords, str):
|
|
61
|
+
keywords = [keywords]
|
|
62
|
+
|
|
63
|
+
payload = {
|
|
64
|
+
"keywords": keywords,
|
|
65
|
+
"scrape_websites": scrape_websites,
|
|
66
|
+
"ultra_mode": ultra_mode
|
|
67
|
+
}
|
|
68
|
+
if title:
|
|
69
|
+
payload["title"] = title
|
|
70
|
+
if max_results:
|
|
71
|
+
payload["max_results"] = max_results
|
|
72
|
+
if webhook_url:
|
|
73
|
+
payload["webhook_url"] = webhook_url
|
|
74
|
+
|
|
75
|
+
return self._client._request("POST", "/maps/jobs", json=payload)
|
|
76
|
+
|
|
77
|
+
def list_jobs(self, page: int = 1, limit: int = 20) -> Dict[str, Any]:
|
|
78
|
+
"""
|
|
79
|
+
List your recent Google Maps scraping jobs with status and progress.
|
|
80
|
+
"""
|
|
81
|
+
params = {"page": page, "limit": limit}
|
|
82
|
+
return self._client._request("GET", "/maps/jobs", params=params)
|
|
83
|
+
|
|
84
|
+
def get_job(self, job_id: str) -> Dict[str, Any]:
|
|
85
|
+
"""
|
|
86
|
+
Retrieve live progress and metadata for a specific scraping job.
|
|
87
|
+
"""
|
|
88
|
+
return self._client._request("GET", f"/maps/jobs/{job_id}")
|
|
89
|
+
|
|
90
|
+
def download_csv(self, job_id: str) -> bytes:
|
|
91
|
+
"""
|
|
92
|
+
Download the full CSV dataset for a completed or paused job.
|
|
93
|
+
"""
|
|
94
|
+
return self._client._request("GET", f"/maps/jobs/{job_id}/download", raw_bytes=True)
|
|
95
|
+
|
|
96
|
+
def pause_job(self, job_id: str) -> Dict[str, Any]:
|
|
97
|
+
"""Pause an active Google Maps scraping job."""
|
|
98
|
+
return self._client._request("POST", f"/maps/jobs/{job_id}/pause")
|
|
99
|
+
|
|
100
|
+
def resume_job(self, job_id: str) -> Dict[str, Any]:
|
|
101
|
+
"""Resume a paused Google Maps scraping job."""
|
|
102
|
+
return self._client._request("POST", f"/maps/jobs/{job_id}/resume")
|
|
103
|
+
|
|
104
|
+
def stop_job(self, job_id: str) -> Dict[str, Any]:
|
|
105
|
+
"""Stop/cancel a scraping job and finalize partial results."""
|
|
106
|
+
return self._client._request("POST", f"/maps/jobs/{job_id}/control")
|
boltscraper/reviews.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Google Reviews Scraper Resource
|
|
3
|
+
Extracts all customer reviews, ratings, reviewer metadata, and owner replies without OAuth.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import List, Union, Optional, Dict, Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ReviewsResource:
|
|
10
|
+
def __init__(self, client):
|
|
11
|
+
self._client = client
|
|
12
|
+
|
|
13
|
+
def search(
|
|
14
|
+
self,
|
|
15
|
+
place_id: str,
|
|
16
|
+
max_reviews: int = 500,
|
|
17
|
+
sort: str = "newest"
|
|
18
|
+
) -> Dict[str, Any]:
|
|
19
|
+
"""
|
|
20
|
+
Synchronously extract customer reviews for a Google Place ID without OAuth.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
place_id: Google Maps Place ID or listing URL
|
|
24
|
+
max_reviews: Maximum reviews to extract (up to 5,000+)
|
|
25
|
+
sort: Sort order ('newest', 'most_relevant', 'highest_rating', 'lowest_rating')
|
|
26
|
+
"""
|
|
27
|
+
payload = {
|
|
28
|
+
"place_id": place_id,
|
|
29
|
+
"max_reviews": max_reviews,
|
|
30
|
+
"sort": sort
|
|
31
|
+
}
|
|
32
|
+
return self._client._request("POST", "/reviews/search", json=payload)
|
|
33
|
+
|
|
34
|
+
def create_job(
|
|
35
|
+
self,
|
|
36
|
+
place_ids: Union[str, List[str]],
|
|
37
|
+
max_reviews_per_place: int = 500,
|
|
38
|
+
sort: str = "newest",
|
|
39
|
+
webhook_url: Optional[str] = None
|
|
40
|
+
) -> Dict[str, Any]:
|
|
41
|
+
"""
|
|
42
|
+
Create an asynchronous batch reviews extraction job.
|
|
43
|
+
"""
|
|
44
|
+
if isinstance(place_ids, str):
|
|
45
|
+
place_ids = [place_ids]
|
|
46
|
+
|
|
47
|
+
payload = {
|
|
48
|
+
"place_ids": place_ids,
|
|
49
|
+
"max_reviews_per_place": max_reviews_per_place,
|
|
50
|
+
"sort": sort
|
|
51
|
+
}
|
|
52
|
+
if webhook_url:
|
|
53
|
+
payload["webhook_url"] = webhook_url
|
|
54
|
+
|
|
55
|
+
return self._client._request("POST", "/reviews/jobs", json=payload)
|
|
56
|
+
|
|
57
|
+
def list_jobs(self, page: int = 1, limit: int = 20) -> Dict[str, Any]:
|
|
58
|
+
"""List recent reviews scraping jobs."""
|
|
59
|
+
params = {"page": page, "limit": limit}
|
|
60
|
+
return self._client._request("GET", "/reviews/jobs", params=params)
|
|
61
|
+
|
|
62
|
+
def get_job(self, job_id: str) -> Dict[str, Any]:
|
|
63
|
+
"""Retrieve progress and metadata for a reviews extraction job."""
|
|
64
|
+
return self._client._request("GET", f"/reviews/jobs/{job_id}")
|
|
65
|
+
|
|
66
|
+
def download_csv(self, job_id: str) -> bytes:
|
|
67
|
+
"""Download reviews CSV export."""
|
|
68
|
+
return self._client._request("GET", f"/reviews/jobs/{job_id}/download", raw_bytes=True)
|
|
69
|
+
|
|
70
|
+
def pause_job(self, job_id: str) -> Dict[str, Any]:
|
|
71
|
+
"""Pause a running reviews scraping job."""
|
|
72
|
+
return self._client._request("POST", f"/reviews/jobs/{job_id}/pause")
|
|
73
|
+
|
|
74
|
+
def resume_job(self, job_id: str) -> Dict[str, Any]:
|
|
75
|
+
"""Resume a paused reviews scraping job."""
|
|
76
|
+
return self._client._request("POST", f"/reviews/jobs/{job_id}/resume")
|
|
77
|
+
|
|
78
|
+
def stop_job(self, job_id: str) -> Dict[str, Any]:
|
|
79
|
+
"""Stop a reviews scraping job."""
|
|
80
|
+
return self._client._request("POST", f"/reviews/jobs/{job_id}/control")
|
boltscraper/webhooks.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Webhooks Resource & Security Verification Helpers
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import hmac
|
|
6
|
+
import hashlib
|
|
7
|
+
from typing import Optional, Dict, Any, Union
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class WebhooksResource:
|
|
11
|
+
def __init__(self, client):
|
|
12
|
+
self._client = client
|
|
13
|
+
|
|
14
|
+
def list(self) -> Dict[str, Any]:
|
|
15
|
+
"""List all registered webhook endpoints."""
|
|
16
|
+
return self._client._request("GET", "/webhooks")
|
|
17
|
+
|
|
18
|
+
def create(
|
|
19
|
+
self,
|
|
20
|
+
name: str,
|
|
21
|
+
webhook_url: str,
|
|
22
|
+
secret: Optional[str] = None,
|
|
23
|
+
platform: str = "all",
|
|
24
|
+
send_auto: bool = True
|
|
25
|
+
) -> Dict[str, Any]:
|
|
26
|
+
"""
|
|
27
|
+
Register a new webhook endpoint.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
name: Friendly label (e.g. 'Production CRM Pipeline')
|
|
31
|
+
webhook_url: Target destination URL
|
|
32
|
+
secret: Optional HMAC secret for payload signature verification
|
|
33
|
+
platform: 'all', 'maps', or 'website'
|
|
34
|
+
send_auto: Automatically dispatch on job completion
|
|
35
|
+
"""
|
|
36
|
+
payload = {
|
|
37
|
+
"name": name,
|
|
38
|
+
"webhook_url": webhook_url,
|
|
39
|
+
"platform": platform,
|
|
40
|
+
"send_auto": send_auto
|
|
41
|
+
}
|
|
42
|
+
if secret:
|
|
43
|
+
payload["secret"] = secret
|
|
44
|
+
return self._client._request("POST", "/webhooks", json=payload)
|
|
45
|
+
|
|
46
|
+
def update(self, webhook_id: str, **kwargs) -> Dict[str, Any]:
|
|
47
|
+
"""Update an existing webhook configuration."""
|
|
48
|
+
return self._client._request("PUT", f"/webhooks/{webhook_id}", json=kwargs)
|
|
49
|
+
|
|
50
|
+
def delete(self, webhook_id: str) -> Dict[str, Any]:
|
|
51
|
+
"""Remove a webhook endpoint."""
|
|
52
|
+
return self._client._request("DELETE", f"/webhooks/{webhook_id}")
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
def verify_signature(
|
|
56
|
+
raw_body: Union[str, bytes],
|
|
57
|
+
signature_header: str,
|
|
58
|
+
secret: str
|
|
59
|
+
) -> bool:
|
|
60
|
+
"""
|
|
61
|
+
Verify incoming webhook HMAC SHA-256 signature from the X-Bolt-Signature header.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
raw_body: Exact raw request body as bytes or string
|
|
65
|
+
signature_header: Value of the 'X-Bolt-Signature' HTTP header (e.g. 'sha256=abcdef...')
|
|
66
|
+
secret: Your configured webhook secret key
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
True if the signature is valid, False otherwise.
|
|
70
|
+
"""
|
|
71
|
+
if not signature_header or not signature_header.startswith("sha256="):
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
if isinstance(raw_body, str):
|
|
75
|
+
raw_body = raw_body.encode("utf-8")
|
|
76
|
+
|
|
77
|
+
expected_sig = signature_header.split("sha256=")[1]
|
|
78
|
+
computed_sig = hmac.new(secret.encode("utf-8"), raw_body, hashlib.sha256).hexdigest()
|
|
79
|
+
return hmac.compare_digest(expected_sig, computed_sig)
|
boltscraper/website.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Website Scraper Resource
|
|
3
|
+
Extracts direct business emails, phone numbers, and social links from company domains.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import List, Union, Optional, Dict, Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class WebsiteResource:
|
|
10
|
+
def __init__(self, client):
|
|
11
|
+
self._client = client
|
|
12
|
+
|
|
13
|
+
def scrape(self, url: str) -> Dict[str, Any]:
|
|
14
|
+
"""
|
|
15
|
+
Synchronously scrape a single website domain for emails, phone numbers, and social media URLs.
|
|
16
|
+
Automatically crawls subpages (/contact, /about, /team, etc.).
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
url: Company website URL (e.g. 'https://stripe.com')
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
Dictionary containing extracted emails, phones, and social profile links.
|
|
23
|
+
"""
|
|
24
|
+
payload = {"url": url}
|
|
25
|
+
return self._client._request("POST", "/website/scrape", json=payload)
|
|
26
|
+
|
|
27
|
+
def create_job(
|
|
28
|
+
self,
|
|
29
|
+
urls: Union[str, List[str]],
|
|
30
|
+
name: Optional[str] = None,
|
|
31
|
+
webhook_url: Optional[str] = None
|
|
32
|
+
) -> Dict[str, Any]:
|
|
33
|
+
"""
|
|
34
|
+
Create an asynchronous batch website extraction task for up to 10,000 URLs.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
urls: Single domain string or list of website URLs
|
|
38
|
+
name: Optional descriptive label for the task
|
|
39
|
+
webhook_url: Optional callback URL when task completes
|
|
40
|
+
"""
|
|
41
|
+
if isinstance(urls, str):
|
|
42
|
+
urls = [urls]
|
|
43
|
+
|
|
44
|
+
payload = {"urls": urls}
|
|
45
|
+
if name:
|
|
46
|
+
payload["name"] = name
|
|
47
|
+
if webhook_url:
|
|
48
|
+
payload["webhook_url"] = webhook_url
|
|
49
|
+
|
|
50
|
+
return self._client._request("POST", "/website/jobs", json=payload)
|
|
51
|
+
|
|
52
|
+
def list_jobs(self, page: int = 1, limit: int = 20) -> Dict[str, Any]:
|
|
53
|
+
"""List recent website contact enrichment jobs."""
|
|
54
|
+
params = {"page": page, "limit": limit}
|
|
55
|
+
return self._client._request("GET", "/website/jobs", params=params)
|
|
56
|
+
|
|
57
|
+
def get_job(self, job_id: str) -> Dict[str, Any]:
|
|
58
|
+
"""Retrieve progress and metadata for a website scraping job."""
|
|
59
|
+
return self._client._request("GET", f"/website/jobs/{job_id}")
|
|
60
|
+
|
|
61
|
+
def download_csv(self, job_id: str) -> bytes:
|
|
62
|
+
"""Download full website contact enrichment CSV dataset."""
|
|
63
|
+
return self._client._request("GET", f"/website/jobs/{job_id}/download", raw_bytes=True)
|
|
64
|
+
|
|
65
|
+
def stop_job(self, job_id: str) -> Dict[str, Any]:
|
|
66
|
+
"""Stop/cancel a website scraping job."""
|
|
67
|
+
return self._client._request("POST", f"/website/jobs/{job_id}/control")
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: boltscraper
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Official Python SDK for Bolt Scraper — Extract business leads, reviews, and direct emails from Google Maps and websites without bans.
|
|
5
|
+
Author-email: Bolt Scraper Team <support@boltscraper.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://boltscraper.com
|
|
8
|
+
Project-URL: Documentation, https://boltscraper.com/api/
|
|
9
|
+
Project-URL: Repository, https://github.com/boltscraper/google-maps-scraper-python
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/boltscraper/google-maps-scraper-python/issues
|
|
11
|
+
Project-URL: Free API Key, https://app.boltscraper.com
|
|
12
|
+
Keywords: google-maps-scraper,google-maps-api,lead-generation,python-scraper,google-places-api,email-extractor,google-reviews-scraper,local-seo,b2b-leads,web-scraping
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Requires-Python: >=3.8
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: requests>=2.25.0
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: requires-python
|
|
30
|
+
|
|
31
|
+
# Google Maps Scraper Python API (`boltscraper`)
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/boltscraper/)
|
|
34
|
+
[](https://pypi.org/project/boltscraper/)
|
|
35
|
+
[](https://opensource.org/licenses/MIT)
|
|
36
|
+
[](https://app.boltscraper.com)
|
|
37
|
+
|
|
38
|
+
The official Python SDK for **[Bolt Scraper](https://boltscraper.com)** — fast, proxy-free, cloud-native data extraction for **Google Maps listings**, **Google customer reviews**, and **company websites**.
|
|
39
|
+
|
|
40
|
+
Extract **60+ business attributes** (names, phones, direct emails, social profiles, hours, ratings, coordinates) without browser automation overhead, CAPTCHA hassles, or IP bans.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Why Developers Use `boltscraper`
|
|
45
|
+
|
|
46
|
+
| Feature | Selenium / BeautifulSoup | Google Places API (New) | Bolt Scraper Python SDK |
|
|
47
|
+
| :--- | :--- | :--- | :--- |
|
|
48
|
+
| **Speed & Setup** | Slow browser overhead | Complex GCP console setup | **5 lines of Python code** |
|
|
49
|
+
| **IP Bans & CAPTCHAs** | Frequent blocks & CAPTCHAs | None (metered API) | **0% bans** (automatic proxy rotation) |
|
|
50
|
+
| **Direct Emails** | Requires custom scraping | **Not offered (0% returned)** | **Included** (crawls company websites) |
|
|
51
|
+
| **Customer Reviews** | DOM fragile / breaks | Max 5 reviews only | **Extracts all 5,000+ reviews** |
|
|
52
|
+
| **Data Storage** | Unstructured HTML | 30-day cache ban | **100% owned indefinitely** |
|
|
53
|
+
| **Pricing** | Proxy costs ($50+/mo) | **$32 – $65+ per 1k requests** | **$0 Free tier** ($0.38 / 1k on paid) |
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Installation
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install boltscraper
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Requires Python 3.8+.
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Quickstart
|
|
68
|
+
|
|
69
|
+
Get your free API key at **[app.boltscraper.com](https://app.boltscraper.com)** (includes 1,000 free lead credits every month).
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from boltscraper import BoltScraper
|
|
73
|
+
|
|
74
|
+
# Initialize client (or set BOLTSCRAPER_API_KEY environment variable)
|
|
75
|
+
client = BoltScraper(api_key="bs_live_YOUR_API_KEY")
|
|
76
|
+
|
|
77
|
+
# 1. Real-time Google Maps search (~2-4 seconds)
|
|
78
|
+
results = client.maps.search("Coffee shops in Seattle", page=1)
|
|
79
|
+
|
|
80
|
+
print(f"Found {results['pagination']['total_results']} places:")
|
|
81
|
+
for place in results["data"]:
|
|
82
|
+
print(f"Name: {place['Name']}")
|
|
83
|
+
print(f"Phone: {place['Phone']} ({place['Phone Type']})")
|
|
84
|
+
print(f"Website: {place['Website']}")
|
|
85
|
+
print(f"Rating: {place['Average Rating']} ({place['Review Count']} reviews)")
|
|
86
|
+
print("-" * 40)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## Core Capabilities & Examples
|
|
92
|
+
|
|
93
|
+
### 1. Synchronous Real-time Maps Search
|
|
94
|
+
Perfect for search bars, AI agent tool calling, and live user queries. Returns instant JSON without background polling:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
response = client.maps.search(
|
|
98
|
+
keyword="Dentists in Austin, TX",
|
|
99
|
+
page=1,
|
|
100
|
+
hl="en"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
for lead in response["data"]:
|
|
104
|
+
print(lead["Name"], lead["Full Address"], lead["Coordinates"])
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### 2. Large-Scale Asynchronous Batch Jobs
|
|
108
|
+
Scrape entire cities or hundreds of queries with automatic website email enrichment and Ultra Mode:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
# Launch background scraping job
|
|
112
|
+
job = client.maps.create_job(
|
|
113
|
+
keywords=["Roofers in Miami, FL", "Solar contractors in Miami, FL"],
|
|
114
|
+
title="Miami Home Services",
|
|
115
|
+
scrape_websites=True, # Crawls company websites for direct emails & socials
|
|
116
|
+
ultra_mode=True, # Multi-point grid scan for up to 3x more results
|
|
117
|
+
webhook_url="https://your-crm.com/api/webhooks/bolt"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
job_id = job["job_id"]
|
|
121
|
+
print(f"Started job: {job_id}")
|
|
122
|
+
|
|
123
|
+
# Check status
|
|
124
|
+
status = client.maps.get_job(job_id)
|
|
125
|
+
print(f"Status: {status['status']} — Extracted: {status['total_leads']} leads")
|
|
126
|
+
|
|
127
|
+
# Download CSV dataset upon completion
|
|
128
|
+
csv_bytes = client.maps.download_csv(job_id)
|
|
129
|
+
with open("miami_leads.csv", "wb") as f:
|
|
130
|
+
f.write(csv_bytes)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### 3. Google Reviews Extraction (Without OAuth)
|
|
134
|
+
Bypass Google's 5-review API limitation and extract all customer reviews, ratings, reviewer levels, and owner responses:
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
# Synchronously extract up to 5,000+ reviews for any Google Place ID
|
|
138
|
+
reviews_data = client.reviews.search(
|
|
139
|
+
place_id="ChIJb8Jg9q-1RIYRAxvq_mZ0gXY",
|
|
140
|
+
max_reviews=500,
|
|
141
|
+
sort="newest" # 'newest', 'most_relevant', 'highest_rating', 'lowest_rating'
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
for review in reviews_data.get("reviews", []):
|
|
145
|
+
print(f"★ {review['rating']} by {review['reviewer_name']}")
|
|
146
|
+
print(f"Text: {review['text']}")
|
|
147
|
+
print(f"Date: {review['date']}")
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### 4. Website Email & Social Link Enrichment
|
|
151
|
+
Turn raw domain lists into enriched B2B contact lists:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
# Instant single-domain enrichment (crawls /contact, /about, /team subpages)
|
|
155
|
+
company = client.website.scrape("https://example.com")
|
|
156
|
+
|
|
157
|
+
print("Direct Emails:", company.get("emails", []))
|
|
158
|
+
print("Phone Numbers:", company.get("phones", []))
|
|
159
|
+
print("Social Links: ", company.get("socials", {}))
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### 5. Webhook Signature Verification
|
|
163
|
+
Verify incoming webhook payloads with HMAC SHA-256 to ensure authenticity:
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
from boltscraper import WebhooksResource
|
|
167
|
+
|
|
168
|
+
# In your FastAPI / Flask / Django webhook handler:
|
|
169
|
+
is_valid = WebhooksResource.verify_signature(
|
|
170
|
+
raw_body=request.body,
|
|
171
|
+
signature_header=request.headers.get("X-Bolt-Signature"),
|
|
172
|
+
secret="YOUR_WEBHOOK_SECRET"
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
if not is_valid:
|
|
176
|
+
return "Invalid signature", 401
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
---
|
|
180
|
+
|
|
181
|
+
## 60+ Extracted Data Points
|
|
182
|
+
|
|
183
|
+
Each Google Maps listing returns comprehensive, production-ready structured data:
|
|
184
|
+
|
|
185
|
+
* **Identity**: Business Name, Category, Sub-categories, Google Category IDs, Place ID, CID, KGMID, Feature ID.
|
|
186
|
+
* **Location**: Full Address, Street Address, City, State, ZIP/Postal Code, Country, Municipality, Latitude, Longitude, Timezone.
|
|
187
|
+
* **Contact**: Formatted Phone, Phone Type (Mobile, Landline, VOIP), International Phone, Phone URI (`tel:`), Website URL, Domain.
|
|
188
|
+
* **Enrichment**: Direct Emails discovered from company websites, Social Media URLs (LinkedIn, Facebook, Instagram, Twitter/X, YouTube).
|
|
189
|
+
* **Reputation & Hours**: Star Rating, Total Reviews Count, Review URL, Operating Status (Open, Closed), Opening Hours (Monday–Sunday schedules).
|
|
190
|
+
* **Media Assets**: Logo URL, Main Photo URL, Photos Page URL, Favicon URL.
|
|
191
|
+
|
|
192
|
+
---
|
|
193
|
+
|
|
194
|
+
## Rate Limits & Error Handling
|
|
195
|
+
|
|
196
|
+
All requests are rate-limited to **60 requests per minute** per API key. The SDK automatically handles `429 Too Many Requests` responses with exponential backoff using the server's `Retry-After` header.
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
from boltscraper.exceptions import (
|
|
200
|
+
AuthenticationError,
|
|
201
|
+
QuotaExceededError,
|
|
202
|
+
RateLimitError,
|
|
203
|
+
BoltScraperError
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
try:
|
|
207
|
+
results = client.maps.search("Plumbers Denver")
|
|
208
|
+
except AuthenticationError:
|
|
209
|
+
print("Invalid API key. Check https://app.boltscraper.com")
|
|
210
|
+
except QuotaExceededError:
|
|
211
|
+
print("Monthly quota exhausted. Upgrade plan at https://boltscraper.com/pricing")
|
|
212
|
+
except RateLimitError as e:
|
|
213
|
+
print(f"Rate limited. Reset in {e.retry_after} seconds.")
|
|
214
|
+
except BoltScraperError as e:
|
|
215
|
+
print(f"API Error ({e.status_code}): {e.message}")
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## Documentation & Resources
|
|
221
|
+
|
|
222
|
+
* **Documentation Portal**: [https://boltscraper.com/api/](https://boltscraper.com/api/)
|
|
223
|
+
* **In-Depth Guides & Tutorials**: [https://boltscraper.com/blogs/](https://boltscraper.com/blogs/)
|
|
224
|
+
* **Pricing Calculator**: [Google Places API Cost Calculator](https://boltscraper.com/tools/google-places-api-pricing-calculator/)
|
|
225
|
+
* **Sign Up (1,000 Free Leads/mo)**: [https://app.boltscraper.com](https://app.boltscraper.com)
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## License
|
|
230
|
+
|
|
231
|
+
MIT License. Copyright (c) 2026 Bolt Scraper.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
boltscraper/__init__.py,sha256=C4oA2VKRHn-A_HHaXEr6cNgwmtLPM2uXsVrK8grqzb4,585
|
|
2
|
+
boltscraper/client.py,sha256=TdgG5PIXLsVDbxksrtDOVH3jK_GImgkmH_af2uXqNV0,5787
|
|
3
|
+
boltscraper/exceptions.py,sha256=reSVw8uSgQRWRu7yo-3aVf3RobCjd7zmD2yQNS1Xr2M,1221
|
|
4
|
+
boltscraper/maps.py,sha256=XfY8sKql5xlXJ75k2IiXRv0ijfQJ9BWvfRMNCNvaX1c,3809
|
|
5
|
+
boltscraper/reviews.py,sha256=zr0f_chHMQjM1ZrHSzEDlvzOk0iad5mx_f4X1znmyDs,2840
|
|
6
|
+
boltscraper/webhooks.py,sha256=4P-xn2iOMX0GVKVchZoTdN7hUj_7A-bV8Too1u4b7lQ,2641
|
|
7
|
+
boltscraper/website.py,sha256=74BlOQn81qMOfWI4wRRE3gVzC7fcDnQi_TQRvDzOXTs,2471
|
|
8
|
+
boltscraper-1.0.0.dist-info/licenses/LICENSE,sha256=m0WjlGcPKwD5Afv6_6vbFpie56LOufldgXLS5QS12bM,1095
|
|
9
|
+
boltscraper-1.0.0.dist-info/METADATA,sha256=nY9IYZqKBBkNsofqZJ3phwwFuVKR2TfNt374S7hExoA,9068
|
|
10
|
+
boltscraper-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
11
|
+
boltscraper-1.0.0.dist-info/top_level.txt,sha256=d8DeYv2GoUSYzjtZCD_SduSZo2EOriZhgUmDgv51Djo,12
|
|
12
|
+
boltscraper-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bolt Scraper (https://boltscraper.com)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
boltscraper
|