scrapemapsapi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapemapsapi/__init__.py +95 -0
- scrapemapsapi/_async_client.py +90 -0
- scrapemapsapi/_base.py +141 -0
- scrapemapsapi/_client.py +91 -0
- scrapemapsapi/errors.py +197 -0
- scrapemapsapi/models.py +489 -0
- scrapemapsapi/places.py +309 -0
- scrapemapsapi/py.typed +0 -0
- scrapemapsapi-0.1.0.dist-info/METADATA +209 -0
- scrapemapsapi-0.1.0.dist-info/RECORD +12 -0
- scrapemapsapi-0.1.0.dist-info/WHEEL +4 -0
- scrapemapsapi-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""ScrapeMapsAPI: official Python SDK for the ScrapeMapsAPI API (Google Maps places and reviews)."""
|
|
2
|
+
|
|
3
|
+
from ._async_client import AsyncScrapeMapsAPI
|
|
4
|
+
from ._base import __version__
|
|
5
|
+
from ._client import ScrapeMapsAPI
|
|
6
|
+
from .errors import (
|
|
7
|
+
APIConnectionError,
|
|
8
|
+
APITimeoutError,
|
|
9
|
+
AuthenticationError,
|
|
10
|
+
CreditsCooldownError,
|
|
11
|
+
InsufficientCreditsError,
|
|
12
|
+
InternalServerError,
|
|
13
|
+
InvalidRequestError,
|
|
14
|
+
NotFoundError,
|
|
15
|
+
PlaceNotFoundError,
|
|
16
|
+
RateLimitError,
|
|
17
|
+
ScrapeMapsAPIError,
|
|
18
|
+
ServiceUnavailableError,
|
|
19
|
+
UpstreamError,
|
|
20
|
+
)
|
|
21
|
+
from .models import (
|
|
22
|
+
Address,
|
|
23
|
+
DayHours,
|
|
24
|
+
Link,
|
|
25
|
+
Location,
|
|
26
|
+
Meta,
|
|
27
|
+
OpeningHours,
|
|
28
|
+
OwnerResponse,
|
|
29
|
+
Pagination,
|
|
30
|
+
Photo,
|
|
31
|
+
Place,
|
|
32
|
+
PlaceIds,
|
|
33
|
+
PlaceMeta,
|
|
34
|
+
PlaceResponse,
|
|
35
|
+
PlaceStatus,
|
|
36
|
+
PlaceSummary,
|
|
37
|
+
ResolvedIds,
|
|
38
|
+
ResolveResponse,
|
|
39
|
+
ResponseInfo,
|
|
40
|
+
Review,
|
|
41
|
+
ReviewAuthor,
|
|
42
|
+
ReviewsMeta,
|
|
43
|
+
ReviewSort,
|
|
44
|
+
ReviewsResponse,
|
|
45
|
+
SearchMeta,
|
|
46
|
+
SearchResponse,
|
|
47
|
+
TimePeriod,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"__version__",
|
|
52
|
+
"ScrapeMapsAPI",
|
|
53
|
+
"AsyncScrapeMapsAPI",
|
|
54
|
+
# errors
|
|
55
|
+
"ScrapeMapsAPIError",
|
|
56
|
+
"APIConnectionError",
|
|
57
|
+
"APITimeoutError",
|
|
58
|
+
"AuthenticationError",
|
|
59
|
+
"CreditsCooldownError",
|
|
60
|
+
"InsufficientCreditsError",
|
|
61
|
+
"InternalServerError",
|
|
62
|
+
"InvalidRequestError",
|
|
63
|
+
"NotFoundError",
|
|
64
|
+
"PlaceNotFoundError",
|
|
65
|
+
"RateLimitError",
|
|
66
|
+
"ServiceUnavailableError",
|
|
67
|
+
"UpstreamError",
|
|
68
|
+
# models
|
|
69
|
+
"Address",
|
|
70
|
+
"DayHours",
|
|
71
|
+
"Link",
|
|
72
|
+
"Location",
|
|
73
|
+
"Meta",
|
|
74
|
+
"OpeningHours",
|
|
75
|
+
"OwnerResponse",
|
|
76
|
+
"Pagination",
|
|
77
|
+
"Photo",
|
|
78
|
+
"Place",
|
|
79
|
+
"PlaceIds",
|
|
80
|
+
"PlaceMeta",
|
|
81
|
+
"PlaceResponse",
|
|
82
|
+
"PlaceStatus",
|
|
83
|
+
"PlaceSummary",
|
|
84
|
+
"ResolvedIds",
|
|
85
|
+
"ResolveResponse",
|
|
86
|
+
"ResponseInfo",
|
|
87
|
+
"Review",
|
|
88
|
+
"ReviewAuthor",
|
|
89
|
+
"ReviewSort",
|
|
90
|
+
"ReviewsMeta",
|
|
91
|
+
"ReviewsResponse",
|
|
92
|
+
"SearchMeta",
|
|
93
|
+
"SearchResponse",
|
|
94
|
+
"TimePeriod",
|
|
95
|
+
]
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Asynchronous ScrapeMapsAPI client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from typing import Any, Mapping, Optional
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
from ._base import (
|
|
11
|
+
DEFAULT_MAX_RETRIES,
|
|
12
|
+
DEFAULT_MAX_RETRY_WAIT,
|
|
13
|
+
DEFAULT_RETRY_BACKOFF,
|
|
14
|
+
DEFAULT_TIMEOUT,
|
|
15
|
+
BaseClient,
|
|
16
|
+
clean_params,
|
|
17
|
+
)
|
|
18
|
+
from .places import AsyncPlaces
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AsyncScrapeMapsAPI(BaseClient):
|
|
22
|
+
"""Asynchronous client for the ScrapeMapsAPI API (Google Maps places and reviews).
|
|
23
|
+
|
|
24
|
+
Usage::
|
|
25
|
+
|
|
26
|
+
from scrapemapsapi import AsyncScrapeMapsAPI
|
|
27
|
+
|
|
28
|
+
async with AsyncScrapeMapsAPI() as client: # reads SCRAPEMAPSAPI_API_KEY
|
|
29
|
+
async for review in client.places.iter_reviews(place_id, max_items=50):
|
|
30
|
+
print(review.rating, review.text)
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
api_key: API key; defaults to the ``SCRAPEMAPSAPI_API_KEY`` environment variable.
|
|
34
|
+
base_url: API root; defaults to ``SCRAPEMAPSAPI_BASE_URL`` or https://api.scrapemapsapi.com.
|
|
35
|
+
timeout: Seconds per request (reviews of large places can take a while upstream).
|
|
36
|
+
max_retries: Retries on 429/502/503 and connection errors (0 disables them).
|
|
37
|
+
retry_backoff: Base delay in seconds of the exponential backoff (with jitter).
|
|
38
|
+
max_retry_wait: Longest wait accepted between retries; a ``Retry-After`` longer than
|
|
39
|
+
this raises immediately instead of sleeping.
|
|
40
|
+
http_client: Your own ``httpx.AsyncClient`` (proxies, transports, tests). Not closed by us.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
api_key: Optional[str] = None,
|
|
46
|
+
*,
|
|
47
|
+
base_url: Optional[str] = None,
|
|
48
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
49
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
50
|
+
retry_backoff: float = DEFAULT_RETRY_BACKOFF,
|
|
51
|
+
max_retry_wait: float = DEFAULT_MAX_RETRY_WAIT,
|
|
52
|
+
http_client: Optional[httpx.AsyncClient] = None,
|
|
53
|
+
) -> None:
|
|
54
|
+
super().__init__(api_key, base_url, timeout, max_retries, retry_backoff, max_retry_wait)
|
|
55
|
+
self._owns_http = http_client is None
|
|
56
|
+
self._http = http_client or httpx.AsyncClient(timeout=timeout)
|
|
57
|
+
self.places = AsyncPlaces(self)
|
|
58
|
+
|
|
59
|
+
async def _get(self, path: str, params: Mapping[str, Any]) -> tuple[dict[str, Any], httpx.Headers]:
|
|
60
|
+
attempt = 0
|
|
61
|
+
while True:
|
|
62
|
+
try:
|
|
63
|
+
response = await self._http.get(
|
|
64
|
+
self._url(path), params=clean_params(params), headers=self._headers, timeout=self.timeout
|
|
65
|
+
)
|
|
66
|
+
except httpx.TransportError as exc:
|
|
67
|
+
wait, error = self._transport_retry_wait(exc, attempt)
|
|
68
|
+
if wait is None:
|
|
69
|
+
raise error from exc
|
|
70
|
+
else:
|
|
71
|
+
if response.is_success:
|
|
72
|
+
body = self._json(response)
|
|
73
|
+
self._record(response)
|
|
74
|
+
return body, response.headers
|
|
75
|
+
wait, error = self._retry_wait(response, attempt)
|
|
76
|
+
if wait is None:
|
|
77
|
+
raise error
|
|
78
|
+
attempt += 1
|
|
79
|
+
await asyncio.sleep(wait)
|
|
80
|
+
|
|
81
|
+
async def close(self) -> None:
|
|
82
|
+
"""Close the underlying HTTP client (only if the SDK created it)."""
|
|
83
|
+
if self._owns_http:
|
|
84
|
+
await self._http.aclose()
|
|
85
|
+
|
|
86
|
+
async def __aenter__(self) -> AsyncScrapeMapsAPI:
|
|
87
|
+
return self
|
|
88
|
+
|
|
89
|
+
async def __aexit__(self, *args: Any) -> None:
|
|
90
|
+
await self.close()
|
scrapemapsapi/_base.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Configuration and logic shared by the sync and async clients (no I/O here)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
from typing import Any, Mapping, Optional
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from .errors import (
|
|
13
|
+
APIConnectionError,
|
|
14
|
+
APITimeoutError,
|
|
15
|
+
AuthenticationError,
|
|
16
|
+
ScrapeMapsAPIError,
|
|
17
|
+
error_from_response,
|
|
18
|
+
)
|
|
19
|
+
from .models import ResponseInfo
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
DEFAULT_BASE_URL = "https://api.scrapemapsapi.com"
|
|
24
|
+
DEFAULT_TIMEOUT = 60.0
|
|
25
|
+
DEFAULT_MAX_RETRIES = 2
|
|
26
|
+
DEFAULT_RETRY_BACKOFF = 0.5
|
|
27
|
+
DEFAULT_MAX_RETRY_WAIT = 30.0
|
|
28
|
+
API_KEY_ENV = "SCRAPEMAPSAPI_API_KEY"
|
|
29
|
+
BASE_URL_ENV = "SCRAPEMAPSAPI_BASE_URL"
|
|
30
|
+
USER_AGENT = f"scrapemapsapi-python/{__version__}"
|
|
31
|
+
|
|
32
|
+
SEARCH_MAX_LIMIT = 20
|
|
33
|
+
REVIEWS_MAX_LIMIT = 20
|
|
34
|
+
|
|
35
|
+
RETRY_STATUSES = frozenset({429, 502, 503})
|
|
36
|
+
# A 429 from the no-credit cooldown lasts minutes and only a top-up lifts it: never retried.
|
|
37
|
+
NO_RETRY_CODES = frozenset({"insufficient_credits_retry_later"})
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def place_path(place: str, suffix: str = "") -> str:
|
|
41
|
+
"""``/v1/places/{place}{suffix}`` with ``place`` URL-encoded (feature IDs contain ``:``)."""
|
|
42
|
+
if not isinstance(place, str) or not place.strip():
|
|
43
|
+
raise ValueError("place must be a non-empty place ID (ChIJ...), CID or feature ID")
|
|
44
|
+
return f"/v1/places/{quote(place.strip(), safe='')}{suffix}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def clean_params(params: Mapping[str, Any]) -> dict[str, Any]:
|
|
48
|
+
"""Drop None values (the API applies its own defaults)."""
|
|
49
|
+
return {k: v for k, v in params.items() if v is not None}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def page_limit(page_size: Optional[int], max_items: Optional[int], seen: int, maximum: int) -> int:
|
|
53
|
+
"""Page size for the next auto-pagination request: never fetch more items than needed."""
|
|
54
|
+
size = min(page_size or maximum, maximum)
|
|
55
|
+
if max_items is not None:
|
|
56
|
+
size = min(size, max_items - seen)
|
|
57
|
+
return max(1, size)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class BaseClient:
|
|
61
|
+
"""Holds configuration, headers and the retry policy."""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
api_key: Optional[str],
|
|
66
|
+
base_url: Optional[str],
|
|
67
|
+
timeout: float,
|
|
68
|
+
max_retries: int,
|
|
69
|
+
retry_backoff: float,
|
|
70
|
+
max_retry_wait: float,
|
|
71
|
+
) -> None:
|
|
72
|
+
api_key = api_key or os.environ.get(API_KEY_ENV)
|
|
73
|
+
if not api_key:
|
|
74
|
+
raise AuthenticationError(f"No API key: pass api_key=... or set the {API_KEY_ENV} environment variable.")
|
|
75
|
+
self.api_key = api_key
|
|
76
|
+
self.base_url = (base_url or os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL).rstrip("/")
|
|
77
|
+
self.timeout = timeout
|
|
78
|
+
self.max_retries = max(0, int(max_retries))
|
|
79
|
+
self.retry_backoff = max(0.0, float(retry_backoff))
|
|
80
|
+
self.max_retry_wait = max(0.0, float(max_retry_wait))
|
|
81
|
+
self.last_response_info: Optional[ResponseInfo] = None
|
|
82
|
+
self._headers = {
|
|
83
|
+
"X-API-Key": self.api_key,
|
|
84
|
+
"Accept": "application/json",
|
|
85
|
+
"User-Agent": USER_AGENT,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
def __repr__(self) -> str:
|
|
89
|
+
return f"{type(self).__name__}(base_url={self.base_url!r})"
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def credits_remaining(self) -> Optional[int]:
|
|
93
|
+
"""Credits left, from the last successful response (None before the first call)."""
|
|
94
|
+
return self.last_response_info.credits_remaining if self.last_response_info else None
|
|
95
|
+
|
|
96
|
+
def _url(self, path: str) -> str:
|
|
97
|
+
return f"{self.base_url}{path}"
|
|
98
|
+
|
|
99
|
+
def _backoff(self, attempt: int) -> float:
|
|
100
|
+
base = self.retry_backoff * (2**attempt)
|
|
101
|
+
return min(base, self.max_retry_wait) * (0.5 + random.random() / 2)
|
|
102
|
+
|
|
103
|
+
def _retry_wait(self, response: httpx.Response, attempt: int) -> tuple[Optional[float], ScrapeMapsAPIError]:
|
|
104
|
+
"""For an error response: ``(seconds to wait before retrying or None, the error)``."""
|
|
105
|
+
error = error_from_response(response)
|
|
106
|
+
if attempt >= self.max_retries or response.status_code not in RETRY_STATUSES or error.code in NO_RETRY_CODES:
|
|
107
|
+
return None, error
|
|
108
|
+
if error.retry_after is not None:
|
|
109
|
+
if error.retry_after > self.max_retry_wait:
|
|
110
|
+
return None, error
|
|
111
|
+
return error.retry_after, error
|
|
112
|
+
return self._backoff(attempt), error
|
|
113
|
+
|
|
114
|
+
def _transport_retry_wait(
|
|
115
|
+
self, exc: httpx.TransportError, attempt: int
|
|
116
|
+
) -> tuple[Optional[float], ScrapeMapsAPIError]:
|
|
117
|
+
error: ScrapeMapsAPIError
|
|
118
|
+
if isinstance(exc, httpx.TimeoutException):
|
|
119
|
+
error = APITimeoutError(f"Request timed out after {self.timeout}s: {exc}")
|
|
120
|
+
else:
|
|
121
|
+
error = APIConnectionError(f"Could not reach {self.base_url}: {exc}")
|
|
122
|
+
if attempt >= self.max_retries:
|
|
123
|
+
return None, error
|
|
124
|
+
return self._backoff(attempt), error
|
|
125
|
+
|
|
126
|
+
def _record(self, response: httpx.Response) -> ResponseInfo:
|
|
127
|
+
info = ResponseInfo.from_headers(response.headers)
|
|
128
|
+
self.last_response_info = info
|
|
129
|
+
return info
|
|
130
|
+
|
|
131
|
+
@staticmethod
|
|
132
|
+
def _json(response: httpx.Response) -> dict[str, Any]:
|
|
133
|
+
try:
|
|
134
|
+
body = response.json()
|
|
135
|
+
except ValueError as e:
|
|
136
|
+
raise ScrapeMapsAPIError(
|
|
137
|
+
f"Invalid JSON in response: {response.text[:200]!r}", status_code=response.status_code
|
|
138
|
+
) from e
|
|
139
|
+
if not isinstance(body, dict):
|
|
140
|
+
raise ScrapeMapsAPIError("Unexpected response shape", status_code=response.status_code, body=body)
|
|
141
|
+
return body
|
scrapemapsapi/_client.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Synchronous ScrapeMapsAPI client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from typing import Any, Mapping, Optional
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
from ._base import (
|
|
11
|
+
DEFAULT_MAX_RETRIES,
|
|
12
|
+
DEFAULT_MAX_RETRY_WAIT,
|
|
13
|
+
DEFAULT_RETRY_BACKOFF,
|
|
14
|
+
DEFAULT_TIMEOUT,
|
|
15
|
+
BaseClient,
|
|
16
|
+
clean_params,
|
|
17
|
+
)
|
|
18
|
+
from .places import Places
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ScrapeMapsAPI(BaseClient):
|
|
22
|
+
"""Synchronous client for the ScrapeMapsAPI API (Google Maps places and reviews).
|
|
23
|
+
|
|
24
|
+
Usage::
|
|
25
|
+
|
|
26
|
+
from scrapemapsapi import ScrapeMapsAPI
|
|
27
|
+
|
|
28
|
+
with ScrapeMapsAPI() as client: # reads SCRAPEMAPSAPI_API_KEY
|
|
29
|
+
page = client.places.search("pizzeria in Naples", limit=5)
|
|
30
|
+
for place in page.data:
|
|
31
|
+
print(place.name, place.rating)
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
api_key: API key; defaults to the ``SCRAPEMAPSAPI_API_KEY`` environment variable.
|
|
35
|
+
base_url: API root; defaults to ``SCRAPEMAPSAPI_BASE_URL`` or https://api.scrapemapsapi.com.
|
|
36
|
+
timeout: Seconds per request (reviews of large places can take a while upstream).
|
|
37
|
+
max_retries: Retries on 429/502/503 and connection errors (0 disables them).
|
|
38
|
+
retry_backoff: Base delay in seconds of the exponential backoff (with jitter).
|
|
39
|
+
max_retry_wait: Longest wait accepted between retries; a ``Retry-After`` longer than
|
|
40
|
+
this raises immediately instead of sleeping.
|
|
41
|
+
http_client: Your own ``httpx.Client`` (proxies, transports, tests). Not closed by us.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
api_key: Optional[str] = None,
|
|
47
|
+
*,
|
|
48
|
+
base_url: Optional[str] = None,
|
|
49
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
50
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
51
|
+
retry_backoff: float = DEFAULT_RETRY_BACKOFF,
|
|
52
|
+
max_retry_wait: float = DEFAULT_MAX_RETRY_WAIT,
|
|
53
|
+
http_client: Optional[httpx.Client] = None,
|
|
54
|
+
) -> None:
|
|
55
|
+
super().__init__(api_key, base_url, timeout, max_retries, retry_backoff, max_retry_wait)
|
|
56
|
+
self._owns_http = http_client is None
|
|
57
|
+
self._http = http_client or httpx.Client(timeout=timeout)
|
|
58
|
+
self.places = Places(self)
|
|
59
|
+
|
|
60
|
+
def _get(self, path: str, params: Mapping[str, Any]) -> tuple[dict[str, Any], httpx.Headers]:
|
|
61
|
+
attempt = 0
|
|
62
|
+
while True:
|
|
63
|
+
try:
|
|
64
|
+
response = self._http.get(
|
|
65
|
+
self._url(path), params=clean_params(params), headers=self._headers, timeout=self.timeout
|
|
66
|
+
)
|
|
67
|
+
except httpx.TransportError as exc:
|
|
68
|
+
wait, error = self._transport_retry_wait(exc, attempt)
|
|
69
|
+
if wait is None:
|
|
70
|
+
raise error from exc
|
|
71
|
+
else:
|
|
72
|
+
if response.is_success:
|
|
73
|
+
body = self._json(response)
|
|
74
|
+
self._record(response)
|
|
75
|
+
return body, response.headers
|
|
76
|
+
wait, error = self._retry_wait(response, attempt)
|
|
77
|
+
if wait is None:
|
|
78
|
+
raise error
|
|
79
|
+
attempt += 1
|
|
80
|
+
time.sleep(wait)
|
|
81
|
+
|
|
82
|
+
def close(self) -> None:
|
|
83
|
+
"""Close the underlying HTTP client (only if the SDK created it)."""
|
|
84
|
+
if self._owns_http:
|
|
85
|
+
self._http.close()
|
|
86
|
+
|
|
87
|
+
def __enter__(self) -> ScrapeMapsAPI:
|
|
88
|
+
return self
|
|
89
|
+
|
|
90
|
+
def __exit__(self, *args: Any) -> None:
|
|
91
|
+
self.close()
|
scrapemapsapi/errors.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Typed exceptions raised by the ScrapeMapsAPI SDK.
|
|
2
|
+
|
|
3
|
+
Every API error carries the machine-readable ``code`` from the response body
|
|
4
|
+
(``{"detail": {"error": code, "message": str}}``), the HTTP ``status_code`` and,
|
|
5
|
+
when the server sent one, ``retry_after`` (seconds, from the ``Retry-After`` header).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any, Mapping, Optional
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ScrapeMapsAPIError(Exception):
|
|
16
|
+
"""Base class for every error raised by the SDK."""
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
message: str,
|
|
21
|
+
*,
|
|
22
|
+
status_code: Optional[int] = None,
|
|
23
|
+
code: Optional[str] = None,
|
|
24
|
+
body: Any = None,
|
|
25
|
+
retry_after: Optional[float] = None,
|
|
26
|
+
) -> None:
|
|
27
|
+
self.message = message
|
|
28
|
+
self.status_code = status_code
|
|
29
|
+
self.code = code
|
|
30
|
+
self.body = body
|
|
31
|
+
self.retry_after = retry_after
|
|
32
|
+
super().__init__(message)
|
|
33
|
+
|
|
34
|
+
def __str__(self) -> str:
|
|
35
|
+
parts = [self.message]
|
|
36
|
+
if self.code:
|
|
37
|
+
parts.append(f"[{self.code}]")
|
|
38
|
+
if self.status_code:
|
|
39
|
+
parts.append(f"(HTTP {self.status_code})")
|
|
40
|
+
return " ".join(parts)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class APIConnectionError(ScrapeMapsAPIError):
|
|
44
|
+
"""The server could not be reached (DNS, connection refused, TLS, ...)."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class APITimeoutError(APIConnectionError):
|
|
48
|
+
"""The request timed out."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class InvalidRequestError(ScrapeMapsAPIError):
|
|
52
|
+
"""400/422: invalid place identifier, cursor, location or parameters."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class AuthenticationError(ScrapeMapsAPIError):
|
|
56
|
+
"""401: missing or invalid API key (also raised when no key is configured)."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class InsufficientCreditsError(ScrapeMapsAPIError):
|
|
60
|
+
"""402: not enough credits. ``available`` and ``required`` are set when the API sends them."""
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def available(self) -> Optional[int]:
|
|
64
|
+
return _detail_int(self.body, "available")
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def required(self) -> Optional[int]:
|
|
68
|
+
return _detail_int(self.body, "required")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class NotFoundError(ScrapeMapsAPIError):
|
|
72
|
+
"""404: the place does not exist."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
PlaceNotFoundError = NotFoundError
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class RateLimitError(ScrapeMapsAPIError):
|
|
79
|
+
"""429: too many requests. ``retry_after`` tells how many seconds to wait."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class CreditsCooldownError(RateLimitError, InsufficientCreditsError):
|
|
83
|
+
"""429 ``insufficient_credits_retry_later``: the key kept calling with no credits and is in a
|
|
84
|
+
cooldown. Top up credits; retrying sooner than ``retry_after`` will not help."""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class UpstreamError(ScrapeMapsAPIError):
|
|
88
|
+
"""502: Google Maps answered with an error. You were not charged."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class ServiceUnavailableError(UpstreamError):
|
|
92
|
+
"""503: Google Maps is temporarily unreachable. You were not charged."""
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class InternalServerError(ScrapeMapsAPIError):
|
|
96
|
+
"""500 and other unexpected 5xx responses."""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
_CODE_MAP: dict[str, type[ScrapeMapsAPIError]] = {
|
|
100
|
+
"invalid_place_id": InvalidRequestError,
|
|
101
|
+
"invalid_cursor": InvalidRequestError,
|
|
102
|
+
"invalid_location": InvalidRequestError,
|
|
103
|
+
"validation_error": InvalidRequestError,
|
|
104
|
+
"missing_api_key": AuthenticationError,
|
|
105
|
+
"invalid_api_key": AuthenticationError,
|
|
106
|
+
"insufficient_credits": InsufficientCreditsError,
|
|
107
|
+
"place_not_found": NotFoundError,
|
|
108
|
+
"rate_limited": RateLimitError,
|
|
109
|
+
"rate_limit_exceeded": RateLimitError,
|
|
110
|
+
"insufficient_credits_retry_later": CreditsCooldownError,
|
|
111
|
+
"upstream_error": UpstreamError,
|
|
112
|
+
"upstream_unavailable": ServiceUnavailableError,
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
_STATUS_MAP: dict[int, type[ScrapeMapsAPIError]] = {
|
|
116
|
+
400: InvalidRequestError,
|
|
117
|
+
401: AuthenticationError,
|
|
118
|
+
402: InsufficientCreditsError,
|
|
119
|
+
404: NotFoundError,
|
|
120
|
+
422: InvalidRequestError,
|
|
121
|
+
429: RateLimitError,
|
|
122
|
+
502: UpstreamError,
|
|
123
|
+
503: ServiceUnavailableError,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _detail_int(body: Any, key: str) -> Optional[int]:
|
|
128
|
+
if isinstance(body, dict) and isinstance(body.get("detail"), dict):
|
|
129
|
+
value = body["detail"].get(key)
|
|
130
|
+
if isinstance(value, int):
|
|
131
|
+
return value
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def parse_retry_after(headers: Mapping[str, str]) -> Optional[float]:
|
|
136
|
+
"""Seconds from a ``Retry-After`` header (only the delta-seconds form is used by the API)."""
|
|
137
|
+
value = headers.get("retry-after")
|
|
138
|
+
if not value:
|
|
139
|
+
return None
|
|
140
|
+
try:
|
|
141
|
+
return max(0.0, float(value))
|
|
142
|
+
except ValueError:
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def parse_error_body(status_code: int, body: Any, text: str = "") -> tuple[Optional[str], str]:
|
|
147
|
+
"""Extract ``(code, message)`` from an error body, whatever its shape."""
|
|
148
|
+
code: Optional[str] = None
|
|
149
|
+
message = text.strip() or f"HTTP {status_code}"
|
|
150
|
+
if isinstance(body, dict):
|
|
151
|
+
detail = body.get("detail", body)
|
|
152
|
+
if isinstance(detail, dict):
|
|
153
|
+
code = detail.get("error") if isinstance(detail.get("error"), str) else None
|
|
154
|
+
if isinstance(detail.get("message"), str):
|
|
155
|
+
message = detail["message"]
|
|
156
|
+
elif isinstance(detail, list): # FastAPI 422: [{"loc": [...], "msg": "...", ...}]
|
|
157
|
+
code = "validation_error"
|
|
158
|
+
msgs = []
|
|
159
|
+
for item in detail:
|
|
160
|
+
if isinstance(item, dict):
|
|
161
|
+
loc = ".".join(str(p) for p in item.get("loc", []) if p not in ("query", "path"))
|
|
162
|
+
msg = item.get("msg", "invalid value")
|
|
163
|
+
msgs.append(f"{loc}: {msg}" if loc else str(msg))
|
|
164
|
+
if msgs:
|
|
165
|
+
message = "; ".join(msgs)
|
|
166
|
+
elif isinstance(detail, str):
|
|
167
|
+
message = detail
|
|
168
|
+
return code, message
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def error_class_for(status_code: int, code: Optional[str]) -> type[ScrapeMapsAPIError]:
|
|
172
|
+
# 5xx: the status decides. A 5xx body can carry an upstream code such as "rate_limited"
|
|
173
|
+
# (Google throttled us) that must not turn into a client-side RateLimitError.
|
|
174
|
+
if code and code in _CODE_MAP and status_code < 500:
|
|
175
|
+
return _CODE_MAP[code]
|
|
176
|
+
if status_code in _STATUS_MAP:
|
|
177
|
+
return _STATUS_MAP[status_code]
|
|
178
|
+
if status_code >= 500:
|
|
179
|
+
return InternalServerError
|
|
180
|
+
return ScrapeMapsAPIError
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def error_from_response(response: httpx.Response) -> ScrapeMapsAPIError:
|
|
184
|
+
"""Build the typed exception for a non-2xx response."""
|
|
185
|
+
try:
|
|
186
|
+
body: Any = response.json()
|
|
187
|
+
except Exception:
|
|
188
|
+
body = None
|
|
189
|
+
code, message = parse_error_body(response.status_code, body, response.text if body is None else "")
|
|
190
|
+
cls = error_class_for(response.status_code, code)
|
|
191
|
+
return cls(
|
|
192
|
+
message,
|
|
193
|
+
status_code=response.status_code,
|
|
194
|
+
code=code,
|
|
195
|
+
body=body,
|
|
196
|
+
retry_after=parse_retry_after(response.headers),
|
|
197
|
+
)
|