prismcrawl 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prismcrawl/__init__.py +40 -0
- prismcrawl/_base.py +274 -0
- prismcrawl/_client.py +92 -0
- prismcrawl/_endpoint.py +149 -0
- prismcrawl/_errors.py +150 -0
- prismcrawl/_html.py +33 -0
- prismcrawl/_resources.py +5622 -0
- prismcrawl/_version.py +1 -0
- prismcrawl/py.typed +0 -0
- prismcrawl/types.py +4294 -0
- prismcrawl-0.1.0.dist-info/METADATA +363 -0
- prismcrawl-0.1.0.dist-info/RECORD +14 -0
- prismcrawl-0.1.0.dist-info/WHEEL +4 -0
- prismcrawl-0.1.0.dist-info/licenses/LICENSE +21 -0
prismcrawl/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Python client for the PrismCrawl API."""
|
|
2
|
+
|
|
3
|
+
from . import types
|
|
4
|
+
from ._base import DEFAULT_BASE_URL
|
|
5
|
+
from ._client import AsyncPrismCrawl, PrismCrawl
|
|
6
|
+
from ._errors import (
|
|
7
|
+
APIConnectionError,
|
|
8
|
+
APIStatusError,
|
|
9
|
+
APITimeoutError,
|
|
10
|
+
AuthenticationError,
|
|
11
|
+
BadRequestError,
|
|
12
|
+
InsufficientCreditsError,
|
|
13
|
+
InternalServerError,
|
|
14
|
+
NotFoundError,
|
|
15
|
+
PayloadTooLargeError,
|
|
16
|
+
PrismCrawlError,
|
|
17
|
+
RateLimitError,
|
|
18
|
+
)
|
|
19
|
+
from ._html import decode_html
|
|
20
|
+
from ._version import __version__
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"PrismCrawl",
|
|
24
|
+
"AsyncPrismCrawl",
|
|
25
|
+
"decode_html",
|
|
26
|
+
"types",
|
|
27
|
+
"DEFAULT_BASE_URL",
|
|
28
|
+
"PrismCrawlError",
|
|
29
|
+
"APIConnectionError",
|
|
30
|
+
"APITimeoutError",
|
|
31
|
+
"APIStatusError",
|
|
32
|
+
"BadRequestError",
|
|
33
|
+
"AuthenticationError",
|
|
34
|
+
"InsufficientCreditsError",
|
|
35
|
+
"NotFoundError",
|
|
36
|
+
"PayloadTooLargeError",
|
|
37
|
+
"RateLimitError",
|
|
38
|
+
"InternalServerError",
|
|
39
|
+
"__version__",
|
|
40
|
+
]
|
prismcrawl/_base.py
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""HTTP transport shared by the sync and async clients."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import email.utils
|
|
7
|
+
import os
|
|
8
|
+
import platform
|
|
9
|
+
import random
|
|
10
|
+
import time
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from typing import Any, Dict, Mapping, Optional
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from ._errors import APIConnectionError, APITimeoutError, PrismCrawlError, make_status_error
|
|
17
|
+
from ._version import __version__
|
|
18
|
+
|
|
19
|
+
DEFAULT_BASE_URL = "https://api.prismcrawl.com"
|
|
20
|
+
DEFAULT_TIMEOUT = 120.0
|
|
21
|
+
DEFAULT_MAX_RETRIES = 2
|
|
22
|
+
API_KEY_ENV = "PRISMCRAWL_API_KEY"
|
|
23
|
+
BASE_URL_ENV = "PRISMCRAWL_BASE_URL"
|
|
24
|
+
|
|
25
|
+
# Don't sleep longer than this for a rate limit; raise RateLimitError instead.
|
|
26
|
+
MAX_RETRY_WAIT = 30.0
|
|
27
|
+
INITIAL_RETRY_DELAY = 0.5
|
|
28
|
+
MAX_RETRY_DELAY = 8.0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _resolve_api_key(api_key: Optional[str]) -> str:
|
|
32
|
+
key = api_key if api_key is not None else os.environ.get(API_KEY_ENV)
|
|
33
|
+
if not key or not key.strip():
|
|
34
|
+
raise PrismCrawlError(
|
|
35
|
+
f"No API key provided. Pass api_key=... or set the {API_KEY_ENV} environment variable. "
|
|
36
|
+
"Create a key in the PrismCrawl dashboard."
|
|
37
|
+
)
|
|
38
|
+
return key.strip()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _user_agent() -> str:
|
|
42
|
+
return f"prismcrawl-python/{__version__} python/{platform.python_version()}"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _retry_after(response: httpx.Response) -> Optional[float]:
|
|
46
|
+
"""Seconds to wait before retrying, from Retry-After or the body's reset_at."""
|
|
47
|
+
header = response.headers.get("retry-after")
|
|
48
|
+
if header:
|
|
49
|
+
try:
|
|
50
|
+
return max(0.0, float(header))
|
|
51
|
+
except ValueError:
|
|
52
|
+
try:
|
|
53
|
+
parsed = email.utils.parsedate_to_datetime(header)
|
|
54
|
+
except (TypeError, ValueError):
|
|
55
|
+
parsed = None
|
|
56
|
+
if parsed is not None:
|
|
57
|
+
return max(0.0, parsed.timestamp() - time.time())
|
|
58
|
+
try:
|
|
59
|
+
body = response.json()
|
|
60
|
+
reset_at = body["error"]["reset_at"]
|
|
61
|
+
reset = datetime.fromisoformat(reset_at.replace("Z", "+00:00"))
|
|
62
|
+
if reset.tzinfo is None:
|
|
63
|
+
reset = reset.replace(tzinfo=timezone.utc)
|
|
64
|
+
return max(0.0, reset.timestamp() - time.time())
|
|
65
|
+
except (ValueError, KeyError, TypeError, AttributeError):
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _backoff(attempt: int) -> float:
|
|
70
|
+
delay = min(INITIAL_RETRY_DELAY * (2**attempt), MAX_RETRY_DELAY)
|
|
71
|
+
return delay * (0.75 + random.random() * 0.5)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _status_retry_delay(response: httpx.Response, attempt: int) -> Optional[float]:
|
|
75
|
+
"""Delay before retrying this response, or None if it should not be retried.
|
|
76
|
+
|
|
77
|
+
Failed requests don't use credits, so 429 and 5xx are safe to retry for any method.
|
|
78
|
+
"""
|
|
79
|
+
status = response.status_code
|
|
80
|
+
if status == 429:
|
|
81
|
+
wait = _retry_after(response)
|
|
82
|
+
if wait is None:
|
|
83
|
+
return _backoff(attempt)
|
|
84
|
+
return wait if wait <= MAX_RETRY_WAIT else None
|
|
85
|
+
if status >= 500:
|
|
86
|
+
return _backoff(attempt)
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _transport_retry_delay(error: httpx.TransportError, method: str, attempt: int) -> Optional[float]:
|
|
91
|
+
# A connect failure means the request was never sent. Anything later (read timeout,
|
|
92
|
+
# dropped connection) may have completed a billable search, so only GETs retry.
|
|
93
|
+
if isinstance(error, (httpx.ConnectError, httpx.ConnectTimeout)) or method == "GET":
|
|
94
|
+
return _backoff(attempt)
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _wrap_transport_error(error: httpx.TransportError) -> APIConnectionError:
|
|
99
|
+
request = getattr(error, "_request", None)
|
|
100
|
+
if isinstance(error, httpx.TimeoutException):
|
|
101
|
+
return APITimeoutError(request=request)
|
|
102
|
+
return APIConnectionError(f"Connection error: {error}", request=request)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _parse(response: httpx.Response, raw_text: bool) -> Any:
|
|
106
|
+
if response.status_code >= 400:
|
|
107
|
+
raise make_status_error(response)
|
|
108
|
+
if raw_text:
|
|
109
|
+
return response.text
|
|
110
|
+
content_type = response.headers.get("content-type", "")
|
|
111
|
+
if "json" in content_type or not content_type:
|
|
112
|
+
return response.json()
|
|
113
|
+
return response.text
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _ClientConfig:
|
|
117
|
+
def __init__(
|
|
118
|
+
self,
|
|
119
|
+
api_key: Optional[str],
|
|
120
|
+
base_url: Optional[str],
|
|
121
|
+
timeout: Optional[float],
|
|
122
|
+
max_retries: int,
|
|
123
|
+
default_headers: Optional[Mapping[str, str]],
|
|
124
|
+
) -> None:
|
|
125
|
+
if max_retries < 0:
|
|
126
|
+
raise ValueError("max_retries must be >= 0")
|
|
127
|
+
self.api_key = _resolve_api_key(api_key)
|
|
128
|
+
self.base_url = (base_url or os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL).rstrip("/")
|
|
129
|
+
self.timeout = timeout
|
|
130
|
+
self.max_retries = max_retries
|
|
131
|
+
self.headers: Dict[str, str] = {
|
|
132
|
+
"x-api-key": self.api_key,
|
|
133
|
+
"accept": "application/json",
|
|
134
|
+
"user-agent": _user_agent(),
|
|
135
|
+
**(default_headers or {}),
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
def build(
|
|
139
|
+
self,
|
|
140
|
+
client: Any,
|
|
141
|
+
method: str,
|
|
142
|
+
path: str,
|
|
143
|
+
json: Optional[Any],
|
|
144
|
+
params: Optional[Mapping[str, str]],
|
|
145
|
+
timeout: Optional[float],
|
|
146
|
+
) -> httpx.Request:
|
|
147
|
+
return client.build_request(
|
|
148
|
+
method,
|
|
149
|
+
self.base_url + path,
|
|
150
|
+
json=json,
|
|
151
|
+
params=params,
|
|
152
|
+
headers=self.headers,
|
|
153
|
+
timeout=self.timeout if timeout is None else timeout,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class BaseClient:
|
|
158
|
+
"""Synchronous transport. Use :class:`prismcrawl.PrismCrawl`."""
|
|
159
|
+
|
|
160
|
+
def __init__(
|
|
161
|
+
self,
|
|
162
|
+
api_key: Optional[str] = None,
|
|
163
|
+
*,
|
|
164
|
+
base_url: Optional[str] = None,
|
|
165
|
+
timeout: Optional[float] = DEFAULT_TIMEOUT,
|
|
166
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
167
|
+
default_headers: Optional[Mapping[str, str]] = None,
|
|
168
|
+
http_client: Optional[httpx.Client] = None,
|
|
169
|
+
) -> None:
|
|
170
|
+
self._config = _ClientConfig(api_key, base_url, timeout, max_retries, default_headers)
|
|
171
|
+
self._owns_http_client = http_client is None
|
|
172
|
+
self._http = http_client or httpx.Client()
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def base_url(self) -> str:
|
|
176
|
+
return self._config.base_url
|
|
177
|
+
|
|
178
|
+
def _request(
|
|
179
|
+
self,
|
|
180
|
+
method: str,
|
|
181
|
+
path: str,
|
|
182
|
+
*,
|
|
183
|
+
json: Optional[Any] = None,
|
|
184
|
+
params: Optional[Mapping[str, str]] = None,
|
|
185
|
+
timeout: Optional[float] = None,
|
|
186
|
+
raw_text: bool = False,
|
|
187
|
+
) -> Any:
|
|
188
|
+
request = self._config.build(self._http, method, path, json, params, timeout)
|
|
189
|
+
attempt = 0
|
|
190
|
+
while True:
|
|
191
|
+
try:
|
|
192
|
+
response = self._http.send(request)
|
|
193
|
+
except httpx.TransportError as error:
|
|
194
|
+
delay = _transport_retry_delay(error, method, attempt)
|
|
195
|
+
if delay is None or attempt >= self._config.max_retries:
|
|
196
|
+
raise _wrap_transport_error(error) from error
|
|
197
|
+
else:
|
|
198
|
+
delay = _status_retry_delay(response, attempt)
|
|
199
|
+
if delay is None or attempt >= self._config.max_retries:
|
|
200
|
+
return _parse(response, raw_text)
|
|
201
|
+
response.close()
|
|
202
|
+
time.sleep(delay)
|
|
203
|
+
attempt += 1
|
|
204
|
+
|
|
205
|
+
def close(self) -> None:
|
|
206
|
+
"""Close the underlying HTTP connection pool."""
|
|
207
|
+
if self._owns_http_client:
|
|
208
|
+
self._http.close()
|
|
209
|
+
|
|
210
|
+
def __enter__(self) -> BaseClient:
|
|
211
|
+
return self
|
|
212
|
+
|
|
213
|
+
def __exit__(self, *exc: object) -> None:
|
|
214
|
+
self.close()
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class AsyncBaseClient:
|
|
218
|
+
"""Asynchronous transport. Use :class:`prismcrawl.AsyncPrismCrawl`."""
|
|
219
|
+
|
|
220
|
+
def __init__(
|
|
221
|
+
self,
|
|
222
|
+
api_key: Optional[str] = None,
|
|
223
|
+
*,
|
|
224
|
+
base_url: Optional[str] = None,
|
|
225
|
+
timeout: Optional[float] = DEFAULT_TIMEOUT,
|
|
226
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
227
|
+
default_headers: Optional[Mapping[str, str]] = None,
|
|
228
|
+
http_client: Optional[httpx.AsyncClient] = None,
|
|
229
|
+
) -> None:
|
|
230
|
+
self._config = _ClientConfig(api_key, base_url, timeout, max_retries, default_headers)
|
|
231
|
+
self._owns_http_client = http_client is None
|
|
232
|
+
self._http = http_client or httpx.AsyncClient()
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def base_url(self) -> str:
|
|
236
|
+
return self._config.base_url
|
|
237
|
+
|
|
238
|
+
async def _request(
|
|
239
|
+
self,
|
|
240
|
+
method: str,
|
|
241
|
+
path: str,
|
|
242
|
+
*,
|
|
243
|
+
json: Optional[Any] = None,
|
|
244
|
+
params: Optional[Mapping[str, str]] = None,
|
|
245
|
+
timeout: Optional[float] = None,
|
|
246
|
+
raw_text: bool = False,
|
|
247
|
+
) -> Any:
|
|
248
|
+
request = self._config.build(self._http, method, path, json, params, timeout)
|
|
249
|
+
attempt = 0
|
|
250
|
+
while True:
|
|
251
|
+
try:
|
|
252
|
+
response = await self._http.send(request)
|
|
253
|
+
except httpx.TransportError as error:
|
|
254
|
+
delay = _transport_retry_delay(error, method, attempt)
|
|
255
|
+
if delay is None or attempt >= self._config.max_retries:
|
|
256
|
+
raise _wrap_transport_error(error) from error
|
|
257
|
+
else:
|
|
258
|
+
delay = _status_retry_delay(response, attempt)
|
|
259
|
+
if delay is None or attempt >= self._config.max_retries:
|
|
260
|
+
return _parse(response, raw_text)
|
|
261
|
+
await response.aclose()
|
|
262
|
+
await asyncio.sleep(delay)
|
|
263
|
+
attempt += 1
|
|
264
|
+
|
|
265
|
+
async def close(self) -> None:
|
|
266
|
+
"""Close the underlying HTTP connection pool."""
|
|
267
|
+
if self._owns_http_client:
|
|
268
|
+
await self._http.aclose()
|
|
269
|
+
|
|
270
|
+
async def __aenter__(self) -> AsyncBaseClient:
|
|
271
|
+
return self
|
|
272
|
+
|
|
273
|
+
async def __aexit__(self, *exc: object) -> None:
|
|
274
|
+
await self.close()
|
prismcrawl/_client.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""The PrismCrawl API clients."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Mapping, Optional
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from ._base import DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT, AsyncBaseClient, BaseClient
|
|
10
|
+
from ._resources import AsyncResources, Resources
|
|
11
|
+
|
|
12
|
+
__all__ = ["PrismCrawl", "AsyncPrismCrawl"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PrismCrawl(BaseClient, Resources):
|
|
16
|
+
"""Synchronous PrismCrawl client.
|
|
17
|
+
|
|
18
|
+
Example:
|
|
19
|
+
>>> from prismcrawl import PrismCrawl
|
|
20
|
+
>>> client = PrismCrawl(api_key="pc_...") # or set PRISMCRAWL_API_KEY
|
|
21
|
+
>>> results = client.google.search(query="best espresso machines")
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
api_key: Your PrismCrawl API key. Defaults to the ``PRISMCRAWL_API_KEY``
|
|
25
|
+
environment variable.
|
|
26
|
+
base_url: API base URL. Defaults to ``PRISMCRAWL_BASE_URL`` or
|
|
27
|
+
``https://api.prismcrawl.com``.
|
|
28
|
+
timeout: Request timeout in seconds, or ``None`` for no timeout.
|
|
29
|
+
max_retries: Retries for connection failures, rate limits, and server errors.
|
|
30
|
+
default_headers: Extra headers sent with every request.
|
|
31
|
+
http_client: A preconfigured ``httpx.Client`` (proxies, transports, ...). The
|
|
32
|
+
caller is responsible for closing it.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
api_key: Optional[str] = None,
|
|
38
|
+
*,
|
|
39
|
+
base_url: Optional[str] = None,
|
|
40
|
+
timeout: Optional[float] = DEFAULT_TIMEOUT,
|
|
41
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
42
|
+
default_headers: Optional[Mapping[str, str]] = None,
|
|
43
|
+
http_client: Optional[httpx.Client] = None,
|
|
44
|
+
) -> None:
|
|
45
|
+
super().__init__(
|
|
46
|
+
api_key,
|
|
47
|
+
base_url=base_url,
|
|
48
|
+
timeout=timeout,
|
|
49
|
+
max_retries=max_retries,
|
|
50
|
+
default_headers=default_headers,
|
|
51
|
+
http_client=http_client,
|
|
52
|
+
)
|
|
53
|
+
self._init_resources(self)
|
|
54
|
+
|
|
55
|
+
def __enter__(self) -> PrismCrawl:
|
|
56
|
+
return self
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class AsyncPrismCrawl(AsyncBaseClient, AsyncResources):
|
|
60
|
+
"""Asynchronous PrismCrawl client.
|
|
61
|
+
|
|
62
|
+
Example:
|
|
63
|
+
>>> from prismcrawl import AsyncPrismCrawl
|
|
64
|
+
>>> async with AsyncPrismCrawl() as client:
|
|
65
|
+
... results = await client.google.search(query="best espresso machines")
|
|
66
|
+
|
|
67
|
+
Takes the same arguments as :class:`PrismCrawl`, with an ``httpx.AsyncClient``
|
|
68
|
+
for ``http_client``.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(
|
|
72
|
+
self,
|
|
73
|
+
api_key: Optional[str] = None,
|
|
74
|
+
*,
|
|
75
|
+
base_url: Optional[str] = None,
|
|
76
|
+
timeout: Optional[float] = DEFAULT_TIMEOUT,
|
|
77
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
78
|
+
default_headers: Optional[Mapping[str, str]] = None,
|
|
79
|
+
http_client: Optional[httpx.AsyncClient] = None,
|
|
80
|
+
) -> None:
|
|
81
|
+
super().__init__(
|
|
82
|
+
api_key,
|
|
83
|
+
base_url=base_url,
|
|
84
|
+
timeout=timeout,
|
|
85
|
+
max_retries=max_retries,
|
|
86
|
+
default_headers=default_headers,
|
|
87
|
+
http_client=http_client,
|
|
88
|
+
)
|
|
89
|
+
self._init_resources(self)
|
|
90
|
+
|
|
91
|
+
async def __aenter__(self) -> AsyncPrismCrawl:
|
|
92
|
+
return self
|
prismcrawl/_endpoint.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Base classes for generated endpoints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, AsyncIterator, ClassVar, Dict, Iterator, Mapping, Optional, Union, overload
|
|
6
|
+
from urllib.parse import quote
|
|
7
|
+
|
|
8
|
+
from typing_extensions import Literal
|
|
9
|
+
|
|
10
|
+
from ._base import AsyncBaseClient, BaseClient
|
|
11
|
+
|
|
12
|
+
Artifact = Literal["json", "html"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _build_body(body: Mapping[str, Any], extra_body: Optional[Mapping[str, Any]]) -> Dict[str, Any]:
|
|
16
|
+
out = {key: value for key, value in body.items() if value is not None}
|
|
17
|
+
if extra_body:
|
|
18
|
+
out.update(extra_body)
|
|
19
|
+
return out
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _history_params(artifact: Optional[str]) -> Optional[Dict[str, str]]:
|
|
23
|
+
if artifact is None:
|
|
24
|
+
return None
|
|
25
|
+
if artifact not in ("json", "html"):
|
|
26
|
+
raise ValueError('artifact must be "json", "html", or None')
|
|
27
|
+
return {"artifact": artifact}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _next_body(kind: Optional[str], body: Dict[str, Any], page: Any) -> Optional[Dict[str, Any]]:
|
|
31
|
+
"""Request body for the page after `page`, or None when there are no more pages."""
|
|
32
|
+
data = page.get("data") if isinstance(page, dict) else None
|
|
33
|
+
content = data.get("content") if isinstance(data, dict) else None
|
|
34
|
+
if not isinstance(content, dict) or not content.get("has_next_page"):
|
|
35
|
+
return None
|
|
36
|
+
if kind == "token":
|
|
37
|
+
token = content.get("next_page_token")
|
|
38
|
+
return {**body, "next_page_token": token} if token else None
|
|
39
|
+
if kind == "page":
|
|
40
|
+
return {**body, "page": int(body.get("page") or 1) + 1}
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class SyncEndpoint:
|
|
45
|
+
_path: ClassVar[str]
|
|
46
|
+
_pagination: ClassVar[Optional[str]] = None
|
|
47
|
+
|
|
48
|
+
def __init__(self, client: BaseClient) -> None:
|
|
49
|
+
self._client = client
|
|
50
|
+
|
|
51
|
+
def _create(
|
|
52
|
+
self, body: Mapping[str, Any], extra_body: Optional[Mapping[str, Any]], timeout: Optional[float]
|
|
53
|
+
) -> Any:
|
|
54
|
+
return self._client._request("POST", self._path, json=_build_body(body, extra_body), timeout=timeout)
|
|
55
|
+
|
|
56
|
+
def _pages(
|
|
57
|
+
self,
|
|
58
|
+
body: Mapping[str, Any],
|
|
59
|
+
extra_body: Optional[Mapping[str, Any]],
|
|
60
|
+
timeout: Optional[float],
|
|
61
|
+
max_pages: Optional[int],
|
|
62
|
+
) -> Iterator[Any]:
|
|
63
|
+
request: Optional[Dict[str, Any]] = _build_body(body, extra_body)
|
|
64
|
+
fetched = 0
|
|
65
|
+
while request is not None and (max_pages is None or fetched < max_pages):
|
|
66
|
+
page = self._client._request("POST", self._path, json=request, timeout=timeout)
|
|
67
|
+
fetched += 1
|
|
68
|
+
yield page
|
|
69
|
+
request = _next_body(self._pagination, request, page)
|
|
70
|
+
|
|
71
|
+
@overload
|
|
72
|
+
def get(self, request_id: str, *, artifact: Literal["html"], timeout: Optional[float] = None) -> str: ...
|
|
73
|
+
@overload
|
|
74
|
+
def get(
|
|
75
|
+
self, request_id: str, *, artifact: Optional[Literal["json"]] = None, timeout: Optional[float] = None
|
|
76
|
+
) -> Dict[str, Any]: ...
|
|
77
|
+
def get(
|
|
78
|
+
self, request_id: str, *, artifact: Optional[Artifact] = None, timeout: Optional[float] = None
|
|
79
|
+
) -> Union[str, Dict[str, Any]]:
|
|
80
|
+
"""Retrieve a previous request by id.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
request_id: The ``request_id`` from an earlier response.
|
|
84
|
+
artifact: ``None`` for request metadata, ``"json"`` for the archived parsed
|
|
85
|
+
response, or ``"html"`` for the archived source HTML (returned as ``str``).
|
|
86
|
+
Requests made with ``zero_trace=True`` have no artifacts.
|
|
87
|
+
timeout: Override the client timeout for this call, in seconds.
|
|
88
|
+
"""
|
|
89
|
+
return self._client._request(
|
|
90
|
+
"GET",
|
|
91
|
+
f"{self._path}/{quote(request_id, safe='')}",
|
|
92
|
+
params=_history_params(artifact),
|
|
93
|
+
timeout=timeout,
|
|
94
|
+
raw_text=artifact == "html",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class AsyncEndpoint:
|
|
99
|
+
_path: ClassVar[str]
|
|
100
|
+
_pagination: ClassVar[Optional[str]] = None
|
|
101
|
+
|
|
102
|
+
def __init__(self, client: AsyncBaseClient) -> None:
|
|
103
|
+
self._client = client
|
|
104
|
+
|
|
105
|
+
async def _create(
|
|
106
|
+
self, body: Mapping[str, Any], extra_body: Optional[Mapping[str, Any]], timeout: Optional[float]
|
|
107
|
+
) -> Any:
|
|
108
|
+
return await self._client._request("POST", self._path, json=_build_body(body, extra_body), timeout=timeout)
|
|
109
|
+
|
|
110
|
+
async def _pages(
|
|
111
|
+
self,
|
|
112
|
+
body: Mapping[str, Any],
|
|
113
|
+
extra_body: Optional[Mapping[str, Any]],
|
|
114
|
+
timeout: Optional[float],
|
|
115
|
+
max_pages: Optional[int],
|
|
116
|
+
) -> AsyncIterator[Any]:
|
|
117
|
+
request: Optional[Dict[str, Any]] = _build_body(body, extra_body)
|
|
118
|
+
fetched = 0
|
|
119
|
+
while request is not None and (max_pages is None or fetched < max_pages):
|
|
120
|
+
page = await self._client._request("POST", self._path, json=request, timeout=timeout)
|
|
121
|
+
fetched += 1
|
|
122
|
+
yield page
|
|
123
|
+
request = _next_body(self._pagination, request, page)
|
|
124
|
+
|
|
125
|
+
@overload
|
|
126
|
+
async def get(self, request_id: str, *, artifact: Literal["html"], timeout: Optional[float] = None) -> str: ...
|
|
127
|
+
@overload
|
|
128
|
+
async def get(
|
|
129
|
+
self, request_id: str, *, artifact: Optional[Literal["json"]] = None, timeout: Optional[float] = None
|
|
130
|
+
) -> Dict[str, Any]: ...
|
|
131
|
+
async def get(
|
|
132
|
+
self, request_id: str, *, artifact: Optional[Artifact] = None, timeout: Optional[float] = None
|
|
133
|
+
) -> Union[str, Dict[str, Any]]:
|
|
134
|
+
"""Retrieve a previous request by id.
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
request_id: The ``request_id`` from an earlier response.
|
|
138
|
+
artifact: ``None`` for request metadata, ``"json"`` for the archived parsed
|
|
139
|
+
response, or ``"html"`` for the archived source HTML (returned as ``str``).
|
|
140
|
+
Requests made with ``zero_trace=True`` have no artifacts.
|
|
141
|
+
timeout: Override the client timeout for this call, in seconds.
|
|
142
|
+
"""
|
|
143
|
+
return await self._client._request(
|
|
144
|
+
"GET",
|
|
145
|
+
f"{self._path}/{quote(request_id, safe='')}",
|
|
146
|
+
params=_history_params(artifact),
|
|
147
|
+
timeout=timeout,
|
|
148
|
+
raw_text=artifact == "html",
|
|
149
|
+
)
|
prismcrawl/_errors.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Exceptions raised by the PrismCrawl client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict, Mapping, Optional
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"PrismCrawlError",
|
|
11
|
+
"APIConnectionError",
|
|
12
|
+
"APITimeoutError",
|
|
13
|
+
"APIStatusError",
|
|
14
|
+
"BadRequestError",
|
|
15
|
+
"AuthenticationError",
|
|
16
|
+
"InsufficientCreditsError",
|
|
17
|
+
"NotFoundError",
|
|
18
|
+
"PayloadTooLargeError",
|
|
19
|
+
"RateLimitError",
|
|
20
|
+
"InternalServerError",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class PrismCrawlError(Exception):
|
|
25
|
+
"""Base class for every error raised by this package."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class APIConnectionError(PrismCrawlError):
|
|
29
|
+
"""The request never produced an HTTP response (DNS, TLS, connection reset, ...)."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, message: str = "Connection error.", *, request: Optional[httpx.Request] = None) -> None:
|
|
32
|
+
super().__init__(message)
|
|
33
|
+
self.request = request
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class APITimeoutError(APIConnectionError):
|
|
37
|
+
"""The request timed out."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, *, request: Optional[httpx.Request] = None) -> None:
|
|
40
|
+
super().__init__("Request timed out.", request=request)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class APIStatusError(PrismCrawlError):
|
|
44
|
+
"""The API returned a non-2xx response.
|
|
45
|
+
|
|
46
|
+
Attributes:
|
|
47
|
+
status_code: HTTP status code.
|
|
48
|
+
code: Machine-readable error code from the response body, such as ``invalid_request``.
|
|
49
|
+
message: Human-readable error message.
|
|
50
|
+
request_id: PrismCrawl request id. Include it when contacting support.
|
|
51
|
+
body: The decoded response body, when it was JSON.
|
|
52
|
+
response: The underlying ``httpx.Response``.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
message: str,
|
|
58
|
+
*,
|
|
59
|
+
response: httpx.Response,
|
|
60
|
+
body: Optional[Any],
|
|
61
|
+
code: Optional[str],
|
|
62
|
+
request_id: Optional[str],
|
|
63
|
+
) -> None:
|
|
64
|
+
super().__init__(message)
|
|
65
|
+
self.message = message
|
|
66
|
+
self.response = response
|
|
67
|
+
self.status_code = response.status_code
|
|
68
|
+
self.body = body
|
|
69
|
+
self.code = code
|
|
70
|
+
self.request_id = request_id
|
|
71
|
+
|
|
72
|
+
def __str__(self) -> str:
|
|
73
|
+
parts = [f"{self.status_code}"]
|
|
74
|
+
if self.code:
|
|
75
|
+
parts.append(self.code)
|
|
76
|
+
text = f"[{' '.join(parts)}] {self.message}"
|
|
77
|
+
if self.request_id:
|
|
78
|
+
text += f" (request_id: {self.request_id})"
|
|
79
|
+
return text
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class BadRequestError(APIStatusError):
|
|
83
|
+
"""400: the request body or pagination token is invalid. No credit is used."""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class AuthenticationError(APIStatusError):
|
|
87
|
+
"""401: the API key is missing, invalid, expired, or revoked."""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class InsufficientCreditsError(APIStatusError):
|
|
91
|
+
"""402: the account has no search credits left."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class NotFoundError(APIStatusError):
|
|
95
|
+
"""404: the request or archived artifact does not exist for this API key."""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class PayloadTooLargeError(APIStatusError):
|
|
99
|
+
"""413: the request body is too large."""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class RateLimitError(APIStatusError):
|
|
103
|
+
"""429: a rate limit was hit.
|
|
104
|
+
|
|
105
|
+
Attributes:
|
|
106
|
+
rule: The limit that was hit, when the API names one.
|
|
107
|
+
reset_at: ISO-8601 time when the limit resets, when the API provides one.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
def __init__(self, message: str, **kwargs: Any) -> None:
|
|
111
|
+
super().__init__(message, **kwargs)
|
|
112
|
+
error = _error_object(self.body)
|
|
113
|
+
self.rule: Optional[str] = error.get("rule")
|
|
114
|
+
self.reset_at: Optional[str] = error.get("reset_at")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class InternalServerError(APIStatusError):
|
|
118
|
+
"""5xx: PrismCrawl could not complete the request. No credit is used."""
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
_STATUS_ERRORS: Mapping[int, type] = {
|
|
122
|
+
400: BadRequestError,
|
|
123
|
+
401: AuthenticationError,
|
|
124
|
+
402: InsufficientCreditsError,
|
|
125
|
+
404: NotFoundError,
|
|
126
|
+
413: PayloadTooLargeError,
|
|
127
|
+
429: RateLimitError,
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _error_object(body: Any) -> Dict[str, Any]:
|
|
132
|
+
if isinstance(body, dict) and isinstance(body.get("error"), dict):
|
|
133
|
+
return body["error"]
|
|
134
|
+
return {}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def make_status_error(response: httpx.Response) -> APIStatusError:
|
|
138
|
+
try:
|
|
139
|
+
body: Any = response.json()
|
|
140
|
+
except ValueError:
|
|
141
|
+
body = response.text or None
|
|
142
|
+
error = _error_object(body)
|
|
143
|
+
message = error.get("message") or response.reason_phrase or "Request failed."
|
|
144
|
+
request_id = response.headers.get("x-request-id")
|
|
145
|
+
if request_id is None and isinstance(body, dict):
|
|
146
|
+
request_id = body.get("request_id")
|
|
147
|
+
cls = _STATUS_ERRORS.get(response.status_code)
|
|
148
|
+
if cls is None:
|
|
149
|
+
cls = InternalServerError if response.status_code >= 500 else APIStatusError
|
|
150
|
+
return cls(message, response=response, body=body, code=error.get("code"), request_id=request_id)
|