enrichednow 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- enrichednow/__init__.py +109 -0
- enrichednow/_base.py +288 -0
- enrichednow/_client.py +301 -0
- enrichednow/_exceptions.py +196 -0
- enrichednow/_models.py +380 -0
- enrichednow/_pagination.py +137 -0
- enrichednow/_resources.py +495 -0
- enrichednow/_version.py +1 -0
- enrichednow/py.typed +1 -0
- enrichednow-0.1.0.dist-info/METADATA +412 -0
- enrichednow-0.1.0.dist-info/RECORD +13 -0
- enrichednow-0.1.0.dist-info/WHEEL +4 -0
- enrichednow-0.1.0.dist-info/licenses/LICENSE +21 -0
enrichednow/__init__.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""The official Python library for the EnrichedNow API.
|
|
2
|
+
|
|
3
|
+
from enrichednow import EnrichedNow
|
|
4
|
+
|
|
5
|
+
client = EnrichedNow() # reads ENRICHEDNOW_API_KEY
|
|
6
|
+
result = client.scrape("ats.board", {"board_url": "https://boards.greenhouse.io/acme"})
|
|
7
|
+
for record in result.records:
|
|
8
|
+
print(record)
|
|
9
|
+
"""
|
|
10
|
+
from ._client import AsyncEnrichedNow, AsyncScrapeResult, EnrichedNow, ScrapeResult
|
|
11
|
+
from ._exceptions import (
|
|
12
|
+
APIConnectionError,
|
|
13
|
+
APIError,
|
|
14
|
+
APITimeoutError,
|
|
15
|
+
AuthenticationError,
|
|
16
|
+
EnrichedNowError,
|
|
17
|
+
MissingAPIKeyError,
|
|
18
|
+
NotFoundError,
|
|
19
|
+
PermissionDeniedError,
|
|
20
|
+
QuotaExceededError,
|
|
21
|
+
RateLimitError,
|
|
22
|
+
RunFailedError,
|
|
23
|
+
ServerError,
|
|
24
|
+
ValidationError,
|
|
25
|
+
WaitTimeoutError,
|
|
26
|
+
)
|
|
27
|
+
from ._models import (
|
|
28
|
+
Change,
|
|
29
|
+
Classification,
|
|
30
|
+
Company,
|
|
31
|
+
CompanyName,
|
|
32
|
+
CompanyRef,
|
|
33
|
+
Event,
|
|
34
|
+
Health,
|
|
35
|
+
History,
|
|
36
|
+
LastRead,
|
|
37
|
+
Location,
|
|
38
|
+
Model,
|
|
39
|
+
Period,
|
|
40
|
+
Posting,
|
|
41
|
+
PostingCounts,
|
|
42
|
+
PostingDetail,
|
|
43
|
+
Run,
|
|
44
|
+
RunDiagnostics,
|
|
45
|
+
Salary,
|
|
46
|
+
Scraper,
|
|
47
|
+
ScraperDetails,
|
|
48
|
+
Sightings,
|
|
49
|
+
Source,
|
|
50
|
+
Usage,
|
|
51
|
+
UsageRow,
|
|
52
|
+
)
|
|
53
|
+
from ._pagination import AsyncPage, AsyncPaginator, AsyncRecordsPage, SyncPage, SyncRecordsPage
|
|
54
|
+
from ._version import __version__
|
|
55
|
+
|
|
56
|
+
__all__ = [
|
|
57
|
+
"__version__",
|
|
58
|
+
# clients
|
|
59
|
+
"EnrichedNow",
|
|
60
|
+
"AsyncEnrichedNow",
|
|
61
|
+
"ScrapeResult",
|
|
62
|
+
"AsyncScrapeResult",
|
|
63
|
+
# errors
|
|
64
|
+
"EnrichedNowError",
|
|
65
|
+
"MissingAPIKeyError",
|
|
66
|
+
"APIConnectionError",
|
|
67
|
+
"APITimeoutError",
|
|
68
|
+
"APIError",
|
|
69
|
+
"AuthenticationError",
|
|
70
|
+
"PermissionDeniedError",
|
|
71
|
+
"NotFoundError",
|
|
72
|
+
"RateLimitError",
|
|
73
|
+
"ValidationError",
|
|
74
|
+
"QuotaExceededError",
|
|
75
|
+
"ServerError",
|
|
76
|
+
"RunFailedError",
|
|
77
|
+
"WaitTimeoutError",
|
|
78
|
+
# pagination
|
|
79
|
+
"SyncPage",
|
|
80
|
+
"AsyncPage",
|
|
81
|
+
"SyncRecordsPage",
|
|
82
|
+
"AsyncRecordsPage",
|
|
83
|
+
"AsyncPaginator",
|
|
84
|
+
# models
|
|
85
|
+
"Model",
|
|
86
|
+
"Health",
|
|
87
|
+
"Scraper",
|
|
88
|
+
"ScraperDetails",
|
|
89
|
+
"Run",
|
|
90
|
+
"RunDiagnostics",
|
|
91
|
+
"Usage",
|
|
92
|
+
"UsageRow",
|
|
93
|
+
"Period",
|
|
94
|
+
"Posting",
|
|
95
|
+
"PostingDetail",
|
|
96
|
+
"Company",
|
|
97
|
+
"CompanyName",
|
|
98
|
+
"CompanyRef",
|
|
99
|
+
"Location",
|
|
100
|
+
"Salary",
|
|
101
|
+
"Source",
|
|
102
|
+
"Classification",
|
|
103
|
+
"History",
|
|
104
|
+
"Event",
|
|
105
|
+
"Change",
|
|
106
|
+
"Sightings",
|
|
107
|
+
"LastRead",
|
|
108
|
+
"PostingCounts",
|
|
109
|
+
]
|
enrichednow/_base.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""What the sync and the async client share: configuration, building a request, reading an error, deciding on a retry."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import email.utils
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import time
|
|
8
|
+
from datetime import date, datetime
|
|
9
|
+
from typing import Any, Callable, Dict, List, Mapping, Optional, Type, TypeVar, Union
|
|
10
|
+
from urllib.parse import quote
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
from ._exceptions import (
|
|
15
|
+
APIConnectionError,
|
|
16
|
+
APIError,
|
|
17
|
+
APITimeoutError,
|
|
18
|
+
AuthenticationError,
|
|
19
|
+
EnrichedNowError,
|
|
20
|
+
MissingAPIKeyError,
|
|
21
|
+
NotFoundError,
|
|
22
|
+
PermissionDeniedError,
|
|
23
|
+
QuotaExceededError,
|
|
24
|
+
RateLimitError,
|
|
25
|
+
ServerError,
|
|
26
|
+
ValidationError,
|
|
27
|
+
)
|
|
28
|
+
from ._pagination import _Page
|
|
29
|
+
from ._version import __version__
|
|
30
|
+
|
|
31
|
+
DEFAULT_BASE_URL = "https://api.enrichednow.com"
|
|
32
|
+
DEFAULT_TIMEOUT = 30.0
|
|
33
|
+
DEFAULT_MAX_RETRIES = 2
|
|
34
|
+
API_KEY_ENV = "ENRICHEDNOW_API_KEY"
|
|
35
|
+
BASE_URL_ENV = "ENRICHEDNOW_BASE_URL"
|
|
36
|
+
|
|
37
|
+
MAX_SERVER_WAIT = 60.0 # the most `wait` the API accepts on starting a run
|
|
38
|
+
WAIT_HEADROOM = 30.0 # how much longer than `wait` the HTTP request itself may take
|
|
39
|
+
DEFAULT_POLL_INTERVAL = 2.0
|
|
40
|
+
DEFAULT_RUN_TIMEOUT = 600.0
|
|
41
|
+
|
|
42
|
+
INITIAL_RETRY_DELAY = 0.5
|
|
43
|
+
MAX_RETRY_DELAY = 8.0
|
|
44
|
+
MAX_RETRY_AFTER = 60.0 # a longer Retry-After is not waited for: the error is raised
|
|
45
|
+
|
|
46
|
+
Timeout = Union[float, httpx.Timeout, None]
|
|
47
|
+
USE_CLIENT_TIMEOUT: Any = object() # "no timeout of its own": the request takes the client's
|
|
48
|
+
Params = Dict[str, Any]
|
|
49
|
+
P = TypeVar("P", bound="_Page[Any]")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def path_segment(value: str) -> str:
|
|
53
|
+
"""A value as one segment of a path: an id with a `/` in it never reaches another address."""
|
|
54
|
+
if not isinstance(value, str) or not value:
|
|
55
|
+
raise ValueError("an id must be a non-empty string")
|
|
56
|
+
return quote(value, safe="")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def query_value(value: Any) -> Any:
|
|
60
|
+
if isinstance(value, bool):
|
|
61
|
+
return "true" if value else "false"
|
|
62
|
+
if isinstance(value, float) and value.is_integer():
|
|
63
|
+
return int(value)
|
|
64
|
+
if isinstance(value, (datetime, date)):
|
|
65
|
+
return value.isoformat()
|
|
66
|
+
if isinstance(value, (list, tuple, set, frozenset)):
|
|
67
|
+
return ",".join(str(item) for item in value)
|
|
68
|
+
return value
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def clean_params(params: Optional[Mapping[str, Any]]) -> Optional[Params]:
|
|
72
|
+
"""Query parameters as sent: what is `None` is left out, booleans are `true`/`false`, dates are ISO 8601."""
|
|
73
|
+
if not params:
|
|
74
|
+
return None
|
|
75
|
+
return {name: query_value(value) for name, value in params.items() if value is not None} or None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def posting_params(
|
|
79
|
+
*,
|
|
80
|
+
company_domain: Optional[str] = None,
|
|
81
|
+
title: Optional[str],
|
|
82
|
+
location: Optional[str],
|
|
83
|
+
country: Optional[str],
|
|
84
|
+
remote: Optional[bool],
|
|
85
|
+
employment_type: Optional[str],
|
|
86
|
+
seniority: Optional[str],
|
|
87
|
+
function: Optional[str],
|
|
88
|
+
status: Optional[str],
|
|
89
|
+
source: Optional[str],
|
|
90
|
+
provenance: Any,
|
|
91
|
+
unmatched: Optional[bool],
|
|
92
|
+
posted_from: Any,
|
|
93
|
+
posted_to: Any,
|
|
94
|
+
first_seen_from: Any,
|
|
95
|
+
first_seen_to: Any,
|
|
96
|
+
last_seen_from: Any,
|
|
97
|
+
last_seen_to: Any,
|
|
98
|
+
sort: Optional[str],
|
|
99
|
+
limit: Optional[int],
|
|
100
|
+
cursor: Optional[str],
|
|
101
|
+
) -> Params:
|
|
102
|
+
return {
|
|
103
|
+
"company_domain": company_domain, "title": title, "location": location, "country": country, "remote": remote,
|
|
104
|
+
"employment_type": employment_type, "seniority": seniority, "function": function, "status": status,
|
|
105
|
+
"source": source, "provenance": provenance, "unmatched": unmatched, "posted_from": posted_from,
|
|
106
|
+
"posted_to": posted_to, "first_seen_from": first_seen_from, "first_seen_to": first_seen_to,
|
|
107
|
+
"last_seen_from": last_seen_from, "last_seen_to": last_seen_to, "sort": sort, "limit": limit, "cursor": cursor,
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def build_page(cls: Type[P], payload: Any, key: str, convert: Optional[Callable[[Any], Any]], fetch: Callable[[Optional[str]], Any]) -> P:
|
|
112
|
+
data: Dict[str, Any] = dict(payload) if isinstance(payload, Mapping) else {}
|
|
113
|
+
rows = data.get(key)
|
|
114
|
+
items: List[Any] = [convert(row) if convert else row for row in rows] if isinstance(rows, list) else []
|
|
115
|
+
cursor = data.get("next_cursor")
|
|
116
|
+
page = cls(items, cursor if isinstance(cursor, str) and cursor else None, data)
|
|
117
|
+
page._fetch = fetch # type: ignore[attr-defined]
|
|
118
|
+
if "expired" in data and hasattr(page, "expired"):
|
|
119
|
+
page.expired = bool(data["expired"])
|
|
120
|
+
return page
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _retry_after(response: httpx.Response) -> Optional[float]:
|
|
124
|
+
value = response.headers.get("retry-after")
|
|
125
|
+
if not value:
|
|
126
|
+
return None
|
|
127
|
+
try:
|
|
128
|
+
return max(0.0, float(value))
|
|
129
|
+
except ValueError:
|
|
130
|
+
pass
|
|
131
|
+
try:
|
|
132
|
+
when = email.utils.parsedate_to_datetime(value)
|
|
133
|
+
except (TypeError, ValueError):
|
|
134
|
+
return None
|
|
135
|
+
return max(0.0, when.timestamp() - time.time())
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def error_from_response(response: httpx.Response) -> APIError:
|
|
139
|
+
"""The typed error for an answer of 4xx or 5xx. The API's errors are `{"error": {"code", "message", "details"}}`."""
|
|
140
|
+
body: Any
|
|
141
|
+
try:
|
|
142
|
+
body = response.json()
|
|
143
|
+
except ValueError:
|
|
144
|
+
body = response.text
|
|
145
|
+
error = body.get("error") if isinstance(body, Mapping) else None
|
|
146
|
+
code = message = None
|
|
147
|
+
details: Mapping[str, Any] = {}
|
|
148
|
+
if isinstance(error, Mapping):
|
|
149
|
+
code = error.get("code") if isinstance(error.get("code"), str) else None
|
|
150
|
+
message = error.get("message") if isinstance(error.get("message"), str) else None
|
|
151
|
+
if isinstance(error.get("details"), Mapping):
|
|
152
|
+
details = error["details"]
|
|
153
|
+
status = response.status_code
|
|
154
|
+
common: Dict[str, Any] = {
|
|
155
|
+
"status_code": status,
|
|
156
|
+
"code": code,
|
|
157
|
+
"details": details,
|
|
158
|
+
"request_id": response.headers.get("x-request-id") or response.headers.get("request-id"),
|
|
159
|
+
"body": body,
|
|
160
|
+
"response": response,
|
|
161
|
+
}
|
|
162
|
+
text = message or f"The API answered {status} {response.reason_phrase}".rstrip() + "."
|
|
163
|
+
if status == 401:
|
|
164
|
+
return AuthenticationError(text, **common)
|
|
165
|
+
if status == 403:
|
|
166
|
+
return PermissionDeniedError(text, **common)
|
|
167
|
+
if status == 404:
|
|
168
|
+
return NotFoundError(text, **common)
|
|
169
|
+
if status == 429:
|
|
170
|
+
return RateLimitError(text, retry_after=_retry_after(response), **common)
|
|
171
|
+
if status == 402:
|
|
172
|
+
return QuotaExceededError(text, **common)
|
|
173
|
+
if 400 <= status < 500:
|
|
174
|
+
return ValidationError(text, **common)
|
|
175
|
+
if status >= 500:
|
|
176
|
+
return ServerError(text, **common)
|
|
177
|
+
return APIError(text, **common)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def error_from_transport(exc: httpx.TransportError) -> APIConnectionError:
|
|
181
|
+
request: Optional[httpx.Request]
|
|
182
|
+
try:
|
|
183
|
+
request = exc.request
|
|
184
|
+
except RuntimeError:
|
|
185
|
+
request = None
|
|
186
|
+
if isinstance(exc, httpx.TimeoutException):
|
|
187
|
+
return APITimeoutError(request=request)
|
|
188
|
+
return APIConnectionError(f"Could not reach the EnrichedNow API: {exc}" if str(exc) else "Could not reach the EnrichedNow API.",
|
|
189
|
+
request=request)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class BaseClient:
|
|
193
|
+
"""Configuration and policy. Neither sends anything: `EnrichedNow` and `AsyncEnrichedNow` do."""
|
|
194
|
+
|
|
195
|
+
def __init__(self, *, api_key: Optional[str], base_url: Optional[str], timeout: Timeout, max_retries: int) -> None:
|
|
196
|
+
key = api_key if api_key is not None else os.environ.get(API_KEY_ENV)
|
|
197
|
+
if not key or not key.strip():
|
|
198
|
+
raise MissingAPIKeyError(
|
|
199
|
+
f"No API key. Pass api_key=... or set the {API_KEY_ENV} environment variable. "
|
|
200
|
+
"Create a key in the console under API keys; it starts with en_live_."
|
|
201
|
+
)
|
|
202
|
+
if not isinstance(max_retries, int) or isinstance(max_retries, bool) or max_retries < 0:
|
|
203
|
+
raise ValueError("max_retries must be a whole number, 0 or more")
|
|
204
|
+
self.api_key = key.strip()
|
|
205
|
+
self.base_url = (base_url or os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL).rstrip("/")
|
|
206
|
+
self.timeout: Timeout = timeout
|
|
207
|
+
self.max_retries = max_retries
|
|
208
|
+
|
|
209
|
+
def __repr__(self) -> str:
|
|
210
|
+
return f"{type(self).__name__}(base_url={self.base_url!r}, api_key='{self.api_key[:8]}…')"
|
|
211
|
+
|
|
212
|
+
@property
|
|
213
|
+
def _headers(self) -> Dict[str, str]:
|
|
214
|
+
return {
|
|
215
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
216
|
+
"Accept": "application/json",
|
|
217
|
+
"User-Agent": f"enrichednow-python/{__version__}",
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
def _url(self, path: str) -> str:
|
|
221
|
+
return f"{self.base_url}{path}"
|
|
222
|
+
|
|
223
|
+
def _timeout_for(self, wait: Optional[float]) -> Timeout:
|
|
224
|
+
"""The HTTP timeout of a request the API holds open for `wait` seconds: never shorter than the hold."""
|
|
225
|
+
if not wait:
|
|
226
|
+
return self.timeout
|
|
227
|
+
needed = float(wait) + WAIT_HEADROOM
|
|
228
|
+
current = self.timeout
|
|
229
|
+
if current is None:
|
|
230
|
+
return None
|
|
231
|
+
if isinstance(current, httpx.Timeout):
|
|
232
|
+
if current.read is None or current.read >= needed:
|
|
233
|
+
return current
|
|
234
|
+
return httpx.Timeout(connect=current.connect, read=needed, write=current.write, pool=current.pool)
|
|
235
|
+
return max(float(current), needed)
|
|
236
|
+
|
|
237
|
+
# --- retries ----------------------------------------------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
def _backoff(self, attempt: int) -> float:
|
|
240
|
+
"""Exponential backoff with jitter: about 0.5 s, 1 s, 2 s, … up to 8 s; each between half of that and all of it."""
|
|
241
|
+
ceiling: float = min(MAX_RETRY_DELAY, INITIAL_RETRY_DELAY * float(2 ** attempt))
|
|
242
|
+
return ceiling * (0.5 + random.random() / 2)
|
|
243
|
+
|
|
244
|
+
def _retry_delay(self, error: EnrichedNowError, *, attempt: int, idempotent: bool) -> Optional[float]:
|
|
245
|
+
"""Seconds to wait before sending the request again, or `None` to raise the error.
|
|
246
|
+
|
|
247
|
+
Retried: a connection that failed, a timeout, 429 and 5xx. A request that is not safe to repeat (starting a
|
|
248
|
+
run) is sent again only when it is certain that the API did not act on it: the connection was never made, or
|
|
249
|
+
the answer was 429 (no run was created).
|
|
250
|
+
"""
|
|
251
|
+
if attempt >= self.max_retries:
|
|
252
|
+
return None
|
|
253
|
+
if isinstance(error, APIConnectionError):
|
|
254
|
+
cause = error.__cause__
|
|
255
|
+
never_sent = isinstance(cause, (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout))
|
|
256
|
+
return self._backoff(attempt) if idempotent or never_sent else None
|
|
257
|
+
if isinstance(error, RateLimitError):
|
|
258
|
+
if error.retry_after is None:
|
|
259
|
+
return self._backoff(attempt)
|
|
260
|
+
return error.retry_after if error.retry_after <= MAX_RETRY_AFTER else None
|
|
261
|
+
if isinstance(error, ServerError):
|
|
262
|
+
if not idempotent:
|
|
263
|
+
return None
|
|
264
|
+
retry_after = _retry_after(error.response) if error.response is not None else None
|
|
265
|
+
if retry_after is not None:
|
|
266
|
+
return retry_after if retry_after <= MAX_RETRY_AFTER else None
|
|
267
|
+
return self._backoff(attempt)
|
|
268
|
+
return None
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def parse_json(response: httpx.Response) -> Any:
|
|
272
|
+
try:
|
|
273
|
+
return response.json()
|
|
274
|
+
except ValueError:
|
|
275
|
+
raise APIError(
|
|
276
|
+
"The API answered with something that is not JSON.",
|
|
277
|
+
status_code=response.status_code,
|
|
278
|
+
body=response.text,
|
|
279
|
+
response=response,
|
|
280
|
+
) from None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def validate_wait(wait: Optional[float]) -> Optional[float]:
|
|
284
|
+
if wait is None:
|
|
285
|
+
return None
|
|
286
|
+
if isinstance(wait, bool) or not isinstance(wait, (int, float)) or not 0 <= wait <= MAX_SERVER_WAIT:
|
|
287
|
+
raise ValueError(f"wait must be a number of seconds from 0 to {MAX_SERVER_WAIT:g}")
|
|
288
|
+
return wait
|
enrichednow/_client.py
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""The two clients: `EnrichedNow` (sync) and `AsyncEnrichedNow` (async), with the same surface."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
import time
|
|
6
|
+
from types import TracebackType
|
|
7
|
+
from typing import Any, Awaitable, Callable, Dict, Iterator, Mapping, Optional, Type
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
from ._base import (
|
|
12
|
+
DEFAULT_MAX_RETRIES,
|
|
13
|
+
DEFAULT_POLL_INTERVAL,
|
|
14
|
+
DEFAULT_RUN_TIMEOUT,
|
|
15
|
+
DEFAULT_TIMEOUT,
|
|
16
|
+
MAX_SERVER_WAIT,
|
|
17
|
+
USE_CLIENT_TIMEOUT,
|
|
18
|
+
BaseClient,
|
|
19
|
+
Timeout,
|
|
20
|
+
clean_params,
|
|
21
|
+
error_from_response,
|
|
22
|
+
error_from_transport,
|
|
23
|
+
parse_json,
|
|
24
|
+
)
|
|
25
|
+
from ._exceptions import EnrichedNowError
|
|
26
|
+
from ._models import Health, Run
|
|
27
|
+
from ._pagination import AsyncPaginator, AsyncRecordsPage, SyncRecordsPage
|
|
28
|
+
from ._resources import (
|
|
29
|
+
AsyncData,
|
|
30
|
+
AsyncRuns,
|
|
31
|
+
AsyncScrapers,
|
|
32
|
+
AsyncUsageResource,
|
|
33
|
+
Data,
|
|
34
|
+
Runs,
|
|
35
|
+
Scrapers,
|
|
36
|
+
UsageResource,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
__all__ = ["EnrichedNow", "AsyncEnrichedNow", "ScrapeResult", "AsyncScrapeResult"]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ScrapeResult:
|
|
43
|
+
"""What `client.scrape()` returns: the run as it ended, and its records.
|
|
44
|
+
|
|
45
|
+
`records` is fetched page by page as it is iterated; iterating the result itself does the same.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(self, client: "EnrichedNow", run: Run) -> None:
|
|
49
|
+
self._client = client
|
|
50
|
+
self.run = run
|
|
51
|
+
self._records: Optional[SyncRecordsPage] = None
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def records(self) -> SyncRecordsPage:
|
|
55
|
+
"""The run's records: iterate for all of them, or use it as the first page."""
|
|
56
|
+
if self._records is None:
|
|
57
|
+
self._records = self._client.runs.records(self.run.id, limit=1000)
|
|
58
|
+
return self._records
|
|
59
|
+
|
|
60
|
+
def __iter__(self) -> Iterator[Dict[str, Any]]:
|
|
61
|
+
return iter(self.records)
|
|
62
|
+
|
|
63
|
+
def __repr__(self) -> str:
|
|
64
|
+
return f"ScrapeResult(run={self.run.id!r}, status={self.run.status!r}, record_count={self.run.record_count!r})"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class AsyncScrapeResult:
|
|
68
|
+
"""What `await client.scrape()` returns: the run as it ended, and its records (`async for record in result.records`)."""
|
|
69
|
+
|
|
70
|
+
def __init__(self, client: "AsyncEnrichedNow", run: Run) -> None:
|
|
71
|
+
self._client = client
|
|
72
|
+
self.run = run
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def records(self) -> AsyncPaginator[Dict[str, Any], AsyncRecordsPage]:
|
|
76
|
+
"""The run's records: `async for` for all of them, or `await` it for the first page."""
|
|
77
|
+
return self._client.runs.records(self.run.id, limit=1000)
|
|
78
|
+
|
|
79
|
+
def __aiter__(self) -> Any:
|
|
80
|
+
return self.records.__aiter__()
|
|
81
|
+
|
|
82
|
+
def __repr__(self) -> str:
|
|
83
|
+
return f"AsyncScrapeResult(run={self.run.id!r}, status={self.run.status!r}, record_count={self.run.record_count!r})"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _server_wait(timeout: Optional[float]) -> float:
|
|
87
|
+
return MAX_SERVER_WAIT if timeout is None else max(0.0, min(MAX_SERVER_WAIT, float(timeout)))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class EnrichedNow(BaseClient):
|
|
91
|
+
"""The EnrichedNow API.
|
|
92
|
+
|
|
93
|
+
client = EnrichedNow() # reads ENRICHEDNOW_API_KEY
|
|
94
|
+
result = client.scrape("ats.board", {"board_url": "https://boards.greenhouse.io/acme"})
|
|
95
|
+
for record in result.records:
|
|
96
|
+
print(record)
|
|
97
|
+
|
|
98
|
+
`timeout` is the HTTP timeout of one request in seconds (or an `httpx.Timeout`). `max_retries` is how many times
|
|
99
|
+
a request that failed in a way worth retrying is sent again. `http_client` is your own `httpx.Client`, for a
|
|
100
|
+
custom transport or TLS settings; the library does not close a client it did not create.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def __init__(
|
|
104
|
+
self,
|
|
105
|
+
api_key: Optional[str] = None,
|
|
106
|
+
*,
|
|
107
|
+
base_url: Optional[str] = None,
|
|
108
|
+
timeout: Timeout = DEFAULT_TIMEOUT,
|
|
109
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
110
|
+
http_client: Optional[httpx.Client] = None,
|
|
111
|
+
) -> None:
|
|
112
|
+
super().__init__(api_key=api_key, base_url=base_url, timeout=timeout, max_retries=max_retries)
|
|
113
|
+
self._owns_http = http_client is None
|
|
114
|
+
self._http = http_client or httpx.Client(follow_redirects=False)
|
|
115
|
+
self._sleep: Callable[[float], None] = time.sleep
|
|
116
|
+
self.scrapers = Scrapers(self)
|
|
117
|
+
self.runs = Runs(self)
|
|
118
|
+
self.usage = UsageResource(self)
|
|
119
|
+
self.data = Data(self)
|
|
120
|
+
|
|
121
|
+
# --- transport ------------------------------------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
def _request(
|
|
124
|
+
self,
|
|
125
|
+
method: str,
|
|
126
|
+
path: str,
|
|
127
|
+
*,
|
|
128
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
129
|
+
json: Any = None,
|
|
130
|
+
idempotent: bool = True,
|
|
131
|
+
timeout: Any = USE_CLIENT_TIMEOUT,
|
|
132
|
+
) -> Any:
|
|
133
|
+
headers = self._headers
|
|
134
|
+
attempt = 0
|
|
135
|
+
while True:
|
|
136
|
+
error: EnrichedNowError
|
|
137
|
+
try:
|
|
138
|
+
response = self._http.request(
|
|
139
|
+
method,
|
|
140
|
+
self._url(path),
|
|
141
|
+
params=clean_params(params),
|
|
142
|
+
json=json,
|
|
143
|
+
headers=headers,
|
|
144
|
+
timeout=self.timeout if timeout is USE_CLIENT_TIMEOUT else timeout,
|
|
145
|
+
)
|
|
146
|
+
except httpx.TransportError as exc:
|
|
147
|
+
error = error_from_transport(exc)
|
|
148
|
+
error.__cause__ = exc
|
|
149
|
+
else:
|
|
150
|
+
if response.is_success:
|
|
151
|
+
return parse_json(response)
|
|
152
|
+
error = error_from_response(response)
|
|
153
|
+
delay = self._retry_delay(error, attempt=attempt, idempotent=idempotent)
|
|
154
|
+
if delay is None:
|
|
155
|
+
raise error
|
|
156
|
+
self._sleep(delay)
|
|
157
|
+
attempt += 1
|
|
158
|
+
|
|
159
|
+
def close(self) -> None:
|
|
160
|
+
"""Close the connections. Not needed when the client is used as a context manager."""
|
|
161
|
+
if self._owns_http:
|
|
162
|
+
self._http.close()
|
|
163
|
+
|
|
164
|
+
def __enter__(self) -> "EnrichedNow":
|
|
165
|
+
return self
|
|
166
|
+
|
|
167
|
+
def __exit__(self, exc_type: Optional[Type[BaseException]], exc: Optional[BaseException], tb: Optional[TracebackType]) -> None:
|
|
168
|
+
self.close()
|
|
169
|
+
|
|
170
|
+
# --- the service ----------------------------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
def health(self) -> Health:
|
|
173
|
+
"""`GET /v1/health`: is the service up."""
|
|
174
|
+
return Health.from_dict(self._request("GET", "/v1/health"))
|
|
175
|
+
|
|
176
|
+
def scrape(
|
|
177
|
+
self,
|
|
178
|
+
scraper_id: str,
|
|
179
|
+
input: Mapping[str, Any],
|
|
180
|
+
*,
|
|
181
|
+
timeout: Optional[float] = DEFAULT_RUN_TIMEOUT,
|
|
182
|
+
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
|
183
|
+
raise_on_failure: bool = True,
|
|
184
|
+
) -> ScrapeResult:
|
|
185
|
+
"""Start a run, wait for it to end, and return it with its records.
|
|
186
|
+
|
|
187
|
+
Raises `RunFailedError` when the run ends `failed`, `unsupported` or `cancelled` (pass
|
|
188
|
+
`raise_on_failure=False` to get the result instead), and `WaitTimeoutError` when it has not ended after
|
|
189
|
+
`timeout` seconds (`None`: wait as long as it takes). A run that ends `partial` is returned: it has records.
|
|
190
|
+
"""
|
|
191
|
+
started = time.monotonic()
|
|
192
|
+
run = self.scrapers.run(scraper_id, input, wait=_server_wait(timeout))
|
|
193
|
+
if run.is_active:
|
|
194
|
+
remaining = None if timeout is None else max(0.0, timeout - (time.monotonic() - started))
|
|
195
|
+
run = self.runs.wait(run.id, timeout=remaining, poll_interval=poll_interval)
|
|
196
|
+
if raise_on_failure:
|
|
197
|
+
run.raise_for_status()
|
|
198
|
+
return ScrapeResult(self, run)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class AsyncEnrichedNow(BaseClient):
|
|
202
|
+
"""The EnrichedNow API, for `asyncio`. The same surface as `EnrichedNow`; every call is awaited.
|
|
203
|
+
|
|
204
|
+
async with AsyncEnrichedNow() as client:
|
|
205
|
+
result = await client.scrape("ats.board", {"board_url": "https://boards.greenhouse.io/acme"})
|
|
206
|
+
async for record in result.records:
|
|
207
|
+
print(record)
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
def __init__(
|
|
211
|
+
self,
|
|
212
|
+
api_key: Optional[str] = None,
|
|
213
|
+
*,
|
|
214
|
+
base_url: Optional[str] = None,
|
|
215
|
+
timeout: Timeout = DEFAULT_TIMEOUT,
|
|
216
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
217
|
+
http_client: Optional[httpx.AsyncClient] = None,
|
|
218
|
+
) -> None:
|
|
219
|
+
super().__init__(api_key=api_key, base_url=base_url, timeout=timeout, max_retries=max_retries)
|
|
220
|
+
self._owns_http = http_client is None
|
|
221
|
+
self._http = http_client or httpx.AsyncClient(follow_redirects=False)
|
|
222
|
+
self._sleep: Callable[[float], Awaitable[None]] = asyncio.sleep
|
|
223
|
+
self.scrapers = AsyncScrapers(self)
|
|
224
|
+
self.runs = AsyncRuns(self)
|
|
225
|
+
self.usage = AsyncUsageResource(self)
|
|
226
|
+
self.data = AsyncData(self)
|
|
227
|
+
|
|
228
|
+
async def _request(
|
|
229
|
+
self,
|
|
230
|
+
method: str,
|
|
231
|
+
path: str,
|
|
232
|
+
*,
|
|
233
|
+
params: Optional[Mapping[str, Any]] = None,
|
|
234
|
+
json: Any = None,
|
|
235
|
+
idempotent: bool = True,
|
|
236
|
+
timeout: Any = USE_CLIENT_TIMEOUT,
|
|
237
|
+
) -> Any:
|
|
238
|
+
headers = self._headers
|
|
239
|
+
attempt = 0
|
|
240
|
+
while True:
|
|
241
|
+
error: EnrichedNowError
|
|
242
|
+
try:
|
|
243
|
+
response = await self._http.request(
|
|
244
|
+
method,
|
|
245
|
+
self._url(path),
|
|
246
|
+
params=clean_params(params),
|
|
247
|
+
json=json,
|
|
248
|
+
headers=headers,
|
|
249
|
+
timeout=self.timeout if timeout is USE_CLIENT_TIMEOUT else timeout,
|
|
250
|
+
)
|
|
251
|
+
except httpx.TransportError as exc:
|
|
252
|
+
error = error_from_transport(exc)
|
|
253
|
+
error.__cause__ = exc
|
|
254
|
+
else:
|
|
255
|
+
if response.is_success:
|
|
256
|
+
return parse_json(response)
|
|
257
|
+
error = error_from_response(response)
|
|
258
|
+
delay = self._retry_delay(error, attempt=attempt, idempotent=idempotent)
|
|
259
|
+
if delay is None:
|
|
260
|
+
raise error
|
|
261
|
+
await self._sleep(delay)
|
|
262
|
+
attempt += 1
|
|
263
|
+
|
|
264
|
+
async def close(self) -> None:
|
|
265
|
+
"""Close the connections. Not needed when the client is used as a context manager."""
|
|
266
|
+
if self._owns_http:
|
|
267
|
+
await self._http.aclose()
|
|
268
|
+
|
|
269
|
+
async def __aenter__(self) -> "AsyncEnrichedNow":
|
|
270
|
+
return self
|
|
271
|
+
|
|
272
|
+
async def __aexit__(self, exc_type: Optional[Type[BaseException]], exc: Optional[BaseException], tb: Optional[TracebackType]) -> None:
|
|
273
|
+
await self.close()
|
|
274
|
+
|
|
275
|
+
async def health(self) -> Health:
|
|
276
|
+
"""`GET /v1/health`: is the service up."""
|
|
277
|
+
return Health.from_dict(await self._request("GET", "/v1/health"))
|
|
278
|
+
|
|
279
|
+
async def scrape(
|
|
280
|
+
self,
|
|
281
|
+
scraper_id: str,
|
|
282
|
+
input: Mapping[str, Any],
|
|
283
|
+
*,
|
|
284
|
+
timeout: Optional[float] = DEFAULT_RUN_TIMEOUT,
|
|
285
|
+
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
|
286
|
+
raise_on_failure: bool = True,
|
|
287
|
+
) -> AsyncScrapeResult:
|
|
288
|
+
"""Start a run, wait for it to end, and return it with its records.
|
|
289
|
+
|
|
290
|
+
Raises `RunFailedError` when the run ends `failed`, `unsupported` or `cancelled` (pass
|
|
291
|
+
`raise_on_failure=False` to get the result instead), and `WaitTimeoutError` when it has not ended after
|
|
292
|
+
`timeout` seconds (`None`: wait as long as it takes). A run that ends `partial` is returned: it has records.
|
|
293
|
+
"""
|
|
294
|
+
started = time.monotonic()
|
|
295
|
+
run = await self.scrapers.run(scraper_id, input, wait=_server_wait(timeout))
|
|
296
|
+
if run.is_active:
|
|
297
|
+
remaining = None if timeout is None else max(0.0, timeout - (time.monotonic() - started))
|
|
298
|
+
run = await self.runs.wait(run.id, timeout=remaining, poll_interval=poll_interval)
|
|
299
|
+
if raise_on_failure:
|
|
300
|
+
run.raise_for_status()
|
|
301
|
+
return AsyncScrapeResult(self, run)
|