enrichednow 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- enrichednow-0.1.0/.github/workflows/ci.yml +58 -0
- enrichednow-0.1.0/.github/workflows/release.yml +64 -0
- enrichednow-0.1.0/.gitignore +13 -0
- enrichednow-0.1.0/LICENSE +21 -0
- enrichednow-0.1.0/PKG-INFO +412 -0
- enrichednow-0.1.0/README.md +383 -0
- enrichednow-0.1.0/SECURITY.md +17 -0
- enrichednow-0.1.0/pyproject.toml +47 -0
- enrichednow-0.1.0/src/enrichednow/__init__.py +109 -0
- enrichednow-0.1.0/src/enrichednow/_base.py +288 -0
- enrichednow-0.1.0/src/enrichednow/_client.py +301 -0
- enrichednow-0.1.0/src/enrichednow/_exceptions.py +196 -0
- enrichednow-0.1.0/src/enrichednow/_models.py +380 -0
- enrichednow-0.1.0/src/enrichednow/_pagination.py +137 -0
- enrichednow-0.1.0/src/enrichednow/_resources.py +495 -0
- enrichednow-0.1.0/src/enrichednow/_version.py +1 -0
- enrichednow-0.1.0/src/enrichednow/py.typed +1 -0
- enrichednow-0.1.0/tests/conftest.py +99 -0
- enrichednow-0.1.0/tests/test_async.py +166 -0
- enrichednow-0.1.0/tests/test_client.py +441 -0
- enrichednow-0.1.0/tests/test_data.py +128 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ci-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
test:
|
|
17
|
+
name: Python ${{ matrix.python }}
|
|
18
|
+
runs-on: ${{ matrix.os }}
|
|
19
|
+
strategy:
|
|
20
|
+
fail-fast: false
|
|
21
|
+
matrix:
|
|
22
|
+
# The oldest and the newest Python the package supports (requires-python and the classifiers).
|
|
23
|
+
include:
|
|
24
|
+
- python: "3.9"
|
|
25
|
+
os: ubuntu-24.04 # the newest runner image that still has a Python 3.9
|
|
26
|
+
- python: "3.13"
|
|
27
|
+
os: ubuntu-latest
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
30
|
+
with:
|
|
31
|
+
persist-credentials: false
|
|
32
|
+
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
|
|
33
|
+
with:
|
|
34
|
+
python-version: ${{ matrix.python }}
|
|
35
|
+
- run: python -m pip install --upgrade pip
|
|
36
|
+
- run: pip install -e ".[dev]"
|
|
37
|
+
- run: pytest
|
|
38
|
+
- name: mypy --strict
|
|
39
|
+
run: mypy
|
|
40
|
+
|
|
41
|
+
build:
|
|
42
|
+
name: Build the wheel
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
46
|
+
with:
|
|
47
|
+
persist-credentials: false
|
|
48
|
+
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
|
|
49
|
+
with:
|
|
50
|
+
python-version: "3.13"
|
|
51
|
+
- run: python -m pip install --upgrade pip build twine
|
|
52
|
+
- run: python -m build
|
|
53
|
+
- run: twine check --strict dist/*
|
|
54
|
+
- name: The wheel installs and imports
|
|
55
|
+
run: |
|
|
56
|
+
python -m venv /tmp/venv
|
|
57
|
+
/tmp/venv/bin/pip install dist/*.whl
|
|
58
|
+
cd /tmp && /tmp/venv/bin/python -c "import enrichednow; print(enrichednow.__version__)"
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Pushing a tag vX.Y.Z publishes that version to PyPI.
|
|
4
|
+
#
|
|
5
|
+
# No token is stored anywhere: PyPI trusts this workflow through OpenID Connect ("trusted publishing"). PyPI must
|
|
6
|
+
# have a trusted publisher for owner "enrichednow", repository "enrichednow-python", workflow "release.yml",
|
|
7
|
+
# environment "pypi". Without one the publish step is refused and nothing is uploaded.
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
tags: ["v*"]
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: release
|
|
17
|
+
cancel-in-progress: false
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
build:
|
|
21
|
+
name: Build
|
|
22
|
+
if: github.repository == 'enrichednow/enrichednow-python'
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
26
|
+
with:
|
|
27
|
+
persist-credentials: false
|
|
28
|
+
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
|
|
29
|
+
with:
|
|
30
|
+
python-version: "3.13"
|
|
31
|
+
- name: The tag is the version in pyproject.toml
|
|
32
|
+
run: |
|
|
33
|
+
version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
|
|
34
|
+
if [ "v$version" != "$GITHUB_REF_NAME" ]; then
|
|
35
|
+
echo "::error::the tag is $GITHUB_REF_NAME but pyproject.toml says $version"
|
|
36
|
+
exit 1
|
|
37
|
+
fi
|
|
38
|
+
- run: python -m pip install --upgrade pip build twine
|
|
39
|
+
- run: pip install -e ".[dev]"
|
|
40
|
+
- run: pytest
|
|
41
|
+
- run: mypy
|
|
42
|
+
- run: python -m build
|
|
43
|
+
- run: twine check --strict dist/*
|
|
44
|
+
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
45
|
+
with:
|
|
46
|
+
name: dist
|
|
47
|
+
path: dist/
|
|
48
|
+
if-no-files-found: error
|
|
49
|
+
|
|
50
|
+
publish:
|
|
51
|
+
name: Publish to PyPI
|
|
52
|
+
needs: build
|
|
53
|
+
runs-on: ubuntu-latest
|
|
54
|
+
environment:
|
|
55
|
+
name: pypi
|
|
56
|
+
url: https://pypi.org/project/enrichednow/
|
|
57
|
+
permissions:
|
|
58
|
+
id-token: write # the OpenID Connect token PyPI checks; this job has no other credential
|
|
59
|
+
steps:
|
|
60
|
+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
61
|
+
with:
|
|
62
|
+
name: dist
|
|
63
|
+
path: dist/
|
|
64
|
+
- uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EnrichedNow
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: enrichednow
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: The official Python library for the EnrichedNow API
|
|
5
|
+
Project-URL: Homepage, https://enrichednow.com
|
|
6
|
+
Project-URL: Documentation, https://docs.enrichednow.com
|
|
7
|
+
Project-URL: Repository, https://github.com/enrichednow/enrichednow-python
|
|
8
|
+
Project-URL: Issues, https://github.com/enrichednow/enrichednow-python/issues
|
|
9
|
+
Author: EnrichedNow
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: api,datasets,enrichednow,jobs
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Requires-Dist: httpx<1,>=0.23
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# EnrichedNow Python library
|
|
31
|
+
|
|
32
|
+
[](https://github.com/enrichednow/enrichednow-python/actions/workflows/ci.yml)
|
|
33
|
+
|
|
34
|
+
The official Python library for the [EnrichedNow API](https://docs.enrichednow.com): run scrapers, read their records,
|
|
35
|
+
and query the job-postings datasets.
|
|
36
|
+
|
|
37
|
+
- Sync and async clients with the same surface
|
|
38
|
+
- Typed models and typed errors; ships `py.typed`
|
|
39
|
+
- Automatic pagination, retries with backoff, and waiting for a run to end
|
|
40
|
+
- One runtime dependency: [`httpx`](https://www.python-httpx.org/)
|
|
41
|
+
- Python 3.9+
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install enrichednow
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Quickstart
|
|
50
|
+
|
|
51
|
+
Create an API key in the console under **API keys** and put it in the environment:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
export ENRICHEDNOW_API_KEY="en_live_..."
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from enrichednow import EnrichedNow
|
|
59
|
+
|
|
60
|
+
client = EnrichedNow() # reads ENRICHEDNOW_API_KEY
|
|
61
|
+
|
|
62
|
+
result = client.scrape("ats.board", {"board_url": "https://boards.greenhouse.io/acme"})
|
|
63
|
+
|
|
64
|
+
print(result.run.status, result.run.record_count)
|
|
65
|
+
for record in result.records:
|
|
66
|
+
print(record["title"])
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
`scrape` starts a run, waits for it to end and hands back the run with its records. Records are fetched page by
|
|
70
|
+
page as you iterate.
|
|
71
|
+
|
|
72
|
+
## The client
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from enrichednow import EnrichedNow
|
|
76
|
+
|
|
77
|
+
client = EnrichedNow(
|
|
78
|
+
api_key="en_live_...", # default: the ENRICHEDNOW_API_KEY environment variable
|
|
79
|
+
base_url="https://api.enrichednow.com", # default
|
|
80
|
+
timeout=30.0, # seconds for one HTTP request (or an httpx.Timeout)
|
|
81
|
+
max_retries=2, # see "Retries"
|
|
82
|
+
)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Without a key, the constructor raises `MissingAPIKeyError`. Keep the key out of your code.
|
|
86
|
+
|
|
87
|
+
The client holds a connection pool. Use it as a context manager, or call `client.close()`, when you are done:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
with EnrichedNow() as client:
|
|
91
|
+
print(client.health().status) # "ok"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
To bring your own `httpx.Client` (custom transport or TLS settings), pass `http_client=...`. The library does not
|
|
95
|
+
close a client it did not create.
|
|
96
|
+
|
|
97
|
+
## Scrapers
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
for scraper in client.scrapers.list(): # the catalogue
|
|
101
|
+
print(scraper.id, scraper.title, scraper.billing_unit)
|
|
102
|
+
|
|
103
|
+
scraper = client.scrapers.get("ats.board") # with JSON Schemas
|
|
104
|
+
print(scraper.input_schema) # what goes in `input`
|
|
105
|
+
print(scraper.record_schema) # the shape of one record
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### Start a run
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
run = client.scrapers.run("careers.company", {"domain": "acme.com"})
|
|
112
|
+
print(run.id, run.status) # "queued"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`wait` (0 to 60 seconds) asks the API to hold the request open until the run ends or the time is up. Either way
|
|
116
|
+
you get the run as it is at that moment, so treat `queued` and `running` as ordinary answers:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
run = client.scrapers.run("careers.company", {"domain": "acme.com"}, wait=60)
|
|
120
|
+
if run.is_active:
|
|
121
|
+
run = client.runs.wait(run.id)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The HTTP timeout of that one request is raised to `wait` + 30 seconds when yours is shorter.
|
|
125
|
+
|
|
126
|
+
## Runs
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
run = client.runs.get("run_...") # the run as it is now
|
|
130
|
+
run = client.runs.wait("run_...", timeout=600, poll_interval=2) # poll until it has ended
|
|
131
|
+
run = client.runs.cancel("run_...") # ask it to stop
|
|
132
|
+
|
|
133
|
+
for run in client.runs.list(scraper_id="ats.board", status="ok"):
|
|
134
|
+
print(run.id, run.created_at, run.record_count)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
A `Run` has `id`, `scraper_id`, `scraper_version`, `status`, `reason`, `input`, `record_count`, `units`,
|
|
138
|
+
`diagnostics`, `created_at`, `started_at` and `finished_at`, plus a few helpers:
|
|
139
|
+
|
|
140
|
+
| | |
|
|
141
|
+
| --- | --- |
|
|
142
|
+
| `run.is_active` | `queued` or `running` |
|
|
143
|
+
| `run.is_terminal` | the run has ended, however it ended |
|
|
144
|
+
| `run.succeeded` | ended `ok` or `partial` |
|
|
145
|
+
| `run.failed` | ended `failed`, `unsupported` or `cancelled` |
|
|
146
|
+
| `run.raise_for_status()` | raises `RunFailedError` when `run.failed`; otherwise returns the run |
|
|
147
|
+
|
|
148
|
+
`runs.wait` returns the run however it ended and raises `WaitTimeoutError` if it is still going after `timeout`
|
|
149
|
+
seconds. The run itself carries on; the error has it on `.run`.
|
|
150
|
+
|
|
151
|
+
Cancelling is cooperative: a run that was already finishing may still complete. Read the status it ended with.
|
|
152
|
+
|
|
153
|
+
### Records
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
for record in client.runs.records(run.id): # every record, page after page
|
|
157
|
+
...
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Records are plain dicts in the shape of the scraper's `record_schema`. They are kept for 7 days after the run
|
|
161
|
+
ended; after that the list is empty and `page.expired` is `True`.
|
|
162
|
+
|
|
163
|
+
## The one-liner
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
result = client.scrape(
|
|
167
|
+
"ats.board",
|
|
168
|
+
{"board_url": "https://boards.greenhouse.io/acme"},
|
|
169
|
+
timeout=600, # give up waiting after this many seconds (None: wait as long as it takes)
|
|
170
|
+
poll_interval=2,
|
|
171
|
+
raise_on_failure=True,
|
|
172
|
+
)
|
|
173
|
+
result.run # the Run, ended
|
|
174
|
+
result.records # iterable of records; `for record in result` works too
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
A run that ends `failed`, `unsupported` or `cancelled` raises `RunFailedError` with `.status`, `.reason` and
|
|
178
|
+
`.run`. Pass `raise_on_failure=False` to get the result instead. A run that ends `partial` is returned: it has
|
|
179
|
+
records, and `result.run.reason` says why it is not complete.
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
from enrichednow import RunFailedError
|
|
183
|
+
|
|
184
|
+
try:
|
|
185
|
+
result = client.scrape("careers.company", {"domain": "acme.com"})
|
|
186
|
+
except RunFailedError as error:
|
|
187
|
+
print(error.status, error.reason, error.run.id)
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
## Usage
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from datetime import date
|
|
194
|
+
|
|
195
|
+
usage = client.usage.get() # this calendar month (UTC), by day
|
|
196
|
+
print(usage.total_units, "of", usage.allowance_units)
|
|
197
|
+
|
|
198
|
+
usage = client.usage.get(from_=date(2026, 11, 1), to="2026-11-18", group_by="scraper")
|
|
199
|
+
for row in usage.rows:
|
|
200
|
+
print(row.key, row.units, row.runs)
|
|
201
|
+
|
|
202
|
+
client.usage.get(product="data") # the data allowance and its usage
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
`group_by` is `day`, `scraper` or `product`. The period's first day is `from_` here (`from` in the API, a reserved
|
|
206
|
+
word in Python); the same goes for `usage.period.from_`.
|
|
207
|
+
|
|
208
|
+
## Datasets
|
|
209
|
+
|
|
210
|
+
The datasets reads live under `client.data`. Each record returned costs one data unit, so stop iterating when you
|
|
211
|
+
have what you need.
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
# Search postings
|
|
215
|
+
for posting in client.data.postings.list(title="software engineer", country="DE", status="open", limit=200):
|
|
216
|
+
print(posting.title_clean, posting.company.name, posting.location.city, posting.url)
|
|
217
|
+
|
|
218
|
+
# One posting, with its description and history
|
|
219
|
+
posting = client.data.postings.get("...")
|
|
220
|
+
print(posting.description, posting.history.events)
|
|
221
|
+
|
|
222
|
+
# Companies
|
|
223
|
+
for company in client.data.companies.list(q="acme", ats="greenhouse"):
|
|
224
|
+
print(company.domain, company.postings.open)
|
|
225
|
+
|
|
226
|
+
company = client.data.companies.get("acme.com")
|
|
227
|
+
for posting in client.data.companies.postings("acme.com", status="open"):
|
|
228
|
+
...
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Posting filters: `company_domain`, `title`, `location`, `country`, `remote`, `employment_type`, `seniority`,
|
|
232
|
+
`function`, `status`, `source`, `provenance`, `unmatched`, `posted_from`, `posted_to`, `first_seen_from`,
|
|
233
|
+
`first_seen_to`, `last_seen_from`, `last_seen_to`, `sort`. Dates and times can be `date` / `datetime` objects or
|
|
234
|
+
ISO 8601 strings; `provenance` can be a string or a list (`["feed", "page"]`).
|
|
235
|
+
|
|
236
|
+
To follow what is new, ask for what changed since you last asked:
|
|
237
|
+
|
|
238
|
+
```python
|
|
239
|
+
new = client.data.postings.list(first_seen_from=last_checked)
|
|
240
|
+
seen = client.data.postings.list(sort="last_seen_at", last_seen_from=last_checked)
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
Two fields are renamed because the models keep the original JSON on `.raw`: the API's `location.raw` and
|
|
244
|
+
`salary.raw` are `location.raw_text` and `salary.raw_text`.
|
|
245
|
+
|
|
246
|
+
## Pagination
|
|
247
|
+
|
|
248
|
+
Every list returns a page. Iterating it yields its items and then those of every later page, fetching each page
|
|
249
|
+
as it is reached:
|
|
250
|
+
|
|
251
|
+
```python
|
|
252
|
+
for run in client.runs.list(limit=100): # `limit` is the size of a page
|
|
253
|
+
...
|
|
254
|
+
|
|
255
|
+
everything = client.runs.records(run_id, limit=1000).to_list()
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
For one page at a time:
|
|
259
|
+
|
|
260
|
+
```python
|
|
261
|
+
page = client.runs.records(run_id, limit=500)
|
|
262
|
+
page.items # this page's records
|
|
263
|
+
page.next_cursor # None on the last page
|
|
264
|
+
page.has_next_page()
|
|
265
|
+
page = page.next_page() # None on the last page
|
|
266
|
+
|
|
267
|
+
for page in client.data.postings.list(status="open").iter_pages():
|
|
268
|
+
save(page.items)
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
To resume later, keep `page.next_cursor` and pass it back as `cursor=` with the same other arguments. A cursor is
|
|
272
|
+
an opaque string; pass it back unchanged.
|
|
273
|
+
|
|
274
|
+
## Errors
|
|
275
|
+
|
|
276
|
+
Two different things can go wrong, and they are reported differently:
|
|
277
|
+
|
|
278
|
+
- **The request failed**: the API answered 4xx or 5xx, or did not answer. The library raises.
|
|
279
|
+
- **The scrape failed**: the request was fine, and the run ended `failed` or `unsupported`. `runs.get` and
|
|
280
|
+
`runs.wait` return that run; `scrape()` and `run.raise_for_status()` raise `RunFailedError`.
|
|
281
|
+
|
|
282
|
+
```text
|
|
283
|
+
EnrichedNowError
|
|
284
|
+
├── MissingAPIKeyError
|
|
285
|
+
├── APIConnectionError the request never got an answer
|
|
286
|
+
│ └── APITimeoutError
|
|
287
|
+
├── APIError the API answered with an error
|
|
288
|
+
│ ├── AuthenticationError 401 unauthorized
|
|
289
|
+
│ │ └── PermissionDeniedError 403 forbidden
|
|
290
|
+
│ ├── NotFoundError 404 not_found
|
|
291
|
+
│ ├── RateLimitError 429 too_many_runs, rate_limited
|
|
292
|
+
│ ├── ValidationError 422 invalid_input, and any other 4xx
|
|
293
|
+
│ │ └── QuotaExceededError 402 quota_exceeded
|
|
294
|
+
│ └── ServerError 5xx internal, data_unavailable
|
|
295
|
+
├── RunFailedError a run ended failed, unsupported or cancelled
|
|
296
|
+
└── WaitTimeoutError a run did not end within the time given
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
An `APIError` carries what the API said:
|
|
300
|
+
|
|
301
|
+
```python
|
|
302
|
+
import enrichednow
|
|
303
|
+
|
|
304
|
+
try:
|
|
305
|
+
client.scrapers.run("ats.board", {"board": "acme"})
|
|
306
|
+
except enrichednow.QuotaExceededError as error:
|
|
307
|
+
print(error.used_units, error.allowance_units)
|
|
308
|
+
except enrichednow.ValidationError as error:
|
|
309
|
+
print(error.status_code) # 422
|
|
310
|
+
print(error.code) # "invalid_input" <- match on this
|
|
311
|
+
print(error.message) # a sentence for a person
|
|
312
|
+
print(error.fields) # [{"path": "input.board_url", "message": "Field required"}, ...]
|
|
313
|
+
except enrichednow.RateLimitError as error:
|
|
314
|
+
print(error.code) # "too_many_runs" or "rate_limited"
|
|
315
|
+
print(error.active_runs, error.max_concurrent_runs, error.retry_after)
|
|
316
|
+
except enrichednow.APIError as error:
|
|
317
|
+
print(error.status_code, error.code, error.message, error.details, error.request_id)
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
| Attribute | |
|
|
321
|
+
| --- | --- |
|
|
322
|
+
| `status_code` | The HTTP status. |
|
|
323
|
+
| `code` | The API's stable error code. |
|
|
324
|
+
| `message` | A sentence for a person. Show it or log it; do not parse it. |
|
|
325
|
+
| `details` | Always a dict. `fields` for `invalid_input`; `used_units` and `allowance_units` for `quota_exceeded`; `active_runs` and `max_concurrent_runs` for `too_many_runs`. |
|
|
326
|
+
| `reason` | `details["reason"]` when the API gives one, such as `email_not_verified` on a 403. |
|
|
327
|
+
| `request_id` | The request id, when the answer has one; otherwise `None`. |
|
|
328
|
+
| `body`, `response` | The parsed body and the `httpx.Response`. |
|
|
329
|
+
|
|
330
|
+
## Retries
|
|
331
|
+
|
|
332
|
+
A request that fails in a way worth retrying is sent again, up to `max_retries` times (default 2), with
|
|
333
|
+
exponential backoff and jitter: about half a second, then a second, then two, up to eight.
|
|
334
|
+
|
|
335
|
+
| Failure | Retried |
|
|
336
|
+
| --- | --- |
|
|
337
|
+
| The connection failed or the request timed out | Yes |
|
|
338
|
+
| `429` | Yes, after the seconds in `Retry-After` when the answer has it |
|
|
339
|
+
| `5xx` | Yes |
|
|
340
|
+
| Any other `4xx` | No: the same request would get the same answer |
|
|
341
|
+
|
|
342
|
+
A `Retry-After` longer than 60 seconds is not waited for: the `RateLimitError` is raised, with the wait on
|
|
343
|
+
`.retry_after`.
|
|
344
|
+
|
|
345
|
+
**Starting a run is the exception.** It is not idempotent: a repeated `POST` starts, and charges for, a second
|
|
346
|
+
run. So `scrapers.run()` and `scrape()` send it again only when no run can exist: the connection was never made,
|
|
347
|
+
or the answer was `429`. After a timeout or a `5xx` the error is raised instead. Look at `client.runs.list()` before
|
|
348
|
+
starting the run again; the first one may well exist.
|
|
349
|
+
|
|
350
|
+
```python
|
|
351
|
+
client = EnrichedNow(max_retries=0) # never retry
|
|
352
|
+
client = EnrichedNow(max_retries=5)
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
## Async
|
|
356
|
+
|
|
357
|
+
`AsyncEnrichedNow` has the same resources, methods and arguments. Every call is awaited; lists are iterated with
|
|
358
|
+
`async for`.
|
|
359
|
+
|
|
360
|
+
```python
|
|
361
|
+
import asyncio
|
|
362
|
+
from enrichednow import AsyncEnrichedNow
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
async def main() -> None:
|
|
366
|
+
async with AsyncEnrichedNow() as client:
|
|
367
|
+
result = await client.scrape("ats.board", {"board_url": "https://boards.greenhouse.io/acme"})
|
|
368
|
+
async for record in result.records:
|
|
369
|
+
print(record["title"])
|
|
370
|
+
|
|
371
|
+
run = await client.scrapers.run("careers.company", {"domain": "acme.com"}, wait=60)
|
|
372
|
+
run = await client.runs.wait(run.id)
|
|
373
|
+
|
|
374
|
+
async for posting in client.data.postings.list(status="open"): # every item, page after page
|
|
375
|
+
...
|
|
376
|
+
|
|
377
|
+
page = await client.runs.list(limit=50) # or one page
|
|
378
|
+
page = await page.next_page()
|
|
379
|
+
|
|
380
|
+
records = await client.runs.records(run.id).to_list()
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
asyncio.run(main())
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
## Models
|
|
387
|
+
|
|
388
|
+
Responses are frozen dataclasses with type hints. Timestamps are `datetime` objects (UTC), days are `date`
|
|
389
|
+
objects. Every model keeps the JSON it came from on `.raw` (also `.to_dict()`), so a field the API adds later is
|
|
390
|
+
there before this library knows its name, and never breaks parsing.
|
|
391
|
+
|
|
392
|
+
```python
|
|
393
|
+
run = client.runs.get("run_...")
|
|
394
|
+
run.finished_at # datetime
|
|
395
|
+
run.diagnostics.facts # dict
|
|
396
|
+
run.raw # the JSON object, as returned
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
## Development
|
|
400
|
+
|
|
401
|
+
```bash
|
|
402
|
+
pip install -e ".[dev]"
|
|
403
|
+
pytest # the tests
|
|
404
|
+
mypy # the strict type check
|
|
405
|
+
python -m build # the wheel and the source distribution, into dist/ (pip install build)
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
The tests run against a mocked transport and make no network calls.
|
|
409
|
+
|
|
410
|
+
## Licence
|
|
411
|
+
|
|
412
|
+
MIT. See [LICENSE](LICENSE). To report a security problem, see [SECURITY.md](SECURITY.md).
|