oalex 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oalex-0.2.0/.gitignore +36 -0
- oalex-0.2.0/CHANGELOG.md +34 -0
- oalex-0.2.0/LICENSE +21 -0
- oalex-0.2.0/PKG-INFO +209 -0
- oalex-0.2.0/README.md +176 -0
- oalex-0.2.0/oalex/__init__.py +34 -0
- oalex-0.2.0/oalex/_backoff.py +103 -0
- oalex-0.2.0/oalex/_cache.py +60 -0
- oalex-0.2.0/oalex/_rate_limit.py +39 -0
- oalex-0.2.0/oalex/client.py +415 -0
- oalex-0.2.0/oalex/errors.py +49 -0
- oalex-0.2.0/oalex/py.typed +0 -0
- oalex-0.2.0/oalex/types.py +252 -0
- oalex-0.2.0/pyproject.toml +76 -0
- oalex-0.2.0/tests/__init__.py +0 -0
- oalex-0.2.0/tests/test_backoff.py +228 -0
- oalex-0.2.0/tests/test_cache.py +83 -0
- oalex-0.2.0/tests/test_client.py +753 -0
- oalex-0.2.0/tests/test_rate_limit.py +55 -0
- oalex-0.2.0/tests/test_types.py +340 -0
oalex-0.2.0/.gitignore
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Python bytecode
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
env/
|
|
16
|
+
|
|
17
|
+
# Test / coverage artifacts
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.mypy_cache/
|
|
20
|
+
.ruff_cache/
|
|
21
|
+
.coverage
|
|
22
|
+
htmlcov/
|
|
23
|
+
.tox/
|
|
24
|
+
|
|
25
|
+
# Editor / OS
|
|
26
|
+
.vscode/
|
|
27
|
+
.idea/
|
|
28
|
+
*.swp
|
|
29
|
+
.DS_Store
|
|
30
|
+
|
|
31
|
+
# Secrets — never commit these
|
|
32
|
+
.env
|
|
33
|
+
.env.*
|
|
34
|
+
*.pem
|
|
35
|
+
*.key
|
|
36
|
+
credentials.json
|
oalex-0.2.0/CHANGELOG.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.2.0 (2026-09-22)
|
|
4
|
+
|
|
5
|
+
OpenAlex retired the mailto polite pool in February 2026 and now meters a daily credit budget. This release catches the client up.
|
|
6
|
+
|
|
7
|
+
### Breaking
|
|
8
|
+
|
|
9
|
+
- A 4xx other than 404 and 429 now raises `OalexRequestError`, which is not a subclass of `OalexUnavailable`. Code that retried on `OalexUnavailable` was retrying bad filters and bad keys forever.
|
|
10
|
+
- `email` is optional. A blank one no longer raises `ValueError`; OpenAlex ignores `mailto` now anyway.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `api_key` argument, falling back to `OPENALEX_API_KEY`. Sent as a Bearer header.
|
|
15
|
+
- `OalexRateLimited` for 429s, with the server's `retry_after`. Subclasses `OalexUnavailable`.
|
|
16
|
+
- `iter_search` for cursor paging past the first page.
|
|
17
|
+
- `fetch_work` accepts `https://openalex.org/W...` URLs; `fetch_doi` accepts a `doi:` prefix.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- A 429 with no credits left (or a `Retry-After` over 30s) is no longer retried.
|
|
22
|
+
- Merged-away ids follow OpenAlex's 301 instead of raising.
|
|
23
|
+
- DOIs containing `#`, `?` or `;` are percent-encoded instead of being truncated.
|
|
24
|
+
- A 200 response that isn't a JSON object is no longer cached for the whole TTL.
|
|
25
|
+
- Retries go through the rate limiter.
|
|
26
|
+
- Concurrent cache writers no longer share a temp file, and a failed cache write logs instead of failing the request.
|
|
27
|
+
|
|
28
|
+
### Build
|
|
29
|
+
|
|
30
|
+
- hatchling instead of setuptools. CI now actually runs Python 3.10 through 3.14 (0.1.0 only ran 3.12).
|
|
31
|
+
|
|
32
|
+
## 0.1.0 (2026-09-22)
|
|
33
|
+
|
|
34
|
+
First release, extracted from research-mcp's OpenAlex adapter.
|
oalex-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 David Burton
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
oalex-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: oalex
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Async Python client for the OpenAlex scholarly works API.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Burton-David/oalex
|
|
6
|
+
Project-URL: Bug Tracker, https://github.com/Burton-David/oalex/issues
|
|
7
|
+
Project-URL: OpenAlex API, https://help.openalex.org/api/
|
|
8
|
+
Author-email: David Burton <david@databurton.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: async,citations,openalex,research,scholarly
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Framework :: AsyncIO
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: httpx>=0.27
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# oalex
|
|
35
|
+
|
|
36
|
+
[](https://github.com/Burton-David/oalex/actions/workflows/ci.yml)
|
|
37
|
+
[](https://www.python.org/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
Async Python client for the [OpenAlex](https://openalex.org/) scholarly works API. Typed, rate-limited, disk-cached, and careful with your daily credit budget.
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import asyncio
|
|
44
|
+
from oalex import Client
|
|
45
|
+
|
|
46
|
+
async def main() -> None:
|
|
47
|
+
async with Client(api_key="your-openalex-key") as oa:
|
|
48
|
+
works = await oa.search("attention is all you need", per_page=5)
|
|
49
|
+
for w in works:
|
|
50
|
+
print(w.id, w.title, w.citation_count)
|
|
51
|
+
|
|
52
|
+
vaswani = await oa.fetch_work("W2626778328")
|
|
53
|
+
if vaswani is not None:
|
|
54
|
+
print(vaswani.title, "cites", len(vaswani.referenced_works), "works")
|
|
55
|
+
|
|
56
|
+
asyncio.run(main())
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Why this exists
|
|
60
|
+
|
|
61
|
+
The OpenAlex HTTP API is clean, but every new client ends up re-solving the same things: rebuilding abstracts from the inverted index, retrying 429s and 5xx without burning the daily budget, following redirects for merged records, and caching so repeat lookups cost nothing. `oalex` does those once.
|
|
62
|
+
|
|
63
|
+
## Features
|
|
64
|
+
|
|
65
|
+
- **Typed.** `Work` and `Author` are frozen dataclasses. `Work.raw` exposes the full payload for fields the typed surface doesn't cover.
|
|
66
|
+
- **Async.** `httpx` under the hood; `async with Client(...)` for clean teardown.
|
|
67
|
+
- **API-key aware.** Pass `api_key=` or set `OPENALEX_API_KEY`. The key travels as a Bearer header, so it never shows up in URLs, cache files, or exception messages.
|
|
68
|
+
- **Rate-limited.** One request per 0.1s by default, well under OpenAlex's 100 requests/second ceiling. Retries wait their turn too.
|
|
69
|
+
- **Disk-cached.** `~/.cache/oalex/` with a 24-hour TTL. Only valid JSON is cached.
|
|
70
|
+
- **Retried.** 1s/2s/4s backoff on 429 and 5xx, honoring `Retry-After` up to 30s. When a 429 says the daily budget is spent (`X-RateLimit-Remaining: 0`, or a `Retry-After` longer than 30s), the client raises `OalexRateLimited` at once instead of sleeping.
|
|
71
|
+
- **Cursor paging.** `iter_search` walks past the first page (and past the 10,000-result offset limit).
|
|
72
|
+
- **Citation graph.** `fetch_referenced` and `fetch_related` resolve OpenAlex's `referenced_works` / `related_works` arrays into full `Work` records.
|
|
73
|
+
|
|
74
|
+
## Install
|
|
75
|
+
|
|
76
|
+
Not on PyPI yet. Install from GitHub:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install git+https://github.com/Burton-David/oalex
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Python 3.10+. The only runtime dependency is [httpx](https://www.python-httpx.org/).
|
|
83
|
+
|
|
84
|
+
## OpenAlex credits
|
|
85
|
+
|
|
86
|
+
Since February 2026 OpenAlex meters usage against a daily budget that resets at midnight UTC. These are the numbers the live API reported in September 2026:
|
|
87
|
+
|
|
88
|
+
| Call | oalex method | Credits |
|
|
89
|
+
|------|--------------|---------|
|
|
90
|
+
| Single work by id or DOI | `fetch_work`, `fetch_doi`, each neighbor in `fetch_referenced` / `fetch_related` | 0 |
|
|
91
|
+
| Search page | `search`, each page of `iter_search` | 10 |
|
|
92
|
+
|
|
93
|
+
A keyless client gets 1,000 credits per day, which is about 100 searches. A [free API key](https://openalex.org/settings/api) raises the budget tenfold. OpenAlex has changed these numbers before; its [pricing page](https://help.openalex.org/access/pricing/) is the source of truth. Cache hits cost nothing.
|
|
94
|
+
|
|
95
|
+
## Usage
|
|
96
|
+
|
|
97
|
+
### Search
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
works = await client.search(
|
|
101
|
+
"graph neural networks",
|
|
102
|
+
per_page=25,
|
|
103
|
+
year_min=2020,
|
|
104
|
+
year_max=2024,
|
|
105
|
+
)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
`search` returns one page. `per_page` is clamped to 1..200. Either year bound can be left off for an open-ended range.
|
|
109
|
+
|
|
110
|
+
For more than one page, iterate:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
async for work in client.iter_search("graph neural networks", max_results=1000):
|
|
114
|
+
print(work.id, work.title)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Each page is a metered call, so set `max_results` on broad queries.
|
|
118
|
+
|
|
119
|
+
### Fetch a single work
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
# By OpenAlex ID: three equivalent forms
|
|
123
|
+
work = await client.fetch_work("W2626778328")
|
|
124
|
+
work = await client.fetch_work("openalex:W2626778328")
|
|
125
|
+
work = await client.fetch_work("https://openalex.org/W2626778328")
|
|
126
|
+
|
|
127
|
+
# By DOI
|
|
128
|
+
work = await client.fetch_doi("10.1038/nature12373")
|
|
129
|
+
work = await client.fetch_doi("https://doi.org/10.1038/nature12373")
|
|
130
|
+
work = await client.fetch_work("doi:10.1038/nature12373")
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`fetch_work` returns `None` for a 404 and for an id prefix it doesn't recognize (`arxiv:`, `pmid:`). A merged-away id follows OpenAlex's redirect and returns the surviving record.
|
|
134
|
+
|
|
135
|
+
### Citation graph
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
# Papers this paper cites
|
|
139
|
+
refs = await client.fetch_referenced("W2626778328", limit=10)
|
|
140
|
+
|
|
141
|
+
# Papers OpenAlex considers similar (topic overlap, not citations)
|
|
142
|
+
related = await client.fetch_related("W2626778328", limit=10)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Neighbors are fetched concurrently, in array order. A neighbor that 404s or keeps failing is skipped, and the next id in the array takes its place. Errors fetching the parent propagate.
|
|
146
|
+
|
|
147
|
+
### Raw payload access
|
|
148
|
+
|
|
149
|
+
`Work.raw` is a read-only mapping of the full OpenAlex response. Reach into it for fields the typed surface doesn't expose:
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
topics = [t["display_name"] for t in work.raw.get("topics", [])]
|
|
153
|
+
source = (work.raw.get("primary_location") or {}).get("source") or {}
|
|
154
|
+
source_id = source.get("id")
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Configuration
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
Client(
|
|
161
|
+
api_key="...", # default: $OPENALEX_API_KEY, else keyless
|
|
162
|
+
cache_dir="/var/cache/oalex", # default: ~/.cache/oalex
|
|
163
|
+
ttl_seconds=24 * 60 * 60, # default: 24h
|
|
164
|
+
min_interval_seconds=0.1, # default: 0.1s between requests
|
|
165
|
+
timeout=30.0, # default: 30s per request
|
|
166
|
+
client=my_httpx_client, # optional: bring your own AsyncClient
|
|
167
|
+
)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
When you pass your own `httpx.AsyncClient`, `oalex` won't close it on exit, and `timeout` is ignored in favor of that client's settings.
|
|
171
|
+
|
|
172
|
+
`email=` is still accepted and sent as `mailto=`. OpenAlex has ignored it since retiring the polite pool, so new code can leave it off.
|
|
173
|
+
|
|
174
|
+
## Errors
|
|
175
|
+
|
|
176
|
+
- `OalexError` is the base class for everything below.
|
|
177
|
+
- `OalexUnavailable` covers transient failures: network errors, 5xx after retries, and a body that isn't a JSON object. Retrying later is reasonable.
|
|
178
|
+
- `OalexRateLimited` is a subclass of `OalexUnavailable` for 429s. `retry_after` holds the server's `Retry-After` hint in seconds when it sent one. A spent daily budget refills at midnight UTC.
|
|
179
|
+
- `OalexRequestError` covers any other 4xx except 404: a bad filter or an invalid API key, for example. `status_code` holds the HTTP status. Retrying the same request won't help.
|
|
180
|
+
|
|
181
|
+
A 404 is not an error: `fetch_work` and `fetch_doi` return `None`, so "OpenAlex doesn't know this id" stays distinct from "OpenAlex is having a bad day."
|
|
182
|
+
|
|
183
|
+
## Limits
|
|
184
|
+
|
|
185
|
+
- Only the `/works` endpoint is wrapped. Authors, sources, and institutions are reachable only through `Work.raw`.
|
|
186
|
+
- The rate limiter is per process. Several processes sharing one API key also share one daily budget, and nothing here coordinates them.
|
|
187
|
+
- The cache stores responses as individual files and never prunes them. Clear `~/.cache/oalex/` yourself if it grows.
|
|
188
|
+
|
|
189
|
+
## Development
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
git clone https://github.com/Burton-David/oalex
|
|
193
|
+
cd oalex
|
|
194
|
+
python -m venv .venv && source .venv/bin/activate
|
|
195
|
+
pip install -e ".[dev]"
|
|
196
|
+
ruff check oalex tests
|
|
197
|
+
mypy oalex
|
|
198
|
+
pytest
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Tests use `httpx.MockTransport` and never touch the live API.
|
|
202
|
+
|
|
203
|
+
## Credits
|
|
204
|
+
|
|
205
|
+
Extracted from [research-mcp](https://github.com/Burton-David/ResearchAssistantMCP)'s OpenAlex source adapter, where the retry policy and disk-cache design were settled first.
|
|
206
|
+
|
|
207
|
+
## License
|
|
208
|
+
|
|
209
|
+
MIT. See `LICENSE`.
|
oalex-0.2.0/README.md
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# oalex
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Burton-David/oalex/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Async Python client for the [OpenAlex](https://openalex.org/) scholarly works API. Typed, rate-limited, disk-cached, and careful with your daily credit budget.
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
import asyncio
|
|
11
|
+
from oalex import Client
|
|
12
|
+
|
|
13
|
+
async def main() -> None:
|
|
14
|
+
async with Client(api_key="your-openalex-key") as oa:
|
|
15
|
+
works = await oa.search("attention is all you need", per_page=5)
|
|
16
|
+
for w in works:
|
|
17
|
+
print(w.id, w.title, w.citation_count)
|
|
18
|
+
|
|
19
|
+
vaswani = await oa.fetch_work("W2626778328")
|
|
20
|
+
if vaswani is not None:
|
|
21
|
+
print(vaswani.title, "cites", len(vaswani.referenced_works), "works")
|
|
22
|
+
|
|
23
|
+
asyncio.run(main())
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Why this exists
|
|
27
|
+
|
|
28
|
+
The OpenAlex HTTP API is clean, but every new client ends up re-solving the same things: rebuilding abstracts from the inverted index, retrying 429s and 5xx without burning the daily budget, following redirects for merged records, and caching so repeat lookups cost nothing. `oalex` does those once.
|
|
29
|
+
|
|
30
|
+
## Features
|
|
31
|
+
|
|
32
|
+
- **Typed.** `Work` and `Author` are frozen dataclasses. `Work.raw` exposes the full payload for fields the typed surface doesn't cover.
|
|
33
|
+
- **Async.** `httpx` under the hood; `async with Client(...)` for clean teardown.
|
|
34
|
+
- **API-key aware.** Pass `api_key=` or set `OPENALEX_API_KEY`. The key travels as a Bearer header, so it never shows up in URLs, cache files, or exception messages.
|
|
35
|
+
- **Rate-limited.** One request per 0.1s by default, well under OpenAlex's 100 requests/second ceiling. Retries wait their turn too.
|
|
36
|
+
- **Disk-cached.** `~/.cache/oalex/` with a 24-hour TTL. Only valid JSON is cached.
|
|
37
|
+
- **Retried.** 1s/2s/4s backoff on 429 and 5xx, honoring `Retry-After` up to 30s. When a 429 says the daily budget is spent (`X-RateLimit-Remaining: 0`, or a `Retry-After` longer than 30s), the client raises `OalexRateLimited` at once instead of sleeping.
|
|
38
|
+
- **Cursor paging.** `iter_search` walks past the first page (and past the 10,000-result offset limit).
|
|
39
|
+
- **Citation graph.** `fetch_referenced` and `fetch_related` resolve OpenAlex's `referenced_works` / `related_works` arrays into full `Work` records.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
Not on PyPI yet. Install from GitHub:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install git+https://github.com/Burton-David/oalex
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Python 3.10+. The only runtime dependency is [httpx](https://www.python-httpx.org/).
|
|
50
|
+
|
|
51
|
+
## OpenAlex credits
|
|
52
|
+
|
|
53
|
+
Since February 2026 OpenAlex meters usage against a daily budget that resets at midnight UTC. These are the numbers the live API reported in September 2026:
|
|
54
|
+
|
|
55
|
+
| Call | oalex method | Credits |
|
|
56
|
+
|------|--------------|---------|
|
|
57
|
+
| Single work by id or DOI | `fetch_work`, `fetch_doi`, each neighbor in `fetch_referenced` / `fetch_related` | 0 |
|
|
58
|
+
| Search page | `search`, each page of `iter_search` | 10 |
|
|
59
|
+
|
|
60
|
+
A keyless client gets 1,000 credits per day, which is about 100 searches. A [free API key](https://openalex.org/settings/api) raises the budget tenfold. OpenAlex has changed these numbers before; its [pricing page](https://help.openalex.org/access/pricing/) is the source of truth. Cache hits cost nothing.
|
|
61
|
+
|
|
62
|
+
## Usage
|
|
63
|
+
|
|
64
|
+
### Search
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
works = await client.search(
|
|
68
|
+
"graph neural networks",
|
|
69
|
+
per_page=25,
|
|
70
|
+
year_min=2020,
|
|
71
|
+
year_max=2024,
|
|
72
|
+
)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`search` returns one page. `per_page` is clamped to 1..200. Either year bound can be left off for an open-ended range.
|
|
76
|
+
|
|
77
|
+
For more than one page, iterate:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
async for work in client.iter_search("graph neural networks", max_results=1000):
|
|
81
|
+
print(work.id, work.title)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Each page is a metered call, so set `max_results` on broad queries.
|
|
85
|
+
|
|
86
|
+
### Fetch a single work
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
# By OpenAlex ID: three equivalent forms
|
|
90
|
+
work = await client.fetch_work("W2626778328")
|
|
91
|
+
work = await client.fetch_work("openalex:W2626778328")
|
|
92
|
+
work = await client.fetch_work("https://openalex.org/W2626778328")
|
|
93
|
+
|
|
94
|
+
# By DOI
|
|
95
|
+
work = await client.fetch_doi("10.1038/nature12373")
|
|
96
|
+
work = await client.fetch_doi("https://doi.org/10.1038/nature12373")
|
|
97
|
+
work = await client.fetch_work("doi:10.1038/nature12373")
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`fetch_work` returns `None` for a 404 and for an id prefix it doesn't recognize (`arxiv:`, `pmid:`). A merged-away id follows OpenAlex's redirect and returns the surviving record.
|
|
101
|
+
|
|
102
|
+
### Citation graph
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
# Papers this paper cites
|
|
106
|
+
refs = await client.fetch_referenced("W2626778328", limit=10)
|
|
107
|
+
|
|
108
|
+
# Papers OpenAlex considers similar (topic overlap, not citations)
|
|
109
|
+
related = await client.fetch_related("W2626778328", limit=10)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Neighbors are fetched concurrently, in array order. A neighbor that 404s or keeps failing is skipped, and the next id in the array takes its place. Errors fetching the parent propagate.
|
|
113
|
+
|
|
114
|
+
### Raw payload access
|
|
115
|
+
|
|
116
|
+
`Work.raw` is a read-only mapping of the full OpenAlex response. Reach into it for fields the typed surface doesn't expose:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
topics = [t["display_name"] for t in work.raw.get("topics", [])]
|
|
120
|
+
source = (work.raw.get("primary_location") or {}).get("source") or {}
|
|
121
|
+
source_id = source.get("id")
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Configuration
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
Client(
|
|
128
|
+
api_key="...", # default: $OPENALEX_API_KEY, else keyless
|
|
129
|
+
cache_dir="/var/cache/oalex", # default: ~/.cache/oalex
|
|
130
|
+
ttl_seconds=24 * 60 * 60, # default: 24h
|
|
131
|
+
min_interval_seconds=0.1, # default: 0.1s between requests
|
|
132
|
+
timeout=30.0, # default: 30s per request
|
|
133
|
+
client=my_httpx_client, # optional: bring your own AsyncClient
|
|
134
|
+
)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
When you pass your own `httpx.AsyncClient`, `oalex` won't close it on exit, and `timeout` is ignored in favor of that client's settings.
|
|
138
|
+
|
|
139
|
+
`email=` is still accepted and sent as `mailto=`. OpenAlex has ignored it since retiring the polite pool, so new code can leave it off.
|
|
140
|
+
|
|
141
|
+
## Errors
|
|
142
|
+
|
|
143
|
+
- `OalexError` is the base class for everything below.
|
|
144
|
+
- `OalexUnavailable` covers transient failures: network errors, 5xx after retries, and a body that isn't a JSON object. Retrying later is reasonable.
|
|
145
|
+
- `OalexRateLimited` is a subclass of `OalexUnavailable` for 429s. `retry_after` holds the server's `Retry-After` hint in seconds when it sent one. A spent daily budget refills at midnight UTC.
|
|
146
|
+
- `OalexRequestError` covers any other 4xx except 404: a bad filter or an invalid API key, for example. `status_code` holds the HTTP status. Retrying the same request won't help.
|
|
147
|
+
|
|
148
|
+
A 404 is not an error: `fetch_work` and `fetch_doi` return `None`, so "OpenAlex doesn't know this id" stays distinct from "OpenAlex is having a bad day."
|
|
149
|
+
|
|
150
|
+
## Limits
|
|
151
|
+
|
|
152
|
+
- Only the `/works` endpoint is wrapped. Authors, sources, and institutions are reachable only through `Work.raw`.
|
|
153
|
+
- The rate limiter is per process. Several processes sharing one API key also share one daily budget, and nothing here coordinates them.
|
|
154
|
+
- The cache stores responses as individual files and never prunes them. Clear `~/.cache/oalex/` yourself if it grows.
|
|
155
|
+
|
|
156
|
+
## Development
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
git clone https://github.com/Burton-David/oalex
|
|
160
|
+
cd oalex
|
|
161
|
+
python -m venv .venv && source .venv/bin/activate
|
|
162
|
+
pip install -e ".[dev]"
|
|
163
|
+
ruff check oalex tests
|
|
164
|
+
mypy oalex
|
|
165
|
+
pytest
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Tests use `httpx.MockTransport` and never touch the live API.
|
|
169
|
+
|
|
170
|
+
## Credits
|
|
171
|
+
|
|
172
|
+
Extracted from [research-mcp](https://github.com/Burton-David/ResearchAssistantMCP)'s OpenAlex source adapter, where the retry policy and disk-cache design were settled first.
|
|
173
|
+
|
|
174
|
+
## License
|
|
175
|
+
|
|
176
|
+
MIT. See `LICENSE`.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""oalex: async Python client for the OpenAlex scholarly works API.
|
|
2
|
+
|
|
3
|
+
Quick start::
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from oalex import Client
|
|
7
|
+
|
|
8
|
+
async def main() -> None:
|
|
9
|
+
async with Client(api_key="...") as oa:
|
|
10
|
+
works = await oa.search("attention is all you need", per_page=5)
|
|
11
|
+
for w in works:
|
|
12
|
+
print(w.id, w.title)
|
|
13
|
+
|
|
14
|
+
asyncio.run(main())
|
|
15
|
+
|
|
16
|
+
See https://help.openalex.org/api/ for the underlying API reference.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from oalex.client import Client
|
|
20
|
+
from oalex.errors import OalexError, OalexRateLimited, OalexRequestError, OalexUnavailable
|
|
21
|
+
from oalex.types import Author, Work, parse_work
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"Author",
|
|
25
|
+
"Client",
|
|
26
|
+
"OalexError",
|
|
27
|
+
"OalexRateLimited",
|
|
28
|
+
"OalexRequestError",
|
|
29
|
+
"OalexUnavailable",
|
|
30
|
+
"Work",
|
|
31
|
+
"parse_work",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""HTTP retry with exponential backoff.
|
|
2
|
+
|
|
3
|
+
OpenAlex returns 429 (rate-limited) and 5xx (server busy) under load.
|
|
4
|
+
Most of these failures are transient; the upstream's Retry-After header
|
|
5
|
+
tells us when the burst has cleared. ``with_backoff`` runs the request,
|
|
6
|
+
retries on retryable statuses (429 + 5xx) and transport errors with a
|
|
7
|
+
1s/2s/4s schedule (three retries, ~7s of waiting in the worst case), and
|
|
8
|
+
honors Retry-After when the server suggests a longer wait.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import logging
|
|
15
|
+
from collections.abc import Awaitable, Callable
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
_log = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
_RETRYABLE_STATUS: frozenset[int] = frozenset({429, 500, 502, 503, 504})
|
|
22
|
+
|
|
23
|
+
# 1s, 2s, 4s: three retries plus the initial attempt gives four tries
|
|
24
|
+
# and a worst-case ~7s wait. Higher caps don't help in practice: when an
|
|
25
|
+
# upstream's 429 cooldown is on the order of minutes, retrying longer
|
|
26
|
+
# just delays the user without succeeding.
|
|
27
|
+
_DEFAULT_DELAYS: tuple[float, ...] = (1.0, 2.0, 4.0)
|
|
28
|
+
|
|
29
|
+
# Retry-After above this means the wait isn't a burst cooldown. On a 429
|
|
30
|
+
# it's almost always the daily credit budget, which resets at midnight UTC,
|
|
31
|
+
# so we hand the response back instead of sleeping on it.
|
|
32
|
+
_MAX_RETRY_AFTER_SECONDS = 30.0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def with_backoff(
|
|
36
|
+
do_request: Callable[[], Awaitable[httpx.Response]],
|
|
37
|
+
*,
|
|
38
|
+
delays: tuple[float, ...] = _DEFAULT_DELAYS,
|
|
39
|
+
) -> httpx.Response:
|
|
40
|
+
"""Run ``do_request()`` with exponential backoff on retryable failures.
|
|
41
|
+
|
|
42
|
+
Returns the final :class:`httpx.Response`, which may still be a 429 or
|
|
43
|
+
5xx once retries run out; the caller decides how to surface it. A
|
|
44
|
+
transport error (connection refused, timeout) is retried the same way
|
|
45
|
+
and re-raised after the last attempt.
|
|
46
|
+
|
|
47
|
+
``delays`` is the sequence of inter-attempt sleeps; total attempts =
|
|
48
|
+
``len(delays) + 1``.
|
|
49
|
+
"""
|
|
50
|
+
last_attempt = len(delays)
|
|
51
|
+
for attempt in range(last_attempt + 1):
|
|
52
|
+
try:
|
|
53
|
+
response = await do_request()
|
|
54
|
+
except httpx.TransportError as exc:
|
|
55
|
+
if attempt == last_attempt:
|
|
56
|
+
raise
|
|
57
|
+
_log.warning(
|
|
58
|
+
"oalex: network error on attempt %d/%d (%s); retrying",
|
|
59
|
+
attempt + 1, last_attempt + 1, exc,
|
|
60
|
+
)
|
|
61
|
+
sleep_for = delays[attempt]
|
|
62
|
+
else:
|
|
63
|
+
if response.status_code not in _RETRYABLE_STATUS or attempt == last_attempt:
|
|
64
|
+
return response
|
|
65
|
+
if _budget_exhausted(response):
|
|
66
|
+
return response
|
|
67
|
+
sleep_for = delays[attempt]
|
|
68
|
+
hint = _parse_retry_after(response.headers.get("retry-after"))
|
|
69
|
+
if hint is not None:
|
|
70
|
+
if hint > _MAX_RETRY_AFTER_SECONDS:
|
|
71
|
+
return response
|
|
72
|
+
sleep_for = max(sleep_for, hint)
|
|
73
|
+
_log.warning(
|
|
74
|
+
"oalex: HTTP %d on attempt %d/%d; retrying",
|
|
75
|
+
response.status_code, attempt + 1, last_attempt + 1,
|
|
76
|
+
)
|
|
77
|
+
_log.info("oalex: backing off %.1fs", sleep_for)
|
|
78
|
+
await asyncio.sleep(sleep_for)
|
|
79
|
+
raise AssertionError("unreachable: the final attempt returns or raises")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _budget_exhausted(response: httpx.Response) -> bool:
|
|
83
|
+
# A 429 also fires for bursts over 100 req/s while credits remain; only
|
|
84
|
+
# a zero balance means waiting seconds is pointless.
|
|
85
|
+
return (
|
|
86
|
+
response.status_code == 429
|
|
87
|
+
and response.headers.get("x-ratelimit-remaining", "").strip() == "0"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _parse_retry_after(value: str | None) -> float | None:
|
|
92
|
+
"""Parse a Retry-After header. Spec allows seconds-as-int OR HTTP-date.
|
|
93
|
+
|
|
94
|
+
We honor seconds. HTTP-date is rare and computing the delta is
|
|
95
|
+
error-prone (timezone, clock skew); falling back to the scheduled
|
|
96
|
+
delay is preferable to parsing it wrong.
|
|
97
|
+
"""
|
|
98
|
+
if value is None:
|
|
99
|
+
return None
|
|
100
|
+
try:
|
|
101
|
+
return float(value.strip())
|
|
102
|
+
except ValueError:
|
|
103
|
+
return None
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Filesystem-backed key/value cache for HTTP responses.
|
|
2
|
+
|
|
3
|
+
Repeated CLI / REPL invocations on the same record waste network time
|
|
4
|
+
and, since OpenAlex started metering list and search calls, daily
|
|
5
|
+
credits too. A small disk cache keyed by the request URL sidesteps
|
|
6
|
+
that. Entries older than ``ttl_seconds`` are treated as misses.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import tempfile
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
_log = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DiskCache:
|
|
22
|
+
"""Filesystem-backed key/value cache. Values are bytes; keys are arbitrary strings."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, directory: str | os.PathLike[str], ttl_seconds: int) -> None:
|
|
25
|
+
self._dir = Path(directory)
|
|
26
|
+
self._dir.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
self._ttl = ttl_seconds
|
|
28
|
+
|
|
29
|
+
def get(self, key: str) -> bytes | None:
|
|
30
|
+
path = self._path_for(key)
|
|
31
|
+
# Another process can prune the directory between stat and read;
|
|
32
|
+
# either way that's a miss, not an error.
|
|
33
|
+
try:
|
|
34
|
+
if (time.time() - path.stat().st_mtime) > self._ttl:
|
|
35
|
+
return None
|
|
36
|
+
return path.read_bytes()
|
|
37
|
+
except FileNotFoundError:
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
def set(self, key: str, value: bytes) -> None:
|
|
41
|
+
path = self._path_for(key)
|
|
42
|
+
# A per-writer temp file: two processes caching the same key must not
|
|
43
|
+
# interleave writes into one shared .tmp before the rename.
|
|
44
|
+
try:
|
|
45
|
+
fd, tmp = tempfile.mkstemp(dir=self._dir, suffix=".tmp")
|
|
46
|
+
try:
|
|
47
|
+
with os.fdopen(fd, "wb") as fh:
|
|
48
|
+
fh.write(value)
|
|
49
|
+
os.replace(tmp, path)
|
|
50
|
+
except BaseException:
|
|
51
|
+
os.unlink(tmp)
|
|
52
|
+
raise
|
|
53
|
+
except OSError as exc:
|
|
54
|
+
# The response is already in hand; a full disk or read-only cache
|
|
55
|
+
# dir shouldn't turn a successful fetch into a failure.
|
|
56
|
+
_log.warning("oalex: could not write cache entry %s: %s", path.name, exc)
|
|
57
|
+
|
|
58
|
+
def _path_for(self, key: str) -> Path:
|
|
59
|
+
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
|
|
60
|
+
return self._dir / f"{digest}.bin"
|