oda_reader 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oda_reader-1.7.0/PKG-INFO +736 -0
- oda_reader-1.7.0/README.md +706 -0
- oda_reader-1.7.0/pyproject.toml +227 -0
- oda_reader-1.7.0/src/oda_reader/__init__.py +163 -0
- oda_reader-1.7.0/src/oda_reader/_cache/README.md +556 -0
- oda_reader-1.7.0/src/oda_reader/_cache/__init__.py +84 -0
- oda_reader-1.7.0/src/oda_reader/_cache/config.py +130 -0
- oda_reader-1.7.0/src/oda_reader/_cache/dataframe.py +241 -0
- oda_reader-1.7.0/src/oda_reader/_cache/legacy.py +213 -0
- oda_reader-1.7.0/src/oda_reader/_cache/manager.py +338 -0
- oda_reader-1.7.0/src/oda_reader/_http_primitives.py +146 -0
- oda_reader-1.7.0/src/oda_reader/aiddata.py +96 -0
- oda_reader-1.7.0/src/oda_reader/codelists/__init__.py +63 -0
- oda_reader-1.7.0/src/oda_reader/codelists/_agencies.py +365 -0
- oda_reader-1.7.0/src/oda_reader/codelists/_categories.py +412 -0
- oda_reader-1.7.0/src/oda_reader/codelists/_fetch.py +589 -0
- oda_reader-1.7.0/src/oda_reader/codelists/_parse.py +631 -0
- oda_reader-1.7.0/src/oda_reader/codelists/_reconcile.py +693 -0
- oda_reader-1.7.0/src/oda_reader/codelists/_types.py +238 -0
- oda_reader-1.7.0/src/oda_reader/common.py +334 -0
- oda_reader-1.7.0/src/oda_reader/cpa.py +78 -0
- oda_reader-1.7.0/src/oda_reader/crs.py +175 -0
- oda_reader-1.7.0/src/oda_reader/dac1.py +51 -0
- oda_reader-1.7.0/src/oda_reader/dac2a.py +111 -0
- oda_reader-1.7.0/src/oda_reader/download/__init__.py +0 -0
- oda_reader-1.7.0/src/oda_reader/download/_deflate64.py +53 -0
- oda_reader-1.7.0/src/oda_reader/download/download_tools.py +725 -0
- oda_reader-1.7.0/src/oda_reader/download/query_builder.py +366 -0
- oda_reader-1.7.0/src/oda_reader/download/version_discovery.py +164 -0
- oda_reader-1.7.0/src/oda_reader/exceptions.py +395 -0
- oda_reader-1.7.0/src/oda_reader/multisystem.py +118 -0
- oda_reader-1.7.0/src/oda_reader/py.typed +0 -0
- oda_reader-1.7.0/src/oda_reader/schemas/__init__.py +0 -0
- oda_reader-1.7.0/src/oda_reader/schemas/crs_translation.py +53 -0
- oda_reader-1.7.0/src/oda_reader/schemas/dac1_translation.py +59 -0
- oda_reader-1.7.0/src/oda_reader/schemas/dac2_translation.py +53 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/_provenance.json +14 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/aidData_schema.json +758 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/area_code_corrections.json +10 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/code_prices_corrections.json +4 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/crs_dotstat.json +507 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/dac1_codes_area.json +443 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/dac1_codes_flow_types.json +5 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/dac1_codes_prices.json +4 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/dac1_dotstat.json +152 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/dac2_codes_area.json +445 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/dac2a_dotstat.json +142 -0
- oda_reader-1.7.0/src/oda_reader/schemas/mappings/multisystem_dotstat.json +202 -0
- oda_reader-1.7.0/src/oda_reader/schemas/multisystem_translation.py +43 -0
- oda_reader-1.7.0/src/oda_reader/schemas/schema_tools.py +243 -0
- oda_reader-1.7.0/src/oda_reader/schemas/xml_tools.py +50 -0
- oda_reader-1.7.0/src/oda_reader/tools.py +42 -0
|
@@ -0,0 +1,736 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oda_reader
|
|
3
|
+
Version: 1.7.0
|
|
4
|
+
Summary: A simple package to import ODA data from the OECD's API and AidData's database
|
|
5
|
+
Author: Jorge Rivera
|
|
6
|
+
Author-email: Jorge Rivera <jorge.rivera@one.org>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Dist: filelock>=3.20.3
|
|
20
|
+
Requires-Dist: joblib>=1.4
|
|
21
|
+
Requires-Dist: openpyxl>=3.1.0
|
|
22
|
+
Requires-Dist: pandas>=2.2.0
|
|
23
|
+
Requires-Dist: platformdirs>=4.5.0
|
|
24
|
+
Requires-Dist: pyarrow>=23.0.1
|
|
25
|
+
Requires-Dist: requests>=2.33.0
|
|
26
|
+
Requires-Dist: requests-cache>=1.2.0
|
|
27
|
+
Requires-Dist: inflate64>=1.0.0
|
|
28
|
+
Requires-Python: >=3.11
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
[](https://pypi.org/project/oda-reader/)
|
|
32
|
+
[](https://anaconda.org/channels/conda-forge/packages/oda-reader)
|
|
33
|
+

|
|
34
|
+

|
|
35
|
+
[](https://github.com/psf/black)
|
|
36
|
+
|
|
37
|
+
# ODA Reader
|
|
38
|
+
|
|
39
|
+
The OECD DAC Data Importer
|
|
40
|
+
|
|
41
|
+
**ODA Reader** is a Python package that simplifies access to the **OECD DAC data**, leveraging
|
|
42
|
+
the **OECD data explorer API** and bulk downloads.
|
|
43
|
+
|
|
44
|
+
It allows for easy, programmatic access to OECD DAC data in python. It is designed for policy
|
|
45
|
+
analysts, data analysts, researchers and students who need easy and programmatic access to
|
|
46
|
+
OECD DAC data.
|
|
47
|
+
|
|
48
|
+
This documentation will walk you through how to set up and use ODA Reader.
|
|
49
|
+
|
|
50
|
+
ODA Reader is a project created and maintained by The ONE Campaign.
|
|
51
|
+
|
|
52
|
+
## Table of Contents
|
|
53
|
+
|
|
54
|
+
1. [Getting Started](#getting-started)
|
|
55
|
+
1. [Features](#features)
|
|
56
|
+
1. [Installation](#installation)
|
|
57
|
+
1. [DAC1](#downloading-dac1-data)
|
|
58
|
+
1. [DAC2a](#downloading-dac2a-data)
|
|
59
|
+
1. [CRS](#downloading-crs-data)
|
|
60
|
+
1. [CPA](#downloading-cpa-data)
|
|
61
|
+
1. [Multisystem](#downloading-multisystem-data)
|
|
62
|
+
1. [Using filters](#using-filters)
|
|
63
|
+
1. [Rate limiting](#rate-limiting)
|
|
64
|
+
1. [Contribute](#contributing-to-oda-reader)
|
|
65
|
+
|
|
66
|
+
## Getting Started
|
|
67
|
+
|
|
68
|
+
**ODA Reader** provides a seamless way to access, download, and process data on Official Development Assistance (ODA)
|
|
69
|
+
and Other Official Flows (OOFs) directly from the OECD API.
|
|
70
|
+
|
|
71
|
+
## Features
|
|
72
|
+
|
|
73
|
+
- **Data Download Tools**: Easily download data from DAC1, DAC2a, the CRS and other datasets.
|
|
74
|
+
- **Query Builder**: Construct complex SDMX API queries easily using `QueryBuilder`.
|
|
75
|
+
- **Bulk download** microdata from the CRS, the Multisystem dataset, and other datasets.
|
|
76
|
+
- **Schema Translation**: Translate OECD data to `.stat` schema for easier integration.
|
|
77
|
+
- **Multi-version Support**: Access multiple versions of dataflows (CRS, DAC1, DAC2, etc.).
|
|
78
|
+
- **Rate Limiting**: API calls automatically pause when the limit (20 requests per minute by default) is reached.
|
|
79
|
+
|
|
80
|
+
## Installation
|
|
81
|
+
|
|
82
|
+
The ODA Reader can be installed directly from the Python Package Index (PyPI) using `pip` or from Anaconda.org using `conda`.
|
|
83
|
+
|
|
84
|
+
To install using `pip`:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install oda-reader
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
To install using `conda`:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
conda install oda-reader
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Basic Usage
|
|
97
|
+
|
|
98
|
+
### Downloading DAC1 Data
|
|
99
|
+
|
|
100
|
+
The `download_dac1()` function allows you to download DAC1 data from the data-explorer API.
|
|
101
|
+
It accepts a few different arguments:
|
|
102
|
+
|
|
103
|
+
- `start_year`: An integer like `2018`, specifying the starting year for the data.
|
|
104
|
+
This parameter is optional - if not provided, the starting date for the dataset is used.
|
|
105
|
+
- `end_year`: An integer like `2022`, specifying the end year for the data.
|
|
106
|
+
This parameter is optional - if not provided, the returned data goes up to the most recent year.
|
|
107
|
+
- `filters`: An optional dictionary containing additional filters to include in the API call.
|
|
108
|
+
See the [Using filters](#using-filters) section for more details.
|
|
109
|
+
- `pre_process`: A boolean to specify if light cleaning of the data should be performed.
|
|
110
|
+
If true, columns will be renamed to unique, machine readable names, and empty columns will be removed
|
|
111
|
+
- `dotstat_codes`: A boolean to specify if the API response should be translated to the dotstat schema.
|
|
112
|
+
For this to work, `pre_process` must be true.
|
|
113
|
+
- `dataflow_version`: The specific schema / dataflow version to be used in the API call.
|
|
114
|
+
This is an advanced parameter and should be used only if necessary to override the default.
|
|
115
|
+
|
|
116
|
+
This basic example will get all available data (all donors, all indicators, etc) from 2018 to 2022:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from oda_reader import download_dac1
|
|
120
|
+
|
|
121
|
+
dac1_data = download_dac1(start_year=2018, end_year=2022)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
You can also use filters to, for example, only get data for specific donors (here France and the United States):
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from oda_reader import download_dac1
|
|
128
|
+
|
|
129
|
+
dac1_data = download_dac1(
|
|
130
|
+
start_year=2018, end_year=2022, filters={"donor": ["FRA","USA"]}
|
|
131
|
+
)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The filtering can get quite specific. For example, the following
|
|
135
|
+
query gets the total grant equivalents of loans from France in 2022 in national currency (current prices)
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from oda_reader import download_dac1
|
|
139
|
+
|
|
140
|
+
dac1_data = download_dac1(
|
|
141
|
+
start_year=2022,
|
|
142
|
+
end_year=2022,
|
|
143
|
+
filters={
|
|
144
|
+
"donor": "FRA",
|
|
145
|
+
"measure": "11017",
|
|
146
|
+
"flow_type": "1160",
|
|
147
|
+
"unit_measure": "XDC",
|
|
148
|
+
"price_base": "V",
|
|
149
|
+
},
|
|
150
|
+
)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
By default, ODA Reader performs basic preprocessing of the returned data, and it converts the response to the OECD.Stat schema. These options can be turned off to get the data exactly as returned by the API.
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from oda_reader import download_dac1
|
|
157
|
+
|
|
158
|
+
dac1_data = download_dac1(pre_process=False, dotstat_codes=False)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Pre-processing converts column names to distinct machine-readable names, and it sets the right data types for further analysis with Pandas. The data can also be pre-processed without translating to the OECD.Stat schema.
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
from oda_reader import download_dac1
|
|
165
|
+
|
|
166
|
+
dac1_data = download_dac1(pre_process=True, dotstat_codes=False)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Downloading DAC2a Data
|
|
170
|
+
|
|
171
|
+
The `download_dac2a()` function allows you to download DAC2a data from the data-explorer API.
|
|
172
|
+
It accepts a few different arguments:
|
|
173
|
+
|
|
174
|
+
- `start_year`: An integer like `2018`, specifying the starting year for the data.
|
|
175
|
+
This parameter is optional - if not provided, the starting date for the dataset is used.
|
|
176
|
+
- `end_year`: An integer like `2022`, specifying the end year for the data.
|
|
177
|
+
This parameter is optional - if not provided, the returned data goes up to the most recent year.
|
|
178
|
+
- `filters`: An optional dictionary containing additional filters to include in the API call.
|
|
179
|
+
See the [Using filters](#using-filters) section for more details.
|
|
180
|
+
- `pre_process`: A boolean to specify if light cleaning of the data should be performed.
|
|
181
|
+
If true, columns will be renamed to unique, machine readable names, and empty columns will be removed
|
|
182
|
+
- `dotstat_codes`: A boolean to specify if the API response should be translated to the dotstat schema.
|
|
183
|
+
For this to work, `pre_process` must be true.
|
|
184
|
+
- `dataflow_version`: The specific schema / dataflow version to be used in the API call.
|
|
185
|
+
This is an advanced parameter and should be used only if necessary to override the default.
|
|
186
|
+
|
|
187
|
+
This basic example will get all available data (all donors, all recipients, indicators, etc) from 2018 to 2022:
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from oda_reader import download_dac2a
|
|
191
|
+
|
|
192
|
+
dac2_data = download_dac2a(start_year=2018, end_year=2022)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
You can also use filters to, for example, only get data for specific recipients:
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from oda_reader import download_dac2a
|
|
199
|
+
|
|
200
|
+
dac2_data = download_dac2a(
|
|
201
|
+
start_year=2018, end_year=2022, filters={"recipient": ["TGO", "NGA"]}
|
|
202
|
+
)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
The filtering can get quite specific. For example, the following
|
|
206
|
+
query gets the imputed multilateral aid from the UK to Guatemala and China in 2022 in constant prices.
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from oda_reader import download_dac2a
|
|
210
|
+
|
|
211
|
+
dac2_data = download_dac2a(
|
|
212
|
+
start_year=2022,
|
|
213
|
+
end_year=2022,
|
|
214
|
+
filters={
|
|
215
|
+
"donor": "GBR",
|
|
216
|
+
"recipient": ["GTM","CHN"],
|
|
217
|
+
"measure": "106",
|
|
218
|
+
"price_base": "Q",
|
|
219
|
+
},
|
|
220
|
+
)
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
By default, ODA Reader performs basic preprocessing of the returned data, and it converts the response to the OECD.Stat schema. These options can be turned off to get the data exactly as returned by the API.
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
from oda_reader import download_dac2a
|
|
227
|
+
|
|
228
|
+
dac2_data = download_dac2a(pre_process=False, dotstat_codes=False)
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Pre-processing converts column names to distinct machine-readable names, and it sets the right data types for further analysis with Pandas. The data can also be pre-processed without translating to the OECD.Stat schema.
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
from oda_reader import download_dac2a
|
|
235
|
+
|
|
236
|
+
dac2_data = download_dac2a(pre_process=True, dotstat_codes=False)
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
### Downloading CRS Data
|
|
240
|
+
|
|
241
|
+
The `download_crs()` function allows you to download CRS data from the data-explorer API.
|
|
242
|
+
It accepts a few different arguments:
|
|
243
|
+
|
|
244
|
+
- `start_year`: An integer like `2018`, specifying the starting year for the data.
|
|
245
|
+
This parameter is optional - if not provided, the starting date for the dataset is used.
|
|
246
|
+
- `end_year`: An integer like `2022`, specifying the end year for the data.
|
|
247
|
+
This parameter is optional - if not provided, the returned data goes up to the most recent year.
|
|
248
|
+
- `filters`: An optional dictionary containing additional filters to include in the API call.
|
|
249
|
+
See the [Using filters](#using-filters) section for more details.
|
|
250
|
+
- `pre_process`: A boolean to specify if light cleaning of the data should be performed.
|
|
251
|
+
If true, columns will be renamed to unique, machine readable names, and empty columns will be removed
|
|
252
|
+
- `dotstat_codes`: A boolean to specify if the API response should be translated to the dotstat schema.For this to work, `pre_process` must be true.
|
|
253
|
+
- `as_grant_equivalent`: A boolean to specify whether the 'flows' or 'grant equivalent' version of the CRS should be returned.
|
|
254
|
+
- `dataflow_version`: The specific schema / dataflow version to be used in the API call.
|
|
255
|
+
This is an advanced parameter and should be used only if necessary to override the default.
|
|
256
|
+
|
|
257
|
+
**Note** the `download_crs` function defaults to getting 'microdata'. That means project-level data. This is different from the approach taken by the data-explorer online, which shows semi-aggregated data. In order to view semi-aggregates, you can add `microdata: False` to the filters.
|
|
258
|
+
|
|
259
|
+
This API is quite slow, and the data can quickly get quite large. It is recommended to use filters to limit the data returned, or to use the bulk download feature (`bulk_download_crs`) to avoid repeated, slow calls to the API.
|
|
260
|
+
|
|
261
|
+
This basic example will get all available data (all donors, all recipients, indicators, etc) from 2018 to 2022. It will return 'flows' data.
|
|
262
|
+
|
|
263
|
+
```python
|
|
264
|
+
from oda_reader import download_crs
|
|
265
|
+
|
|
266
|
+
crs_data = download_crs(start_year=2018, end_year=2022)
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
The same example as grant equivalents would be:
|
|
270
|
+
|
|
271
|
+
```python
|
|
272
|
+
from oda_reader import download_crs
|
|
273
|
+
|
|
274
|
+
crs_data = download_crs(start_year=2018, end_year=2022, as_grant_equivalent=True)
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
You can also use filters to, for example, only get data for specific donors and recipients:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
from oda_reader import download_crs
|
|
281
|
+
|
|
282
|
+
crs_data = download_crs(
|
|
283
|
+
start_year=2018, end_year=2022, filters={"donor": "DEU", "recipient": ["TGO", "NGA"]}
|
|
284
|
+
)
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
The filtering can get quite specific. For example, the following
|
|
288
|
+
query gets disbursements for ODA grants from Germany to Nigeria for
|
|
289
|
+
primary education, provided through multilateral organisations, in
|
|
290
|
+
constant prices:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
from oda_reader import download_crs
|
|
294
|
+
|
|
295
|
+
crs_data = download_crs(
|
|
296
|
+
start_year=2022,
|
|
297
|
+
end_year=2022,
|
|
298
|
+
filters={
|
|
299
|
+
"donor": "DEU",
|
|
300
|
+
"recipient": "NGA",
|
|
301
|
+
"sector": "11220",
|
|
302
|
+
"measure":"11",
|
|
303
|
+
"channel": "40000",
|
|
304
|
+
"price_base": "Q",
|
|
305
|
+
},
|
|
306
|
+
)
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
The data-explorer API can also return semi-aggregates, built from the CRS microdata.
|
|
310
|
+
That is the data that is shown online through the data-explorer.
|
|
311
|
+
|
|
312
|
+
You can get that view of the data using the ODA Reader package. However, the filters must
|
|
313
|
+
be used to avoid double counting.
|
|
314
|
+
|
|
315
|
+
For example, to get all ODA from the United States to Liberia in 2019. In this case
|
|
316
|
+
`channel` and `modality` are set to `_T` (which stands for total). Alternatively, it can
|
|
317
|
+
be set to specific channels or modalities to get semi-aggregates for specific channels
|
|
318
|
+
or modalities.
|
|
319
|
+
|
|
320
|
+
```python
|
|
321
|
+
from oda_reader import download_crs
|
|
322
|
+
|
|
323
|
+
crs_data = download_crs(
|
|
324
|
+
start_year=2019,
|
|
325
|
+
end_year=2019,
|
|
326
|
+
filters={
|
|
327
|
+
"donor": "USA",
|
|
328
|
+
"recipient": "LBR",
|
|
329
|
+
"sector": "1000",
|
|
330
|
+
"measure":"100",
|
|
331
|
+
"channel": "_T",
|
|
332
|
+
"modality": "_T",
|
|
333
|
+
"flow_type": "C",
|
|
334
|
+
"price_base": "V",
|
|
335
|
+
"microdata": False,
|
|
336
|
+
},
|
|
337
|
+
)
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
By default, ODA Reader performs basic preprocessing of the returned data, and it converts the response to the OECD.Stat schema. These options can be turned off to get the data exactly as returned by the API.
|
|
341
|
+
|
|
342
|
+
```python
|
|
343
|
+
from oda_reader import download_crs
|
|
344
|
+
|
|
345
|
+
crs_data = download_crs(pre_process=False, dotstat_codes=False)
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
Pre-processing converts column names to distinct machine-readable names, and it sets the right data types for further analysis with Pandas. The data can also be pre-processed without translating to the OECD.Stat schema.
|
|
349
|
+
|
|
350
|
+
```python
|
|
351
|
+
from oda_reader import download_crs
|
|
352
|
+
|
|
353
|
+
crs_data = download_crs(pre_process=True, dotstat_codes=False)
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
#### Bulk downloading CRS data
|
|
357
|
+
|
|
358
|
+
In many situations, downloading the full CRS may be the most efficient way to conduct analysis. For example, when requesting a lot of data, or when all the project information is needed.
|
|
359
|
+
|
|
360
|
+
For those cases, ODA Reader provides tools for getting the bulk download files provided by the OECD.
|
|
361
|
+
The entire CRS is provided as a parquet file (just over 1GB in size). They also provide a 'reduced'
|
|
362
|
+
version which does not include certain columns in order to result in a smaller file.
|
|
363
|
+
|
|
364
|
+
The `bulk_download_crs()` function allows you to download the full CRS data (as a parquet file)
|
|
365
|
+
It accepts a few different arguments:
|
|
366
|
+
|
|
367
|
+
- `save_to_path`: A string or `Path` object specifying a folder where the parquet file should be
|
|
368
|
+
saved. If not provided, `bulk_download_crs` will return a Pandas DataFrame.
|
|
369
|
+
- `reduced_version`: A boolean which defaults to `False`. If `True` smaller file (removing certain
|
|
370
|
+
columns) is downloaded and saved/returned instead.
|
|
371
|
+
- `as_iterator`: If `True` the function yields one `DataFrame` per row group
|
|
372
|
+
instead of returning the entire file at once. This greatly reduces the peak
|
|
373
|
+
memory usage when working with very large files.
|
|
374
|
+
|
|
375
|
+
**Note** that the files provided by the OECD follow the .Stat schema.
|
|
376
|
+
|
|
377
|
+
To save the full parquet file to `example-folder`:
|
|
378
|
+
|
|
379
|
+
```python
|
|
380
|
+
from oda_reader import bulk_download_crs
|
|
381
|
+
|
|
382
|
+
bulk_download_crs(save_to_path="./example-folder/")
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
To process the file iteratively and keep memory usage low:
|
|
386
|
+
|
|
387
|
+
```python
|
|
388
|
+
from oda_reader import bulk_download_crs
|
|
389
|
+
|
|
390
|
+
for chunk in bulk_download_crs(as_iterator=True):
|
|
391
|
+
# process each chunk (a pandas DataFrame)
|
|
392
|
+
...
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
To keep the full file in memory as a Pandas DataFrame:
|
|
396
|
+
|
|
397
|
+
```python
|
|
398
|
+
from oda_reader import bulk_download_crs
|
|
399
|
+
|
|
400
|
+
full_crs = bulk_download_crs()
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
To download the smaller file to `example-folder`:
|
|
404
|
+
|
|
405
|
+
```python
|
|
406
|
+
from oda_reader import bulk_download_crs
|
|
407
|
+
|
|
408
|
+
bulk_download_crs(save_to_path="./example-folder/", reduced_version=True)
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
To keep the smaller file in memory as a Pandas DataFrame:
|
|
412
|
+
|
|
413
|
+
```python
|
|
414
|
+
from oda_reader import bulk_download_crs
|
|
415
|
+
|
|
416
|
+
full_crs = bulk_download_crs(reduced_version=True)
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
The `download_crs_file()` function allows you to download the CRS data for a specific year
|
|
420
|
+
(as a parquet file). It accepts a few different arguments:
|
|
421
|
+
|
|
422
|
+
- `year`: An integer specifying the year needed (e.g 2019).
|
|
423
|
+
- `save_to_path`: A string or `Path` object specifying a folder where the parquet file should be
|
|
424
|
+
saved. If not provided, `download_crs_file` will return a Pandas DataFrame.
|
|
425
|
+
|
|
426
|
+
**Note** that the files provided by the OECD follow the .Stat schema.
|
|
427
|
+
|
|
428
|
+
To save the full parquet file to `example-folder`:
|
|
429
|
+
|
|
430
|
+
```python
|
|
431
|
+
from oda_reader import download_crs_file
|
|
432
|
+
|
|
433
|
+
download_crs_file(year=2022, save_to_path="./example-folder/")
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
For older years, the years are grouped in a single file. For example:
|
|
437
|
+
|
|
438
|
+
- 2004-05
|
|
439
|
+
- 2002-03
|
|
440
|
+
- 2000-01
|
|
441
|
+
- 1995-99
|
|
442
|
+
- 1973-94
|
|
443
|
+
|
|
444
|
+
In those cases, the year string can be passed as `year` to download the file.
|
|
445
|
+
For example, for 1995-1999:
|
|
446
|
+
|
|
447
|
+
```python
|
|
448
|
+
from oda_reader import download_crs_file
|
|
449
|
+
|
|
450
|
+
download_crs_file(year="1995-99", save_to_path="./example-folder/")
|
|
451
|
+
```
|
|
452
|
+
|
|
453
|
+
To keep the file in memory as a Pandas DataFrame, for 2017 for example:
|
|
454
|
+
|
|
455
|
+
```python
|
|
456
|
+
from oda_reader import download_crs_file
|
|
457
|
+
|
|
458
|
+
crs_data = download_crs_file(year=2017)
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
### Downloading CPA Data
|
|
462
|
+
|
|
463
|
+
**Country Programmable Aid (CPA)** is the share of aid that donors programme at country level. ODA Reader
|
|
464
|
+
downloads CPA directly from the OECD SDMX API (dataflow `DSD_CPA@DF_CRS_CPA`). CPA is activity-level
|
|
465
|
+
data and shares the same schema and filter set as the CRS.
|
|
466
|
+
|
|
467
|
+
The `download_cpa()` function accepts the following arguments:
|
|
468
|
+
|
|
469
|
+
- `start_year`: An integer like `2018`, specifying the starting year for the data.
|
|
470
|
+
This parameter is optional - if not provided, the starting date for the dataset is used.
|
|
471
|
+
- `end_year`: An integer like `2022`, specifying the end year for the data.
|
|
472
|
+
This parameter is optional - if not provided, the returned data goes up to the most recent year.
|
|
473
|
+
- `filters`: An optional dictionary containing additional filters to include in the API call.
|
|
474
|
+
See the [Using filters](#using-filters) section for more details.
|
|
475
|
+
- `pre_process`: A boolean to specify if light cleaning of the data should be performed.
|
|
476
|
+
If true, columns will be renamed to unique, machine readable names, and empty columns will be removed.
|
|
477
|
+
- `dotstat_codes`: A boolean to specify if the API response should be translated to the dotstat schema.
|
|
478
|
+
For this to work, `pre_process` must be true.
|
|
479
|
+
- `dataflow_version`: The specific schema / dataflow version to be used in the API call.
|
|
480
|
+
This is an advanced parameter and should be used only if necessary to override the default.
|
|
481
|
+
|
|
482
|
+
**Note** `download_cpa` defaults to microdata (`microdata=True`, i.e. `MD_DIM=DD`), returning
|
|
483
|
+
project-level CPA records — the same default as `download_crs`.
|
|
484
|
+
|
|
485
|
+
This basic example will get CPA data for 2022:
|
|
486
|
+
|
|
487
|
+
```python
|
|
488
|
+
from oda_reader import download_cpa
|
|
489
|
+
|
|
490
|
+
cpa_data = download_cpa(start_year=2022, end_year=2022)
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
You can also use filters to, for example, only get data for a specific donor:
|
|
494
|
+
|
|
495
|
+
```python
|
|
496
|
+
from oda_reader import download_cpa
|
|
497
|
+
|
|
498
|
+
cpa_data = download_cpa(start_year=2022, end_year=2022, filters={"donor": "USA"})
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
The available filters for CPA are the same as for CRS and can be retrieved with
|
|
502
|
+
`get_available_filters("cpa")`.
|
|
503
|
+
|
|
504
|
+
### Downloading Multisystem Data
|
|
505
|
+
|
|
506
|
+
The `download_multisystem()` function allows you to download _Members total use of the
|
|
507
|
+
multilateral system (Multisystem)_ data from the data-explorer API.
|
|
508
|
+
|
|
509
|
+
It accepts a few different arguments:
|
|
510
|
+
|
|
511
|
+
- `start_year`: An integer like `2018`, specifying the starting year for the data.
|
|
512
|
+
This parameter is optional - if not provided, the starting date for the dataset is used.
|
|
513
|
+
- `end_year`: An integer like `2022`, specifying the end year for the data.
|
|
514
|
+
This parameter is optional - if not provided, the returned data goes up to the most recent year.
|
|
515
|
+
- `filters`: An optional dictionary containing additional filters to include in the API call.
|
|
516
|
+
See the [Using filters](#using-filters) section for more details.
|
|
517
|
+
- `pre_process`: A boolean to specify if light cleaning of the data should be performed.
|
|
518
|
+
If true, columns will be renamed to unique, machine readable names, and empty columns will be removed
|
|
519
|
+
- `dotstat_codes`: A boolean to specify if the API response should be translated to the dotstat schema. For this to work, `pre_process` must be true.
|
|
520
|
+
- `dataflow_version`: The specific schema / dataflow version to be used in the API call.
|
|
521
|
+
This is an advanced parameter and should be used only if necessary to override the default.
|
|
522
|
+
|
|
523
|
+
This API is quite slow, and the data can quickly get quite large. It is recommended to use filters to limit the data returned, or to use the bulk download feature (`bulk_download_multisystem`) to avoid repeated, slow calls to the API.
|
|
524
|
+
|
|
525
|
+
This basic example will get all available data (all donors, all recipients, indicators, etc) from 2018 to 2022.
|
|
526
|
+
|
|
527
|
+
```python
|
|
528
|
+
from oda_reader import download_multisystem
|
|
529
|
+
|
|
530
|
+
multisystem = download_multisystem(start_year=2018, end_year=2022)
|
|
531
|
+
```
|
|
532
|
+
|
|
533
|
+
You can also use filters to, for example, only get data for specific donors and recipients:
|
|
534
|
+
|
|
535
|
+
```python
|
|
536
|
+
from oda_reader import download_multisystem
|
|
537
|
+
|
|
538
|
+
multisystem = download_multisystem(
|
|
539
|
+
start_year=2018, end_year=2022, filters={"donor": "ITA", "recipient": ["TGO", "NGA"]}
|
|
540
|
+
)
|
|
541
|
+
```
|
|
542
|
+
|
|
543
|
+
The filtering can get quite specific. For example, the following
|
|
544
|
+
query gets core multilateral contributions from Canada to the World Bank International
|
|
545
|
+
Development Association (IDA), as disbursements in current prices.
|
|
546
|
+
|
|
547
|
+
```python
|
|
548
|
+
from oda_reader import download_multisystem
|
|
549
|
+
|
|
550
|
+
multisystem = download_multisystem(
|
|
551
|
+
start_year=2015,
|
|
552
|
+
end_year=2015,
|
|
553
|
+
filters={
|
|
554
|
+
"donor": "CAN",
|
|
555
|
+
"channel": "44002",
|
|
556
|
+
"flow_type": "D",
|
|
557
|
+
"price_base": "Q"
|
|
558
|
+
},
|
|
559
|
+
)
|
|
560
|
+
```
|
|
561
|
+
|
|
562
|
+
By default, ODA Reader performs basic preprocessing of the returned data, and it converts the response to the OECD.Stat schema. These options can be turned off to get the data exactly as returned by the API.
|
|
563
|
+
|
|
564
|
+
```python
|
|
565
|
+
from oda_reader import download_multisystem
|
|
566
|
+
|
|
567
|
+
multisystem = download_multisystem(pre_process=False, dotstat_codes=False)
|
|
568
|
+
```
|
|
569
|
+
|
|
570
|
+
Pre-processing converts column names to distinct machine-readable names, and it sets the right data types for further analysis with Pandas. The data can also be pre-processed without translating to the OECD.Stat schema.
|
|
571
|
+
|
|
572
|
+
```python
|
|
573
|
+
from oda_reader import download_multisystem
|
|
574
|
+
|
|
575
|
+
multisystem = download_multisystem(pre_process=True, dotstat_codes=False)
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
#### Bulk downloading Multisystem data
|
|
579
|
+
|
|
580
|
+
In many situations, downloading the full Multisystem data may be the most efficient way to conduct analysis. For example, when requesting a lot of data, or when all the project information is needed.
|
|
581
|
+
|
|
582
|
+
For those cases, ODA Reader provides tools for getting the bulk download files provided by the OECD.
|
|
583
|
+
The entire Multisystem dataset is provided as a parquet file.
|
|
584
|
+
|
|
585
|
+
The `bulk_download_multisystem()` function allows you to download the full Multisystem data (as a parquet file).
|
|
586
|
+
|
|
587
|
+
It accepts a couple of arguments:
|
|
588
|
+
|
|
589
|
+
- `save_to_path`: A string or `Path` object specifying a folder where the parquet file should be
|
|
590
|
+
saved. If not provided, `bulk_download_multisystem` will return a Pandas DataFrame.
|
|
591
|
+
- `as_iterator`: If `True` yields `DataFrame` chunks instead of loading the full
|
|
592
|
+
file into memory at once.
|
|
593
|
+
|
|
594
|
+
**Note** that the files provided by the OECD follow the .Stat schema.
|
|
595
|
+
|
|
596
|
+
To save the full parquet file to `example-folder`:
|
|
597
|
+
|
|
598
|
+
```python
|
|
599
|
+
from oda_reader import bulk_download_multisystem
|
|
600
|
+
|
|
601
|
+
bulk_download_multisystem(save_to_path="./example-folder/")
|
|
602
|
+
```
|
|
603
|
+
|
|
604
|
+
To process the data iteratively:
|
|
605
|
+
|
|
606
|
+
```python
|
|
607
|
+
from oda_reader import bulk_download_multisystem
|
|
608
|
+
|
|
609
|
+
for chunk in bulk_download_multisystem(as_iterator=True):
|
|
610
|
+
# process each chunk here
|
|
611
|
+
...
|
|
612
|
+
```
|
|
613
|
+
|
|
614
|
+
To keep the full file in memory as a Pandas DataFrame:
|
|
615
|
+
|
|
616
|
+
```python
|
|
617
|
+
from oda_reader import bulk_download_multisystem
|
|
618
|
+
|
|
619
|
+
full_multisystem = bulk_download_multisystem()
|
|
620
|
+
```
|
|
621
|
+
|
|
622
|
+
## Using filters
|
|
623
|
+
|
|
624
|
+
When using ODA Reader, you can apply filters to refine the data you retrieve from the API. This applies to all tools except for the bulk download functions.
|
|
625
|
+
|
|
626
|
+
Filters allow you to specify subsets of data, making it easy to focus on the information that is most relevant to your needs.
|
|
627
|
+
|
|
628
|
+
Filters are specified as a dictionary, with keys representing the filter categories (such as donor, recipient, sector, etc.) and values representing the criteria to match, provided as single values (like a year, or a code), or lists of values (like multiple donors or multiple sectors).
|
|
629
|
+
|
|
630
|
+
You can use the `get_available_filters()` function to see the available filter parameters that
|
|
631
|
+
can be used for a specific dataset. Note that (for now) all filter values must be provided using
|
|
632
|
+
the data-explorer schema.
|
|
633
|
+
|
|
634
|
+
For example, to get the available filters for DAC1:
|
|
635
|
+
|
|
636
|
+
```python
|
|
637
|
+
from oda_reader import get_available_filters
|
|
638
|
+
|
|
639
|
+
dac1_filters = get_available_filters(source="dac1")
|
|
640
|
+
```
|
|
641
|
+
|
|
642
|
+
By default, the dictionary is also printed. To only return the object without printing,
|
|
643
|
+
you can set `quiet` as `True`.
|
|
644
|
+
|
|
645
|
+
```python
|
|
646
|
+
from oda_reader import get_available_filters
|
|
647
|
+
|
|
648
|
+
dac1_filters = get_available_filters(source="dac1", quiet=True)
|
|
649
|
+
```
|
|
650
|
+
|
|
651
|
+
The same applies to other sources:
|
|
652
|
+
|
|
653
|
+
```python
|
|
654
|
+
from oda_reader import get_available_filters
|
|
655
|
+
|
|
656
|
+
dac2a_filters = get_available_filters(source="dac2a")
|
|
657
|
+
crs_filters = get_available_filters(source="crs")
|
|
658
|
+
multisystem_filters = get_available_filters(source="multisystem")
|
|
659
|
+
```
|
|
660
|
+
|
|
661
|
+
## Rate limiting
|
|
662
|
+
|
|
663
|
+
ODA Reader limits outgoing requests to avoid hitting the OECD API too often.
|
|
664
|
+
Network calls pause automatically when the limit (20 calls per minute by default)
|
|
665
|
+
is reached. The limit can be changed via the `API_RATE_LIMITER` object:
|
|
666
|
+
|
|
667
|
+
```python
|
|
668
|
+
from oda_reader import API_RATE_LIMITER
|
|
669
|
+
|
|
670
|
+
API_RATE_LIMITER.max_calls = 10
|
|
671
|
+
API_RATE_LIMITER.period = 60
|
|
672
|
+
```
|
|
673
|
+
|
|
674
|
+
## Contributing to ODA Reader
|
|
675
|
+
|
|
676
|
+
Thank you for your interest in contributing to ODA Reader. We welcome contributions from everyone to help improve this project.
|
|
677
|
+
|
|
678
|
+
Note that this project is not associated with, nor maintained by, the OECD.
|
|
679
|
+
|
|
680
|
+
### Submitting Ideas and Reporting Issues
|
|
681
|
+
|
|
682
|
+
If you have an idea for a new feature, additional functionality, or if you have encountered a bug, please feel free to submit an issue to initiate a discussion. This helps ensure alignment and prevents duplicated efforts.
|
|
683
|
+
|
|
684
|
+
### Contributing Code
|
|
685
|
+
|
|
686
|
+
To contribute code, you can fork the repository, implement your changes, and then open a pull request (PR). Please ensure that you submit an issue beforehand to discuss your proposed changes.
|
|
687
|
+
|
|
688
|
+
Your contributions are invaluable in making ODA Reader better for everyone.
|
|
689
|
+
|
|
690
|
+
### Development setup
|
|
691
|
+
|
|
692
|
+
This project uses [uv](https://docs.astral.sh/uv/) and follows the
|
|
693
|
+
[`bblocks-projects`](https://github.com/ONEcampaign/bblocks-projects) standard
|
|
694
|
+
(linting with [ruff](https://docs.astral.sh/ruff/), type checking with
|
|
695
|
+
[ty](https://github.com/astral-sh/ty), and pre-commit hooks).
|
|
696
|
+
|
|
697
|
+
```bash
|
|
698
|
+
uv sync --all-groups # install runtime + dev/test/docs dependencies
|
|
699
|
+
uv run pre-commit install # enable the git hooks
|
|
700
|
+
|
|
701
|
+
# Run the quality checks locally (these also run in CI):
|
|
702
|
+
uv run ruff check .
|
|
703
|
+
uv run ruff format --check .
|
|
704
|
+
uv run ty check src/oda_reader
|
|
705
|
+
uv run pytest -m "not integration"
|
|
706
|
+
```
|
|
707
|
+
|
|
708
|
+
### Releasing
|
|
709
|
+
|
|
710
|
+
Releases are published to [PyPI](https://pypi.org/project/oda_reader/)
|
|
711
|
+
automatically by the `release.yml` GitHub Actions workflow using
|
|
712
|
+
[trusted publishing](https://docs.pypi.org/trusted-publishers/) (OIDC — no API
|
|
713
|
+
tokens or secrets). The workflow runs only on tags matching `v*`, builds the
|
|
714
|
+
sdist and wheel with `uv build`, and publishes from the `pypi` deployment
|
|
715
|
+
environment.
|
|
716
|
+
|
|
717
|
+
To cut a release:
|
|
718
|
+
|
|
719
|
+
1. Bump `version` in `pyproject.toml` and add a dated entry to `CHANGELOG.md`.
|
|
720
|
+
|
|
721
|
+
1. Merge those changes to `main`.
|
|
722
|
+
|
|
723
|
+
1. Tag the release and push the tag:
|
|
724
|
+
|
|
725
|
+
```bash
|
|
726
|
+
git tag v1.2.3
|
|
727
|
+
git push origin v1.2.3
|
|
728
|
+
```
|
|
729
|
+
|
|
730
|
+
1. The `release.yml` workflow builds and publishes to PyPI. Watch the run in the
|
|
731
|
+
repository's **Actions** tab.
|
|
732
|
+
|
|
733
|
+
The tag version should match the `version` in `pyproject.toml`. Trusted
|
|
734
|
+
publishing is configured on the PyPI project (publisher: `ONEcampaign/oda_reader`,
|
|
735
|
+
workflow `release.yml`, environment `pypi`); no maintainer credentials are
|
|
736
|
+
needed to publish.
|