undata 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- undata-0.1.0/.gitignore +18 -0
- undata-0.1.0/CHANGELOG.md +54 -0
- undata-0.1.0/CITATION.cff +18 -0
- undata-0.1.0/LICENSE +21 -0
- undata-0.1.0/PKG-INFO +157 -0
- undata-0.1.0/README.md +122 -0
- undata-0.1.0/examples/01_first_table.py +10 -0
- undata-0.1.0/examples/02_compare_countries.py +8 -0
- undata-0.1.0/examples/03_whole_region.py +8 -0
- undata-0.1.0/examples/04_sdg_target.py +11 -0
- undata-0.1.0/examples/05_missing_data.py +8 -0
- undata-0.1.0/examples/06_names_and_regions.py +11 -0
- undata-0.1.0/examples/07_check_the_search_box.py +9 -0
- undata-0.1.0/examples/08_know_your_source.py +12 -0
- undata-0.1.0/examples/09_save_and_cite.py +8 -0
- undata-0.1.0/examples/10_about_an_indicator.py +5 -0
- undata-0.1.0/pyproject.toml +138 -0
- undata-0.1.0/src/undata/__init__.py +86 -0
- undata-0.1.0/src/undata/_endpoints.py +167 -0
- undata-0.1.0/src/undata/_http.py +210 -0
- undata-0.1.0/src/undata/_session.py +58 -0
- undata-0.1.0/src/undata/_version.py +24 -0
- undata-0.1.0/src/undata/ask.py +98 -0
- undata-0.1.0/src/undata/data.py +238 -0
- undata-0.1.0/src/undata/dcid.py +196 -0
- undata-0.1.0/src/undata/errors.py +149 -0
- undata-0.1.0/src/undata/export.py +92 -0
- undata-0.1.0/src/undata/indicators.py +209 -0
- undata-0.1.0/src/undata/places.py +343 -0
- undata-0.1.0/src/undata/py.typed +0 -0
- undata-0.1.0/src/undata/sdg.py +156 -0
- undata-0.1.0/tests/__init__.py +0 -0
- undata-0.1.0/tests/conftest.py +37 -0
- undata-0.1.0/tests/fake_api.py +202 -0
- undata-0.1.0/tests/live/__init__.py +0 -0
- undata-0.1.0/tests/live/test_friendly_api.py +92 -0
- undata-0.1.0/tests/live/test_platform_contract.py +128 -0
- undata-0.1.0/tests/test_data.py +128 -0
- undata-0.1.0/tests/test_dcid.py +99 -0
- undata-0.1.0/tests/test_endpoints.py +161 -0
- undata-0.1.0/tests/test_http.py +170 -0
- undata-0.1.0/tests/test_indicators.py +111 -0
- undata-0.1.0/tests/test_places.py +148 -0
- undata-0.1.0/tests/test_sdg_ask_export.py +168 -0
undata-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.coverage
|
|
8
|
+
coverage.xml
|
|
9
|
+
htmlcov/
|
|
10
|
+
.pytest_cache/
|
|
11
|
+
.mypy_cache/
|
|
12
|
+
.ruff_cache/
|
|
13
|
+
.hypothesis/
|
|
14
|
+
src/undata/_version.py
|
|
15
|
+
.DS_Store
|
|
16
|
+
|
|
17
|
+
# Local secrets. Never commit.
|
|
18
|
+
.env
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes are recorded here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and versions follow
|
|
5
|
+
[Semantic Versioning](https://semver.org/). Below 1.0, minor versions may change the API.
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] - 2026-10-13
|
|
10
|
+
|
|
11
|
+
First release.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- The friendly API, for people who need numbers rather than an API:
|
|
16
|
+
`un.search`, `un.about`, `un.get`, `un.latest`, `un.countries`, `un.regions`,
|
|
17
|
+
`un.sdg.goals/targets/indicators/series`, `un.ask`, `un.save` and `un.cite`. Tables
|
|
18
|
+
have the same columns every time, with unit and source on every row.
|
|
19
|
+
- Places by name: UN names, everyday names, ISO codes and regions all work, and tables
|
|
20
|
+
use UN M49 short names. Unknown names raise `PlaceNotFoundError` with suggestions;
|
|
21
|
+
names inside several countries raise `AmbiguousPlaceError`. Nothing is substituted.
|
|
22
|
+
- Plain-English warnings: `MissingDataWarning` names countries with no data (never
|
|
23
|
+
zero-filled), `MixedYearsWarning` flags "latest" values from different years, and
|
|
24
|
+
`PlaceGuessedWarning` flags a place data.un.org's search box chose itself, read from
|
|
25
|
+
its own `placeSource` field.
|
|
26
|
+
- `un.save` writes Excel with `data`, `sources` and `no data` sheets, or CSV.
|
|
27
|
+
- Search re-ranks results so names containing all your words come first, and hides
|
|
28
|
+
series Data Commons imported from elsewhere unless `all_sources=True`.
|
|
29
|
+
- Ten examples rewritten on the friendly API, and the examples page rebuilt for
|
|
30
|
+
colleagues new to data work.
|
|
31
|
+
|
|
32
|
+
- Transport layer: single-line JSON bodies, GET-only observations, percent-encoded
|
|
33
|
+
relation expressions, retries on 429/502/503/504 honouring `Retry-After`, and
|
|
34
|
+
`BlockedRequestError`, which explains a 403 instead of implying an API key exists.
|
|
35
|
+
- One wrapper per API route, including graph walks that follow `nextToken` past the
|
|
36
|
+
500-value page cap.
|
|
37
|
+
- The dcid grammar: `parse_dcid`, `build_dcid`, `describe_dcid`, `child_countries_of`.
|
|
38
|
+
- A live contract suite that pins the platform behaviours the package depends on,
|
|
39
|
+
including that variable search ranks Data Commons' older `sdg/...` copy above the
|
|
40
|
+
UN's own `undata/sdg/...` series.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
|
|
44
|
+
- A request mentioning Western Sahara (`ESH`) is refused by the firewall on GET, which
|
|
45
|
+
made any request for Africa fail. Data is now fetched by POST, which it accepts.
|
|
46
|
+
- The 403 message named only the POST triggers; it now lists all three.
|
|
47
|
+
- The platform's "Latin America" region contains no countries, so it is no longer
|
|
48
|
+
offered; "Latin America and the Caribbean" is suggested instead.
|
|
49
|
+
|
|
50
|
+
- Spaces in query strings are sent as `%20`. httpx sends `+`, which this server reads
|
|
51
|
+
literally, so `resolve("Viet Nam")` reached it as `Viet+Nam`.
|
|
52
|
+
|
|
53
|
+
[Unreleased]: https://github.com/nerd-factory/undata/compare/v0.1.0...HEAD
|
|
54
|
+
[0.1.0]: https://github.com/nerd-factory/undata/releases/tag/v0.1.0
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use undata in published work, please cite it as below, and cite the agency that published the data."
|
|
3
|
+
title: "undata: a Python client for the UN System Data Commons"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Islam
|
|
7
|
+
given-names: Mafizul
|
|
8
|
+
- name: "Nerd Lab contributors"
|
|
9
|
+
repository-code: "https://github.com/nerd-factory/undata"
|
|
10
|
+
url: "https://nerd-factory.github.io/undata/"
|
|
11
|
+
version: 0.1.0
|
|
12
|
+
date-released: 2026-10-13
|
|
13
|
+
license: MIT
|
|
14
|
+
keywords:
|
|
15
|
+
- United Nations
|
|
16
|
+
- Data Commons
|
|
17
|
+
- SDG
|
|
18
|
+
- official statistics
|
undata-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mafizul Islam and the Nerd Lab contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
undata-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: undata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: UN data in Python, without the guessing: a friendly, unofficial client for the UN System Data Commons (data.un.org).
|
|
5
|
+
Project-URL: Homepage, https://nerd-factory.github.io/undata/
|
|
6
|
+
Project-URL: Documentation, https://nerd-factory.github.io/undata/
|
|
7
|
+
Project-URL: Source, https://github.com/nerd-factory/undata
|
|
8
|
+
Project-URL: Issues, https://github.com/nerd-factory/undata/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/nerd-factory/undata/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Release notes, https://github.com/nerd-factory/undata/releases
|
|
11
|
+
Project-URL: Field guide, https://mafiatun.github.io/un-data-commons-field-guide/
|
|
12
|
+
Author-email: Mafizul Islam <islam50@un.org>
|
|
13
|
+
Maintainer: Nerd Lab contributors
|
|
14
|
+
License-Expression: MIT
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Keywords: api-client,data-commons,official-statistics,open-data,pandas,sdg,statistics,sustainable-development-goals,un-data,united-nations
|
|
17
|
+
Classifier: Development Status :: 3 - Alpha
|
|
18
|
+
Classifier: Intended Audience :: Developers
|
|
19
|
+
Classifier: Intended Audience :: Science/Research
|
|
20
|
+
Classifier: Natural Language :: English
|
|
21
|
+
Classifier: Operating System :: OS Independent
|
|
22
|
+
Classifier: Programming Language :: Python :: 3
|
|
23
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
27
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
28
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
29
|
+
Classifier: Typing :: Typed
|
|
30
|
+
Requires-Python: >=3.11
|
|
31
|
+
Requires-Dist: httpx>=0.27
|
|
32
|
+
Requires-Dist: openpyxl>=3.1
|
|
33
|
+
Requires-Dist: pandas>=2.2.2
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
<div align="center">
|
|
37
|
+
|
|
38
|
+
<picture>
|
|
39
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/nerd-factory/nerd-lab/main/assets/logo/nerd-lab-offwhite.svg">
|
|
40
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/nerd-factory/nerd-lab/main/assets/logo/nerd-lab-ink.svg">
|
|
41
|
+
<img alt="NERD LAB" src="https://raw.githubusercontent.com/nerd-factory/nerd-lab/main/assets/logo/nerd-lab-offwhite.svg" width="300">
|
|
42
|
+
</picture>
|
|
43
|
+
|
|
44
|
+
# undata
|
|
45
|
+
|
|
46
|
+
**UN data in Python, without the guessing.**
|
|
47
|
+
|
|
48
|
+
[](https://github.com/nerd-factory/undata/actions/workflows/ci.yml)
|
|
49
|
+
[](https://github.com/nerd-factory/undata/actions/workflows/live.yml)
|
|
50
|
+
[](https://github.com/nerd-factory/undata/blob/main/pyproject.toml)
|
|
51
|
+
[](https://github.com/nerd-factory/undata/blob/main/LICENSE)
|
|
52
|
+
|
|
53
|
+
**[Ten examples, with real output →](https://nerd-factory.github.io/undata/)**
|
|
54
|
+
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
Type a country's name, get a table. undata is a Python package for the
|
|
58
|
+
[UN System Data Commons](https://data.un.org), which holds statistics from 26 UN System
|
|
59
|
+
entities. It is written for colleagues who need numbers for a report, not a course in APIs.
|
|
60
|
+
|
|
61
|
+
> **Alpha.** It works today; the API may still change before 1.0.
|
|
62
|
+
> **Unofficial.** A [Nerd Lab](https://github.com/nerd-factory) experiment, not affiliated with
|
|
63
|
+
> or endorsed by the United Nations. The data belongs to the agencies that publish it.
|
|
64
|
+
|
|
65
|
+
## Three lines to a spreadsheet
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install undata
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
import undata as un
|
|
73
|
+
|
|
74
|
+
table = un.get("undata/sdg/SH_STA_MORT.SEX--F", ["Bangladesh", "Viet Nam"])
|
|
75
|
+
un.save(table, "maternal-mortality.xlsx") # data on one sheet, citations on the next
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
In Google Colab, run `!pip install undata` in the first cell. For the latest unreleased
|
|
79
|
+
changes: `pip install "git+https://github.com/nerd-factory/undata"`.
|
|
80
|
+
|
|
81
|
+
Don't know the indicator's id? `un.search("maternal mortality")` lists matching indicators
|
|
82
|
+
with their names. Copy the id from the first column.
|
|
83
|
+
|
|
84
|
+
## Everything on one screen
|
|
85
|
+
|
|
86
|
+
| Function | What it does |
|
|
87
|
+
|---|---|
|
|
88
|
+
| `un.search("words")` | Find indicators by what they measure. UN collections only, best match first. |
|
|
89
|
+
| `un.about(indicator)` | What an indicator is, whose it is, its unit, and which years it covers. |
|
|
90
|
+
| `un.get(indicator, countries, years)` | The numbers: one row per country and year, with unit and source. |
|
|
91
|
+
| `un.latest(indicator, countries)` | Each country's most recent value, with a warning if those years differ. |
|
|
92
|
+
| `un.countries(region)` · `un.regions()` | The names it knows, with UN names and ISO codes. |
|
|
93
|
+
| `un.sdg.goals()` · `targets(16)` · `indicators("16.1")` · `series("16.1.2")` | Walk from an SDG goal down to indicators you can fetch. |
|
|
94
|
+
| `un.ask("a question")` | See what data.un.org's search box decided, including whether it guessed the place. |
|
|
95
|
+
| `un.save(table, "file.xlsx")` · `un.cite(table)` | Save to Excel or CSV with sources; write citations for a report. |
|
|
96
|
+
|
|
97
|
+
Wherever a function asks for countries you can write names the way you would in an email
|
|
98
|
+
(`"Viet Nam"`, `"Vietnam"`, `"VNM"`), a list of them, or a region such as `"Southern Asia"`
|
|
99
|
+
or `"Africa"`. Every function explains itself: `help(un.get)`.
|
|
100
|
+
|
|
101
|
+
Every table has the same columns:
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
country iso3 year value unit indicator_name indicator source source_url
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## The careful parts are built in
|
|
108
|
+
|
|
109
|
+
- **It never guesses a place.** data.un.org's search box answers a question about "Bengal"
|
|
110
|
+
with figures for the United States. undata stops and says it does not know "Bengal".
|
|
111
|
+
- **Gaps are named, never zero.** Countries without data are left out of the table and listed
|
|
112
|
+
in a `MissingDataWarning`, so a chart cannot show them as zeros.
|
|
113
|
+
- **UN names in, UN names out.** Tables say Türkiye, Viet Nam and Côte d'Ivoire, whatever
|
|
114
|
+
spelling you typed.
|
|
115
|
+
- **The source is on every row.** The same statistic can come from several agencies with
|
|
116
|
+
different numbers.
|
|
117
|
+
- **Mixed years are flagged.** "Latest" means each country's own latest year; a
|
|
118
|
+
`MixedYearsWarning` says when those differ.
|
|
119
|
+
- **It works where other clients fail.** The platform's firewall blocks Google's official
|
|
120
|
+
`datacommons-client` (any POST body containing `"select"`) and any GET request that
|
|
121
|
+
mentions Western Sahara (`ESH`, mistaken for a shell command). undata routes around both.
|
|
122
|
+
|
|
123
|
+
Each behaviour is pinned by a test in [tests/live/](https://github.com/nerd-factory/undata/tree/main/tests/live) that runs against the real
|
|
124
|
+
platform every night and opens an issue if the platform changes. Most of them were found
|
|
125
|
+
while writing the [UN Data Commons Field Guide](https://mafiatun.github.io/un-data-commons-field-guide/).
|
|
126
|
+
|
|
127
|
+
## Examples
|
|
128
|
+
|
|
129
|
+
Ten runnable scripts in [examples/](https://github.com/nerd-factory/undata/tree/main/examples): a first table, comparing a region, a whole
|
|
130
|
+
continent, an SDG target to its numbers, gaps, names, the search box, sources, Excel and
|
|
131
|
+
citations, and checking an indicator. They run against the live platform every night, and
|
|
132
|
+
[the examples page](https://nerd-factory.github.io/undata/) shows each with its real output.
|
|
133
|
+
|
|
134
|
+
## For developers
|
|
135
|
+
|
|
136
|
+
The friendly functions sit on three layers you can use directly: `undata.Transport` (the
|
|
137
|
+
only code that touches the network), `undata._endpoints` (one thin wrapper per API route),
|
|
138
|
+
and `undata.dcid` (the identifier grammar). [PLAN.md](https://github.com/nerd-factory/undata/blob/main/PLAN.md) covers the design and the
|
|
139
|
+
road to 1.0.
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
git clone https://github.com/nerd-factory/undata && cd undata
|
|
143
|
+
uv sync # Python 3.11+, installs dev tools
|
|
144
|
+
uv run pytest # unit tests and doctests, offline
|
|
145
|
+
uv run pytest -m live # pins the live platform's behaviour (network)
|
|
146
|
+
uv run ruff check . && uv run ruff format --check . && uv run mypy
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
See [CONTRIBUTING.md](https://github.com/nerd-factory/undata/blob/main/CONTRIBUTING.md). Questions that feel too basic are welcome in
|
|
150
|
+
[Nerd Lab Discussions](https://github.com/nerd-factory/nerd-lab/discussions); they are not
|
|
151
|
+
too basic.
|
|
152
|
+
|
|
153
|
+
## Licence
|
|
154
|
+
|
|
155
|
+
Code: [MIT](https://github.com/nerd-factory/undata/blob/main/LICENSE). Data fetched with it is governed by the
|
|
156
|
+
[UN Data Commons terms of use](https://data.un.org/undatacommons/terms-of-use) and by each
|
|
157
|
+
contributing agency's own terms.
|
undata-0.1.0/README.md
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/nerd-factory/nerd-lab/main/assets/logo/nerd-lab-offwhite.svg">
|
|
5
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/nerd-factory/nerd-lab/main/assets/logo/nerd-lab-ink.svg">
|
|
6
|
+
<img alt="NERD LAB" src="https://raw.githubusercontent.com/nerd-factory/nerd-lab/main/assets/logo/nerd-lab-offwhite.svg" width="300">
|
|
7
|
+
</picture>
|
|
8
|
+
|
|
9
|
+
# undata
|
|
10
|
+
|
|
11
|
+
**UN data in Python, without the guessing.**
|
|
12
|
+
|
|
13
|
+
[](https://github.com/nerd-factory/undata/actions/workflows/ci.yml)
|
|
14
|
+
[](https://github.com/nerd-factory/undata/actions/workflows/live.yml)
|
|
15
|
+
[](https://github.com/nerd-factory/undata/blob/main/pyproject.toml)
|
|
16
|
+
[](https://github.com/nerd-factory/undata/blob/main/LICENSE)
|
|
17
|
+
|
|
18
|
+
**[Ten examples, with real output →](https://nerd-factory.github.io/undata/)**
|
|
19
|
+
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
Type a country's name, get a table. undata is a Python package for the
|
|
23
|
+
[UN System Data Commons](https://data.un.org), which holds statistics from 26 UN System
|
|
24
|
+
entities. It is written for colleagues who need numbers for a report, not a course in APIs.
|
|
25
|
+
|
|
26
|
+
> **Alpha.** It works today; the API may still change before 1.0.
|
|
27
|
+
> **Unofficial.** A [Nerd Lab](https://github.com/nerd-factory) experiment, not affiliated with
|
|
28
|
+
> or endorsed by the United Nations. The data belongs to the agencies that publish it.
|
|
29
|
+
|
|
30
|
+
## Three lines to a spreadsheet
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install undata
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
import undata as un
|
|
38
|
+
|
|
39
|
+
table = un.get("undata/sdg/SH_STA_MORT.SEX--F", ["Bangladesh", "Viet Nam"])
|
|
40
|
+
un.save(table, "maternal-mortality.xlsx") # data on one sheet, citations on the next
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
In Google Colab, run `!pip install undata` in the first cell. For the latest unreleased
|
|
44
|
+
changes: `pip install "git+https://github.com/nerd-factory/undata"`.
|
|
45
|
+
|
|
46
|
+
Don't know the indicator's id? `un.search("maternal mortality")` lists matching indicators
|
|
47
|
+
with their names. Copy the id from the first column.
|
|
48
|
+
|
|
49
|
+
## Everything on one screen
|
|
50
|
+
|
|
51
|
+
| Function | What it does |
|
|
52
|
+
|---|---|
|
|
53
|
+
| `un.search("words")` | Find indicators by what they measure. UN collections only, best match first. |
|
|
54
|
+
| `un.about(indicator)` | What an indicator is, whose it is, its unit, and which years it covers. |
|
|
55
|
+
| `un.get(indicator, countries, years)` | The numbers: one row per country and year, with unit and source. |
|
|
56
|
+
| `un.latest(indicator, countries)` | Each country's most recent value, with a warning if those years differ. |
|
|
57
|
+
| `un.countries(region)` · `un.regions()` | The names it knows, with UN names and ISO codes. |
|
|
58
|
+
| `un.sdg.goals()` · `targets(16)` · `indicators("16.1")` · `series("16.1.2")` | Walk from an SDG goal down to indicators you can fetch. |
|
|
59
|
+
| `un.ask("a question")` | See what data.un.org's search box decided, including whether it guessed the place. |
|
|
60
|
+
| `un.save(table, "file.xlsx")` · `un.cite(table)` | Save to Excel or CSV with sources; write citations for a report. |
|
|
61
|
+
|
|
62
|
+
Wherever a function asks for countries you can write names the way you would in an email
|
|
63
|
+
(`"Viet Nam"`, `"Vietnam"`, `"VNM"`), a list of them, or a region such as `"Southern Asia"`
|
|
64
|
+
or `"Africa"`. Every function explains itself: `help(un.get)`.
|
|
65
|
+
|
|
66
|
+
Every table has the same columns:
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
country iso3 year value unit indicator_name indicator source source_url
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## The careful parts are built in
|
|
73
|
+
|
|
74
|
+
- **It never guesses a place.** data.un.org's search box answers a question about "Bengal"
|
|
75
|
+
with figures for the United States. undata stops and says it does not know "Bengal".
|
|
76
|
+
- **Gaps are named, never zero.** Countries without data are left out of the table and listed
|
|
77
|
+
in a `MissingDataWarning`, so a chart cannot show them as zeros.
|
|
78
|
+
- **UN names in, UN names out.** Tables say Türkiye, Viet Nam and Côte d'Ivoire, whatever
|
|
79
|
+
spelling you typed.
|
|
80
|
+
- **The source is on every row.** The same statistic can come from several agencies with
|
|
81
|
+
different numbers.
|
|
82
|
+
- **Mixed years are flagged.** "Latest" means each country's own latest year; a
|
|
83
|
+
`MixedYearsWarning` says when those differ.
|
|
84
|
+
- **It works where other clients fail.** The platform's firewall blocks Google's official
|
|
85
|
+
`datacommons-client` (any POST body containing `"select"`) and any GET request that
|
|
86
|
+
mentions Western Sahara (`ESH`, mistaken for a shell command). undata routes around both.
|
|
87
|
+
|
|
88
|
+
Each behaviour is pinned by a test in [tests/live/](https://github.com/nerd-factory/undata/tree/main/tests/live) that runs against the real
|
|
89
|
+
platform every night and opens an issue if the platform changes. Most of them were found
|
|
90
|
+
while writing the [UN Data Commons Field Guide](https://mafiatun.github.io/un-data-commons-field-guide/).
|
|
91
|
+
|
|
92
|
+
## Examples
|
|
93
|
+
|
|
94
|
+
Ten runnable scripts in [examples/](https://github.com/nerd-factory/undata/tree/main/examples): a first table, comparing a region, a whole
|
|
95
|
+
continent, an SDG target to its numbers, gaps, names, the search box, sources, Excel and
|
|
96
|
+
citations, and checking an indicator. They run against the live platform every night, and
|
|
97
|
+
[the examples page](https://nerd-factory.github.io/undata/) shows each with its real output.
|
|
98
|
+
|
|
99
|
+
## For developers
|
|
100
|
+
|
|
101
|
+
The friendly functions sit on three layers you can use directly: `undata.Transport` (the
|
|
102
|
+
only code that touches the network), `undata._endpoints` (one thin wrapper per API route),
|
|
103
|
+
and `undata.dcid` (the identifier grammar). [PLAN.md](https://github.com/nerd-factory/undata/blob/main/PLAN.md) covers the design and the
|
|
104
|
+
road to 1.0.
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
git clone https://github.com/nerd-factory/undata && cd undata
|
|
108
|
+
uv sync # Python 3.11+, installs dev tools
|
|
109
|
+
uv run pytest # unit tests and doctests, offline
|
|
110
|
+
uv run pytest -m live # pins the live platform's behaviour (network)
|
|
111
|
+
uv run ruff check . && uv run ruff format --check . && uv run mypy
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
See [CONTRIBUTING.md](https://github.com/nerd-factory/undata/blob/main/CONTRIBUTING.md). Questions that feel too basic are welcome in
|
|
115
|
+
[Nerd Lab Discussions](https://github.com/nerd-factory/nerd-lab/discussions); they are not
|
|
116
|
+
too basic.
|
|
117
|
+
|
|
118
|
+
## Licence
|
|
119
|
+
|
|
120
|
+
Code: [MIT](https://github.com/nerd-factory/undata/blob/main/LICENSE). Data fetched with it is governed by the
|
|
121
|
+
[UN Data Commons terms of use](https://data.un.org/undatacommons/terms-of-use) and by each
|
|
122
|
+
contributing agency's own terms.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Your first table: find an indicator by name, then get it for two countries."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
# 1. Find the indicator. Copy its id from the "indicator" column.
|
|
6
|
+
print(un.search("maternal mortality").head(3).to_string(index=False))
|
|
7
|
+
|
|
8
|
+
# 2. Get it. Country names work as you would write them.
|
|
9
|
+
table = un.get("undata/sdg/SH_STA_MORT.SEX--F", ["Bangladesh", "Viet Nam"], years=(2020, 2023))
|
|
10
|
+
print(table[["country", "year", "value", "unit"]].to_string(index=False))
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Compare the countries of a region on one indicator, each at its latest year."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
table = un.latest("undata/undphdro/HDI_hdi", "Southern Asia")
|
|
6
|
+
|
|
7
|
+
ranked = table.sort_values("value", ascending=False)
|
|
8
|
+
print(ranked[["country", "year", "value"]].to_string(index=False))
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""A whole continent in one line. Regions stand for every country in them."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
table = un.latest("undata/sdg/EG_ACS_ELEC", "Africa") # % with access to electricity
|
|
6
|
+
|
|
7
|
+
print(len(table), "countries, latest years:", sorted(table["year"].unique().tolist()))
|
|
8
|
+
print(table.nsmallest(3, "value")[["country", "year", "value", "unit"]].to_string(index=False))
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""From an SDG target your mandate names, to the numbers."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
print(un.sdg.indicators("16.1").to_string(index=False))
|
|
6
|
+
|
|
7
|
+
series = un.sdg.series("16.1.2")
|
|
8
|
+
print(series[series["total"]][["indicator", "name"]].to_string(index=False))
|
|
9
|
+
|
|
10
|
+
rates = un.latest("undata/sdg/VC_DTH_TOTR", ["Ukraine", "Sudan", "Myanmar"])
|
|
11
|
+
print(rates[["country", "year", "value"]].to_string(index=False))
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Countries without data are named, never turned into zeros."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
table = un.latest("undata/sdg/VC_DTH_TOTR", ["Ukraine", "Norway", "Japan", "Mali"])
|
|
6
|
+
|
|
7
|
+
print(table[["country", "year", "value"]].to_string(index=False))
|
|
8
|
+
print("No data for:", [row["country"] for row in table.attrs["missing"]])
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Names, UN names, codes and regions all work. Unknown names are never guessed."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
print(un.countries("Western Africa").head(4).to_string(index=False))
|
|
6
|
+
|
|
7
|
+
for name in ["Vietnam", "Türkiye", "Ivory Coast", "COD", "Bengal"]:
|
|
8
|
+
try:
|
|
9
|
+
print(f"{name!r:<14} ->", un.places.resolve(name).name)
|
|
10
|
+
except un.PlaceNotFoundError as error:
|
|
11
|
+
print(f"{name!r:<14} -> {error}")
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""See what data.un.org's search box decided, before you trust its chart."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
answer = un.ask("child mortality in Bengal")
|
|
6
|
+
print(answer.place, "| guessed:", answer.place_was_guessed)
|
|
7
|
+
|
|
8
|
+
answer = un.ask("child mortality in Bangladesh")
|
|
9
|
+
print(answer.place, "| guessed:", answer.place_was_guessed)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""One statistic, three agencies. The source column says whose number you are using."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
MATERNAL_MORTALITY_RATIO = [
|
|
6
|
+
"undata/sdg/SH_STA_MORT.SEX--F", # Global SDG Indicators Database
|
|
7
|
+
"undata/unicef/MNCH_MMR.SEX--F", # UNICEF
|
|
8
|
+
"undata/undphdro/GII_mmr.SEX--F", # UNDP Human Development Report Office
|
|
9
|
+
]
|
|
10
|
+
table = un.latest(MATERNAL_MORTALITY_RATIO, "Bangladesh")
|
|
11
|
+
|
|
12
|
+
print(table[["source", "year", "value"]].to_string(index=False))
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Save to Excel with a sheet of citations, and cite the data in a report."""
|
|
2
|
+
|
|
3
|
+
import undata as un
|
|
4
|
+
|
|
5
|
+
table = un.get("undata/sdg/SH_STA_MORT.SEX--F", "Southern Asia", years=(2015, 2023))
|
|
6
|
+
|
|
7
|
+
un.save(table, "maternal-mortality.xlsx") # sheets: data, sources
|
|
8
|
+
print(un.cite(table))
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25", "hatch-vcs>=0.4"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "undata"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "UN data in Python, without the guessing: a friendly, unofficial client for the UN System Data Commons (data.un.org)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Mafizul Islam", email = "islam50@un.org" }]
|
|
13
|
+
maintainers = [{ name = "Nerd Lab contributors" }]
|
|
14
|
+
requires-python = ">=3.11"
|
|
15
|
+
dependencies = ["httpx>=0.27", "pandas>=2.2.2", "openpyxl>=3.1"]
|
|
16
|
+
keywords = [
|
|
17
|
+
"united-nations",
|
|
18
|
+
"un-data",
|
|
19
|
+
"data-commons",
|
|
20
|
+
"sdg",
|
|
21
|
+
"sustainable-development-goals",
|
|
22
|
+
"official-statistics",
|
|
23
|
+
"statistics",
|
|
24
|
+
"open-data",
|
|
25
|
+
"pandas",
|
|
26
|
+
"api-client",
|
|
27
|
+
]
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Development Status :: 3 - Alpha",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"Intended Audience :: Science/Research",
|
|
32
|
+
"Natural Language :: English",
|
|
33
|
+
"Operating System :: OS Independent",
|
|
34
|
+
"Programming Language :: Python :: 3",
|
|
35
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
36
|
+
"Programming Language :: Python :: 3.11",
|
|
37
|
+
"Programming Language :: Python :: 3.12",
|
|
38
|
+
"Programming Language :: Python :: 3.13",
|
|
39
|
+
"Programming Language :: Python :: 3.14",
|
|
40
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
41
|
+
"Typing :: Typed",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://nerd-factory.github.io/undata/"
|
|
46
|
+
Documentation = "https://nerd-factory.github.io/undata/"
|
|
47
|
+
Source = "https://github.com/nerd-factory/undata"
|
|
48
|
+
Issues = "https://github.com/nerd-factory/undata/issues"
|
|
49
|
+
Changelog = "https://github.com/nerd-factory/undata/blob/main/CHANGELOG.md"
|
|
50
|
+
"Release notes" = "https://github.com/nerd-factory/undata/releases"
|
|
51
|
+
"Field guide" = "https://mafiatun.github.io/un-data-commons-field-guide/"
|
|
52
|
+
|
|
53
|
+
[dependency-groups]
|
|
54
|
+
dev = [
|
|
55
|
+
"pytest>=8.3",
|
|
56
|
+
"pytest-cov>=6.0",
|
|
57
|
+
"respx>=0.22",
|
|
58
|
+
"hypothesis>=6.120",
|
|
59
|
+
"mypy>=1.13",
|
|
60
|
+
"ruff>=0.8",
|
|
61
|
+
"pandas-stubs>=2.2",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[tool.hatch.version]
|
|
65
|
+
source = "vcs"
|
|
66
|
+
|
|
67
|
+
[tool.hatch.build.hooks.vcs]
|
|
68
|
+
version-file = "src/undata/_version.py"
|
|
69
|
+
|
|
70
|
+
[tool.hatch.build.targets.sdist]
|
|
71
|
+
# Tests and examples ship in the sdist so downstream packagers (conda-forge, Linux
|
|
72
|
+
# distributions) can run them. The website and CI configuration do not.
|
|
73
|
+
include = [
|
|
74
|
+
"/src",
|
|
75
|
+
"/tests",
|
|
76
|
+
"/examples",
|
|
77
|
+
"/README.md",
|
|
78
|
+
"/CHANGELOG.md",
|
|
79
|
+
"/CITATION.cff",
|
|
80
|
+
"/LICENSE",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
# ── Tests ──────────────────────────────────────────────────────────────
|
|
84
|
+
[tool.pytest.ini_options]
|
|
85
|
+
minversion = "8.0"
|
|
86
|
+
testpaths = ["src", "tests"]
|
|
87
|
+
addopts = [
|
|
88
|
+
"-ra", "--strict-markers", "--strict-config", "--doctest-modules", "-m", "not live",
|
|
89
|
+
]
|
|
90
|
+
markers = [
|
|
91
|
+
"live: calls the real data.un.org API (run nightly, or with `-m live`)",
|
|
92
|
+
]
|
|
93
|
+
filterwarnings = ["error"]
|
|
94
|
+
xfail_strict = true
|
|
95
|
+
|
|
96
|
+
[tool.coverage.run]
|
|
97
|
+
source = ["undata"]
|
|
98
|
+
branch = true
|
|
99
|
+
omit = ["src/undata/_version.py"]
|
|
100
|
+
|
|
101
|
+
[tool.coverage.report]
|
|
102
|
+
show_missing = true
|
|
103
|
+
skip_covered = true
|
|
104
|
+
exclude_also = ["if TYPE_CHECKING:", "@overload", "raise NotImplementedError"]
|
|
105
|
+
|
|
106
|
+
# ── Lint and types ─────────────────────────────────────────────────────
|
|
107
|
+
[tool.ruff]
|
|
108
|
+
line-length = 100
|
|
109
|
+
target-version = "py311"
|
|
110
|
+
extend-exclude = ["src/undata/_version.py"]
|
|
111
|
+
|
|
112
|
+
[tool.ruff.lint]
|
|
113
|
+
select = [
|
|
114
|
+
"E", "W", "F", "I", "B", "UP", "SIM", "C4", "PT", "RUF", "N", "D", "S", "PL", "TRY",
|
|
115
|
+
]
|
|
116
|
+
ignore = [
|
|
117
|
+
"D105", # magic methods are self-describing
|
|
118
|
+
"D107", # __init__ is documented on the class
|
|
119
|
+
"PLR0913", # endpoint wrappers mirror the API's own parameters
|
|
120
|
+
"TRY003", # long messages are the point: errors here explain what to do next
|
|
121
|
+
]
|
|
122
|
+
|
|
123
|
+
[tool.ruff.lint.pydocstyle]
|
|
124
|
+
convention = "google"
|
|
125
|
+
|
|
126
|
+
[tool.ruff.lint.per-file-ignores]
|
|
127
|
+
"tests/**" = ["D", "S101", "PLR2004", "PLC0415"]
|
|
128
|
+
"examples/**" = ["D103", "S101"]
|
|
129
|
+
|
|
130
|
+
[tool.mypy]
|
|
131
|
+
strict = true
|
|
132
|
+
python_version = "3.11"
|
|
133
|
+
files = ["src", "tests"]
|
|
134
|
+
exclude = ["src/undata/_version.py"]
|
|
135
|
+
|
|
136
|
+
[[tool.mypy.overrides]]
|
|
137
|
+
module = ["undata._version"]
|
|
138
|
+
ignore_missing_imports = true
|