pyckup 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyckup-0.1.2 → pyckup-0.1.3}/PKG-INFO +8 -2
- pyckup-0.1.3/pyckup/sources/__init__.py +1 -0
- pyckup-0.1.3/pyckup/sources/owid.py +245 -0
- pyckup-0.1.3/pyckup/util.py +25 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup.egg-info/PKG-INFO +8 -2
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup.egg-info/SOURCES.txt +5 -1
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup.egg-info/requires.txt +1 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/setup.cfg +2 -1
- pyckup-0.1.3/tests/test_offline.py +131 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/LICENSE +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/README.md +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup/__init__.py +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup/base.py +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup.egg-info/dependency_links.txt +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup.egg-info/not-zip-safe +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/pyckup.egg-info/top_level.txt +0 -0
- {pyckup-0.1.2 → pyckup-0.1.3}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pyckup
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
|
|
5
5
|
Home-page: https://github.com/i2mint/pyckup
|
|
6
6
|
License: apache-2.0
|
|
@@ -8,6 +8,12 @@ Keywords: data,data access,data preparation,machine learning,artificial intellig
|
|
|
8
8
|
Platform: any
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
License-File: LICENSE
|
|
11
|
+
Requires-Dist: dol
|
|
12
|
+
Requires-Dist: graze
|
|
13
|
+
Requires-Dist: config2py
|
|
14
|
+
Dynamic: description
|
|
15
|
+
Dynamic: description-content-type
|
|
16
|
+
Dynamic: license-file
|
|
11
17
|
|
|
12
18
|
# pyckup
|
|
13
19
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Data sources"""
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OWID data source.
|
|
3
|
+
|
|
4
|
+
You can see a list of datas here: https://ourworldindata.org/data
|
|
5
|
+
|
|
6
|
+
TODO: Write a function that lists all the available datas?
|
|
7
|
+
TODO: Write a dol interface to metadata (json) and data (csv or zip), like with haggle (kaggle interface), etc.
|
|
8
|
+
|
|
9
|
+
See: https://docs.owid.io/projects/etl/api/chart-api/
|
|
10
|
+
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Optional, Tuple, List
|
|
17
|
+
import zipfile
|
|
18
|
+
|
|
19
|
+
# graze helpers
|
|
20
|
+
from graze.base import url_to_file_download, return_filepath
|
|
21
|
+
|
|
22
|
+
from pyckup.util import downloads_dir
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
DFLT_REFRESH = False
|
|
26
|
+
|
|
27
|
+
owid_downloads_dir = os.path.join(downloads_dir, 'owid')
|
|
28
|
+
Path(owid_downloads_dir).mkdir(parents=True, exist_ok=True)
|
|
29
|
+
owid_downloads_dir_path = str(owid_downloads_dir)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def slug_to_url_and_filename(slug, kind='json', rootdir=owid_downloads_dir_path):
|
|
33
|
+
"""The OWID url for a chart ``slug`` and the local path to store it at.
|
|
34
|
+
|
|
35
|
+
``kind`` is one of 'json' (metadata), 'csv', 'zip' or 'html'.
|
|
36
|
+
|
|
37
|
+
>>> url, path = slug_to_url_and_filename('co2', 'csv', rootdir='/tmp')
|
|
38
|
+
>>> url
|
|
39
|
+
'https://ourworldindata.org/grapher/co2.csv'
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
base = f'https://ourworldindata.org/grapher/{slug}'
|
|
43
|
+
|
|
44
|
+
if kind == 'json':
|
|
45
|
+
url = f'{base}.metadata.json'
|
|
46
|
+
path = os.path.join(rootdir, f'{slug}.metadata.json')
|
|
47
|
+
elif kind == 'csv':
|
|
48
|
+
url = f'{base}.csv'
|
|
49
|
+
path = os.path.join(rootdir, f'{slug}.csv')
|
|
50
|
+
elif kind == 'zip':
|
|
51
|
+
url = f'{base}.zip'
|
|
52
|
+
path = os.path.join(rootdir, f'{slug}.zip')
|
|
53
|
+
elif kind == 'html':
|
|
54
|
+
url = f'{base}'
|
|
55
|
+
path = os.path.join(rootdir, f'{slug}.html')
|
|
56
|
+
else:
|
|
57
|
+
raise ValueError(
|
|
58
|
+
f"Unknown kind: {kind} (should be 'json', 'csv', 'zip', or 'html')"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
return url, path
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def acquire_owid_data(
|
|
65
|
+
slug: str, *, refresh: bool | None = DFLT_REFRESH
|
|
66
|
+
) -> list[tuple[str, Path | None, Path | None]]:
|
|
67
|
+
"""Download OWID CSV + JSON metadata for each chart slug.
|
|
68
|
+
|
|
69
|
+
Returns list of tuples: (json_path_or_None, csv_path_or_None)
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
attempted_zip = set() # track slug ZIP fallback attempts to avoid duplicates
|
|
73
|
+
|
|
74
|
+
def _is_forbidden_error(err: Exception) -> bool:
|
|
75
|
+
msg = str(err)
|
|
76
|
+
return '403' in msg or 'Forbidden' in msg
|
|
77
|
+
|
|
78
|
+
def _is_non_redistributable(err: Exception) -> bool:
|
|
79
|
+
# Detect server message that explicitly forbids redistribution
|
|
80
|
+
msg = str(err).lower()
|
|
81
|
+
return 'non-redistributable' in msg or 'not allowed to re-share' in msg
|
|
82
|
+
|
|
83
|
+
def _download_from_zip(
|
|
84
|
+
slug: str, csv_path: Path, json_path: Path
|
|
85
|
+
) -> tuple[Path | None, Path | None]:
|
|
86
|
+
zip_url, tmp_zip_path = slug_to_url_and_filename(slug, kind='zip')
|
|
87
|
+
tmp_zip_path = Path(tmp_zip_path)
|
|
88
|
+
|
|
89
|
+
# Avoid re-attempting ZIP if we already tried once
|
|
90
|
+
if slug in attempted_zip and not refresh:
|
|
91
|
+
return None, None
|
|
92
|
+
|
|
93
|
+
attempted_zip.add(slug)
|
|
94
|
+
|
|
95
|
+
try:
|
|
96
|
+
zip_ret = url_to_file_download(
|
|
97
|
+
zip_url,
|
|
98
|
+
filepath=str(tmp_zip_path),
|
|
99
|
+
overwrite=bool(refresh),
|
|
100
|
+
return_func=return_filepath,
|
|
101
|
+
)
|
|
102
|
+
if not zip_ret:
|
|
103
|
+
return None, None
|
|
104
|
+
zip_file_path = Path(zip_ret)
|
|
105
|
+
if not zip_file_path.exists():
|
|
106
|
+
return None, None
|
|
107
|
+
except Exception as e:
|
|
108
|
+
# If server forbids redistribution, bail early and don't spam retries
|
|
109
|
+
if _is_non_redistributable(e):
|
|
110
|
+
print(
|
|
111
|
+
f"OWID ZIP fallback: data for '{slug}' is non-redistributable; skipping ZIP fallback."
|
|
112
|
+
)
|
|
113
|
+
return None, None
|
|
114
|
+
print(f"OWID ZIP fallback for '{slug}' failed: {e}")
|
|
115
|
+
return None, None
|
|
116
|
+
|
|
117
|
+
csv_ok, json_ok = None, None
|
|
118
|
+
try:
|
|
119
|
+
with zipfile.ZipFile(zip_file_path, 'r') as zf:
|
|
120
|
+
csv_member = f'{slug}.csv'
|
|
121
|
+
meta_member = f'{slug}.metadata.json'
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
with zf.open(csv_member) as fsrc, open(csv_path, 'wb') as fdst:
|
|
125
|
+
fdst.write(fsrc.read())
|
|
126
|
+
csv_ok = csv_path
|
|
127
|
+
print(
|
|
128
|
+
f'✓ OWID CSV (from ZIP): {csv_path.name} ({csv_path.stat().st_size:,} bytes)'
|
|
129
|
+
)
|
|
130
|
+
except Exception as e:
|
|
131
|
+
print(f"Could not extract CSV from ZIP for '{slug}': {e}")
|
|
132
|
+
|
|
133
|
+
try:
|
|
134
|
+
with zf.open(meta_member) as fsrc, open(json_path, 'wb') as fdst:
|
|
135
|
+
fdst.write(fsrc.read())
|
|
136
|
+
json_ok = json_path
|
|
137
|
+
print(
|
|
138
|
+
f'✓ OWID JSON (from ZIP): {json_path.name} ({json_path.stat().st_size:,} bytes)'
|
|
139
|
+
)
|
|
140
|
+
except Exception as e:
|
|
141
|
+
print(f"Could not extract metadata JSON from ZIP for '{slug}': {e}")
|
|
142
|
+
except Exception as e:
|
|
143
|
+
print(f"OWID ZIP for '{slug}' could not be read: {e}")
|
|
144
|
+
finally:
|
|
145
|
+
try:
|
|
146
|
+
if zip_file_path.exists():
|
|
147
|
+
zip_file_path.unlink()
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
|
|
151
|
+
return csv_ok, json_ok
|
|
152
|
+
|
|
153
|
+
json_url, json_path = slug_to_url_and_filename(slug, kind='json')
|
|
154
|
+
csv_url, csv_path = slug_to_url_and_filename(slug, kind='csv')
|
|
155
|
+
|
|
156
|
+
# ---- CSV ----
|
|
157
|
+
csv_file = None
|
|
158
|
+
if Path(csv_path).exists() and not refresh:
|
|
159
|
+
csv_file = csv_path
|
|
160
|
+
else:
|
|
161
|
+
# Attempt direct CSV
|
|
162
|
+
print(f"Acquiring OWID CSV for '{slug}' -> {csv_url}")
|
|
163
|
+
try:
|
|
164
|
+
csv_ret = url_to_file_download(
|
|
165
|
+
csv_url,
|
|
166
|
+
filepath=str(csv_path),
|
|
167
|
+
overwrite=bool(refresh),
|
|
168
|
+
return_func=return_filepath,
|
|
169
|
+
)
|
|
170
|
+
csv_file = Path(csv_ret) if csv_ret else None
|
|
171
|
+
if csv_file and csv_file.exists():
|
|
172
|
+
print(
|
|
173
|
+
f'✓ OWID CSV: {csv_file.name} ({csv_file.stat().st_size:,} bytes)'
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
csv_file = None
|
|
177
|
+
except Exception as e:
|
|
178
|
+
if _is_non_redistributable(e):
|
|
179
|
+
print(
|
|
180
|
+
f"OWID CSV for '{slug}' is non-redistributable; not available for download."
|
|
181
|
+
)
|
|
182
|
+
csv_file = None
|
|
183
|
+
elif _is_forbidden_error(e):
|
|
184
|
+
# Only try ZIP fallback if we haven't already attempted it for this slug
|
|
185
|
+
if slug not in attempted_zip:
|
|
186
|
+
print(
|
|
187
|
+
f"CSV download forbidden for '{slug}', attempting ZIP fallback…"
|
|
188
|
+
)
|
|
189
|
+
csv_file, json_file_from_zip = _download_from_zip(
|
|
190
|
+
slug, csv_path, json_path
|
|
191
|
+
)
|
|
192
|
+
else:
|
|
193
|
+
csv_file = None
|
|
194
|
+
json_file_from_zip = None
|
|
195
|
+
else:
|
|
196
|
+
print(f"OWID CSV for '{slug}' failed: {e}")
|
|
197
|
+
csv_file = None
|
|
198
|
+
json_file_from_zip = None
|
|
199
|
+
|
|
200
|
+
# ---- JSON metadata ----
|
|
201
|
+
json_file: Path | None = None
|
|
202
|
+
if Path(json_path).exists() and not refresh:
|
|
203
|
+
json_file = json_path
|
|
204
|
+
else:
|
|
205
|
+
if 'json_file_from_zip' in locals() and json_file_from_zip:
|
|
206
|
+
json_file = json_file_from_zip
|
|
207
|
+
else:
|
|
208
|
+
try:
|
|
209
|
+
print(f"Acquiring OWID JSON for '{slug}' -> {json_url}")
|
|
210
|
+
json_ret = url_to_file_download(
|
|
211
|
+
json_url,
|
|
212
|
+
filepath=str(json_path),
|
|
213
|
+
overwrite=bool(refresh),
|
|
214
|
+
return_func=return_filepath,
|
|
215
|
+
)
|
|
216
|
+
json_file = Path(json_ret) if json_ret else None
|
|
217
|
+
if json_file and json_file.exists():
|
|
218
|
+
try:
|
|
219
|
+
meta = json.loads(json_file.read_text(encoding='utf-8'))
|
|
220
|
+
title = meta.get('chart', {}).get('title', slug)
|
|
221
|
+
print(
|
|
222
|
+
f'✓ OWID JSON: {json_file.name} ({json_file.stat().st_size:,} bytes)'
|
|
223
|
+
)
|
|
224
|
+
print(f' • Chart title: {title}')
|
|
225
|
+
except Exception:
|
|
226
|
+
pass
|
|
227
|
+
else:
|
|
228
|
+
json_file = None
|
|
229
|
+
except Exception as e:
|
|
230
|
+
if _is_non_redistributable(e):
|
|
231
|
+
print(
|
|
232
|
+
f"OWID JSON for '{slug}' appears non-redistributable; skipping."
|
|
233
|
+
)
|
|
234
|
+
json_file = None
|
|
235
|
+
elif _is_forbidden_error(e) and slug not in attempted_zip:
|
|
236
|
+
print(
|
|
237
|
+
f"Metadata download forbidden for '{slug}', attempting ZIP fallback…"
|
|
238
|
+
)
|
|
239
|
+
_, json_file_zip = _download_from_zip(slug, csv_path, json_path)
|
|
240
|
+
json_file = json_file_zip
|
|
241
|
+
else:
|
|
242
|
+
print(f"OWID JSON for '{slug}' failed: {e}")
|
|
243
|
+
json_file = None
|
|
244
|
+
|
|
245
|
+
return json_file, csv_file
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Utils for pyckyp"""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from importlib.resources import files
|
|
5
|
+
from functools import partial
|
|
6
|
+
|
|
7
|
+
from config2py import simple_config_getter, get_app_config_folder, process_path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
pkg_name = 'pyckup'
|
|
11
|
+
|
|
12
|
+
# ---------------- PKG DATA -----------------------------------------------------
|
|
13
|
+
|
|
14
|
+
get_config = simple_config_getter(pkg_name)
|
|
15
|
+
|
|
16
|
+
data_files = files(pkg_name) / 'data'
|
|
17
|
+
app_data_dir = os.environ.get(
|
|
18
|
+
f'{pkg_name.upper()}_APP_DATA_DIR', get_app_config_folder(pkg_name)
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
# ---------------- LOCAL DATA -----------------------------------------------------
|
|
22
|
+
app_data_dir = process_path(app_data_dir, ensure_dir_exists=True)
|
|
23
|
+
djoin = partial(os.path.join, app_data_dir)
|
|
24
|
+
|
|
25
|
+
downloads_dir = process_path(djoin('downloads'), ensure_dir_exists=True)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pyckup
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
|
|
5
5
|
Home-page: https://github.com/i2mint/pyckup
|
|
6
6
|
License: apache-2.0
|
|
@@ -8,6 +8,12 @@ Keywords: data,data access,data preparation,machine learning,artificial intellig
|
|
|
8
8
|
Platform: any
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
License-File: LICENSE
|
|
11
|
+
Requires-Dist: dol
|
|
12
|
+
Requires-Dist: graze
|
|
13
|
+
Requires-Dist: config2py
|
|
14
|
+
Dynamic: description
|
|
15
|
+
Dynamic: description-content-type
|
|
16
|
+
Dynamic: license-file
|
|
11
17
|
|
|
12
18
|
# pyckup
|
|
13
19
|
|
|
@@ -4,9 +4,13 @@ setup.cfg
|
|
|
4
4
|
setup.py
|
|
5
5
|
pyckup/__init__.py
|
|
6
6
|
pyckup/base.py
|
|
7
|
+
pyckup/util.py
|
|
7
8
|
pyckup.egg-info/PKG-INFO
|
|
8
9
|
pyckup.egg-info/SOURCES.txt
|
|
9
10
|
pyckup.egg-info/dependency_links.txt
|
|
10
11
|
pyckup.egg-info/not-zip-safe
|
|
11
12
|
pyckup.egg-info/requires.txt
|
|
12
|
-
pyckup.egg-info/top_level.txt
|
|
13
|
+
pyckup.egg-info/top_level.txt
|
|
14
|
+
pyckup/sources/__init__.py
|
|
15
|
+
pyckup/sources/owid.py
|
|
16
|
+
tests/test_offline.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[metadata]
|
|
2
2
|
name = pyckup
|
|
3
|
-
version = 0.1.
|
|
3
|
+
version = 0.1.3
|
|
4
4
|
url = https://github.com/i2mint/pyckup
|
|
5
5
|
platforms = any
|
|
6
6
|
description_file = README.md
|
|
@@ -24,6 +24,7 @@ zip_safe = False
|
|
|
24
24
|
install_requires =
|
|
25
25
|
dol
|
|
26
26
|
graze
|
|
27
|
+
config2py
|
|
27
28
|
|
|
28
29
|
[egg_info]
|
|
29
30
|
tag_build =
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Offline tests for pyckup: protocol dispatch, local files, and the OWID acquirer with a fake downloader."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import zipfile
|
|
5
|
+
from functools import partial
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from pyckup import grab
|
|
11
|
+
from pyckup import base
|
|
12
|
+
from pyckup.sources import owid
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_grab_local_file_and_folder(tmp_path):
|
|
16
|
+
f = tmp_path / 'a.bin'
|
|
17
|
+
f.write_bytes(b'xyz')
|
|
18
|
+
assert grab(str(f)) == b'xyz'
|
|
19
|
+
assert grab('file://' + str(f)) == b'xyz'
|
|
20
|
+
folder = grab(str(tmp_path))
|
|
21
|
+
assert 'a.bin' in folder and folder['a.bin'] == b'xyz'
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_grab_custom_and_unknown_protocol():
|
|
25
|
+
g = partial(grab, protocols={'foo': str.upper})
|
|
26
|
+
assert g('foo://bar') == 'FOO://BAR'
|
|
27
|
+
with pytest.raises(KeyError, match='Unrecognized protocol: nope'):
|
|
28
|
+
g('nope://x')
|
|
29
|
+
assert grab('x://y', protocols=lambda: {'x': lambda k: k[::-1]}) == 'y//:x'
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_url_2_bytes_reads_in_chunks(tmp_path):
|
|
33
|
+
f = tmp_path / 'big.bin'
|
|
34
|
+
f.write_bytes(b'0123456789' * 300)
|
|
35
|
+
assert base.url_2_bytes(f.as_uri(), chk_size=7) == b'0123456789' * 300
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_slug_to_url_and_filename(tmp_path):
|
|
39
|
+
url, path = owid.slug_to_url_and_filename('co2', 'csv', rootdir=str(tmp_path))
|
|
40
|
+
assert url == 'https://ourworldindata.org/grapher/co2.csv'
|
|
41
|
+
assert path == str(tmp_path / 'co2.csv')
|
|
42
|
+
assert owid.slug_to_url_and_filename('co2', 'zip', str(tmp_path))[0].endswith(
|
|
43
|
+
'.zip'
|
|
44
|
+
)
|
|
45
|
+
assert owid.slug_to_url_and_filename('co2', 'html', str(tmp_path))[0].endswith(
|
|
46
|
+
'/co2'
|
|
47
|
+
)
|
|
48
|
+
assert owid.slug_to_url_and_filename('co2', 'json', str(tmp_path))[0].endswith(
|
|
49
|
+
'co2.metadata.json'
|
|
50
|
+
)
|
|
51
|
+
with pytest.raises(ValueError, match='Unknown kind'):
|
|
52
|
+
owid.slug_to_url_and_filename('co2', 'pdf', str(tmp_path))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@pytest.fixture
|
|
56
|
+
def rooted(tmp_path, monkeypatch):
|
|
57
|
+
orig = owid.slug_to_url_and_filename
|
|
58
|
+
monkeypatch.setattr(
|
|
59
|
+
owid,
|
|
60
|
+
'slug_to_url_and_filename',
|
|
61
|
+
lambda slug, kind='json': orig(slug, kind, str(tmp_path)),
|
|
62
|
+
)
|
|
63
|
+
return tmp_path
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _fake_downloader(contents, fail_on=()):
|
|
67
|
+
def download(url, *, filepath, overwrite, return_func):
|
|
68
|
+
for marker, exc in fail_on:
|
|
69
|
+
if url.endswith(marker):
|
|
70
|
+
raise exc
|
|
71
|
+
Path(filepath).write_bytes(
|
|
72
|
+
contents[url.rsplit('.', 1)[-1] if url.endswith('.csv') else 'json']
|
|
73
|
+
)
|
|
74
|
+
return filepath
|
|
75
|
+
|
|
76
|
+
return download
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_acquire_downloads_csv_and_json(rooted, monkeypatch):
|
|
80
|
+
monkeypatch.setattr(
|
|
81
|
+
owid,
|
|
82
|
+
'url_to_file_download',
|
|
83
|
+
_fake_downloader({'csv': b'a,b\n1,2\n', 'json': b'{"chart": {"title": "T"}}'}),
|
|
84
|
+
)
|
|
85
|
+
json_file, csv_file = owid.acquire_owid_data('co2')
|
|
86
|
+
assert Path(csv_file).read_bytes() == b'a,b\n1,2\n'
|
|
87
|
+
assert Path(json_file).read_text().startswith('{"chart"')
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_acquire_uses_existing_files_without_downloading(rooted, monkeypatch):
|
|
91
|
+
(rooted / 'co2.csv').write_text('x')
|
|
92
|
+
(rooted / 'co2.metadata.json').write_text('{}')
|
|
93
|
+
|
|
94
|
+
def boom(*a, **k):
|
|
95
|
+
raise AssertionError('should not download')
|
|
96
|
+
|
|
97
|
+
monkeypatch.setattr(owid, 'url_to_file_download', boom)
|
|
98
|
+
json_file, csv_file = owid.acquire_owid_data('co2')
|
|
99
|
+
assert (
|
|
100
|
+
Path(csv_file).name == 'co2.csv' and Path(json_file).name == 'co2.metadata.json'
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_acquire_non_redistributable_gives_none(rooted, monkeypatch, capsys):
|
|
105
|
+
err = RuntimeError('403 non-redistributable')
|
|
106
|
+
monkeypatch.setattr(
|
|
107
|
+
owid,
|
|
108
|
+
'url_to_file_download',
|
|
109
|
+
_fake_downloader({}, fail_on=[('.csv', err), ('.json', err)]),
|
|
110
|
+
)
|
|
111
|
+
assert owid.acquire_owid_data('co2') == (None, None)
|
|
112
|
+
assert 'non-redistributable' in capsys.readouterr().out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_acquire_forbidden_falls_back_to_zip(rooted, monkeypatch):
|
|
116
|
+
buf = io.BytesIO()
|
|
117
|
+
with zipfile.ZipFile(buf, 'w') as z:
|
|
118
|
+
z.writestr('co2.csv', 'a\n1\n')
|
|
119
|
+
z.writestr('co2.metadata.json', '{}')
|
|
120
|
+
zbytes = buf.getvalue()
|
|
121
|
+
|
|
122
|
+
def download(url, *, filepath, overwrite, return_func):
|
|
123
|
+
if url.endswith('.zip'):
|
|
124
|
+
Path(filepath).write_bytes(zbytes)
|
|
125
|
+
return filepath
|
|
126
|
+
raise RuntimeError('403 Forbidden')
|
|
127
|
+
|
|
128
|
+
monkeypatch.setattr(owid, 'url_to_file_download', download)
|
|
129
|
+
json_file, csv_file = owid.acquire_owid_data('co2')
|
|
130
|
+
assert Path(csv_file).read_text() == 'a\n1\n'
|
|
131
|
+
assert Path(json_file).read_text() == '{}'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|