pyckup 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: pyckup
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
5
5
  Home-page: https://github.com/i2mint/pyckup
6
6
  License: apache-2.0
@@ -8,6 +8,12 @@ Keywords: data,data access,data preparation,machine learning,artificial intellig
8
8
  Platform: any
9
9
  Description-Content-Type: text/markdown
10
10
  License-File: LICENSE
11
+ Requires-Dist: dol
12
+ Requires-Dist: graze
13
+ Requires-Dist: config2py
14
+ Dynamic: description
15
+ Dynamic: description-content-type
16
+ Dynamic: license-file
11
17
 
12
18
  # pyckup
13
19
 
@@ -0,0 +1 @@
1
+ """Data sources"""
@@ -0,0 +1,245 @@
1
+ """
2
+ OWID data source.
3
+
4
+ You can see a list of datas here: https://ourworldindata.org/data
5
+
6
+ TODO: Write a function that lists all the available datas?
7
+ TODO: Write a dol interface to metadata (json) and data (csv or zip), like with haggle (kaggle interface), etc.
8
+
9
+ See: https://docs.owid.io/projects/etl/api/chart-api/
10
+
11
+ """
12
+
13
+ import os
14
+ import json
15
+ from pathlib import Path
16
+ from typing import Optional, Tuple, List
17
+ import zipfile
18
+
19
+ # graze helpers
20
+ from graze.base import url_to_file_download, return_filepath
21
+
22
+ from pyckup.util import downloads_dir
23
+
24
+
25
+ DFLT_REFRESH = False
26
+
27
+ owid_downloads_dir = os.path.join(downloads_dir, 'owid')
28
+ Path(owid_downloads_dir).mkdir(parents=True, exist_ok=True)
29
+ owid_downloads_dir_path = str(owid_downloads_dir)
30
+
31
+
32
+ def slug_to_url_and_filename(slug, kind='json', rootdir=owid_downloads_dir_path):
33
+ """The OWID url for a chart ``slug`` and the local path to store it at.
34
+
35
+ ``kind`` is one of 'json' (metadata), 'csv', 'zip' or 'html'.
36
+
37
+ >>> url, path = slug_to_url_and_filename('co2', 'csv', rootdir='/tmp')
38
+ >>> url
39
+ 'https://ourworldindata.org/grapher/co2.csv'
40
+ """
41
+
42
+ base = f'https://ourworldindata.org/grapher/{slug}'
43
+
44
+ if kind == 'json':
45
+ url = f'{base}.metadata.json'
46
+ path = os.path.join(rootdir, f'{slug}.metadata.json')
47
+ elif kind == 'csv':
48
+ url = f'{base}.csv'
49
+ path = os.path.join(rootdir, f'{slug}.csv')
50
+ elif kind == 'zip':
51
+ url = f'{base}.zip'
52
+ path = os.path.join(rootdir, f'{slug}.zip')
53
+ elif kind == 'html':
54
+ url = f'{base}'
55
+ path = os.path.join(rootdir, f'{slug}.html')
56
+ else:
57
+ raise ValueError(
58
+ f"Unknown kind: {kind} (should be 'json', 'csv', 'zip', or 'html')"
59
+ )
60
+
61
+ return url, path
62
+
63
+
64
+ def acquire_owid_data(
65
+ slug: str, *, refresh: bool | None = DFLT_REFRESH
66
+ ) -> list[tuple[str, Path | None, Path | None]]:
67
+ """Download OWID CSV + JSON metadata for each chart slug.
68
+
69
+ Returns list of tuples: (json_path_or_None, csv_path_or_None)
70
+ """
71
+
72
+ attempted_zip = set() # track slug ZIP fallback attempts to avoid duplicates
73
+
74
+ def _is_forbidden_error(err: Exception) -> bool:
75
+ msg = str(err)
76
+ return '403' in msg or 'Forbidden' in msg
77
+
78
+ def _is_non_redistributable(err: Exception) -> bool:
79
+ # Detect server message that explicitly forbids redistribution
80
+ msg = str(err).lower()
81
+ return 'non-redistributable' in msg or 'not allowed to re-share' in msg
82
+
83
+ def _download_from_zip(
84
+ slug: str, csv_path: Path, json_path: Path
85
+ ) -> tuple[Path | None, Path | None]:
86
+ zip_url, tmp_zip_path = slug_to_url_and_filename(slug, kind='zip')
87
+ tmp_zip_path = Path(tmp_zip_path)
88
+
89
+ # Avoid re-attempting ZIP if we already tried once
90
+ if slug in attempted_zip and not refresh:
91
+ return None, None
92
+
93
+ attempted_zip.add(slug)
94
+
95
+ try:
96
+ zip_ret = url_to_file_download(
97
+ zip_url,
98
+ filepath=str(tmp_zip_path),
99
+ overwrite=bool(refresh),
100
+ return_func=return_filepath,
101
+ )
102
+ if not zip_ret:
103
+ return None, None
104
+ zip_file_path = Path(zip_ret)
105
+ if not zip_file_path.exists():
106
+ return None, None
107
+ except Exception as e:
108
+ # If server forbids redistribution, bail early and don't spam retries
109
+ if _is_non_redistributable(e):
110
+ print(
111
+ f"OWID ZIP fallback: data for '{slug}' is non-redistributable; skipping ZIP fallback."
112
+ )
113
+ return None, None
114
+ print(f"OWID ZIP fallback for '{slug}' failed: {e}")
115
+ return None, None
116
+
117
+ csv_ok, json_ok = None, None
118
+ try:
119
+ with zipfile.ZipFile(zip_file_path, 'r') as zf:
120
+ csv_member = f'{slug}.csv'
121
+ meta_member = f'{slug}.metadata.json'
122
+
123
+ try:
124
+ with zf.open(csv_member) as fsrc, open(csv_path, 'wb') as fdst:
125
+ fdst.write(fsrc.read())
126
+ csv_ok = csv_path
127
+ print(
128
+ f'✓ OWID CSV (from ZIP): {csv_path.name} ({csv_path.stat().st_size:,} bytes)'
129
+ )
130
+ except Exception as e:
131
+ print(f"Could not extract CSV from ZIP for '{slug}': {e}")
132
+
133
+ try:
134
+ with zf.open(meta_member) as fsrc, open(json_path, 'wb') as fdst:
135
+ fdst.write(fsrc.read())
136
+ json_ok = json_path
137
+ print(
138
+ f'✓ OWID JSON (from ZIP): {json_path.name} ({json_path.stat().st_size:,} bytes)'
139
+ )
140
+ except Exception as e:
141
+ print(f"Could not extract metadata JSON from ZIP for '{slug}': {e}")
142
+ except Exception as e:
143
+ print(f"OWID ZIP for '{slug}' could not be read: {e}")
144
+ finally:
145
+ try:
146
+ if zip_file_path.exists():
147
+ zip_file_path.unlink()
148
+ except Exception:
149
+ pass
150
+
151
+ return csv_ok, json_ok
152
+
153
+ json_url, json_path = slug_to_url_and_filename(slug, kind='json')
154
+ csv_url, csv_path = slug_to_url_and_filename(slug, kind='csv')
155
+
156
+ # ---- CSV ----
157
+ csv_file = None
158
+ if Path(csv_path).exists() and not refresh:
159
+ csv_file = csv_path
160
+ else:
161
+ # Attempt direct CSV
162
+ print(f"Acquiring OWID CSV for '{slug}' -> {csv_url}")
163
+ try:
164
+ csv_ret = url_to_file_download(
165
+ csv_url,
166
+ filepath=str(csv_path),
167
+ overwrite=bool(refresh),
168
+ return_func=return_filepath,
169
+ )
170
+ csv_file = Path(csv_ret) if csv_ret else None
171
+ if csv_file and csv_file.exists():
172
+ print(
173
+ f'✓ OWID CSV: {csv_file.name} ({csv_file.stat().st_size:,} bytes)'
174
+ )
175
+ else:
176
+ csv_file = None
177
+ except Exception as e:
178
+ if _is_non_redistributable(e):
179
+ print(
180
+ f"OWID CSV for '{slug}' is non-redistributable; not available for download."
181
+ )
182
+ csv_file = None
183
+ elif _is_forbidden_error(e):
184
+ # Only try ZIP fallback if we haven't already attempted it for this slug
185
+ if slug not in attempted_zip:
186
+ print(
187
+ f"CSV download forbidden for '{slug}', attempting ZIP fallback…"
188
+ )
189
+ csv_file, json_file_from_zip = _download_from_zip(
190
+ slug, csv_path, json_path
191
+ )
192
+ else:
193
+ csv_file = None
194
+ json_file_from_zip = None
195
+ else:
196
+ print(f"OWID CSV for '{slug}' failed: {e}")
197
+ csv_file = None
198
+ json_file_from_zip = None
199
+
200
+ # ---- JSON metadata ----
201
+ json_file: Path | None = None
202
+ if Path(json_path).exists() and not refresh:
203
+ json_file = json_path
204
+ else:
205
+ if 'json_file_from_zip' in locals() and json_file_from_zip:
206
+ json_file = json_file_from_zip
207
+ else:
208
+ try:
209
+ print(f"Acquiring OWID JSON for '{slug}' -> {json_url}")
210
+ json_ret = url_to_file_download(
211
+ json_url,
212
+ filepath=str(json_path),
213
+ overwrite=bool(refresh),
214
+ return_func=return_filepath,
215
+ )
216
+ json_file = Path(json_ret) if json_ret else None
217
+ if json_file and json_file.exists():
218
+ try:
219
+ meta = json.loads(json_file.read_text(encoding='utf-8'))
220
+ title = meta.get('chart', {}).get('title', slug)
221
+ print(
222
+ f'✓ OWID JSON: {json_file.name} ({json_file.stat().st_size:,} bytes)'
223
+ )
224
+ print(f' • Chart title: {title}')
225
+ except Exception:
226
+ pass
227
+ else:
228
+ json_file = None
229
+ except Exception as e:
230
+ if _is_non_redistributable(e):
231
+ print(
232
+ f"OWID JSON for '{slug}' appears non-redistributable; skipping."
233
+ )
234
+ json_file = None
235
+ elif _is_forbidden_error(e) and slug not in attempted_zip:
236
+ print(
237
+ f"Metadata download forbidden for '{slug}', attempting ZIP fallback…"
238
+ )
239
+ _, json_file_zip = _download_from_zip(slug, csv_path, json_path)
240
+ json_file = json_file_zip
241
+ else:
242
+ print(f"OWID JSON for '{slug}' failed: {e}")
243
+ json_file = None
244
+
245
+ return json_file, csv_file
@@ -0,0 +1,25 @@
1
+ """Utils for pyckyp"""
2
+
3
+ import os
4
+ from importlib.resources import files
5
+ from functools import partial
6
+
7
+ from config2py import simple_config_getter, get_app_config_folder, process_path
8
+
9
+
10
+ pkg_name = 'pyckup'
11
+
12
+ # ---------------- PKG DATA -----------------------------------------------------
13
+
14
+ get_config = simple_config_getter(pkg_name)
15
+
16
+ data_files = files(pkg_name) / 'data'
17
+ app_data_dir = os.environ.get(
18
+ f'{pkg_name.upper()}_APP_DATA_DIR', get_app_config_folder(pkg_name)
19
+ )
20
+
21
+ # ---------------- LOCAL DATA -----------------------------------------------------
22
+ app_data_dir = process_path(app_data_dir, ensure_dir_exists=True)
23
+ djoin = partial(os.path.join, app_data_dir)
24
+
25
+ downloads_dir = process_path(djoin('downloads'), ensure_dir_exists=True)
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: pyckup
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Tools to provide easy access to prepared data to data scientists that can't be asked.
5
5
  Home-page: https://github.com/i2mint/pyckup
6
6
  License: apache-2.0
@@ -8,6 +8,12 @@ Keywords: data,data access,data preparation,machine learning,artificial intellig
8
8
  Platform: any
9
9
  Description-Content-Type: text/markdown
10
10
  License-File: LICENSE
11
+ Requires-Dist: dol
12
+ Requires-Dist: graze
13
+ Requires-Dist: config2py
14
+ Dynamic: description
15
+ Dynamic: description-content-type
16
+ Dynamic: license-file
11
17
 
12
18
  # pyckup
13
19
 
@@ -4,9 +4,13 @@ setup.cfg
4
4
  setup.py
5
5
  pyckup/__init__.py
6
6
  pyckup/base.py
7
+ pyckup/util.py
7
8
  pyckup.egg-info/PKG-INFO
8
9
  pyckup.egg-info/SOURCES.txt
9
10
  pyckup.egg-info/dependency_links.txt
10
11
  pyckup.egg-info/not-zip-safe
11
12
  pyckup.egg-info/requires.txt
12
- pyckup.egg-info/top_level.txt
13
+ pyckup.egg-info/top_level.txt
14
+ pyckup/sources/__init__.py
15
+ pyckup/sources/owid.py
16
+ tests/test_offline.py
@@ -1,2 +1,3 @@
1
1
  dol
2
2
  graze
3
+ config2py
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = pyckup
3
- version = 0.1.2
3
+ version = 0.1.3
4
4
  url = https://github.com/i2mint/pyckup
5
5
  platforms = any
6
6
  description_file = README.md
@@ -24,6 +24,7 @@ zip_safe = False
24
24
  install_requires =
25
25
  dol
26
26
  graze
27
+ config2py
27
28
 
28
29
  [egg_info]
29
30
  tag_build =
@@ -0,0 +1,131 @@
1
+ """Offline tests for pyckup: protocol dispatch, local files, and the OWID acquirer with a fake downloader."""
2
+
3
+ import io
4
+ import zipfile
5
+ from functools import partial
6
+ from pathlib import Path
7
+
8
+ import pytest
9
+
10
+ from pyckup import grab
11
+ from pyckup import base
12
+ from pyckup.sources import owid
13
+
14
+
15
+ def test_grab_local_file_and_folder(tmp_path):
16
+ f = tmp_path / 'a.bin'
17
+ f.write_bytes(b'xyz')
18
+ assert grab(str(f)) == b'xyz'
19
+ assert grab('file://' + str(f)) == b'xyz'
20
+ folder = grab(str(tmp_path))
21
+ assert 'a.bin' in folder and folder['a.bin'] == b'xyz'
22
+
23
+
24
+ def test_grab_custom_and_unknown_protocol():
25
+ g = partial(grab, protocols={'foo': str.upper})
26
+ assert g('foo://bar') == 'FOO://BAR'
27
+ with pytest.raises(KeyError, match='Unrecognized protocol: nope'):
28
+ g('nope://x')
29
+ assert grab('x://y', protocols=lambda: {'x': lambda k: k[::-1]}) == 'y//:x'
30
+
31
+
32
+ def test_url_2_bytes_reads_in_chunks(tmp_path):
33
+ f = tmp_path / 'big.bin'
34
+ f.write_bytes(b'0123456789' * 300)
35
+ assert base.url_2_bytes(f.as_uri(), chk_size=7) == b'0123456789' * 300
36
+
37
+
38
+ def test_slug_to_url_and_filename(tmp_path):
39
+ url, path = owid.slug_to_url_and_filename('co2', 'csv', rootdir=str(tmp_path))
40
+ assert url == 'https://ourworldindata.org/grapher/co2.csv'
41
+ assert path == str(tmp_path / 'co2.csv')
42
+ assert owid.slug_to_url_and_filename('co2', 'zip', str(tmp_path))[0].endswith(
43
+ '.zip'
44
+ )
45
+ assert owid.slug_to_url_and_filename('co2', 'html', str(tmp_path))[0].endswith(
46
+ '/co2'
47
+ )
48
+ assert owid.slug_to_url_and_filename('co2', 'json', str(tmp_path))[0].endswith(
49
+ 'co2.metadata.json'
50
+ )
51
+ with pytest.raises(ValueError, match='Unknown kind'):
52
+ owid.slug_to_url_and_filename('co2', 'pdf', str(tmp_path))
53
+
54
+
55
+ @pytest.fixture
56
+ def rooted(tmp_path, monkeypatch):
57
+ orig = owid.slug_to_url_and_filename
58
+ monkeypatch.setattr(
59
+ owid,
60
+ 'slug_to_url_and_filename',
61
+ lambda slug, kind='json': orig(slug, kind, str(tmp_path)),
62
+ )
63
+ return tmp_path
64
+
65
+
66
+ def _fake_downloader(contents, fail_on=()):
67
+ def download(url, *, filepath, overwrite, return_func):
68
+ for marker, exc in fail_on:
69
+ if url.endswith(marker):
70
+ raise exc
71
+ Path(filepath).write_bytes(
72
+ contents[url.rsplit('.', 1)[-1] if url.endswith('.csv') else 'json']
73
+ )
74
+ return filepath
75
+
76
+ return download
77
+
78
+
79
+ def test_acquire_downloads_csv_and_json(rooted, monkeypatch):
80
+ monkeypatch.setattr(
81
+ owid,
82
+ 'url_to_file_download',
83
+ _fake_downloader({'csv': b'a,b\n1,2\n', 'json': b'{"chart": {"title": "T"}}'}),
84
+ )
85
+ json_file, csv_file = owid.acquire_owid_data('co2')
86
+ assert Path(csv_file).read_bytes() == b'a,b\n1,2\n'
87
+ assert Path(json_file).read_text().startswith('{"chart"')
88
+
89
+
90
+ def test_acquire_uses_existing_files_without_downloading(rooted, monkeypatch):
91
+ (rooted / 'co2.csv').write_text('x')
92
+ (rooted / 'co2.metadata.json').write_text('{}')
93
+
94
+ def boom(*a, **k):
95
+ raise AssertionError('should not download')
96
+
97
+ monkeypatch.setattr(owid, 'url_to_file_download', boom)
98
+ json_file, csv_file = owid.acquire_owid_data('co2')
99
+ assert (
100
+ Path(csv_file).name == 'co2.csv' and Path(json_file).name == 'co2.metadata.json'
101
+ )
102
+
103
+
104
+ def test_acquire_non_redistributable_gives_none(rooted, monkeypatch, capsys):
105
+ err = RuntimeError('403 non-redistributable')
106
+ monkeypatch.setattr(
107
+ owid,
108
+ 'url_to_file_download',
109
+ _fake_downloader({}, fail_on=[('.csv', err), ('.json', err)]),
110
+ )
111
+ assert owid.acquire_owid_data('co2') == (None, None)
112
+ assert 'non-redistributable' in capsys.readouterr().out
113
+
114
+
115
+ def test_acquire_forbidden_falls_back_to_zip(rooted, monkeypatch):
116
+ buf = io.BytesIO()
117
+ with zipfile.ZipFile(buf, 'w') as z:
118
+ z.writestr('co2.csv', 'a\n1\n')
119
+ z.writestr('co2.metadata.json', '{}')
120
+ zbytes = buf.getvalue()
121
+
122
+ def download(url, *, filepath, overwrite, return_func):
123
+ if url.endswith('.zip'):
124
+ Path(filepath).write_bytes(zbytes)
125
+ return filepath
126
+ raise RuntimeError('403 Forbidden')
127
+
128
+ monkeypatch.setattr(owid, 'url_to_file_download', download)
129
+ json_file, csv_file = owid.acquire_owid_data('co2')
130
+ assert Path(csv_file).read_text() == 'a\n1\n'
131
+ assert Path(json_file).read_text() == '{}'
File without changes
File without changes
File without changes
File without changes
File without changes