hdx-python-pipelineutils 0.0.6__tar.gz → 0.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/.github/workflows/publish.yaml +2 -2
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/.github/workflows/run-python-tests.yaml +4 -4
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/.pre-commit-config.yaml +2 -2
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/CLAUDE.md +2 -2
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/PKG-INFO +5 -6
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/pyproject.toml +6 -7
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/__init__.py +0 -19
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/_version.py +2 -2
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/hapi_admins.py +19 -1
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/lookup.py +1 -1
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/reader.py +4 -26
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/sector_configuration.yaml +1 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/conftest.py +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/test_hapi_admins.py +11 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/test_lookup.py +2 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/test_readers.py +108 -104
- hdx_python_pipelineutils-0.0.8/uv.lock +2225 -0
- hdx_python_pipelineutils-0.0.6/uv.lock +0 -2273
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/.gitignore +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/LICENSE +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/README.md +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/org_type.py +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/org_type_configuration.yaml +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/sector.py +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/download-global-pcode-lengths.csv +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/download-global-pcodes-adm-1-2.csv +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/education_closures_school_closures.csv +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/global-coordination-groups-beta.json +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/hno_2017_sahel_nutrition.csv +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/hno_2017_sahel_people_in_need.xlsx +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/org_type_organization_types_beta_csv_no_hxl.csv +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/organization-types-beta.json +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/sahel-humanitarian-needs-overview.json +0 -0
- {hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/sector_global_coordination_groups_beta_csv_no_hxl.csv +0 -0
{hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/.github/workflows/publish.yaml
RENAMED
|
@@ -17,7 +17,7 @@ jobs:
|
|
|
17
17
|
contents: read
|
|
18
18
|
|
|
19
19
|
steps:
|
|
20
|
-
- uses: actions/checkout@
|
|
20
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
21
21
|
|
|
22
22
|
- name: Get history and tags for versioning to work
|
|
23
23
|
run: |
|
|
@@ -25,7 +25,7 @@ jobs:
|
|
|
25
25
|
git fetch --depth=1 origin +refs/tags/*:refs/tags/*
|
|
26
26
|
|
|
27
27
|
- name: Install uv
|
|
28
|
-
uses: astral-sh/setup-uv@
|
|
28
|
+
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
|
|
29
29
|
|
|
30
30
|
- name: Build with uv
|
|
31
31
|
run: uv build
|
|
@@ -16,10 +16,10 @@ jobs:
|
|
|
16
16
|
pull-requests: write
|
|
17
17
|
|
|
18
18
|
steps:
|
|
19
|
-
- uses: actions/checkout@
|
|
19
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
20
20
|
|
|
21
21
|
- name: Install uv
|
|
22
|
-
uses: astral-sh/setup-uv@
|
|
22
|
+
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
|
|
23
23
|
with:
|
|
24
24
|
enable-cache: true
|
|
25
25
|
python-version: "3.13"
|
|
@@ -39,13 +39,13 @@ jobs:
|
|
|
39
39
|
run: uv run pytest
|
|
40
40
|
|
|
41
41
|
- name: Publish Unit Test Results
|
|
42
|
-
uses: EnricoMi/publish-unit-test-result-action@v2
|
|
42
|
+
uses: EnricoMi/publish-unit-test-result-action@7622c5bf3d112a72ab4fa3951c2aa5723d8ab92d # v2.25.0
|
|
43
43
|
if: always()
|
|
44
44
|
with:
|
|
45
45
|
files: test-results.xml
|
|
46
46
|
|
|
47
47
|
- name: Publish in Coveralls
|
|
48
|
-
uses: coverallsapp/github-action@v2
|
|
48
|
+
uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8
|
|
49
49
|
if: always()
|
|
50
50
|
with:
|
|
51
51
|
flag-name: tests
|
|
@@ -10,7 +10,7 @@ repos:
|
|
|
10
10
|
- id: check-ast
|
|
11
11
|
|
|
12
12
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
13
|
-
rev: v0.
|
|
13
|
+
rev: v0.17.3
|
|
14
14
|
hooks:
|
|
15
15
|
# Run the linter.
|
|
16
16
|
- id: ruff-check
|
|
@@ -19,7 +19,7 @@ repos:
|
|
|
19
19
|
- id: ruff-format
|
|
20
20
|
|
|
21
21
|
- repo: https://github.com/astral-sh/uv-pre-commit
|
|
22
|
-
rev: 0.
|
|
22
|
+
rev: 0.13.6
|
|
23
23
|
hooks:
|
|
24
24
|
# Ensure the lockfile is up-to-date with pyproject.toml
|
|
25
25
|
- id: uv-lock
|
|
@@ -37,7 +37,7 @@ Source lives under `src/hdx/pipelineutils/`:
|
|
|
37
37
|
- **`org_type.py`** — `OrgType(Lookup)`: resolves organisation-type codes using `org_type_configuration.yaml`.
|
|
38
38
|
- **`sector.py`** — `Sector(Lookup)`: resolves sector codes using `sector_configuration.yaml`.
|
|
39
39
|
- **`hapi_admins.py`** — `complete_admins()`: populates adm-level names from p-codes and vice-versa using `hdx-python-country` `AdminLevel` objects; returns the resolved level and a list of warnings.
|
|
40
|
-
- **`__init__.py`** — shared helpers: `string_params_to_dict`, `
|
|
40
|
+
- **`__init__.py`** — shared helpers: `string_params_to_dict`, `get_startend_dates_from_time_period`.
|
|
41
41
|
|
|
42
42
|
## Testing
|
|
43
43
|
|
|
@@ -45,7 +45,7 @@ Tests are in `tests/` and use `pytest`. No live HDX connection is required — d
|
|
|
45
45
|
|
|
46
46
|
## Code Style
|
|
47
47
|
|
|
48
|
-
Formatted and linted with `ruff` (rules: E, F, I, UP; line-length not enforced). Python ≥ 3.
|
|
48
|
+
Formatted and linted with `ruff` (rules: E, F, I, UP; line-length not enforced). Python ≥ 3.11. Use `X | Y` union syntax (PEP 604), not `Optional`/`Union`. Google-style docstrings with `Args:` and `Returns:` sections.
|
|
49
49
|
|
|
50
50
|
## Collaboration Style
|
|
51
51
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: hdx-python-pipelineutils
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.8
|
|
4
4
|
Summary: HDX Python Pipeline Utilities
|
|
5
5
|
Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-pipelineutils
|
|
6
6
|
Author-email: Michael Rans <rans@email.com>
|
|
@@ -18,16 +18,15 @@ Classifier: Operating System :: Unix
|
|
|
18
18
|
Classifier: Programming Language :: Python
|
|
19
19
|
Classifier: Programming Language :: Python :: 3
|
|
20
20
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
23
22
|
Classifier: Programming Language :: Python :: 3.12
|
|
24
23
|
Classifier: Programming Language :: Python :: 3.13
|
|
25
24
|
Classifier: Programming Language :: Python :: 3.14
|
|
26
25
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
27
|
-
Requires-Python: >=3.
|
|
28
|
-
Requires-Dist: hdx-python-api>=6.
|
|
29
|
-
Requires-Dist: hdx-python-country>=4.2.
|
|
30
|
-
Requires-Dist: hdx-python-utilities>=4.2.
|
|
26
|
+
Requires-Python: >=3.11
|
|
27
|
+
Requires-Dist: hdx-python-api>=6.7.8
|
|
28
|
+
Requires-Dist: hdx-python-country>=4.2.7
|
|
29
|
+
Requires-Dist: hdx-python-utilities>=4.2.6
|
|
31
30
|
Provides-Extra: docs
|
|
32
31
|
Requires-Dist: mkapi; extra == 'docs'
|
|
33
32
|
Description-Content-Type: text/markdown
|
|
@@ -18,7 +18,6 @@ classifiers = [
|
|
|
18
18
|
"Programming Language :: Python",
|
|
19
19
|
"Programming Language :: Python :: 3",
|
|
20
20
|
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
-
"Programming Language :: Python :: 3.10",
|
|
22
21
|
"Programming Language :: Python :: 3.11",
|
|
23
22
|
"Programming Language :: Python :: 3.12",
|
|
24
23
|
"Programming Language :: Python :: 3.13",
|
|
@@ -33,13 +32,13 @@ classifiers = [
|
|
|
33
32
|
]
|
|
34
33
|
readme = "README.md"
|
|
35
34
|
dynamic = ["version"]
|
|
36
|
-
requires-python = ">=3.
|
|
35
|
+
requires-python = ">=3.11"
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
dependencies = [
|
|
40
|
-
"hdx-python-api>=6.
|
|
41
|
-
"hdx-python-country>=4.2.
|
|
42
|
-
"hdx-python-utilities>=4.2.
|
|
39
|
+
"hdx-python-api>=6.7.8",
|
|
40
|
+
"hdx-python-country>=4.2.7",
|
|
41
|
+
"hdx-python-utilities>=4.2.6",
|
|
43
42
|
]
|
|
44
43
|
|
|
45
44
|
[project.optional-dependencies]
|
|
@@ -50,7 +49,7 @@ dev = [
|
|
|
50
49
|
"pytest",
|
|
51
50
|
"pytest-cov",
|
|
52
51
|
"pre-commit",
|
|
53
|
-
"ruff==0.
|
|
52
|
+
"ruff==0.17.3",
|
|
54
53
|
]
|
|
55
54
|
|
|
56
55
|
[project.urls]
|
|
@@ -81,7 +80,7 @@ allow-direct-references = true
|
|
|
81
80
|
# ----------------------------------------------------------------------------
|
|
82
81
|
|
|
83
82
|
[tool.ruff]
|
|
84
|
-
target-version = "
|
|
83
|
+
target-version = "py311"
|
|
85
84
|
src = ["src"]
|
|
86
85
|
exclude = ["_version.py"]
|
|
87
86
|
|
{hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/__init__.py
RENAMED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
import re
|
|
2
1
|
from datetime import datetime
|
|
3
2
|
|
|
4
3
|
from hdx.data.dataset import Dataset
|
|
5
4
|
|
|
6
|
-
template = re.compile("{{.*?}}")
|
|
7
|
-
|
|
8
5
|
|
|
9
6
|
def string_params_to_dict(string: str) -> dict[str, str]:
|
|
10
7
|
params = {}
|
|
@@ -16,22 +13,6 @@ def string_params_to_dict(string: str) -> dict[str, str]:
|
|
|
16
13
|
return params
|
|
17
14
|
|
|
18
15
|
|
|
19
|
-
def match_template(input: str) -> tuple[str | None, str | None]:
|
|
20
|
-
"""Try to match {{XXX}} in input string
|
|
21
|
-
|
|
22
|
-
Args:
|
|
23
|
-
input: String in which to look for template
|
|
24
|
-
|
|
25
|
-
Returns:
|
|
26
|
-
(Matched string with brackets, matched string without brackets)
|
|
27
|
-
"""
|
|
28
|
-
match = template.search(input)
|
|
29
|
-
if match:
|
|
30
|
-
template_string = match.group()
|
|
31
|
-
return template_string, template_string[2:-2]
|
|
32
|
-
return None, None
|
|
33
|
-
|
|
34
|
-
|
|
35
16
|
def get_startend_dates_from_time_period(
|
|
36
17
|
dataset: Dataset, today: datetime | None = None
|
|
37
18
|
) -> dict | None:
|
{hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/_version.py
RENAMED
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 0,
|
|
21
|
+
__version__ = version = '0.0.8'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 0, 8)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -38,11 +38,29 @@ def complete_admins(
|
|
|
38
38
|
warnings.append(f"PCode unknown {adm_code}->''")
|
|
39
39
|
return ""
|
|
40
40
|
|
|
41
|
+
def get_pcode_from_names(level: int) -> str | None:
|
|
42
|
+
# Exact matching only, so that a wrong parent does not scope the
|
|
43
|
+
# child lookup to the wrong area
|
|
44
|
+
pcode = None
|
|
45
|
+
for j in range(level + 1):
|
|
46
|
+
if adm_codes[j] in admins[j].pcodes:
|
|
47
|
+
pcode = adm_codes[j]
|
|
48
|
+
elif provider_adm_names[j]:
|
|
49
|
+
pcode, _ = admins[j].get_pcode(
|
|
50
|
+
countryiso3,
|
|
51
|
+
provider_adm_names[j],
|
|
52
|
+
parent=pcode,
|
|
53
|
+
fuzzy_match=False,
|
|
54
|
+
)
|
|
55
|
+
else:
|
|
56
|
+
pcode = None
|
|
57
|
+
return pcode
|
|
58
|
+
|
|
41
59
|
for i, provider_adm_name in reversed(list(enumerate(provider_adm_names))):
|
|
42
60
|
adm_code = adm_codes[i]
|
|
43
61
|
parent = admins[i].pcode_to_parent.get(adm_code)
|
|
44
62
|
if not parent and i > 0:
|
|
45
|
-
parent = adm_codes[i - 1]
|
|
63
|
+
parent = adm_codes[i - 1] or get_pcode_from_names(i - 1) or ""
|
|
46
64
|
if not provider_adm_name:
|
|
47
65
|
provider_adm_name = ""
|
|
48
66
|
provider_adm_names[i] = ""
|
{hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/lookup.py
RENAMED
|
@@ -59,7 +59,7 @@ class Lookup:
|
|
|
59
59
|
|
|
60
60
|
reader = Read.get_reader()
|
|
61
61
|
datasetinfo = self._configuration["datasetinfo"]
|
|
62
|
-
|
|
62
|
+
_headers, iterator = reader.read(
|
|
63
63
|
datasetinfo, file_prefix=self._configuration["file_prefix"]
|
|
64
64
|
)
|
|
65
65
|
code_key = self._configuration["code_key"]
|
{hdx_python_pipelineutils-0.0.6 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/reader.py
RENAMED
|
@@ -16,7 +16,7 @@ from hdx.utilities.retriever import Retrieve
|
|
|
16
16
|
from hdx.utilities.saver import save_json
|
|
17
17
|
from slugify import slugify
|
|
18
18
|
|
|
19
|
-
from . import get_startend_dates_from_time_period
|
|
19
|
+
from . import get_startend_dates_from_time_period
|
|
20
20
|
|
|
21
21
|
logger = logging.getLogger(__name__)
|
|
22
22
|
|
|
@@ -68,8 +68,8 @@ class Read(Retrieve):
|
|
|
68
68
|
temp_dir: Path | str,
|
|
69
69
|
save: bool = False,
|
|
70
70
|
use_saved: bool = False,
|
|
71
|
-
ignore: Sequence[str] =
|
|
72
|
-
rate_limit: dict | None = {"calls": 1, "period": 0.1},
|
|
71
|
+
ignore: Sequence[str] = (),
|
|
72
|
+
rate_limit: dict | None = {"calls": 1, "period": 0.1}, # noqa: B006
|
|
73
73
|
today: datetime | None = None,
|
|
74
74
|
**kwargs: Any,
|
|
75
75
|
):
|
|
@@ -151,25 +151,6 @@ class Read(Retrieve):
|
|
|
151
151
|
"""
|
|
152
152
|
return cls.get_retriever(name)
|
|
153
153
|
|
|
154
|
-
@staticmethod
|
|
155
|
-
def get_url(url: str, **kwargs: Any) -> str:
|
|
156
|
-
"""Get url from a string replacing any template arguments
|
|
157
|
-
|
|
158
|
-
Args:
|
|
159
|
-
url: Url to read
|
|
160
|
-
**kwargs: Variables to use when evaluating template arguments
|
|
161
|
-
|
|
162
|
-
Returns:
|
|
163
|
-
Url with any template arguments replaced
|
|
164
|
-
"""
|
|
165
|
-
for kwarg in kwargs:
|
|
166
|
-
globals()[kwarg] = kwargs[kwarg]
|
|
167
|
-
template_string, match_string = match_template(url)
|
|
168
|
-
if template_string:
|
|
169
|
-
replace_string = eval(match_string)
|
|
170
|
-
url = url.replace(template_string, replace_string)
|
|
171
|
-
return url
|
|
172
|
-
|
|
173
154
|
def clone(self, downloader: Download) -> "Read":
|
|
174
155
|
"""Clone a given reader but use the given downloader
|
|
175
156
|
|
|
@@ -225,8 +206,6 @@ class Read(Retrieve):
|
|
|
225
206
|
if compression:
|
|
226
207
|
kwargs["compression"] = compression
|
|
227
208
|
url = datasetinfo["url"]
|
|
228
|
-
if isinstance(url, list):
|
|
229
|
-
url = [self.get_url(x, **kwargs) for x in url]
|
|
230
209
|
filename = kwargs.get("filename")
|
|
231
210
|
if not filename:
|
|
232
211
|
filename = datasetinfo.get("filename")
|
|
@@ -348,8 +327,7 @@ class Read(Retrieve):
|
|
|
348
327
|
"""
|
|
349
328
|
filename = name.lower()
|
|
350
329
|
file_type = f".{format}"
|
|
351
|
-
|
|
352
|
-
filename = filename[: -len(file_type)]
|
|
330
|
+
filename = filename.removesuffix(file_type)
|
|
353
331
|
return f"{slugify(filename, separator='_')}{file_type}"
|
|
354
332
|
|
|
355
333
|
def construct_filename_and_download(
|
|
File without changes
|
|
@@ -96,6 +96,17 @@ class TestAdmins:
|
|
|
96
96
|
assert adm_level == 2
|
|
97
97
|
assert warnings == []
|
|
98
98
|
|
|
99
|
+
provider_adm_names = ["Kabul", "Qara Bagh"]
|
|
100
|
+
adm_codes = ["", ""]
|
|
101
|
+
adm_names = ["", ""]
|
|
102
|
+
adm_level, warnings = complete_admins(
|
|
103
|
+
admins, countryiso3, provider_adm_names, adm_codes, adm_names
|
|
104
|
+
)
|
|
105
|
+
assert adm_codes == ["AF01", "AF0114"]
|
|
106
|
+
assert adm_names == ["Kabul", "Qara Bagh"]
|
|
107
|
+
assert adm_level == 2
|
|
108
|
+
assert warnings == []
|
|
109
|
+
|
|
99
110
|
provider_adm_names = ["Kabal", "Paghman"]
|
|
100
111
|
adm_codes = ["AF01", ""]
|
|
101
112
|
adm_names = ["", ""]
|
|
@@ -7,6 +7,8 @@ class TestLookup:
|
|
|
7
7
|
sector = Sector()
|
|
8
8
|
assert sector.get_code("child protection") == "PRO-CPN"
|
|
9
9
|
assert sector.get_code("gestion des sites daccueil temporaires") == "SHL"
|
|
10
|
+
assert sector.get_code("Cash multi-sectoriel") == "Cash"
|
|
11
|
+
assert sector.get_code("Abris et Non vivres") == "SHL"
|
|
10
12
|
assert "Intersectoral" in sector.get_code_to_name()
|
|
11
13
|
assert sector.get_name("PRO-MIN") == "Mine Action"
|
|
12
14
|
assert sector.get_name("xxx") is None
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from datetime import
|
|
1
|
+
from datetime import UTC, datetime
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
from hdx.data.dataset import Dataset
|
|
@@ -12,8 +12,9 @@ from hdx.pipelineutils.reader import Read
|
|
|
12
12
|
|
|
13
13
|
class TestReaders:
|
|
14
14
|
def test_clone(self):
|
|
15
|
-
with
|
|
16
|
-
|
|
15
|
+
with (
|
|
16
|
+
Download(user_agent="test") as downloader,
|
|
17
|
+
Read(
|
|
17
18
|
downloader,
|
|
18
19
|
"fallback_dir",
|
|
19
20
|
"saved_dir",
|
|
@@ -22,12 +23,13 @@ class TestReaders:
|
|
|
22
23
|
use_saved=False,
|
|
23
24
|
prefix="population",
|
|
24
25
|
today=parse_date("2021-02-01"),
|
|
25
|
-
) as reader
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
) as reader,
|
|
27
|
+
):
|
|
28
|
+
clone_reader = reader.clone(downloader)
|
|
29
|
+
for property, value in vars(reader).items():
|
|
30
|
+
if property == "downloader":
|
|
31
|
+
continue
|
|
32
|
+
assert getattr(clone_reader, property) == value
|
|
31
33
|
|
|
32
34
|
def test_read_dataset(self, configuration, monkeypatch):
|
|
33
35
|
def read_from_hdx(dataset_name, _):
|
|
@@ -44,45 +46,47 @@ class TestReaders:
|
|
|
44
46
|
dataset.add_update_resource(resource)
|
|
45
47
|
return dataset
|
|
46
48
|
|
|
47
|
-
with
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
49
|
+
with (
|
|
50
|
+
temp_dir("TestReader") as temp_folder,
|
|
51
|
+
Download(user_agent="test") as downloader,
|
|
52
|
+
):
|
|
53
|
+
munged_url = "https://docs.google.com/spreadsheets/d/1NjSI2LaS3SqbgYc0HdD8oIb7lofGtiHgoKKATCpwVdY/export?format=csv&gid=1088874596"
|
|
54
|
+
with Read(
|
|
55
|
+
downloader,
|
|
56
|
+
temp_folder,
|
|
57
|
+
temp_folder,
|
|
58
|
+
temp_folder,
|
|
59
|
+
save=True,
|
|
60
|
+
use_saved=False,
|
|
61
|
+
prefix="test",
|
|
62
|
+
today=parse_date("2021-02-01"),
|
|
63
|
+
) as reader:
|
|
64
|
+
monkeypatch.setattr(Dataset, "read_from_hdx", read_from_hdx)
|
|
65
|
+
dataset_name = "None"
|
|
66
|
+
dataset = reader.read_dataset(dataset_name)
|
|
67
|
+
assert dataset is None
|
|
68
|
+
dataset_name = "Test Dataset"
|
|
69
|
+
dataset = reader.read_dataset(dataset_name)
|
|
70
|
+
assert dataset["name"] == dataset_name
|
|
71
|
+
assert dataset.get_resource()["url"] == munged_url
|
|
72
|
+
monkeypatch.delattr(Dataset, "read_from_hdx")
|
|
73
|
+
with Read(
|
|
74
|
+
downloader,
|
|
75
|
+
temp_folder,
|
|
76
|
+
temp_folder,
|
|
77
|
+
temp_folder,
|
|
78
|
+
save=False,
|
|
79
|
+
use_saved=True,
|
|
80
|
+
prefix="test",
|
|
81
|
+
today=parse_date("2021-02-01"),
|
|
82
|
+
) as reader:
|
|
83
|
+
dataset_name = "None"
|
|
84
|
+
dataset = reader.read_dataset(dataset_name)
|
|
85
|
+
assert dataset is None
|
|
86
|
+
dataset_name = "Test Dataset"
|
|
87
|
+
dataset = reader.read_dataset(dataset_name)
|
|
88
|
+
assert dataset["name"] == dataset_name
|
|
89
|
+
assert dataset.get_resource()["url"] == munged_url
|
|
86
90
|
|
|
87
91
|
def test_search_datasets(self, configuration, monkeypatch):
|
|
88
92
|
filename = "TestDataset"
|
|
@@ -102,50 +106,50 @@ class TestReaders:
|
|
|
102
106
|
datasets.append(dataset)
|
|
103
107
|
return datasets
|
|
104
108
|
|
|
105
|
-
with
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
109
|
+
with (
|
|
110
|
+
temp_dir("TestReader") as temp_folder,
|
|
111
|
+
Download(user_agent="test") as downloader,
|
|
112
|
+
):
|
|
113
|
+
with Read(
|
|
114
|
+
downloader,
|
|
115
|
+
temp_folder,
|
|
116
|
+
temp_folder,
|
|
117
|
+
temp_folder,
|
|
118
|
+
save=True,
|
|
119
|
+
use_saved=False,
|
|
120
|
+
prefix="test",
|
|
121
|
+
today=parse_date("2021-02-01"),
|
|
122
|
+
) as reader:
|
|
123
|
+
monkeypatch.setattr(Dataset, "search_in_hdx", search_in_hdx)
|
|
124
|
+
datasets = reader.search_datasets(filename)
|
|
125
|
+
assert len(datasets) == 2
|
|
126
|
+
dataset = datasets[0]
|
|
127
|
+
assert dataset["name"] == f"{filename}_0"
|
|
128
|
+
assert dataset.get_resource()["url"] == f"{filename}_0.json"
|
|
129
|
+
dataset = datasets[1]
|
|
130
|
+
assert dataset["name"] == f"{filename}_1"
|
|
131
|
+
assert dataset.get_resource()["url"] == f"{filename}_1.json"
|
|
132
|
+
monkeypatch.delattr(Dataset, "search_in_hdx")
|
|
133
|
+
with Read(
|
|
134
|
+
downloader,
|
|
135
|
+
temp_folder,
|
|
136
|
+
temp_folder,
|
|
137
|
+
temp_folder,
|
|
138
|
+
save=False,
|
|
139
|
+
use_saved=True,
|
|
140
|
+
prefix="test",
|
|
141
|
+
today=parse_date("2021-02-01"),
|
|
142
|
+
) as reader:
|
|
143
|
+
datasets = reader.search_datasets(filename)
|
|
144
|
+
assert len(datasets) == 2
|
|
145
|
+
dataset = datasets[0]
|
|
146
|
+
assert dataset["name"] == f"{filename}_0"
|
|
147
|
+
assert dataset.get_resource()["url"] == f"{filename}_0.json"
|
|
148
|
+
dataset = datasets[1]
|
|
149
|
+
assert dataset["name"] == f"{filename}_1"
|
|
150
|
+
assert dataset.get_resource()["url"] == f"{filename}_1.json"
|
|
145
151
|
|
|
146
152
|
def test_read(self, configuration):
|
|
147
|
-
url = Read.get_url("http://{{var}}", var="hello")
|
|
148
|
-
assert url == "http://hello"
|
|
149
153
|
datasetinfo = {
|
|
150
154
|
"name": "test",
|
|
151
155
|
"dataset": "sahel-humanitarian-needs-overview",
|
|
@@ -180,8 +184,8 @@ class TestReaders:
|
|
|
180
184
|
"hdx_provider_name": "OCHA West and Central Africa (ROWCA)",
|
|
181
185
|
"license": "[Creative Commons Attribution International](http://www.opendefinition.org/licenses/cc-by)",
|
|
182
186
|
"time_period": {
|
|
183
|
-
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=
|
|
184
|
-
"start": datetime(2016, 9, 1, 0, 0, tzinfo=
|
|
187
|
+
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
|
|
188
|
+
"start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
|
|
185
189
|
},
|
|
186
190
|
"title": "Sahel : Humanitarian Needs Overview",
|
|
187
191
|
},
|
|
@@ -190,19 +194,19 @@ class TestReaders:
|
|
|
190
194
|
"name": "HNO -2017 -Sahel-nutrition.csv",
|
|
191
195
|
"format": "csv",
|
|
192
196
|
"hdx_id": "2527ac5b-66fe-46f0-8b9b-7086d2c4ddd3",
|
|
193
|
-
"update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=
|
|
197
|
+
"update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=UTC),
|
|
194
198
|
},
|
|
195
199
|
"headers": 1,
|
|
196
200
|
"name": "test",
|
|
197
201
|
"source": "Multiple organisations",
|
|
198
202
|
"source_date": {
|
|
199
|
-
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=
|
|
200
|
-
"start": datetime(2016, 9, 1, 0, 0, tzinfo=
|
|
203
|
+
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
|
|
204
|
+
"start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
|
|
201
205
|
},
|
|
202
206
|
"source_url": "https://data.humdata.org/dataset/sahel-humanitarian-needs-overview",
|
|
203
207
|
"time_period": {
|
|
204
|
-
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=
|
|
205
|
-
"start": datetime(2016, 9, 1, 0, 0, tzinfo=
|
|
208
|
+
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
|
|
209
|
+
"start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
|
|
206
210
|
},
|
|
207
211
|
"url": "https://data.humdata.org/dataset/47f6ef46-500f-421a-9fa2-fefd93facf95/resource/2527ac5b-66fe-46f0-8b9b-7086d2c4ddd3/download/hno-2017-sahel-nutrition.csv",
|
|
208
212
|
}
|
|
@@ -251,8 +255,8 @@ class TestReaders:
|
|
|
251
255
|
"hdx_provider_name": "OCHA West and Central Africa (ROWCA)",
|
|
252
256
|
"license": "[Creative Commons Attribution International](http://www.opendefinition.org/licenses/cc-by)",
|
|
253
257
|
"time_period": {
|
|
254
|
-
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=
|
|
255
|
-
"start": datetime(2016, 9, 1, 0, 0, tzinfo=
|
|
258
|
+
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
|
|
259
|
+
"start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
|
|
256
260
|
},
|
|
257
261
|
"title": "Sahel : Humanitarian Needs Overview",
|
|
258
262
|
},
|
|
@@ -261,7 +265,7 @@ class TestReaders:
|
|
|
261
265
|
"name": "HNO-2017-Sahel- People in need.xlsx",
|
|
262
266
|
"format": "xlsx",
|
|
263
267
|
"hdx_id": "d9248be4-7bfb-4a81-a7aa-c035dcb737a2",
|
|
264
|
-
"update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=
|
|
268
|
+
"update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=UTC),
|
|
265
269
|
},
|
|
266
270
|
"headers": 1,
|
|
267
271
|
"name": "test",
|
|
@@ -269,13 +273,13 @@ class TestReaders:
|
|
|
269
273
|
"sheet": 1,
|
|
270
274
|
"source": "Multiple organisations",
|
|
271
275
|
"source_date": {
|
|
272
|
-
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=
|
|
273
|
-
"start": datetime(2016, 9, 1, 0, 0, tzinfo=
|
|
276
|
+
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
|
|
277
|
+
"start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
|
|
274
278
|
},
|
|
275
279
|
"source_url": "https://data.humdata.org/dataset/sahel-humanitarian-needs-overview",
|
|
276
280
|
"time_period": {
|
|
277
|
-
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=
|
|
278
|
-
"start": datetime(2016, 9, 1, 0, 0, tzinfo=
|
|
281
|
+
"end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
|
|
282
|
+
"start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
|
|
279
283
|
},
|
|
280
284
|
"url": "https://data.humdata.org/dataset/47f6ef46-500f-421a-9fa2-fefd93facf95/resource/d9248be4-7bfb-4a81-a7aa-c035dcb737a2/download/hno-2017-sahel-people-in-need.xlsx",
|
|
281
285
|
}
|