hdx-python-pipelineutils 0.0.7__tar.gz → 0.0.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/.github/workflows/publish.yaml +2 -2
  2. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/.github/workflows/run-python-tests.yaml +4 -4
  3. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/.pre-commit-config.yaml +2 -2
  4. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/CLAUDE.md +2 -2
  5. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/PKG-INFO +5 -6
  6. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/pyproject.toml +6 -7
  7. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/__init__.py +0 -19
  8. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/_version.py +2 -2
  9. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/lookup.py +1 -1
  10. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/reader.py +4 -26
  11. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/conftest.py +0 -0
  12. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/test_readers.py +108 -104
  13. hdx_python_pipelineutils-0.0.8/uv.lock +2225 -0
  14. hdx_python_pipelineutils-0.0.7/uv.lock +0 -2273
  15. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/.gitignore +0 -0
  16. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/LICENSE +0 -0
  17. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/README.md +0 -0
  18. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/hapi_admins.py +0 -0
  19. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/org_type.py +0 -0
  20. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/org_type_configuration.yaml +0 -0
  21. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/sector.py +0 -0
  22. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/src/hdx/pipelineutils/sector_configuration.yaml +0 -0
  23. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/download-global-pcode-lengths.csv +0 -0
  24. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/download-global-pcodes-adm-1-2.csv +0 -0
  25. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/education_closures_school_closures.csv +0 -0
  26. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/global-coordination-groups-beta.json +0 -0
  27. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/hno_2017_sahel_nutrition.csv +0 -0
  28. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/hno_2017_sahel_people_in_need.xlsx +0 -0
  29. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/org_type_organization_types_beta_csv_no_hxl.csv +0 -0
  30. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/organization-types-beta.json +0 -0
  31. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/sahel-humanitarian-needs-overview.json +0 -0
  32. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/fixtures/input/sector_global_coordination_groups_beta_csv_no_hxl.csv +0 -0
  33. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/test_hapi_admins.py +0 -0
  34. {hdx_python_pipelineutils-0.0.7 → hdx_python_pipelineutils-0.0.8}/tests/test_lookup.py +0 -0
@@ -17,7 +17,7 @@ jobs:
17
17
  contents: read
18
18
 
19
19
  steps:
20
- - uses: actions/checkout@v6
20
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
21
21
 
22
22
  - name: Get history and tags for versioning to work
23
23
  run: |
@@ -25,7 +25,7 @@ jobs:
25
25
  git fetch --depth=1 origin +refs/tags/*:refs/tags/*
26
26
 
27
27
  - name: Install uv
28
- uses: astral-sh/setup-uv@v7
28
+ uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
29
29
 
30
30
  - name: Build with uv
31
31
  run: uv build
@@ -16,10 +16,10 @@ jobs:
16
16
  pull-requests: write
17
17
 
18
18
  steps:
19
- - uses: actions/checkout@v6
19
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
20
20
 
21
21
  - name: Install uv
22
- uses: astral-sh/setup-uv@v7
22
+ uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
23
23
  with:
24
24
  enable-cache: true
25
25
  python-version: "3.13"
@@ -39,13 +39,13 @@ jobs:
39
39
  run: uv run pytest
40
40
 
41
41
  - name: Publish Unit Test Results
42
- uses: EnricoMi/publish-unit-test-result-action@v2
42
+ uses: EnricoMi/publish-unit-test-result-action@7622c5bf3d112a72ab4fa3951c2aa5723d8ab92d # v2.25.0
43
43
  if: always()
44
44
  with:
45
45
  files: test-results.xml
46
46
 
47
47
  - name: Publish in Coveralls
48
- uses: coverallsapp/github-action@v2
48
+ uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8
49
49
  if: always()
50
50
  with:
51
51
  flag-name: tests
@@ -10,7 +10,7 @@ repos:
10
10
  - id: check-ast
11
11
 
12
12
  - repo: https://github.com/astral-sh/ruff-pre-commit
13
- rev: v0.15.7
13
+ rev: v0.17.3
14
14
  hooks:
15
15
  # Run the linter.
16
16
  - id: ruff-check
@@ -19,7 +19,7 @@ repos:
19
19
  - id: ruff-format
20
20
 
21
21
  - repo: https://github.com/astral-sh/uv-pre-commit
22
- rev: 0.10.12
22
+ rev: 0.13.6
23
23
  hooks:
24
24
  # Ensure the lockfile is up-to-date with pyproject.toml
25
25
  - id: uv-lock
@@ -37,7 +37,7 @@ Source lives under `src/hdx/pipelineutils/`:
37
37
  - **`org_type.py`** — `OrgType(Lookup)`: resolves organisation-type codes using `org_type_configuration.yaml`.
38
38
  - **`sector.py`** — `Sector(Lookup)`: resolves sector codes using `sector_configuration.yaml`.
39
39
  - **`hapi_admins.py`** — `complete_admins()`: populates adm-level names from p-codes and vice-versa using `hdx-python-country` `AdminLevel` objects; returns the resolved level and a list of warnings.
40
- - **`__init__.py`** — shared helpers: `string_params_to_dict`, `match_template` (extracts `{{...}}` placeholders), `get_startend_dates_from_time_period`.
40
+ - **`__init__.py`** — shared helpers: `string_params_to_dict`, `get_startend_dates_from_time_period`.
41
41
 
42
42
  ## Testing
43
43
 
@@ -45,7 +45,7 @@ Tests are in `tests/` and use `pytest`. No live HDX connection is required — d
45
45
 
46
46
  ## Code Style
47
47
 
48
- Formatted and linted with `ruff` (rules: E, F, I, UP; line-length not enforced). Python ≥ 3.10. Use `X | Y` union syntax (PEP 604), not `Optional`/`Union`. Google-style docstrings with `Args:` and `Returns:` sections.
48
+ Formatted and linted with `ruff` (rules: E, F, I, UP; line-length not enforced). Python ≥ 3.11. Use `X | Y` union syntax (PEP 604), not `Optional`/`Union`. Google-style docstrings with `Args:` and `Returns:` sections.
49
49
 
50
50
  ## Collaboration Style
51
51
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: hdx-python-pipelineutils
3
- Version: 0.0.7
3
+ Version: 0.0.8
4
4
  Summary: HDX Python Pipeline Utilities
5
5
  Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-pipelineutils
6
6
  Author-email: Michael Rans <rans@email.com>
@@ -18,16 +18,15 @@ Classifier: Operating System :: Unix
18
18
  Classifier: Programming Language :: Python
19
19
  Classifier: Programming Language :: Python :: 3
20
20
  Classifier: Programming Language :: Python :: 3 :: Only
21
- Classifier: Programming Language :: Python :: 3.10
22
21
  Classifier: Programming Language :: Python :: 3.11
23
22
  Classifier: Programming Language :: Python :: 3.12
24
23
  Classifier: Programming Language :: Python :: 3.13
25
24
  Classifier: Programming Language :: Python :: 3.14
26
25
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
27
- Requires-Python: >=3.10
28
- Requires-Dist: hdx-python-api>=6.6.7
29
- Requires-Dist: hdx-python-country>=4.2.1
30
- Requires-Dist: hdx-python-utilities>=4.2.0
26
+ Requires-Python: >=3.11
27
+ Requires-Dist: hdx-python-api>=6.7.8
28
+ Requires-Dist: hdx-python-country>=4.2.7
29
+ Requires-Dist: hdx-python-utilities>=4.2.6
31
30
  Provides-Extra: docs
32
31
  Requires-Dist: mkapi; extra == 'docs'
33
32
  Description-Content-Type: text/markdown
@@ -18,7 +18,6 @@ classifiers = [
18
18
  "Programming Language :: Python",
19
19
  "Programming Language :: Python :: 3",
20
20
  "Programming Language :: Python :: 3 :: Only",
21
- "Programming Language :: Python :: 3.10",
22
21
  "Programming Language :: Python :: 3.11",
23
22
  "Programming Language :: Python :: 3.12",
24
23
  "Programming Language :: Python :: 3.13",
@@ -33,13 +32,13 @@ classifiers = [
33
32
  ]
34
33
  readme = "README.md"
35
34
  dynamic = ["version"]
36
- requires-python = ">=3.10"
35
+ requires-python = ">=3.11"
37
36
 
38
37
 
39
38
  dependencies = [
40
- "hdx-python-api>=6.6.7",
41
- "hdx-python-country>=4.2.1",
42
- "hdx-python-utilities>=4.2.0",
39
+ "hdx-python-api>=6.7.8",
40
+ "hdx-python-country>=4.2.7",
41
+ "hdx-python-utilities>=4.2.6",
43
42
  ]
44
43
 
45
44
  [project.optional-dependencies]
@@ -50,7 +49,7 @@ dev = [
50
49
  "pytest",
51
50
  "pytest-cov",
52
51
  "pre-commit",
53
- "ruff==0.15.7",
52
+ "ruff==0.17.3",
54
53
  ]
55
54
 
56
55
  [project.urls]
@@ -81,7 +80,7 @@ allow-direct-references = true
81
80
  # ----------------------------------------------------------------------------
82
81
 
83
82
  [tool.ruff]
84
- target-version = "py310"
83
+ target-version = "py311"
85
84
  src = ["src"]
86
85
  exclude = ["_version.py"]
87
86
 
@@ -1,10 +1,7 @@
1
- import re
2
1
  from datetime import datetime
3
2
 
4
3
  from hdx.data.dataset import Dataset
5
4
 
6
- template = re.compile("{{.*?}}")
7
-
8
5
 
9
6
  def string_params_to_dict(string: str) -> dict[str, str]:
10
7
  params = {}
@@ -16,22 +13,6 @@ def string_params_to_dict(string: str) -> dict[str, str]:
16
13
  return params
17
14
 
18
15
 
19
- def match_template(input: str) -> tuple[str | None, str | None]:
20
- """Try to match {{XXX}} in input string
21
-
22
- Args:
23
- input: String in which to look for template
24
-
25
- Returns:
26
- (Matched string with brackets, matched string without brackets)
27
- """
28
- match = template.search(input)
29
- if match:
30
- template_string = match.group()
31
- return template_string, template_string[2:-2]
32
- return None, None
33
-
34
-
35
16
  def get_startend_dates_from_time_period(
36
17
  dataset: Dataset, today: datetime | None = None
37
18
  ) -> dict | None:
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.0.7'
22
- __version_tuple__ = version_tuple = (0, 0, 7)
21
+ __version__ = version = '0.0.8'
22
+ __version_tuple__ = version_tuple = (0, 0, 8)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -59,7 +59,7 @@ class Lookup:
59
59
 
60
60
  reader = Read.get_reader()
61
61
  datasetinfo = self._configuration["datasetinfo"]
62
- headers, iterator = reader.read(
62
+ _headers, iterator = reader.read(
63
63
  datasetinfo, file_prefix=self._configuration["file_prefix"]
64
64
  )
65
65
  code_key = self._configuration["code_key"]
@@ -16,7 +16,7 @@ from hdx.utilities.retriever import Retrieve
16
16
  from hdx.utilities.saver import save_json
17
17
  from slugify import slugify
18
18
 
19
- from . import get_startend_dates_from_time_period, match_template
19
+ from . import get_startend_dates_from_time_period
20
20
 
21
21
  logger = logging.getLogger(__name__)
22
22
 
@@ -68,8 +68,8 @@ class Read(Retrieve):
68
68
  temp_dir: Path | str,
69
69
  save: bool = False,
70
70
  use_saved: bool = False,
71
- ignore: Sequence[str] = tuple(),
72
- rate_limit: dict | None = {"calls": 1, "period": 0.1},
71
+ ignore: Sequence[str] = (),
72
+ rate_limit: dict | None = {"calls": 1, "period": 0.1}, # noqa: B006
73
73
  today: datetime | None = None,
74
74
  **kwargs: Any,
75
75
  ):
@@ -151,25 +151,6 @@ class Read(Retrieve):
151
151
  """
152
152
  return cls.get_retriever(name)
153
153
 
154
- @staticmethod
155
- def get_url(url: str, **kwargs: Any) -> str:
156
- """Get url from a string replacing any template arguments
157
-
158
- Args:
159
- url: Url to read
160
- **kwargs: Variables to use when evaluating template arguments
161
-
162
- Returns:
163
- Url with any template arguments replaced
164
- """
165
- for kwarg in kwargs:
166
- globals()[kwarg] = kwargs[kwarg]
167
- template_string, match_string = match_template(url)
168
- if template_string:
169
- replace_string = eval(match_string)
170
- url = url.replace(template_string, replace_string)
171
- return url
172
-
173
154
  def clone(self, downloader: Download) -> "Read":
174
155
  """Clone a given reader but use the given downloader
175
156
 
@@ -225,8 +206,6 @@ class Read(Retrieve):
225
206
  if compression:
226
207
  kwargs["compression"] = compression
227
208
  url = datasetinfo["url"]
228
- if isinstance(url, list):
229
- url = [self.get_url(x, **kwargs) for x in url]
230
209
  filename = kwargs.get("filename")
231
210
  if not filename:
232
211
  filename = datasetinfo.get("filename")
@@ -348,8 +327,7 @@ class Read(Retrieve):
348
327
  """
349
328
  filename = name.lower()
350
329
  file_type = f".{format}"
351
- if filename.endswith(file_type):
352
- filename = filename[: -len(file_type)]
330
+ filename = filename.removesuffix(file_type)
353
331
  return f"{slugify(filename, separator='_')}{file_type}"
354
332
 
355
333
  def construct_filename_and_download(
@@ -1,4 +1,4 @@
1
- from datetime import datetime, timezone
1
+ from datetime import UTC, datetime
2
2
 
3
3
  import pytest
4
4
  from hdx.data.dataset import Dataset
@@ -12,8 +12,9 @@ from hdx.pipelineutils.reader import Read
12
12
 
13
13
  class TestReaders:
14
14
  def test_clone(self):
15
- with Download(user_agent="test") as downloader:
16
- with Read(
15
+ with (
16
+ Download(user_agent="test") as downloader,
17
+ Read(
17
18
  downloader,
18
19
  "fallback_dir",
19
20
  "saved_dir",
@@ -22,12 +23,13 @@ class TestReaders:
22
23
  use_saved=False,
23
24
  prefix="population",
24
25
  today=parse_date("2021-02-01"),
25
- ) as reader:
26
- clone_reader = reader.clone(downloader)
27
- for property, value in vars(reader).items():
28
- if property == "downloader":
29
- continue
30
- assert getattr(clone_reader, property) == value
26
+ ) as reader,
27
+ ):
28
+ clone_reader = reader.clone(downloader)
29
+ for property, value in vars(reader).items():
30
+ if property == "downloader":
31
+ continue
32
+ assert getattr(clone_reader, property) == value
31
33
 
32
34
  def test_read_dataset(self, configuration, monkeypatch):
33
35
  def read_from_hdx(dataset_name, _):
@@ -44,45 +46,47 @@ class TestReaders:
44
46
  dataset.add_update_resource(resource)
45
47
  return dataset
46
48
 
47
- with temp_dir("TestReader") as temp_folder:
48
- with Download(user_agent="test") as downloader:
49
- munged_url = "https://docs.google.com/spreadsheets/d/1NjSI2LaS3SqbgYc0HdD8oIb7lofGtiHgoKKATCpwVdY/export?format=csv&gid=1088874596"
50
- with Read(
51
- downloader,
52
- temp_folder,
53
- temp_folder,
54
- temp_folder,
55
- save=True,
56
- use_saved=False,
57
- prefix="test",
58
- today=parse_date("2021-02-01"),
59
- ) as reader:
60
- monkeypatch.setattr(Dataset, "read_from_hdx", read_from_hdx)
61
- dataset_name = "None"
62
- dataset = reader.read_dataset(dataset_name)
63
- assert dataset is None
64
- dataset_name = "Test Dataset"
65
- dataset = reader.read_dataset(dataset_name)
66
- assert dataset["name"] == dataset_name
67
- assert dataset.get_resource()["url"] == munged_url
68
- monkeypatch.delattr(Dataset, "read_from_hdx")
69
- with Read(
70
- downloader,
71
- temp_folder,
72
- temp_folder,
73
- temp_folder,
74
- save=False,
75
- use_saved=True,
76
- prefix="test",
77
- today=parse_date("2021-02-01"),
78
- ) as reader:
79
- dataset_name = "None"
80
- dataset = reader.read_dataset(dataset_name)
81
- assert dataset is None
82
- dataset_name = "Test Dataset"
83
- dataset = reader.read_dataset(dataset_name)
84
- assert dataset["name"] == dataset_name
85
- assert dataset.get_resource()["url"] == munged_url
49
+ with (
50
+ temp_dir("TestReader") as temp_folder,
51
+ Download(user_agent="test") as downloader,
52
+ ):
53
+ munged_url = "https://docs.google.com/spreadsheets/d/1NjSI2LaS3SqbgYc0HdD8oIb7lofGtiHgoKKATCpwVdY/export?format=csv&gid=1088874596"
54
+ with Read(
55
+ downloader,
56
+ temp_folder,
57
+ temp_folder,
58
+ temp_folder,
59
+ save=True,
60
+ use_saved=False,
61
+ prefix="test",
62
+ today=parse_date("2021-02-01"),
63
+ ) as reader:
64
+ monkeypatch.setattr(Dataset, "read_from_hdx", read_from_hdx)
65
+ dataset_name = "None"
66
+ dataset = reader.read_dataset(dataset_name)
67
+ assert dataset is None
68
+ dataset_name = "Test Dataset"
69
+ dataset = reader.read_dataset(dataset_name)
70
+ assert dataset["name"] == dataset_name
71
+ assert dataset.get_resource()["url"] == munged_url
72
+ monkeypatch.delattr(Dataset, "read_from_hdx")
73
+ with Read(
74
+ downloader,
75
+ temp_folder,
76
+ temp_folder,
77
+ temp_folder,
78
+ save=False,
79
+ use_saved=True,
80
+ prefix="test",
81
+ today=parse_date("2021-02-01"),
82
+ ) as reader:
83
+ dataset_name = "None"
84
+ dataset = reader.read_dataset(dataset_name)
85
+ assert dataset is None
86
+ dataset_name = "Test Dataset"
87
+ dataset = reader.read_dataset(dataset_name)
88
+ assert dataset["name"] == dataset_name
89
+ assert dataset.get_resource()["url"] == munged_url
86
90
 
87
91
  def test_search_datasets(self, configuration, monkeypatch):
88
92
  filename = "TestDataset"
@@ -102,50 +106,50 @@ class TestReaders:
102
106
  datasets.append(dataset)
103
107
  return datasets
104
108
 
105
- with temp_dir("TestReader") as temp_folder:
106
- with Download(user_agent="test") as downloader:
107
- with Read(
108
- downloader,
109
- temp_folder,
110
- temp_folder,
111
- temp_folder,
112
- save=True,
113
- use_saved=False,
114
- prefix="test",
115
- today=parse_date("2021-02-01"),
116
- ) as reader:
117
- monkeypatch.setattr(Dataset, "search_in_hdx", search_in_hdx)
118
- datasets = reader.search_datasets(filename)
119
- assert len(datasets) == 2
120
- dataset = datasets[0]
121
- assert dataset["name"] == f"{filename}_0"
122
- assert dataset.get_resource()["url"] == f"{filename}_0.json"
123
- dataset = datasets[1]
124
- assert dataset["name"] == f"{filename}_1"
125
- assert dataset.get_resource()["url"] == f"{filename}_1.json"
126
- monkeypatch.delattr(Dataset, "search_in_hdx")
127
- with Read(
128
- downloader,
129
- temp_folder,
130
- temp_folder,
131
- temp_folder,
132
- save=False,
133
- use_saved=True,
134
- prefix="test",
135
- today=parse_date("2021-02-01"),
136
- ) as reader:
137
- datasets = reader.search_datasets(filename)
138
- assert len(datasets) == 2
139
- dataset = datasets[0]
140
- assert dataset["name"] == f"{filename}_0"
141
- assert dataset.get_resource()["url"] == f"{filename}_0.json"
142
- dataset = datasets[1]
143
- assert dataset["name"] == f"{filename}_1"
144
- assert dataset.get_resource()["url"] == f"{filename}_1.json"
109
+ with (
110
+ temp_dir("TestReader") as temp_folder,
111
+ Download(user_agent="test") as downloader,
112
+ ):
113
+ with Read(
114
+ downloader,
115
+ temp_folder,
116
+ temp_folder,
117
+ temp_folder,
118
+ save=True,
119
+ use_saved=False,
120
+ prefix="test",
121
+ today=parse_date("2021-02-01"),
122
+ ) as reader:
123
+ monkeypatch.setattr(Dataset, "search_in_hdx", search_in_hdx)
124
+ datasets = reader.search_datasets(filename)
125
+ assert len(datasets) == 2
126
+ dataset = datasets[0]
127
+ assert dataset["name"] == f"{filename}_0"
128
+ assert dataset.get_resource()["url"] == f"{filename}_0.json"
129
+ dataset = datasets[1]
130
+ assert dataset["name"] == f"{filename}_1"
131
+ assert dataset.get_resource()["url"] == f"{filename}_1.json"
132
+ monkeypatch.delattr(Dataset, "search_in_hdx")
133
+ with Read(
134
+ downloader,
135
+ temp_folder,
136
+ temp_folder,
137
+ temp_folder,
138
+ save=False,
139
+ use_saved=True,
140
+ prefix="test",
141
+ today=parse_date("2021-02-01"),
142
+ ) as reader:
143
+ datasets = reader.search_datasets(filename)
144
+ assert len(datasets) == 2
145
+ dataset = datasets[0]
146
+ assert dataset["name"] == f"{filename}_0"
147
+ assert dataset.get_resource()["url"] == f"{filename}_0.json"
148
+ dataset = datasets[1]
149
+ assert dataset["name"] == f"{filename}_1"
150
+ assert dataset.get_resource()["url"] == f"{filename}_1.json"
145
151
 
146
152
  def test_read(self, configuration):
147
- url = Read.get_url("http://{{var}}", var="hello")
148
- assert url == "http://hello"
149
153
  datasetinfo = {
150
154
  "name": "test",
151
155
  "dataset": "sahel-humanitarian-needs-overview",
@@ -180,8 +184,8 @@ class TestReaders:
180
184
  "hdx_provider_name": "OCHA West and Central Africa (ROWCA)",
181
185
  "license": "[Creative Commons Attribution International](http://www.opendefinition.org/licenses/cc-by)",
182
186
  "time_period": {
183
- "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=timezone.utc),
184
- "start": datetime(2016, 9, 1, 0, 0, tzinfo=timezone.utc),
187
+ "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
188
+ "start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
185
189
  },
186
190
  "title": "Sahel : Humanitarian Needs Overview",
187
191
  },
@@ -190,19 +194,19 @@ class TestReaders:
190
194
  "name": "HNO -2017 -Sahel-nutrition.csv",
191
195
  "format": "csv",
192
196
  "hdx_id": "2527ac5b-66fe-46f0-8b9b-7086d2c4ddd3",
193
- "update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=timezone.utc),
197
+ "update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=UTC),
194
198
  },
195
199
  "headers": 1,
196
200
  "name": "test",
197
201
  "source": "Multiple organisations",
198
202
  "source_date": {
199
- "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=timezone.utc),
200
- "start": datetime(2016, 9, 1, 0, 0, tzinfo=timezone.utc),
203
+ "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
204
+ "start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
201
205
  },
202
206
  "source_url": "https://data.humdata.org/dataset/sahel-humanitarian-needs-overview",
203
207
  "time_period": {
204
- "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=timezone.utc),
205
- "start": datetime(2016, 9, 1, 0, 0, tzinfo=timezone.utc),
208
+ "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
209
+ "start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
206
210
  },
207
211
  "url": "https://data.humdata.org/dataset/47f6ef46-500f-421a-9fa2-fefd93facf95/resource/2527ac5b-66fe-46f0-8b9b-7086d2c4ddd3/download/hno-2017-sahel-nutrition.csv",
208
212
  }
@@ -251,8 +255,8 @@ class TestReaders:
251
255
  "hdx_provider_name": "OCHA West and Central Africa (ROWCA)",
252
256
  "license": "[Creative Commons Attribution International](http://www.opendefinition.org/licenses/cc-by)",
253
257
  "time_period": {
254
- "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=timezone.utc),
255
- "start": datetime(2016, 9, 1, 0, 0, tzinfo=timezone.utc),
258
+ "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
259
+ "start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
256
260
  },
257
261
  "title": "Sahel : Humanitarian Needs Overview",
258
262
  },
@@ -261,7 +265,7 @@ class TestReaders:
261
265
  "name": "HNO-2017-Sahel- People in need.xlsx",
262
266
  "format": "xlsx",
263
267
  "hdx_id": "d9248be4-7bfb-4a81-a7aa-c035dcb737a2",
264
- "update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=timezone.utc),
268
+ "update_date": datetime(2017, 3, 10, 10, 8, 37, tzinfo=UTC),
265
269
  },
266
270
  "headers": 1,
267
271
  "name": "test",
@@ -269,13 +273,13 @@ class TestReaders:
269
273
  "sheet": 1,
270
274
  "source": "Multiple organisations",
271
275
  "source_date": {
272
- "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=timezone.utc),
273
- "start": datetime(2016, 9, 1, 0, 0, tzinfo=timezone.utc),
276
+ "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
277
+ "start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
274
278
  },
275
279
  "source_url": "https://data.humdata.org/dataset/sahel-humanitarian-needs-overview",
276
280
  "time_period": {
277
- "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=timezone.utc),
278
- "start": datetime(2016, 9, 1, 0, 0, tzinfo=timezone.utc),
281
+ "end": datetime(2016, 9, 1, 23, 59, 59, tzinfo=UTC),
282
+ "start": datetime(2016, 9, 1, 0, 0, tzinfo=UTC),
279
283
  },
280
284
  "url": "https://data.humdata.org/dataset/47f6ef46-500f-421a-9fa2-fefd93facf95/resource/d9248be4-7bfb-4a81-a7aa-c035dcb737a2/download/hno-2017-sahel-people-in-need.xlsx",
281
285
  }