hdx-python-scraper 2.7.0__tar.gz → 2.7.1__tar.gz

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.7.0
+Version: 2.7.1
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>
@@ -26,9 +26,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Requires-Dist: gspread
-Requires-Dist: hdx-python-api>=6.6.0
-Requires-Dist: hdx-python-country>=4.0.0
-Requires-Dist: hdx-python-utilities>=4.0.0
+Requires-Dist: hdx-python-api>=6.6.1
+Requires-Dist: hdx-python-country>=4.0.1
+Requires-Dist: hdx-python-utilities>=4.0.2
 Requires-Dist: regex
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.10"
 
 dependencies = [
-    "hdx-python-api>=6.6.0",
-    "hdx-python-country>=4.0.0",
-    "hdx-python-utilities>=4.0.0",
+    "hdx-python-api>=6.6.1",
+    "hdx-python-country>=4.0.1",
+    "hdx-python-utilities>=4.0.2",
     "gspread",
     "regex",
 ]

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/requirements.txt

@@ -42,13 +42,13 @@ google-auth-oauthlib==1.2.2
     # via gspread
 gspread==6.2.1
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.6.0
+hdx-python-api==6.6.1
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==4.0.0
+hdx-python-country==4.0.1
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==4.0.0
+hdx-python-utilities==4.0.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
@@ -146,7 +146,7 @@ referencing==0.37.0
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2025.11.3
+regex==2026.1.15
     # via hdx-python-scraper (pyproject.toml)
 requests==2.32.5
     # via

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.7.0'
-__version_tuple__ = version_tuple = (2, 7, 0)
+__version__ = version = '2.7.1'
+__version_tuple__ = version_tuple = (2, 7, 1)
 
 __commit_id__ = commit_id = None

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/outputs/excelfile.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 
 from openpyxl import Workbook
 
@@ -24,7 +25,7 @@ class ExcelFile(BaseOutput):
     """
 
     def __init__(
-        self, excel_path: str, tabs: dict[str, str], updatetabs: list[str]
+        self, excel_path: Path | str, tabs: dict[str, str], updatetabs: list[str]
     ) -> None:
         super().__init__(updatetabs)
         self.workbook = Workbook()

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/outputs/json.py

@@ -1,5 +1,5 @@
 import logging
-from os.path import join
+from pathlib import Path
 from typing import Any
 
 from hdx.utilities.dictandlist import dict_of_lists_add
@@ -184,7 +184,7 @@ class JsonFile(BaseOutput):
                         newrow[hxl_row[key]] = row[key]
                 self.add_data_row(name, newrow)
 
-    def save(self, folder: str | None = None, **kwargs: Any) -> list[str]:
+    def save(self, folder: Path | str | None = None, **kwargs: Any) -> list[Path]:
         """Save JSON file and any addition subsets of that JSON defined in the additional configuration
 
         Args:
@@ -197,7 +197,8 @@ class JsonFile(BaseOutput):
         filepaths = []
         filepath = self.configuration["output"]
         if folder:
-            filepath = join(folder, filepath)
+            folder = Path(folder)
+            filepath = folder / filepath
         logger.info(f"Writing JSON to {filepath}")
         save_json(self.json, filepath)
         filepaths.append(filepath)
@@ -262,7 +263,7 @@ class JsonFile(BaseOutput):
                 continue
             filedetailspath = filedetails["filepath"]
             if folder:
-                filedetailspath = join(folder, filedetailspath)
+                filedetailspath = folder / filedetailspath
             logger.info(f"Writing JSON to {filedetailspath}")
             save_json(json, filedetailspath)
             filepaths.append(filedetailspath)

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/runner.py

@@ -2,6 +2,7 @@ import logging
 from collections.abc import Callable, Sequence
 from copy import copy
 from datetime import datetime
+from pathlib import Path
 from traceback import format_exc
 from typing import Any, Optional
 
@@ -422,7 +423,7 @@ class Runner:
     def add_resource_downloader(
         self,
         datasetinfo: dict,
-        folder: str = "",
+        folder: Path | str = "",
         force_add_to_run: bool = False,
     ) -> str:
         """Add resource downloader to the run. If running specific scrapers rather than
@@ -445,7 +446,7 @@ class Runner:
     def add_resource_downloaders(
         self,
         configuration: dict,
-        folder: str = "",
+        folder: Path | str = "",
         force_add_to_run: bool = False,
     ) -> list[str]:
         """Add multiple resource downloaders to the run. If running specific scrapers

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/scrapers/resource_downloader.py

@@ -1,5 +1,5 @@
 import logging
-from os.path import join
+from pathlib import Path
 from shutil import copy2
 
 from slugify import slugify
@@ -19,11 +19,11 @@ class ResourceDownloader(BaseScraper):
         folder: Folder to which to download. Default is "".
     """
 
-    def __init__(self, datasetinfo, folder):
+    def __init__(self, datasetinfo: dict, folder: Path | str):
         # ResourceDownloader only outputs to sources
         name = f"resource_downloader_{slugify(datasetinfo['hxltag'].lower(), separator='_')}"
         super().__init__(name, datasetinfo, {})
-        self.folder = folder
+        self.folder = Path(folder)
 
     def run(self) -> None:
         """Runs one resource downloader given dataset information
@@ -35,7 +35,7 @@ class ResourceDownloader(BaseScraper):
         resource = reader.read_hdx_metadata(self.datasetinfo)
         url, path = reader.download_resource(resource, file_prefix=self.name)
         logger.info(f"Downloading {url} to {path}")
-        copy2(path, join(self.folder, self.datasetinfo["filename"]))
+        copy2(path, self.folder / self.datasetinfo["filename"])
 
     def add_sources(self) -> None:
         """Add source for resource download

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/utilities/fallbacks.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 
 from hdx.utilities.loader import LoadError, load_json
 
@@ -25,7 +26,7 @@ class Fallbacks:
     @classmethod
     def add(
         cls,
-        fallbacks_path: str,
+        fallbacks_path: Path | str,
         levels_mapping: dict[str, str] = default_levels_mapping,
         sources_key: str = "sources",
         admin_name_mapping: dict[str, str] = default_admin_name_mapping,

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/utilities/lookup.py

@@ -1,5 +1,6 @@
 import logging
 from copy import copy
+from pathlib import Path
 
 from hdx.utilities.loader import load_yaml
 from hdx.utilities.matching import get_code_from_name
@@ -20,7 +21,7 @@ class Lookup:
         classobject: Child class
     """
 
-    def __init__(self, yaml_config_path: str, classobject: type):
+    def __init__(self, yaml_config_path: Path | str, classobject: type):
         configuration = load_yaml(script_dir_plus_file(yaml_config_path, classobject))
         self._configuration = configuration
         initial_lookup = configuration.get("initial_lookup", {})

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/src/hdx/scraper/framework/utilities/reader.py

@@ -2,7 +2,7 @@ import glob
 import logging
 from collections.abc import Iterator, Sequence
 from datetime import datetime
-from os.path import join
+from pathlib import Path
 from typing import Any
 from urllib.parse import parse_qsl
 
@@ -41,9 +41,9 @@ class Read(Retrieve):
     def __init__(
         self,
         downloader: Download,
-        fallback_dir: str,
-        saved_dir: str,
-        temp_dir: str,
+        fallback_dir: Path | str,
+        saved_dir: Path | str,
+        temp_dir: Path | str,
         save: bool = False,
         use_saved: bool = False,
         prefix: str = "",
@@ -65,9 +65,9 @@ class Read(Retrieve):
     @classmethod
     def create_readers(
         cls,
-        fallback_dir: str,
-        saved_dir: str,
-        temp_dir: str,
+        fallback_dir: Path | str,
+        saved_dir: Path | str,
+        temp_dir: Path | str,
         save: bool = False,
         use_saved: bool = False,
         ignore: Sequence[str] = tuple(),
@@ -275,7 +275,7 @@ class Read(Retrieve):
         Returns:
             The dataset that was read or None
         """
-        saved_path = join(self.saved_dir, f"{dataset_name}.json")
+        saved_path = self.saved_dir / f"{dataset_name}.json"
         if self.use_saved:
             logger.info(f"Using saved dataset {dataset_name} in {saved_path}")
             dataset = Dataset.load_from_json(saved_path)
@@ -319,7 +319,7 @@ class Read(Retrieve):
             list of datasets resulting from query
         """
 
-        saved_path = join(self.saved_dir, filename)
+        saved_path = self.saved_dir / filename
         if self.use_saved:
             logger.info(
                 f"Using saved datasets in {filename}_n.json in {self.saved_dir}"
@@ -461,7 +461,7 @@ class Read(Retrieve):
         url = resource["url"]
         try:
             _, path = self.download_resource(resource, **kwargs)
-            data = hxl.data(path, InputOptions(allow_local=True)).cache()
+            data = hxl.data(str(path), InputOptions(allow_local=True)).cache()
             data.display_tags
             return data
         except hxl.HXLException:
@@ -488,7 +488,7 @@ class Read(Retrieve):
         """
         try:
             _, path = self.construct_filename_and_download(name, format, url, **kwargs)
-            return hxl.info(path, InputOptions(allow_local=True))
+            return hxl.info(str(path), InputOptions(allow_local=True))
         except hxl.HXLException:
             logger.warning(f"Could not process {url}. Maybe there are no HXL tags?")
             return None

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/tests/hdx/scraper/framework/conftest.py

@@ -1,6 +1,6 @@
 """Global fixtures"""
 
-from os.path import join
+from pathlib import Path
 
 import pytest
 from hdx.api.configuration import Configuration
@@ -18,12 +18,12 @@ from . import bool_assert
 
 @pytest.fixture(scope="session")
 def fixtures():
-    return join("tests", "fixtures")
+    return Path("tests") / "fixtures"
 
 
 @pytest.fixture(scope="session")
 def input_folder(fixtures):
-    return join(fixtures, "input")
+    return fixtures / "input"
 
 
 @pytest.fixture(scope="session")
@@ -32,7 +32,7 @@ def configuration(fixtures, input_folder):
         hdx_read_only=True,
         hdx_site="prod",
         user_agent="test",
-        project_config_yaml=join("tests", "config", "project_configuration.yaml"),
+        project_config_yaml=Path("tests") / "config" / "project_configuration.yaml",
     )
     Locations.set_validlocations(
         [
@@ -58,7 +58,7 @@ def configuration(fixtures, input_folder):
     if save:
         Read.create_readers(
             "",
-            join(fixtures, "tmp"),
+            fixtures / "tmp",
             "",
             save=True,
             use_saved=False,
@@ -88,9 +88,9 @@ def configuration(fixtures, input_folder):
 
 @pytest.fixture(scope="function")
 def fallbacks_json(input_folder):
-    path = join(input_folder, "fallbacks.json")
+    path = input_folder / "fallbacks.json"
     Fallbacks.add(path, sources_key="sources")
-    return path
+    return str(path)
 
 
 def check_scrapers(

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/tests/hdx/scraper/framework/outputs/test_output.py

@@ -1,6 +1,5 @@
 import filecmp
 from os import getenv
-from os.path import join
 
 import numpy as np
 import pandas
@@ -29,7 +28,7 @@ class TestOutput:
             tabs = configuration["tabs"]
             sheetname = list(tabs.values())[0]
             noout = BaseOutput(tabs)
-            excelout = ExcelFile(join(tempdir, "test_output.xlsx"), tabs, tabs)
+            excelout = ExcelFile(tempdir / "test_output.xlsx", tabs, tabs)
             gsheet_auth = getenv("GSHEET_AUTH")
             if not gsheet_auth:
                 raise ValueError("No gsheet authorisation supplied!")
@@ -78,16 +77,16 @@ class TestOutput:
             result = googletab.get("A1:C3")
             result[2][2] = int(result[2][2])
             assert result == output
-            assert filecmp.cmp(filepaths[0], join(fixtures, "test_scraper_all.json"))
+            assert filecmp.cmp(filepaths[0], fixtures / "test_scraper_all.json")
             assert filecmp.cmp(
                 filepaths[1],
-                join(fixtures, "test_scraper_population.json"),
+                fixtures / "test_scraper_population.json",
             )
             assert filecmp.cmp(
                 filepaths[2],
-                join(fixtures, "test_scraper_population.json"),
+                fixtures / "test_scraper_population.json",
             )
-            assert filecmp.cmp(filepaths[3], join(fixtures, "test_scraper_other.json"))
+            assert filecmp.cmp(filepaths[3], fixtures / "test_scraper_other.json")
 
             jsonout.json = {}
             df = pandas.DataFrame(output[2:], columns=output[0])
@@ -101,16 +100,16 @@ class TestOutput:
             result = googletab.get("A1:C3")
             result[2][2] = int(result[2][2])
             assert result == output
-            assert filecmp.cmp(filepaths[0], join(fixtures, "test_scraper_all.json"))
+            assert filecmp.cmp(filepaths[0], fixtures / "test_scraper_all.json")
             assert filecmp.cmp(
                 filepaths[1],
-                join(fixtures, "test_scraper_population.json"),
+                fixtures / "test_scraper_population.json",
             )
             assert filecmp.cmp(
                 filepaths[2],
-                join(fixtures, "test_scraper_population.json"),
+                fixtures / "test_scraper_population.json",
             )
-            assert filecmp.cmp(filepaths[3], join(fixtures, "test_scraper_other.json"))
+            assert filecmp.cmp(filepaths[3], fixtures / "test_scraper_other.json")
 
             df = pandas.DataFrame(output[1:], columns=output[0])
             googleout.update_tab("national", df, limit=2)

{hdx_python_scraper-2.7.0 → hdx_python_scraper-2.7.1}/tests/hdx/scraper/framework/scrapers/test_resource_downloaders.py

@@ -1,4 +1,4 @@
-from os.path import exists, join
+from os.path import exists
 
 import pytest
 from hdx.utilities.dateparse import parse_date
@@ -11,7 +11,7 @@ from hdx.scraper.framework.runner import Runner
 class TestResourceDownloader:
     @pytest.fixture(scope="function")
     def output_file(self, fixtures):
-        return join(fixtures, "test_output.xlsx")
+        return fixtures / "test_output.xlsx"
 
     def test_resourcedownloader(self, configuration, output_file):
         with temp_dir(