hdx-python-country 3.7.8__tar.gz → 3.8.0__tar.gz

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hdx-python-country
-Version: 3.7.8
+Version: 3.8.0
 Summary: HDX Python country code and exchange rate (fx) utilities
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-country
 Author: Michael Rans
@@ -25,7 +25,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.8
-Requires-Dist: hdx-python-utilities>=3.7.3
+Requires-Dist: hdx-python-utilities>=3.7.4
 Requires-Dist: libhxl>=5.2.1
 Requires-Dist: pyphonetics
 Provides-Extra: dev

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/documentation/main.md

@@ -130,6 +130,10 @@ object has been set up with parents). Keys take the form "STRING_TO_REPLACE",
 *admin_fuzzy_dont* is a list of names for which fuzzy matching should not be
 tried
 
+A Retrieve object can be passed in the *retriever* parameter that enables
+saving data downloaded to a file or loading previously saved data depending
+on how the Retrieve object is configured.
+
 Once an AdminLevel object is constructed, one of three setup methods must be
 called: *setup_from_admin_info*, *setup_from_libhxl_dataset* or
 *setup_from_url*.

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-utilities>=3.7.3",
+    "hdx-python-utilities>=3.7.4",
     "libhxl>=5.2.1",
     "pyphonetics",
 ]

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/requirements.txt

@@ -24,17 +24,17 @@ distlib==0.3.8
     # via virtualenv
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.15.4
+filelock==3.16.1
     # via virtualenv
-frictionless==5.17.1
+frictionless==5.18.0
     # via hdx-python-utilities
-hdx-python-utilities==3.7.3
+hdx-python-utilities==3.7.4
     # via hdx-python-country (pyproject.toml)
-humanize==4.10.0
+humanize==4.11.0
     # via frictionless
-identify==2.6.0
+identify==2.6.1
     # via pre-commit
-idna==3.8
+idna==3.10
     # via requests
 ijson==3.3.0
     # via hdx-python-utilities
@@ -74,7 +74,7 @@ packaging==24.1
     # via pytest
 petl==1.7.15
     # via frictionless
-platformdirs==4.2.2
+platformdirs==4.3.6
     # via virtualenv
 pluggy==1.5.0
     # via pytest
@@ -82,17 +82,17 @@ ply==3.11
     # via
     #   jsonpath-ng
     #   libhxl
-pre-commit==3.8.0
+pre-commit==4.0.0
     # via hdx-python-country (pyproject.toml)
-pydantic==2.9.0
+pydantic==2.9.2
     # via frictionless
-pydantic-core==2.23.2
+pydantic-core==2.23.4
     # via pydantic
 pygments==2.18.0
     # via rich
 pyphonetics==0.5.3
     # via hdx-python-country (pyproject.toml)
-pytest==8.3.2
+pytest==8.3.3
     # via
     #   hdx-python-country (pyproject.toml)
     #   pytest-cov
@@ -127,7 +127,7 @@ requests-file==2.1.0
     # via hdx-python-utilities
 rfc3986==2.0.0
     # via frictionless
-rich==13.8.0
+rich==13.9.2
     # via typer
 rpds-py==0.20.0
     # via
@@ -139,7 +139,7 @@ ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
 shellingham==1.5.4
     # via typer
-simpleeval==0.9.13
+simpleeval==1.0.0
     # via frictionless
 six==1.16.0
     # via
@@ -163,19 +163,17 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   typer
-tzdata==2024.1
-    # via pydantic
 unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.2
+urllib3==2.2.3
     # via
     #   libhxl
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.3
+virtualenv==20.26.6
     # via pre-commit
 wheel==0.44.0
     # via libhxl

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/src/hdx/location/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.7.8'
-__version_tuple__ = version_tuple = (3, 7, 8)
+__version__ = version = '3.8.0'
+__version_tuple__ = version_tuple = (3, 8, 0)

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/src/hdx/location/adminlevel.py

@@ -8,7 +8,9 @@ from hxl.input import HXLIOException
 
 from hdx.location.country import Country
 from hdx.location.phonetics import Phonetics
+from hdx.utilities.base_downloader import DownloadError
 from hdx.utilities.dictandlist import dict_of_sets_add
+from hdx.utilities.retriever import Retrieve
 from hdx.utilities.text import multiple_replace, normalise
 from hdx.utilities.typehint import ListTuple
 
@@ -30,10 +32,20 @@ class AdminLevel:
     admin_fuzzy_dont is a list of names for which fuzzy matching should not be
     tried
 
+    The admin_level_overrides parameter allows manually overriding the returned
+    admin level for given countries. It is a dictionary with iso3s as keys and
+    admin level numbers as values.
+
+    The retriever parameter accepts an object of type Retrieve (or inherited
+    classes). It is used to allow either that admin data from urls is saved
+    to files or to enable already saved files to be used instead of downloading
+    from urls.
+
     Args:
         admin_config (Dict): Configuration dictionary. Defaults to {}.
         admin_level (int): Admin level. Defaults to 1.
         admin_level_overrides (Dict): Countries at other admin levels.
+        retriever (Optional[Retrieve]): Retriever object to use for loading/saving files. Defaults to None.
     """
 
     pcode_regex = re.compile(r"^([a-zA-Z]{2,3})(\d+)$")
@@ -47,9 +59,11 @@ class AdminLevel:
         admin_config: Dict = {},
         admin_level: int = 1,
         admin_level_overrides: Dict = {},
+        retriever: Optional[Retrieve] = None,
     ) -> None:
         self.admin_level = admin_level
         self.admin_level_overrides = admin_level_overrides
+        self.retriever: Optional[Retrieve] = retriever
         self.countries_fuzzy_try = admin_config.get("countries_fuzzy_try")
         self.admin_name_mappings = admin_config.get("admin_name_mappings", {})
         self.admin_name_replacements = admin_config.get(
@@ -101,23 +115,39 @@ class AdminLevel:
             admin_url = cls._admin_url_default
         cls._admin_url = admin_url
 
-    @classmethod
-    def get_libhxl_dataset(cls, admin_url: str = _admin_url) -> hxl.Dataset:
+    @staticmethod
+    def get_libhxl_dataset(
+        url: str = _admin_url, retriever: Optional[Retrieve] = None
+    ) -> hxl.Dataset:
         """
         Get libhxl Dataset object given a URL which defaults to global p-codes
         dataset on HDX.
 
         Args:
-            admin_url (str): URL from which to load data. Defaults to global p-codes dataset.
+            url (str): URL from which to load data. Defaults to internal admin url.
+            retriever (Optional[Retrieve]): Retriever object to use for loading file. Defaults to None.
 
         Returns:
-            None
-        """
+            hxl.Dataset: HXL Dataset object
+        """
+        if retriever:
+            try:
+                url_to_use = retriever.download_file(url)
+            except DownloadError:
+                logger.exception(
+                    f"Setup of libhxl Dataset object with {url} failed!"
+                )
+                raise
+        else:
+            url_to_use = url
         try:
-            return hxl.data(admin_url, InputOptions(encoding="utf-8"))
-        except HXLIOException:
+            return hxl.data(
+                url_to_use,
+                InputOptions(InputOptions(allow_local=True, encoding="utf-8")),
+            )
+        except (FileNotFoundError, HXLIOException):
             logger.exception(
-                f"Setup of libhxl Dataset object with {admin_url} failed!"
+                f"Setup of libhxl Dataset object with {url} failed!"
             )
             raise
 
@@ -245,7 +275,7 @@ class AdminLevel:
         Returns:
             None
         """
-        admin_info = self.get_libhxl_dataset(admin_url)
+        admin_info = self.get_libhxl_dataset(admin_url, self.retriever)
         self.setup_from_libhxl_dataset(admin_info, countryiso3s)
 
     def load_pcode_formats(self, formats_url: str = _formats_url) -> None:
@@ -258,7 +288,7 @@ class AdminLevel:
         Returns:
             None
         """
-        formats_info = self.get_libhxl_dataset(formats_url)
+        formats_info = self.get_libhxl_dataset(formats_url, self.retriever)
         for row in formats_info:
             pcode_format = [int(row.get("#country+len"))]
             for admin_no in range(1, 4):

{hdx_python_country-3.7.8 → hdx_python_country-3.8.0}/tests/hdx/location/test_adminlevel.py

@@ -3,28 +3,35 @@
 from os.path import join
 
 import pytest
-from hxl.input import HXLIOException
 
 from hdx.location.adminlevel import AdminLevel
+from hdx.utilities.base_downloader import DownloadError
+from hdx.utilities.downloader import Download
 from hdx.utilities.loader import load_yaml
+from hdx.utilities.path import temp_dir
+from hdx.utilities.retriever import Retrieve
 
 
 class TestAdminLevel:
+    @pytest.fixture(scope="class")
+    def fixtures_dir(self):
+        return join("tests", "fixtures")
+
     @pytest.fixture(scope="function")
-    def config(self):
-        return load_yaml(join("tests", "fixtures", "adminlevel.yaml"))
+    def config(self, fixtures_dir):
+        return load_yaml(join(fixtures_dir, "adminlevel.yaml"))
 
     @pytest.fixture(scope="function")
-    def config_parent(self):
-        return load_yaml(join("tests", "fixtures", "adminlevelparent.yaml"))
+    def config_parent(self, fixtures_dir):
+        return load_yaml(join(fixtures_dir, "adminlevelparent.yaml"))
 
     @pytest.fixture(scope="function")
-    def url(self):
-        return "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-country/blank_adm_name/tests/fixtures/global_pcodes_adm_1_2.csv"
+    def url(self, fixtures_dir):
+        return join(fixtures_dir, "download-global-pcodes-adm-1-2.csv")
 
     @pytest.fixture(scope="function")
-    def formats_url(self):
-        return "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-country/blank_adm_name/tests/fixtures/global_pcode_lengths.csv"
+    def formats_url(self, fixtures_dir):
+        return join(fixtures_dir, "download-global-pcode-lengths.csv")
 
     def test_adminlevel(self, config):
         adminone = AdminLevel(config)
@@ -367,16 +374,50 @@ class TestAdminLevel:
             "MWI", "Blantyre city", parent="MW3", logname="test"
         ) == (None, False)
 
-    def test_adminlevel_with_url(self, config, url):
-        adminone = AdminLevel(config)
-        with pytest.raises(HXLIOException):
+    def test_adminlevel_with_url(self, config, url, fixtures_dir):
+        adminone = AdminLevel(config, admin_level_overrides={"YEM": 5})
+        assert adminone.get_admin_level("YEM") == 5
+        with pytest.raises(FileNotFoundError):
             adminone.setup_from_url("fake_url")
+        adminone = AdminLevel(config)
         AdminLevel.set_default_admin_url()
         assert AdminLevel._admin_url == AdminLevel._admin_url_default
         AdminLevel.set_default_admin_url(url)
         assert AdminLevel._admin_url == url
         adminone.setup_from_url(countryiso3s=("YEM",))
         assert len(adminone.get_pcode_list()) == 22
+
+        with temp_dir(
+            "TestAdminLevelRetriever",
+            delete_on_success=True,
+            delete_on_failure=False,
+        ) as tempdir:
+            with Download(user_agent="test") as downloader:
+                retriever = Retrieve(
+                    downloader,
+                    tempdir,
+                    fixtures_dir,
+                    tempdir,
+                    save=False,
+                    use_saved=False,
+                )
+                adminone = AdminLevel(config, retriever=retriever)
+                with pytest.raises(DownloadError):
+                    adminone.setup_from_url("fake_url")
+                retriever = Retrieve(
+                    downloader,
+                    tempdir,
+                    fixtures_dir,
+                    tempdir,
+                    save=False,
+                    use_saved=True,
+                )
+                adminone = AdminLevel(config, retriever=retriever)
+                with pytest.raises(FileNotFoundError):
+                    adminone.setup_from_url("fake_url")
+                adminone.setup_from_url(countryiso3s=("YEM",))
+                assert len(adminone.get_pcode_list()) == 22
+
         adminone = AdminLevel(config)
         adminone.setup_from_url()
         assert adminone.get_admin_level("YEM") == 1