hdx-python-scraper 2.5.9__tar.gz → 2.6.0__tar.gz

{hdx_python_scraper-2.5.9 → hdx_python_scraper-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.5.9
+Version: 2.6.0
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>

{hdx_python_scraper-2.5.9 → hdx_python_scraper-2.6.0}/requirements.txt

@@ -113,7 +113,7 @@ nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.14
     # via quantulum3
-numpy==2.2.2
+numpy==2.2.3
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib

{hdx_python_scraper-2.5.9 → hdx_python_scraper-2.6.0}/src/hdx/scraper/framework/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.5.9'
-__version_tuple__ = version_tuple = (2, 5, 9)
+__version__ = version = '2.6.0'
+__version_tuple__ = version_tuple = (2, 6, 0)

hdx_python_scraper-2.6.0/src/hdx/scraper/framework/utilities/hapi_admins.py

@@ -0,0 +1,75 @@
+from typing import List, Tuple
+
+from hdx.location.adminlevel import AdminLevel
+
+
+def complete_admins(
+    admins: List[AdminLevel],
+    countryiso3: str,
+    provider_adm_names: List,
+    adm_codes: List,
+    adm_names: List,
+) -> Tuple[int, List[str]]:
+    warnings = []
+    parent = None
+    adm_level = len(provider_adm_names)
+    for i, provider_adm_name in reversed(list(enumerate(provider_adm_names))):
+        adm_code = adm_codes[i]
+        if not provider_adm_name:
+            provider_adm_name = ""
+            provider_adm_names[i] = ""
+        if parent:
+            pcode = admins[i + 1].pcode_to_parent.get(parent)
+            warntxt = "parent"
+        elif provider_adm_name:
+            pcode, _ = admins[i].get_pcode(
+                countryiso3, provider_adm_name, parent=parent
+            )
+            warntxt = f"provider_adm{i + 1}_name"
+        else:
+            pcode = None
+        if adm_code:
+            if adm_code not in admins[i].pcodes:
+                if pcode:
+                    warnings.append(
+                        f"PCode unknown {adm_code}->{pcode} ({warntxt})"
+                    )
+                    adm_code = pcode
+                else:
+                    warnings.append(f"PCode unknown {adm_code}->''")
+                    adm_code = ""
+            elif pcode and adm_code != pcode:
+                if parent:
+                    warnings.append(
+                        f"PCode mismatch {adm_code}->{pcode} ({warntxt})"
+                    )
+                    adm_code = pcode
+                else:
+                    warnings.append(
+                        f"PCode mismatch {adm_code} != {provider_adm_name}"
+                    )
+        elif pcode:
+            adm_code = pcode
+        else:
+            adm_code = ""
+        adm_codes[i] = adm_code
+        if adm_code:
+            adm_names[i] = admins[i].pcode_to_name.get(adm_code, "")
+            parent = adm_code
+        else:
+            adm_names[i] = ""
+            if provider_adm_name == "":
+                adm_level -= 1
+    return adm_level, warnings
+
+
+def pad_admins(
+    provider_adm_names: List[str],
+    adm_codes: List[str],
+    adm_names: List[str],
+    adm_level: int = 2,
+) -> None:
+    for i in range(len(provider_adm_names), adm_level):
+        provider_adm_names.append("")
+        adm_codes.append("")
+        adm_names.append("")

hdx_python_scraper-2.6.0/tests/hdx/scraper/framework/utilities/test_hapi_admins.py

@@ -0,0 +1,184 @@
+from hdx.location.adminlevel import AdminLevel
+from hdx.scraper.framework.utilities.hapi_admins import (
+    complete_admins,
+    pad_admins,
+)
+from hdx.scraper.framework.utilities.reader import Read
+
+
+class TestAdmins:
+    def test_complete_admins(self, configuration):
+        countryiso3 = "AFG"
+        admins = []
+        reader = Read.get_reader("hdx")
+        for i in range(2):
+            admin = AdminLevel(admin_level=i + 1, retriever=reader)
+            admin.setup_from_url(countryiso3s=(countryiso3,))
+            admin.load_pcode_formats()
+            admins.append(admin)
+
+        provider_adm_names = ["", ""]
+        adm_codes = ["", ""]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["", ""]
+        assert adm_names == ["", ""]
+        assert adm_level == 0
+        assert warnings == []
+
+        provider_adm_names = ["Kabul", ""]
+        adm_codes = ["AF01", ""]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", ""]
+        assert adm_names == ["Kabul", ""]
+        assert adm_level == 1
+        assert warnings == []
+
+        provider_adm_names = ["Kabul", ""]
+        adm_codes = ["AF01", "AF010X"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", ""]
+        assert adm_names == ["Kabul", ""]
+        assert adm_level == 1
+        assert warnings == ["PCode unknown AF010X->''"]
+
+        provider_adm_names = ["Kabal", "Paghman"]
+        adm_codes = ["AF01", ""]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["Kabul", "Paghman"]
+        assert adm_level == 2
+        assert warnings == []
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF01", "AF01XX"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["Kabul", "Paghman"]
+        assert adm_level == 2
+        assert warnings == [
+            "PCode unknown AF01XX->AF0102 (provider_adm2_name)"
+        ]
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["", "AF0102"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["Kabul", "Paghman"]
+        assert adm_level == 2
+        assert warnings == []
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF0X", "AF0102"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["Kabul", "Paghman"]
+        assert adm_level == 2
+        assert warnings == ["PCode unknown AF0X->AF01 (parent)"]
+
+        provider_adm_names = ["Kabul", ""]
+        adm_codes = ["AF01", "AF0102"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert provider_adm_names == ["Kabul", ""]
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["Kabul", "Paghman"]
+        assert adm_level == 2
+        assert warnings == []
+
+        provider_adm_names = ["Kabul", None]
+        adm_codes = ["AF01", None]
+        adm_names = [None, None]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", ""]
+        assert adm_names == ["Kabul", ""]
+        assert adm_level == 1
+        assert warnings == []
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF02", "AF0102"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["Kabul", "Paghman"]
+        assert adm_level == 2
+        assert warnings == ["PCode mismatch AF02->AF01 (parent)"]
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF01", "AF0103"]
+        adm_names = ["", ""]
+        adm_level, warnings = complete_admins(
+            admins, countryiso3, provider_adm_names, adm_codes, adm_names
+        )
+        assert provider_adm_names == ["Kabul", "Paghman"]
+        assert adm_codes == ["AF01", "AF0103"]
+        assert adm_names == ["Kabul", "Chahar Asyab"]
+        assert adm_level == 2
+        assert warnings == ["PCode mismatch AF0103 != Paghman"]
+
+    def test_pad_admins(self):
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF01", "AF0102"]
+        adm_names = ["", ""]
+        pad_admins(provider_adm_names, adm_codes, adm_names, 3)
+        assert provider_adm_names == ["Kabul", "Paghman", ""]
+        assert adm_codes == ["AF01", "AF0102", ""]
+        assert adm_names == ["", "", ""]
+
+        provider_adm_names = ["Kabul"]
+        adm_codes = ["AF01"]
+        adm_names = [""]
+        pad_admins(provider_adm_names, adm_codes, adm_names)
+        assert provider_adm_names == ["Kabul", ""]
+        assert adm_codes == ["AF01", ""]
+        assert adm_names == ["", ""]
+
+        provider_adm_names = []
+        adm_codes = []
+        adm_names = []
+        pad_admins(provider_adm_names, adm_codes, adm_names)
+        assert provider_adm_names == ["", ""]
+        assert adm_codes == ["", ""]
+        assert adm_names == ["", ""]
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF01", "AF0102"]
+        adm_names = ["", ""]
+        pad_admins(provider_adm_names, adm_codes, adm_names, 1)
+        assert provider_adm_names == ["Kabul", "Paghman"]
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["", ""]
+
+        provider_adm_names = ["Kabul", "Paghman"]
+        adm_codes = ["AF01", "AF0102"]
+        adm_names = ["", ""]
+        pad_admins(provider_adm_names, adm_codes, adm_names, 2)
+        assert provider_adm_names == ["Kabul", "Paghman"]
+        assert adm_codes == ["AF01", "AF0102"]
+        assert adm_names == ["", ""]

hdx_python_scraper-2.5.9/src/hdx/scraper/framework/utilities/hapi_admins.py

@@ -1,29 +0,0 @@
-from typing import List
-
-from hdx.location.adminlevel import AdminLevel
-
-
-def complete_admins(
-    admins: List[AdminLevel],
-    countryiso3: str,
-    provider_adm_names: List[str],
-    adm_codes: List[str],
-    adm_names: List[str],
-) -> None:
-    parent = None
-    for i, adm_code in enumerate(adm_codes):
-        if adm_code:
-            adm_name = admins[i].pcode_to_name.get(adm_code, "")
-            adm_names[i] = adm_name
-            continue
-        provider_adm_name = provider_adm_names[i]
-        if not provider_adm_name:
-            continue
-        adm_code, _ = admins[i].get_pcode(
-            countryiso3, provider_adm_name, parent=parent
-        )
-        if adm_code:
-            adm_codes[i] = adm_code
-            adm_name = admins[i].pcode_to_name.get(adm_code, "")
-            adm_names[i] = adm_name
-            parent = adm_code

hdx_python_scraper-2.5.9/tests/hdx/scraper/framework/utilities/test_hapi_admins.py

@@ -1,23 +0,0 @@
-from hdx.location.adminlevel import AdminLevel
-from hdx.scraper.framework.utilities.hapi_admins import complete_admins
-from hdx.scraper.framework.utilities.reader import Read
-
-
-class TestAdmins:
-    def test_complete_admins(self, configuration):
-        countryiso3 = "AFG"
-        admins = []
-        reader = Read.get_reader("hdx")
-        for i in range(2):
-            admin = AdminLevel(admin_level=i + 1, retriever=reader)
-            admin.setup_from_url(countryiso3s=(countryiso3,))
-            admin.load_pcode_formats()
-            admins.append(admin)
-        provider_adm_names = ["Kabal", "Paghman"]
-        adm_codes = ["AF01", ""]
-        adm_names = ["", ""]
-        complete_admins(
-            admins, countryiso3, provider_adm_names, adm_codes, adm_names
-        )
-        assert adm_codes == ["AF01", "AF0102"]
-        assert adm_names == ["Kabul", "Paghman"]