hdx-python-scraper 2.5.8__tar.gz → 2.6.0__tar.gz

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/.config/pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
         exclude: test_scraper_.*\.json
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.6
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.5.15
+    rev: 0.5.29
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.5.8
+Version: 2.6.0
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/pyproject.toml

@@ -95,7 +95,7 @@ run = """
        """
 
 [tool.hatch.envs.hatch-static-analysis]
-dependencies = ["ruff==0.8.6"]
+dependencies = ["ruff==0.9.6"]
 
 [tool.hatch.envs.hatch-static-analysis.scripts]
 format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/requirements.txt

@@ -22,7 +22,7 @@ ckanapi==4.8
     # via hdx-python-api
 click==8.1.8
     # via typer
-coverage==7.6.10
+coverage==7.6.12
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
@@ -65,7 +65,7 @@ hdx-python-utilities==3.8.3
     #   hdx-python-country
 humanize==4.11.0
     # via frictionless
-identify==2.6.6
+identify==2.6.7
     # via pre-commit
 idna==3.10
     # via
@@ -113,7 +113,7 @@ nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.14
     # via quantulum3
-numpy==2.2.2
+numpy==2.2.3
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -264,7 +264,7 @@ urllib3==2.3.0
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.29.1
+virtualenv==20.29.2
     # via pre-commit
 wheel==0.45.1
     # via libhxl

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/src/hdx/scraper/framework/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.5.8'
-__version_tuple__ = version_tuple = (2, 5, 8)
+__version__ = version = '2.6.0'
+__version_tuple__ = version_tuple = (2, 6, 0)

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/src/hdx/scraper/framework/outputs/json.py

@@ -215,7 +215,7 @@ class JsonFile(BaseOutput):
                     if tab not in remove:
                         tabs.append({"tab": tab})
             for tabdetails in tabs:
-                key = f'{tabdetails["tab"]}{self.suffix}'
+                key = f"{tabdetails['tab']}{self.suffix}"
                 newjson = self.json.get(key)
                 filters = tabdetails.get("filters", {})
                 hxltags = tabdetails.get("output")

hdx_python_scraper-2.6.0/src/hdx/scraper/framework/utilities/hapi_admins.py

@@ -0,0 +1,75 @@
+from typing import List, Tuple
+
+from hdx.location.adminlevel import AdminLevel
+
+
+def complete_admins(
+    admins: List[AdminLevel],
+    countryiso3: str,
+    provider_adm_names: List,
+    adm_codes: List,
+    adm_names: List,
+) -> Tuple[int, List[str]]:
+    warnings = []
+    parent = None
+    adm_level = len(provider_adm_names)
+    for i, provider_adm_name in reversed(list(enumerate(provider_adm_names))):
+        adm_code = adm_codes[i]
+        if not provider_adm_name:
+            provider_adm_name = ""
+            provider_adm_names[i] = ""
+        if parent:
+            pcode = admins[i + 1].pcode_to_parent.get(parent)
+            warntxt = "parent"
+        elif provider_adm_name:
+            pcode, _ = admins[i].get_pcode(
+                countryiso3, provider_adm_name, parent=parent
+            )
+            warntxt = f"provider_adm{i + 1}_name"
+        else:
+            pcode = None
+        if adm_code:
+            if adm_code not in admins[i].pcodes:
+                if pcode:
+                    warnings.append(
+                        f"PCode unknown {adm_code}->{pcode} ({warntxt})"
+                    )
+                    adm_code = pcode
+                else:
+                    warnings.append(f"PCode unknown {adm_code}->''")
+                    adm_code = ""
+            elif pcode and adm_code != pcode:
+                if parent:
+                    warnings.append(
+                        f"PCode mismatch {adm_code}->{pcode} ({warntxt})"
+                    )
+                    adm_code = pcode
+                else:
+                    warnings.append(
+                        f"PCode mismatch {adm_code} != {provider_adm_name}"
+                    )
+        elif pcode:
+            adm_code = pcode
+        else:
+            adm_code = ""
+        adm_codes[i] = adm_code
+        if adm_code:
+            adm_names[i] = admins[i].pcode_to_name.get(adm_code, "")
+            parent = adm_code
+        else:
+            adm_names[i] = ""
+            if provider_adm_name == "":
+                adm_level -= 1
+    return adm_level, warnings
+
+
+def pad_admins(
+    provider_adm_names: List[str],
+    adm_codes: List[str],
+    adm_names: List[str],
+    adm_level: int = 2,
+) -> None:
+    for i in range(len(provider_adm_names), adm_level):
+        provider_adm_names.append("")
+        adm_codes.append("")
+        adm_names.append("")

{hdx_python_scraper-2.5.8 → hdx_python_scraper-2.6.0}/src/hdx/scraper/framework/utilities/lookup.py

@@ -60,5 +60,10 @@ class Lookup:
             unmatched=self._unmatched,
         )
 
+    def get_name(
+        self, code: str, default: Optional[str] = None
+    ) -> Optional[str]:
+        return self._code_to_name.get(code, default)
+
     def get_code_to_name(self) -> Dict[str, str]:
         return self._code_to_name

hdx_python_scraper-2.6.0/tests/fixtures/input/download-global-pcode-lengths.csv

@@ -0,0 +1,158 @@
+Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
+#country+code,#country+len,#adm1+len,#adm2+len,#adm3+len,#adm4+len,#adm5+len
+AFG,2,2,2,,,
+AGO,2,2,3,3,,
+ALB,2,2,2,2,,
+ARE,2,2,,,,
+ARG,2,3,3,,,
+ARM,2,2,1,3,,
+ATG,2,2,,,,
+AZE,2,8,,,,
+BDI,3,3,3,,,
+BEN,2,2,2,,,
+BES,3,1,,,,
+BFA,2,2,2,2,,
+BGD,2,2,2,2,2,
+BGR,2,3,3,,,
+BLR,2,3,3,,,
+BLZ,2,2,,,,
+BMU,2,2,1,,,
+BOL,2,2,2,2,,
+BRA,2,2,5,,,
+BRB,2,2,,,,
+BTN,2,3,2,,,
+BWA,2,2,2,2,,
+CAF,2,2,1,1,2,
+CHL,2,2,1,2,,
+CHN,2,3,3,,,
+CIV,2,2,2,2,,
+CMR,2,3,3,3,,
+COD,2,2,2,,,
+COG,2,2,2,,,
+COL,2,2,3,,,
+COM,2,1,1,1,,
+CPV,2,2,2,,,
+CRI,2,1,2,2,,
+CUB,2,2,2,,,
+CUW,2,2,,,,
+CYM,2,2,,,,
+DJI,2,2,2,,,
+DMA,2,2,,,,
+DOM,2,2,2,2,2,
+DZA,2,3,3,,,
+ECU,2,2,2,2,,
+EGY,2,2,2,,,
+ERI,2,1,2,,,
+ESH,2,2,,,,
+ETH,2,2,2,2,,
+FJI,2,1,2,2,,
+FSM,2,1,2,,,
+GAB,3,3,3,,,
+GEO,2,2,2,,,
+GHA,2,2,2,,,
+GIN,2,3,3,2,,
+GLP,2,2,2,,,
+GMB,2,2,2,2,,
+GNB,2,2,2,,,
+GNQ,2,3,3,,,
+GRD,2,2,,,,
+GTM,2,2,2,,,
+GUF,2,1,3,,,
+GUY,2,2,2,,,
+HND,2,2,2,,,
+HTI,2,2,2,3,,
+HUN,2,3,3,,,
+IDN,2,2,2,3,,
+IRN,2,3,3,,,
+IRQ,2,3,3,3,,
+JAM,2,2,3,,,
+KAZ,3,3,3,,,
+KEN,2,3,3,,,
+KGZ,2,11,0,0,,
+KHM,2,2,2,2,,
+KIR,2,1,2,,,
+KNA,2,2,,,,
+KWT,2,2,,,,
+LAO,2,2,2,,,
+LBN,2,1,1,1,,
+LBR,2,2,2,,,
+LBY,2,2,2,,,
+LCA,2,2,9,,,
+LKA,2,1,1,2,3,
+LSO,2,1,2,,,
+MAR,2,3,3,7,1,
+MDA,2,3,,,,
+MDG,2,2,7|3,-1|3,3,
+MDV,2,3,3,3,,
+MEX,2,2,3,,,
+MHL,2,2,2,,,
+MLI,2,2,2,2,,
+MMR,3,3,4,-1,3,3
+MNG,2,2,2,,,
+MOZ,2,2,2,2,,
+MRT,2,2,1,2,,
+MSR,2,2,,,,
+MTQ,2,2,2,,,
+MUS,2,2,,,,
+MWI,2,1,2,2,,
+MYS,2,2,2,,,
+NAM,2,2,2,,,
+NER,2,3,3,3,,
+NGA,2,3,3,3,,
+NIC,2,2,2,,,
+NPL,2,2,2,3,,
+OMN,2,2,2,,,
+PAK,2,1,2,2,,
+PAN,2,2,2,2,,
+PER,2,2,2,2,,
+PHL,2,2,3,2,3,
+PNG,2,2,2,2,,
+POL,2,3,3,,,
+PRI,2,2,,,,
+PRK,2,2,2,,,
+PRY,2,2,2,,,
+PSE,2,2,2,,,
+QAT,3,3,3,,,
+ROU,2,3,3,,,
+RUS,2,3,3,,,
+RWA,2,1,1,2,2,
+SAU,2,2,,,,
+SDN,2,2,3,,,
+SEN,2,2,2,2,,
+SLB,2,2,4,4,,
+SLE,2,2,2,2,2,
+SLV,2,2,2,,,
+SOM,2,2,2,,,
+SSD,2,2,2,2,,
+STP,2,2,2,,,
+SUR,2,2,2,,,
+SVK,2,3,3,,,
+SWZ,2,1,2,,,
+SXM,2,1,,,,
+SYC,2,1,1,4,,
+SYR,2,2,2,2,,
+TCA,2,1,2,,,
+TCD,2,2,2,2,,
+TGO,2,2,2,2,,
+THA,2,2,2,2,,
+TJK,0,7,0,,,
+TLS,2,2,2,2,,
+TON,2,1,1,2,,
+TTO,2,2,,,,
+TUN,2,1,1,2,2,
+TUR,3,3,3,3,3,
+TZA,2,2,2,3|4,,
+UGA,2,1,3,2,2,
+UKR,2,2,2,3,3,
+URY,2,2,3,,,
+UZB,2,2,3,,,
+VCT,2,1,2,,,
+VEN,2,2,2,2,,
+VGB,2,2,,,,
+VIR,2,3,5,,,
+VNM,2,3,2,,,
+VUT,2,2,3,,,
+YEM,2,2,2,2,,
+ZAF,2,1,2,1,3,
+ZMB,2,3,3,3,3,
+ZWE,2,2,2,2,,