hdx-python-scraper 2.6.2__tar.gz → 2.6.3__tar.gz

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hdx-python-scraper
-Version: 2.6.2
+Version: 2.6.3
 Summary: HDX Python scraper utilities to assemble data from multiple sources
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-scraper
 Author-email: Michael Rans <rans@email.com>

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.3}/src/hdx/scraper/framework/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.6.2'
-__version_tuple__ = version_tuple = (2, 6, 2)
+__version__ = version = '2.6.3'
+__version_tuple__ = version_tuple = (2, 6, 3)

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.3}/src/hdx/scraper/framework/utilities/hapi_admins.py

@@ -31,8 +31,20 @@ def complete_admins(
     warnings = []
     child = None
     adm_level = len(provider_adm_names)
+
+    def check_unknown_pcode(adm_code: str, pcode: str) -> str:
+        if pcode:
+            warnings.append(f"PCode unknown {adm_code}->{pcode} ({warntxt})")
+            return pcode
+        else:
+            warnings.append(f"PCode unknown {adm_code}->''")
+            return ""
+
     for i, provider_adm_name in reversed(list(enumerate(provider_adm_names))):
         adm_code = adm_codes[i]
+        parent = admins[i].pcode_to_parent.get(adm_code)
+        if not parent and i > 0:
+            parent = adm_codes[i - 1]
         if not provider_adm_name:
             provider_adm_name = ""
             provider_adm_names[i] = ""
@@ -40,9 +52,6 @@ def complete_admins(
             pcode = admins[i + 1].pcode_to_parent.get(child)
             warntxt = "parent"
         elif provider_adm_name:
-            parent = admins[i].pcode_to_parent.get(adm_code)
-            if not parent and i > 0:
-                parent = adm_codes[i - 1]
             pcode, _ = admins[i].get_pcode(
                 countryiso3,
                 provider_adm_name,
@@ -54,14 +63,19 @@ def complete_admins(
             pcode = None
         if adm_code:
             if adm_code not in admins[i].pcodes:
-                if pcode:
-                    warnings.append(
-                        f"PCode unknown {adm_code}->{pcode} ({warntxt})"
+                if admins[i].looks_like_pcode(adm_code):
+                    adj_adm_code = admins[i].convert_admin_pcode_length(
+                        countryiso3, adm_code, parent=parent
                     )
-                    adm_code = pcode
+                    if adj_adm_code:
+                        warnings.append(
+                            f"PCode length {adm_code}->{adj_adm_code}"
+                        )
+                        adm_code = adj_adm_code
+                    else:
+                        adm_code = check_unknown_pcode(adm_code, pcode)
                 else:
-                    warnings.append(f"PCode unknown {adm_code}->''")
-                    adm_code = ""
+                    adm_code = check_unknown_pcode(adm_code, pcode)
             elif pcode and adm_code != pcode:
                 if child:
                     warnings.append(

{hdx_python_scraper-2.6.2 → hdx_python_scraper-2.6.3}/tests/fixtures/input/download-global-pcode-lengths.csv

@@ -1,158 +1,158 @@
-Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
+Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
 #country+code,#country+len,#adm1+len,#adm2+len,#adm3+len,#adm4+len,#adm5+len
 AFG,2,2,2,,,
-AGO,2,2,3,3,,
 ALB,2,2,2,2,,
-ARE,2,2,,,,
+DZA,2,3,3,,,
+AGO,2,2,3,3,,
 ARG,2,3,3,,,
 ARM,2,2,1,3,,
-ATG,2,2,,,,
 AZE,2,8,,,,
-BDI,3,3,3,,,
-BEN,2,2,2,,,
-BES,3,1,,,,
-BFA,2,2,2,2,,
 BGD,2,2,2,2,2,
-BGR,2,3,3,,,
+BRB,2,2,,,,
 BLR,2,3,3,,,
 BLZ,2,2,,,,
+BEN,2,2,2,,,
 BMU,2,2,1,,,
-BOL,2,2,2,2,,
-BRA,2,2,5,,,
-BRB,2,2,,,,
 BTN,2,3,2,,,
+BOL,2,2,2,2,,
+BES,3,1,,,,
 BWA,2,2,2,2,,
-CAF,2,2,1,1,2,
+BRA,2,2,5,,,
+BGR,2,3,3,,,
+BFA,2,2,2,2,,
+BDI,3,3,3,,,
+KHM,2,2,2,2,,
+CMR,2,3,3,3,,
+CPV,2,2,2,,,
+CYM,2,2,,,,
+CAF,2,2,1,1,,
+TCD,3,2,2,,,
 CHL,2,2,1,2,,
 CHN,2,3,3,,,
-CIV,2,2,2,2,,
-CMR,2,3,3,3,,
-COD,2,2,2,,,
-COG,2,2,2,,,
 COL,2,2,3,,,
 COM,2,1,1,1,,
-CPV,2,2,2,,,
+COG,2,2,2,,,
+COD,2,2,2,,,
 CRI,2,1,2,2,,
+CIV,2,2,2,2,,
 CUB,2,2,2,,,
 CUW,2,2,,,,
-CYM,2,2,,,,
 DJI,2,2,2,,,
 DMA,2,2,,,,
 DOM,2,2,2,2,2,
-DZA,2,3,3,,,
-ECU,2,2,2,2,,
-EGY,2,2,2,,,
+ECU,2,2,3,2,,
+EGY,2,2,2,2,,
+SLV,2,2,3,3,,
+GNQ,2,3,3,,,
 ERI,2,1,2,,,
-ESH,2,2,,,,
 ETH,2,2,2,2,,
 FJI,2,1,2,2,,
-FSM,2,1,2,,,
+GUF,2,1,3,,,
 GAB,3,3,3,,,
+GMB,2,2,2,2,,
 GEO,2,2,2,,,
 GHA,2,2,2,,,
-GIN,2,3,3,2,,
-GLP,2,2,2,,,
-GMB,2,2,2,2,,
-GNB,2,2,2,,,
-GNQ,2,3,3,,,
+GRC,2,2,2,,,
 GRD,2,2,,,,
+GLP,2,2,2,,,
 GTM,2,2,2,,,
-GUF,2,1,3,,,
+GIN,2,3,3,2,,
+GNB,2,2,2,,,
 GUY,2,2,2,,,
-HND,2,2,2,,,
 HTI,2,2,2,3,,
+HND,2,2,2,,,
 HUN,2,3,3,,,
 IDN,2,2,2,3,,
 IRN,2,3,3,,,
 IRQ,2,3,3,3,,
-JAM,2,2,3,,,
+JAM,2,2,2,2,,
 KAZ,3,3,3,,,
 KEN,2,3,3,,,
-KGZ,2,11,0,0,,
-KHM,2,2,2,2,,
 KIR,2,1,2,,,
-KNA,2,2,,,,
+PRK,2,2,2,,,
 KWT,2,2,,,,
+KGZ,2,11,0,0,,
 LAO,2,2,2,,,
-LBN,2,1,1,1,,
+LBN,2,1,1,,,
+LSO,2,1,2,,,
 LBR,2,2,2,,,
 LBY,2,2,2,,,
-LCA,2,2,9,,,
-LKA,2,1,1,2,3,
-LSO,2,1,2,,,
-MAR,2,3,3,7,1,
-MDA,2,3,,,,
-MDG,2,2,7|3,-1|3,3,
-MDV,2,3,3,3,,
-MEX,2,2,3,,,
-MHL,2,2,2,,,
+MKD,2,3,,,,
+MDG,2,2,3|7,3|-1,3,
+MWI,2,1,2,2,,
+MYS,2,2,2,,,
+MDV,2,3,3,,,
 MLI,2,2,2,2,,
-MMR,3,3,4,-1,3,3
-MNG,2,2,2,,,
-MOZ,2,2,2,2,,
-MRT,2,2,1,2,,
-MSR,2,2,,,,
+MHL,2,2,2,,,
 MTQ,2,2,2,,,
+MRT,2,2,1,2,,
 MUS,2,2,,,,
-MWI,2,1,2,2,,
-MYS,2,2,2,,,
+MEX,2,2,3,,,
+FSM,2,1,2,,,
+MDA,2,3,,,,
+MNG,2,2,2,,,
+MSR,2,2,,,,
+MAR,2,3,3,7,1,
+MOZ,2,2,2,2,,
+MMR,3,3,4,-1,,
 NAM,2,2,2,,,
+NPL,2,2,2,3,,
+NIC,2,2,2,,,
 NER,2,3,3,3,,
 NGA,2,3,3,3,,
-NIC,2,2,2,,,
-NPL,2,2,2,3,,
 OMN,2,2,2,,,
 PAK,2,1,2,2,,
+PSE,2,2,2,,,
 PAN,2,2,2,2,,
+PNG,2,2,2,2,,
+PRY,2,2,2,,,
 PER,2,2,2,2,,
 PHL,2,2,3,2,3,
-PNG,2,2,2,2,,
 POL,2,3,3,,,
-PRI,2,2,,,,
-PRK,2,2,2,,,
-PRY,2,2,2,,,
-PSE,2,2,2,,,
-QAT,3,3,3,,,
+QAT,3,3,3,3,,
 ROU,2,3,3,,,
 RUS,2,3,3,,,
 RWA,2,1,1,2,2,
+KNA,2,2,,,,
+LCA,2,2,9,,,
+VCT,2,1,2,,,
+STP,2,2,2,,,
 SAU,2,2,,,,
-SDN,2,2,3,,,
 SEN,2,2,2,2,,
-SLB,2,2,4,4,,
+SYC,2,1,1,4,,
 SLE,2,2,2,2,2,
-SLV,2,2,2,,,
-SOM,2,2,2,,,
+SXM,2,1,,,,
+SVK,2,3,3,,,
+SLB,2,2,4,4,,
+SOM,2,2,2|7,,,
+ZAF,2,1,2,1,3,
+LKA,2,1,1,2,3,
 SSD,2,2,2,2,,
-STP,2,2,2,,,
+SDN,2,2,3,,,
 SUR,2,2,2,,,
-SVK,2,3,3,,,
 SWZ,2,1,2,,,
-SXM,2,1,,,,
-SYC,2,1,1,4,,
 SYR,2,2,2,2,,
-TCA,2,1,2,,,
-TCD,2,2,2,2,,
-TGO,2,2,2,2,,
-THA,2,2,2,2,,
 TJK,0,7,0,,,
+TZA,2,2,2,4|3,,
+THA,2,2,2,2,,
 TLS,2,2,2,2,,
+TGO,2,2,2,2,,
 TON,2,1,1,2,,
 TTO,2,2,,,,
 TUN,2,1,1,2,2,
-TUR,3,3,3,3,3,
-TZA,2,2,2,3|4,,
-UGA,2,1,3,2,2,
+TUR,3,3,3,,,
+TCA,2,1,2,,,
+UGA,2,1,3,,2,
 UKR,2,2,2,3,3,
+ARE,2,2,,,,
 URY,2,2,3,,,
 UZB,2,2,3,,,
-VCT,2,1,2,,,
+VUT,2,2,3,,,
 VEN,2,2,2,2,,
-VGB,2,2,,,,
-VIR,2,3,5,,,
 VNM,2,3,2,,,
-VUT,2,2,3,,,
+VGB,2,2,,,,
+ESH,2,2,,,,
 YEM,2,2,2,2,,
-ZAF,2,1,2,1,3,
 ZMB,2,3,3,3,3,
 ZWE,2,2,2,2,,
+VIR,2,3,5,,,