datamarket 0.9.39__py3-none-any.whl → 0.9.40__py3-none-any.whl

datamarket/interfaces/nominatim.py

@@ -17,6 +17,7 @@ from ..utils.strings import normalize
 # PARAMETERS
 
 JARO_WINKLER_THRESHOLD = 0.85
+CLOSE_KM = 2.0
 
 ########################################################################################################################
 # CLASSES
@@ -145,55 +146,86 @@ class Nominatim:
         parsed_nominatim_result: Dict[str, Optional[str]],
         parsed_geonames_result: Dict[str, Optional[str]],
         nominatim_address_province_raw: Optional[str],
-    ) -> Tuple[Optional[str], Optional[str]]:
+        dist_nominatim: float,  # distance Nominatim ↔ input (km)
+        dist_geonames: float,  # distance GeoNames ↔ input (km)
+    ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
         """
-        Determines the postcode and its derived province based on comparisons
-        between Nominatim and GeoNames data, and Nominatim's raw address province.
+        Decide the authoritative postcode, the province derived from it and the associated state.
+
+        Strategy:
+        1. Derive province from each postcode.
+        2. Validate each postcode–province pair:
+        • Nominatim: compare with raw province string (if present).
+        • GeoNames: multi-step validation (raw province, then Nominatim-derived
+            province when Nominatim coords are close, then distance fallback).
+        3. Return the postcode/province that passes validation with precedence:
+        Nominatim > GeoNames. Returns (None, None, None) if neither passes.
         """
+
+        # --- Extract postcodes ---
         nominatim_postcode = parsed_nominatim_result.get("postcode")
         geonames_postcode = parsed_geonames_result.get("postcode")
 
-        province_from_nominatim_postcode = self.geonames.get_province_from_postcode(nominatim_postcode)
-        province_from_geonames_postcode = self.geonames.get_province_from_postcode(geonames_postcode)
+        # --- Province derived from each postcode ---
+        province_from_nominatim_pc = self.geonames.get_province_from_postcode(nominatim_postcode)
+        province_from_geonames_pc = self.geonames.get_province_from_postcode(geonames_postcode)
 
-        norm_raw_nominatim_province = (
-            normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
-        )
-        norm_province_from_nominatim_postcode = (
-            normalize(province_from_nominatim_postcode) if province_from_nominatim_postcode else ""
-        )
-        norm_province_from_geonames_postcode = (
-            normalize(province_from_geonames_postcode) if province_from_geonames_postcode else ""
-        )
+        # --- Normalised strings for similarity comparisons ---
+        norm_raw_province = normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
+        norm_province_from_nominatim_pc = normalize(province_from_nominatim_pc) if province_from_nominatim_pc else ""
+        norm_province_from_geonames_pc = normalize(province_from_geonames_pc) if province_from_geonames_pc else ""
 
-        selected_postcode = None
-        selected_province_from_postcode = None
+        # --- Distance heuristics ---
+        nominatim_is_close = dist_nominatim < CLOSE_KM
+        geonames_is_close = dist_geonames < CLOSE_KM
 
-        # If provinces derived from Nominatim and GeoNames postcodes differ
-        nominatim_postcode_province_matches = False
-        if norm_province_from_nominatim_postcode and norm_raw_nominatim_province:
-            nominatim_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_nominatim_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
+        # --- Validate Nominatim postcode ---
+        nominatim_pc_valid = False
+        if norm_province_from_nominatim_pc and norm_raw_province:
+            nominatim_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_nominatim_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
             )
 
-        geonames_postcode_province_matches = False
-        if norm_province_from_geonames_postcode and norm_raw_nominatim_province:
-            geonames_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_geonames_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
-            )
+        # --- Validate GeoNames postcode ---
+        geonames_pc_valid = False
 
-        # Prefer GeoNames postcode if its province matches Nominatim's raw address province,
-        # and Nominatim's own postcode-derived province does not.
-        if nominatim_postcode_province_matches:
-            selected_postcode = nominatim_postcode
-            selected_province_from_postcode = province_from_nominatim_postcode
-        if geonames_postcode_province_matches and not nominatim_postcode_province_matches:
-            selected_postcode = geonames_postcode
-            selected_province_from_postcode = province_from_geonames_postcode
+        # 1) Compare with raw province string (if exists)
+        if norm_province_from_geonames_pc and norm_raw_province:
+            geonames_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_geonames_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
+            )
 
-        return selected_postcode, selected_province_from_postcode
+        # 2) If no raw province, compare with province from Nominatim PC **only when** Nominatim is close
+        if not geonames_pc_valid and not norm_raw_province and nominatim_is_close:
+            if norm_province_from_geonames_pc and norm_province_from_nominatim_pc:
+                geonames_pc_valid = (
+                    jaro_winkler_similarity(norm_province_from_geonames_pc, norm_province_from_nominatim_pc)
+                    > JARO_WINKLER_THRESHOLD
+                )
+
+        # 3) Fallback: accept GeoNames PC if its coordinates are very close
+        if not geonames_pc_valid and geonames_is_close and geonames_postcode:
+            geonames_pc_valid = True
+
+        # --- Select authoritative tuple ---
+        postcode = None
+        province = None
+        state = None
+
+        if nominatim_pc_valid:
+            postcode = nominatim_postcode
+            province = province_from_nominatim_pc
+            state = parsed_nominatim_result.get("state")
+            if not state and geonames_pc_valid:
+                state = parsed_geonames_result.get("state")
+        elif geonames_pc_valid:
+            postcode = geonames_postcode
+            province = province_from_geonames_pc
+            state = parsed_geonames_result.get("state")
+            if not state and nominatim_pc_valid:
+                state = parsed_nominatim_result.get("state")
+
+        return postcode, province, state
 
     def _select_final_result(
         self,
@@ -203,21 +235,41 @@ class Nominatim:
         dist_geonames: float,
         authoritative_postcode: Optional[str],
         authoritative_province_from_postcode: Optional[str],
-        nominatim_province: Optional[str],
+        authoritative_state: Optional[str],
     ) -> Dict[str, Optional[str]]:
         """
-        Selects the final address result based on distances and applies the authoritative postcode/province.
+        Choose the address block (Nominatim vs GeoNames) based on distance,
+        then apply the authoritative postcode/province.
+
+        Rules:
+        • Pick the source with the smaller finite distance.
+        • Always overwrite 'postcode' if authoritative_postcode is present.
+        • Overwrite 'province' only when authoritative_province_from_postcode is not None.
+        • If both distances are ∞, return an empty address.
         """
+
+        # ------------------------------------------------------------------ #
+        # 1. Decide the base address block                                  #
+        # ------------------------------------------------------------------ #
         if dist_nominatim <= dist_geonames and dist_nominatim != float("inf"):
             final_result = parsed_nominatim_result
-            final_result["postcode"] = authoritative_postcode
-            final_result["province"] = nominatim_province
         elif dist_geonames < dist_nominatim and dist_geonames != float("inf"):
             final_result = parsed_geonames_result
+        else:
+            return self._get_empty_address_result()
+
+        # ------------------------------------------------------------------ #
+        # 2. Apply authoritative postcode / province                        #
+        # ------------------------------------------------------------------ #
+        if authoritative_postcode:
             final_result["postcode"] = authoritative_postcode
+
+        if authoritative_province_from_postcode:
             final_result["province"] = authoritative_province_from_postcode
-        else:
-            final_result = self._get_empty_address_result()
+
+        if authoritative_province_from_postcode:
+            final_result["state"] = authoritative_state
+
         return final_result
 
     def _get_district_quarter(self, raw_json: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
@@ -250,12 +302,6 @@ class Nominatim:
         parsed_nominatim_result = self._parse_nominatim_result(nominatim_response)
         parsed_geonames_result = self._parse_geonames_result(geonames_response)
 
-        # Determine authoritative postcode
-        nominatim_province = parsed_nominatim_result.get("province")
-        selected_postcode, selected_province_from_postcode = self._select_postcode_and_derived_province(
-            parsed_nominatim_result, parsed_geonames_result, nominatim_province
-        )
-
         # Calculate distances
         nominatim_response_lat = nominatim_response.get("lat")
         nominatim_response_lon = nominatim_response.get("lon")
@@ -272,6 +318,12 @@ class Nominatim:
         dist_nominatim = self._calculate_distance(nominatim_response_lat, nominatim_response_lon, input_coords)
         dist_geonames = self._calculate_distance(geonames_response_lat, geonames_response_lon, input_coords)
 
+        # Determine authoritative postcode
+        nominatim_province = parsed_nominatim_result.get("province")
+        selected_postcode, selected_province_from_postcode, selected_state = self._select_postcode_and_derived_province(
+            parsed_nominatim_result, parsed_geonames_result, nominatim_province, dist_nominatim, dist_geonames
+        )
+
         # Select final result
         final_result = self._select_final_result(
             parsed_nominatim_result,
@@ -280,7 +332,7 @@ class Nominatim:
             dist_geonames,
             selected_postcode,
             selected_province_from_postcode,
-            nominatim_province,
+            selected_state,
         )
 
         return final_result

{datamarket-0.9.39.dist-info → datamarket-0.9.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.39
+Version: 0.9.40
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.39.dist-info → datamarket-0.9.40.dist-info}/RECORD

@@ -4,7 +4,7 @@ datamarket/interfaces/alchemy.py,sha256=mQwjDqBpz1QHRV2JTCALvn5iK_ky69oE2Gw-EtRX
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
-datamarket/interfaces/nominatim.py,sha256=XzqBFVBzGU2BIFnFueZ56tk4JhaQgj5dvFalnCG6Zxk,12417
+datamarket/interfaces/nominatim.py,sha256=d_KIrgzTusVYnw0Fk3YWCjrzlT9sI_bObGG-wOr__as,14726
 datamarket/interfaces/peerdb.py,sha256=cwYwvO740GyaPo9zLAwJsf3UeJDGDiYzjQVM9Q6s-_g,23652
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
@@ -24,7 +24,7 @@ datamarket/utils/strings/normalization.py,sha256=QLZ-THzjGOK9eWPPR1PrsffwQkSOx_M
 datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.39.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.39.dist-info/METADATA,sha256=hSvX9DNsR_mJwh_Wcx1HbU5o3LTK2nGctn-kzQ-ZERo,6961
-datamarket-0.9.39.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.9.39.dist-info/RECORD,,
+datamarket-0.9.40.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.40.dist-info/METADATA,sha256=odG5B_7jOuXZFBNe260K0HP7vkDntMf8yAUOgSxNOws,6961
+datamarket-0.9.40.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.40.dist-info/RECORD,,