datamarket 0.9.45__tar.gz → 0.9.47__tar.gz

{datamarket-0.9.45 → datamarket-0.9.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.45
+Version: 0.9.47
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -54,6 +54,7 @@ Provides-Extra: pyspark
 Provides-Extra: pytest
 Provides-Extra: rapidfuzz
 Provides-Extra: retry
+Provides-Extra: rnet
 Provides-Extra: shapely
 Provides-Extra: soda-core-mysql
 Provides-Extra: soda-core-postgres
@@ -115,6 +116,7 @@ Requires-Dist: python-string-utils (>=1.0.0,<2.0.0)
 Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0) ; extra == "rapidfuzz"
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
+Requires-Dist: rnet (>=2.0.0,<3.0.0) ; extra == "rnet"
 Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
 Requires-Dist: soda-core-mysql-utf8-hotfix (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"

{datamarket-0.9.45 → datamarket-0.9.47}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.45"
+version = "0.9.47"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -86,6 +86,7 @@ pandarallel = { version = "^1.0.0", optional = true }
 pyrate-limiter = { version = "^3.0.0", optional = true }
 pyproj = { version = "^3.0.0", optional = true }
 sqlparse = { version = "~0.5.0", optional = true }
+rnet = { version = "^2.0.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -133,6 +134,7 @@ pandarallel = ["pandarallel"]
 pyrate-limiter = ["pyrate-limiter"]
 pyproj = ["pyproj"]
 sqlparse = ["sqlparse"]
+rnet = ["rnet"]
 
 # Interface groups
 aws = ["boto3"]

{datamarket-0.9.45 → datamarket-0.9.47}/src/datamarket/interfaces/nominatim.py

@@ -10,7 +10,7 @@ import requests
 from geopy.distance import geodesic
 from jellyfish import jaro_winkler_similarity
 
-from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES
+from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES, STATES
 from ..utils.strings import normalize
 
 ########################################################################################################################
@@ -141,6 +141,24 @@ class Nominatim:
             "number": None,
         }
 
+    @staticmethod
+    def _canonicalize_state(state: Optional[str]) -> Optional[str]:
+        """
+        Canonicalize the state name using similarity. The most similar canonical state name is
+        returned if the similarity score is above the threshold.
+        """
+        if not state:
+            return None
+        norm_state = normalize(state)
+        best_match = None
+        best_score = 0.0
+        for canonical in STATES:
+            score = jaro_winkler_similarity(norm_state, normalize(canonical))
+            if score > best_score:
+                best_score = score
+                best_match = canonical
+        return best_match if best_score > JARO_WINKLER_THRESHOLD else None
+
     def _select_postcode_and_derived_province(
         self,
         parsed_nominatim_result: Dict[str, Optional[str]],
@@ -225,6 +243,9 @@ class Nominatim:
             if not state and nominatim_pc_valid:
                 state = parsed_nominatim_result.get("state")
 
+        # Canonicalize
+        state = self._canonicalize_state(state)
+
         return postcode, province, state
 
     def _select_final_result(

{datamarket-0.9.45 → datamarket-0.9.47}/src/datamarket/params/nominatim.py

@@ -54,3 +54,25 @@ POSTCODES = {
     "51": "Ceuta",
     "52": "Melilla",
 }
+
+STATES = { 
+    "Andalucía",
+    "Aragón",
+    "Asturias",
+    "Baleares",
+    "Canarias",
+    "Cantabria",
+    "Castilla-La Mancha",
+    "Castilla y León",
+    "Cataluña",
+    "Ceuta", 
+    "Comunidad Valenciana",
+    "Extremadura",
+    "Galicia",
+    "La Rioja",
+    "Madrid",
+    "Melilla",
+    "Murcia",
+    "Navarra",
+    "País Vasco" 
+}