datamarket 0.9.44__tar.gz → 0.9.46__tar.gz

{datamarket-0.9.44 → datamarket-0.9.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.44
+Version: 0.9.46
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -57,6 +57,7 @@ Provides-Extra: retry
 Provides-Extra: shapely
 Provides-Extra: soda-core-mysql
 Provides-Extra: soda-core-postgres
+Provides-Extra: sqlparse
 Provides-Extra: tqdm
 Provides-Extra: undetected-chromedriver
 Provides-Extra: xmltodict
@@ -119,6 +120,7 @@ Requires-Dist: soda-core-mysql-utf8-hotfix (>=3.0.0,<4.0.0) ; extra == "soda-cor
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
 Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
 Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
+Requires-Dist: sqlparse (>=0.5.0,<0.6.0) ; extra == "sqlparse"
 Requires-Dist: stem (>=1.0.0,<2.0.0)
 Requires-Dist: tenacity (>=9.0.0,<10.0.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"

{datamarket-0.9.44 → datamarket-0.9.46}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.44"
+version = "0.9.46"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -85,6 +85,7 @@ spacy-langdetect = { version = "~0.1.0", optional = true }
 pandarallel = { version = "^1.0.0", optional = true }
 pyrate-limiter = { version = "^3.0.0", optional = true }
 pyproj = { version = "^3.0.0", optional = true }
+sqlparse = { version = "~0.5.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -131,6 +132,7 @@ camoufox = ["camoufox", "browserforge", "playwright"]
 pandarallel = ["pandarallel"]
 pyrate-limiter = ["pyrate-limiter"]
 pyproj = ["pyproj"]
+sqlparse = ["sqlparse"]
 
 # Interface groups
 aws = ["boto3"]

{datamarket-0.9.44 → datamarket-0.9.46}/src/datamarket/interfaces/nominatim.py

@@ -10,7 +10,7 @@ import requests
 from geopy.distance import geodesic
 from jellyfish import jaro_winkler_similarity
 
-from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES
+from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES, STATES
 from ..utils.strings import normalize
 
 ########################################################################################################################
@@ -141,6 +141,24 @@ class Nominatim:
             "number": None,
         }
 
+    @staticmethod
+    def _canonicalize_state(state: Optional[str]) -> Optional[str]:
+        """
+        Canonicalize the state name using similarity. The most similar canonical state name is
+        returned if the similarity score is above the threshold.
+        """
+        if not state:
+            return None
+        norm_state = normalize(state)
+        best_match = None
+        best_score = 0.0
+        for canonical in STATES:
+            score = jaro_winkler_similarity(norm_state, normalize(canonical))
+            if score > best_score:
+                best_score = score
+                best_match = canonical
+        return best_match if best_score > JARO_WINKLER_THRESHOLD else None
+
     def _select_postcode_and_derived_province(
         self,
         parsed_nominatim_result: Dict[str, Optional[str]],
@@ -225,6 +243,9 @@ class Nominatim:
             if not state and nominatim_pc_valid:
                 state = parsed_nominatim_result.get("state")
 
+        # Canonicalize
+        state = self._canonicalize_state(state)
+
         return postcode, province, state
 
     def _select_final_result(

{datamarket-0.9.44 → datamarket-0.9.46}/src/datamarket/params/nominatim.py

@@ -54,3 +54,25 @@ POSTCODES = {
     "51": "Ceuta",
     "52": "Melilla",
 }
+
+STATES = { 
+    "Andalucía",
+    "Aragón",
+    "Asturias",
+    "Baleares",
+    "Canarias",
+    "Cantabria",
+    "Castilla-La Mancha",
+    "Castilla y León",
+    "Cataluña",
+    "Ceuta", 
+    "Comunidad Valenciana",
+    "Extremadura",
+    "Galicia",
+    "La Rioja",
+    "Madrid",
+    "Melilla",
+    "Murcia",
+    "Navarra",
+    "País Vasco" 
+}