datamarket 0.7.91__py3-none-any.whl → 0.7.92__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -195,6 +195,29 @@ class AlchemyInterface:
 
         query_results.update({column_name: default_value}, synchronize_session=False)
 
+    @staticmethod
+    def _log_integrity_error(ex: IntegrityError, alchemy_obj, action="insert"):
+        """
+            Compact, readable IntegrityError logger using SQLSTATE codes.
+            Consult https://www.postgresql.org/docs/current/errcodes-appendix.html for details.
+        """
+
+        PG_ERROR_LABELS = {
+        "23000": "Integrity constraint violation",
+        "23001": "Restrict violation",
+        "23502": "NOT NULL violation",
+        "23503": "Foreign key violation",
+        "23505": "Unique violation",
+        "23514": "Check constraint violation",
+        "23P01": "Exclusion constraint violation",
+        }
+        code = getattr(ex.orig, "pgcode", None)
+        label = PG_ERROR_LABELS.get(code, "Integrity error (unspecified)")
+
+        # Log one clean message with trace + the raw DB message separately
+        logger.error(f"{label} trying to {action} {alchemy_obj}\nPostgreSQL message: {ex.orig}")
+
+
     def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
@@ -205,10 +228,10 @@ class AlchemyInterface:
                 if not silent:
                     logger.info(f"adding {alchemy_obj}...")
                 self.session.add(alchemy_obj)
-        except IntegrityError:
+        except IntegrityError as ex:
             # Rollback is handled automatically by begin_nested() context manager on error
             if not silent:
-                logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
+                self._log_integrity_error(ex, alchemy_obj, action="insert")
             # Do not re-raise, allow outer transaction/loop to continue
             return False
 
@@ -264,10 +287,10 @@ class AlchemyInterface:
             # Use a savepoint (nested transaction)
             with self.session.begin_nested():
                 self.session.execute(statement)
-        except IntegrityError:
+        except IntegrityError as ex:
             # Rollback is handled automatically by begin_nested() context manager on error
             if not silent:
-                logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
+                self._log_integrity_error(ex, alchemy_obj, action="upsert")
             # Do not re-raise, allow outer transaction/loop to continue
             return False
 

datamarket/interfaces/nominatim.py

@@ -10,8 +10,9 @@ import requests
 from geopy.distance import geodesic
 from jellyfish import jaro_winkler_similarity
 
-from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES, STATES
+from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES
 from ..utils.strings import normalize
+from ..utils.nominatim import standardize_admin_division
 
 ########################################################################################################################
 # PARAMETERS
@@ -141,24 +142,6 @@ class Nominatim:
             "number": None,
         }
 
-    @staticmethod
-    def _canonicalize_state(state: Optional[str]) -> Optional[str]:
-        """
-        Canonicalize the state name using similarity. The most similar canonical state name is
-        returned if the similarity score is above the threshold.
-        """
-        if not state:
-            return None
-        norm_state = normalize(state)
-        best_match = None
-        best_score = 0.0
-        for canonical in STATES:
-            score = jaro_winkler_similarity(norm_state, normalize(canonical))
-            if score > best_score:
-                best_score = score
-                best_match = canonical
-        return best_match if best_score > JARO_WINKLER_THRESHOLD else None
-
     def _select_postcode_and_derived_province(
         self,
         parsed_nominatim_result: Dict[str, Optional[str]],
@@ -243,9 +226,6 @@ class Nominatim:
             if not state and nominatim_pc_valid:
                 state = parsed_nominatim_result.get("state")
 
-        # Canonicalize
-        state = self._canonicalize_state(state)
-
         return postcode, province, state
 
     def _select_final_result(
@@ -355,7 +335,16 @@ class Nominatim:
             selected_province_from_postcode,
             selected_state,
         )
-
+        
+        # Standardize
+        final_result["province"] = standardize_admin_division(
+            name=final_result["province"], 
+            level="province",
+            country_code=final_result["country_code"])
+        final_result["state"] = standardize_admin_division(
+            name=final_result["state"], 
+            level="state",
+            country_code=final_result["country_code"])
         return final_result
 
 
@@ -369,4 +358,4 @@ class NominatimInterface(Nominatim):
 
             super().__init__(self.nominatim_endpoint, self.geonames_endpoint)
         else:
-            logger.warning("no osm section in config")
+            logger.warning("no osm section in config")

datamarket/params/nominatim.py

@@ -55,24 +55,90 @@ POSTCODES = {
     "52": "Melilla",
 }
 
+# Mapping of normalized names (for comparison) to standardized names (for storing)
+# for each corresponding country code
 STATES = {
-    "Andalucía",
-    "Aragón",
-    "Asturias",
-    "Baleares",
-    "Canarias",
-    "Cantabria",
-    "Castilla-La Mancha",
-    "Castilla y León",
-    "Cataluña",
-    "Ceuta",
-    "Comunidad Valenciana",
-    "Extremadura",
-    "Galicia",
-    "La Rioja",
-    "Madrid",
-    "Melilla",
-    "Murcia",
-    "Navarra",
-    "País Vasco",
+    "es": {
+        "andalucia": "Andalucía",
+        "aragon": "Aragón",
+        "asturias": "Asturias",
+        "baleares": "Baleares",
+        "canarias": "Canarias",
+        "cantabria": "Cantabria",
+        "castilla la mancha": "Castilla-La Mancha",
+        "castilla y leon": "Castilla y León",
+        "cataluna": "Cataluña",
+        "ceuta": "Ceuta",
+        "comunidad valenciana": "Comunidad Valenciana",
+        "extremadura": "Extremadura",
+        "galicia": "Galicia",
+        "la rioja": "La Rioja",
+        "madrid": "Comunidad de Madrid",
+        "melilla": "Melilla",
+        "murcia": "Murcia",
+        "navarra": "Navarra",
+        "pais vasco": "País Vasco",
+        "euskadi": "País Vasco",    # Alias not caught by rapidfuzz
+    }
 }
+
+PROVINCES = {
+    "es": {
+        "alava": "Álava",
+        "araba": "Álava", # Alias not caught by rapidfuzz
+        "albacete": "Albacete",
+        "alicante": "Alicante",
+        "almeria": "Almería",
+        "asturias": "Asturias",
+        "avila": "Ávila",
+        "badajoz": "Badajoz",
+        "barcelona": "Barcelona",
+        "bizkaia": "Vizcaya",
+        "burgos": "Burgos",
+        "caceres": "Cáceres",
+        "cadiz": "Cádiz",
+        "cantabria": "Cantabria",
+        "castellon": "Castellón",
+        "ceuta": "Ceuta", # Considered province by opensm and/or geonames
+        "ciudad real": "Ciudad Real",
+        "cordoba": "Córdoba",
+        "cuenca": "Cuenca",
+        "gipuzkoa": "Gipuzkoa",
+        "gerona": "Gerona",
+        "granada": "Granada",
+        "guadalajara": "Guadalajara",
+        "huelva": "Huelva",
+        "huesca": "Huesca",
+        "islas baleares": "Islas Baleares",
+        "jaen": "Jaén",
+        "la coruna": "La Coruña",
+        "la rioja": "La Rioja",
+        "las palmas": "Las Palmas",
+        "leon": "León",
+        "lerida": "Lérida",
+        "lugo": "Lugo",
+        "madrid": "Madrid",
+        "malaga": "Málaga",
+        "melilla": "Melilla", # Considered province by opensm and/or geonames
+        "murcia": "Murcia",
+        "navarra": "Navarra",
+        "orense": "Orense",
+        "palencia": "Palencia",
+        "pontevedra": "Pontevedra",
+        "salamanca": "Salamanca",
+        "santa cruz de tenerife": "Santa Cruz de Tenerife",
+        "segovia": "Segovia",
+        "sevilla": "Sevilla",
+        "soria": "Soria",
+        "tarragona": "Tarragona",
+        "teruel": "Teruel",
+        "toledo": "Toledo",
+        "valencia": "Valencia",
+        "valladolid": "Valladolid",
+        "zamora": "Zamora",
+        "zaragoza": "Zaragoza",
+    }
+}
+
+# Cutoff score for rapidfuzz in the name standardization function
+STANDARD_THRESHOLD = 40

datamarket/utils/main.py

@@ -9,6 +9,7 @@ import re
 import shlex
 import subprocess
 import time
+from babel.numbers import parse_decimal
 
 from bs4 import BeautifulSoup
 import pendulum
@@ -89,6 +90,19 @@ def text_to_int(text):
         return num
 
 
+def text_to_float(text: str | None, locale: str = "es_ES") -> float | None:
+    if not text:
+        return None
+    match = re.search(r"\d(?:[\d\s.,]*\d)?", text)
+    if not match:
+        return None
+    number_str = match.group(0).replace(" ", "")
+    try:
+        return float(parse_decimal(number_str, locale=locale))
+    except Exception:
+        return None
+
+
 def sleep_out_interval(from_h, to_h, tz="Europe/Madrid", seconds=1800):
     while pendulum.now(tz=tz).hour >= to_h or pendulum.now(tz=tz).hour < from_h:
         logger.warning("time to sleep and not scrape anything...")

datamarket/utils/nominatim.py

@@ -0,0 +1,38 @@
+from typing import Optional, Literal
+from rapidfuzz import fuzz, process
+from ..params.nominatim import STATES, PROVINCES, STANDARD_THRESHOLD
+from .strings import normalize
+
+def standardize_admin_division(
+    name: str,
+    level: Literal["province", "state"] = "province",
+    country_code: str = "es"
+) -> Optional[str]:
+    """
+    Normalize and standardize administrative divisions of a given country using RapidFuzz.
+    Uses normalized dict keys for comparison and returns dict values with the official names.
+    """
+    if not name:
+        return None
+
+    country_code = country_code.lower()
+    mapping = STATES.get(country_code) if level == "state" else PROVINCES.get(country_code)
+
+    if not mapping: # If country is not standardized, return raw name
+        return name
+
+    normalized_name = normalize(name) # Essential for rapidfuzz to work well
+    result = process.extractOne(
+        normalized_name,
+        mapping.keys(), # Compare with the normalized names in the dict
+        scorer=fuzz.WRatio,
+        score_cutoff=STANDARD_THRESHOLD,
+    )
+
+    if not result:
+        return None
+
+    best_key, score, _ = result
+    
+    # Return the standardized name corresponding to the normalized name
+    return mapping[best_key]

datamarket/utils/strings/normalization.py

@@ -215,4 +215,4 @@ def normalize(
     if naming is NamingConvention.PASCAL:
         return camelize(underscored)
 
-    return underscored
+    return underscored

{datamarket-0.7.91.dist-info → datamarket-0.7.92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.7.91
+Version: 0.7.92
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -64,6 +64,7 @@ Provides-Extra: undetected-chromedriver
 Provides-Extra: xmltodict
 Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0)
 Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
+Requires-Dist: babel (>=2.0.0,<3.0.0)
 Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: boto3 (>=1.35.0,<1.36.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
 Requires-Dist: browserforge (>=1.2.0,<2.0.0) ; extra == "camoufox"

{datamarket-0.7.91.dist-info → datamarket-0.7.92.dist-info}/RECORD

@@ -2,32 +2,33 @@ datamarket/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/exceptions/__init__.py,sha256=-Vu-RZNKjW6fYCLqbUJTkKNuHeA8Yi_gyR50oZNaA_8,33
 datamarket/exceptions/main.py,sha256=MP5ql6M7DoMbBf-Dg_2ohcUFdWXgzv-dXHntPPit31s,453
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=mQwjDqBpz1QHRV2JTCALvn5iK_ky69oE2Gw-EtRXsqQ,14664
+datamarket/interfaces/alchemy.py,sha256=TquEsTeYmiFaiyaTgIx0yF3McQeRdY5YVMsTOXrsT9s,15592
 datamarket/interfaces/aws.py,sha256=zfFEtIz6PsLMjKPOUcYuVgBcBIUkuNI2xKW-QturS30,4314
 datamarket/interfaces/azure.py,sha256=PnPlo95skYiq63qYa4QDvEnVYi2JblPmMSfbTsmXhFs,4937
 datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjHk,4840
 datamarket/interfaces/ftp.py,sha256=K219-PP21EhQo1A1LkvRLahlrw2-pf4svBN0LogZaJE,2813
-datamarket/interfaces/nominatim.py,sha256=HLk0FcdfbOVCF_i71l-Hlb17swL0W1a3Gg2n5OLD0tM,15507
+datamarket/interfaces/nominatim.py,sha256=TjS9O2U446XuPUzfP65NwDSG-RDNqmYb6-NKikM-34w,15187
 datamarket/interfaces/peerdb.py,sha256=sO451wEGNb_0DDwchZ6eBVYKltqHM5XKau-WsfspXzA,23640
 datamarket/interfaces/proxy.py,sha256=Uu-dHvpQOLNBZPGHAanLXnKT1789ArcHfOw8exECt34,5398
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/params/nominatim.py,sha256=fELSriTLSpotoT_k6Ft98fnD8EKHA8WopxPYPLSRfgs,1531
+datamarket/params/nominatim.py,sha256=g_zx4WYumj8F2Z9xSCvLXLGv-PuAckCGnSxU821u2O8,3986
 datamarket/utils/__init__.py,sha256=FHLh-Qp9XpM4LkAocppCf_llW2CWVVghGorkqxqt1wk,34
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=WweHHt3Ti-tVXdmLnpNYGsYpyTaCx_o1mvnL7_NomVY,5450
+datamarket/utils/main.py,sha256=KYHjDOps6_Q3TFV_Jj7MLj-L9Evx05AXELCvp06BARU,5857
+datamarket/utils/nominatim.py,sha256=R0yuEQ6aOG_eM1D-PeMlTV0GtvhWoAkzpR-xYx8U_g0,1247
 datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/utils/playwright/async_api.py,sha256=UbA2D4ScBtYeMfrRjly4RO-s8wXIub9c05J1eoOCpsQ,5782
 datamarket/utils/playwright/sync_api.py,sha256=Tw_-KLB3vipFuEQwcX8iCbj7giCzcwXB-bhl_ncR-2Q,5542
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/strings/__init__.py,sha256=b6TYOT9v7y9ID-lDyZk4E8BH2uIPbsF2ZSLGjCQ1MCQ,43
-datamarket/utils/strings/normalization.py,sha256=rj0wfJSjqcCRp-ruHqc5pylO3_TOmY5_V1lKzkyWoAA,8991
+datamarket/utils/strings/normalization.py,sha256=4HSs-8y1k36_xett0nqN2ND_A3xFn4591n5I-lhpaLM,8990
 datamarket/utils/strings/obfuscation.py,sha256=Jo-x3f2Cb75983smmpcdPqUlBrLCTyrnmH3FPlgUUjM,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.7.91.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.7.91.dist-info/METADATA,sha256=t_iQOXjo_CHGC7XwOBnPtWTTWypi-MaXrQBawy103vM,7348
-datamarket-0.7.91.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.7.91.dist-info/RECORD,,
+datamarket-0.7.92.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.7.92.dist-info/METADATA,sha256=OtPM2d_9vLMdOfAACyAnS2UpLDgJKRcDk-fp5ERKTG4,7386
+datamarket-0.7.92.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.7.92.dist-info/RECORD,,