datamarket 0.7.89__py3-none-any.whl → 0.7.125__py3-none-any.whl

datamarket/exceptions/__init__.py

@@ -1 +1 @@
-from .main import *  # noqa: F403
+from .main import *  # noqa: F403

datamarket/exceptions/main.py

@@ -2,13 +2,117 @@
 # CLASSES
 
 
-class RedirectionDetectedError(Exception):
-    def __init__(self, message="Redirection detected!"):
+from typing import Optional
+
+from requests import Request, Response
+from requests.exceptions import HTTPError
+
+
+class ManagedHTTPError(HTTPError):
+    """Signal that this HTTP status was handled and should not be retried."""
+
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        *args,
+        **kwargs,
+    ):
+        self.response = response
+        self.request = request or getattr(response, "request", None)
+
+        # Build a safe default message
+        if not message:
+            status = getattr(self.response, "status_code", "unknown")
+            url = getattr(self.request, "url", "unknown")
+            message = f"HTTP {status} for {url}"
+
+        self.message = message
+
+        super().__init__(message, *args, response=response, **kwargs)
+
+
+class IgnoredHTTPError(ManagedHTTPError):
+    """Exception type that signals the error should be ignored by retry logic."""
+
+    pass
+
+
+class NotFoundError(ManagedHTTPError):
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        *args,
+        **kwargs,
+    ):
+        if not message:
+            status = getattr(response, "status_code", 404)
+            req = request or getattr(response, "request", None)
+            url = getattr(req, "url", "unknown")
+            message = f"HTTP {status} for {url}"
+        super().__init__(message, response, request, *args, **kwargs)
+
+
+class BadRequestError(ManagedHTTPError):
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        *args,
+        **kwargs,
+    ):
+        if not message:
+            status = getattr(response, "status_code", 400)
+            req = request or getattr(response, "request", None)
+            url = getattr(req, "url", "unknown")
+            message = f"HTTP {status} for {url}"
+        super().__init__(message, response, request, *args, **kwargs)
+
+
+class EmptyResponseError(ManagedHTTPError):
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        *args,
+        **kwargs,
+    ):
+        if not message:
+            req = request or getattr(response, "request", None)
+            url = getattr(req, "url", "unknown")
+            message = f"Empty response for {url}"
+        super().__init__(message, response, request, *args, **kwargs)
+
+
+class RedirectionDetectedError(ManagedHTTPError):
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        *args,
+        **kwargs,
+    ):
+        if not message:
+            status = getattr(response, "status_code", 300)
+            req = request or getattr(response, "request", None)
+            url = getattr(req, "url", "unknown")
+            message = f"HTTP {status} for {url}"
+        super().__init__(message, response, request, *args, **kwargs)
+
+
+class NoWorkingProxiesError(Exception):
+    def __init__(self, message="No working proxies available"):
         self.message = message
         super().__init__(self.message)
 
 
-class NotFoundError(Exception):
-    def __init__(self, message="Not found!"):
+class EnsureNewIPTimeoutError(Exception):
+    def __init__(self, message="Timed out waiting for new IP"):
         self.message = message
         super().__init__(self.message)

datamarket/interfaces/alchemy.py

@@ -3,6 +3,7 @@
 
 import logging
 from collections.abc import MutableMapping
+from enum import Enum, auto
 from typing import Any, Iterator, List, Optional, Type, TypeVar, Union
 from urllib.parse import quote_plus
 
@@ -12,7 +13,6 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import DeclarativeMeta
 from sqlalchemy.orm import Session, sessionmaker
 from sqlalchemy.sql.expression import ClauseElement
-from enum import Enum, auto
 
 ########################################################################################################################
 # CLASSES
@@ -195,6 +195,31 @@ class AlchemyInterface:
 
         query_results.update({column_name: default_value}, synchronize_session=False)
 
+    @staticmethod
+    def _log_integrity_error(ex: IntegrityError, alchemy_obj, action="insert"):
+        """
+        Compact, readable IntegrityError logger using SQLSTATE codes.
+        Consult https://www.postgresql.org/docs/current/errcodes-appendix.html for details.
+        """
+
+        PG_ERROR_LABELS = {
+            "23000": "Integrity constraint violation",
+            "23001": "Restrict violation",
+            "23502": "NOT NULL violation",
+            "23503": "Foreign key violation",
+            "23505": "Unique violation",
+            "23514": "Check constraint violation",
+            "23P01": "Exclusion constraint violation",
+        }
+        code = getattr(ex.orig, "pgcode", None)
+        label = PG_ERROR_LABELS.get(code, "Integrity error (unspecified)")
+
+        # Log one clean message with trace + the raw DB message separately
+        if code == "23505":  # A simple info log for unique violations
+            logger.info(f"{label} trying to {action} {alchemy_obj}")
+        else:
+            logger.error(f"{label} trying to {action} {alchemy_obj}\nPostgreSQL message: {ex.orig}")
+
     def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
@@ -205,10 +230,10 @@ class AlchemyInterface:
                 if not silent:
                     logger.info(f"adding {alchemy_obj}...")
                 self.session.add(alchemy_obj)
-        except IntegrityError:
+        except IntegrityError as ex:
             # Rollback is handled automatically by begin_nested() context manager on error
             if not silent:
-                logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
+                self._log_integrity_error(ex, alchemy_obj, action="insert")
             # Do not re-raise, allow outer transaction/loop to continue
             return False
 
@@ -264,10 +289,10 @@ class AlchemyInterface:
             # Use a savepoint (nested transaction)
             with self.session.begin_nested():
                 self.session.execute(statement)
-        except IntegrityError:
+        except IntegrityError as ex:
             # Rollback is handled automatically by begin_nested() context manager on error
             if not silent:
-                logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
+                self._log_integrity_error(ex, alchemy_obj, action="upsert")
             # Do not re-raise, allow outer transaction/loop to continue
             return False
 

datamarket/interfaces/aws.py

@@ -3,8 +3,9 @@
 
 import io
 import logging
+from typing import Any, Dict, List, Optional
+
 import boto3
-from typing import Optional, List, Dict, Any
 
 ########################################################################################################################
 # CLASSES
@@ -82,6 +83,14 @@ class AWSInterface:
                 return
         logger.warning(f"Profile {profile_name} not found")
 
+    def get_bucket_url(self) -> Optional[str]:
+        """Return active bucket URL."""
+        if not self.bucket:
+            logger.warning("No active bucket selected")
+            return None
+        region = self.s3_client.meta.region_name
+        return f"https://{self.bucket}.s3.{region}.amazonaws.com"
+
     def get_file(self, s3_path: str):
         if not self.bucket:
             logger.warning("No active bucket selected")
@@ -92,14 +101,27 @@ class AWSInterface:
             logger.info(f"{s3_path} does not exist")
             return None
 
+    def file_exists(self, s3_path: str) -> bool:
+        if not self.bucket:
+            logger.warning("No active bucket selected")
+            return False
+        try:
+            self.s3_client.head_object(Bucket=self.bucket, Key=s3_path)
+            return True
+        except self.s3_client.exceptions.NoSuchKey:
+            return False
+        except Exception as e:
+            logger.error(f"Error checking existence of {s3_path}: {e}")
+            raise
+
     def read_file_as_bytes(self, s3_path: str) -> Optional[io.BytesIO]:
         obj = self.get_file(s3_path)
         if not obj:
             return None
         return io.BytesIO(obj["Body"].read())
 
-    def upload_file(self, local_path: str, s3_path: str) -> None:
+    def upload_file(self, local_path: str, s3_path: str, **kwargs) -> None:
         if not self.bucket:
             logger.warning("No active bucket selected")
             return
-        self.s3.Bucket(self.bucket).upload_file(local_path, s3_path)
+        self.s3.Bucket(self.bucket).upload_file(local_path, s3_path, **kwargs)

datamarket/interfaces/azure.py

@@ -5,7 +5,7 @@ import logging
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-from azure.storage.blob import BlobServiceClient
+from azure.storage.blob import BlobServiceClient, ContainerClient
 from pendulum import now
 
 ########################################################################################################################
@@ -24,22 +24,26 @@ class AzureBlobInterface:
                 profile_name = section.split(":", 1)[1]
                 connection_string = self.config[section].get("connection_string")
                 container_name = self.config[section].get("container_name")
+                sas_container_url = self.config[section].get("sas_container_url")
+
+                if sas_container_url:
+                    session = ContainerClient.from_container_url(sas_container_url)
+                elif connection_string and container_name:
+                    session = BlobServiceClient.from_connection_string(connection_string).get_container_client(
+                        container_name
+                    )
 
                 self.profiles.append(
                     {
                         "profile": profile_name,
                         "container_name": container_name,
-                        "session": BlobServiceClient.from_connection_string(
-                            connection_string
-                        ).get_container_client(container_name),
+                        "session": session,
                     }
                 )
 
         if not self.profiles:
             logger.warning("No Azure profiles found in config file")
-        self.current_profile: Optional[Dict[str, Any]] = (
-            self.profiles[0] if self.profiles else None
-        )
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
 
     def switch_profile(self, profile_name: str) -> None:
         for profile in self.profiles:
@@ -54,7 +58,7 @@ class AzureBlobInterface:
         remote_folder,
         remote_file=None,
         upload_file_info=False,
-        **kwargs,
+        **file_info_data,
     ):
         if not remote_file:
             remote_file = Path(local_file).name
@@ -66,16 +70,16 @@ class AzureBlobInterface:
             blob_client.upload_blob(data, overwrite=True)
 
         if upload_file_info:
-            self.upload_file_info(remote_path, **kwargs)
+            self.upload_file_info(remote_path, **file_info_data)
 
-    def upload_file_info(self, remote_path, **kwargs):
+    def upload_file_info(self, remote_path, **file_info_data):
         summary_file = remote_path.split(".")[0] + "_resumen.csv"
         blob_client = self.current_profile["session"].get_blob_client(summary_file)
 
         new_record = {
             "file": remote_path,
-            "num_rows": kwargs["num_rows"],
-            "schema_version": kwargs["schema_version"],
+            "num_rows": file_info_data.get("num_rows"),
+            "schema_version": file_info_data.get("schema_version"),
             "upload_date": now(tz="Europe/Madrid").to_datetime_string(),
         }
 
@@ -109,14 +113,10 @@ class AzureBlobInterface:
             if blob_client.exists():
                 properties = blob_client.get_blob_properties()
                 if properties.size > 100:  # Check if size is greater than 100 bytes
-                    logger.debug(
-                        f"Blob '{remote_path}' exists and is not empty (size: {properties.size})."
-                    )
+                    logger.debug(f"Blob '{remote_path}' exists and is not empty (size: {properties.size}).")
                     return True
                 else:
-                    logger.debug(
-                        f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes."
-                    )
+                    logger.debug(f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes.")
                     return False
             else:
                 logger.debug(f"Blob '{remote_path}' does not exist.")

datamarket/interfaces/ftp.py

@@ -20,22 +20,14 @@ class FTPInterface:
             if section.startswith("ftp:"):
                 profile_name = section.split(":", 1)[1]
                 ftps = self.config[section]["ftps"].lower() == "true"
-                ftp_conn = (
-                    FTP_TLS(self.config[section]["server"])
-                    if ftps
-                    else FTP(self.config[section]["server"])
-                )  # noqa: S321
-                ftp_conn.login(
-                    self.config[section]["username"], self.config[section]["password"]
-                )
+                ftp_conn = FTP_TLS(self.config[section]["server"]) if ftps else FTP(self.config[section]["server"])  # noqa: S321
+                ftp_conn.login(self.config[section]["username"], self.config[section]["password"])
                 self.profiles.append({"profile": profile_name, "session": ftp_conn})
 
         if not self.profiles:
             logger.warning("no ftp section in config")
 
-        self.current_profile: Optional[Dict[str, Any]] = (
-            self.profiles[0] if self.profiles else None
-        )
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
         self.ftp = self.current_profile["session"] if self.current_profile else None
 
     def switch_profile(self, profile_name: str) -> None:

datamarket/interfaces/nominatim.py

@@ -10,7 +10,14 @@ import requests
 from geopy.distance import geodesic
 from jellyfish import jaro_winkler_similarity
 
-from ..params.nominatim import CITY_TO_PROVINCE, POSTCODES, STATES
+from ..params.nominatim import (
+    CITY_TO_PROVINCE,
+    MADRID_DISTRICT_DIRECT_PATCH,
+    MADRID_DISTRICT_QUARTER_PATCH,
+    MADRID_QUARTER_DIRECT_PATCH,
+    POSTCODES,
+)
+from ..utils.nominatim import standardize_admin_division
 from ..utils.strings import normalize
 
 ########################################################################################################################
@@ -141,24 +148,6 @@ class Nominatim:
             "number": None,
         }
 
-    @staticmethod
-    def _canonicalize_state(state: Optional[str]) -> Optional[str]:
-        """
-        Canonicalize the state name using similarity. The most similar canonical state name is
-        returned if the similarity score is above the threshold.
-        """
-        if not state:
-            return None
-        norm_state = normalize(state)
-        best_match = None
-        best_score = 0.0
-        for canonical in STATES:
-            score = jaro_winkler_similarity(norm_state, normalize(canonical))
-            if score > best_score:
-                best_score = score
-                best_match = canonical
-        return best_match if best_score > JARO_WINKLER_THRESHOLD else None
-
     def _select_postcode_and_derived_province(
         self,
         parsed_nominatim_result: Dict[str, Optional[str]],
@@ -214,7 +203,7 @@ class Nominatim:
             )
 
         # 2) If no raw province, compare with province from Nominatim PC **only when** Nominatim is close
-        if not geonames_pc_valid and not norm_raw_province and nominatim_is_close:
+        if not geonames_pc_valid and not norm_raw_province and nominatim_is_close:  # noqa: SIM102
             if norm_province_from_geonames_pc and norm_province_from_nominatim_pc:
                 geonames_pc_valid = (
                     jaro_winkler_similarity(norm_province_from_geonames_pc, norm_province_from_nominatim_pc)
@@ -243,9 +232,6 @@ class Nominatim:
             if not state and nominatim_pc_valid:
                 state = parsed_nominatim_result.get("state")
 
-        # Canonicalize
-        state = self._canonicalize_state(state)
-
         return postcode, province, state
 
     def _select_final_result(
@@ -293,14 +279,32 @@ class Nominatim:
 
         return final_result
 
+    @staticmethod
+    def _patch_district(raw_district: str, raw_quarter: str = None):
+        """
+        Patches the district name, optionally using the quarter for specific patches.
+        """
+        if raw_quarter:
+            # If raw_quarter is provided, use the tuple (district, quarter) as the key.
+            key = (raw_district, raw_quarter)
+            return MADRID_DISTRICT_QUARTER_PATCH.get(key, raw_district)
+        else:
+            return MADRID_DISTRICT_DIRECT_PATCH.get(raw_district, raw_district)
+
+    @staticmethod
+    def _patch_quarter(raw_quarter: str):
+        """
+        Patches the quarter name directly.
+        """
+        return MADRID_QUARTER_DIRECT_PATCH.get(raw_quarter, raw_quarter)
+
     def _get_district_quarter(self, raw_json: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
         district = self._get_attribute(raw_json, ["city_district", "suburb", "borough"])
         quarter = self._get_attribute(raw_json, ["quarter", "neighbourhood"])
-
-        if not district and quarter:
-            district = quarter
-            quarter = None
-
+        if (city := raw_json.get("city")) and city == "Madrid":
+            mid_district = self._patch_district(district)
+            quarter = self._patch_quarter(quarter)
+            district = self._patch_district(mid_district, quarter)
         return district, quarter
 
     def geocode(self, address: str) -> List[Dict[str, Any]]:
@@ -356,6 +360,13 @@ class Nominatim:
             selected_state,
         )
 
+        # Standardize
+        final_result["province"] = standardize_admin_division(
+            name=final_result["province"], level="province", country_code=final_result["country_code"]
+        )
+        final_result["state"] = standardize_admin_division(
+            name=final_result["state"], level="state", country_code=final_result["country_code"]
+        )
         return final_result