datamarket 0.9.39__tar.gz → 0.9.41__tar.gz

{datamarket-0.9.39 → datamarket-0.9.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.39
+Version: 0.9.41
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -48,6 +48,7 @@ Provides-Extra: proxy
 Provides-Extra: pyarrow
 Provides-Extra: pydrive2
 Provides-Extra: pymupdf
+Provides-Extra: pyrate-limiter
 Provides-Extra: pysocks
 Provides-Extra: pyspark
 Provides-Extra: pytest
@@ -105,6 +106,7 @@ Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
 Requires-Dist: pycountry (>=24.0.0,<25.0.0)
 Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
 Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
+Requires-Dist: pyrate-limiter (>=3.0.0,<4.0.0) ; extra == "pyrate-limiter"
 Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"
 Requires-Dist: pyspark (>=3.0.0,<4.0.0) ; extra == "pyspark"
 Requires-Dist: pytest (>=8.0.0,<9.0.0) ; extra == "pytest"

{datamarket-0.9.39 → datamarket-0.9.41}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.39"
+version = "0.9.41"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -83,6 +83,7 @@ presidio-anonymizer = { version = "^2.0.0", optional = true }
 spacy = { version = "^3.0.0", optional = true }
 spacy-langdetect = { version = "~0.1.0", optional = true }
 pandarallel = { version = "^1.0.0", optional = true }
+pyrate-limiter = { version = "^3.0.0", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -128,6 +129,7 @@ openpyxl = ["openpyxl"]
 httpx = ["httpx"]
 camoufox = ["camoufox", "browserforge", "playwright"]
 pandarallel = ["pandarallel"]
+pyrate-limiter = ["pyrate-limiter"]
 
 # Interface groups
 aws = ["boto3"]

datamarket-0.9.41/src/datamarket/exceptions/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

datamarket-0.9.41/src/datamarket/exceptions/main.py

@@ -0,0 +1,14 @@
+########################################################################################################################
+# CLASSES
+
+
+class RedirectionDetectedError(Exception):
+    def __init__(self, message="Redirection detected!"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NotFoundError(Exception):
+    def __init__(self, message="Not found!"):
+        self.message = message
+        super().__init__(self.message)

{datamarket-0.9.39 → datamarket-0.9.41}/src/datamarket/interfaces/nominatim.py

@@ -17,6 +17,7 @@ from ..utils.strings import normalize
 # PARAMETERS
 
 JARO_WINKLER_THRESHOLD = 0.85
+CLOSE_KM = 2.0
 
 ########################################################################################################################
 # CLASSES
@@ -145,55 +146,86 @@ class Nominatim:
         parsed_nominatim_result: Dict[str, Optional[str]],
         parsed_geonames_result: Dict[str, Optional[str]],
         nominatim_address_province_raw: Optional[str],
-    ) -> Tuple[Optional[str], Optional[str]]:
+        dist_nominatim: float,  # distance Nominatim ↔ input (km)
+        dist_geonames: float,  # distance GeoNames ↔ input (km)
+    ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
         """
-        Determines the postcode and its derived province based on comparisons
-        between Nominatim and GeoNames data, and Nominatim's raw address province.
+        Decide the authoritative postcode, the province derived from it and the associated state.
+
+        Strategy:
+        1. Derive province from each postcode.
+        2. Validate each postcode–province pair:
+        • Nominatim: compare with raw province string (if present).
+        • GeoNames: multi-step validation (raw province, then Nominatim-derived
+            province when Nominatim coords are close, then distance fallback).
+        3. Return the postcode/province that passes validation with precedence:
+        Nominatim > GeoNames. Returns (None, None, None) if neither passes.
         """
+
+        # --- Extract postcodes ---
         nominatim_postcode = parsed_nominatim_result.get("postcode")
         geonames_postcode = parsed_geonames_result.get("postcode")
 
-        province_from_nominatim_postcode = self.geonames.get_province_from_postcode(nominatim_postcode)
-        province_from_geonames_postcode = self.geonames.get_province_from_postcode(geonames_postcode)
+        # --- Province derived from each postcode ---
+        province_from_nominatim_pc = self.geonames.get_province_from_postcode(nominatim_postcode)
+        province_from_geonames_pc = self.geonames.get_province_from_postcode(geonames_postcode)
 
-        norm_raw_nominatim_province = (
-            normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
-        )
-        norm_province_from_nominatim_postcode = (
-            normalize(province_from_nominatim_postcode) if province_from_nominatim_postcode else ""
-        )
-        norm_province_from_geonames_postcode = (
-            normalize(province_from_geonames_postcode) if province_from_geonames_postcode else ""
-        )
+        # --- Normalised strings for similarity comparisons ---
+        norm_raw_province = normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
+        norm_province_from_nominatim_pc = normalize(province_from_nominatim_pc) if province_from_nominatim_pc else ""
+        norm_province_from_geonames_pc = normalize(province_from_geonames_pc) if province_from_geonames_pc else ""
 
-        selected_postcode = None
-        selected_province_from_postcode = None
+        # --- Distance heuristics ---
+        nominatim_is_close = dist_nominatim < CLOSE_KM
+        geonames_is_close = dist_geonames < CLOSE_KM
 
-        # If provinces derived from Nominatim and GeoNames postcodes differ
-        nominatim_postcode_province_matches = False
-        if norm_province_from_nominatim_postcode and norm_raw_nominatim_province:
-            nominatim_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_nominatim_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
+        # --- Validate Nominatim postcode ---
+        nominatim_pc_valid = False
+        if norm_province_from_nominatim_pc and norm_raw_province:
+            nominatim_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_nominatim_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
             )
 
-        geonames_postcode_province_matches = False
-        if norm_province_from_geonames_postcode and norm_raw_nominatim_province:
-            geonames_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_geonames_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
-            )
+        # --- Validate GeoNames postcode ---
+        geonames_pc_valid = False
 
-        # Prefer GeoNames postcode if its province matches Nominatim's raw address province,
-        # and Nominatim's own postcode-derived province does not.
-        if nominatim_postcode_province_matches:
-            selected_postcode = nominatim_postcode
-            selected_province_from_postcode = province_from_nominatim_postcode
-        if geonames_postcode_province_matches and not nominatim_postcode_province_matches:
-            selected_postcode = geonames_postcode
-            selected_province_from_postcode = province_from_geonames_postcode
+        # 1) Compare with raw province string (if exists)
+        if norm_province_from_geonames_pc and norm_raw_province:
+            geonames_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_geonames_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
+            )
 
-        return selected_postcode, selected_province_from_postcode
+        # 2) If no raw province, compare with province from Nominatim PC **only when** Nominatim is close
+        if not geonames_pc_valid and not norm_raw_province and nominatim_is_close:
+            if norm_province_from_geonames_pc and norm_province_from_nominatim_pc:
+                geonames_pc_valid = (
+                    jaro_winkler_similarity(norm_province_from_geonames_pc, norm_province_from_nominatim_pc)
+                    > JARO_WINKLER_THRESHOLD
+                )
+
+        # 3) Fallback: accept GeoNames PC if its coordinates are very close
+        if not geonames_pc_valid and geonames_is_close and geonames_postcode:
+            geonames_pc_valid = True
+
+        # --- Select authoritative tuple ---
+        postcode = None
+        province = None
+        state = None
+
+        if nominatim_pc_valid:
+            postcode = nominatim_postcode
+            province = province_from_nominatim_pc
+            state = parsed_nominatim_result.get("state")
+            if not state and geonames_pc_valid:
+                state = parsed_geonames_result.get("state")
+        elif geonames_pc_valid:
+            postcode = geonames_postcode
+            province = province_from_geonames_pc
+            state = parsed_geonames_result.get("state")
+            if not state and nominatim_pc_valid:
+                state = parsed_nominatim_result.get("state")
+
+        return postcode, province, state
 
     def _select_final_result(
         self,
@@ -203,21 +235,41 @@ class Nominatim:
         dist_geonames: float,
         authoritative_postcode: Optional[str],
         authoritative_province_from_postcode: Optional[str],
-        nominatim_province: Optional[str],
+        authoritative_state: Optional[str],
     ) -> Dict[str, Optional[str]]:
         """
-        Selects the final address result based on distances and applies the authoritative postcode/province.
+        Choose the address block (Nominatim vs GeoNames) based on distance,
+        then apply the authoritative postcode/province.
+
+        Rules:
+        • Pick the source with the smaller finite distance.
+        • Always overwrite 'postcode' if authoritative_postcode is present.
+        • Overwrite 'province' only when authoritative_province_from_postcode is not None.
+        • If both distances are ∞, return an empty address.
         """
+
+        # ------------------------------------------------------------------ #
+        # 1. Decide the base address block                                  #
+        # ------------------------------------------------------------------ #
         if dist_nominatim <= dist_geonames and dist_nominatim != float("inf"):
             final_result = parsed_nominatim_result
-            final_result["postcode"] = authoritative_postcode
-            final_result["province"] = nominatim_province
         elif dist_geonames < dist_nominatim and dist_geonames != float("inf"):
             final_result = parsed_geonames_result
+        else:
+            return self._get_empty_address_result()
+
+        # ------------------------------------------------------------------ #
+        # 2. Apply authoritative postcode / province                        #
+        # ------------------------------------------------------------------ #
+        if authoritative_postcode:
             final_result["postcode"] = authoritative_postcode
+
+        if authoritative_province_from_postcode:
             final_result["province"] = authoritative_province_from_postcode
-        else:
-            final_result = self._get_empty_address_result()
+
+        if authoritative_province_from_postcode:
+            final_result["state"] = authoritative_state
+
         return final_result
 
     def _get_district_quarter(self, raw_json: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
@@ -250,12 +302,6 @@ class Nominatim:
         parsed_nominatim_result = self._parse_nominatim_result(nominatim_response)
         parsed_geonames_result = self._parse_geonames_result(geonames_response)
 
-        # Determine authoritative postcode
-        nominatim_province = parsed_nominatim_result.get("province")
-        selected_postcode, selected_province_from_postcode = self._select_postcode_and_derived_province(
-            parsed_nominatim_result, parsed_geonames_result, nominatim_province
-        )
-
         # Calculate distances
         nominatim_response_lat = nominatim_response.get("lat")
         nominatim_response_lon = nominatim_response.get("lon")
@@ -272,6 +318,12 @@ class Nominatim:
         dist_nominatim = self._calculate_distance(nominatim_response_lat, nominatim_response_lon, input_coords)
         dist_geonames = self._calculate_distance(geonames_response_lat, geonames_response_lon, input_coords)
 
+        # Determine authoritative postcode
+        nominatim_province = parsed_nominatim_result.get("province")
+        selected_postcode, selected_province_from_postcode, selected_state = self._select_postcode_and_derived_province(
+            parsed_nominatim_result, parsed_geonames_result, nominatim_province, dist_nominatim, dist_geonames
+        )
+
         # Select final result
         final_result = self._select_final_result(
             parsed_nominatim_result,
@@ -280,7 +332,7 @@ class Nominatim:
             dist_geonames,
             selected_postcode,
             selected_province_from_postcode,
-            nominatim_province,
+            selected_state,
         )
 
         return final_result

datamarket-0.9.41/src/datamarket/utils/__init__.py

@@ -0,0 +1 @@
+from .main import *  # noqa: F403

{datamarket-0.9.39 → datamarket-0.9.41}/src/datamarket/utils/main.py

@@ -13,8 +13,26 @@ from pathlib import Path
 from typing import Any, Literal, Self, Union
 
 import pendulum
+import requests
+from bs4 import BeautifulSoup
 from croniter import croniter
 from dynaconf import Dynaconf, add_converter
+from requests.exceptions import ProxyError
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    retry_if_not_exception_type,
+    stop_after_attempt,
+    stop_after_delay,
+    wait_exponential,
+)
+
+from ..exceptions import NotFoundError, RedirectionDetectedError
+from ..interfaces.proxy import ProxyInterface
+
+########################################################################################################################
+# FUNCTIONS
 
 logger = logging.getLogger(__name__)
 
@@ -26,9 +44,7 @@ class NoProjectFoundError(Exception):
 
 class NoPackageFoundError(Exception):
     def __init__(self):
-        super().__init__(
-            "A project was detected but it has no packages inside the 'src' directory"
-        )
+        super().__init__("A project was detected but it has no packages inside the 'src' directory")
 
 
 ########################################################################################################################
@@ -84,8 +100,8 @@ class Project:
 
         try:
             self.pkg_name = next((self.path / "src").glob("*")).name
-        except StopIteration:
-            raise NoPackageFoundError()
+        except StopIteration as e:
+            raise NoPackageFoundError() from e
 
         self.env_name = f"{self.pkg_name}_env"
         self.config_path = self.path / self.CONFIG_FILE_NAME
@@ -227,3 +243,61 @@ def parse_field(dict_struct, field_path, format_method=None):
         if field_value is None:
             return None
     return format_method(field_value) if format_method else field_value
+
+
+@retry(
+    retry=retry_if_not_exception_type((NotFoundError, RedirectionDetectedError, ProxyError)),
+    wait=wait_exponential(exp_base=3, multiplier=3, max=60),
+    stop=stop_after_attempt(5),
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+    reraise=True,
+)
+def get_data(
+    url: str,
+    method: str = "GET",
+    output: str = "json",
+    sleep: tuple = (6, 3),
+    proxy_interface: ProxyInterface = None,
+    use_auth_proxies: bool = False,
+    max_proxy_delay: int = 1800,
+    **kwargs,
+):
+    retry_type = retry_if_exception_type(ProxyError)
+    wait = wait_exponential(exp_base=3, multiplier=3, max=60)
+    stop = stop_after_delay(max_proxy_delay)
+    before_sleep = before_sleep_log(logger, logging.WARNING)
+
+    @retry(retry=retry_type, wait=wait, stop=stop, before_sleep=before_sleep, reraise=True)
+    def _fetch_with_proxy_retry(url, method, proxy_interface, use_auth, **params):
+        logger.info(f"Fetching data from {url} ...")
+        proxy_cfg = None
+        if proxy_interface:
+            host, port, user, pwd = proxy_interface.get_proxies(raw=True, use_auth=use_auth)
+            if host and port:
+                proxy_url = f"http://{host}:{port}"
+                proxy_auth_url = f"http://{user}:{pwd}@{host}:{port}"
+                proxy_cfg = {"http": proxy_url, "https": proxy_url}
+                if user and pwd:
+                    proxy_cfg = {"http": proxy_auth_url, "https": proxy_auth_url}
+                logger.info(f"Using proxy: {proxy_url}")
+        response = getattr(requests, method.lower())(url, proxies=proxy_cfg, **params)
+        return response
+
+    params = {"timeout": 30} | kwargs
+    r = _fetch_with_proxy_retry(url, method, proxy_interface, use_auth_proxies, **params)
+
+    ban_sleep(*sleep)
+
+    if r.status_code == 404:
+        raise NotFoundError(f"404 Not Found error for {url}")
+    r.raise_for_status()
+    r.encoding = "utf-8"
+
+    if output == "json":
+        return r.json()
+    elif output == "text":
+        return r.text
+    elif output == "soup":
+        return BeautifulSoup(r.content, "html.parser")
+    elif output == "response":
+        return r

datamarket-0.9.39/src/datamarket/utils/__init__.py

@@ -1 +0,0 @@
-from .main import *