datamarket 0.9.38__tar.gz → 0.9.40__tar.gz

{datamarket-0.9.38 → datamarket-0.9.40}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.38
+Version: 0.9.40
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
-Home-page: https://datamarket.es
 License: GPL-3.0-or-later
 Author: DataMarket
 Author-email: techsupport@datamarket.es
@@ -97,7 +96,7 @@ Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
 Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"
 Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
-Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
+Requires-Dist: playwright (==1.47.0) ; extra == "playwright" or extra == "camoufox"
 Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
 Requires-Dist: presidio-analyzer[phonenumbers] (>=2.0.0,<3.0.0) ; extra == "pii"
 Requires-Dist: presidio-anonymizer (>=2.0.0,<3.0.0) ; extra == "pii"
@@ -114,18 +113,18 @@ Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0) ; extra == "rapidfuzz"
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
 Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
-Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
+Requires-Dist: soda-core-mysql-utf8-hotfix (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
 Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
 Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
 Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
 Requires-Dist: tenacity (>=9.0.0,<10.0.0)
-Requires-Dist: tf-playwright-stealth (>=1.0.0,<2.0.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
 Requires-Dist: typer (>=0.15.0,<0.16.0)
 Requires-Dist: unidecode (>=1.0.0,<2.0.0)
 Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
 Project-URL: Documentation, https://github.com/Data-Market/datamarket
+Project-URL: Homepage, https://datamarket.es
 Project-URL: Repository, https://github.com/Data-Market/datamarket
 Description-Content-Type: text/markdown
 

{datamarket-0.9.38 → datamarket-0.9.40}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.38"
+version = "0.9.40"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -43,7 +43,7 @@ pytest = { version = "^8.0.0", optional = true }
 playwright = { version = "1.47.0", optional = true }
 tf-playwright-stealth = { version = "^1.0.0", optional = true }
 soda-core-postgres = { version = "^3.0.0", optional = true }
-soda-core-mysql = { version = "^3.0.0", optional = true }
+soda-core-mysql-utf8-hotfix = { version = "^3.0.0", optional = true }
 fake-useragent = { version = "^2.0.0", optional = true }
 pydrive2 = { version = "^1.0.0", optional = true }
 clickhouse-driver = { version = "~0.2.0", optional = true }
@@ -94,7 +94,7 @@ pytest = ["pytest"]
 playwright = ["playwright"]
 playwright-stealth = ["playwright-stealth"]
 soda-core-postgres = ["soda-core-postgres"]
-soda-core-mysql = ["soda-core-mysql"]
+soda-core-mysql = ["soda-core-mysql-utf8-hotfix"]
 fake-useragent = ["fake-useragent"]
 pydrive2 = ["pydrive2"]
 clickhouse-driver = ["clickhouse-driver"]
@@ -126,7 +126,7 @@ google-auth-oauthlib = ["google-auth-oauthlib"]
 dnspython = ["dnspython"]
 openpyxl = ["openpyxl"]
 httpx = ["httpx"]
-camoufox = ["camoufox"]
+camoufox = ["camoufox", "browserforge", "playwright"]
 pandarallel = ["pandarallel"]
 
 # Interface groups

{datamarket-0.9.38 → datamarket-0.9.40}/src/datamarket/interfaces/nominatim.py

@@ -17,6 +17,7 @@ from ..utils.strings import normalize
 # PARAMETERS
 
 JARO_WINKLER_THRESHOLD = 0.85
+CLOSE_KM = 2.0
 
 ########################################################################################################################
 # CLASSES
@@ -145,55 +146,86 @@ class Nominatim:
         parsed_nominatim_result: Dict[str, Optional[str]],
         parsed_geonames_result: Dict[str, Optional[str]],
         nominatim_address_province_raw: Optional[str],
-    ) -> Tuple[Optional[str], Optional[str]]:
+        dist_nominatim: float,  # distance Nominatim ↔ input (km)
+        dist_geonames: float,  # distance GeoNames ↔ input (km)
+    ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
         """
-        Determines the postcode and its derived province based on comparisons
-        between Nominatim and GeoNames data, and Nominatim's raw address province.
+        Decide the authoritative postcode, the province derived from it and the associated state.
+
+        Strategy:
+        1. Derive province from each postcode.
+        2. Validate each postcode–province pair:
+        • Nominatim: compare with raw province string (if present).
+        • GeoNames: multi-step validation (raw province, then Nominatim-derived
+            province when Nominatim coords are close, then distance fallback).
+        3. Return the postcode/province that passes validation with precedence:
+        Nominatim > GeoNames. Returns (None, None, None) if neither passes.
         """
+
+        # --- Extract postcodes ---
         nominatim_postcode = parsed_nominatim_result.get("postcode")
         geonames_postcode = parsed_geonames_result.get("postcode")
 
-        province_from_nominatim_postcode = self.geonames.get_province_from_postcode(nominatim_postcode)
-        province_from_geonames_postcode = self.geonames.get_province_from_postcode(geonames_postcode)
+        # --- Province derived from each postcode ---
+        province_from_nominatim_pc = self.geonames.get_province_from_postcode(nominatim_postcode)
+        province_from_geonames_pc = self.geonames.get_province_from_postcode(geonames_postcode)
 
-        norm_raw_nominatim_province = (
-            normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
-        )
-        norm_province_from_nominatim_postcode = (
-            normalize(province_from_nominatim_postcode) if province_from_nominatim_postcode else ""
-        )
-        norm_province_from_geonames_postcode = (
-            normalize(province_from_geonames_postcode) if province_from_geonames_postcode else ""
-        )
+        # --- Normalised strings for similarity comparisons ---
+        norm_raw_province = normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
+        norm_province_from_nominatim_pc = normalize(province_from_nominatim_pc) if province_from_nominatim_pc else ""
+        norm_province_from_geonames_pc = normalize(province_from_geonames_pc) if province_from_geonames_pc else ""
 
-        selected_postcode = None
-        selected_province_from_postcode = None
+        # --- Distance heuristics ---
+        nominatim_is_close = dist_nominatim < CLOSE_KM
+        geonames_is_close = dist_geonames < CLOSE_KM
 
-        # If provinces derived from Nominatim and GeoNames postcodes differ
-        nominatim_postcode_province_matches = False
-        if norm_province_from_nominatim_postcode and norm_raw_nominatim_province:
-            nominatim_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_nominatim_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
+        # --- Validate Nominatim postcode ---
+        nominatim_pc_valid = False
+        if norm_province_from_nominatim_pc and norm_raw_province:
+            nominatim_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_nominatim_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
             )
 
-        geonames_postcode_province_matches = False
-        if norm_province_from_geonames_postcode and norm_raw_nominatim_province:
-            geonames_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_geonames_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
-            )
+        # --- Validate GeoNames postcode ---
+        geonames_pc_valid = False
 
-        # Prefer GeoNames postcode if its province matches Nominatim's raw address province,
-        # and Nominatim's own postcode-derived province does not.
-        if nominatim_postcode_province_matches:
-            selected_postcode = nominatim_postcode
-            selected_province_from_postcode = province_from_nominatim_postcode
-        if geonames_postcode_province_matches and not nominatim_postcode_province_matches:
-            selected_postcode = geonames_postcode
-            selected_province_from_postcode = province_from_geonames_postcode
+        # 1) Compare with raw province string (if exists)
+        if norm_province_from_geonames_pc and norm_raw_province:
+            geonames_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_geonames_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
+            )
 
-        return selected_postcode, selected_province_from_postcode
+        # 2) If no raw province, compare with province from Nominatim PC **only when** Nominatim is close
+        if not geonames_pc_valid and not norm_raw_province and nominatim_is_close:
+            if norm_province_from_geonames_pc and norm_province_from_nominatim_pc:
+                geonames_pc_valid = (
+                    jaro_winkler_similarity(norm_province_from_geonames_pc, norm_province_from_nominatim_pc)
+                    > JARO_WINKLER_THRESHOLD
+                )
+
+        # 3) Fallback: accept GeoNames PC if its coordinates are very close
+        if not geonames_pc_valid and geonames_is_close and geonames_postcode:
+            geonames_pc_valid = True
+
+        # --- Select authoritative tuple ---
+        postcode = None
+        province = None
+        state = None
+
+        if nominatim_pc_valid:
+            postcode = nominatim_postcode
+            province = province_from_nominatim_pc
+            state = parsed_nominatim_result.get("state")
+            if not state and geonames_pc_valid:
+                state = parsed_geonames_result.get("state")
+        elif geonames_pc_valid:
+            postcode = geonames_postcode
+            province = province_from_geonames_pc
+            state = parsed_geonames_result.get("state")
+            if not state and nominatim_pc_valid:
+                state = parsed_nominatim_result.get("state")
+
+        return postcode, province, state
 
     def _select_final_result(
         self,
@@ -203,21 +235,41 @@ class Nominatim:
         dist_geonames: float,
         authoritative_postcode: Optional[str],
         authoritative_province_from_postcode: Optional[str],
-        nominatim_province: Optional[str],
+        authoritative_state: Optional[str],
     ) -> Dict[str, Optional[str]]:
         """
-        Selects the final address result based on distances and applies the authoritative postcode/province.
+        Choose the address block (Nominatim vs GeoNames) based on distance,
+        then apply the authoritative postcode/province.
+
+        Rules:
+        • Pick the source with the smaller finite distance.
+        • Always overwrite 'postcode' if authoritative_postcode is present.
+        • Overwrite 'province' only when authoritative_province_from_postcode is not None.
+        • If both distances are ∞, return an empty address.
         """
+
+        # ------------------------------------------------------------------ #
+        # 1. Decide the base address block                                  #
+        # ------------------------------------------------------------------ #
         if dist_nominatim <= dist_geonames and dist_nominatim != float("inf"):
             final_result = parsed_nominatim_result
-            final_result["postcode"] = authoritative_postcode
-            final_result["province"] = nominatim_province
         elif dist_geonames < dist_nominatim and dist_geonames != float("inf"):
             final_result = parsed_geonames_result
+        else:
+            return self._get_empty_address_result()
+
+        # ------------------------------------------------------------------ #
+        # 2. Apply authoritative postcode / province                        #
+        # ------------------------------------------------------------------ #
+        if authoritative_postcode:
             final_result["postcode"] = authoritative_postcode
+
+        if authoritative_province_from_postcode:
             final_result["province"] = authoritative_province_from_postcode
-        else:
-            final_result = self._get_empty_address_result()
+
+        if authoritative_province_from_postcode:
+            final_result["state"] = authoritative_state
+
         return final_result
 
     def _get_district_quarter(self, raw_json: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
@@ -250,12 +302,6 @@ class Nominatim:
         parsed_nominatim_result = self._parse_nominatim_result(nominatim_response)
         parsed_geonames_result = self._parse_geonames_result(geonames_response)
 
-        # Determine authoritative postcode
-        nominatim_province = parsed_nominatim_result.get("province")
-        selected_postcode, selected_province_from_postcode = self._select_postcode_and_derived_province(
-            parsed_nominatim_result, parsed_geonames_result, nominatim_province
-        )
-
         # Calculate distances
         nominatim_response_lat = nominatim_response.get("lat")
         nominatim_response_lon = nominatim_response.get("lon")
@@ -272,6 +318,12 @@ class Nominatim:
         dist_nominatim = self._calculate_distance(nominatim_response_lat, nominatim_response_lon, input_coords)
         dist_geonames = self._calculate_distance(geonames_response_lat, geonames_response_lon, input_coords)
 
+        # Determine authoritative postcode
+        nominatim_province = parsed_nominatim_result.get("province")
+        selected_postcode, selected_province_from_postcode, selected_state = self._select_postcode_and_derived_province(
+            parsed_nominatim_result, parsed_geonames_result, nominatim_province, dist_nominatim, dist_geonames
+        )
+
         # Select final result
         final_result = self._select_final_result(
             parsed_nominatim_result,
@@ -280,7 +332,7 @@ class Nominatim:
             dist_geonames,
             selected_postcode,
             selected_province_from_postcode,
-            nominatim_province,
+            selected_state,
         )
 
         return final_result

datamarket-0.9.40/src/datamarket/utils/playwright/async_api.py

@@ -0,0 +1,23 @@
+########################################################################################################################
+# IMPORTS
+
+import asyncio
+from random import randint
+
+from playwright.async_api import Page
+
+
+########################################################################################################################
+# FUNCTIONS
+
+
+async def human_type(page: Page, text: str, delay: int = 100):
+    for char in text:
+        await page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+
+
+async def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, sleep=True):
+    for _ in range(count):
+        await page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+        if sleep:
+            await asyncio.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311

datamarket-0.9.40/src/datamarket/utils/playwright/sync_api.py

@@ -0,0 +1,23 @@
+########################################################################################################################
+# IMPORTS
+
+import time
+from random import randint
+
+from playwright.sync_api import Page
+
+
+########################################################################################################################
+# FUNCTIONS
+
+
+def human_type(page: Page, text: str, delay: int = 100):
+    for char in text:
+        page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+
+
+def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, sleep=True):
+    for _ in range(count):
+        page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+        if sleep:
+            time.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311