datamarket 0.9.38__py3-none-any.whl → 0.9.40__py3-none-any.whl

datamarket/interfaces/nominatim.py

@@ -17,6 +17,7 @@ from ..utils.strings import normalize
 # PARAMETERS
 
 JARO_WINKLER_THRESHOLD = 0.85
+CLOSE_KM = 2.0
 
 ########################################################################################################################
 # CLASSES
@@ -145,55 +146,86 @@ class Nominatim:
         parsed_nominatim_result: Dict[str, Optional[str]],
         parsed_geonames_result: Dict[str, Optional[str]],
         nominatim_address_province_raw: Optional[str],
-    ) -> Tuple[Optional[str], Optional[str]]:
+        dist_nominatim: float,  # distance Nominatim ↔ input (km)
+        dist_geonames: float,  # distance GeoNames ↔ input (km)
+    ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
         """
-        Determines the postcode and its derived province based on comparisons
-        between Nominatim and GeoNames data, and Nominatim's raw address province.
+        Decide the authoritative postcode, the province derived from it and the associated state.
+
+        Strategy:
+        1. Derive province from each postcode.
+        2. Validate each postcode–province pair:
+        • Nominatim: compare with raw province string (if present).
+        • GeoNames: multi-step validation (raw province, then Nominatim-derived
+            province when Nominatim coords are close, then distance fallback).
+        3. Return the postcode/province that passes validation with precedence:
+        Nominatim > GeoNames. Returns (None, None, None) if neither passes.
         """
+
+        # --- Extract postcodes ---
         nominatim_postcode = parsed_nominatim_result.get("postcode")
         geonames_postcode = parsed_geonames_result.get("postcode")
 
-        province_from_nominatim_postcode = self.geonames.get_province_from_postcode(nominatim_postcode)
-        province_from_geonames_postcode = self.geonames.get_province_from_postcode(geonames_postcode)
+        # --- Province derived from each postcode ---
+        province_from_nominatim_pc = self.geonames.get_province_from_postcode(nominatim_postcode)
+        province_from_geonames_pc = self.geonames.get_province_from_postcode(geonames_postcode)
 
-        norm_raw_nominatim_province = (
-            normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
-        )
-        norm_province_from_nominatim_postcode = (
-            normalize(province_from_nominatim_postcode) if province_from_nominatim_postcode else ""
-        )
-        norm_province_from_geonames_postcode = (
-            normalize(province_from_geonames_postcode) if province_from_geonames_postcode else ""
-        )
+        # --- Normalised strings for similarity comparisons ---
+        norm_raw_province = normalize(nominatim_address_province_raw) if nominatim_address_province_raw else ""
+        norm_province_from_nominatim_pc = normalize(province_from_nominatim_pc) if province_from_nominatim_pc else ""
+        norm_province_from_geonames_pc = normalize(province_from_geonames_pc) if province_from_geonames_pc else ""
 
-        selected_postcode = None
-        selected_province_from_postcode = None
+        # --- Distance heuristics ---
+        nominatim_is_close = dist_nominatim < CLOSE_KM
+        geonames_is_close = dist_geonames < CLOSE_KM
 
-        # If provinces derived from Nominatim and GeoNames postcodes differ
-        nominatim_postcode_province_matches = False
-        if norm_province_from_nominatim_postcode and norm_raw_nominatim_province:
-            nominatim_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_nominatim_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
+        # --- Validate Nominatim postcode ---
+        nominatim_pc_valid = False
+        if norm_province_from_nominatim_pc and norm_raw_province:
+            nominatim_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_nominatim_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
             )
 
-        geonames_postcode_province_matches = False
-        if norm_province_from_geonames_postcode and norm_raw_nominatim_province:
-            geonames_postcode_province_matches = (
-                jaro_winkler_similarity(norm_province_from_geonames_postcode, norm_raw_nominatim_province)
-                > JARO_WINKLER_THRESHOLD
-            )
+        # --- Validate GeoNames postcode ---
+        geonames_pc_valid = False
 
-        # Prefer GeoNames postcode if its province matches Nominatim's raw address province,
-        # and Nominatim's own postcode-derived province does not.
-        if nominatim_postcode_province_matches:
-            selected_postcode = nominatim_postcode
-            selected_province_from_postcode = province_from_nominatim_postcode
-        if geonames_postcode_province_matches and not nominatim_postcode_province_matches:
-            selected_postcode = geonames_postcode
-            selected_province_from_postcode = province_from_geonames_postcode
+        # 1) Compare with raw province string (if exists)
+        if norm_province_from_geonames_pc and norm_raw_province:
+            geonames_pc_valid = (
+                jaro_winkler_similarity(norm_province_from_geonames_pc, norm_raw_province) > JARO_WINKLER_THRESHOLD
+            )
 
-        return selected_postcode, selected_province_from_postcode
+        # 2) If no raw province, compare with province from Nominatim PC **only when** Nominatim is close
+        if not geonames_pc_valid and not norm_raw_province and nominatim_is_close:
+            if norm_province_from_geonames_pc and norm_province_from_nominatim_pc:
+                geonames_pc_valid = (
+                    jaro_winkler_similarity(norm_province_from_geonames_pc, norm_province_from_nominatim_pc)
+                    > JARO_WINKLER_THRESHOLD
+                )
+
+        # 3) Fallback: accept GeoNames PC if its coordinates are very close
+        if not geonames_pc_valid and geonames_is_close and geonames_postcode:
+            geonames_pc_valid = True
+
+        # --- Select authoritative tuple ---
+        postcode = None
+        province = None
+        state = None
+
+        if nominatim_pc_valid:
+            postcode = nominatim_postcode
+            province = province_from_nominatim_pc
+            state = parsed_nominatim_result.get("state")
+            if not state and geonames_pc_valid:
+                state = parsed_geonames_result.get("state")
+        elif geonames_pc_valid:
+            postcode = geonames_postcode
+            province = province_from_geonames_pc
+            state = parsed_geonames_result.get("state")
+            if not state and nominatim_pc_valid:
+                state = parsed_nominatim_result.get("state")
+
+        return postcode, province, state
 
     def _select_final_result(
         self,
@@ -203,21 +235,41 @@ class Nominatim:
         dist_geonames: float,
         authoritative_postcode: Optional[str],
         authoritative_province_from_postcode: Optional[str],
-        nominatim_province: Optional[str],
+        authoritative_state: Optional[str],
     ) -> Dict[str, Optional[str]]:
         """
-        Selects the final address result based on distances and applies the authoritative postcode/province.
+        Choose the address block (Nominatim vs GeoNames) based on distance,
+        then apply the authoritative postcode/province.
+
+        Rules:
+        • Pick the source with the smaller finite distance.
+        • Always overwrite 'postcode' if authoritative_postcode is present.
+        • Overwrite 'province' only when authoritative_province_from_postcode is not None.
+        • If both distances are ∞, return an empty address.
         """
+
+        # ------------------------------------------------------------------ #
+        # 1. Decide the base address block                                  #
+        # ------------------------------------------------------------------ #
         if dist_nominatim <= dist_geonames and dist_nominatim != float("inf"):
             final_result = parsed_nominatim_result
-            final_result["postcode"] = authoritative_postcode
-            final_result["province"] = nominatim_province
         elif dist_geonames < dist_nominatim and dist_geonames != float("inf"):
             final_result = parsed_geonames_result
+        else:
+            return self._get_empty_address_result()
+
+        # ------------------------------------------------------------------ #
+        # 2. Apply authoritative postcode / province                        #
+        # ------------------------------------------------------------------ #
+        if authoritative_postcode:
             final_result["postcode"] = authoritative_postcode
+
+        if authoritative_province_from_postcode:
             final_result["province"] = authoritative_province_from_postcode
-        else:
-            final_result = self._get_empty_address_result()
+
+        if authoritative_province_from_postcode:
+            final_result["state"] = authoritative_state
+
         return final_result
 
     def _get_district_quarter(self, raw_json: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
@@ -250,12 +302,6 @@ class Nominatim:
         parsed_nominatim_result = self._parse_nominatim_result(nominatim_response)
         parsed_geonames_result = self._parse_geonames_result(geonames_response)
 
-        # Determine authoritative postcode
-        nominatim_province = parsed_nominatim_result.get("province")
-        selected_postcode, selected_province_from_postcode = self._select_postcode_and_derived_province(
-            parsed_nominatim_result, parsed_geonames_result, nominatim_province
-        )
-
         # Calculate distances
         nominatim_response_lat = nominatim_response.get("lat")
         nominatim_response_lon = nominatim_response.get("lon")
@@ -272,6 +318,12 @@ class Nominatim:
         dist_nominatim = self._calculate_distance(nominatim_response_lat, nominatim_response_lon, input_coords)
         dist_geonames = self._calculate_distance(geonames_response_lat, geonames_response_lon, input_coords)
 
+        # Determine authoritative postcode
+        nominatim_province = parsed_nominatim_result.get("province")
+        selected_postcode, selected_province_from_postcode, selected_state = self._select_postcode_and_derived_province(
+            parsed_nominatim_result, parsed_geonames_result, nominatim_province, dist_nominatim, dist_geonames
+        )
+
         # Select final result
         final_result = self._select_final_result(
             parsed_nominatim_result,
@@ -280,7 +332,7 @@ class Nominatim:
             dist_geonames,
             selected_postcode,
             selected_province_from_postcode,
-            nominatim_province,
+            selected_state,
         )
 
         return final_result

datamarket/utils/playwright/async_api.py

@@ -0,0 +1,23 @@
+########################################################################################################################
+# IMPORTS
+
+import asyncio
+from random import randint
+
+from playwright.async_api import Page
+
+
+########################################################################################################################
+# FUNCTIONS
+
+
+async def human_type(page: Page, text: str, delay: int = 100):
+    for char in text:
+        await page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+
+
+async def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, sleep=True):
+    for _ in range(count):
+        await page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+        if sleep:
+            await asyncio.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311

datamarket/utils/playwright/sync_api.py

@@ -0,0 +1,23 @@
+########################################################################################################################
+# IMPORTS
+
+import time
+from random import randint
+
+from playwright.sync_api import Page
+
+
+########################################################################################################################
+# FUNCTIONS
+
+
+def human_type(page: Page, text: str, delay: int = 100):
+    for char in text:
+        page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+
+
+def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, sleep=True):
+    for _ in range(count):
+        page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+        if sleep:
+            time.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311

{datamarket-0.9.38.dist-info → datamarket-0.9.40.dist-info}/METADATA

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.38
+Version: 0.9.40
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
-Home-page: https://datamarket.es
 License: GPL-3.0-or-later
 Author: DataMarket
 Author-email: techsupport@datamarket.es
@@ -97,7 +96,7 @@ Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
 Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"
 Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pillow (>=11.0.0,<12.0.0) ; extra == "pillow"
-Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
+Requires-Dist: playwright (==1.47.0) ; extra == "playwright" or extra == "camoufox"
 Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
 Requires-Dist: presidio-analyzer[phonenumbers] (>=2.0.0,<3.0.0) ; extra == "pii"
 Requires-Dist: presidio-anonymizer (>=2.0.0,<3.0.0) ; extra == "pii"
@@ -114,18 +113,18 @@ Requires-Dist: rapidfuzz (>=3.0.0,<4.0.0) ; extra == "rapidfuzz"
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: retry (>=0.9.0,<0.10.0) ; extra == "retry"
 Requires-Dist: shapely (>=2.0.0,<3.0.0) ; extra == "shapely"
-Requires-Dist: soda-core-mysql (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
+Requires-Dist: soda-core-mysql-utf8-hotfix (>=3.0.0,<4.0.0) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (>=3.0.0,<4.0.0) ; extra == "soda-core-postgres"
 Requires-Dist: spacy (>=3.0.0,<4.0.0) ; extra == "pii"
 Requires-Dist: spacy-langdetect (>=0.1.0,<0.2.0) ; extra == "pii"
 Requires-Dist: stem (>=1.0.0,<2.0.0) ; extra == "stem" or extra == "proxy"
 Requires-Dist: tenacity (>=9.0.0,<10.0.0)
-Requires-Dist: tf-playwright-stealth (>=1.0.0,<2.0.0)
 Requires-Dist: tqdm (>=4.0.0,<5.0.0) ; extra == "tqdm"
 Requires-Dist: typer (>=0.15.0,<0.16.0)
 Requires-Dist: unidecode (>=1.0.0,<2.0.0)
 Requires-Dist: xmltodict (>=0.14.0,<0.15.0) ; extra == "xmltodict"
 Project-URL: Documentation, https://github.com/Data-Market/datamarket
+Project-URL: Homepage, https://datamarket.es
 Project-URL: Repository, https://github.com/Data-Market/datamarket
 Description-Content-Type: text/markdown
 

{datamarket-0.9.38.dist-info → datamarket-0.9.40.dist-info}/RECORD

@@ -4,7 +4,7 @@ datamarket/interfaces/alchemy.py,sha256=mQwjDqBpz1QHRV2JTCALvn5iK_ky69oE2Gw-EtRX
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
-datamarket/interfaces/nominatim.py,sha256=XzqBFVBzGU2BIFnFueZ56tk4JhaQgj5dvFalnCG6Zxk,12417
+datamarket/interfaces/nominatim.py,sha256=d_KIrgzTusVYnw0Fk3YWCjrzlT9sI_bObGG-wOr__as,14726
 datamarket/interfaces/peerdb.py,sha256=cwYwvO740GyaPo9zLAwJsf3UeJDGDiYzjQVM9Q6s-_g,23652
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
@@ -14,6 +14,9 @@ datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
 datamarket/utils/main.py,sha256=j8wnAxeLvijdRU9M4V6HunWH7vgWWHP4u4xamzkWcUU,7009
+datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamarket/utils/playwright/async_api.py,sha256=pWfVj-ItfIeZBxG7WiUHKSeZDcHQFUQ5mrNGyIh1IdA,883
+datamarket/utils/playwright/sync_api.py,sha256=lIGm8mLxhFg04LVNdF8SO_9yjOLBnWe5pPry6ZFMnIg,846
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/strings/__init__.py,sha256=b6TYOT9v7y9ID-lDyZk4E8BH2uIPbsF2ZSLGjCQ1MCQ,43
@@ -21,7 +24,7 @@ datamarket/utils/strings/normalization.py,sha256=QLZ-THzjGOK9eWPPR1PrsffwQkSOx_M
 datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.38.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.38.dist-info/METADATA,sha256=rY2O90LCgMiAmWuussQ8xylApqA6jj1hhSB9FGT0Jpk,6968
-datamarket-0.9.38.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-datamarket-0.9.38.dist-info/RECORD,,
+datamarket-0.9.40.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.40.dist-info/METADATA,sha256=odG5B_7jOuXZFBNe260K0HP7vkDntMf8yAUOgSxNOws,6961
+datamarket-0.9.40.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.40.dist-info/RECORD,,

{datamarket-0.9.38.dist-info → datamarket-0.9.40.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any