datamarket 0.8.5__tar.gz → 0.8.7__tar.gz

{datamarket-0.8.5 → datamarket-0.8.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.8.5
+Version: 0.8.7
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -55,6 +55,7 @@ Provides-Extra: pytest
 Provides-Extra: rapidfuzz
 Provides-Extra: retry
 Provides-Extra: shapely
+Provides-Extra: soda-core-mysql
 Provides-Extra: soda-core-postgres
 Provides-Extra: stem
 Provides-Extra: tqdm
@@ -86,7 +87,7 @@ Requires-Dist: httpx[http2] (==0.28.1) ; extra == "httpx"
 Requires-Dist: jinja2 (==3.1.5)
 Requires-Dist: json5 (==0.9.25) ; extra == "json5"
 Requires-Dist: lxml[html-clean] (==5.3.0) ; extra == "lxml"
-Requires-Dist: nodriver (==0.37) ; extra == "nodriver"
+Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
 Requires-Dist: openpyxl (==3.1.5) ; extra == "openpyxl"
 Requires-Dist: pandas (==2.2.3) ; extra == "pandas"
 Requires-Dist: pandera (==0.20.4) ; extra == "pandera"
@@ -106,6 +107,7 @@ Requires-Dist: rapidfuzz (==3.10.1) ; extra == "rapidfuzz"
 Requires-Dist: requests (==2.32.3)
 Requires-Dist: retry (==0.9.2) ; extra == "retry"
 Requires-Dist: shapely (==2.0.6) ; extra == "shapely"
+Requires-Dist: soda-core-mysql (==3.4.4) ; extra == "soda-core-mysql"
 Requires-Dist: soda-core-postgres (==3.4.1) ; extra == "soda-core-postgres"
 Requires-Dist: stem (==1.8.2) ; extra == "stem" or extra == "proxy"
 Requires-Dist: tenacity (==9.0.0)

{datamarket-0.8.5 → datamarket-0.8.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.8.5"
+version = "0.8.7"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -38,6 +38,7 @@ pytest = { version = "8.3.3", optional = true }
 playwright = { version = "1.47.0", optional = true }
 playwright-stealth = { version = "1.0.6", optional = true }
 soda-core-postgres = { version = "3.4.1", optional = true }
+soda-core-mysql = { version = "3.4.4", optional = true }
 fake-useragent = { version = "1.5.1", optional = true }
 pydrive2 = { version = "1.20.0", optional = true }
 clickhouse-driver = { version = "0.2.9", optional = true }
@@ -46,7 +47,7 @@ click = { version = "8.1.7", optional = true }
 rapidfuzz = { version = "3.10.1", optional = true }
 demjson3 = { version = "3.0.6", optional = true }
 geopy = { version = "2.4.1", optional = true }
-nodriver = { version = "0.37", optional = true }
+nodriver = { version = "0.38.post1", optional = true }
 undetected-chromedriver = { version = "3.5.5", optional = true }
 retry = { version = "0.9.2", optional = true }
 shapely = { version = "2.0.6", optional = true }
@@ -82,6 +83,7 @@ pytest = ["pytest"]
 playwright = ["playwright"]
 playwright-stealth = ["playwright-stealth"]
 soda-core-postgres = ["soda-core-postgres"]
+soda-core-mysql = ["soda-core-mysql"]
 fake-useragent = ["fake-useragent"]
 pydrive2 = ["pydrive2"]
 clickhouse-driver = ["clickhouse-driver"]

{datamarket-0.8.5 → datamarket-0.8.7}/src/datamarket/interfaces/peerdb.py

@@ -216,52 +216,26 @@ class TransientS3:
             self.config = config["peerdb-s3"]
             self.bucket_name = self.config["bucket"]
             self.session = boto3.Session(profile_name=self.config["profile"])
-            self.s3_client = self.session.client("s3")
+            self.s3_resource = self.session.resource("s3")
             self.credentials = self.session.get_credentials()
             self.access_key = self.credentials.access_key
             self.secret_key = self.credentials.secret_key
             self.region_name = self.session.region_name
-            self.endpoint_url = self.s3_client.meta.endpoint_url
+            self.endpoint_url = self.s3_resource.meta.endpoint_url
         else:
             logger.warning("no peerdb-s3 section in config")
 
     def delete_paths_with_schema(self, schema_name):
         logger.info(f"Deleting paths containing '{schema_name}' from S3")
 
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter="/")
+        bucket = self.s3_resource.Bucket(self.bucket_name)
 
-        for page in pages:
-            if "CommonPrefixes" in page:
-                for prefix in page["CommonPrefixes"]:
-                    folder = prefix["Prefix"]
-                    if schema_name in folder:
-                        self._delete_folder_contents(folder)
+        for prefix in [schema_name, f"clone_{schema_name}"]:
+            objects_to_delete = bucket.objects.filter(Prefix=prefix)
+            objects_to_delete.delete()
 
         logger.info(f"Deleted paths containing '{schema_name}' from S3")
 
-    def _delete_folder_contents(self, folder):
-        logger.info(f"Deleting contents of folder: {folder}")
-
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=folder)
-
-        delete_us = dict(Objects=[])
-        for page in pages:
-            if "Contents" in page:
-                for obj in page["Contents"]:
-                    delete_us["Objects"].append(dict(Key=obj["Key"]))
-
-                    # AWS limits to deleting 1000 objects at a time
-                    if len(delete_us["Objects"]) >= 1000:
-                        self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-                        delete_us = dict(Objects=[])
-
-        if len(delete_us["Objects"]):
-            self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-
-        logger.info(f"Deleted contents of folder: {folder}")
-
 
 class PeerDBInterface:
     def __init__(self, config):
@@ -308,11 +282,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ['localhost', '127.0.0.1']:
+        if host in ["localhost", "127.0.0.1"]:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r'(localhost|127\.0\.0\.1)'
+        url_pattern = r"(localhost|127\.0\.0\.1)"
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)

{datamarket-0.8.5 → datamarket-0.8.7}/src/datamarket/utils/main.py

@@ -1,6 +1,7 @@
 ########################################################################################################################
 # IMPORTS
 
+import asyncio
 import configparser
 import inspect
 import logging
@@ -124,6 +125,12 @@ def ban_sleep(max_time, min_time=0):
     time.sleep(sleep_time)
 
 
+async def ban_sleep_async(max_time, min_time=0):
+    sleep_time = int(random.uniform(min_time, max_time))  # noqa: S311
+    logger.info(f"sleeping for {sleep_time} seconds...")
+    await asyncio.sleep(sleep_time)
+
+
 def run_bash_command(command):
     p = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)