datamarket 0.8.6__py3-none-any.whl → 0.8.8__py3-none-any.whl

datamarket/interfaces/peerdb.py

@@ -216,52 +216,26 @@ class TransientS3:
             self.config = config["peerdb-s3"]
             self.bucket_name = self.config["bucket"]
             self.session = boto3.Session(profile_name=self.config["profile"])
-            self.s3_client = self.session.client("s3")
+            self.s3_resource = self.session.resource("s3")
             self.credentials = self.session.get_credentials()
             self.access_key = self.credentials.access_key
             self.secret_key = self.credentials.secret_key
             self.region_name = self.session.region_name
-            self.endpoint_url = self.s3_client.meta.endpoint_url
+            self.endpoint_url = self.s3_resource.meta.endpoint_url
         else:
             logger.warning("no peerdb-s3 section in config")
 
     def delete_paths_with_schema(self, schema_name):
         logger.info(f"Deleting paths containing '{schema_name}' from S3")
 
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter="/")
+        bucket = self.s3_resource.Bucket(self.bucket_name)
 
-        for page in pages:
-            if "CommonPrefixes" in page:
-                for prefix in page["CommonPrefixes"]:
-                    folder = prefix["Prefix"]
-                    if schema_name in folder:
-                        self._delete_folder_contents(folder)
+        for prefix in [schema_name, f"clone_{schema_name}"]:
+            objects_to_delete = bucket.objects.filter(Prefix=prefix)
+            objects_to_delete.delete()
 
         logger.info(f"Deleted paths containing '{schema_name}' from S3")
 
-    def _delete_folder_contents(self, folder):
-        logger.info(f"Deleting contents of folder: {folder}")
-
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=folder)
-
-        delete_us = dict(Objects=[])
-        for page in pages:
-            if "Contents" in page:
-                for obj in page["Contents"]:
-                    delete_us["Objects"].append(dict(Key=obj["Key"]))
-
-                    # AWS limits to deleting 1000 objects at a time
-                    if len(delete_us["Objects"]) >= 1000:
-                        self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-                        delete_us = dict(Objects=[])
-
-        if len(delete_us["Objects"]):
-            self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
-
-        logger.info(f"Deleted contents of folder: {folder}")
-
 
 class PeerDBInterface:
     def __init__(self, config):
@@ -308,11 +282,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ['localhost', '127.0.0.1']:
+        if host in ["localhost", "127.0.0.1"]:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r'(localhost|127\.0\.0\.1)'
+        url_pattern = r"(localhost|127\.0\.0\.1)"
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)

datamarket/utils/main.py

@@ -1,6 +1,7 @@
 ########################################################################################################################
 # IMPORTS
 
+import asyncio
 import configparser
 import inspect
 import logging
@@ -67,7 +68,8 @@ def get_config(config_file: Path, tz: str = "Europe/Madrid"):
     if Path(config_file).suffix == ".ini":
         logger.warning("Using legacy INI config reader. Please migrate to TOML")
         cfg = configparser.RawConfigParser()
-        return cfg.read(config_file)
+        cfg.read(config_file)
+        return cfg
 
     add_converter("read", read_converter)
 
@@ -124,6 +126,12 @@ def ban_sleep(max_time, min_time=0):
     time.sleep(sleep_time)
 
 
+async def ban_sleep_async(max_time, min_time=0):
+    sleep_time = int(random.uniform(min_time, max_time))  # noqa: S311
+    logger.info(f"sleeping for {sleep_time} seconds...")
+    await asyncio.sleep(sleep_time)
+
+
 def run_bash_command(command):
     p = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 

{datamarket-0.8.6.dist-info → datamarket-0.8.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.8.6
+Version: 0.8.8
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.8.6.dist-info → datamarket-0.8.8.dist-info}/RECORD

@@ -5,7 +5,7 @@ datamarket/interfaces/aws.py,sha256=UztVuBn561DnU1AcjyJ16UAIS1BUD5HUxiQ4gc9EhtM,
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=9GQgiNBBK7njkv8ytHQaP9YLB9kI5vnUFA5gtz9J7As,1859
 datamarket/interfaces/nominatim.py,sha256=_gFJ04D-ju5xn3wuaGT5Pj5jhf4F5eINpxOpuQL_dIQ,3664
-datamarket/interfaces/peerdb.py,sha256=kJ3hdzGEACTfA_HM1b_I2rExY0x6ijCq5eOP1WObpLc,21821
+datamarket/interfaces/peerdb.py,sha256=hGQ9TXKq9k2xEShz7n6iV-x66bhYylBaWoCc-I2VtN0,20705
 datamarket/interfaces/proxy.py,sha256=8EJaW8zAMzUMIRLkdAcMkTO9qZXPIubE6vyB5ZXcRtU,3352
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,11 +13,11 @@ datamarket/params/nominatim.py,sha256=pBYRfoBkkLBg2INbFymefmYSzaAVujQSpEro5c1hD_
 datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=fygFEz_lGUPrR39HOrcgE6aPX2ODPyTQNJOsgRQLN3g,5533
+datamarket/utils/main.py,sha256=agWVJ5ZFZjVrBNuMpnxN2F_edA3mMJop6dVHPBBkOqU,5775
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.8.6.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.8.6.dist-info/METADATA,sha256=FCi1xRdJYfiOJw6S8o9XlvVIEJ48yepjIaxm21_eN44,6284
-datamarket-0.8.6.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-datamarket-0.8.6.dist-info/RECORD,,
+datamarket-0.8.8.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.8.8.dist-info/METADATA,sha256=Kgq1Quj4MHqEk0xauolE8YM8wJBMR3ULQwvkSGfLt7s,6284
+datamarket-0.8.8.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+datamarket-0.8.8.dist-info/RECORD,,