datamarket 0.8.8__tar.gz → 0.9.1__tar.gz

{datamarket-0.8.8 → datamarket-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.8.8
+Version: 0.9.1
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: alchemy
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
 Provides-Extra: boto3
@@ -62,7 +63,7 @@ Provides-Extra: tqdm
 Provides-Extra: undetected-chromedriver
 Provides-Extra: unidecode
 Provides-Extra: xmltodict
-Requires-Dist: SQLAlchemy (==2.0.36)
+Requires-Dist: SQLAlchemy (==2.0.36) ; extra == "alchemy"
 Requires-Dist: azure-storage-blob (==12.23.1) ; extra == "azure-storage-blob"
 Requires-Dist: beautifulsoup4 (==4.12.3)
 Requires-Dist: boto3 (==1.35.53) ; extra == "boto3" or extra == "aws" or extra == "peerdb"

{datamarket-0.8.8 → datamarket-0.9.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.8.8"
+version = "0.9.1"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -17,7 +17,6 @@ classifiers = [
 [tool.poetry.dependencies]
 python = "^3.9"
 typer = "0.12.5"
-SQLAlchemy = "2.0.36"
 psycopg2-binary = "2.9.10"
 requests = "2.32.3"
 tenacity = "9.0.0"
@@ -71,6 +70,7 @@ google-auth-oauthlib = { version = "1.2.1", optional = true }
 dnspython = { version = "2.7.0", optional = true }
 openpyxl = { version = "3.1.5", optional = true }
 httpx = { extras = ["http2"], version = "0.28.1", optional = true }
+SQLAlchemy = { version = "2.0.36", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -122,6 +122,7 @@ aws = ["boto3"]
 drive = ["pydrive2"]
 peerdb = ["boto3", "clickhouse-driver"]
 proxy = ["stem"]
+alchemy = ["SQLAlchemy"]
 
 
 [build-system]

{datamarket-0.8.8 → datamarket-0.9.1}/src/datamarket/interfaces/peerdb.py

@@ -212,30 +212,56 @@ class ClickhousePeer:
 
 class TransientS3:
     def __init__(self, config):
-        if "peerdb-s3" in config:
-            self.config = config["peerdb-s3"]
+        if section := config.get("peerdb", {}).get("s3"):
+            self.config = section
             self.bucket_name = self.config["bucket"]
             self.session = boto3.Session(profile_name=self.config["profile"])
-            self.s3_resource = self.session.resource("s3")
+            self.s3_client = self.session.client("s3")
             self.credentials = self.session.get_credentials()
             self.access_key = self.credentials.access_key
             self.secret_key = self.credentials.secret_key
             self.region_name = self.session.region_name
-            self.endpoint_url = self.s3_resource.meta.endpoint_url
+            self.endpoint_url = self.s3_client.meta.endpoint_url
         else:
-            logger.warning("no peerdb-s3 section in config")
+            logger.warning("no peerdb.s3 section in config")
 
     def delete_paths_with_schema(self, schema_name):
         logger.info(f"Deleting paths containing '{schema_name}' from S3")
 
-        bucket = self.s3_resource.Bucket(self.bucket_name)
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter="/")
 
-        for prefix in [schema_name, f"clone_{schema_name}"]:
-            objects_to_delete = bucket.objects.filter(Prefix=prefix)
-            objects_to_delete.delete()
+        for page in pages:
+            if "CommonPrefixes" in page:
+                for prefix in page["CommonPrefixes"]:
+                    folder = prefix["Prefix"]
+                    if schema_name in folder:
+                        self._delete_folder_contents(folder)
 
         logger.info(f"Deleted paths containing '{schema_name}' from S3")
 
+    def _delete_folder_contents(self, folder):
+        logger.info(f"Deleting contents of folder: {folder}")
+
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=folder)
+
+        delete_us = dict(Objects=[])
+        for page in pages:
+            if "Contents" in page:
+                for obj in page["Contents"]:
+                    delete_us["Objects"].append(dict(Key=obj["Key"]))
+
+                    # AWS limits to deleting 1000 objects at a time
+                    if len(delete_us["Objects"]) >= 1000:
+                        self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
+                        delete_us = dict(Objects=[])
+
+        if len(delete_us["Objects"]):
+            self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
+
+        logger.info(f"Deleted contents of folder: {folder}")
+
 
 class PeerDBInterface:
     def __init__(self, config):
@@ -282,11 +308,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ["localhost", "127.0.0.1"]:
+        if host in ['localhost', '127.0.0.1']:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r"(localhost|127\.0\.0\.1)"
+        url_pattern = r'(localhost|127\.0\.0\.1)'
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)