datamarket 0.9.4__tar.gz → 0.9.6__tar.gz

{datamarket-0.9.4 → datamarket-0.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.4
+Version: 0.9.6
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -63,7 +63,7 @@ Provides-Extra: xmltodict
 Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0) ; extra == "alchemy"
 Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
 Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
-Requires-Dist: boto3 (>=1.0.0,<2.0.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: boto3 (>=1.35.0,<1.36.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
 Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
 Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
 Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"

{datamarket-0.9.4 → datamarket-0.9.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.4"
+version = "0.9.6"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -27,7 +27,7 @@ croniter = "^3.0.0"
 dynaconf = "^3.0.0"
 jinja2 = "^3.0.0"
 
-boto3 = { version = "^1.0.0", optional = true }
+boto3 = { version = "~1.35.0", optional = true }
 unidecode = { version = "^1.0.0", optional = true }
 lxml = { extras = ["html-clean"], version = "^5.0.0", optional = true }
 tqdm = { version = "^4.0.0", optional = true }

{datamarket-0.9.4 → datamarket-0.9.6}/src/datamarket/interfaces/aws.py

@@ -4,8 +4,7 @@
 import io
 import logging
 import boto3
-
-from ..utils.main import Config
+from dynaconf import Dynaconf
 
 ########################################################################################################################
 # CLASSES
@@ -14,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class AWSInterface:
-    def __init__(self, config: Config) -> None:
+    def __init__(self, config: Dynaconf) -> None:
         self.profiles = []
         self.config = config
 

{datamarket-0.9.4 → datamarket-0.9.6}/src/datamarket/interfaces/ftp.py

@@ -5,16 +5,15 @@ import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
 
-from ..utils.main import Config
+from dynaconf import Dynaconf
 
 ########################################################################################################################
 # CLASSES
 
 logger = logging.getLogger(__name__)
 
-
 class FTPInterface:
-    def __init__(self, config: Config):
+    def __init__(self, config: Dynaconf):
         if "ftp" in config:
             self.config = config["ftp"]
 
@@ -23,7 +22,7 @@ class FTPInterface:
             logger.warning("no ftp section in config")
 
     def get_ftp(self):
-        if self.config["ftps"].lower() == "true":
+        if self.config["ftps"]:
             ftp_conn = FTP_TLS(self.config["server"])
 
         else:

{datamarket-0.9.4 → datamarket-0.9.6}/src/datamarket/interfaces/peerdb.py

@@ -216,26 +216,52 @@ class TransientS3:
             self.config = section
             self.bucket_name = self.config["bucket"]
             self.session = boto3.Session(profile_name=self.config["profile"])
-            self.s3_resource = self.session.resource("s3")
+            self.s3_client = self.session.client("s3")
             self.credentials = self.session.get_credentials()
             self.access_key = self.credentials.access_key
             self.secret_key = self.credentials.secret_key
             self.region_name = self.session.region_name
-            self.endpoint_url = self.s3_resource.meta.endpoint_url
+            self.endpoint_url = self.s3_client.meta.endpoint_url
         else:
             logger.warning("no peerdb.s3 section in config")
 
     def delete_paths_with_schema(self, schema_name):
         logger.info(f"Deleting paths containing '{schema_name}' from S3")
 
-        bucket = self.s3_resource.Bucket(self.bucket_name)
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(Bucket=self.bucket_name, Delimiter="/")
 
-        for prefix in [schema_name, f"clone_{schema_name}"]:
-            objects_to_delete = bucket.objects.filter(Prefix=prefix)
-            objects_to_delete.delete()
+        for page in pages:
+            if "CommonPrefixes" in page:
+                for prefix in page["CommonPrefixes"]:
+                    folder = prefix["Prefix"]
+                    if schema_name in folder:
+                        self._delete_folder_contents(folder)
 
         logger.info(f"Deleted paths containing '{schema_name}' from S3")
 
+    def _delete_folder_contents(self, folder):
+        logger.info(f"Deleting contents of folder: {folder}")
+
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(Bucket=self.bucket_name, Prefix=folder)
+
+        delete_us = dict(Objects=[])
+        for page in pages:
+            if "Contents" in page:
+                for obj in page["Contents"]:
+                    delete_us["Objects"].append(dict(Key=obj["Key"]))
+
+                    # AWS limits to deleting 1000 objects at a time
+                    if len(delete_us["Objects"]) >= 1000:
+                        self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
+                        delete_us = dict(Objects=[])
+
+        if len(delete_us["Objects"]):
+            self.s3_client.delete_objects(Bucket=self.bucket_name, Delete=delete_us)
+
+        logger.info(f"Deleted contents of folder: {folder}")
+
 
 class PeerDBInterface:
     def __init__(self, config):
@@ -282,11 +308,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ["localhost", "127.0.0.1"]:
+        if host in ['localhost', '127.0.0.1']:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r"(localhost|127\.0\.0\.1)"
+        url_pattern = r'(localhost|127\.0\.0\.1)'
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)

{datamarket-0.9.4 → datamarket-0.9.6}/src/datamarket/utils/main.py

@@ -2,6 +2,7 @@
 # IMPORTS
 
 import asyncio
+from dataclasses import dataclass
 import inspect
 import logging
 import random
@@ -11,23 +12,35 @@ import shutil
 import subprocess
 import time
 from pathlib import Path
-from typing import Literal, Union
+from typing import Literal, Optional, Union
 
 import pendulum
 from croniter import croniter
 from configparser import RawConfigParser
 from dynaconf import Dynaconf, add_converter
 
+logger = logging.getLogger(__name__)
+
 ########################################################################################################################
-# FUNCTIONS
+# CLASSES
+
+
+@dataclass
+class ProjectMetadata:
+    cmd_prefix: str
+    package_name: str
+    env_name: str
+    path: Path
+    config_path: Path
 
-logger = logging.getLogger(__name__)
 
-Config = Union[RawConfigParser, Dynaconf]
+########################################################################################################################
+# FUNCTIONS
 
 
 def get_granular_date(
-    granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str], tz: str = "Europe/Madrid"
+    granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str],
+    tz: str = "Europe/Madrid",
 ) -> pendulum.DateTime:
     """
     Returns the most recent date based on the given granularity or a custom cron expression.
@@ -67,8 +80,10 @@ def read_converter(path_str: str):
 
 
 def get_config(
-    config_file: Path, tz: str = "Europe/Madrid"
-) -> Union[RawConfigParser, Dynaconf]:
+    config_file: Optional[Path] = None, tz: str = "Europe/Madrid"
+) -> Dynaconf:
+    config_file = config_file or get_project_metadata().config_path
+
     if Path(config_file).suffix == ".ini":
         logger.warning("Using legacy INI config reader. Please migrate to TOML")
         cfg = RawConfigParser()
@@ -106,17 +121,20 @@ def get_config(
 
     return config
 
-
-def get_project_metadata():
+def get_project_metadata() -> ProjectMetadata:
     caller_frame = inspect.stack()[1]
     current_file_parts = Path(caller_frame.filename).resolve().parts
     src_index = current_file_parts.index("src")
+
     cmd_prefix = "dix vnc run --" if shutil.which("dix") else ""
-    pkg_name = current_file_parts[src_index + 1]
-    env_name = f"{pkg_name}_env"
+    package_name = current_file_parts[src_index + 1]
+    env_name = f"{package_name}_env"
     project_path = Path(*current_file_parts[:src_index])
+    config_path = project_path / "config.toml"
 
-    return {"cmd_prefix": cmd_prefix, "pkg_name": pkg_name, "env_name": env_name, "project_path": project_path}
+    return ProjectMetadata(
+        cmd_prefix, package_name, env_name, project_path, config_path
+    )
 
 
 def set_logger(level):