datamarket 0.6.0__py3-none-any.whl → 0.10.3__py3-none-any.whl

datamarket/interfaces/aws.py

@@ -3,6 +3,8 @@
 
 import io
 import logging
+from typing import Any, Dict, List, Optional
+
 import boto3
 
 ########################################################################################################################
@@ -12,34 +14,44 @@ logger = logging.getLogger(__name__)
 
 
 class AWSInterface:
-    def __init__(self, config):
-        self.profiles = []
+    def __init__(self, config) -> None:
+        self.profiles: List[Dict[str, Any]] = []
         self.config = config
 
-        for section in self.config.sections():
+        for section in getattr(self.config, "sections", lambda: [])():
             if section.startswith("aws:"):
                 profile_name = section.split(":", 1)[1]
+                bucket_value = self.config[section].get("buckets", "")
+                buckets = [b.strip() for b in bucket_value.split(",") if b.strip()]
+                session = boto3.Session(profile_name=profile_name)
+
                 self.profiles.append(
                     {
                         "profile": profile_name,
-                        "bucket": self.config[section]["bucket"],
-                        "session": boto3.Session(profile_name=profile_name),
+                        "buckets": buckets,
+                        "session": session,
                     }
                 )
 
         if not self.profiles:
             logger.warning("No AWS profiles found in config file")
 
-        self.current_profile = self.profiles[0] if self.profiles else None
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
         self._update_resources()
 
-    def _update_resources(self):
+    def _update_resources(self) -> None:
+        """Refresh S3 resources for the current profile and set default bucket (first in list)"""
         if self.current_profile:
             self.s3 = self.current_profile["session"].resource("s3")
             self.s3_client = self.s3.meta.client
-            self.bucket = self.current_profile["bucket"]
+            buckets = self.current_profile.get("buckets", [])
+            self.bucket = buckets[0] if buckets else None
+        else:
+            self.s3 = None
+            self.s3_client = None
+            self.bucket = None
 
-    def switch_profile(self, profile_name):
+    def switch_profile(self, profile_name: str) -> None:
         for profile in self.profiles:
             if profile["profile"] == profile_name:
                 self.current_profile = profile
@@ -47,14 +59,69 @@ class AWSInterface:
                 return
         logger.warning(f"Profile {profile_name} not found")
 
-    def get_file(self, s3_path):
+    def switch_bucket(self, bucket: str) -> None:
+        if not self.current_profile:
+            logger.warning("No current AWS profile to switch bucket on")
+            return
+
+        buckets = self.current_profile.get("buckets") or []
+        if bucket not in buckets:
+            logger.warning(f"Bucket {bucket} not found in profile {self.current_profile.get('profile')}")
+            return
+
+        self.bucket = bucket
+
+    def switch_bucket_for_profile(self, profile_name: str, bucket: str) -> None:
+        """
+        Select a profile and then switch its active bucket.
+        """
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                self._update_resources()  # sets default bucket & s3 clients
+                self.switch_bucket(bucket)  # only sets self.bucket if valid
+                return
+        logger.warning(f"Profile {profile_name} not found")
+
+    def get_bucket_url(self) -> Optional[str]:
+        """Return active bucket URL."""
+        if not self.bucket:
+            logger.warning("No active bucket selected")
+            return None
+        region = self.s3_client.meta.region_name
+        return f"https://{self.bucket}.s3.{region}.amazonaws.com"
+
+    def get_file(self, s3_path: str):
+        if not self.bucket:
+            logger.warning("No active bucket selected")
+            return None
         try:
             return self.s3.Object(self.bucket, s3_path).get()
         except self.s3_client.exceptions.NoSuchKey:
             logger.info(f"{s3_path} does not exist")
+            return None
+
+    def file_exists(self, s3_path: str) -> bool:
+        if not self.bucket:
+            logger.warning("No active bucket selected")
+            return False
+        try:
+            self.s3_client.head_object(Bucket=self.bucket, Key=s3_path)
+            return True
+        except self.s3_client.exceptions.NoSuchKey:
+            return False
+        except Exception as e:
+            logger.error(f"Error checking existence of {s3_path}: {e}")
+            raise
 
-    def read_file_as_bytes(self, s3_path):
-        return io.BytesIO(self.get_file(s3_path)["Body"].read())
+    def read_file_as_bytes(self, s3_path: str) -> Optional[io.BytesIO]:
+        obj = self.get_file(s3_path)
+        if not obj:
+            return None
+        return io.BytesIO(obj["Body"].read())
 
-    def upload_file(self, local_path, s3_path):
-        self.s3.Bucket(self.bucket).upload_file(local_path, s3_path)
+    def upload_file(self, local_path: str, s3_path: str, **kwargs) -> None:
+        if not self.bucket:
+            logger.warning("No active bucket selected")
+            return
+        self.s3.Bucket(self.bucket).upload_file(local_path, s3_path, **kwargs)

datamarket/interfaces/azure.py

@@ -0,0 +1,127 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from azure.storage.blob import BlobServiceClient, ContainerClient
+from pendulum import now
+
+########################################################################################################################
+# CLASSES
+
+logger = logging.getLogger(__name__)
+
+
+class AzureBlobInterface:
+    def __init__(self, config):
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("azure:"):
+                profile_name = section.split(":", 1)[1]
+                connection_string = self.config[section].get("connection_string")
+                container_name = self.config[section].get("container_name")
+                sas_container_url = self.config[section].get("sas_container_url")
+
+                if sas_container_url:
+                    session = ContainerClient.from_container_url(sas_container_url)
+                elif connection_string and container_name:
+                    session = BlobServiceClient.from_connection_string(connection_string).get_container_client(
+                        container_name
+                    )
+
+                self.profiles.append(
+                    {
+                        "profile": profile_name,
+                        "container_name": container_name,
+                        "session": session,
+                    }
+                )
+
+        if not self.profiles:
+            logger.warning("No Azure profiles found in config file")
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
+
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                return
+        logger.warning(f"Profile {profile_name} not found")
+
+    def upload_file(
+        self,
+        local_file,
+        remote_folder,
+        remote_file=None,
+        upload_file_info=False,
+        **file_info_data,
+    ):
+        if not remote_file:
+            remote_file = Path(local_file).name
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        with open(local_file, "rb") as data:
+            blob_client.upload_blob(data, overwrite=True)
+
+        if upload_file_info:
+            self.upload_file_info(remote_path, **file_info_data)
+
+    def upload_file_info(self, remote_path, **file_info_data):
+        summary_file = remote_path.split(".")[0] + "_resumen.csv"
+        blob_client = self.current_profile["session"].get_blob_client(summary_file)
+
+        new_record = {
+            "file": remote_path,
+            "num_rows": file_info_data.get("num_rows"),
+            "schema_version": file_info_data.get("schema_version"),
+            "upload_date": now(tz="Europe/Madrid").to_datetime_string(),
+        }
+
+        new_record_str = "file,num_rows,schema_version,upload_date\n"
+        new_record_str += ",".join([str(v) for v in new_record.values()]) + "\n"
+
+        blob_client.upload_blob(new_record_str, overwrite=True)
+
+    def download_file(self, local_file, remote_path):
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        blob_data = blob_client.download_blob()
+        with open(local_file, "wb") as f:
+            blob_data.readinto(f)
+
+    def check_file_exists_and_not_empty(self, remote_file, remote_folder):
+        """
+        Checks if a blob exists in the specified folder and has a size greater than 100 bytes.
+
+        Args:
+            remote_file (str): The name of the file (blob) to check.
+            remote_folder (str): The folder (prefix) where the file is located.
+
+        Returns:
+            bool: True if the blob exists and has a size greater than 100, False otherwise.
+        """
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        try:
+            blob_client = self.current_profile["session"].get_blob_client(remote_path)
+            if blob_client.exists():
+                properties = blob_client.get_blob_properties()
+                if properties.size > 100:  # Check if size is greater than 100 bytes
+                    logger.debug(f"Blob '{remote_path}' exists and is not empty (size: {properties.size}).")
+                    return True
+                else:
+                    logger.debug(f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes.")
+                    return False
+            else:
+                logger.debug(f"Blob '{remote_path}' does not exist.")
+                return False
+        except Exception as e:
+            logger.error(f"Error checking blob '{remote_path}': {e}")
+            # In case of error, assume it doesn't exist or is empty to allow upload attempt
+            return False

datamarket/interfaces/drive.py

@@ -18,13 +18,9 @@ class DriveInterface:
         if "drive" in config:
             self.config = config["drive"]
 
-            GoogleAuth.DEFAULT_SETTINGS[
-                "client_config_file"
-            ] = f'{self.config["config_path"]}/credentials.json'
+            GoogleAuth.DEFAULT_SETTINGS["client_config_file"] = f"{self.config['config_path']}/credentials.json"
 
-            self.gauth = GoogleAuth(
-                settings_file=f'{self.config["config_path"]}/settings.yaml'
-            )
+            self.gauth = GoogleAuth(settings_file=f"{self.config['config_path']}/settings.yaml")
             self.gauth.LocalWebserverAuth()
 
             self.drive = GoogleDrive(self.gauth)
@@ -48,24 +44,78 @@ class DriveInterface:
                 logger.info(f"deleting old {filename}...")
                 drive_file.Delete(param={"supportsTeamDrives": True})
 
+    def _create_remote_dir_tree(self, base_folder_id, path_parts):
+        """
+        Ensure the nested folders described by path_parts exist under base_folder_id.
+        Returns the folder_id of the deepest folder (or base_folder_id if path_parts is empty).
+        """
+        parent_id = base_folder_id
+        for part in path_parts:
+            part = part.strip()
+            if not part:
+                continue
+
+            query = (
+                f"'{parent_id}' in parents and title = '{part}'"
+                " and mimeType = 'application/vnd.google-apps.folder' and trashed=false"
+            )
+            results = self.drive.ListFile(
+                {
+                    "q": query,
+                    "corpora": "teamDrive",
+                    "teamDriveId": self.team_id,
+                    "includeTeamDriveItems": True,
+                    "supportsTeamDrives": True,
+                }
+            ).GetList()
+
+            if results:
+                parent_id = results[0]["id"]
+            else:
+                folder_metadata = {
+                    "title": part,
+                    "mimeType": "application/vnd.google-apps.folder",
+                    "parents": [
+                        {
+                            "kind": "drive#fileLink",
+                            "teamDriveId": self.team_id,
+                            "id": parent_id,
+                        }
+                    ],
+                }
+                folder = self.drive.CreateFile(folder_metadata)
+                folder.Upload(param={"supportsTeamDrives": True})
+                parent_id = folder["id"]
+
+        return parent_id
+
     def upload_file(self, local_filename, drive_filename, folder_id):
-        self.delete_old_files(drive_filename, folder_id)
+        drive_filename = drive_filename.strip("/")
+        parts = drive_filename.split("/")
+        if len(parts) > 1:
+            *folders, filename = parts
+            target_folder_id = self._create_remote_dir_tree(folder_id, folders)
+        else:
+            filename = parts[0]
+            target_folder_id = folder_id
+
+        self.delete_old_files(filename, target_folder_id)
 
         f = self.drive.CreateFile(
             {
-                "title": drive_filename,
+                "title": filename,
                 "parents": [
                     {
                         "kind": "drive#fileLink",
                         "teamDriveId": self.team_id,
-                        "id": folder_id,
+                        "id": target_folder_id,
                     }
                 ],
             }
         )
         f.SetContentFile(local_filename)
 
-        logger.info(f"uploading {drive_filename} to folder: {folder_id}...")
+        logger.info(f"uploading {drive_filename} to folder: {target_folder_id}...")
         f.Upload(param={"supportsTeamDrives": True})
 
     def validate_file(self, filename, folder_id):

datamarket/interfaces/ftp.py

@@ -4,6 +4,7 @@
 import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
+from typing import Any, Dict, List, Optional
 
 ########################################################################################################################
 # CLASSES
@@ -13,29 +14,37 @@ logger = logging.getLogger(__name__)
 
 class FTPInterface:
     def __init__(self, config):
-        if "ftp" in config:
-            self.config = config["ftp"]
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("ftp:"):
+                profile_name = section.split(":", 1)[1]
+                ftps = self.config[section]["ftps"].lower() == "true"
+                ftp_conn = FTP_TLS(self.config[section]["server"]) if ftps else FTP(self.config[section]["server"])  # noqa: S321
+                ftp_conn.login(self.config[section]["username"], self.config[section]["password"])
+                self.profiles.append({"profile": profile_name, "session": ftp_conn})
 
-            self.ftp = self.get_ftp()
-        else:
+        if not self.profiles:
             logger.warning("no ftp section in config")
 
-    def get_ftp(self):
-        if self.config["ftps"].lower() == "true":
-            ftp_conn = FTP_TLS(self.config["server"])
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
+        self.ftp = self.current_profile["session"] if self.current_profile else None
 
-        else:
-            ftp_conn = FTP(self.config["server"])
-
-        ftp_conn.login(self.config["username"], self.config["password"])
-
-        return ftp_conn
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                self.ftp = profile["session"]
+                return
+        logger.warning(f"Profile {profile_name} not found")
 
     def upload_file(self, local_file, remote_folder, remote_file=None):
         if not remote_file:
             remote_file = Path(local_file).name
 
-        self.ftp.cwd(remote_folder)
+        self._create_remote_dir_tree(full_path=f"/{remote_folder}{remote_file}")
+
+        self.ftp.cwd(f"/{remote_folder}")
 
         with open(local_file, "rb") as f:
             self.ftp.storbinary(f"STOR {remote_file}", f)
@@ -43,3 +52,17 @@ class FTPInterface:
     def download_file(self, local_file, remote_file):
         with open(local_file, "wb") as f:
             self.ftp.retrbinary(f"RETR {remote_file}", f.write)
+
+    def _create_remote_dir_tree(self, full_path):
+        dir_tree = full_path.split("/")[0:-1]  # Exclude filename
+
+        for part in dir_tree:
+            if not part:
+                continue
+
+            try:
+                self.ftp.cwd(part)
+            except Exception as e:
+                logger.warning(f"Error while creating remote directory: {e}")
+                self.ftp.mkd(part)
+                self.ftp.cwd(part)