datamarket 0.7.41__py3-none-any.whl → 0.7.125__py3-none-any.whl

datamarket/interfaces/azure.py

@@ -0,0 +1,127 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from azure.storage.blob import BlobServiceClient, ContainerClient
+from pendulum import now
+
+########################################################################################################################
+# CLASSES
+
+logger = logging.getLogger(__name__)
+
+
+class AzureBlobInterface:
+    def __init__(self, config):
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("azure:"):
+                profile_name = section.split(":", 1)[1]
+                connection_string = self.config[section].get("connection_string")
+                container_name = self.config[section].get("container_name")
+                sas_container_url = self.config[section].get("sas_container_url")
+
+                if sas_container_url:
+                    session = ContainerClient.from_container_url(sas_container_url)
+                elif connection_string and container_name:
+                    session = BlobServiceClient.from_connection_string(connection_string).get_container_client(
+                        container_name
+                    )
+
+                self.profiles.append(
+                    {
+                        "profile": profile_name,
+                        "container_name": container_name,
+                        "session": session,
+                    }
+                )
+
+        if not self.profiles:
+            logger.warning("No Azure profiles found in config file")
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
+
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                return
+        logger.warning(f"Profile {profile_name} not found")
+
+    def upload_file(
+        self,
+        local_file,
+        remote_folder,
+        remote_file=None,
+        upload_file_info=False,
+        **file_info_data,
+    ):
+        if not remote_file:
+            remote_file = Path(local_file).name
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        with open(local_file, "rb") as data:
+            blob_client.upload_blob(data, overwrite=True)
+
+        if upload_file_info:
+            self.upload_file_info(remote_path, **file_info_data)
+
+    def upload_file_info(self, remote_path, **file_info_data):
+        summary_file = remote_path.split(".")[0] + "_resumen.csv"
+        blob_client = self.current_profile["session"].get_blob_client(summary_file)
+
+        new_record = {
+            "file": remote_path,
+            "num_rows": file_info_data.get("num_rows"),
+            "schema_version": file_info_data.get("schema_version"),
+            "upload_date": now(tz="Europe/Madrid").to_datetime_string(),
+        }
+
+        new_record_str = "file,num_rows,schema_version,upload_date\n"
+        new_record_str += ",".join([str(v) for v in new_record.values()]) + "\n"
+
+        blob_client.upload_blob(new_record_str, overwrite=True)
+
+    def download_file(self, local_file, remote_path):
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        blob_data = blob_client.download_blob()
+        with open(local_file, "wb") as f:
+            blob_data.readinto(f)
+
+    def check_file_exists_and_not_empty(self, remote_file, remote_folder):
+        """
+        Checks if a blob exists in the specified folder and has a size greater than 100 bytes.
+
+        Args:
+            remote_file (str): The name of the file (blob) to check.
+            remote_folder (str): The folder (prefix) where the file is located.
+
+        Returns:
+            bool: True if the blob exists and has a size greater than 100, False otherwise.
+        """
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        try:
+            blob_client = self.current_profile["session"].get_blob_client(remote_path)
+            if blob_client.exists():
+                properties = blob_client.get_blob_properties()
+                if properties.size > 100:  # Check if size is greater than 100 bytes
+                    logger.debug(f"Blob '{remote_path}' exists and is not empty (size: {properties.size}).")
+                    return True
+                else:
+                    logger.debug(f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes.")
+                    return False
+            else:
+                logger.debug(f"Blob '{remote_path}' does not exist.")
+                return False
+        except Exception as e:
+            logger.error(f"Error checking blob '{remote_path}': {e}")
+            # In case of error, assume it doesn't exist or is empty to allow upload attempt
+            return False

datamarket/interfaces/drive.py

@@ -18,13 +18,9 @@ class DriveInterface:
         if "drive" in config:
             self.config = config["drive"]
 
-            GoogleAuth.DEFAULT_SETTINGS[
-                "client_config_file"
-            ] = f'{self.config["config_path"]}/credentials.json'
+            GoogleAuth.DEFAULT_SETTINGS["client_config_file"] = f"{self.config['config_path']}/credentials.json"
 
-            self.gauth = GoogleAuth(
-                settings_file=f'{self.config["config_path"]}/settings.yaml'
-            )
+            self.gauth = GoogleAuth(settings_file=f"{self.config['config_path']}/settings.yaml")
             self.gauth.LocalWebserverAuth()
 
             self.drive = GoogleDrive(self.gauth)
@@ -48,24 +44,78 @@ class DriveInterface:
                 logger.info(f"deleting old {filename}...")
                 drive_file.Delete(param={"supportsTeamDrives": True})
 
+    def _create_remote_dir_tree(self, base_folder_id, path_parts):
+        """
+        Ensure the nested folders described by path_parts exist under base_folder_id.
+        Returns the folder_id of the deepest folder (or base_folder_id if path_parts is empty).
+        """
+        parent_id = base_folder_id
+        for part in path_parts:
+            part = part.strip()
+            if not part:
+                continue
+
+            query = (
+                f"'{parent_id}' in parents and title = '{part}'"
+                " and mimeType = 'application/vnd.google-apps.folder' and trashed=false"
+            )
+            results = self.drive.ListFile(
+                {
+                    "q": query,
+                    "corpora": "teamDrive",
+                    "teamDriveId": self.team_id,
+                    "includeTeamDriveItems": True,
+                    "supportsTeamDrives": True,
+                }
+            ).GetList()
+
+            if results:
+                parent_id = results[0]["id"]
+            else:
+                folder_metadata = {
+                    "title": part,
+                    "mimeType": "application/vnd.google-apps.folder",
+                    "parents": [
+                        {
+                            "kind": "drive#fileLink",
+                            "teamDriveId": self.team_id,
+                            "id": parent_id,
+                        }
+                    ],
+                }
+                folder = self.drive.CreateFile(folder_metadata)
+                folder.Upload(param={"supportsTeamDrives": True})
+                parent_id = folder["id"]
+
+        return parent_id
+
     def upload_file(self, local_filename, drive_filename, folder_id):
-        self.delete_old_files(drive_filename, folder_id)
+        drive_filename = drive_filename.strip("/")
+        parts = drive_filename.split("/")
+        if len(parts) > 1:
+            *folders, filename = parts
+            target_folder_id = self._create_remote_dir_tree(folder_id, folders)
+        else:
+            filename = parts[0]
+            target_folder_id = folder_id
+
+        self.delete_old_files(filename, target_folder_id)
 
         f = self.drive.CreateFile(
             {
-                "title": drive_filename,
+                "title": filename,
                 "parents": [
                     {
                         "kind": "drive#fileLink",
                         "teamDriveId": self.team_id,
-                        "id": folder_id,
+                        "id": target_folder_id,
                     }
                 ],
             }
         )
         f.SetContentFile(local_filename)
 
-        logger.info(f"uploading {drive_filename} to folder: {folder_id}...")
+        logger.info(f"uploading {drive_filename} to folder: {target_folder_id}...")
         f.Upload(param={"supportsTeamDrives": True})
 
     def validate_file(self, filename, folder_id):

datamarket/interfaces/ftp.py

@@ -4,6 +4,7 @@
 import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
+from typing import Any, Dict, List, Optional
 
 ########################################################################################################################
 # CLASSES
@@ -13,31 +14,37 @@ logger = logging.getLogger(__name__)
 
 class FTPInterface:
     def __init__(self, config):
-        if "ftp" in config:
-            self.config = config["ftp"]
-
-            self.ftp = self.get_ftp()
-        else:
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("ftp:"):
+                profile_name = section.split(":", 1)[1]
+                ftps = self.config[section]["ftps"].lower() == "true"
+                ftp_conn = FTP_TLS(self.config[section]["server"]) if ftps else FTP(self.config[section]["server"])  # noqa: S321
+                ftp_conn.login(self.config[section]["username"], self.config[section]["password"])
+                self.profiles.append({"profile": profile_name, "session": ftp_conn})
+
+        if not self.profiles:
             logger.warning("no ftp section in config")
 
-    def get_ftp(self):
-        if self.config["ftps"].lower() == "true":
-            ftp_conn = FTP_TLS(self.config["server"])
-
-        else:
-            ftp_conn = FTP(self.config["server"])
-
-        ftp_conn.login(self.config["username"], self.config["password"])
+        self.current_profile: Optional[Dict[str, Any]] = self.profiles[0] if self.profiles else None
+        self.ftp = self.current_profile["session"] if self.current_profile else None
 
-        return ftp_conn
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                self.ftp = profile["session"]
+                return
+        logger.warning(f"Profile {profile_name} not found")
 
     def upload_file(self, local_file, remote_folder, remote_file=None):
         if not remote_file:
             remote_file = Path(local_file).name
 
-        self._create_remote_dir_tree(full_path=f"{remote_folder}{remote_file}")
+        self._create_remote_dir_tree(full_path=f"/{remote_folder}{remote_file}")
 
-        self.ftp.cwd(remote_folder)
+        self.ftp.cwd(f"/{remote_folder}")
 
         with open(local_file, "rb") as f:
             self.ftp.storbinary(f"STOR {remote_file}", f)