datamarket 0.9.51__py3-none-any.whl → 0.9.52__py3-none-any.whl

datamarket/interfaces/azure.py

@@ -0,0 +1,127 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from azure.storage.blob import BlobServiceClient
+from pendulum import now
+
+########################################################################################################################
+# CLASSES
+
+logger = logging.getLogger(__name__)
+
+
+class AzureBlobInterface:
+    def __init__(self, config):
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("azure:"):
+                profile_name = section.split(":", 1)[1]
+                connection_string = self.config[section].get("connection_string")
+                container_name = self.config[section].get("container_name")
+
+                self.profiles.append(
+                    {
+                        "profile": profile_name,
+                        "container_name": container_name,
+                        "session": BlobServiceClient.from_connection_string(
+                            connection_string
+                        ).get_container_client(container_name),
+                    }
+                )
+
+        if not self.profiles:
+            logger.warning("No Azure profiles found in config file")
+        self.current_profile: Optional[Dict[str, Any]] = (
+            self.profiles[0] if self.profiles else None
+        )
+
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                return
+        logger.warning(f"Profile {profile_name} not found")
+
+    def upload_file(
+        self,
+        local_file,
+        remote_folder,
+        remote_file=None,
+        upload_file_info=False,
+        **kwargs,
+    ):
+        if not remote_file:
+            remote_file = Path(local_file).name
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        with open(local_file, "rb") as data:
+            blob_client.upload_blob(data, overwrite=True)
+
+        if upload_file_info:
+            self.upload_file_info(remote_path, **kwargs)
+
+    def upload_file_info(self, remote_path, **kwargs):
+        summary_file = remote_path.split(".")[0] + "_resumen.csv"
+        blob_client = self.current_profile["session"].get_blob_client(summary_file)
+
+        new_record = {
+            "file": remote_path,
+            "num_rows": kwargs["num_rows"],
+            "schema_version": kwargs["schema_version"],
+            "upload_date": now(tz="Europe/Madrid").to_datetime_string(),
+        }
+
+        new_record_str = "file,num_rows,schema_version,upload_date\n"
+        new_record_str += ",".join([str(v) for v in new_record.values()]) + "\n"
+
+        blob_client.upload_blob(new_record_str, overwrite=True)
+
+    def download_file(self, local_file, remote_path):
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        blob_data = blob_client.download_blob()
+        with open(local_file, "wb") as f:
+            blob_data.readinto(f)
+
+    def check_file_exists_and_not_empty(self, remote_file, remote_folder):
+        """
+        Checks if a blob exists in the specified folder and has a size greater than 100 bytes.
+
+        Args:
+            remote_file (str): The name of the file (blob) to check.
+            remote_folder (str): The folder (prefix) where the file is located.
+
+        Returns:
+            bool: True if the blob exists and has a size greater than 100, False otherwise.
+        """
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        try:
+            blob_client = self.current_profile["session"].get_blob_client(remote_path)
+            if blob_client.exists():
+                properties = blob_client.get_blob_properties()
+                if properties.size > 100:  # Check if size is greater than 100 bytes
+                    logger.debug(
+                        f"Blob '{remote_path}' exists and is not empty (size: {properties.size})."
+                    )
+                    return True
+                else:
+                    logger.debug(
+                        f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes."
+                    )
+                    return False
+            else:
+                logger.debug(f"Blob '{remote_path}' does not exist.")
+                return False
+        except Exception as e:
+            logger.error(f"Error checking blob '{remote_path}': {e}")
+            # In case of error, assume it doesn't exist or is empty to allow upload attempt
+            return False

datamarket/interfaces/ftp.py

@@ -4,39 +4,53 @@
 import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
-
-from dynaconf import Dynaconf
+from typing import Any, Dict, List, Optional
 
 ########################################################################################################################
 # CLASSES
 
 logger = logging.getLogger(__name__)
 
-class FTPInterface:
-    def __init__(self, config: Dynaconf):
-        if "ftp" in config:
-            self.config = config["ftp"]
 
-            self.ftp = self.get_ftp()
-        else:
+class FTPInterface:
+    def __init__(self, config):
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("ftp:"):
+                profile_name = section.split(":", 1)[1]
+                ftps = self.config[section]["ftps"].lower() == "true"
+                ftp_conn = (
+                    FTP_TLS(self.config[section]["server"])
+                    if ftps
+                    else FTP(self.config[section]["server"])
+                )  # noqa: S321
+                ftp_conn.login(
+                    self.config[section]["username"], self.config[section]["password"]
+                )
+                self.profiles.append({"profile": profile_name, "session": ftp_conn})
+
+        if not self.profiles:
             logger.warning("no ftp section in config")
 
-    def get_ftp(self):
-        if self.config["ftps"]:
-            ftp_conn = FTP_TLS(self.config["server"])
+        self.current_profile: Optional[Dict[str, Any]] = (
+            self.profiles[0] if self.profiles else None
+        )
+        self.ftp = self.current_profile["session"]
 
-        else:
-            ftp_conn = FTP(self.config["server"])
-
-        ftp_conn.login(self.config["username"], self.config["password"])
-
-        return ftp_conn
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                self.ftp = profile["session"]
+                return
+        logger.warning(f"Profile {profile_name} not found")
 
     def upload_file(self, local_file, remote_folder, remote_file=None):
         if not remote_file:
             remote_file = Path(local_file).name
 
-        self._create_remote_dir_tree(full_path=f"{remote_folder}{remote_file}")
+        self._create_remote_dir_tree(full_path=f"/{remote_folder}{remote_file}")
 
         self.ftp.cwd(remote_folder)
 
@@ -59,4 +73,4 @@ class FTPInterface:
             except Exception as e:
                 logger.warning(f"Error while creating remote directory: {e}")
                 self.ftp.mkd(part)
-                self.ftp.cwd(part)
+                self.ftp.cwd(part)

{datamarket-0.9.51.dist-info → datamarket-0.9.52.dist-info}/METADATA

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: datamarket
-Version: 0.9.51
+Version: 0.9.52
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
+License-File: LICENSE
 Author: DataMarket
 Author-email: techsupport@datamarket.es
 Requires-Python: >=3.12,<3.13

{datamarket-0.9.51.dist-info → datamarket-0.9.52.dist-info}/RECORD

@@ -4,8 +4,9 @@ datamarket/exceptions/main.py,sha256=MP5ql6M7DoMbBf-Dg_2ohcUFdWXgzv-dXHntPPit31s
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamarket/interfaces/alchemy.py,sha256=mQwjDqBpz1QHRV2JTCALvn5iK_ky69oE2Gw-EtRXsqQ,14664
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
+datamarket/interfaces/azure.py,sha256=4lxjL4O4nGO8aDmzY8m9x6vzgYpzQaBWwhERrTuwtqA,4936
 datamarket/interfaces/drive.py,sha256=3nhx3THr2SHNWKYwme9F2nPpvsqyEMFIxz0whF2FjHk,4840
-datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
+datamarket/interfaces/ftp.py,sha256=t4zU3ccKo2-8R3nZvM0gqgWbLMRr21jYS5bYpX0jjWk,2772
 datamarket/interfaces/nominatim.py,sha256=HLk0FcdfbOVCF_i71l-Hlb17swL0W1a3Gg2n5OLD0tM,15507
 datamarket/interfaces/peerdb.py,sha256=cwYwvO740GyaPo9zLAwJsf3UeJDGDiYzjQVM9Q6s-_g,23652
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
@@ -26,7 +27,7 @@ datamarket/utils/strings/normalization.py,sha256=rj0wfJSjqcCRp-ruHqc5pylO3_TOmY5
 datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.51.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.51.dist-info/METADATA,sha256=UPV2cxDlddvKHiKY5tSt-dDkA7reLhyIX1KAIfcjxag,7326
-datamarket-0.9.51.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.9.51.dist-info/RECORD,,
+datamarket-0.9.52.dist-info/METADATA,sha256=80X3F8UhM8-vlWqkp3_72Tmh0TzGJudtngDOt5x7Kcc,7348
+datamarket-0.9.52.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+datamarket-0.9.52.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.52.dist-info/RECORD,,

{datamarket-0.9.51.dist-info → datamarket-0.9.52.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any