datamarket 0.9.51__tar.gz → 0.9.53__tar.gz

{datamarket-0.9.51 → datamarket-0.9.53}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: datamarket
-Version: 0.9.51
+Version: 0.9.53
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
+License-File: LICENSE
 Author: DataMarket
 Author-email: techsupport@datamarket.es
 Requires-Python: >=3.12,<3.13

{datamarket-0.9.51 → datamarket-0.9.53}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.51"
+version = "0.9.53"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

datamarket-0.9.53/src/datamarket/interfaces/azure.py

@@ -0,0 +1,127 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from azure.storage.blob import BlobServiceClient
+from pendulum import now
+
+########################################################################################################################
+# CLASSES
+
+logger = logging.getLogger(__name__)
+
+
+class AzureBlobInterface:
+    def __init__(self, config):
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("azure:"):
+                profile_name = section.split(":", 1)[1]
+                connection_string = self.config[section].get("connection_string")
+                container_name = self.config[section].get("container_name")
+
+                self.profiles.append(
+                    {
+                        "profile": profile_name,
+                        "container_name": container_name,
+                        "session": BlobServiceClient.from_connection_string(
+                            connection_string
+                        ).get_container_client(container_name),
+                    }
+                )
+
+        if not self.profiles:
+            logger.warning("No Azure profiles found in config file")
+        self.current_profile: Optional[Dict[str, Any]] = (
+            self.profiles[0] if self.profiles else None
+        )
+
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                return
+        logger.warning(f"Profile {profile_name} not found")
+
+    def upload_file(
+        self,
+        local_file,
+        remote_folder,
+        remote_file=None,
+        upload_file_info=False,
+        **kwargs,
+    ):
+        if not remote_file:
+            remote_file = Path(local_file).name
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        with open(local_file, "rb") as data:
+            blob_client.upload_blob(data, overwrite=True)
+
+        if upload_file_info:
+            self.upload_file_info(remote_path, **kwargs)
+
+    def upload_file_info(self, remote_path, **kwargs):
+        summary_file = remote_path.split(".")[0] + "_resumen.csv"
+        blob_client = self.current_profile["session"].get_blob_client(summary_file)
+
+        new_record = {
+            "file": remote_path,
+            "num_rows": kwargs["num_rows"],
+            "schema_version": kwargs["schema_version"],
+            "upload_date": now(tz="Europe/Madrid").to_datetime_string(),
+        }
+
+        new_record_str = "file,num_rows,schema_version,upload_date\n"
+        new_record_str += ",".join([str(v) for v in new_record.values()]) + "\n"
+
+        blob_client.upload_blob(new_record_str, overwrite=True)
+
+    def download_file(self, local_file, remote_path):
+        blob_client = self.current_profile["session"].get_blob_client(remote_path)
+        blob_data = blob_client.download_blob()
+        with open(local_file, "wb") as f:
+            blob_data.readinto(f)
+
+    def check_file_exists_and_not_empty(self, remote_file, remote_folder):
+        """
+        Checks if a blob exists in the specified folder and has a size greater than 100 bytes.
+
+        Args:
+            remote_file (str): The name of the file (blob) to check.
+            remote_folder (str): The folder (prefix) where the file is located.
+
+        Returns:
+            bool: True if the blob exists and has a size greater than 100, False otherwise.
+        """
+
+        remote_path = f"{remote_folder}/{remote_file}" if remote_folder else remote_file
+
+        try:
+            blob_client = self.current_profile["session"].get_blob_client(remote_path)
+            if blob_client.exists():
+                properties = blob_client.get_blob_properties()
+                if properties.size > 100:  # Check if size is greater than 100 bytes
+                    logger.debug(
+                        f"Blob '{remote_path}' exists and is not empty (size: {properties.size})."
+                    )
+                    return True
+                else:
+                    logger.debug(
+                        f"Blob '{remote_path}' exists but size ({properties.size}) is not > 100 bytes."
+                    )
+                    return False
+            else:
+                logger.debug(f"Blob '{remote_path}' does not exist.")
+                return False
+        except Exception as e:
+            logger.error(f"Error checking blob '{remote_path}': {e}")
+            # In case of error, assume it doesn't exist or is empty to allow upload attempt
+            return False

datamarket-0.9.53/src/datamarket/interfaces/ftp.py

@@ -0,0 +1,76 @@
+########################################################################################################################
+# IMPORTS
+
+import logging
+from ftplib import FTP, FTP_TLS
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+########################################################################################################################
+# CLASSES
+
+logger = logging.getLogger(__name__)
+
+
+class FTPInterface:
+    def __init__(self, config):
+        self.profiles: List[Dict[str, Any]] = []
+        self.config = config
+        for section in getattr(self.config, "sections", lambda: [])():
+            if section.startswith("ftp:"):
+                profile_name = section.split(":", 1)[1]
+                ftps = self.config[section]["ftps"].lower() == "true"
+                ftp_conn = (
+                    FTP_TLS(self.config[section]["server"])
+                    if ftps
+                    else FTP(self.config[section]["server"])
+                )  # noqa: S321
+                ftp_conn.login(
+                    self.config[section]["username"], self.config[section]["password"]
+                )
+                self.profiles.append({"profile": profile_name, "session": ftp_conn})
+
+        if not self.profiles:
+            logger.warning("no ftp section in config")
+
+        self.current_profile: Optional[Dict[str, Any]] = (
+            self.profiles[0] if self.profiles else None
+        )
+        self.ftp = self.current_profile["session"]
+
+    def switch_profile(self, profile_name: str) -> None:
+        for profile in self.profiles:
+            if profile["profile"] == profile_name:
+                self.current_profile = profile
+                self.ftp = profile["session"]
+                return
+        logger.warning(f"Profile {profile_name} not found")
+
+    def upload_file(self, local_file, remote_folder, remote_file=None):
+        if not remote_file:
+            remote_file = Path(local_file).name
+
+        self._create_remote_dir_tree(full_path=f"/{remote_folder}{remote_file}")
+
+        self.ftp.cwd(f"/{remote_folder}")
+
+        with open(local_file, "rb") as f:
+            self.ftp.storbinary(f"STOR {remote_file}", f)
+
+    def download_file(self, local_file, remote_file):
+        with open(local_file, "wb") as f:
+            self.ftp.retrbinary(f"RETR {remote_file}", f.write)
+
+    def _create_remote_dir_tree(self, full_path):
+        dir_tree = full_path.split("/")[0:-1]  # Exclude filename
+
+        for part in dir_tree:
+            if not part:
+                continue
+
+            try:
+                self.ftp.cwd(part)
+            except Exception as e:
+                logger.warning(f"Error while creating remote directory: {e}")
+                self.ftp.mkd(part)
+                self.ftp.cwd(part)

datamarket-0.9.51/src/datamarket/interfaces/ftp.py

@@ -1,62 +0,0 @@
-########################################################################################################################
-# IMPORTS
-
-import logging
-from ftplib import FTP, FTP_TLS
-from pathlib import Path
-
-from dynaconf import Dynaconf
-
-########################################################################################################################
-# CLASSES
-
-logger = logging.getLogger(__name__)
-
-class FTPInterface:
-    def __init__(self, config: Dynaconf):
-        if "ftp" in config:
-            self.config = config["ftp"]
-
-            self.ftp = self.get_ftp()
-        else:
-            logger.warning("no ftp section in config")
-
-    def get_ftp(self):
-        if self.config["ftps"]:
-            ftp_conn = FTP_TLS(self.config["server"])
-
-        else:
-            ftp_conn = FTP(self.config["server"])
-
-        ftp_conn.login(self.config["username"], self.config["password"])
-
-        return ftp_conn
-
-    def upload_file(self, local_file, remote_folder, remote_file=None):
-        if not remote_file:
-            remote_file = Path(local_file).name
-
-        self._create_remote_dir_tree(full_path=f"{remote_folder}{remote_file}")
-
-        self.ftp.cwd(remote_folder)
-
-        with open(local_file, "rb") as f:
-            self.ftp.storbinary(f"STOR {remote_file}", f)
-
-    def download_file(self, local_file, remote_file):
-        with open(local_file, "wb") as f:
-            self.ftp.retrbinary(f"RETR {remote_file}", f.write)
-
-    def _create_remote_dir_tree(self, full_path):
-        dir_tree = full_path.split("/")[0:-1]  # Exclude filename
-
-        for part in dir_tree:
-            if not part:
-                continue
-
-            try:
-                self.ftp.cwd(part)
-            except Exception as e:
-                logger.warning(f"Error while creating remote directory: {e}")
-                self.ftp.mkd(part)
-                self.ftp.cwd(part)