sibi-dst 0.3.35__py3-none-any.whl → 0.3.37__py3-none-any.whl

sibi_dst/utils/data_wrapper.py

@@ -205,6 +205,7 @@ class DataWrapper:
         try:
             # Get file info
             info = self.fs.info(file_path)
+            self.logger.info(f"File info for {file_path}: {info}")
 
             # Determine the modification time from available keys
             file_modification_time = None
@@ -218,6 +219,7 @@ class DataWrapper:
                     info["LastModified"] if isinstance(info["LastModified"], datetime.datetime)
                     else datetime.datetime.strptime(info["LastModified"], "%Y-%m-%dT%H:%M:%S.%fZ")
                 )
+                self.logger.info(f"S3 File modification time: {file_modification_datetime}")
             else:
                 self.logger.warning(f"Modification time not available for {file_path}.")
                 return True  # Assume file is too old if we cannot determine its age
@@ -233,10 +235,12 @@ class DataWrapper:
 
         except FileNotFoundError:
             self.logger.warning(f"File {file_path} not found.")
+            if self.ignore_missing:
+                return False
             return True  # File is considered old if it doesn't exist
         except Exception as e:
             self.logger.error(f"Error checking file age for {file_path}: {str(e)}")
-            return True  #
+            return True
 
     def process_date(self, date: datetime.date):
         """

sibi_dst/utils/parquet_saver.py

@@ -1,9 +1,17 @@
+import base64
 from pathlib import Path
 from typing import Optional
 
 import pyarrow as pa
 import fsspec
+import warnings
+import hashlib
+from s3fs import S3FileSystem
 
+from fsspec import filesystem
+
+# Suppress the specific UserWarning message
+warnings.filterwarnings("ignore")
 from sibi_dst.utils import Logger
 
 
@@ -20,8 +28,8 @@ class ParquetSaver:
         self.parquet_storage_path = parquet_storage_path.rstrip("/")
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
-        # Default to the local filesystem if `fs` is not provided
-        self.fs = fs or fsspec.filesystem("file")
+        self.fs = fs
+        self.protocol = self.parquet_storage_path.split(":")[0]
 
     def save_to_parquet(self, parquet_filename: Optional[str] = None, clear_existing=True):
         """
@@ -30,7 +38,7 @@ class ParquetSaver:
         :param clear_existing: Whether to clear existing files in the target directory.
         """
         full_path = self._construct_full_path(parquet_filename)
-
+        self.logger.info(f"Save method for :{full_path}")
         # Ensure directory exists and clear if necessary
         self._ensure_directory_exists(full_path, clear_existing=clear_existing)
 
@@ -99,27 +107,90 @@ class ParquetSaver:
         :param clear_existing: Whether to clear existing files/directories.
         """
         directory = "/".join(full_path.split("/")[:-1])
-
         if self.fs.exists(directory):
+            self.logger.info(f"Directory already exists: {directory}")
             if clear_existing:
-                self.logger.info(f"Clearing existing directory: {directory}")
-                self.fs.rm(directory, recursive=True)
-        else:
+                self._clear_directory(directory)
+
+        if not self.fs.exists(directory):
             self.logger.info(f"Creating directory: {directory}")
             self.fs.mkdirs(directory, exist_ok=True)
 
     def _save_dataframe_to_parquet(self, full_path: str, schema: pa.Schema):
         """Save the DataFrame to Parquet using the specified schema."""
-        if self.fs.exists(full_path):
-            self.logger.info(f"Overwriting existing file: {full_path}")
-            self.fs.rm(full_path, recursive=True)
-
-        self.logger.info(f"Saving Parquet file to: {full_path}")
+        #self._clear_directory(full_path)
+        self.logger.info(f"Saving/Overwrite Parquet file to: {full_path}")
         self.df_result.to_parquet(
-            full_path,
-            engine="pyarrow",
-            schema=schema,
-            storage_options=self.fs.storage_options if hasattr(self.fs, "storage_options") else None,
-            write_index=False,
+             path=full_path,
+             engine="pyarrow",
+             schema=schema,
+             #overwrite=True,
+             filesystem=self.fs,
+             write_index=False,
         )
 
+    def _clear_directory(self, directory: str):
+        """
+        Clears the specified directory by removing all the files within it. If the
+        directory is not associated with the "s3" protocol, it will be removed using
+        the local filesystem's functionality. For directories using the "s3" protocol,
+        the bucket name and prefix are parsed, and files are deleted using the S3
+        client's `delete_objects` method.
+
+        :param directory: The directory path to clear. If the protocol is not "s3", it
+                          represents a local filesystem path. Otherwise, it is assumed
+                          to be an S3 path (e.g., "s3://bucket_name/prefix").
+        :type directory: str
+        """
+        if self.protocol != "s3":
+            if self.fs.exists(directory):
+                self.logger.info(f"Clearing directory: {directory}")
+                self.fs.rm(directory, recursive=True)
+            return
+        # Parse bucket name and prefix
+        bucket_name, prefix = self._parse_s3_path(directory)
+
+        # List files in the directory
+        files = self.fs.ls(directory, detail=True)
+        if not files:
+            self.logger.info(f"No files to delete in directory: {directory}")
+            return
+
+        # Generate the delete payload
+        objects_to_delete = [{"Key": file["name"].replace(f"{bucket_name}/", "", 1)} for file in files]
+        delete_payload = {
+            "Objects": objects_to_delete,
+            "Quiet": True,
+        }
+
+        # Calculate Content-MD5
+        payload_bytes = str(delete_payload).encode("utf-8")
+        md5_hash = hashlib.md5(payload_bytes).digest()
+        content_md5 = base64.b64encode(md5_hash).decode("utf-8")
+
+        # Use the underlying s3 client to delete objects
+        try:
+            self.fs.s3.delete_objects(
+                Bucket=bucket_name,
+                Delete=delete_payload,
+                ContentMD5=content_md5,
+            )
+            self.logger.info(f"Successfully deleted files in {directory}.")
+        except Exception as e:
+            self.logger.error(f"Failed to delete objects in {directory}: {e}")
+            pass
+
+    @staticmethod
+    def _parse_s3_path(s3_path: str):
+        """
+        Parse an S3 path into bucket name and prefix.
+        :param s3_path: Full S3 path (e.g., s3://bucket-name/path/).
+        :return: Tuple of bucket name and prefix.
+        """
+        if not s3_path.startswith("s3://"):
+            raise ValueError("Invalid S3 path. Must start with 's3://'.")
+        path_parts = s3_path[5:].split("/", 1)
+        bucket_name = path_parts[0]
+        prefix = path_parts[1] if len(path_parts) > 1 else ""
+        return bucket_name, prefix
+

sibi_dst/utils/storage_manager.py

@@ -15,6 +15,7 @@ class StorageManager:
         self.storage_path = storage_path.rstrip("/")
         self.fs_type = fs_type
         self.fs_options = fs_options or {}
+
         self.fs = fsspec.filesystem(fs_type, **self.fs_options)
         self.depot_paths = {}
         self.depot_name = None

{sibi_dst-0.3.35.dist-info → sibi_dst-0.3.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.35
+Version: 0.3.37
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.35.dist-info → sibi_dst-0.3.37.dist-info}/RECORD

@@ -42,14 +42,14 @@ sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWd
 sibi_dst/utils/clickhouse_writer.py,sha256=syXGN9NG1FS8soHuMj6QNRqTRWi-thuYUF-_BWDc_KI,9883
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_utils.py,sha256=j-lEKt6EJL2fm0z7adcjtVG7yFYLRpQL8xSgh2CVmJg,8769
-sibi_dst/utils/data_wrapper.py,sha256=Ybmn9V7XYuPdUliMz1QrwrXraR_YFOzSr2zJmZOVmWM,16462
+sibi_dst/utils/data_wrapper.py,sha256=V9p1CKvlmT3yyfZ5d2BIJSvz0Ee1TIA4hufpTIYo6v8,16684
 sibi_dst/utils/date_utils.py,sha256=ei7WgzIUk1tRa3sHniaVm_lNmfTGq12b_HzmMV91k18,12407
 sibi_dst/utils/df_utils.py,sha256=OFEtcwVKIilvf9qVf-IfIOHp4jcFAHX5l2IDGudhPZg,10989
 sibi_dst/utils/file_utils.py,sha256=JpsybYj3XvVJisSBeVU6YSaZnYRm4_6YWTI3TLnnY4Y,1257
 sibi_dst/utils/filepath_generator.py,sha256=volVm0SSlBrtZp1RpTHxyui5rj5asNcVsWEBRY5FOUQ,6673
 sibi_dst/utils/log_utils.py,sha256=XUbeXa1JsOlcEJyW8jnBlWo295rLUnuYi-HMzyhHwJg,3145
-sibi_dst/utils/parquet_saver.py,sha256=Wj3jDlAXHACma6ea1zfuH0I3AJS2OVuwBG3lY2yZ27k,5238
-sibi_dst/utils/storage_manager.py,sha256=qHo5vTv-dr1roRr_mOcprSTdlAfH4Q2Dy5tQUz06Pnk,4228
-sibi_dst-0.3.35.dist-info/METADATA,sha256=emeZrFZZi6ixveE_vAwcb_ZD8AU24YnepxB6H1kMSb4,2564
-sibi_dst-0.3.35.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.35.dist-info/RECORD,,
+sibi_dst/utils/parquet_saver.py,sha256=_QkXL3IiC2b4m7sxHpCSeqPwBWxXeiP5sH_WheSMEm4,8042
+sibi_dst/utils/storage_manager.py,sha256=-zlMrRo_6o6mCd_OHknKqNQl7m0I9VW89grDAUO1V5c,4229
+sibi_dst-0.3.37.dist-info/METADATA,sha256=A09_8PykbkfOjpCHIpiij4Wn-nlS2tU6gsR0-MngZUQ,2564
+sibi_dst-0.3.37.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.37.dist-info/RECORD,,