sibi-dst 0.3.34__tar.gz → 0.3.36__tar.gz

{sibi_dst-0.3.34 → sibi_dst-0.3.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.34
+Version: 0.3.36
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
+Requires-Dist: boto3 (>=1.36.3,<2.0.0)
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)

{sibi_dst-0.3.34 → sibi_dst-0.3.36}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.34"
+version = "0.3.36"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -43,6 +43,7 @@ geopandas = "^1.0.1"
 osmnx = "^2.0.1"
 gunicorn = "^23.0.0"
 uvicorn-worker = "^0.3.0"
+boto3 = "^1.36.3"
 
 
 [build-system]

{sibi_dst-0.3.34 → sibi_dst-0.3.36}/sibi_dst/utils/data_wrapper.py

@@ -205,6 +205,7 @@ class DataWrapper:
         try:
             # Get file info
             info = self.fs.info(file_path)
+            self.logger.info(f"File info for {file_path}: {info}")
 
             # Determine the modification time from available keys
             file_modification_time = None
@@ -218,6 +219,7 @@ class DataWrapper:
                     info["LastModified"] if isinstance(info["LastModified"], datetime.datetime)
                     else datetime.datetime.strptime(info["LastModified"], "%Y-%m-%dT%H:%M:%S.%fZ")
                 )
+                self.logger.info(f"S3 File modification time: {file_modification_datetime}")
             else:
                 self.logger.warning(f"Modification time not available for {file_path}.")
                 return True  # Assume file is too old if we cannot determine its age
@@ -233,10 +235,12 @@ class DataWrapper:
 
         except FileNotFoundError:
             self.logger.warning(f"File {file_path} not found.")
+            if self.ignore_missing:
+                return False
             return True  # File is considered old if it doesn't exist
         except Exception as e:
             self.logger.error(f"Error checking file age for {file_path}: {str(e)}")
-            return True  #
+            return True
 
     def process_date(self, date: datetime.date):
         """

{sibi_dst-0.3.34 → sibi_dst-0.3.36}/sibi_dst/utils/parquet_saver.py

@@ -1,9 +1,17 @@
+import base64
 from pathlib import Path
 from typing import Optional
 
 import pyarrow as pa
 import fsspec
+import warnings
+import hashlib
+from s3fs import S3FileSystem
 
+from fsspec import filesystem
+
+# Suppress the specific UserWarning message
+warnings.filterwarnings("ignore")
 from sibi_dst.utils import Logger
 
 
@@ -20,8 +28,8 @@ class ParquetSaver:
         self.parquet_storage_path = parquet_storage_path.rstrip("/")
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
-        # Default to the local filesystem if `fs` is not provided
-        self.fs = fs or fsspec.filesystem("file")
+        self.fs = fs
+        self.protocol = self.parquet_storage_path.split(":")[0]
 
     def save_to_parquet(self, parquet_filename: Optional[str] = None, clear_existing=True):
         """
@@ -30,7 +38,7 @@ class ParquetSaver:
         :param clear_existing: Whether to clear existing files in the target directory.
         """
         full_path = self._construct_full_path(parquet_filename)
-
+        self.logger.info(f"Save method for :{full_path}")
         # Ensure directory exists and clear if necessary
         self._ensure_directory_exists(full_path, clear_existing=clear_existing)
 
@@ -38,7 +46,9 @@ class ParquetSaver:
         schema = self._define_schema()
         self._convert_dtypes(schema)
         self._save_dataframe_to_parquet(full_path, schema)
-        self.fs.close()
+        # Close the filesystem if the close method exists
+        if hasattr(self.fs, 'close') and callable(getattr(self.fs, 'close', None)):
+            self.fs.close()
 
     def _define_schema(self) -> pa.Schema:
         """Define a PyArrow schema dynamically based on df_result column types."""
@@ -97,27 +107,90 @@ class ParquetSaver:
         :param clear_existing: Whether to clear existing files/directories.
         """
         directory = "/".join(full_path.split("/")[:-1])
-
         if self.fs.exists(directory):
+            self.logger.info(f"Directory already exists: {directory}")
             if clear_existing:
-                self.logger.info(f"Clearing existing directory: {directory}")
-                self.fs.rm(directory, recursive=True)
-        else:
+                self._clear_directory(directory)
+
+        if not self.fs.exists(directory):
             self.logger.info(f"Creating directory: {directory}")
             self.fs.mkdirs(directory, exist_ok=True)
 
     def _save_dataframe_to_parquet(self, full_path: str, schema: pa.Schema):
         """Save the DataFrame to Parquet using the specified schema."""
-        if self.fs.exists(full_path):
-            self.logger.info(f"Overwriting existing file: {full_path}")
-            self.fs.rm(full_path, recursive=True)
-
-        self.logger.info(f"Saving Parquet file to: {full_path}")
+        #self._clear_directory(full_path)
+        self.logger.info(f"Saving/Overwrite Parquet file to: {full_path}")
         self.df_result.to_parquet(
-            full_path,
-            engine="pyarrow",
-            schema=schema,
-            storage_options=self.fs.storage_options if hasattr(self.fs, "storage_options") else None,
-            write_index=False,
+             path=full_path,
+             engine="pyarrow",
+             schema=schema,
+             #overwrite=True,
+             filesystem=self.fs,
+             write_index=False,
         )
 
+    def _clear_directory(self, directory: str):
+        """
+        Clears the specified directory by removing all the files within it. If the
+        directory is not associated with the "s3" protocol, it will be removed using
+        the local filesystem's functionality. For directories using the "s3" protocol,
+        the bucket name and prefix are parsed, and files are deleted using the S3
+        client's `delete_objects` method.
+
+        :param directory: The directory path to clear. If the protocol is not "s3", it
+                          represents a local filesystem path. Otherwise, it is assumed
+                          to be an S3 path (e.g., "s3://bucket_name/prefix").
+        :type directory: str
+        """
+        if self.protocol != "s3":
+            if self.fs.exists(directory):
+                self.logger.info(f"Clearing directory: {directory}")
+                self.fs.rm(directory, recursive=True)
+            return
+        # Parse bucket name and prefix
+        bucket_name, prefix = self._parse_s3_path(directory)
+
+        # List files in the directory
+        files = self.fs.ls(directory, detail=True)
+        if not files:
+            self.logger.info(f"No files to delete in directory: {directory}")
+            return
+
+        # Generate the delete payload
+        objects_to_delete = [{"Key": file["name"].replace(f"{bucket_name}/", "", 1)} for file in files]
+        delete_payload = {
+            "Objects": objects_to_delete,
+            "Quiet": True,
+        }
+
+        # Calculate Content-MD5
+        payload_bytes = str(delete_payload).encode("utf-8")
+        md5_hash = hashlib.md5(payload_bytes).digest()
+        content_md5 = base64.b64encode(md5_hash).decode("utf-8")
+
+        # Use the underlying s3 client to delete objects
+        try:
+            self.fs.s3.delete_objects(
+                Bucket=bucket_name,
+                Delete=delete_payload,
+                ContentMD5=content_md5,
+            )
+            self.logger.info(f"Successfully deleted files in {directory}.")
+        except Exception as e:
+            self.logger.error(f"Failed to delete objects in {directory}: {e}")
+            pass
+
+    @staticmethod
+    def _parse_s3_path(s3_path: str):
+        """
+        Parse an S3 path into bucket name and prefix.
+        :param s3_path: Full S3 path (e.g., s3://bucket-name/path/).
+        :return: Tuple of bucket name and prefix.
+        """
+        if not s3_path.startswith("s3://"):
+            raise ValueError("Invalid S3 path. Must start with 's3://'.")
+        path_parts = s3_path[5:].split("/", 1)
+        bucket_name = path_parts[0]
+        prefix = path_parts[1] if len(path_parts) > 1 else ""
+        return bucket_name, prefix
+

{sibi_dst-0.3.34 → sibi_dst-0.3.36}/sibi_dst/utils/storage_manager.py

@@ -15,6 +15,7 @@ class StorageManager:
         self.storage_path = storage_path.rstrip("/")
         self.fs_type = fs_type
         self.fs_options = fs_options or {}
+
         self.fs = fsspec.filesystem(fs_type, **self.fs_options)
         self.depot_paths = {}
         self.depot_name = None