tgedr-dataops 0.0.37__py3-none-any.whl → 1.0.2__py3-none-any.whl

tgedr_dataops/source/pd_df_s3_source.py

@@ -0,0 +1,130 @@
+"""Module for reading pandas DataFrames from S3 sources.
+
+This module provides:
+- PdDfS3Source: class for reading CSV and Excel files from S3 into pandas DataFrames.
+"""
+from io import StringIO
+import logging
+from typing import Any, ClassVar
+import pandas as pd
+
+from tgedr_dataops.commons.utils_fs import process_s3_url
+from tgedr_dataops.source.abstract_s3_file_source import AbstractS3FileSource
+from tgedr_dataops_abs.source import SourceException
+
+logger = logging.getLogger()
+
+
+class PdDfS3Source(AbstractS3FileSource):
+    """class used to read a pandas dataframe from a csv file in s3."""
+
+    CONTEXT_KEY_FILE_FORMAT = "file_format"
+    CONTEXT_KEY_SEPARATOR = "sep"
+    CONTEXT_KEY_NO_HEADER = "no_header"
+    CONTEXT_KEY_COLUMN_NAMES = "column_names"
+    CONTEXT_KEY_SCHEMA_TYPES = "schema_types"
+    DEFAULT_FORMAT = "csv"
+    FORMATS: ClassVar[list[str]] = ["csv", "xlsx"]
+    DEFAULT_SEPARATOR = ","
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the PdDfS3Source with optional configuration.
+
+        Parameters
+        ----------
+        config : dict[str, Any], optional
+            Configuration dictionary for S3 connection, by default None.
+        """
+        super().__init__(config=config)
+
+    def get(self, context: dict[str, Any] | None = None) -> pd.DataFrame:
+        """Retrieve and load a pandas DataFrame from S3.
+
+        Supports CSV and Excel formats.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing:
+            - 'source' or CONTEXT_KEY_URL: S3 URL
+            - 'file_format': File format (csv or xlsx), defaults to csv
+            - 'sep': CSV separator, defaults to comma
+            - 'no_header': If present, CSV has no header row
+            - 'column_names': List of column names for CSV
+            - 'schema_types': Dictionary of column types for CSV
+
+        Returns
+        -------
+        pd.DataFrame
+            The loaded pandas DataFrame.
+
+        Raises
+        ------
+        SourceException
+            If source URL is missing or if file_format is unsupported.
+        """
+        logger.info(f"[get|in] ({context})")
+        result: pd.DataFrame = None
+
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        _format: str = self.DEFAULT_FORMAT
+        if self.CONTEXT_KEY_FILE_FORMAT in context:
+            _format = context[self.CONTEXT_KEY_FILE_FORMAT]
+            if _format not in self.FORMATS:
+                raise SourceException(f"[get] invalid format: {_format}")
+
+        result = self.__read_csv(context=context) if "csv" == _format else self.__read_excel(context=context)
+
+        logger.info(f"[get|out] => {result}")
+        return result
+
+    def __read_csv(self, context: dict[str, Any] | None = None) -> pd.DataFrame:
+        """Read a CSV file from S3 into a pandas DataFrame.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary with S3 URL and CSV reading options.
+
+        Returns
+        -------
+        pd.DataFrame
+            The loaded DataFrame from CSV.
+        """
+        logger.info(f"[__read_csv|in] ({context})")
+
+        _, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
+
+        obj = self._client.get_object(Bucket=bucket, Key=key)
+        data = obj["Body"].read().decode("utf-8")
+
+        header = 0 if self.CONTEXT_KEY_NO_HEADER not in context else None
+        names = context.get(self.CONTEXT_KEY_COLUMN_NAMES, None)
+        dtype = context.get(self.CONTEXT_KEY_SCHEMA_TYPES, None)
+        sep = context.get(self.CONTEXT_KEY_SEPARATOR, self.DEFAULT_SEPARATOR)
+
+        result: pd.DataFrame = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
+
+        logger.info(f"[__read_csv|out] => {result}")
+        return result
+
+    def __read_excel(self, context: dict[str, Any] | None = None) -> pd.DataFrame:
+        """Read an Excel file from S3 into a pandas DataFrame.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing S3 URL.
+
+        Returns
+        -------
+        pd.DataFrame
+            The loaded DataFrame from Excel.
+        """
+        logger.info(f"[__read_excel|in] ({context})")
+        src = context[self.CONTEXT_KEY_URL]
+        result: pd.DataFrame = pd.read_excel(src, engine="openpyxl")
+        logger.info(f"[__read_excel|out] => {result}")
+        return result

{tgedr/dataops → tgedr_dataops}/source/s3_file_copy.py

@@ -1,17 +1,22 @@
+"""Module for copying files between S3 buckets.
+
+This module provides:
+- S3FileCopy: class for copying objects/files from one S3 bucket to another
+"""
 import logging
-import os
-from typing import Any, Dict, List, Optional
-from tgedr.dataops.commons.s3_connector import S3Connector
+from pathlib import Path
+from typing import Any
+from tgedr_dataops.commons.s3_connector import S3Connector
 
-from tgedr.dataops.source.source import Source, SourceException
-from tgedr.dataops.commons.utils_fs import process_s3_path, resolve_s3_protocol
+from tgedr_dataops_abs.source import Source, SourceException
+from tgedr_dataops.commons.utils_fs import process_s3_path, resolve_s3_protocol
 
 
 logger = logging.getLogger(__name__)
 
 
 class S3FileCopy(Source, S3Connector):
-    """class used to copy objects/files from an s3 bucket to another s3 bucket"""
+    """class used to copy objects/files from an s3 bucket to another s3 bucket."""
 
     CONTEXT_KEY_SOURCE = "source"
     CONTEXT_KEY_TARGET = "target"
@@ -19,14 +24,38 @@ class S3FileCopy(Source, S3Connector):
     CONTEXT_KEY_SUFFIX = "suffix"
     CONTEXT_KEY_PRESERVE_SOURCE_KEY = "preserve_source_key"
 
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the S3FileCopy source.
+
+        Parameters
+        ----------
+        config : dict[str, Any], optional
+            Configuration dictionary containing optional AWS credentials.
+        """
         Source.__init__(self, config=config)
         S3Connector.__init__(self)
 
-    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
+    def list(self, context: dict[str, Any] | None = None) -> list[str]:
+        """List objects in an S3 bucket matching the given prefix.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' (source S3 URL) and optionally 'suffix' to filter results.
+
+        Returns
+        -------
+        list[str]
+            List of S3 URLs matching the criteria.
+
+        Raises
+        ------
+        SourceException
+            If the 'source' key is missing from the context.
+        """
         logger.info(f"[list|in] ({context})")
 
-        result: List[str] = []
+        result: list[str] = []
         if self.CONTEXT_KEY_SOURCE not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
 
@@ -48,10 +77,27 @@ class S3FileCopy(Source, S3Connector):
         logger.info(f"[list|out] => result len: {len(result)}")
         return result
 
-    def get(self, context: Optional[Dict[str, Any]] = None) -> Any:
+    def get(self, context: dict[str, Any] | None = None) -> Any:
+        """Copy a file from one S3 location to another.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' (source S3 URL) and 'target' (target S3 URL).
+
+        Returns
+        -------
+        Any
+            The target S3 URL.
+
+        Raises
+        ------
+        SourceException
+            If source or target context is missing.
+        """
         logger.info(f"[get|in] ({context})")
 
-        result: List[str] = []
+        result: list[str] = []
         if self.CONTEXT_KEY_FILES not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_FILES}")
         if self.CONTEXT_KEY_TARGET not in context:
@@ -59,9 +105,7 @@ class S3FileCopy(Source, S3Connector):
 
         preserve_source_key = False
         if self.CONTEXT_KEY_PRESERVE_SOURCE_KEY in context:
-            preserve_source_key = (
-                True if (str(context[self.CONTEXT_KEY_PRESERVE_SOURCE_KEY]).lower() in ["1", "true"]) else False
-            )
+            preserve_source_key = str(context[self.CONTEXT_KEY_PRESERVE_SOURCE_KEY]).lower() in ["1", "true"]
         logger.info(f"[get] preserve_source_key: {preserve_source_key}")
 
         target_bucket, target_key = process_s3_path(context[self.CONTEXT_KEY_TARGET])
@@ -69,31 +113,23 @@ class S3FileCopy(Source, S3Connector):
         protocol = "" if s3_protocol is None else s3_protocol
 
         files = context[self.CONTEXT_KEY_FILES]
-        multiple_files: bool = True if (0 < len(files)) else False
-
         for file in files:
             src_bucket, src_key = process_s3_path(file)
-
             src = {"Bucket": src_bucket, "Key": src_key}
 
-            if True == preserve_source_key:
-                key = os.path.join(target_key, src_key)
+            target_key = target_key.rstrip("/") if target_key.endswith("/") else target_key
+            if preserve_source_key:
+                key = str(Path(target_key) / src_key)
             else:
-                key = target_key
-                if not multiple_files:
-                    if target_key.endswith("/"):
-                        src_key_leaf = os.path.basename(src_key)
-                        key += src_key_leaf
-                else:
-                    src_key_leaf = os.path.basename(src_key)
-                    key = os.path.join(key, src_key_leaf)
+                src_key_leaf = Path(src_key).name
+                key = str(Path(target_key) / src_key_leaf)
 
             logger.info(
                 f"[get] copying... src bucket: {src_bucket}   src key: {src_key}   target bucket: {target_bucket}   target key: {key}"
             )
             self._client.copy(src, target_bucket, key)
 
-            result.append(os.path.join(protocol, target_bucket, key))
+            result.append(str(Path(protocol) / target_bucket / key))
 
         logger.info(f"[get|out] => {result}")
         return result

tgedr_dataops/source/s3_file_extended_source.py

@@ -0,0 +1,68 @@
+"""S3 file extended source module for retrieving objects from S3 with metadata support.
+
+This module provides:
+- S3FileExtendedSource: class for retrieving S3 objects/files with metadata extraction
+"""
+import logging
+from typing import Any, ClassVar
+
+from tgedr_dataops.source.s3_file_source import S3FileSource
+from tgedr_dataops_abs.source import SourceException
+from tgedr_dataops.commons.utils_fs import process_s3_path
+
+
+logger = logging.getLogger(__name__)
+
+
+class S3FileExtendedSource(S3FileSource):
+    """Class used to retrieve objects/files from S3 bucket to local filesystem."""
+
+    METADATA_KEYS: ClassVar[list[str]] = ["LastModified", "ContentLength", "ETag", "VersionId", "ContentType"]
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the S3FileExtendedSource.
+
+        Parameters
+        ----------
+        config : dict[str, Any], optional
+            Configuration dictionary containing optional AWS credentials.
+        """
+        super().__init__(config=config)
+
+    def get_metadata(self, context: dict[str, Any] | None = None) -> dict[str, Any]:
+        """Retrieve metadata for an S3 object.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' S3 URL.
+
+        Returns
+        -------
+        dict[str, Any]
+            Dictionary containing object metadata including size, last modified time, etc.
+
+        Raises
+        ------
+        SourceException
+            If source context is missing.
+        """
+        logger.info(f"[get_metadata|in] ({context})")
+
+        result: dict[str, Any] = {}
+        if self.CONTEXT_KEY_SOURCE not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
+
+        bucket, key = process_s3_path(context[self.CONTEXT_KEY_SOURCE])
+
+        o = self._client.head_object(Bucket=bucket, Key=key)
+
+        for key in list(o.keys()):
+            if key in self.METADATA_KEYS:
+                if key == "LastModified":
+                    result[key] = int(o[key].timestamp())
+                else:
+                    result[key] = o[key]
+
+        logger.info(f"[get_metadata|out] => result len: {len(result)}")
+        return result

{tgedr/dataops → tgedr_dataops}/source/s3_file_source.py

@@ -1,31 +1,60 @@
+"""S3 file source module for retrieving objects from S3 buckets.
+
+This module provides:
+- S3FileSource: A source implementation for listing and downloading files from S3.
+"""
 import logging
-import os
-from typing import Any, Dict, List, Optional
-from tgedr.dataops.commons.s3_connector import S3Connector
+from pathlib import Path
+from typing import Any
+from tgedr_dataops.commons.s3_connector import S3Connector
 
-from tgedr.dataops.source.source import Source, SourceException
-from tgedr.dataops.commons.utils_fs import remove_s3_protocol, resolve_s3_protocol
+from tgedr_dataops_abs.source import Source, SourceException
+from tgedr_dataops.commons.utils_fs import remove_s3_protocol, resolve_s3_protocol
 
 
 logger = logging.getLogger(__name__)
 
 
 class S3FileSource(Source, S3Connector):
-    """class used to retrieve objects/files from s3 bucket to local fs location"""
+    """class used to retrieve objects/files from s3 bucket to local fs location."""
 
     CONTEXT_KEY_SOURCE = "source"
     CONTEXT_KEY_TARGET = "target"
     CONTEXT_KEY_FILES = "files"
     CONTEXT_KEY_SUFFIX = "suffix"
 
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the S3FileSource.
+
+        Parameters
+        ----------
+        config : dict[str, Any], optional
+            Configuration dictionary containing optional AWS credentials.
+        """
         Source.__init__(self, config=config)
         S3Connector.__init__(self)
 
-    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
+    def list(self, context: dict[str, Any] | None = None) -> list[str]:
+        """List files in the S3 bucket.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' (S3 URL) and optionally 'suffix' for filtering.
+
+        Returns
+        -------
+        list[str]
+            List of S3 object URLs in the source bucket/prefix.
+
+        Raises
+        ------
+        SourceException
+            If source context is missing.
+        """
         logger.info(f"[list|in] ({context})")
 
-        result: List[str] = []
+        result: list[str] = []
         if self.CONTEXT_KEY_SOURCE not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
 
@@ -50,42 +79,49 @@ class S3FileSource(Source, S3Connector):
         logger.info(f"[list|out] => result len: {len(result)}")
         return result
 
-    def get(self, context: Optional[Dict[str, Any]] = None) -> Any:
+    def get(self, context: dict[str, Any] | None = None) -> Any:
+        """Download a file from S3 to local filesystem.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' (S3 URL) and 'target' (local path).
+
+        Returns
+        -------
+        Any
+            The local target file path.
+
+        Raises
+        ------
+        SourceException
+            If source or target context is missing.
+        """
         logger.info(f"[get|in] ({context})")
 
-        result: List[str] = []
+        result: list[str] = []
         if self.CONTEXT_KEY_FILES not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_FILES}")
         if self.CONTEXT_KEY_TARGET not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_TARGET}")
 
-        preserve_structure = True
-
         files = context[self.CONTEXT_KEY_FILES]
         target = context[self.CONTEXT_KEY_TARGET]
 
-        target_is_dir: bool = False
-        if os.path.isdir(target):
-            target_is_dir = True
+        target_is_dir: bool = Path(target).is_dir()
+        target = target.rstrip("/") if target.endswith("/") else target
 
         for file in files:
             path_elements = remove_s3_protocol(file).split("/")
             bucket = path_elements[0]
             key = "/".join(path_elements[1:])
             filename = path_elements[-1]
-
-            if target_is_dir:
-                if preserve_structure:
-                    local_file = os.path.join(target, key)
-                else:
-                    local_file = os.path.join(target, filename)
-            else:
-                local_file = target
+            local_file = str(Path(target) / filename) if target_is_dir else target
 
             # assure we have that path there
-            local_folder = os.path.dirname(local_file)
-            if not os.path.isdir(local_folder):
-                os.makedirs(local_folder)
+            local_folder = Path(local_file).parent
+            if not local_folder.is_dir():
+                local_folder.mkdir(parents=True)
 
             logger.info(f"[get] bucket: {bucket}   key: {key}   file: {file}   local_file: {local_file}")
             self._client.download_file(Bucket=bucket, Key=key, Filename=local_file)