tgedr-dataops 0.0.37__py3-none-any.whl → 1.0.1__py3-none-any.whl

{tgedr/dataops → tgedr_dataops}/commons/s3_connector.py

@@ -1,17 +1,30 @@
+"""AWS S3 connector module.
+
+This module provides the S3Connector class for establishing and managing
+connections to AWS S3 resources.
+"""
 import os
 import boto3
 
 
 class S3Connector:
-    """utility base class to be extended, providing a connection session with aws s3 resources"""
+    """utility base class to be extended, providing a connection session with aws s3 resources."""
 
-    def __init__(self):
+    def __init__(self) -> None:
+        """Initialize the S3Connector with empty session, resource, and client attributes."""
         self.__resource = None
         self.__session = None
         self.__client = None
 
     @property
-    def _session(self):  # pragma: no cover
+    def _session(self) -> boto3.Session:
+        """Get or create a boto3 session.
+
+        Returns
+        -------
+        boto3.Session
+            The boto3 session instance, using credentials if configured.
+        """
         if self.__session is None:
             if "1" == os.getenv("S3_CONNECTOR_USE_CREDENTIALS", default="0"):
                 self.__session = boto3.Session(
@@ -26,13 +39,27 @@ class S3Connector:
         return self.__session
 
     @property
-    def _resource(self):
+    def _resource(self) -> boto3.resources.base.ServiceResource:
+        """Get or create an S3 resource.
+
+        Returns
+        -------
+        boto3.resources.base.ServiceResource
+            The boto3 S3 resource instance.
+        """
         if self.__resource is None:
             self.__resource = self._session.resource("s3")
         return self.__resource
 
     @property
-    def _client(self):
+    def _client(self) -> boto3.client:
+        """Get or create an S3 client.
+
+        Returns
+        -------
+        boto3.client
+            The boto3 S3 client instance.
+        """
         if self.__client is None:
             self.__client = self._session.client("s3")
         return self.__client

tgedr_dataops/commons/utils_fs.py

@@ -0,0 +1,187 @@
+"""Filesystem utility functions for file and path operations.
+
+This module provides utilities for:
+- Creating temporary files and directories
+- Parsing and processing S3 URLs and paths
+- Extracting URL protocols
+- Generating file hashes
+"""
+import tempfile
+import re
+import hashlib
+from pathlib import Path
+
+
+def temp_dir(root: str | None = None, suffix: str | None = None, prefix: str | None = None) -> str:
+    """Create a temporary directory and return its path.
+
+    Parameters
+    ----------
+    root : str | None
+        Directory where the temporary directory will be created.
+    suffix : str | None
+        Suffix for the temporary directory name.
+    prefix : str | None
+        Prefix for the temporary directory name.
+
+    Returns
+    -------
+    str
+        Path to the created temporary directory.
+    """
+    return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=root)
+
+
+def temp_file(
+    root: str | None = None, suffix: str | None = None, prefix: str | None = None,
+    discard_handle: bool = True
+) -> str | tuple[int, str]:
+    """Create a temporary file and return its path or handle and path.
+
+    Parameters
+    ----------
+    root : str | None
+        Directory where the temporary file will be created.
+    suffix : str | None
+        Suffix for the temporary file name.
+    prefix : str | None
+        Prefix for the temporary file name.
+    discard_handle : bool
+        If True, return only the file path. If False, return tuple of (handle, path).
+
+    Returns
+    -------
+    str | tuple[int, str]
+        File path if discard_handle is True, otherwise tuple of (file handle, file path).
+    """
+    h, f = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=root)
+    if discard_handle:
+        return f
+    return (h, f)
+
+
+def resolve_url_protocol(url: str) -> str:
+    """Extract the protocol from a URL.
+
+    Parameters
+    ----------
+    url : str
+        The URL to extract the protocol from.
+
+    Returns
+    -------
+    str
+        The protocol (e.g., 'http://', 'https://') or None if no protocol is found.
+    """
+    result = None
+    group_match = re.search("(.*://).*", url)
+    if group_match is not None:
+        result = group_match.group(1)
+    return result
+
+
+def resolve_s3_protocol(url: str) -> str:
+    """Extract the S3 protocol from a URL.
+
+    Parameters
+    ----------
+    url : str
+        The URL to extract the S3 protocol from.
+
+    Returns
+    -------
+    str
+        The S3 protocol (e.g., 's3://', 's3a://') or None if no S3 protocol is found.
+    """
+    result = None
+    group_match = re.search("(s3[a]?://).*", url)
+    if group_match is not None:
+        result = group_match.group(1)
+    return result
+
+
+def remove_s3_protocol(url: str) -> str:
+    """Remove the S3 protocol prefix from a URL.
+
+    Parameters
+    ----------
+    url : str
+        The S3 URL to remove the protocol from.
+
+    Returns
+    -------
+    str
+        The URL without the S3 protocol prefix (s3:// or s3a://).
+    """
+    if url.startswith("s3://"):
+        result = url[5:]
+    elif url.startswith("s3a://"):
+        result = url[6:]
+    else:
+        result = url
+    return result
+
+
+def process_s3_path(path: str) -> tuple[str, str]:
+    """Extract bucket and key from an S3 path.
+
+    Parameters
+    ----------
+    path : str
+        The S3 path to process (with or without protocol).
+
+    Returns
+    -------
+    tuple[str, str]
+        A tuple containing (bucket, key).
+    """
+    no_protocol_path = remove_s3_protocol(path)
+    path_elements = no_protocol_path.split("/")
+    bucket = path_elements[0]
+    key = "/".join(path_elements[1:])
+    return (bucket, key)
+
+
+def process_s3_url(url: str) -> tuple[str, str, str]:
+    """Extract protocol, bucket, and key from an S3 URL.
+
+    Parameters
+    ----------
+    url : str
+        The S3 URL to process.
+
+    Returns
+    -------
+    tuple[str, str, str]
+        A tuple containing (protocol, bucket, key).
+    """
+    protocol = resolve_s3_protocol(url)
+    no_protocol_url = remove_s3_protocol(url)
+    path_elements = no_protocol_url.split("/")
+    bucket = path_elements[0]
+    key = "/".join(path_elements[1:])
+    return ("" if protocol is None else protocol, bucket, key)
+
+
+def hash_file(filepath: str, hash_func=hashlib.sha256) -> str:  # noqa: ANN001
+    """Generate a hash for a file.
+
+    Args:
+        filepath (str): The path to the file.
+        hash_func: A hashlib hash function, e.g., hashlib.md5().
+
+    Returns:
+        str: The hexadecimal hash string of the file.
+    """
+    # Initialize the hash object
+    hasher = hash_func()
+
+    # Open the file in binary read mode
+    with Path(filepath).open("rb") as file:
+        # Read the file in chunks to avoid using too much memory
+        chunk_size = 8192
+        while chunk := file.read(chunk_size):
+            hasher.update(chunk)
+
+    # Return the hexadecimal digest of the hash
+    return hasher.hexdigest()

tgedr_dataops/quality/pandas_validation.py

@@ -0,0 +1,21 @@
+"""Pandas DataFrame validation implementation module.
+
+This module provides the Pandas-specific implementation of Great Expectations validation.
+"""
+from great_expectations.execution_engine import ExecutionEngine
+from great_expectations.execution_engine import PandasExecutionEngine
+from tgedr_dataops_abs.great_expectations_validation import GreatExpectationsValidation
+
+
+class PandasValidation(GreatExpectationsValidation):
+    """Pandas DataFrame validation implementation."""
+
+    def _get_execution_engine(self, batch_data_dict: dict) -> ExecutionEngine:
+        """Get the execution engine used by the validation implementation.
+
+        Returns
+        -------
+        ExecutionEngine
+            The execution engine instance.
+        """
+        return PandasExecutionEngine(batch_data_dict=batch_data_dict)

tgedr_dataops/sink/local_fs_file_sink.py

@@ -0,0 +1,77 @@
+"""Local filesystem sink implementation for persisting files.
+
+This module provides:
+- LocalFsFileSink: sink class for saving/persisting files to local filesystem.
+"""
+
+import logging
+import shutil
+from pathlib import Path
+from typing import Any
+
+from tgedr_dataops_abs.sink import Sink, SinkException
+
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFsFileSink(Sink):
+    """sink class used to save/persist an object/file to a local fs location."""
+
+    CONTEXT_SOURCE_PATH = "source"
+    CONTEXT_TARGET_PATH = "target"
+
+    def put(self, context: dict[str, Any] | None = None) -> Any:
+        """Copy a file from source to target on local filesystem.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' and 'target' paths.
+
+        Raises
+        ------
+        SinkException
+            If source or target context is missing.
+        """
+        logger.info(f"[put|in] ({context})")
+
+        if self.CONTEXT_SOURCE_PATH not in context:
+            raise SinkException(f"you must provide context for {self.CONTEXT_SOURCE_PATH}")
+        if self.CONTEXT_TARGET_PATH not in context:
+            raise SinkException(f"you must provide context for {self.CONTEXT_TARGET_PATH}")
+
+        source = context[self.CONTEXT_SOURCE_PATH]
+        target = context[self.CONTEXT_TARGET_PATH]
+
+        shutil.copy(source, target)
+        logger.info("[put|out]")
+
+    def delete(self, context: dict[str, Any] | None = None) -> None:
+        """Delete a file or directory from local filesystem.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'target' path.
+
+        Raises
+        ------
+        SinkException
+            If target context is missing or target is neither file nor directory.
+        """
+        logger.info(f"[delete|in] ({context})")
+
+        if self.CONTEXT_TARGET_PATH not in context:
+            raise SinkException(f"you must provide context for {self.CONTEXT_TARGET_PATH}")
+
+        target = Path(context[self.CONTEXT_TARGET_PATH])
+
+        if target.is_file():
+            target.unlink()
+        elif target.is_dir():
+            shutil.rmtree(target)
+        else:
+            raise SinkException(f"[delete] is it a dir or a folder? {target}")
+
+        logger.info("[delete|out]")

{tgedr/dataops → tgedr_dataops}/sink/s3_file_sink.py

@@ -1,26 +1,50 @@
+"""S3 sink implementation for persisting files to S3.
+
+This module provides:
+- S3FileSink: sink class for saving/persisting files to S3 buckets.
+"""
 import logging
-import os
-from typing import Any, Dict, Optional
-from tgedr.dataops.commons.s3_connector import S3Connector
+from pathlib import Path
+from typing import Any
+from tgedr_dataops.commons.s3_connector import S3Connector
 
-from tgedr.dataops.sink.sink import Sink, SinkException
-from tgedr.dataops.commons.utils_fs import remove_s3_protocol
+from tgedr_dataops_abs.sink import Sink, SinkException
+from tgedr_dataops.commons.utils_fs import remove_s3_protocol
 
 
 logger = logging.getLogger(__name__)
 
 
 class S3FileSink(Sink, S3Connector):
-    """sink class used to save/persist a local object/file to an s3 bucket"""
+    """sink class used to save/persist a local object/file to an s3 bucket."""
 
     CONTEXT_SOURCE_PATH = "source"
     CONTEXT_TARGET_PATH = "target"
 
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the S3FileSink.
+
+        Parameters
+        ----------
+        config : dict[str, Any], optional
+            Configuration dictionary.
+        """
         Sink.__init__(self, config=config)
         S3Connector.__init__(self)
 
-    def put(self, context: Optional[Dict[str, Any]] = None) -> Any:
+    def put(self, context: dict[str, Any] | None = None) -> Any:
+        """Upload a local file to S3 bucket.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' (local file) and 'target' (S3 URL) paths.
+
+        Raises
+        ------
+        SinkException
+            If source or target context is missing, or if source is a directory.
+        """
         logger.info(f"[put|in] ({context})")
 
         if self.CONTEXT_SOURCE_PATH not in context:
@@ -29,7 +53,7 @@ class S3FileSink(Sink, S3Connector):
             raise SinkException(f"you must provide context for {self.CONTEXT_TARGET_PATH}")
 
         source = context[self.CONTEXT_SOURCE_PATH]
-        if os.path.isdir(source):
+        if Path(source).is_dir():
             raise SinkException("source can't be a folder, must be a file")
 
         target = remove_s3_protocol(context[self.CONTEXT_TARGET_PATH])
@@ -38,7 +62,7 @@ class S3FileSink(Sink, S3Connector):
         target_key = "/".join(target_elements[1:])
 
         if target_key.endswith("/"):
-            target_file = os.path.basename(source)
+            target_file = Path(source).name
             target_key = target_key + target_file
 
         logger.info(f"[put] uploading {source} to key: {target_key} in bucket: {target_bucket}")
@@ -46,7 +70,19 @@ class S3FileSink(Sink, S3Connector):
 
         logger.info("[put|out]")
 
-    def delete(self, context: Optional[Dict[str, Any]] = None):
+    def delete(self, context: dict[str, Any] | None = None) -> None:
+        """Delete an object from S3 bucket.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'target' (S3 URL) path.
+
+        Raises
+        ------
+        SinkException
+            If target context is missing.
+        """
         logger.info(f"[delete|in] ({context})")
 
         if self.CONTEXT_TARGET_PATH not in context:

tgedr_dataops/source/abstract_s3_file_source.py

@@ -0,0 +1,72 @@
+"""Abstract base class for S3 file sources.
+
+This module provides:
+- AbstractS3FileSource: abstract class for reading file sources from S3.
+"""
+from abc import ABC
+import logging
+from typing import Any
+
+from tgedr_dataops.commons.s3_connector import S3Connector
+from tgedr_dataops.commons.utils_fs import process_s3_url
+from tgedr_dataops_abs.source import Source, SourceException
+
+
+logger = logging.getLogger()
+
+
+class AbstractS3FileSource(Source, S3Connector, ABC):
+    """abstract class used to read file sources from s3."""
+
+    CONTEXT_KEY_URL = "url"
+    CONTEXT_KEY_SUFFIX = "suffix"
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the S3 file source.
+
+        Parameters
+        ----------
+        config : dict[str, Any], optional
+            Configuration dictionary for the source.
+        """
+        Source.__init__(self, config=config)
+        S3Connector.__init__(self)
+
+    def list(self, context: dict[str, Any] | None = None) -> list[str]:
+        """List objects in the S3 bucket.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' S3 URL.
+
+        Returns
+        -------
+        list[str]
+            List of S3 object keys in the source bucket/prefix.
+
+        Raises
+        ------
+        SourceException
+            If source context is missing.
+        """
+        logger.info(f"[list|in] ({context})")
+
+        result: list[str] = []
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
+
+        objs = self._client.list_objects_v2(Bucket=bucket, Prefix=key)
+        result = [
+            (protocol + bucket + "/" + entry["Key"]) for entry in objs["Contents"] if not (entry["Key"]).endswith("/")
+        ]
+
+        if self.CONTEXT_KEY_SUFFIX in context:
+            suffix: str = context[self.CONTEXT_KEY_SUFFIX]
+            result = [f for f in result if f.endswith(suffix)]
+
+        logger.debug(f"[list|out] => {result}")
+        logger.info(f"[list|out] => result len: {len(result)}")
+        return result

tgedr_dataops/source/local_fs_file_source.py

@@ -0,0 +1,108 @@
+"""Local filesystem source implementation for file operations.
+
+This module provides:
+- LocalFsFileSource: A source class for retrieving and listing local files.
+"""
+import logging
+import shutil
+from pathlib import Path
+from typing import Any
+
+from tgedr_dataops_abs.source import Source, SourceException
+
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFsFileSource(Source):
+    """source class used to retrieve local objects/files to a another local fs location."""
+
+    CONTEXT_KEY_SOURCE = "source"
+    CONTEXT_KEY_TARGET = "target"
+    CONTEXT_KEY_SUFFIX = "suffix"
+    CONTEXT_KEY_FILES = "files"
+
+    def list(self, context: dict[str, Any] | None = None) -> list[str]:
+        """List files in the local filesystem directory.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' path.
+
+        Returns
+        -------
+        list[str]
+            List of file paths in the source directory.
+
+        Raises
+        ------
+        SourceException
+            If source context is missing or if source path is not a directory.
+        """
+        logger.info(f"[list|in] ({context})")
+        result: list[str] = []
+        if self.CONTEXT_KEY_SOURCE not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
+
+        source = context[self.CONTEXT_KEY_SOURCE]
+        source_path = Path(source)
+        if source_path.is_dir():
+            suffix = None
+            if self.CONTEXT_KEY_SUFFIX in context:
+                suffix = context[self.CONTEXT_KEY_SUFFIX]
+                result: list[str] = [str(file) for file in source_path.iterdir() if file.name.endswith(suffix)]
+            else:
+                result: list[str] = [str(file) for file in source_path.iterdir()]
+        elif source_path.is_file():
+            result: list[str] = [source]
+
+        logger.debug(f"[list|out] => {result}")
+        logger.info(f"[list|out] => result len: {len(result)}")
+        return result
+
+    def get(self, context: dict[str, Any] | None = None) -> Any:
+        """Retrieve file(s) from local filesystem.
+
+        Parameters
+        ----------
+        context : dict[str, Any], optional
+            Context dictionary containing 'source' path.
+
+        Returns
+        -------
+        Any
+            List of file paths if source is a directory, or single file path if source is a file.
+
+        Raises
+        ------
+        SourceException
+            If source context is missing.
+        """
+        logger.info(f"[get|in] ({context})")
+
+        if self.CONTEXT_KEY_FILES not in context or self.CONTEXT_KEY_TARGET not in context:
+            raise SourceException(f"{self.CONTEXT_KEY_FILES} and {self.CONTEXT_KEY_TARGET} must be provided in config")
+        files = context[self.CONTEXT_KEY_FILES]
+        target = context[self.CONTEXT_KEY_TARGET]
+
+        if "list" != type(files).__name__:
+            if "str" == type(files).__name__:
+                files = [files]
+            else:
+                raise SourceException("files argument must be a list of strings or a string")
+
+        target_is_dir: bool = False
+        if Path(target).is_dir():
+            target_is_dir = True
+
+        result: list[str] = []
+
+        for file in files:
+            basename = Path(file).name
+            new_file = str(Path(target) / basename) if target_is_dir else target
+            shutil.copy(file, new_file)
+            result.append(new_file)
+
+        logger.info("[get|out] => {result}")
+        return result