boto3-assist 0.1.14__py3-none-any.whl → 0.2.1__py3-none-any.whl

boto3_assist/s3/s3.py

@@ -0,0 +1,476 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+import os
+import tempfile
+import time
+from typing import Any, Dict, List, Optional
+
+from aws_lambda_powertools import Logger
+from botocore.exceptions import ClientError
+
+from boto3_assist.errors.custom_exceptions import InvalidHttpMethod
+from boto3_assist.s3.s3_connection import S3Connection
+from boto3_assist.utilities.datetime_utility import DatetimeUtility
+from boto3_assist.utilities.file_operations import FileOperations
+from boto3_assist.utilities.http_utility import HttpUtility
+
+logger = Logger(child=True)
+
+
+class S3(S3Connection):
+    """Common S3 Actions"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_end_point_url: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        """_summary_
+
+        Args:
+            aws_profile (Optional[str], optional): _description_. Defaults to None.
+            aws_region (Optional[str], optional): _description_. Defaults to None.
+            aws_end_point_url (Optional[str], optional): _description_. Defaults to None.
+            aws_access_key_id (Optional[str], optional): _description_. Defaults to None.
+            aws_secret_access_key (Optional[str], optional): _description_. Defaults to None.
+        """
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_end_point_url=aws_end_point_url,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+    def generate_presigned_url(
+        self,
+        bucket_name: str,
+        key_path: str,
+        user_id: str,
+        file_name: str,
+        meta_data: dict | None = None,
+        expiration=3600,
+        method_type="POST",
+    ) -> Dict[str, Any]:
+        """
+        Create a signed URL for uploading a file to S3.
+        :param bucket_name: The name of the S3 bucket.
+        :param user_id: The user ID of the user uploading the file.
+        :param file_name: The file name of the file being uploaded.
+        :param aws_profile: The name of the AWS profile to use.
+        :param aws_region: The name of the AWS region to use.
+        :param expiration: The number of seconds the URL is valid for.
+        :return: The signed URL.
+        """
+        start = DatetimeUtility.get_utc_now()
+        logger.debug(
+            f"Creating signed URL for bucket {bucket_name} for user {user_id} and file {file_name} at {start} UTC"
+        )
+
+        file_extension = FileOperations.get_file_extension(file_name)
+
+        local_meta = {
+            "user_id": f"{user_id}",
+            "file_name": f"{file_name}",
+            "extension": f"{file_extension}",
+            "method": "pre-signed-upload",
+        }
+
+        if not meta_data:
+            meta_data = local_meta
+        else:
+            meta_data.update(local_meta)
+
+        object_key = key_path
+        method_type = method_type.upper()
+
+        signed_url: str | Dict[str, Any]
+        if method_type == "PUT":
+            signed_url = self.client.generate_presigned_url(
+                "put_object",
+                Params={
+                    "Bucket": f"{bucket_name}",
+                    "Key": f"{object_key}",
+                    # NOTE: if you include the ContentType or Metadata then its required in the when they upload the file
+                    # Otherwise you will get a `SignatureDoesNotMatch` error
+                    # for now I'm commenting it out.
+                    #'ContentType': 'application/octet-stream',
+                    #'ACL': 'private',
+                    # "Metadata": meta_data,
+                },
+                ExpiresIn=expiration,  # URL is valid for x seconds
+            )
+        elif method_type == "POST":
+            signed_url = self.client.generate_presigned_post(
+                bucket_name,
+                object_key,
+                ExpiresIn=expiration,  # URL is valid for x seconds
+            )
+        elif method_type == "GET":
+            signed_url = self.client.generate_presigned_url(
+                "get_object",
+                Params={
+                    "Bucket": f"{bucket_name}",
+                    "Key": f"{object_key}",
+                },
+                ExpiresIn=expiration,  # URL is valid for x seconds
+            )
+        else:
+            raise InvalidHttpMethod(
+                f'Unknown method type was referenced.  valid types are "PUT", "POST", "GET" , "{method_type}" as used '
+            )
+
+        end = DatetimeUtility.get_utc_now()
+        logger.debug(f"Signed URL created in {end-start}")
+
+        response = {
+            "signed_url": signed_url,
+            "object_key": object_key,
+            "meta_data": meta_data,
+        }
+
+        return response
+
+    def upload_file(
+        self,
+        bucket: str,
+        key: str,
+        local_file_path: str,
+        throw_error_on_failure: bool = False,
+    ) -> str | None:
+        """
+        Uploads a file to s3. Returns the full s3 path s3://<bucket>/<key>
+        """
+
+        if key.startswith("/"):
+            # remove the first slash
+            key = key[1:]
+
+        # build the path
+        s3_path = f"s3://{bucket}/{key}"
+
+        logger.debug(
+            {
+                "metric_filter": "upload_file_to_s3",
+                "bucket": bucket,
+                "key": key,
+                "local_file_path": local_file_path,
+            }
+        )
+        try:
+            self.client.upload_file(local_file_path, bucket, key)
+
+        except ClientError as ce:
+            error = {
+                "metric_filter": "upload_file_to_s3_failure",
+                "s3 upload": "failure",
+                "bucket": bucket,
+                "key": key,
+                "local_file_path": local_file_path,
+            }
+            logger.error(error)
+
+            if throw_error_on_failure:
+                raise RuntimeError(error) from ce
+
+            return None
+
+        return s3_path
+
+    def download_file(
+        self,
+        bucket: str,
+        object_key: str,
+        local_directory: str | None = None,
+        local_file_path: str | None = None,
+        retry_attempts: int = 3,
+        retry_sleep: int = 5,
+    ) -> str:
+        """Download a file from s3"""
+        exception: Exception | None = None
+
+        if retry_attempts == 0:
+            retry_attempts = 1
+
+        for i in range(retry_attempts):
+            exception = None
+            try:
+                path = self.download_file_no_retries(
+                    bucket=bucket,
+                    object_key=object_key,
+                    local_directory=local_directory,
+                    local_file_path=local_file_path,
+                )
+                if path and os.path.exists(path):
+                    return path
+
+            except Exception as e:  # pylint: disable=w0718
+                logger.warning(
+                    {
+                        "action": "download_file",
+                        "result": "failure",
+                        "exception": str(e),
+                        "attempt": i + 1,
+                        "retry_attempts": retry_attempts,
+                    }
+                )
+
+                exception = e
+
+                # sleep for a bit
+                attempt = i + 1
+                time.sleep(attempt * retry_sleep)
+
+        if exception:
+            logger.exception(
+                {
+                    "action": "download_file",
+                    "result": "failure",
+                    "exception": str(exception),
+                    "retry_attempts": retry_attempts,
+                }
+            )
+
+            raise exception from exception
+
+        raise RuntimeError("Unable to download file")
+
+    def download_file_no_retries(
+        self,
+        bucket: str,
+        object_key: str,
+        local_directory: str | None = None,
+        local_file_path: str | None = None,
+    ) -> str:
+        """
+        Downloads a file from s3
+
+        Args:
+            bucket (str): s3 bucket
+            object_key (str): the s3 object key
+            local_directory (str, optional): Local directory to download to. Defaults to None.
+            If None, we'll use a local tmp directory.
+
+        Raises:
+            e:
+
+        Returns:
+            str: Path to the downloaded file.
+        """
+
+        decoded_object_key: str
+        try:
+            logger.debug(
+                {
+                    "action": "downloading file",
+                    "bucket": bucket,
+                    "object_key": object_key,
+                    "local_directory": local_directory,
+                }
+            )
+            return self.__download_file(
+                bucket, object_key, local_directory, local_file_path
+            )
+        except FileNotFoundError:
+            logger.warning(
+                {
+                    "metric_filter": "download_file_error",
+                    "error": "FileNotFoundError",
+                    "message": "attempting to find it decoded",
+                    "bucket": bucket,
+                    "object_key": object_key,
+                }
+            )
+
+            # attempt to decode the object_key
+            decoded_object_key = HttpUtility.decode_url(object_key)
+
+            logger.error(
+                {
+                    "metric_filter": "download_file_error",
+                    "error": "FileNotFoundError",
+                    "message": "attempting to find it decoded",
+                    "bucket": bucket,
+                    "object_key": object_key,
+                    "decoded_object_key": decoded_object_key,
+                }
+            )
+
+            return self.__download_file(bucket, decoded_object_key, local_directory)
+
+        except Exception as e:
+            logger.error(
+                {
+                    "metric_filter": "download_file_error",
+                    "error": str(e),
+                    "bucket": bucket,
+                    "decoded_object_key": decoded_object_key,
+                }
+            )
+            raise e
+
+    def stream_file(self, bucket_name: str, object_key: str) -> Dict[str, Any]:
+        """
+        Gets a file from s3 and returns the response.
+        The "Body" is a streaming body object.  You can read it like a file.
+        For example:
+
+        with response["Body"] as f:
+            data = f.read()
+            print(data)
+
+        """
+
+        logger.debug(
+            {
+                "source": "download_file",
+                "action": "downloading a file from s3",
+                "bucket": bucket_name,
+                "key": object_key,
+            }
+        )
+
+        response: Dict[str, Any] = {}
+        error = None
+
+        try:
+            response = dict(self.client.get_object(Bucket=bucket_name, Key=object_key))
+
+            logger.debug(
+                {"metric_filter": "s3_download_response", "response": str(response)}
+            )
+
+        except Exception as e:  # pylint: disable=W0718
+            error = str(e)
+            logger.error({"metric_filter": "s3_download_error", "error": str(e)})
+            raise RuntimeError(
+                {
+                    "metric_filter": "s3_download_error",
+                    "error": str(e),
+                    "bucket": bucket_name,
+                    "key": object_key,
+                }
+            ) from e
+
+        finally:
+            logger.debug(
+                {
+                    "source": "download_file",
+                    "action": "downloading a file from s3",
+                    "bucket": bucket_name,
+                    "key": object_key,
+                    "response": response,
+                    "errors": error,
+                }
+            )
+
+        return response
+
+    def __download_file(
+        self,
+        bucket: str,
+        key: str,
+        local_directory: str | None = None,
+        local_file_path: str | None = None,
+    ):
+        if local_directory and local_file_path:
+            raise ValueError(
+                "Only one of local_directory or local_file_path can be provided"
+            )
+
+        if local_directory and not os.path.exists(local_directory):
+            FileOperations.makedirs(local_directory)
+
+        if local_file_path and not os.path.exists(os.path.dirname(local_file_path)):
+            FileOperations.makedirs(os.path.dirname(local_file_path))
+
+        file_name = self.__get_file_name_from_path(key)
+        if local_directory is None and local_file_path is None:
+            local_path = self.get_local_path_for_file(file_name)
+        elif local_directory:
+            local_path = os.path.join(local_directory, file_name)
+        else:
+            local_path = local_file_path
+
+        logger.debug(
+            {
+                "source": "download_file",
+                "action": "downloading a file from s3",
+                "bucket": bucket,
+                "key": key,
+                "file_name": file_name,
+                "local_path": local_path,
+            }
+        )
+
+        error: str | None = None
+        try:
+            self.client.download_file(bucket, key, local_path)
+
+        except Exception as e:  # pylint: disable=W0718
+            error = str(e)
+            logger.error({"metric_filter": "s3_download_error", "error": str(e)})
+
+        file_exist = os.path.exists(local_path)
+
+        logger.debug(
+            {
+                "source": "download_file",
+                "action": "downloading a file from s3",
+                "bucket": bucket,
+                "key": key,
+                "file_name": file_name,
+                "local_path": local_path,
+                "file_downloaded": file_exist,
+                "errors": error,
+            }
+        )
+
+        if not file_exist:
+            raise FileNotFoundError("File Failed to download (does not exist) from S3.")
+
+        return local_path
+
+    def __get_file_name_from_path(self, path: str) -> str:
+        """
+        Get a file name from the path
+
+        Args:
+            path (str): a file path
+
+        Returns:
+            str: the file name
+        """
+        return path.rsplit("/")[-1]
+
+    def get_local_path_for_file(self, file_name: str):
+        """
+        Get a local temp location for a file.
+        This is designed to work with lambda functions.
+        The /tmp directory is the only writeable location for lambda functions.
+        """
+        temp_dir = self.get_temp_directory()
+        # use /tmp it's the only writeable location for lambda
+        local_path = os.path.join(temp_dir, file_name)
+        return local_path
+
+    def get_temp_directory(self):
+        """
+        Determines the appropriate temporary directory based on the environment.
+        If running in AWS Lambda, returns '/tmp'.
+        Otherwise, returns the system's standard temp directory.
+        """
+        if "AWS_LAMBDA_FUNCTION_NAME" in os.environ:
+            # In AWS Lambda environment
+            return "/tmp"
+        else:
+            # Not in AWS Lambda, use the system's default temp directory
+            return tempfile.gettempdir()

boto3_assist/s3/s3_connection.py

@@ -0,0 +1,120 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+from boto3_assist.boto3session import Boto3SessionManager
+from boto3_assist.environment_services.environment_variables import (
+    EnvironmentVariables,
+)
+from boto3_assist.connection_tracker import ConnectionTracker
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client, S3ServiceResource
+else:
+    S3Client = object
+    S3ServiceResource = object
+
+
+logger = Logger()
+tracker: ConnectionTracker = ConnectionTracker(service_name="s3")
+
+
+class S3Connection:
+    """Connection"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_end_point_url: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        self.aws_profile = aws_profile or EnvironmentVariables.AWS.profile()
+        self.aws_region = aws_region or EnvironmentVariables.AWS.region()
+        self.end_point_url = (
+            aws_end_point_url or EnvironmentVariables.AWS.endpoint_url()
+        )
+        self.aws_access_key_id = (
+            aws_access_key_id or EnvironmentVariables.AWS.aws_access_key_id()
+        )
+        self.aws_secret_access_key = (
+            aws_secret_access_key or EnvironmentVariables.AWS.aws_secret_access_key()
+        )
+        self.__session: Boto3SessionManager | None = None
+        self.__client: S3Client | None = None
+        self.__resource: S3ServiceResource | None = None
+
+        self.raise_on_error: bool = True
+
+    def setup(self, setup_source: Optional[str] = None) -> None:
+        """
+        Setup the environment.  Automatically called via init.
+        You can run setup at anytime with new parameters.
+        Args: setup_source: Optional[str] = None
+            Defines the source of the setup.  Useful for logging.
+        Returns: None
+        """
+
+        self.__session = Boto3SessionManager(
+            service_name="s3",
+            aws_profile=self.aws_profile,
+            aws_region=self.aws_region,
+            aws_access_key_id=self.aws_access_key_id,
+            aws_secret_access_key=self.aws_secret_access_key,
+            aws_endpoint_url=self.end_point_url,
+        )
+
+        tracker.increment_connection()
+
+        self.raise_on_error = False
+
+    @property
+    def session(self) -> Boto3SessionManager:
+        """Session"""
+        if self.__session is None:
+            self.setup(setup_source="session init")
+
+        if self.__session is None:
+            raise RuntimeError("Session is not available")
+        return self.__session
+
+    @property
+    def client(self) -> S3Client:
+        """Client Connection"""
+        if self.__client is None:
+            logger.info("Creating Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: S3Client):
+        logger.info("Setting Client")
+        self.__client = value
+
+    @property
+    def resource(self) -> S3ServiceResource:
+        """Resource Connection"""
+        if self.__resource is None:
+            logger.info("Creating Resource")
+            self.__resource = self.session.resource
+
+        if self.raise_on_error and self.__resource is None:
+            raise RuntimeError("Resource is not available")
+
+        return self.__resource
+
+    @resource.setter
+    def resource(self, value: S3ServiceResource):
+        logger.info("Setting Resource")
+        self.__resource = value

boto3_assist/utilities/file_operations.py

@@ -0,0 +1,105 @@
+import os
+import shutil
+import json
+import zipfile
+from typing import List, Any, Dict
+from pathlib import Path
+import re
+
+from aws_lambda_powertools import Logger
+
+
+logger = Logger()
+
+
+class FileOperations:
+    """
+    General File Operations
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    @staticmethod
+    def makedirs(path):
+        """Create a directory and all sub directories."""
+        abs_path = os.path.abspath(path)
+        os.makedirs(abs_path, exist_ok=True)
+
+    @staticmethod
+    def clean_directory(path: str):
+        """Clean / Delete all files and directories and sub directories"""
+        if path is None:
+            return
+        if path == "/":
+            raise ValueError("Cannot delete root directory")
+
+        abs_path = os.path.abspath(path)
+        if os.path.exists(abs_path):
+            items = os.listdir(abs_path)
+            for item in items:
+                path = os.path.join(abs_path, item)
+                if os.path.exists(path):
+                    try:
+                        if os.path.isdir(path):
+                            shutil.rmtree(path)
+                        elif os.path.isfile(path):
+                            os.remove(path)
+
+                    except Exception as e:  # pylint: disable=W0718
+                        logger.exception(f"clean up error {str(e)}")
+
+    @staticmethod
+    def get_directory_name(path: str):
+        """
+        Get the directory path from a path that is either a directory
+        or a path to a file.
+        """
+        dirname = os.path.dirname(path)
+        return dirname
+
+    @staticmethod
+    def write_to_file(path: str, data: str, append: bool = False) -> str:
+        """
+        Write to a file
+
+        """
+        return FileOperations.write_file(path=path, output=data, append=append)
+
+    @staticmethod
+    def write_file(path: str, output: str, append: bool = False) -> str:
+        """
+        Writes to a file
+        Args:
+            path (str): path
+            output (str): text to write to the file
+            append (bool): if true this operation will append to the file
+                otherwise it will overwrite. the default is to overwrite
+        Returns:
+            str: path to the file
+        """
+        dirname = FileOperations.get_directory_name(path)
+        FileOperations.makedirs(dirname)
+        mode = "a" if append else "w"
+
+        if output is None:
+            output = ""
+        with open(path, mode=mode, encoding="utf-8") as file:
+            file.write(output)
+
+        return path
+
+    @staticmethod
+    def get_file_extension(file_name: str, include_dot: bool = False):
+        """Get the extension of a file"""
+        logger.debug(f"getting extension for {file_name}")
+        # get the last part of a string after a period .
+        extention = os.path.splitext(file_name)[1]
+        logger.debug(f"extention is {extention}")
+
+        if not include_dot:
+            if str(extention).startswith("."):
+                extention = str(extention).removeprefix(".")
+                logger.debug(f"extension after prefix removal: {extention}")
+
+        return extention