boto3-assist 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

boto3_assist/connection_tracker.py

@@ -67,8 +67,8 @@ class ConnectionTracker:
 
             if not self.issue_stack_trace:
                 stack_trace_message = (
-                    f"\nTo add additional information to the log and determine where additional connections are being created, "
-                    f"set the environment variable {self.__stack_trace_env_var} to true.\n"
+                    f"📄 NOTE: To add additional information 👀 to the log and determine where additional connections are being created: "
+                    f"set the environment variable 👉{self.__stack_trace_env_var}👈 to true ✅. \n"
                 )
             else:
                 stack = "\n".join(traceback.format_stack())
@@ -83,8 +83,8 @@ class ConnectionTracker:
                 "instead of creating a new one. Connections are expensive in terms of time and latency. "
                 "If you are seeing performance issues, check how and where you are creating your "
                 "connections. You should be able to pass the connection to your other objects "
-                "and reuse your boto3 connections."
-                "\n\nMOCK Testing may show this message as well, in which case you can dismiss this warning.\n\n"
+                "and reuse your boto3 connections. "
+                "\n🧪 MOCK Testing may show this message as well, in which case you can dismiss this warning.🧪\n"
                 f"{stack_trace_message}"
             )
 

boto3_assist/s3/s3.py

@@ -4,20 +4,13 @@ Maintainers: Eric Wilson
 MIT License.  See Project Root for the license information.
 """
 
-import os
-import tempfile
-import time
-import io
-from typing import Any, Dict, List, Optional
+from typing import Optional, cast
 
 from aws_lambda_powertools import Logger
-from botocore.exceptions import ClientError
 
-from boto3_assist.errors.custom_exceptions import InvalidHttpMethod
 from boto3_assist.s3.s3_connection import S3Connection
-from boto3_assist.utilities.datetime_utility import DatetimeUtility
-from boto3_assist.utilities.file_operations import FileOperations
-from boto3_assist.utilities.http_utility import HttpUtility
+from boto3_assist.s3.s3_object import S3Object
+from boto3_assist.s3.s3_bucket import S3Bucket
 
 logger = Logger(child=True)
 
@@ -51,494 +44,21 @@ class S3(S3Connection):
             aws_secret_access_key=aws_secret_access_key,
         )
 
-    def create_bucket(self, *, bucket_name: str) -> None:
-        """
-        Create an S3 bucket
-        :param bucket_name: Bucket to create
-        :return: True if bucket is created, else False
-        """
-        try:
-            self.client.create_bucket(Bucket=bucket_name)
-            logger.info(f"Bucket {bucket_name} created")
-        except ClientError as e:
-            logger.exception(e)
-            raise e
-
-    def generate_presigned_url(
-        self,
-        *,
-        bucket_name: str,
-        key_path: str,
-        user_id: str,
-        file_name: str,
-        meta_data: dict | None = None,
-        expiration=3600,
-        method_type="POST",
-    ) -> Dict[str, Any]:
-        """
-        Create a signed URL for uploading a file to S3.
-        :param bucket_name: The name of the S3 bucket.
-        :param user_id: The user ID of the user uploading the file.
-        :param file_name: The file name of the file being uploaded.
-        :param aws_profile: The name of the AWS profile to use.
-        :param aws_region: The name of the AWS region to use.
-        :param expiration: The number of seconds the URL is valid for.
-        :return: The signed URL.
-        """
-        start = DatetimeUtility.get_utc_now()
-        logger.debug(
-            f"Creating signed URL for bucket {bucket_name} for user {user_id} and file {file_name} at {start} UTC"
-        )
-
-        file_extension = FileOperations.get_file_extension(file_name)
-
-        local_meta = {
-            "user_id": f"{user_id}",
-            "file_name": f"{file_name}",
-            "extension": f"{file_extension}",
-            "method": "pre-signed-upload",
-        }
-
-        if not meta_data:
-            meta_data = local_meta
-        else:
-            meta_data.update(local_meta)
-
-        key = key_path
-        method_type = method_type.upper()
-
-        signed_url: str | Dict[str, Any]
-        if method_type == "PUT":
-            signed_url = self.client.generate_presigned_url(
-                "put_object",
-                Params={
-                    "Bucket": f"{bucket_name}",
-                    "Key": f"{key}",
-                    # NOTE: if you include the ContentType or Metadata then its required in the when they upload the file
-                    # Otherwise you will get a `SignatureDoesNotMatch` error
-                    # for now I'm commenting it out.
-                    #'ContentType': 'application/octet-stream',
-                    #'ACL': 'private',
-                    # "Metadata": meta_data,
-                },
-                ExpiresIn=expiration,  # URL is valid for x seconds
-            )
-        elif method_type == "POST":
-            signed_url = self.client.generate_presigned_post(
-                bucket_name,
-                key,
-                ExpiresIn=expiration,  # URL is valid for x seconds
-            )
-        elif method_type == "GET":
-            signed_url = self.client.generate_presigned_url(
-                "get_object",
-                Params={
-                    "Bucket": f"{bucket_name}",
-                    "Key": f"{key}",
-                },
-                ExpiresIn=expiration,  # URL is valid for x seconds
-            )
-        else:
-            raise InvalidHttpMethod(
-                f'Unknown method type was referenced.  valid types are "PUT", "POST", "GET" , "{method_type}" as used '
-            )
-
-        end = DatetimeUtility.get_utc_now()
-        logger.debug(f"Signed URL created in {end-start}")
-
-        response = {
-            "signed_url": signed_url,
-            "key": key,
-            "meta_data": meta_data,
-        }
-
-        return response
-
-    def upload_file_obj(self, *, bucket: str, key: str, file_obj: bytes | str) -> str:
-        """
-        Uploads a file object to s3. Returns the full s3 path s3://<bucket>/<key>
-        """
-
-        if key.startswith("/"):
-            # remove the first slash
-            key = key[1:]
-
-        logger.debug(
-            {
-                "metric_filter": "upload_file_to_s3",
-                "bucket": bucket,
-                "key": key,
-            }
-        )
-        try:
-            # convert if necessary
-            file_obj: bytes = (
-                file_obj.encode("utf-8") if isinstance(file_obj, str) else file_obj
-            )
-            self.client.upload_fileobj(
-                Fileobj=io.BytesIO(file_obj), Bucket=bucket, Key=key
-            )
-
-        except ClientError as ce:
-            error = {
-                "metric_filter": "upload_file_to_s3_failure",
-                "s3 upload": "failure",
-                "bucket": bucket,
-                "key": key,
-            }
-            logger.error(error)
-            raise RuntimeError(error) from ce
-
-        return f"s3://{bucket}/{key}"
-
-    def upload_file(
-        self,
-        *,
-        bucket: str,
-        key: str,
-        local_file_path: str,
-        throw_error_on_failure: bool = False,
-    ) -> str | None:
-        """
-        Uploads a file to s3. Returns the full s3 path s3://<bucket>/<key>
-        """
-
-        if key.startswith("/"):
-            # remove the first slash
-            key = key[1:]
-
-        # build the path
-        s3_path = f"s3://{bucket}/{key}"
-
-        logger.debug(
-            {
-                "metric_filter": "upload_file_to_s3",
-                "bucket": bucket,
-                "key": key,
-                "local_file_path": local_file_path,
-            }
-        )
-        try:
-            self.client.upload_file(local_file_path, bucket, key)
-
-        except ClientError as ce:
-            error = {
-                "metric_filter": "upload_file_to_s3_failure",
-                "s3 upload": "failure",
-                "bucket": bucket,
-                "key": key,
-                "local_file_path": local_file_path,
-            }
-            logger.error(error)
-
-            if throw_error_on_failure:
-                raise RuntimeError(error) from ce
-
-            return None
-
-        return s3_path
-
-    def download_file(
-        self,
-        *,
-        bucket: str,
-        key: str,
-        local_directory: str | None = None,
-        local_file_path: str | None = None,
-        retry_attempts: int = 3,
-        retry_sleep: int = 5,
-    ) -> str:
-        """Download a file from s3"""
-        exception: Exception | None = None
-
-        if retry_attempts == 0:
-            retry_attempts = 1
-
-        for i in range(retry_attempts):
-            exception = None
-            try:
-                path = self.download_file_no_retries(
-                    bucket=bucket,
-                    key=key,
-                    local_directory=local_directory,
-                    local_file_path=local_file_path,
-                )
-                if path and os.path.exists(path):
-                    return path
-
-            except Exception as e:  # pylint: disable=w0718
-                logger.warning(
-                    {
-                        "action": "download_file",
-                        "result": "failure",
-                        "exception": str(e),
-                        "attempt": i + 1,
-                        "retry_attempts": retry_attempts,
-                    }
-                )
-
-                exception = e
-
-                # sleep for a bit
-                attempt = i + 1
-                time.sleep(attempt * retry_sleep)
-
-        if exception:
-            logger.exception(
-                {
-                    "action": "download_file",
-                    "result": "failure",
-                    "exception": str(exception),
-                    "retry_attempts": retry_attempts,
-                }
-            )
-
-            raise exception from exception
-
-        raise RuntimeError("Unable to download file")
-
-    def download_file_no_retries(
-        self,
-        bucket: str,
-        key: str,
-        local_directory: str | None = None,
-        local_file_path: str | None = None,
-    ) -> str:
-        """
-        Downloads a file from s3
-
-        Args:
-            bucket (str): s3 bucket
-            key (str): the s3 object key
-            local_directory (str, optional): Local directory to download to. Defaults to None.
-            If None, we'll use a local tmp directory.
-
-        Raises:
-            e:
-
-        Returns:
-            str: Path to the downloaded file.
-        """
-
-        decoded_object_key: str
-        try:
-            logger.debug(
-                {
-                    "action": "downloading file",
-                    "bucket": bucket,
-                    "key": key,
-                    "local_directory": local_directory,
-                }
-            )
-            return self.__download_file(bucket, key, local_directory, local_file_path)
-        except FileNotFoundError:
-            logger.warning(
-                {
-                    "metric_filter": "download_file_error",
-                    "error": "FileNotFoundError",
-                    "message": "attempting to find it decoded",
-                    "bucket": bucket,
-                    "key": key,
-                }
-            )
-
-            # attempt to decode the key
-            decoded_object_key = HttpUtility.decode_url(key)
-
-            logger.error(
-                {
-                    "metric_filter": "download_file_error",
-                    "error": "FileNotFoundError",
-                    "message": "attempting to find it decoded",
-                    "bucket": bucket,
-                    "key": key,
-                    "decoded_object_key": decoded_object_key,
-                }
-            )
-
-            return self.__download_file(bucket, decoded_object_key, local_directory)
-
-        except Exception as e:
-            logger.error(
-                {
-                    "metric_filter": "download_file_error",
-                    "error": str(e),
-                    "bucket": bucket,
-                    "decoded_object_key": decoded_object_key,
-                }
-            )
-            raise e
-
-    def stream_file(self, bucket_name: str, key: str) -> Dict[str, Any]:
-        """
-        Gets a file from s3 and returns the response.
-        The "Body" is a streaming body object.  You can read it like a file.
-        For example:
-
-        with response["Body"] as f:
-            data = f.read()
-            print(data)
-
-        """
-
-        logger.debug(
-            {
-                "source": "download_file",
-                "action": "downloading a file from s3",
-                "bucket": bucket_name,
-                "key": key,
-            }
-        )
-
-        response: Dict[str, Any] = {}
-        error = None
-
-        try:
-            response = dict(self.client.get_object(Bucket=bucket_name, Key=key))
-
-            logger.debug(
-                {"metric_filter": "s3_download_response", "response": str(response)}
-            )
-
-        except Exception as e:  # pylint: disable=W0718
-            error = str(e)
-            logger.error({"metric_filter": "s3_download_error", "error": str(e)})
-            raise RuntimeError(
-                {
-                    "metric_filter": "s3_download_error",
-                    "error": str(e),
-                    "bucket": bucket_name,
-                    "key": key,
-                }
-            ) from e
-
-        finally:
-            logger.debug(
-                {
-                    "source": "download_file",
-                    "action": "downloading a file from s3",
-                    "bucket": bucket_name,
-                    "key": key,
-                    "response": response,
-                    "errors": error,
-                }
-            )
-
-        return response
-
-    def __download_file(
-        self,
-        bucket: str,
-        key: str,
-        local_directory: str | None = None,
-        local_file_path: str | None = None,
-    ):
-        if local_directory and local_file_path:
-            raise ValueError(
-                "Only one of local_directory or local_file_path can be provided"
-            )
-
-        if local_directory and not os.path.exists(local_directory):
-            FileOperations.makedirs(local_directory)
-
-        if local_file_path and not os.path.exists(os.path.dirname(local_file_path)):
-            FileOperations.makedirs(os.path.dirname(local_file_path))
-
-        file_name = self.__get_file_name_from_path(key)
-        if local_directory is None and local_file_path is None:
-            local_path = self.get_local_path_for_file(file_name)
-        elif local_directory:
-            local_path = os.path.join(local_directory, file_name)
-        else:
-            local_path = local_file_path
-
-        logger.debug(
-            {
-                "source": "download_file",
-                "action": "downloading a file from s3",
-                "bucket": bucket,
-                "key": key,
-                "file_name": file_name,
-                "local_path": local_path,
-            }
-        )
-
-        error: str | None = None
-        try:
-            self.client.download_file(bucket, key, local_path)
-
-        except Exception as e:  # pylint: disable=W0718
-            error = str(e)
-            logger.error({"metric_filter": "s3_download_error", "error": str(e)})
-
-        file_exist = os.path.exists(local_path)
-
-        logger.debug(
-            {
-                "source": "download_file",
-                "action": "downloading a file from s3",
-                "bucket": bucket,
-                "key": key,
-                "file_name": file_name,
-                "local_path": local_path,
-                "file_downloaded": file_exist,
-                "errors": error,
-            }
-        )
-
-        if not file_exist:
-            raise FileNotFoundError("File Failed to download (does not exist) from S3.")
-
-        return local_path
-
-    def __get_file_name_from_path(self, path: str) -> str:
-        """
-        Get a file name from the path
-
-        Args:
-            path (str): a file path
-
-        Returns:
-            str: the file name
-        """
-        return path.rsplit("/")[-1]
-
-    def get_local_path_for_file(self, file_name: str):
-        """
-        Get a local temp location for a file.
-        This is designed to work with lambda functions.
-        The /tmp directory is the only writeable location for lambda functions.
-        """
-        temp_dir = self.get_temp_directory()
-        # use /tmp it's the only writeable location for lambda
-        local_path = os.path.join(temp_dir, file_name)
-        return local_path
-
-    def get_temp_directory(self):
-        """
-        Determines the appropriate temporary directory based on the environment.
-        If running in AWS Lambda, returns '/tmp'.
-        Otherwise, returns the system's standard temp directory.
-        """
-        if "AWS_LAMBDA_FUNCTION_NAME" in os.environ:
-            # In AWS Lambda environment
-            return "/tmp"
-        else:
-            # Not in AWS Lambda, use the system's default temp directory
-            return tempfile.gettempdir()
-
-    def encode(
-        self, text: str, encoding: str = "utf-8", errors: str = "strict"
-    ) -> bytes:
-        """
-        Encodes a string for s3
-        """
-        return text.encode(encoding=encoding, errors=errors)
-
-    def decode(
-        self, file_obj: bytes, encoding: str = "utf-8", errors: str = "strict"
-    ) -> str:
-        """
-        Decodes bytes to a string
-        """
-        return file_obj.decode(encoding=encoding, errors=errors)
+        self.__s3_object: S3Object | None = None
+        self.__s3_bucket: S3Bucket | None = None
+
+    @property
+    def object(self) -> S3Object:
+        """s3 object"""
+        if self.__s3_object is None:
+            connection = cast(S3Connection, self)
+            self.__s3_object = S3Object(connection)
+        return self.__s3_object
+
+    @property
+    def bucket(self) -> S3Bucket:
+        """s3 bucket"""
+        if self.__s3_bucket is None:
+            connection = cast(S3Connection, self)
+            self.__s3_bucket = S3Bucket(connection)
+        return self.__s3_bucket

boto3_assist/s3/s3_bucket.py

@@ -0,0 +1,67 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Any, Dict
+
+from aws_lambda_powertools import Logger
+from botocore.exceptions import ClientError
+
+
+from boto3_assist.s3.s3_connection import S3Connection
+
+logger = Logger(child=True)
+
+
+class S3Bucket:
+    """Common S3 Actions"""
+
+    def __init__(self, connection: S3Connection):
+        self.connection = connection or S3Connection()
+
+    def create(self, *, bucket_name: str) -> Dict[str, Any]:
+        """
+        Create an S3 bucket
+        :param bucket_name: Bucket to create
+        :return: True if bucket is created, else False
+        """
+        try:
+            response = self.connection.client.create_bucket(Bucket=bucket_name)
+            logger.info(f"Bucket {bucket_name} created")
+
+            return dict(response)
+        except ClientError as e:
+            logger.exception(e)
+            raise e
+
+    def enable_versioning(self, *, bucket_name: str) -> None:
+        """
+        Enable versioning on an S3 bucket
+        :param bucket_name: Bucket to enable versioning on
+        :return: None
+        """
+        try:
+            self.connection.client.put_bucket_versioning(
+                Bucket=bucket_name, VersioningConfiguration={"Status": "Enabled"}
+            )
+            logger.info(f"Versioning enabled on bucket {bucket_name}")
+        except ClientError as e:
+            logger.exception(e)
+            raise e
+
+    def disable_versioning(self, *, bucket_name: str) -> None:
+        """
+        Disable versioning on an S3 bucket
+        :param bucket_name: Bucket to disable versioning on
+        :return: None
+        """
+        try:
+            self.connection.client.put_bucket_versioning(
+                Bucket=bucket_name, VersioningConfiguration={"Status": "Suspended"}
+            )
+            logger.info(f"Versioning disabled on bucket {bucket_name}")
+        except ClientError as e:
+            logger.exception(e)
+            raise e

boto3_assist/s3/s3_connection.py

@@ -8,10 +8,7 @@ from typing import Optional
 from typing import TYPE_CHECKING
 
 from aws_lambda_powertools import Logger
-from boto3_assist.boto3session import Boto3SessionManager
-from boto3_assist.environment_services.environment_variables import (
-    EnvironmentVariables,
-)
+
 from boto3_assist.connection import Connection
 
 if TYPE_CHECKING:

boto3_assist/s3/s3_object.py

@@ -0,0 +1,606 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+import os
+import tempfile
+import time
+import io
+from typing import Any, Dict, Optional, List
+
+from aws_lambda_powertools import Logger
+from botocore.exceptions import ClientError
+
+from boto3_assist.errors.custom_exceptions import InvalidHttpMethod
+from boto3_assist.s3.s3_connection import S3Connection
+from boto3_assist.utilities.datetime_utility import DatetimeUtility
+from boto3_assist.utilities.file_operations import FileOperations
+from boto3_assist.utilities.http_utility import HttpUtility
+
+
+logger = Logger(child=True)
+
+
+class S3Object:
+    """S3 Object Actions"""
+
+    def __init__(self, connection: S3Connection):
+        self.connection = connection or S3Connection()
+
+    def delete(self, *, bucket_name: str, key: str) -> Dict[str, Any]:
+        """
+        Deletes an object key
+
+        Args:
+            bucket_name (str): The AWS Bucket Name
+            key (str): The Object Key
+        """
+        s3 = self.connection.client
+        # see if the object exists
+        try:
+            response = s3.head_object(Bucket=bucket_name, Key=key)
+            response = s3.delete_object(Bucket=bucket_name, Key=key)
+        except s3.exceptions.NoSuchKey:
+            response = {"ResponseMetadata": {"HTTPStatusCode": 404}}
+        except s3.exceptions.ClientError as e:
+            if e.response.get("Error", {}).get("Code") == "404":
+                response = {"ResponseMetadata": {"HTTPStatusCode": 404}}
+            else:
+                raise e
+
+        return dict(response)
+
+    def delete_all_versions(
+        self, *, bucket_name: str, key: str, include_deleted: bool = False
+    ) -> List[str]:
+        """
+        Deletes an object key and all the versions for that object key
+
+        Args:
+            bucket_name (str): The AWS Bucket Name
+            key (str): The Object Kuye
+            include_deleted (bool, optional): Should deleted files be removed as well.
+                If True it will look for the object keys with the deleted marker and remove it.
+                Defaults to False.
+        """
+        s3 = self.connection.client
+        paginator = s3.get_paginator("list_object_versions")
+        files: List[str] = []
+
+        for page in paginator.paginate(Bucket=bucket_name, Prefix=key):
+            # Delete object versions
+            if "Versions" in page:
+                for version in page["Versions"]:
+                    s3.delete_object(
+                        Bucket=bucket_name,
+                        Key=version["Key"],
+                        VersionId=version["VersionId"],
+                    )
+
+                    files.append(f"{version['Key']} - {version['VersionId']}")
+
+                if include_deleted:
+                    # delete a previous files that may have just been a soft delete.
+                    if "DeleteMarkers" in page:
+                        for marker in page["DeleteMarkers"]:
+                            s3.delete_object(
+                                Bucket=bucket_name,
+                                Key=marker["Key"],
+                                VersionId=marker["VersionId"],
+                            )
+
+                            files.append(
+                                f"{marker['Key']}:{marker['VersionId']}:delete-marker"
+                            )
+            else:
+                response = self.delete(bucket_name=bucket_name, key=key)
+                if response["ResponseMetadata"]["HTTPStatusCode"] == 404:
+                    return files
+
+                files.append(key)
+
+        return files
+
+    def generate_presigned_url(
+        self,
+        *,
+        bucket_name: str,
+        key_path: str,
+        file_name: str,
+        meta_data: Optional[dict] = None,
+        expiration: int = 3600,
+        method_type: str = "POST",
+        user_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Create a signed URL for uploading a file to S3.
+        :param bucket_name: The name of the S3 bucket.
+        :param user_id: The user ID of the user uploading the file.
+        :param file_name: The file name of the file being uploaded.
+        :param aws_profile: The name of the AWS profile to use.
+        :param aws_region: The name of the AWS region to use.
+        :param expiration: The number of seconds the URL is valid for.
+        :return: The signed URL.
+        """
+        start = DatetimeUtility.get_utc_now()
+        logger.debug(
+            f"Creating signed URL for bucket {bucket_name} for user {user_id} and file {file_name} at {start} UTC"
+        )
+
+        file_extension = FileOperations.get_file_extension(file_name)
+
+        local_meta = {
+            "user_id": f"{user_id}",
+            "file_name": f"{file_name}",
+            "extension": f"{file_extension}",
+            "method": "pre-signed-upload",
+        }
+
+        if not meta_data:
+            meta_data = local_meta
+        else:
+            meta_data.update(local_meta)
+
+        key = key_path
+        method_type = method_type.upper()
+
+        signed_url: str | Dict[str, Any]
+        if method_type == "PUT":
+            signed_url = self.connection.client.generate_presigned_url(
+                "put_object",
+                Params={
+                    "Bucket": f"{bucket_name}",
+                    "Key": f"{key}",
+                    # NOTE: if you include the ContentType or Metadata then its required in the when they upload the file
+                    # Otherwise you will get a `SignatureDoesNotMatch` error
+                    # for now I'm commenting it out.
+                    #'ContentType': 'application/octet-stream',
+                    #'ACL': 'private',
+                    # "Metadata": meta_data,
+                },
+                ExpiresIn=expiration,  # URL is valid for x seconds
+            )
+        elif method_type == "POST":
+            signed_url = self.connection.client.generate_presigned_post(
+                bucket_name,
+                key,
+                ExpiresIn=expiration,  # URL is valid for x seconds
+            )
+        elif method_type == "GET":
+            signed_url = self.connection.client.generate_presigned_url(
+                "get_object",
+                Params={
+                    "Bucket": f"{bucket_name}",
+                    "Key": f"{key}",
+                },
+                ExpiresIn=expiration,  # URL is valid for x seconds
+            )
+        else:
+            raise InvalidHttpMethod(
+                f'Unknown method type was referenced.  valid types are "PUT", "POST", "GET" , "{method_type}" as used '
+            )
+
+        end = DatetimeUtility.get_utc_now()
+        logger.debug(f"Signed URL created in {end-start}")
+
+        response = {
+            "signed_url": signed_url,
+            "key": key,
+            "meta_data": meta_data,
+        }
+
+        return response
+
+    def upload_file_obj(self, *, bucket: str, key: str, file_obj: bytes | str) -> str:
+        """
+        Uploads a file object to s3. Returns the full s3 path s3://<bucket>/<key>
+        """
+
+        if key.startswith("/"):
+            # remove the first slash
+            key = key[1:]
+
+        logger.debug(
+            {
+                "metric_filter": "upload_file_to_s3",
+                "bucket": bucket,
+                "key": key,
+            }
+        )
+        try:
+            # convert if necessary
+            file_obj: bytes = (
+                file_obj.encode("utf-8") if isinstance(file_obj, str) else file_obj
+            )
+            self.connection.client.upload_fileobj(
+                Fileobj=io.BytesIO(file_obj), Bucket=bucket, Key=key
+            )
+
+        except ClientError as ce:
+            error = {
+                "metric_filter": "upload_file_to_s3_failure",
+                "s3 upload": "failure",
+                "bucket": bucket,
+                "key": key,
+            }
+            logger.error(error)
+            raise RuntimeError(error) from ce
+
+        return f"s3://{bucket}/{key}"
+
+    def upload_file(
+        self,
+        *,
+        bucket: str,
+        key: str,
+        local_file_path: str,
+        throw_error_on_failure: bool = False,
+    ) -> str | None:
+        """
+        Uploads a file to s3. Returns the full s3 path s3://<bucket>/<key>
+        """
+
+        if key.startswith("/"):
+            # remove the first slash
+            key = key[1:]
+
+        # build the path
+        s3_path = f"s3://{bucket}/{key}"
+
+        logger.debug(
+            {
+                "metric_filter": "upload_file_to_s3",
+                "bucket": bucket,
+                "key": key,
+                "local_file_path": local_file_path,
+            }
+        )
+        try:
+            self.connection.client.upload_file(local_file_path, bucket, key)
+
+        except ClientError as ce:
+            error = {
+                "metric_filter": "upload_file_to_s3_failure",
+                "s3 upload": "failure",
+                "bucket": bucket,
+                "key": key,
+                "local_file_path": local_file_path,
+            }
+            logger.error(error)
+
+            if throw_error_on_failure:
+                raise RuntimeError(error) from ce
+
+            return None
+
+        return s3_path
+
+    def download_file(
+        self,
+        *,
+        bucket: str,
+        key: str,
+        local_directory: str | None = None,
+        local_file_path: str | None = None,
+        retry_attempts: int = 3,
+        retry_sleep: int = 5,
+    ) -> str:
+        """Download a file from s3"""
+        exception: Exception | None = None
+
+        if retry_attempts == 0:
+            retry_attempts = 1
+
+        for i in range(retry_attempts):
+            exception = None
+            try:
+                path = self.download_file_no_retries(
+                    bucket=bucket,
+                    key=key,
+                    local_directory=local_directory,
+                    local_file_path=local_file_path,
+                )
+                if path and os.path.exists(path):
+                    return path
+
+            except Exception as e:  # pylint: disable=w0718
+                logger.warning(
+                    {
+                        "action": "download_file",
+                        "result": "failure",
+                        "exception": str(e),
+                        "attempt": i + 1,
+                        "retry_attempts": retry_attempts,
+                    }
+                )
+
+                exception = e
+
+                # sleep for a bit
+                attempt = i + 1
+                time.sleep(attempt * retry_sleep)
+
+        if exception:
+            logger.exception(
+                {
+                    "action": "download_file",
+                    "result": "failure",
+                    "exception": str(exception),
+                    "retry_attempts": retry_attempts,
+                }
+            )
+
+            raise exception from exception
+
+        raise RuntimeError("Unable to download file")
+
+    def download_file_no_retries(
+        self,
+        bucket: str,
+        key: str,
+        local_directory: str | None = None,
+        local_file_path: str | None = None,
+    ) -> str:
+        """
+        Downloads a file from s3
+
+        Args:
+            bucket (str): s3 bucket
+            key (str): the s3 object key
+            local_directory (str, optional): Local directory to download to. Defaults to None.
+            If None, we'll use a local tmp directory.
+
+        Raises:
+            e:
+
+        Returns:
+            str: Path to the downloaded file.
+        """
+
+        decoded_object_key: str
+        try:
+            logger.debug(
+                {
+                    "action": "downloading file",
+                    "bucket": bucket,
+                    "key": key,
+                    "local_directory": local_directory,
+                }
+            )
+            return self.__download_file(bucket, key, local_directory, local_file_path)
+        except FileNotFoundError:
+            logger.warning(
+                {
+                    "metric_filter": "download_file_error",
+                    "error": "FileNotFoundError",
+                    "message": "attempting to find it decoded",
+                    "bucket": bucket,
+                    "key": key,
+                }
+            )
+
+            # attempt to decode the key
+            decoded_object_key = HttpUtility.decode_url(key)
+
+            logger.error(
+                {
+                    "metric_filter": "download_file_error",
+                    "error": "FileNotFoundError",
+                    "message": "attempting to find it decoded",
+                    "bucket": bucket,
+                    "key": key,
+                    "decoded_object_key": decoded_object_key,
+                }
+            )
+
+            return self.__download_file(bucket, decoded_object_key, local_directory)
+
+        except Exception as e:
+            logger.error(
+                {
+                    "metric_filter": "download_file_error",
+                    "error": str(e),
+                    "bucket": bucket,
+                    "decoded_object_key": decoded_object_key,
+                }
+            )
+            raise e
+
+    def stream_file(self, bucket_name: str, key: str) -> Dict[str, Any]:
+        """
+        Gets a file from s3 and returns the response.
+        The "Body" is a streaming body object.  You can read it like a file.
+        For example:
+
+        with response["Body"] as f:
+            data = f.read()
+            print(data)
+
+        """
+
+        logger.debug(
+            {
+                "source": "download_file",
+                "action": "downloading a file from s3",
+                "bucket": bucket_name,
+                "key": key,
+            }
+        )
+
+        response: Dict[str, Any] = {}
+        error = None
+
+        try:
+            response = dict(
+                self.connection.client.get_object(Bucket=bucket_name, Key=key)
+            )
+
+            logger.debug(
+                {"metric_filter": "s3_download_response", "response": str(response)}
+            )
+
+        except Exception as e:  # pylint: disable=W0718
+            error = str(e)
+            logger.error({"metric_filter": "s3_download_error", "error": str(e)})
+            raise RuntimeError(
+                {
+                    "metric_filter": "s3_download_error",
+                    "error": str(e),
+                    "bucket": bucket_name,
+                    "key": key,
+                }
+            ) from e
+
+        finally:
+            logger.debug(
+                {
+                    "source": "download_file",
+                    "action": "downloading a file from s3",
+                    "bucket": bucket_name,
+                    "key": key,
+                    "response": response,
+                    "errors": error,
+                }
+            )
+
+        return response
+
+    def __download_file(
+        self,
+        bucket: str,
+        key: str,
+        local_directory: str | None = None,
+        local_file_path: str | None = None,
+    ):
+        if local_directory and local_file_path:
+            raise ValueError(
+                "Only one of local_directory or local_file_path can be provided"
+            )
+
+        if local_directory and not os.path.exists(local_directory):
+            FileOperations.makedirs(local_directory)
+
+        if local_file_path and not os.path.exists(os.path.dirname(local_file_path)):
+            FileOperations.makedirs(os.path.dirname(local_file_path))
+
+        file_name = self.__get_file_name_from_path(key)
+        if local_directory is None and local_file_path is None:
+            local_path = self.get_local_path_for_file(file_name)
+        elif local_directory:
+            local_path = os.path.join(local_directory, file_name)
+        else:
+            local_path = local_file_path
+
+        logger.debug(
+            {
+                "source": "download_file",
+                "action": "downloading a file from s3",
+                "bucket": bucket,
+                "key": key,
+                "file_name": file_name,
+                "local_path": local_path,
+            }
+        )
+
+        error: str | None = None
+        try:
+            self.connection.client.download_file(bucket, key, local_path)
+
+        except Exception as e:  # pylint: disable=W0718
+            error = str(e)
+            logger.error({"metric_filter": "s3_download_error", "error": str(e)})
+
+        file_exist = os.path.exists(local_path)
+
+        logger.debug(
+            {
+                "source": "download_file",
+                "action": "downloading a file from s3",
+                "bucket": bucket,
+                "key": key,
+                "file_name": file_name,
+                "local_path": local_path,
+                "file_downloaded": file_exist,
+                "errors": error,
+            }
+        )
+
+        if not file_exist:
+            raise FileNotFoundError("File Failed to download (does not exist) from S3.")
+
+        return local_path
+
+    def __get_file_name_from_path(self, path: str) -> str:
+        """
+        Get a file name from the path
+
+        Args:
+            path (str): a file path
+
+        Returns:
+            str: the file name
+        """
+        return path.rsplit("/")[-1]
+
+    def get_local_path_for_file(self, file_name: str):
+        """
+        Get a local temp location for a file.
+        This is designed to work with lambda functions.
+        The /tmp directory is the only writeable location for lambda functions.
+        """
+        temp_dir = self.get_temp_directory()
+        # use /tmp it's the only writeable location for lambda
+        local_path = os.path.join(temp_dir, file_name)
+        return local_path
+
+    def get_temp_directory(self):
+        """
+        Determines the appropriate temporary directory based on the environment.
+        If running in AWS Lambda, returns '/tmp'.
+        Otherwise, returns the system's standard temp directory.
+        """
+        if "AWS_LAMBDA_FUNCTION_NAME" in os.environ:
+            # In AWS Lambda environment
+            return "/tmp"
+        else:
+            # Not in AWS Lambda, use the system's default temp directory
+            return tempfile.gettempdir()
+
+    def encode(
+        self, text: str, encoding: str = "utf-8", errors: str = "strict"
+    ) -> bytes:
+        """
+        Encodes a string for s3
+        """
+        return text.encode(encoding=encoding, errors=errors)
+
+    def decode(
+        self, file_obj: bytes, encoding: str = "utf-8", errors: str = "strict"
+    ) -> str:
+        """
+        Decodes bytes to a string
+        """
+        return file_obj.decode(encoding=encoding, errors=errors)
+
+    def list_versions(self, bucket: str, prefix: str = "") -> List[str]:
+        """
+        List all versions of objects in an S3 bucket with a given prefix.
+
+        Args:
+            bucket (str): The name of the S3 bucket.
+            prefix (str, optional): The prefix to filter objects by. Defaults to "".
+
+        Returns:
+            list: A list of dictionaries containing information about each object version.
+        """
+        versions = []
+        paginator = self.connection.client.get_paginator("list_object_versions")
+        page_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
+
+        for page in page_iterator:
+            if "Versions" in page:
+                versions.extend(page["Versions"])
+
+        return versions

boto3_assist/utilities/string_utility.py

@@ -195,6 +195,36 @@ class StringUtility:
         hash_object.update(encoded_string)
         return hash_object.hexdigest()
 
+    @staticmethod
+    def generate_idempotent_uuid(
+        namespace: uuid.UUID | str, unique_string: str, case_sensitive: bool = False
+    ) -> str:
+        """
+        Generates an idempotnent UUID, which is useful for creates
+
+        Args:
+            namespace (GUID | str): A namespace for your id, it must be a UUID or a string in a UUID format
+            unique_string (str): A unique string like an email address, a tenant name.
+                Use a combination for more granularity:
+                    tenant-name:email
+                    vendor:product-name
+                    vendor:product-id
+                    etc
+
+        Returns:
+            str: a string representation of a UUID
+        """
+        if isinstance(namespace, str):
+            namespace = uuid.UUID(namespace)
+
+        if not unique_string:
+            raise ValueError("unique_string cannot be empty")
+
+        if not case_sensitive:
+            unique_string = unique_string.lower()
+
+        return str(uuid.uuid5(namespace, unique_string))
+
     @staticmethod
     def get_size_in_kb(input_string: str | dict) -> float:
         """

boto3_assist/version.py

@@ -1 +1 @@
-__version__ = '0.6.1'
+__version__ = '0.7.0'

{boto3_assist-0.6.1.dist-info → boto3_assist-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boto3_assist
-Version: 0.6.1
+Version: 0.7.0
 Summary: Additional boto3 wrappers to make your life a little easier
 Author-email: Eric Wilson <boto3-assist@geekcafe.com>
 License-File: LICENSE-EXPLAINED.txt

{boto3_assist-0.6.1.dist-info → boto3_assist-0.7.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
 boto3_assist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 boto3_assist/boto3session.py,sha256=Q9sByNC0r_aMQfHnIEnxtTaiCMUqikm8UeSTxV7-Np0,6632
 boto3_assist/connection.py,sha256=CNGkAIRyfrELoWrV0ziQBA3oHacNFuLL3i8faUPRiO0,3486
-boto3_assist/connection_tracker.py,sha256=bfImvNVX-0Lhb-ombOurWUpNLdI0qVDil-kokBdIFkY,4345
+boto3_assist/connection_tracker.py,sha256=UgfR9RlvXf3A4ssMr3gDMpw89ka8mSRvJn4M34SzhbU,4378
 boto3_assist/http_status_codes.py,sha256=G0zRSWenwavYKETvDF9tNVUXQz3Ae2gXdBETYbjvJe8,3284
-boto3_assist/version.py,sha256=gd3s3RotD0_KL90Tua-YkOr60Jm2C2_wvlEhAT08068,22
+boto3_assist/version.py,sha256=09hol5ovL6ArILJIJZ-MiscnOjKa6RjaETmtdgJpW2c,22
 boto3_assist/aws_lambda/event_info.py,sha256=OkZ4WzuGaHEu_T8sB188KBgShAJhZpWASALKRGBOhMg,14648
 boto3_assist/cloudwatch/cloudwatch_connection.py,sha256=mnGWaLSQpHh5EeY7Ek_2o9JKHJxOELIYtQVMX1IaHn4,2480
 boto3_assist/cloudwatch/cloudwatch_connection_tracker.py,sha256=mzRtO1uHrcfJNh1XrGEiXdTqxwEP8d1RqJkraMNkgK0,410
@@ -35,8 +35,10 @@ boto3_assist/environment_services/environment_loader.py,sha256=zCA4mRdVWMLKzjDRv
 boto3_assist/environment_services/environment_variables.py,sha256=4ccBKdPt6O7hcRT3zBHd8vqu8yQU8udmoD5RLAT3iMs,6801
 boto3_assist/errors/custom_exceptions.py,sha256=zC2V2Y4PUtKj3uLPn8mB-JessksKWJWvKM9kp1dmvt8,760
 boto3_assist/models/serializable_model.py,sha256=ZMrRJRvJWLY8PBSKK_nPCgYKv1qUxDPEVdcADKbIHsI,266
-boto3_assist/s3/s3.py,sha256=DFCJs5z1mMIT8nZfnqPyr_cvhi9-FePuYH--tzD7b5E,17104
-boto3_assist/s3/s3_connection.py,sha256=FI1AhZV4UbTXQRTb4TqL9mv88Gt018rPZVFBvLetVAw,2163
+boto3_assist/s3/s3.py,sha256=ESTPXtyDi8mrwHaYNWjQLNGTuTUV4CxKDqw-O_KGzKs,2052
+boto3_assist/s3/s3_bucket.py,sha256=GfyBbuI5BWz_ybwU_nDqUZiC0wt24PNt49GKZmb05OY,2018
+boto3_assist/s3/s3_connection.py,sha256=0JgEDNoDFPQTo5hQe-lS8mWnFBJ2S8MDSl0LPG__lZo,2008
+boto3_assist/s3/s3_object.py,sha256=kH9apiVZ-lWGtcSqcLvaAI-2g011ecrvg8wP7y9u140,19482
 boto3_assist/utilities/datetime_utility.py,sha256=dgAMB9VqakrYIPXlSoVQiLSsc_yhrJK4gMfJO9mX90w,11112
 boto3_assist/utilities/dictionaroy_utility.py,sha256=PjUrerEd6uhmw37A-k7xe_DWHvXZGGoMqT6xjUqmWBI,893
 boto3_assist/utilities/file_operations.py,sha256=Zy8fu8fpuVNf7U9NimrLdy5FRF71XSI159cnRdzmzGY,3411
@@ -44,9 +46,9 @@ boto3_assist/utilities/http_utility.py,sha256=koFv7Va-8ng-47Nt1K2Sh7Ti95e62IYs9V
 boto3_assist/utilities/logging_utility.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 boto3_assist/utilities/numbers_utility.py,sha256=KIiNkBSRbfNWvtXG5SdHp625LTiW12VtADUa4ZlWMFo,8709
 boto3_assist/utilities/serialization_utility.py,sha256=KJhit0lI1lr8hhndW6UhKQSZD3c2RH555Ni7eP-e5Ms,16557
-boto3_assist/utilities/string_utility.py,sha256=sBY80aQO-fTRanlHryZFMQBxdo6OvLRvnZjZrQepHlI,9283
-boto3_assist-0.6.1.dist-info/METADATA,sha256=MVB3qF040hI48bRH8xhXXkyX9aewgYUUiGtUubi2l-0,1728
-boto3_assist-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-boto3_assist-0.6.1.dist-info/licenses/LICENSE-EXPLAINED.txt,sha256=WFREvTpfTjPjDHpOLADxJpCKpIla3Ht87RUUGii4ODU,606
-boto3_assist-0.6.1.dist-info/licenses/LICENSE.txt,sha256=PXDhFWS5L5aOTkVhNvoitHKbAkgxqMI2uUPQyrnXGiI,1105
-boto3_assist-0.6.1.dist-info/RECORD,,
+boto3_assist/utilities/string_utility.py,sha256=5BpDaqGZI8cSM-3YFQLU1fKcWcqG9r1_GPgDstCWFIs,10318
+boto3_assist-0.7.0.dist-info/METADATA,sha256=0cBmMT5N4of5YzMQhUlK3S_ovOEaxRItYa_JQnjH16I,1728
+boto3_assist-0.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+boto3_assist-0.7.0.dist-info/licenses/LICENSE-EXPLAINED.txt,sha256=WFREvTpfTjPjDHpOLADxJpCKpIla3Ht87RUUGii4ODU,606
+boto3_assist-0.7.0.dist-info/licenses/LICENSE.txt,sha256=PXDhFWS5L5aOTkVhNvoitHKbAkgxqMI2uUPQyrnXGiI,1105
+boto3_assist-0.7.0.dist-info/RECORD,,