boto3-assist 0.6.0__tar.gz → 0.7.0__tar.gz

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/.gitignore

@@ -163,3 +163,5 @@ cython_debug/
 
 .imports
 .env.development
+.unittest
+.unittests

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/.vscode/settings.json

@@ -12,8 +12,7 @@
     "python.analysis.extraPaths": [                
         "${workspaceFolder}",
         "${workspaceFolder}/examples",
-        "${workspaceFolder}/src",
-        "${workspaceFolder}/src/boto3_assist",        
+        "${workspaceFolder}/src",                
         "${workspaceFolder}/tests",
         "${workspaceFolder}/devops",
         "${workspaceFolder}/devops/cdk",   

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boto3_assist
-Version: 0.6.0
+Version: 0.7.0
 Summary: Additional boto3 wrappers to make your life a little easier
 Author-email: Eric Wilson <boto3-assist@geekcafe.com>
 License-File: LICENSE-EXPLAINED.txt

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["src/boto3_assist"]
 
 [project]
 name = "boto3_assist"
-version = "0.6.0"
+version = "0.7.0"
 
 authors = [
   { name="Eric Wilson", email="boto3-assist@geekcafe.com" }

boto3_assist-0.7.0/run_unit_tests.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+python --version
+python -m venv .unittest
+source ./.unittest/bin/activate
+which python
+
+
+pip install --upgrade pip
+pip install -r ./requirements.txt
+pip install -r ./requirements-dev.txt
+
+
+echo "running unit test"
+python -m unittest discover -s tests -p "*_test.py"
+
+if [ $? -eq 0 ]; then
+    echo "Tests passed successfully"
+else
+    echo "No tests found or tests failed"
+    exit 1
+fi

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/src/boto3_assist/connection_tracker.py

@@ -67,8 +67,8 @@ class ConnectionTracker:
 
             if not self.issue_stack_trace:
                 stack_trace_message = (
-                    f"\nTo add additional information to the log and determine where additional connections are being created, "
-                    f"set the environment variable {self.__stack_trace_env_var} to true.\n"
+                    f"📄 NOTE: To add additional information 👀 to the log and determine where additional connections are being created: "
+                    f"set the environment variable 👉{self.__stack_trace_env_var}👈 to true ✅. \n"
                 )
             else:
                 stack = "\n".join(traceback.format_stack())
@@ -83,8 +83,8 @@ class ConnectionTracker:
                 "instead of creating a new one. Connections are expensive in terms of time and latency. "
                 "If you are seeing performance issues, check how and where you are creating your "
                 "connections. You should be able to pass the connection to your other objects "
-                "and reuse your boto3 connections."
-                "\n\nMOCK Testing may show this message as well, in which case you can dismiss this warning.\n\n"
+                "and reuse your boto3 connections. "
+                "\n🧪 MOCK Testing may show this message as well, in which case you can dismiss this warning.🧪\n"
                 f"{stack_trace_message}"
             )
 

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/src/boto3_assist/models/serializable_model.py

@@ -5,5 +5,5 @@ MIT License.  See Project Root for the license information.
 """
 
 from __future__ import annotations
-from typing import TypeVar, Dict, Any
-from boto3_assist.utilities.serialization_utility import SerializableModel
+
+from boto3_assist.utilities.serialization_utility import SerializableModel  # noqa: F401 # pylint: disable=unused-import

boto3_assist-0.7.0/src/boto3_assist/s3/s3.py

@@ -0,0 +1,64 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional, cast
+
+from aws_lambda_powertools import Logger
+
+from boto3_assist.s3.s3_connection import S3Connection
+from boto3_assist.s3.s3_object import S3Object
+from boto3_assist.s3.s3_bucket import S3Bucket
+
+logger = Logger(child=True)
+
+
+class S3(S3Connection):
+    """Common S3 Actions"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_end_point_url: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        """_summary_
+
+        Args:
+            aws_profile (Optional[str], optional): _description_. Defaults to None.
+            aws_region (Optional[str], optional): _description_. Defaults to None.
+            aws_end_point_url (Optional[str], optional): _description_. Defaults to None.
+            aws_access_key_id (Optional[str], optional): _description_. Defaults to None.
+            aws_secret_access_key (Optional[str], optional): _description_. Defaults to None.
+        """
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_end_point_url=aws_end_point_url,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__s3_object: S3Object | None = None
+        self.__s3_bucket: S3Bucket | None = None
+
+    @property
+    def object(self) -> S3Object:
+        """s3 object"""
+        if self.__s3_object is None:
+            connection = cast(S3Connection, self)
+            self.__s3_object = S3Object(connection)
+        return self.__s3_object
+
+    @property
+    def bucket(self) -> S3Bucket:
+        """s3 bucket"""
+        if self.__s3_bucket is None:
+            connection = cast(S3Connection, self)
+            self.__s3_bucket = S3Bucket(connection)
+        return self.__s3_bucket

boto3_assist-0.7.0/src/boto3_assist/s3/s3_bucket.py

@@ -0,0 +1,67 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Any, Dict
+
+from aws_lambda_powertools import Logger
+from botocore.exceptions import ClientError
+
+
+from boto3_assist.s3.s3_connection import S3Connection
+
+logger = Logger(child=True)
+
+
+class S3Bucket:
+    """Common S3 Actions"""
+
+    def __init__(self, connection: S3Connection):
+        self.connection = connection or S3Connection()
+
+    def create(self, *, bucket_name: str) -> Dict[str, Any]:
+        """
+        Create an S3 bucket
+        :param bucket_name: Bucket to create
+        :return: True if bucket is created, else False
+        """
+        try:
+            response = self.connection.client.create_bucket(Bucket=bucket_name)
+            logger.info(f"Bucket {bucket_name} created")
+
+            return dict(response)
+        except ClientError as e:
+            logger.exception(e)
+            raise e
+
+    def enable_versioning(self, *, bucket_name: str) -> None:
+        """
+        Enable versioning on an S3 bucket
+        :param bucket_name: Bucket to enable versioning on
+        :return: None
+        """
+        try:
+            self.connection.client.put_bucket_versioning(
+                Bucket=bucket_name, VersioningConfiguration={"Status": "Enabled"}
+            )
+            logger.info(f"Versioning enabled on bucket {bucket_name}")
+        except ClientError as e:
+            logger.exception(e)
+            raise e
+
+    def disable_versioning(self, *, bucket_name: str) -> None:
+        """
+        Disable versioning on an S3 bucket
+        :param bucket_name: Bucket to disable versioning on
+        :return: None
+        """
+        try:
+            self.connection.client.put_bucket_versioning(
+                Bucket=bucket_name, VersioningConfiguration={"Status": "Suspended"}
+            )
+            logger.info(f"Versioning disabled on bucket {bucket_name}")
+        except ClientError as e:
+            logger.exception(e)
+            raise e

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/src/boto3_assist/s3/s3_connection.py

@@ -8,10 +8,7 @@ from typing import Optional
 from typing import TYPE_CHECKING
 
 from aws_lambda_powertools import Logger
-from boto3_assist.boto3session import Boto3SessionManager
-from boto3_assist.environment_services.environment_variables import (
-    EnvironmentVariables,
-)
+
 from boto3_assist.connection import Connection
 
 if TYPE_CHECKING:

boto3_assist-0.6.0/src/boto3_assist/s3/s3.py → boto3_assist-0.7.0/src/boto3_assist/s3/s3_object.py

@@ -8,7 +8,7 @@ import os
 import tempfile
 import time
 import io
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional, List
 
 from aws_lambda_powertools import Logger
 from botocore.exceptions import ClientError
@@ -19,61 +19,100 @@ from boto3_assist.utilities.datetime_utility import DatetimeUtility
 from boto3_assist.utilities.file_operations import FileOperations
 from boto3_assist.utilities.http_utility import HttpUtility
 
+
 logger = Logger(child=True)
 
 
-class S3(S3Connection):
-    """Common S3 Actions"""
+class S3Object:
+    """S3 Object Actions"""
 
-    def __init__(
-        self,
-        *,
-        aws_profile: Optional[str] = None,
-        aws_region: Optional[str] = None,
-        aws_end_point_url: Optional[str] = None,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-    ) -> None:
-        """_summary_
+    def __init__(self, connection: S3Connection):
+        self.connection = connection or S3Connection()
 
-        Args:
-            aws_profile (Optional[str], optional): _description_. Defaults to None.
-            aws_region (Optional[str], optional): _description_. Defaults to None.
-            aws_end_point_url (Optional[str], optional): _description_. Defaults to None.
-            aws_access_key_id (Optional[str], optional): _description_. Defaults to None.
-            aws_secret_access_key (Optional[str], optional): _description_. Defaults to None.
+    def delete(self, *, bucket_name: str, key: str) -> Dict[str, Any]:
         """
-        super().__init__(
-            aws_profile=aws_profile,
-            aws_region=aws_region,
-            aws_end_point_url=aws_end_point_url,
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
-        )
+        Deletes an object key
 
-    def create_bucket(self, *, bucket_name: str) -> None:
-        """
-        Create an S3 bucket
-        :param bucket_name: Bucket to create
-        :return: True if bucket is created, else False
+        Args:
+            bucket_name (str): The AWS Bucket Name
+            key (str): The Object Key
         """
+        s3 = self.connection.client
+        # see if the object exists
         try:
-            self.client.create_bucket(Bucket=bucket_name)
-            logger.info(f"Bucket {bucket_name} created")
-        except ClientError as e:
-            logger.exception(e)
-            raise e
+            response = s3.head_object(Bucket=bucket_name, Key=key)
+            response = s3.delete_object(Bucket=bucket_name, Key=key)
+        except s3.exceptions.NoSuchKey:
+            response = {"ResponseMetadata": {"HTTPStatusCode": 404}}
+        except s3.exceptions.ClientError as e:
+            if e.response.get("Error", {}).get("Code") == "404":
+                response = {"ResponseMetadata": {"HTTPStatusCode": 404}}
+            else:
+                raise e
+
+        return dict(response)
+
+    def delete_all_versions(
+        self, *, bucket_name: str, key: str, include_deleted: bool = False
+    ) -> List[str]:
+        """
+        Deletes an object key and all the versions for that object key
+
+        Args:
+            bucket_name (str): The AWS Bucket Name
+            key (str): The Object Kuye
+            include_deleted (bool, optional): Should deleted files be removed as well.
+                If True it will look for the object keys with the deleted marker and remove it.
+                Defaults to False.
+        """
+        s3 = self.connection.client
+        paginator = s3.get_paginator("list_object_versions")
+        files: List[str] = []
+
+        for page in paginator.paginate(Bucket=bucket_name, Prefix=key):
+            # Delete object versions
+            if "Versions" in page:
+                for version in page["Versions"]:
+                    s3.delete_object(
+                        Bucket=bucket_name,
+                        Key=version["Key"],
+                        VersionId=version["VersionId"],
+                    )
+
+                    files.append(f"{version['Key']} - {version['VersionId']}")
+
+                if include_deleted:
+                    # delete a previous files that may have just been a soft delete.
+                    if "DeleteMarkers" in page:
+                        for marker in page["DeleteMarkers"]:
+                            s3.delete_object(
+                                Bucket=bucket_name,
+                                Key=marker["Key"],
+                                VersionId=marker["VersionId"],
+                            )
+
+                            files.append(
+                                f"{marker['Key']}:{marker['VersionId']}:delete-marker"
+                            )
+            else:
+                response = self.delete(bucket_name=bucket_name, key=key)
+                if response["ResponseMetadata"]["HTTPStatusCode"] == 404:
+                    return files
+
+                files.append(key)
+
+        return files
 
     def generate_presigned_url(
         self,
         *,
         bucket_name: str,
         key_path: str,
-        user_id: str,
         file_name: str,
-        meta_data: dict | None = None,
-        expiration=3600,
-        method_type="POST",
+        meta_data: Optional[dict] = None,
+        expiration: int = 3600,
+        method_type: str = "POST",
+        user_id: Optional[str] = None,
     ) -> Dict[str, Any]:
         """
         Create a signed URL for uploading a file to S3.
@@ -109,7 +148,7 @@ class S3(S3Connection):
 
         signed_url: str | Dict[str, Any]
         if method_type == "PUT":
-            signed_url = self.client.generate_presigned_url(
+            signed_url = self.connection.client.generate_presigned_url(
                 "put_object",
                 Params={
                     "Bucket": f"{bucket_name}",
@@ -124,13 +163,13 @@ class S3(S3Connection):
                 ExpiresIn=expiration,  # URL is valid for x seconds
             )
         elif method_type == "POST":
-            signed_url = self.client.generate_presigned_post(
+            signed_url = self.connection.client.generate_presigned_post(
                 bucket_name,
                 key,
                 ExpiresIn=expiration,  # URL is valid for x seconds
             )
         elif method_type == "GET":
-            signed_url = self.client.generate_presigned_url(
+            signed_url = self.connection.client.generate_presigned_url(
                 "get_object",
                 Params={
                     "Bucket": f"{bucket_name}",
@@ -175,7 +214,7 @@ class S3(S3Connection):
             file_obj: bytes = (
                 file_obj.encode("utf-8") if isinstance(file_obj, str) else file_obj
             )
-            self.client.upload_fileobj(
+            self.connection.client.upload_fileobj(
                 Fileobj=io.BytesIO(file_obj), Bucket=bucket, Key=key
             )
 
@@ -219,7 +258,7 @@ class S3(S3Connection):
             }
         )
         try:
-            self.client.upload_file(local_file_path, bucket, key)
+            self.connection.client.upload_file(local_file_path, bucket, key)
 
         except ClientError as ce:
             error = {
@@ -394,7 +433,9 @@ class S3(S3Connection):
         error = None
 
         try:
-            response = dict(self.client.get_object(Bucket=bucket_name, Key=key))
+            response = dict(
+                self.connection.client.get_object(Bucket=bucket_name, Key=key)
+            )
 
             logger.debug(
                 {"metric_filter": "s3_download_response", "response": str(response)}
@@ -465,7 +506,7 @@ class S3(S3Connection):
 
         error: str | None = None
         try:
-            self.client.download_file(bucket, key, local_path)
+            self.connection.client.download_file(bucket, key, local_path)
 
         except Exception as e:  # pylint: disable=W0718
             error = str(e)
@@ -542,3 +583,24 @@ class S3(S3Connection):
         Decodes bytes to a string
         """
         return file_obj.decode(encoding=encoding, errors=errors)
+
+    def list_versions(self, bucket: str, prefix: str = "") -> List[str]:
+        """
+        List all versions of objects in an S3 bucket with a given prefix.
+
+        Args:
+            bucket (str): The name of the S3 bucket.
+            prefix (str, optional): The prefix to filter objects by. Defaults to "".
+
+        Returns:
+            list: A list of dictionaries containing information about each object version.
+        """
+        versions = []
+        paginator = self.connection.client.get_paginator("list_object_versions")
+        page_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
+
+        for page in page_iterator:
+            if "Versions" in page:
+                versions.extend(page["Versions"])
+
+        return versions

{boto3_assist-0.6.0 → boto3_assist-0.7.0}/src/boto3_assist/utilities/serialization_utility.py

@@ -1,16 +1,15 @@
 """Serialization Utility"""
 
-from datetime import datetime
-from decimal import Decimal
-from typing import Dict, List, TypeVar, Any
-import json
-import jsons
 import datetime as dt
 import decimal
 import inspect
+import json
 import uuid
-from aws_lambda_powertools import Logger
+from datetime import datetime
+from decimal import Decimal
+from typing import Any, Dict, List, TypeVar
 
+from aws_lambda_powertools import Logger
 
 T = TypeVar("T")
 
@@ -53,6 +52,15 @@ class SerializableModel:
             instance=self, serialize_fn=lambda x: x, include_none=True
         )
 
+    def to_wide_dictionary(self) -> Dict:
+        """
+        Dumps an object to dictionary structure
+        """
+
+        dump = Serialization.to_wide_dictionary(model=self)
+
+        return dump
+
 
 class JsonEncoder(json.JSONEncoder):
     """
@@ -108,6 +116,32 @@ class Serialization:
 
         return dump
 
+    @staticmethod
+    def to_wide_dictionary(model: object) -> Dict:
+        """
+        Dumps an object to dictionary structure
+        """
+
+        dump = Serialization.to_dict(
+            instance=model, serialize_fn=lambda x: x, include_none=True
+        )
+
+        # have a dictionary now let's flatten out
+        flat_dict = {}
+        for key, value in dump.items():
+            if isinstance(value, dict):
+                for sub_key, sub_value in value.items():
+                    flat_dict[f"{key}_{sub_key}"] = sub_value
+            elif isinstance(value, list):
+                for i, sub_value in enumerate(value):
+                    sub_dict = Serialization.to_wide_dictionary(sub_value)
+                    for sub_key, sub_value in sub_dict.items():
+                        flat_dict[f"{key}_{i}_{sub_key}"] = sub_value
+            else:
+                flat_dict[key] = value
+
+        return flat_dict
+
     @staticmethod
     def map(source: object, target: T, coerce: bool = True) -> T | None:
         """Map an object from one object to another"""
@@ -118,12 +152,92 @@ class Serialization:
             source_dict = Serialization.convert_object_to_dict(source)
             if not isinstance(source_dict, dict):
                 return None
-        return Serialization.load_properties(
+        return Serialization._load_properties(
             source=source_dict, target=target, coerce=coerce
         )
 
     @staticmethod
-    def load_properties(
+    def to_wide_dictionary_list(
+        data: Dict[str, Any] | List[Dict[str, Any]],
+        remove_collisions: bool = True,
+        raise_error_on_collision: bool = False,
+    ) -> List[Dict[str, Any]]:
+        """
+        Converts a dictionary or list of dictionaries to a list of dictionaries.
+
+        :param data: Dictionary or list of dictionaries to be converted
+        :param remove_collisions: If True, removes duplicate keys from the dictionaries
+        :return: List of dictionaries
+        """
+
+        collisions = []
+
+        def recursive_flatten(prefix, obj):
+            """
+            Recursively flattens a JSON object.
+
+            :param prefix: Current key prefix
+            :param obj: Object to flatten
+            :return: List of flattened dictionaries
+            """
+            if isinstance(obj, list):
+                result = []
+                for _, item in enumerate(obj):
+                    x = recursive_flatten("", item)
+                    result.extend(x)
+                return result
+            elif isinstance(obj, dict):
+                result = [{}]
+                for key, value in obj.items():
+                    sub_result = recursive_flatten(
+                        f"{prefix}_{key}" if prefix else key, value
+                    )
+                    new_result = []
+                    for entry in result:
+                        for sub_entry in sub_result:
+                            # remove any collisions
+
+                            for k in entry:
+                                if k in sub_entry:
+                                    if k not in collisions:
+                                        logger.debug(f"Collision detected: {k}")
+                                        collisions.append(k)
+                            merged = entry.copy()
+                            merged.update(sub_entry)
+                            new_result.append(merged)
+                    result = new_result
+                return result
+            else:
+                return [{prefix: obj}] if prefix else []
+
+        results = recursive_flatten("", data)
+        if remove_collisions:
+            results = Serialization.remove_collisions(results, collisions)
+
+        if raise_error_on_collision and len(collisions) > 0:
+            raise ValueError(f"Duplicate keys detected: {collisions}")
+
+        return results
+
+    @staticmethod
+    def remove_collisions(
+        data: List[Dict[str, Any]], collisions: List[str]
+    ) -> List[Dict[str, Any]]:
+        """
+        Removes collisions from a list of dictionaries.
+
+        :param data: List of dictionaries
+        :param collisions: List of collision keys
+        :return: List of dictionaries with collisions removed
+        """
+        for c in collisions:
+            for r in data:
+                if c in r:
+                    del r[c]
+        return data
+
+    @staticmethod
+    def _load_properties(
         source: dict,
         target: T,
         coerce: bool = True,
@@ -183,9 +297,9 @@ class Serialization:
                         attr.clear()
                         attr.extend(value)
                     elif isinstance(attr, dict) and isinstance(value, dict):
-                        Serialization.load_properties(value, attr, coerce=coerce)
+                        Serialization._load_properties(value, attr, coerce=coerce)
                     elif hasattr(attr, "__dict__") and isinstance(value, dict):
-                        Serialization.load_properties(value, attr, coerce=coerce)
+                        Serialization._load_properties(value, attr, coerce=coerce)
                     else:
                         setattr(target, key, value)
                 except ValueError as e:
@@ -230,12 +344,18 @@ class Serialization:
 
     @staticmethod
     def to_dict(
-        instance: SerializableModel,
+        instance: SerializableModel | dict,
         serialize_fn,
         include_none: bool = True,
     ) -> Dict[str, Any]:
         """To Dict / Dictionary"""
 
+        if instance is None:
+            return {}
+
+        if isinstance(instance, dict):
+            return instance
+
         def is_primitive(value):
             """Check if the value is a primitive data type."""
             return isinstance(value, (str, int, bool, type(None)))