boto3-assist 0.1.13__py3-none-any.whl → 0.2.0__py3-none-any.whl

boto3_assist/boto3session.py

@@ -45,6 +45,7 @@ class Boto3SessionManager:
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
         self.aws_session_token = aws_session_token
+
         self.__session: Any = None
         self.__client: Any = None
         self.__resource: Any = None
@@ -90,7 +91,7 @@ class Boto3SessionManager:
 
     def __get_aws_session(
         self, aws_profile: Optional[str] = None, aws_region: Optional[str] = None
-    ) -> boto3.Session:
+    ) -> boto3.Session | None:
         """Get a boto3 session for AWS."""
         logger.debug({"profile": aws_profile, "region": aws_region})
         try:
@@ -153,7 +154,7 @@ class Boto3SessionManager:
             )
         return self.__resource
 
-    def __create_boto3_session(self) -> boto3.Session:
+    def __create_boto3_session(self) -> boto3.Session | None:
         try:
             session = boto3.Session(
                 profile_name=self.aws_profile,

boto3_assist/cloudwatch/cloudwatch_connection.py

@@ -0,0 +1,86 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+from boto3_assist.boto3session import Boto3SessionManager
+from boto3_assist.environment_services.environment_variables import (
+    EnvironmentVariables,
+)
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+from boto3_assist.connection import Connection
+
+if TYPE_CHECKING:
+    from mypy_boto3_cloudwatch import CloudWatchClient, CloudWatchServiceResource
+else:
+    CloudWatchClient = object
+    CloudWatchServiceResource = object
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class CloudWatchConnection(Connection):
+    """CW Environment"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            service_name="cloudwatch",
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__client: CloudWatchClient | None = None
+        self.__resource: CloudWatchServiceResource | None = None
+
+        self.raise_on_error: bool = True
+
+    @property
+    def client(self) -> CloudWatchClient:
+        """CloudWatch Client Connection"""
+        if self.__client is None:
+            logger.info("Creating CloudWatch Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("CloudWatch Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: CloudWatchClient):
+        logger.info("Setting CloudWatch Client")
+        self.__client = value
+
+    @property
+    def resource(self) -> CloudWatchServiceResource:
+        """CloudWatch Resource Connection"""
+        if self.__resource is None:
+            logger.info("Creating CloudWatch Resource")
+            self.__resource = self.session.resource
+
+        if self.raise_on_error and self.__resource is None:
+            raise RuntimeError("CloudWatch Resource is not available")
+
+        return self.__resource
+
+    @resource.setter
+    def resource(self, value: CloudWatchServiceResource):
+        logger.info("Setting CloudWatch Resource")
+        self.__resource = value

boto3_assist/cloudwatch/cloudwatch_connection_tracker.py

@@ -0,0 +1,17 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from boto3_assist.connection_tracker import ConnectionTracker
+
+
+class CloudWatchConnectionTracker(ConnectionTracker):
+    """
+    Tracks CloudWatch Connection Requests.
+    Useful in for performance tuning and debugging.
+    """
+
+    def __init__(self) -> None:
+        super().__init__("CloudWatch")

boto3_assist/cloudwatch/cloudwatch_log_connection.py

@@ -0,0 +1,62 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+from boto3_assist.connection import Connection
+
+if TYPE_CHECKING:
+    from mypy_boto3_logs import CloudWatchLogsClient
+else:
+    CloudWatchLogsClient = object
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class CloudWatchConnection(Connection):
+    """CW Logs Environment"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            service_name="logs",
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__client: CloudWatchLogsClient | None = None
+
+    @property
+    def client(self) -> CloudWatchLogsClient:
+        """CloudWatch Client Connection"""
+        if self.__client is None:
+            logger.debug("Creating CloudWatch Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("CloudWatch Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: CloudWatchLogsClient):
+        logger.debug("Setting CloudWatch Client")
+        self.__client = value

boto3_assist/cloudwatch/cloudwatch_logs.py

@@ -0,0 +1,39 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional, List, Dict, Any
+from boto3_assist.cloudwatch.cloudwatch_log_connection import CloudWatchConnection
+
+
+class CloudWatchLogs(CloudWatchConnection):
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+    def list_log_groups(self):
+        """Retrieve all log groups in the AWS account."""
+        log_groups: List[Dict[str, Any]] = []
+        paginator = self.client.get_paginator("describe_log_groups")
+        for page in paginator.paginate():
+            log_groups.extend(page["logGroups"])  # type: ignore[arg-type]
+        return log_groups
+
+
+def main():
+    query: CloudWatchLogs = CloudWatchLogs()
+    result = query.list_log_groups()
+    print(result)

boto3_assist/cloudwatch/cloudwatch_query.py

@@ -0,0 +1,191 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+import os
+from datetime import datetime, timedelta, UTC
+from typing import Optional, Dict, Any, List
+from boto3_assist.cloudwatch.cloudwatch_connection import CloudWatchConnection
+from boto3_assist.cloudwatch.cloudwatch_logs import CloudWatchLogs
+
+
+class CloudWatchQuery(CloudWatchConnection):
+    """Query Cloud Watch"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__cw_logs: CloudWatchLogs | None = None
+
+    @property
+    def cw_logs(self) -> CloudWatchLogs:
+        """CloudWatch Logs Connection"""
+        if self.__cw_logs is None:
+            self.__cw_logs = CloudWatchLogs(
+                aws_profile=self.aws_profile,
+                aws_region=self.aws_region,
+                aws_access_key_id=self.aws_access_key_id,
+                aws_secret_access_key=self.aws_secret_access_key,
+            )
+        return self.__cw_logs
+
+    def get_log_group_size(
+        self, log_group_name: str, start_time: datetime, end_time: datetime
+    ) -> Dict[str, Any]:
+        """
+        Get the log group size for a given period of time
+        Args:
+            log_group_name (str): _description_
+            start_time (datetime): _description_
+            end_time (datetime): _description_
+
+        Returns:
+            _type_: _description_
+        """
+        response = self.client.get_metric_data(
+            MetricDataQueries=[
+                {
+                    "Id": "storedBytes",
+                    "MetricStat": {
+                        "Metric": {
+                            "Namespace": "AWS/Logs",
+                            # "MetricName": "StoredBytes",
+                            "MetricName": "IncomingBytes",
+                            "Dimensions": [
+                                {"Name": "LogGroupName", "Value": log_group_name}
+                            ],
+                        },
+                        "Period": 86400,  # Daily data
+                        "Stat": "Sum",
+                    },
+                    "ReturnData": True,
+                },
+            ],
+            StartTime=start_time,
+            EndTime=end_time,
+        )
+
+        # Extract the total size in bytes for the period
+        size: float = 0.0
+        if response["MetricDataResults"]:
+            # Access the first MetricDataResult
+            metric_data_result = response["MetricDataResults"][0]
+            # Sum the values if they exist
+            size = (
+                sum(metric_data_result["Values"]) if metric_data_result["Values"] else 0
+            )
+        else:
+            size = 0
+
+        size_mb = size / (1024 * 1024)
+        size_gb = size_mb / 1024
+        resp: Dict[str, Any] = {
+            "LogGroupName": log_group_name,
+            "Size": {
+                "Bytes": size,
+                "MB": size_mb,
+                "GB": size_gb,
+            },
+            "StartDate": start_time.isoformat(),
+            "EndDate": end_time.isoformat(),
+        }
+
+        return resp
+
+    def get_log_sizes(
+        self,
+        start_date_time: datetime | None = None,
+        end_date_time: datetime | None = None,
+        days: int | None = 7,
+        top: int = 0,
+    ) -> List[Dict[str, Any]]:
+        """
+        Gets the log sizes for all log groups
+
+        Args:
+            start_date_time (datetime | None, optional): The Start Date. Defaults to None.
+                If None it's set to now in UTC time - the days field
+            end_date_time (datetime | None, optional): he Start Date. Defaults to None.
+                If None it's set to not in UTC time
+            days (int | None, optional): The days offset. Defaults to 7.
+            top (int, optional): If greater than zero it will return the top x after sorting
+                Defaults to 0.
+
+        Returns:
+            list: _description_
+        """
+        if not days:
+            days = 7
+        start_time = start_date_time or (datetime.now(UTC) - timedelta(days=days))
+        end_time = end_date_time or datetime.now(UTC)
+
+        # Step 1: List all log groups
+        log_groups = self.cw_logs.list_log_groups()
+        log_group_sizes = []
+
+        # Step 2: Get sizes for each log group
+        for log_group in log_groups:
+            log_group_name = log_group["logGroupName"]
+
+            size_info = self.get_log_group_size(log_group_name, start_time, end_time)
+            log_group_sizes.append(size_info)
+
+        # Step 3: Sort by size
+        # top_log_groups = sorted(log_group_sizes, key=lambda x: x[1], reverse=True)
+        top_log_groups = sorted(
+            log_group_sizes,
+            key=lambda x: x.get("Size", {}).get("Bytes", 0),
+            reverse=True,
+        )
+        if top and top > 0:
+            # find the top x if provided
+            top_log_groups = top_log_groups[:top]
+
+        return top_log_groups
+
+
+def main():
+    log_group = os.environ.get("LOG_GROUP_QUERY_SAMPLE", "<enter-log-group-here>")
+    start = datetime.now() - timedelta(days=7)  # Last 30 days
+    end = datetime.now()
+    cw_query: CloudWatchQuery = CloudWatchQuery()
+    result = cw_query.get_log_group_size(log_group, start, end)
+    print(result)
+
+    top = 25
+    days = 7
+    top_log_groups = cw_query.get_log_sizes(top=top, days=days)
+    print(f"Top {top} log groups by size for the last week:")
+
+    for top_log_group in top_log_groups:
+        log_group_name = top_log_group["LogGroupName"]
+        size_in_bytes = top_log_group.get("Size", {}).get("Bytes", 0)
+        size_in_megs = top_log_group.get("Size", {}).get("MB", 0)
+        size_in_gigs = top_log_group.get("Size", {}).get("GB", 0)
+        size: str = ""
+        if size_in_gigs > 1:
+            size = f"{size_in_gigs:.2f} GB"
+        elif size_in_megs > 1:
+            size = f"{size_in_megs:.2f} MB"
+        else:
+            size = f"{size_in_bytes} bytes"
+
+        print(f"{size}: {log_group_name}")
+
+
+if __name__ == "__main__":
+    main()

boto3_assist/connection.py

@@ -0,0 +1,101 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+
+from aws_lambda_powertools import Logger
+from boto3_assist.boto3session import Boto3SessionManager
+from boto3_assist.environment_services.environment_variables import (
+    EnvironmentVariables,
+)
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class Connection:
+    """Boto 3 Connection"""
+
+    def __init__(
+        self,
+        *,
+        service_name: Optional[str] = None,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        self.aws_profile = aws_profile or EnvironmentVariables.AWS.profile()
+        self.aws_region = aws_region or EnvironmentVariables.AWS.region()
+
+        self.aws_access_key_id = (
+            aws_access_key_id or EnvironmentVariables.AWS.DynamoDB.aws_access_key_id()
+        )
+        self.aws_secret_access_key = (
+            aws_secret_access_key
+            or EnvironmentVariables.AWS.DynamoDB.aws_secret_access_key()
+        )
+        self.__session: Boto3SessionManager | None = None
+
+        self.__service_name: str | None = service_name
+        self.raise_on_error: bool = True
+
+    def setup(self, setup_source: Optional[str] = None) -> None:
+        """
+        Setup the environment.  Automatically called via init.
+        You can run setup at anytime with new parameters.
+        Args: setup_source: Optional[str] = None
+            Defines the source of the setup.  Useful for logging.
+        Returns: None
+        """
+
+        logger.debug(
+            {
+                "metric_filter": f"{self.service_name}_connection_setup",
+                "source": f"{self.service_name} Connection",
+                "aws_profile": self.aws_profile,
+                "aws_region": self.aws_region,
+                "setup_source": setup_source,
+            }
+        )
+
+        self.__session = Boto3SessionManager(
+            service_name=self.service_name,
+            aws_profile=self.aws_profile,
+            aws_region=self.aws_region,
+            aws_access_key_id=self.aws_access_key_id,
+            aws_secret_access_key=self.aws_secret_access_key,
+        )
+
+        tracker.increment_connection()
+
+        self.raise_on_error = EnvironmentVariables.AWS.DynamoDB.raise_on_error_setting()
+
+    @property
+    def service_name(self) -> str:
+        """Service Name"""
+        if self.__service_name is None:
+            raise RuntimeError("Service Name is not available")
+        return self.__service_name
+
+    @service_name.setter
+    def service_name(self, value: str):
+        logger.debug("Setting Service Name")
+        self.__service_name = value
+
+    @property
+    def session(self) -> Boto3SessionManager:
+        """Session"""
+        if self.__session is None:
+            self.setup(setup_source="session init")
+
+        if self.__session is None:
+            raise RuntimeError("Session is not available")
+        return self.__session

boto3_assist/connection_tracker.py

@@ -16,7 +16,7 @@ class ConnectionTracker:
 
     def __init__(self, service_name: str) -> None:
         self.__stack_trace_env_var: str = "BOTO3_ASSIST_CONNECTION_STACK_TRACE"
-        self.__connection_couter: int = 0
+        self.__connection_counter: int = 0
         self.__service_name: str = service_name
         self.__issue_stack_trace: bool | None = None
 
@@ -31,7 +31,7 @@ class ConnectionTracker:
 
     def increment_connection(self) -> None:
         """Increments the connection counter"""
-        self.__connection_couter += 1
+        self.__connection_counter += 1
 
         if self.connection_count > 1:
             service_message = ""
@@ -53,27 +53,27 @@ class ConnectionTracker:
 
             self.__log_warning(
                 f"{service_message}"
-                f"Your dynamodb connection count is {self.connection_count}. "
+                f"Your boto3 connection count is {self.connection_count}. "
                 "Under most circumstances you should be able to use the same connection "
                 "vs. creating a new one.  Connections are expensive in terms of time / latency. "
                 "If you are seeing perforance issues, check how and where you are creating your "
-                "connections.  You should be able to pass the .db connection to your other objects "
-                "and reuse your dynamodb boto connections."
+                "connections.  You should be able to pass the connection to your other objects "
+                "and reuse your boto3 connections."
                 f"{stack_trace_message}"
             )
 
     def decrement_connection(self) -> None:
         """Decrements the connection counter"""
-        self.__connection_couter -= 1
+        self.__connection_counter -= 1
 
     @property
     def connection_count(self) -> int:
         """Returns the current connection count"""
-        return self.__connection_couter
+        return self.__connection_counter
 
     def reset(self) -> None:
         """Resets the connection counter"""
-        self.__connection_couter = 0
+        self.__connection_counter = 0
 
     def __log_warning(self, message: str) -> None:
         """Logs a warning message"""

boto3_assist/dynamodb/dynamodb.py

@@ -5,7 +5,7 @@ MIT License.  See Project Root for the license information.
 """
 
 import os
-from typing import List, Optional, overload
+from typing import List, Optional, overload, Dict, Any
 
 from aws_lambda_powertools import Tracer, Logger
 from boto3.dynamodb.conditions import (
@@ -77,7 +77,7 @@ class DynamoDB(DynamoDBConnection):
             dict: The Response from DynamoDB's put_item actions.
             It does not return the saved object, only the response.
         """
-        response = None
+        response: Dict[str, Any] = {}
 
         try:
             if not isinstance(item, dict):
@@ -93,13 +93,13 @@ class DynamoDB(DynamoDBConnection):
 
             if isinstance(item, dict) and isinstance(next(iter(item.values())), dict):
                 # Use boto3.client syntax
-                response = self.dynamodb_client.put_item(
-                    TableName=table_name, Item=item
+                response = dict(
+                    self.dynamodb_client.put_item(TableName=table_name, Item=item)
                 )
             else:
                 # Use boto3.resource syntax
                 table = self.dynamodb_resource.Table(table_name)
-                response = table.put_item(Item=item)
+                response = dict(table.put_item(Item=item))  # type: ignore[arg-type]
 
         except Exception as e:  # pylint: disable=w0718
             logger.exception(
@@ -117,7 +117,7 @@ class DynamoDB(DynamoDBConnection):
 
         if self.log_dynamodb_item_size:
             size_bytes: int = StringUtility.get_size_in_bytes(item)
-            size_kb: int = StringUtility.get_size_in_kb(item)
+            size_kb: float = StringUtility.get_size_in_kb(item)
             logger.info({"item_size": {"bytes": size_bytes, "kb": f"{size_kb:.2f}kb"}})
 
             if size_kb > 390:
@@ -173,7 +173,7 @@ class DynamoDB(DynamoDBConnection):
         expression_attribute_names: Optional[dict] = None,
         source: Optional[str] = None,
         call_type: str = "resource",
-    ) -> dict:
+    ) -> Dict[str, Any]:
         """
         Description:
             generic get_item dynamoDb call
@@ -186,12 +186,15 @@ class DynamoDB(DynamoDBConnection):
             if table_name is None:
                 raise ValueError("table_name must be provided when model is used.")
             if key is not None:
-                raise ValueError("key cannot be provided when model is used.")
+                raise ValueError(
+                    "key cannot be provided when model is used. "
+                    "When using the model, we'll automatically use the key defined."
+                )
             key = model.indexes.primary.key()
             if do_projections:
                 projection_expression = model.projection_expression
                 expression_attribute_names = model.projection_expression_attribute_names
-        elif key is None or table_name is None:
+        elif key is None and model is None:
             raise ValueError("Either 'key'  or 'model'  must be provided.")
 
         response = None
@@ -205,12 +208,18 @@ class DynamoDB(DynamoDBConnection):
             # only pass in args that aren't none
             valid_kwargs = {k: v for k, v in kwargs.items() if v is not None}
 
+            if table_name is None:
+                raise ValueError("table_name must be provided.")
             if call_type == "resource":
                 table = self.dynamodb_resource.Table(table_name)
-                response = table.get_item(Key=key, **valid_kwargs)
+                response = dict(table.get_item(Key=key, **valid_kwargs))  # type: ignore[arg-type]
             elif call_type == "client":
-                response = self.dynamodb_client.get_item(
-                    Key=key, TableName=table_name, **valid_kwargs
+                response = dict(
+                    self.dynamodb_client.get_item(
+                        Key=key,
+                        TableName=table_name,
+                        **valid_kwargs,  # type: ignore[arg-type]
+                    )
                 )
             else:
                 raise ValueError(
@@ -246,10 +255,12 @@ class DynamoDB(DynamoDBConnection):
             dict: dynamodb response dictionary
         """
         table = self.dynamodb_resource.Table(table_name)
-        response = table.update_item(
-            Key=key,
-            UpdateExpression=update_expression,
-            ExpressionAttributeValues=expression_attribute_values,
+        response = dict(
+            table.update_item(
+                Key=key,
+                UpdateExpression=update_expression,
+                ExpressionAttributeValues=expression_attribute_values,
+            )
         )
 
         return response
@@ -307,7 +318,7 @@ class DynamoDB(DynamoDBConnection):
             raise ValueError("Query failed: table_name must be provided.")
 
         table = self.dynamodb_resource.Table(table_name)
-        response = table.query(**kwargs)
+        response = dict(table.query(**kwargs))
 
         return response
 

boto3_assist/dynamodb/dynamodb_connection.py

@@ -92,6 +92,9 @@ class DynamoDBConnection:
         """Session"""
         if self.__session is None:
             self.setup(setup_source="session init")
+
+        if self.__session is None:
+            raise RuntimeError("Session is not available")
         return self.__session
 
     @property

boto3_assist/dynamodb/dynamodb_index.py

@@ -34,6 +34,29 @@ class DynamoDBIndexes:
         """Add a GSI/LSI index"""
         if index.name is None:
             raise ValueError("Index name cannot be None")
+
+        # if the index already exists, raise an exception
+        if index.name in self.__indexes:
+            raise ValueError(f"Index {index.name} already exists")
+        if index.name == DynamoDBIndexes.PRIMARY_INDEX:
+            raise ValueError(f"Index {index.name} is reserved for the primary index")
+        if index.partition_key is None:
+            raise ValueError("Index must have a partition key")
+
+        # check if the index.partition_key.attribute_name is already in the index
+        for _, v in self.__indexes.items():
+            if v.partition_key.attribute_name == index.partition_key.attribute_name:
+                raise ValueError(
+                    f"Index {index.name} already exists with partition key {index.partition_key.attribute_name}"
+                )
+        # check if the gsi1.sort_key.attribute_name exists
+        if index.sort_key is not None:
+            for _, v in self.__indexes.items():
+                if v.sort_key.attribute_name == index.sort_key.attribute_name:
+                    raise ValueError(
+                        f"Index {index.name} already exists with sort key {index.sort_key.attribute_name}"
+                    )
+
         self.__indexes[index.name] = index
 
     def get(self, index_name: str) -> DynamoDBIndex:

boto3_assist/dynamodb/dynamodb_iservice.py

@@ -7,6 +7,10 @@ from boto3_assist.dynamodb.dynamodb_model_base import DynamoDBModelBase
 class IDynamoDBService(ABC):
     """DynamoDB Service Interface"""
 
+    def __init__(self, db: DynamoDB) -> None:
+        self.db = db
+        super().__init__()
+
     @property
     @abstractmethod
     def db(self) -> DynamoDB:

boto3_assist/dynamodb/dynamodb_key.py

@@ -59,4 +59,6 @@ class DynamoDBKey:
                 break
             else:
                 parts.append(f"{prefix}{value}")
-        return "#".join(parts)
+        key_str = "#".join(parts)
+
+        return key_str

boto3_assist/dynamodb/dynamodb_model_base.py

@@ -9,7 +9,7 @@ import datetime as dt
 import decimal
 import inspect
 import uuid
-
+import base64
 from typing import TypeVar, List
 from boto3.dynamodb.types import TypeSerializer
 from boto3_assist.utilities.serialization_utility import Serialization
@@ -64,16 +64,28 @@ class DynamoDBModelBase:
     @exclude_from_serialization
     def projection_expression(self) -> str | None:
         """Gets the projection expression"""
+        prop_list: List[str] = []
         if self.__projection_expression is None and self.auto_generate_projections:
             props = self.to_dictionary()
             # turn props to a list[str]
             prop_list = list(props.keys())
+        else:
+            if self.__projection_expression:
+                prop_list = self.__projection_expression.split(",")
+                prop_list = [p.strip() for p in prop_list]
 
-            transformed_list = self.__reserved_words.tranform_projections(prop_list)
-            self.projection_expression = ",".join(transformed_list)
+        if len(prop_list) == 0:
+            return None
+
+        transformed_list = self.__reserved_words.tranform_projections(prop_list)
+        self.projection_expression = ",".join(transformed_list)
 
         return self.__projection_expression
 
+    @projection_expression.setter
+    def projection_expression(self, value: str | None):
+        self.__projection_expression = value
+
     @property
     @exclude_from_serialization
     def auto_generate_projections(self) -> bool:
@@ -84,10 +96,6 @@ class DynamoDBModelBase:
     def auto_generate_projections(self, value: bool):
         self.__auto_generate_projections = value
 
-    @projection_expression.setter
-    def projection_expression(self, value: str | None):
-        self.__projection_expression = value
-
     @property
     @exclude_from_serialization
     def projection_expression_attribute_names(self) -> dict | None:
@@ -105,6 +113,12 @@ class DynamoDBModelBase:
             self.projection_expression_attribute_names = (
                 self.__reserved_words.transform_attributes(prop_list)
             )
+        else:
+            if self.projection_expression:
+                expression_list = self.projection_expression.replace("#", "").split(",")
+                self.projection_expression_attribute_names = (
+                    self.__reserved_words.transform_attributes(expression_list)
+                )
 
         return self.__projection_expression_attribute_names
 
@@ -325,6 +339,8 @@ class DynamoDBSerializer:
 
         # Add instance variables
         for attr, value in instance.__dict__.items():
+            if str(attr) == "T":
+                continue
             # don't get the private properties
             if not str(attr).startswith("_"):
                 if value is not None or include_none:
@@ -346,6 +362,10 @@ class DynamoDBSerializer:
                 if exclude:
                     continue
 
+                # Skip TypeVar T or instances of DynamoDBModelBase
+                if str(name) == "T":
+                    continue
+
                 # don't get the private properties
                 if not str(name).startswith("_"):
                     value = getattr(instance, name)

boto3_assist/dynamodb/dynamodb_reserved_words.py

@@ -3,7 +3,10 @@ from typing import List
 
 
 class DynamoDBReservedWords:
-    """Reserved Word"""
+    """
+    Reserved Words used by DynamoDB that can cause issues,
+    so they will need to be transformed under certain conditions, such as when doing projections
+    """
 
     def __init__(self) -> None:
         self.__list: List[str] = self.__read_list()
@@ -31,13 +34,13 @@ class DynamoDBReservedWords:
         if isinstance(projections, str):
             projections = projections.split(",")
 
-        # any project that exists add a # infront of it
+        # any projection that exists add a # infront of it
         projections = ["#" + p if self.is_reserved_word(p) else p for p in projections]
         return projections
 
-    def transform_attributes(self, projections: List[str] | str) -> dict:
+    def transform_attributes(self, projections: List[str] | str) -> dict | None:
         """Transforms a dict of attributes to remove reserved words"""
-        transformed_attributes: dict | None = {}
+        transformed_attributes: dict = {}
         if isinstance(projections, str):
             projections = projections.split(",")
         for item in projections:

boto3_assist/dynamodb/dynamodb_reserved_words.txt

@@ -1,4 +1,3 @@
-
 ABORT
 ABSOLUTE
 ACTION

boto3_assist/ec2/ec2_connection.py

@@ -84,6 +84,9 @@ class EC2Connection:
         """Session"""
         if self.__session is None:
             self.setup(setup_source="session init")
+
+        if self.__session is None:
+            raise RuntimeError("Session is not available")
         return self.__session
 
     @property

boto3_assist/environment_services/environment_loader.py

@@ -1,22 +1,42 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
 import os
-from typing import IO, Optional, Union
+
+from typing import List, Union, Optional, IO
+from pathlib import Path
 from dotenv import load_dotenv
+from aws_lambda_powertools import Logger
+
+
+logger = Logger(__name__)
+
+DEBUGGING = os.getenv("DEBUGGING", "false").lower() == "true"
 
 StrPath = Union[str, "os.PathLike[str]"]
 
 
 class EnvironmentLoader:
+    """Environment Loader"""
+
     def __init__(self) -> None:
         pass
 
     def load_environment_file(
         self,
+        *,
+        starting_path: Optional[str] = None,
+        file_name: Optional[str] = None,
         path: Optional[StrPath] = None,
         stream: Optional[IO[str]] = None,
         verbose: bool = False,
         override: bool = True,
         interpolate: bool = True,
         encoding: Optional[str] = "utf-8",
+        raise_error_if_not_found: bool = False,
     ) -> bool:
         """
         Loads an environment file into memory. This simply passes off to load_dotenv in dotenv.
@@ -37,11 +57,55 @@ class EnvironmentLoader:
         If both `dotenv_path` and `stream` are `None`, `find_dotenv()` is used to find the
         .env file.
         """
-        return load_dotenv(
-            dotenv_path=path,
+
+        if not starting_path:
+            starting_path = __file__
+
+        if file_name is None:
+            file_name = ".env"
+
+        new_path: str | StrPath | None = path or self.find_file(
+            starting_path=starting_path,
+            file_name=file_name,
+            raise_error_if_not_found=raise_error_if_not_found,
+        )
+
+        loaded = load_dotenv(
+            dotenv_path=new_path,
             stream=stream,
             verbose=verbose,
             override=override,
             interpolate=interpolate,
             encoding=encoding,
         )
+
+        if DEBUGGING:
+            env_vars = os.environ
+            logger.debug(f"Loaded environment file: {path}")
+            print(env_vars)
+
+        return loaded
+
+    def find_file(
+        self, starting_path: str, file_name: str, raise_error_if_not_found: bool = True
+    ) -> str | None:
+        """Searches the project directory structor for a file"""
+        parents = 10
+        starting_path = starting_path or __file__
+
+        paths: List[str] = []
+        for parent in range(parents):
+            path = Path(starting_path).parents[parent].absolute()
+            print(f"searching: {path}")
+            tmp = os.path.join(path, file_name)
+            paths.append(tmp)
+            if os.path.exists(tmp):
+                return tmp
+
+        if raise_error_if_not_found:
+            searched_paths = "\n".join(paths)
+            raise RuntimeError(
+                f"Failed to locate environment file: {file_name} in: \n {searched_paths}"
+            )
+
+        return None

boto3_assist/environment_services/environment_variables.py

@@ -27,6 +27,8 @@ class EnvironmentVariables:
             gets the aws region from an environment var
             """
             value = os.getenv("AWS_REGION")
+            if not value:
+                value = None
             return value
 
         @staticmethod
@@ -36,6 +38,8 @@ class EnvironmentVariables:
             This should only be set with temporty creds and only for development purposes
             """
             value = os.getenv("AWS_PROFILE")
+            if not value:
+                value = None
             return value
 
         @staticmethod

boto3_assist/version.py

@@ -1 +1 @@
-__version__ = '0.1.13'
+__version__ = '0.2.0'

{boto3_assist-0.1.13.dist-info → boto3_assist-0.2.0.dist-info}/METADATA

@@ -1,10 +1,8 @@
 Metadata-Version: 2.3
 Name: boto3_assist
-Version: 0.1.13
+Version: 0.2.0
 Summary: Additional boto3 wrappers to make your life a little easier
 Author-email: Eric Wilson <boto3-assist@geekcafe.com>
-License-File: LICENSE-EXPLAINED.txt
-License-File: LICENSE.txt
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3

boto3_assist-0.2.0.dist-info/RECORD

@@ -0,0 +1,38 @@
+boto3_assist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+boto3_assist/boto3session.py,sha256=NWhWtYR3143thEbTpoklkwdz77-fTMs-QsoQdqfRm6E,6430
+boto3_assist/connection.py,sha256=EJlGueLIYqMSKs7aQlThK1S0Zkb8dOYBWch1iRZdgUI,3233
+boto3_assist/connection_tracker.py,sha256=_s1t7h2DOi3CCIHIr_HIKyGjku65WR-HJ_v8vJHDvO0,2977
+boto3_assist/version.py,sha256=FVHPBGkfhbQDi_z3v0PiKJrXXqXOx0vGW_1VaqNJi7U,22
+boto3_assist/cloudwatch/cloudwatch_connection.py,sha256=kiEmn3PVL9hbTrLehb2Xak9pEY8KygKoRO6YxdpYcbk,2634
+boto3_assist/cloudwatch/cloudwatch_connection_tracker.py,sha256=mzRtO1uHrcfJNh1XrGEiXdTqxwEP8d1RqJkraMNkgK0,410
+boto3_assist/cloudwatch/cloudwatch_log_connection.py,sha256=qQMZHjUJ6gA8wU9utjQhOURXNSPH2RjxSoAy83bvoCs,1737
+boto3_assist/cloudwatch/cloudwatch_logs.py,sha256=VtI0OnFjX1l4RYVvA8tvveGkPwAogtrplnflZ4dQSNM,1204
+boto3_assist/cloudwatch/cloudwatch_query.py,sha256=uNhSb1Gfp99v8BaHmCnCKs63j4MMU4WveqBavCJyhGY,6409
+boto3_assist/dynamodb/dynamodb.py,sha256=WZ0tmY-YAfY4aAvc1HNqizcLl_qeuBUj54K3BdtsUpQ,15127
+boto3_assist/dynamodb/dynamodb_connection.py,sha256=Gq5B830hAWAjWrvx0cGm0WpPaIr5usceSsykR-HU0tc,4513
+boto3_assist/dynamodb/dynamodb_connection_tracker.py,sha256=0BWHRfi5_vjkJLuCSX6sYwvA6wc7BSYCQnGrzbhfyKA,404
+boto3_assist/dynamodb/dynamodb_helpers.py,sha256=ajpTJ5bJOm9PDgE2Zx9p2zkTRFV4xswqJRS81SOTn1s,12198
+boto3_assist/dynamodb/dynamodb_importer.py,sha256=nCKsyRQeMqDSf0Q5mQ_X_oVIg4PRnu0hcUzZnBli610,3471
+boto3_assist/dynamodb/dynamodb_index.py,sha256=IihgaUVeSUJOFvgDx-xOQRC0cxQblfie7CM_Yj_K9o4,7498
+boto3_assist/dynamodb/dynamodb_iservice.py,sha256=O9Aj0PFEvcuk2vhARifWTFnUwcQW5EXzwZS478Hm-N0,796
+boto3_assist/dynamodb/dynamodb_key.py,sha256=YD7o1EUlwVBQ55p9YCTKqAUU_np4nqtLIHnmp-BeolM,1803
+boto3_assist/dynamodb/dynamodb_model_base.py,sha256=D74Wjh3YTewkGD0_oZ_9gGsjwgUMhz-oTSRIRdpx4VE,14273
+boto3_assist/dynamodb/dynamodb_model_base_interfaces.py,sha256=yT4zDRI8vP15WVOHnCvY3FsEy_QSIta5-bnUby70Xow,747
+boto3_assist/dynamodb/dynamodb_reindexer.py,sha256=bCj6KIU0fQOgjkkiq9yF51PFZZr4Y9Lu3-hPlmsPG0Y,6164
+boto3_assist/dynamodb/dynamodb_reserved_words.py,sha256=p0irNBSqGe4rd2FwWQqbRJWrNr4svdbWiyIXmz9lj4c,1937
+boto3_assist/dynamodb/dynamodb_reserved_words.txt,sha256=rvctS63Cv3i9SHmPq2Unmj6RZyQ-OMqxUXsNhtbg1is,4136
+boto3_assist/dynamodb/readme.md,sha256=wNMzdRwk0qRV0kE88UUYnJos3pEK0HNjEIVkq2PATf8,1490
+boto3_assist/dynamodb/troubleshooting.md,sha256=uGpBaBUt_MyzjzwFOLOe0udTgcvaOpiTFxfj7ilLNkM,136
+boto3_assist/ec2/ec2_connection.py,sha256=VT_hnu_behs9mf1Prggo5U9c9KAwbnLBTSZkmu4gp30,3277
+boto3_assist/environment_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+boto3_assist/environment_services/environment_loader.py,sha256=jvG1xwMtgkZqu70NbjG1b1IefKiWgaidjZZoqsSfULk,3370
+boto3_assist/environment_services/environment_variables.py,sha256=4ccBKdPt6O7hcRT3zBHd8vqu8yQU8udmoD5RLAT3iMs,6801
+boto3_assist/utilities/datetime_utility.py,sha256=TbqGQkJDTahqvaZAIV550nhYnW1Bsq0Hdu3Go6P4RRs,10282
+boto3_assist/utilities/logging_utility.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+boto3_assist/utilities/serialization_utility.py,sha256=s_QQRIhtwIE7xN5nU13mNk2wtWyErBX_Sg7n0gbHj-M,4308
+boto3_assist/utilities/string_utility.py,sha256=w8l063UT3GE48tuJopETyZrjG4CgAzWkyDWMAYMg5Og,7432
+boto3_assist-0.2.0.dist-info/METADATA,sha256=BCVBJGfQPqRIO-FvqKhyPOEaJZcJ1U0AlcRs9xFeb68,1742
+boto3_assist-0.2.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+boto3_assist-0.2.0.dist-info/licenses/LICENSE-EXPLAINED.txt,sha256=WFREvTpfTjPjDHpOLADxJpCKpIla3Ht87RUUGii4ODU,606
+boto3_assist-0.2.0.dist-info/licenses/LICENSE.txt,sha256=PXDhFWS5L5aOTkVhNvoitHKbAkgxqMI2uUPQyrnXGiI,1105
+boto3_assist-0.2.0.dist-info/RECORD,,

{boto3_assist-0.1.13.dist-info → boto3_assist-0.2.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.26.3
 Root-Is-Purelib: true
 Tag: py3-none-any

boto3_assist-0.1.13.dist-info/RECORD

@@ -1,32 +0,0 @@
-boto3_assist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-boto3_assist/boto3session.py,sha256=J20E2lkqJOaey4Ohi-xRe2xABj7QsA8DrggzK29-5es,6415
-boto3_assist/connection_tracker.py,sha256=ggCp2gQ7Y-2fHSGFk8QCHJz-Rq1yVNO4aRYgrYTP178,2987
-boto3_assist/version.py,sha256=Vx7xxEndWksf5h9xRevQfOixX-CU35SwBXGCh74SioU,23
-boto3_assist/dynamodb/dynamodb.py,sha256=vV0HvFCESnVK6BcAfcrWsMsNxIIsctvDX_3xdzOVCT0,14623
-boto3_assist/dynamodb/dynamodb_connection.py,sha256=JMCmWOsMzy45rikGl3Z2xqlG2vUTEKSYqi6dpsfJ750,4418
-boto3_assist/dynamodb/dynamodb_connection_tracker.py,sha256=0BWHRfi5_vjkJLuCSX6sYwvA6wc7BSYCQnGrzbhfyKA,404
-boto3_assist/dynamodb/dynamodb_helpers.py,sha256=ajpTJ5bJOm9PDgE2Zx9p2zkTRFV4xswqJRS81SOTn1s,12198
-boto3_assist/dynamodb/dynamodb_importer.py,sha256=nCKsyRQeMqDSf0Q5mQ_X_oVIg4PRnu0hcUzZnBli610,3471
-boto3_assist/dynamodb/dynamodb_index.py,sha256=LRQgSci222s-pU-JXgnaAoOa71ABX9h3uJPeCVPl1GE,6315
-boto3_assist/dynamodb/dynamodb_iservice.py,sha256=2AuaKxt7DUZbB-GpBBtPtPMpAlgZkumkAldm8vy7-sg,701
-boto3_assist/dynamodb/dynamodb_key.py,sha256=X3I3gUPx2T858vjRDi9SN8qn8ez5UJUo0vZiKBeeUWg,1776
-boto3_assist/dynamodb/dynamodb_model_base.py,sha256=x4jrH10-uooeQfdKhWQ6htXhA95bvCBET8jKrM7TQD0,13492
-boto3_assist/dynamodb/dynamodb_model_base_interfaces.py,sha256=yT4zDRI8vP15WVOHnCvY3FsEy_QSIta5-bnUby70Xow,747
-boto3_assist/dynamodb/dynamodb_reindexer.py,sha256=bCj6KIU0fQOgjkkiq9yF51PFZZr4Y9Lu3-hPlmsPG0Y,6164
-boto3_assist/dynamodb/dynamodb_reserved_words.py,sha256=iiud7ijpER5MgKHIP_NIgkDmHVy_57VRQzNHRipv0as,1786
-boto3_assist/dynamodb/dynamodb_reserved_words.txt,sha256=CanugFPh7rpgJ_jH9k4t8ANycrZJWTjT09sWiW4LC68,4137
-boto3_assist/dynamodb/readme.md,sha256=wNMzdRwk0qRV0kE88UUYnJos3pEK0HNjEIVkq2PATf8,1490
-boto3_assist/dynamodb/troubleshooting.md,sha256=uGpBaBUt_MyzjzwFOLOe0udTgcvaOpiTFxfj7ilLNkM,136
-boto3_assist/ec2/ec2_connection.py,sha256=KN1AivKY6yYpx_AjmaA3aQUZtUwKIvWk4FO8OPq1aTY,3182
-boto3_assist/environment_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-boto3_assist/environment_services/environment_loader.py,sha256=aW-PvlNi8EuavKPXDAi5txLbJFVRkBpkFIgmnR0OWDw,1581
-boto3_assist/environment_services/environment_variables.py,sha256=29ujkcFdgVgDgTK_pcVkpcFofEOYTbMXYyRim5vLj-k,6691
-boto3_assist/utilities/datetime_utility.py,sha256=TbqGQkJDTahqvaZAIV550nhYnW1Bsq0Hdu3Go6P4RRs,10282
-boto3_assist/utilities/logging_utility.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-boto3_assist/utilities/serialization_utility.py,sha256=s_QQRIhtwIE7xN5nU13mNk2wtWyErBX_Sg7n0gbHj-M,4308
-boto3_assist/utilities/string_utility.py,sha256=w8l063UT3GE48tuJopETyZrjG4CgAzWkyDWMAYMg5Og,7432
-boto3_assist-0.1.13.dist-info/METADATA,sha256=1Cg2vIjtiWjlTFmAqEhIhKUSrW3fUtW1DztU6FGRZp4,1805
-boto3_assist-0.1.13.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-boto3_assist-0.1.13.dist-info/licenses/LICENSE-EXPLAINED.txt,sha256=WFREvTpfTjPjDHpOLADxJpCKpIla3Ht87RUUGii4ODU,606
-boto3_assist-0.1.13.dist-info/licenses/LICENSE.txt,sha256=PXDhFWS5L5aOTkVhNvoitHKbAkgxqMI2uUPQyrnXGiI,1105
-boto3_assist-0.1.13.dist-info/RECORD,,