boto3-assist 0.1.14__tar.gz → 0.2.0__tar.gz

boto3_assist-0.2.0/.env.unittest

@@ -0,0 +1,17 @@
+# let's make sure we don't hit any actual aws resources
+# when running moto mock tests
+AWS_ACCESS_KEY_ID="testing"
+AWS_SECRET_ACCESS_KEY="testing"
+AWS_SECURITY_TOKEN="testing"
+AWS_SESSION_TOKEN="testing"
+AWS_DEFAULT_REGION="us-east-1"
+AWS_REGION="us-east-1"
+# So far it looks like I need a profile or I get an error like the one below
+# "The config profile (<profile>) could not be found" 
+# "The config profile () could not be found" <- if using the default profile
+# AWS_PROFILE="default"
+# AWS_DEFAULT_PROFILE="default"
+# I don't want to use default profile, so I'm using a mock profile
+# However it must exist in ~/.aws/config or you'll get the same error above
+AWS_PROFILE="moto-mock-tests"
+AWS_DEFAULT_PROFILE="moto-mock-tests"

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/.gitignore

@@ -161,4 +161,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-.imports
+.imports
+.env.development

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/PKG-INFO

@@ -1,10 +1,8 @@
 Metadata-Version: 2.3
 Name: boto3_assist
-Version: 0.1.14
+Version: 0.2.0
 Summary: Additional boto3 wrappers to make your life a little easier
 Author-email: Eric Wilson <boto3-assist@geekcafe.com>
-License-File: LICENSE-EXPLAINED.txt
-License-File: LICENSE.txt
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3

boto3_assist-0.2.0/examples/cloudwatch/log_report.py

@@ -0,0 +1,62 @@
+import os
+import json
+import csv
+from typing import List, Dict, Any
+from pathlib import Path
+from boto3_assist.environment_services.environment_loader import EnvironmentLoader
+from boto3_assist.ec2.ec2_connection import EC2Connection
+from boto3_assist.cloudwatch.cloudwatch_query import CloudWatchQuery
+
+
+class LogReport(CloudWatchQuery):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def export(self, regions: List[Dict[str, Any]], file_format: str = "csv"):
+        """
+        Export the regions to a file
+        Args:
+            regions (List[Dict[str, Any]]): _description_
+            file_format (str, optional): _description_. Defaults to "csv".
+        """
+        if file_format == "csv":
+            self.__export_regions_to_csv(regions)
+        elif file_format == "json":
+            self.__export_regions_to_json(regions)
+
+    def __export_regions_to_csv(self, regions, filename="aws_regions_with_status.csv"):
+        with open(filename, "w", newline="", encoding="utf-8") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["RegionName", "OptInStatus"])
+            for region in regions:
+                writer.writerow([region["RegionName"], region["OptInStatus"]])
+
+    def __export_regions_to_json(
+        self, regions, filename="aws_regions_with_status.json"
+    ):
+        with open(filename, "w", encoding="utf-8") as jsonfile:
+            json.dump(regions, jsonfile, indent=4)
+
+
+def main():
+    """Main"""
+
+    env_file_name: str = os.getenv("ENVRIONMENT_FILE", ".env")
+    path = os.path.join(str(Path(__file__).parents[2].absolute()), env_file_name)
+    el: EnvironmentLoader = EnvironmentLoader()
+    if not os.path.exists(path=path):
+        raise FileNotFoundError("Failed to find the environmetn file")
+    loaded: bool = el.load_environment_file(path=path)
+    if not loaded:
+        raise RuntimeError("Failed to load my local environment")
+
+    report: CloudWatchQuery = CloudWatchQuery()
+
+    # Retrieve AWS regions with their status
+    logs = report.get_log_sizes()
+
+    print(logs)
+
+
+if __name__ == "__main__":
+    main()

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/examples/dynamodb/models/order_item_model.py

@@ -33,14 +33,18 @@ class OrderItem(DynamoDBModelBase):
         self.__setup_indexes()
 
     def __setup_indexes(self):
-        # user id
+        # the primary key will be made off of the
+        # order.id and this item.id
+        # this will allow for a 1 to many search on the items related to an order
         primay: DynamoDBIndex = DynamoDBIndex()
         primay.name = "primary"
+        # create a partition key off of the order key
         primay.partition_key.attribute_name = "pk"
         primay.partition_key.value = lambda: DynamoDBKey.build_key(
             ("order", self.order_id)
         )
 
+        # create the sort key off of this items id
         primay.sort_key.attribute_name = "sk"
         primay.sort_key.value = lambda: DynamoDBKey.build_key(("item", self.id))
         self.indexes.add_primary(primay)

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/examples/dynamodb/order_example/main.py

@@ -5,17 +5,18 @@ DynamoDB Example
 import json
 import os
 import random
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import List
-from datetime import datetime, timedelta, UTC
+
 from boto3_assist.dynamodb.dynamodb import DynamoDB
 from boto3_assist.environment_services.environment_loader import EnvironmentLoader
 from boto3_assist.utilities.serialization_utility import JsonEncoder
 from boto3_assist.utilities.string_utility import StringUtility
-from examples.dynamodb.services.table_service import DynamoDBTableService
-from examples.dynamodb.services.order_service import OrderService, Order
-from examples.dynamodb.services.order_item_service import OrderItemService, OrderItem
 from examples.dynamodb.models.product_model import Product
+from examples.dynamodb.services.order_item_service import OrderItem, OrderItemService
+from examples.dynamodb.services.order_service import Order, OrderService
+from examples.dynamodb.services.table_service import DynamoDBTableService
 
 
 class DynamoDBExample:
@@ -154,7 +155,7 @@ def main():
     el: EnvironmentLoader = EnvironmentLoader()
     if not os.path.exists(path=path):
         raise FileNotFoundError("Failed to find the environmetn file")
-    loaded: bool = el.load_environment_file(path)
+    loaded: bool = el.load_environment_file(path=path)
     if not loaded:
         raise RuntimeError("Failed to load my local environment")
 

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/examples/dynamodb/services/order_item_service.py

@@ -40,7 +40,7 @@ class OrderItemService:
 
     def list(self, order_id: str) -> list:
         """
-        Lists users using a global secondary index.
+        Lists using a global secondary index.
 
         Args:
             user_id (str): Gets orders by a user id.

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/examples/dynamodb/user_post_example/main.py

@@ -181,7 +181,7 @@ def main():
     el: EnvironmentLoader = EnvironmentLoader()
     if not os.path.exists(path=path):
         raise FileNotFoundError("Failed to find the environmetn file")
-    loaded: bool = el.load_environment_file(path)
+    loaded: bool = el.load_environment_file(path=path)
     if not loaded:
         raise RuntimeError("Failed to load my local environment")
 

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/examples/ec2/regions_report.py

@@ -60,7 +60,7 @@ def main():
     el: EnvironmentLoader = EnvironmentLoader()
     if not os.path.exists(path=path):
         raise FileNotFoundError("Failed to find the environmetn file")
-    loaded: bool = el.load_environment_file(path)
+    loaded: bool = el.load_environment_file(path=path)
     if not loaded:
         raise RuntimeError("Failed to load my local environment")
 

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["src/boto3_assist"]
 
 [project]
 name = "boto3_assist"
-version = "0.1.14"
+version = "0.2.0"
 authors = [
   { name="Eric Wilson", email="boto3-assist@geekcafe.com" }
 ]

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/requirements-dev.txt

@@ -3,6 +3,7 @@ build
 mypy
 mypy_boto3_dynamodb
 mypy_boto3_ec2
+mypy_boto3_cloudwatch
 moto [dynamodb2] # mocks for unit tests
 # setuptools
 types-python-dateutil

{boto3_assist-0.1.14 → boto3_assist-0.2.0}/src/boto3_assist/boto3session.py

@@ -45,6 +45,7 @@ class Boto3SessionManager:
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
         self.aws_session_token = aws_session_token
+
         self.__session: Any = None
         self.__client: Any = None
         self.__resource: Any = None
@@ -90,7 +91,7 @@ class Boto3SessionManager:
 
     def __get_aws_session(
         self, aws_profile: Optional[str] = None, aws_region: Optional[str] = None
-    ) -> boto3.Session:
+    ) -> boto3.Session | None:
         """Get a boto3 session for AWS."""
         logger.debug({"profile": aws_profile, "region": aws_region})
         try:
@@ -153,7 +154,7 @@ class Boto3SessionManager:
             )
         return self.__resource
 
-    def __create_boto3_session(self) -> boto3.Session:
+    def __create_boto3_session(self) -> boto3.Session | None:
         try:
             session = boto3.Session(
                 profile_name=self.aws_profile,

boto3_assist-0.2.0/src/boto3_assist/cloudwatch/cloudwatch_connection.py

@@ -0,0 +1,86 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+from boto3_assist.boto3session import Boto3SessionManager
+from boto3_assist.environment_services.environment_variables import (
+    EnvironmentVariables,
+)
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+from boto3_assist.connection import Connection
+
+if TYPE_CHECKING:
+    from mypy_boto3_cloudwatch import CloudWatchClient, CloudWatchServiceResource
+else:
+    CloudWatchClient = object
+    CloudWatchServiceResource = object
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class CloudWatchConnection(Connection):
+    """CW Environment"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            service_name="cloudwatch",
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__client: CloudWatchClient | None = None
+        self.__resource: CloudWatchServiceResource | None = None
+
+        self.raise_on_error: bool = True
+
+    @property
+    def client(self) -> CloudWatchClient:
+        """CloudWatch Client Connection"""
+        if self.__client is None:
+            logger.info("Creating CloudWatch Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("CloudWatch Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: CloudWatchClient):
+        logger.info("Setting CloudWatch Client")
+        self.__client = value
+
+    @property
+    def resource(self) -> CloudWatchServiceResource:
+        """CloudWatch Resource Connection"""
+        if self.__resource is None:
+            logger.info("Creating CloudWatch Resource")
+            self.__resource = self.session.resource
+
+        if self.raise_on_error and self.__resource is None:
+            raise RuntimeError("CloudWatch Resource is not available")
+
+        return self.__resource
+
+    @resource.setter
+    def resource(self, value: CloudWatchServiceResource):
+        logger.info("Setting CloudWatch Resource")
+        self.__resource = value

boto3_assist-0.2.0/src/boto3_assist/cloudwatch/cloudwatch_connection_tracker.py

@@ -0,0 +1,17 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from boto3_assist.connection_tracker import ConnectionTracker
+
+
+class CloudWatchConnectionTracker(ConnectionTracker):
+    """
+    Tracks CloudWatch Connection Requests.
+    Useful in for performance tuning and debugging.
+    """
+
+    def __init__(self) -> None:
+        super().__init__("CloudWatch")

boto3_assist-0.2.0/src/boto3_assist/cloudwatch/cloudwatch_log_connection.py

@@ -0,0 +1,62 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+from boto3_assist.connection import Connection
+
+if TYPE_CHECKING:
+    from mypy_boto3_logs import CloudWatchLogsClient
+else:
+    CloudWatchLogsClient = object
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class CloudWatchConnection(Connection):
+    """CW Logs Environment"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            service_name="logs",
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__client: CloudWatchLogsClient | None = None
+
+    @property
+    def client(self) -> CloudWatchLogsClient:
+        """CloudWatch Client Connection"""
+        if self.__client is None:
+            logger.debug("Creating CloudWatch Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("CloudWatch Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: CloudWatchLogsClient):
+        logger.debug("Setting CloudWatch Client")
+        self.__client = value

boto3_assist-0.2.0/src/boto3_assist/cloudwatch/cloudwatch_logs.py

@@ -0,0 +1,39 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional, List, Dict, Any
+from boto3_assist.cloudwatch.cloudwatch_log_connection import CloudWatchConnection
+
+
+class CloudWatchLogs(CloudWatchConnection):
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+    def list_log_groups(self):
+        """Retrieve all log groups in the AWS account."""
+        log_groups: List[Dict[str, Any]] = []
+        paginator = self.client.get_paginator("describe_log_groups")
+        for page in paginator.paginate():
+            log_groups.extend(page["logGroups"])  # type: ignore[arg-type]
+        return log_groups
+
+
+def main():
+    query: CloudWatchLogs = CloudWatchLogs()
+    result = query.list_log_groups()
+    print(result)

boto3_assist-0.2.0/src/boto3_assist/cloudwatch/cloudwatch_query.py

@@ -0,0 +1,191 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+import os
+from datetime import datetime, timedelta, UTC
+from typing import Optional, Dict, Any, List
+from boto3_assist.cloudwatch.cloudwatch_connection import CloudWatchConnection
+from boto3_assist.cloudwatch.cloudwatch_logs import CloudWatchLogs
+
+
+class CloudWatchQuery(CloudWatchConnection):
+    """Query Cloud Watch"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__cw_logs: CloudWatchLogs | None = None
+
+    @property
+    def cw_logs(self) -> CloudWatchLogs:
+        """CloudWatch Logs Connection"""
+        if self.__cw_logs is None:
+            self.__cw_logs = CloudWatchLogs(
+                aws_profile=self.aws_profile,
+                aws_region=self.aws_region,
+                aws_access_key_id=self.aws_access_key_id,
+                aws_secret_access_key=self.aws_secret_access_key,
+            )
+        return self.__cw_logs
+
+    def get_log_group_size(
+        self, log_group_name: str, start_time: datetime, end_time: datetime
+    ) -> Dict[str, Any]:
+        """
+        Get the log group size for a given period of time
+        Args:
+            log_group_name (str): _description_
+            start_time (datetime): _description_
+            end_time (datetime): _description_
+
+        Returns:
+            _type_: _description_
+        """
+        response = self.client.get_metric_data(
+            MetricDataQueries=[
+                {
+                    "Id": "storedBytes",
+                    "MetricStat": {
+                        "Metric": {
+                            "Namespace": "AWS/Logs",
+                            # "MetricName": "StoredBytes",
+                            "MetricName": "IncomingBytes",
+                            "Dimensions": [
+                                {"Name": "LogGroupName", "Value": log_group_name}
+                            ],
+                        },
+                        "Period": 86400,  # Daily data
+                        "Stat": "Sum",
+                    },
+                    "ReturnData": True,
+                },
+            ],
+            StartTime=start_time,
+            EndTime=end_time,
+        )
+
+        # Extract the total size in bytes for the period
+        size: float = 0.0
+        if response["MetricDataResults"]:
+            # Access the first MetricDataResult
+            metric_data_result = response["MetricDataResults"][0]
+            # Sum the values if they exist
+            size = (
+                sum(metric_data_result["Values"]) if metric_data_result["Values"] else 0
+            )
+        else:
+            size = 0
+
+        size_mb = size / (1024 * 1024)
+        size_gb = size_mb / 1024
+        resp: Dict[str, Any] = {
+            "LogGroupName": log_group_name,
+            "Size": {
+                "Bytes": size,
+                "MB": size_mb,
+                "GB": size_gb,
+            },
+            "StartDate": start_time.isoformat(),
+            "EndDate": end_time.isoformat(),
+        }
+
+        return resp
+
+    def get_log_sizes(
+        self,
+        start_date_time: datetime | None = None,
+        end_date_time: datetime | None = None,
+        days: int | None = 7,
+        top: int = 0,
+    ) -> List[Dict[str, Any]]:
+        """
+        Gets the log sizes for all log groups
+
+        Args:
+            start_date_time (datetime | None, optional): The Start Date. Defaults to None.
+                If None it's set to now in UTC time - the days field
+            end_date_time (datetime | None, optional): he Start Date. Defaults to None.
+                If None it's set to not in UTC time
+            days (int | None, optional): The days offset. Defaults to 7.
+            top (int, optional): If greater than zero it will return the top x after sorting
+                Defaults to 0.
+
+        Returns:
+            list: _description_
+        """
+        if not days:
+            days = 7
+        start_time = start_date_time or (datetime.now(UTC) - timedelta(days=days))
+        end_time = end_date_time or datetime.now(UTC)
+
+        # Step 1: List all log groups
+        log_groups = self.cw_logs.list_log_groups()
+        log_group_sizes = []
+
+        # Step 2: Get sizes for each log group
+        for log_group in log_groups:
+            log_group_name = log_group["logGroupName"]
+
+            size_info = self.get_log_group_size(log_group_name, start_time, end_time)
+            log_group_sizes.append(size_info)
+
+        # Step 3: Sort by size
+        # top_log_groups = sorted(log_group_sizes, key=lambda x: x[1], reverse=True)
+        top_log_groups = sorted(
+            log_group_sizes,
+            key=lambda x: x.get("Size", {}).get("Bytes", 0),
+            reverse=True,
+        )
+        if top and top > 0:
+            # find the top x if provided
+            top_log_groups = top_log_groups[:top]
+
+        return top_log_groups
+
+
+def main():
+    log_group = os.environ.get("LOG_GROUP_QUERY_SAMPLE", "<enter-log-group-here>")
+    start = datetime.now() - timedelta(days=7)  # Last 30 days
+    end = datetime.now()
+    cw_query: CloudWatchQuery = CloudWatchQuery()
+    result = cw_query.get_log_group_size(log_group, start, end)
+    print(result)
+
+    top = 25
+    days = 7
+    top_log_groups = cw_query.get_log_sizes(top=top, days=days)
+    print(f"Top {top} log groups by size for the last week:")
+
+    for top_log_group in top_log_groups:
+        log_group_name = top_log_group["LogGroupName"]
+        size_in_bytes = top_log_group.get("Size", {}).get("Bytes", 0)
+        size_in_megs = top_log_group.get("Size", {}).get("MB", 0)
+        size_in_gigs = top_log_group.get("Size", {}).get("GB", 0)
+        size: str = ""
+        if size_in_gigs > 1:
+            size = f"{size_in_gigs:.2f} GB"
+        elif size_in_megs > 1:
+            size = f"{size_in_megs:.2f} MB"
+        else:
+            size = f"{size_in_bytes} bytes"
+
+        print(f"{size}: {log_group_name}")
+
+
+if __name__ == "__main__":
+    main()