boto3-assist 0.1.14__py3-none-any.whl → 0.2.1__py3-none-any.whl

boto3_assist/boto3session.py

@@ -45,6 +45,7 @@ class Boto3SessionManager:
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
         self.aws_session_token = aws_session_token
+
         self.__session: Any = None
         self.__client: Any = None
         self.__resource: Any = None
@@ -90,7 +91,7 @@ class Boto3SessionManager:
 
     def __get_aws_session(
         self, aws_profile: Optional[str] = None, aws_region: Optional[str] = None
-    ) -> boto3.Session:
+    ) -> boto3.Session | None:
         """Get a boto3 session for AWS."""
         logger.debug({"profile": aws_profile, "region": aws_region})
         try:
@@ -153,7 +154,7 @@ class Boto3SessionManager:
             )
         return self.__resource
 
-    def __create_boto3_session(self) -> boto3.Session:
+    def __create_boto3_session(self) -> boto3.Session | None:
         try:
             session = boto3.Session(
                 profile_name=self.aws_profile,

boto3_assist/cloudwatch/cloudwatch_connection.py

@@ -0,0 +1,84 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+
+
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+from boto3_assist.connection import Connection
+
+if TYPE_CHECKING:
+    from mypy_boto3_cloudwatch import CloudWatchClient, CloudWatchServiceResource
+else:
+    CloudWatchClient = object
+    CloudWatchServiceResource = object
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class CloudWatchConnection(Connection):
+    """CW Environment"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            service_name="cloudwatch",
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__client: CloudWatchClient | None = None
+        self.__resource: CloudWatchServiceResource | None = None
+
+        self.raise_on_error: bool = True
+
+    @property
+    def client(self) -> CloudWatchClient:
+        """CloudWatch Client Connection"""
+        if self.__client is None:
+            logger.info("Creating CloudWatch Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("CloudWatch Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: CloudWatchClient):
+        logger.info("Setting CloudWatch Client")
+        self.__client = value
+
+    @property
+    def resource(self) -> CloudWatchServiceResource:
+        """CloudWatch Resource Connection"""
+        if self.__resource is None:
+            logger.info("Creating CloudWatch Resource")
+            self.__resource = self.session.resource
+
+        if self.raise_on_error and self.__resource is None:
+            raise RuntimeError("CloudWatch Resource is not available")
+
+        return self.__resource
+
+    @resource.setter
+    def resource(self, value: CloudWatchServiceResource):
+        logger.info("Setting CloudWatch Resource")
+        self.__resource = value

boto3_assist/cloudwatch/cloudwatch_connection_tracker.py

@@ -0,0 +1,17 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from boto3_assist.connection_tracker import ConnectionTracker
+
+
+class CloudWatchConnectionTracker(ConnectionTracker):
+    """
+    Tracks CloudWatch Connection Requests.
+    Useful in for performance tuning and debugging.
+    """
+
+    def __init__(self) -> None:
+        super().__init__("CloudWatch")

boto3_assist/cloudwatch/cloudwatch_log_connection.py

@@ -0,0 +1,62 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools import Logger
+
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+from boto3_assist.connection import Connection
+
+if TYPE_CHECKING:
+    from mypy_boto3_logs import CloudWatchLogsClient
+else:
+    CloudWatchLogsClient = object
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class CloudWatchConnection(Connection):
+    """CW Logs Environment"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            service_name="logs",
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__client: CloudWatchLogsClient | None = None
+
+    @property
+    def client(self) -> CloudWatchLogsClient:
+        """CloudWatch Client Connection"""
+        if self.__client is None:
+            logger.debug("Creating CloudWatch Client")
+            self.__client = self.session.client
+
+        if self.raise_on_error and self.__client is None:
+            raise RuntimeError("CloudWatch Client is not available")
+        return self.__client
+
+    @client.setter
+    def client(self, value: CloudWatchLogsClient):
+        logger.debug("Setting CloudWatch Client")
+        self.__client = value

boto3_assist/cloudwatch/cloudwatch_logs.py

@@ -0,0 +1,39 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional, List, Dict, Any
+from boto3_assist.cloudwatch.cloudwatch_log_connection import CloudWatchConnection
+
+
+class CloudWatchLogs(CloudWatchConnection):
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+    def list_log_groups(self):
+        """Retrieve all log groups in the AWS account."""
+        log_groups: List[Dict[str, Any]] = []
+        paginator = self.client.get_paginator("describe_log_groups")
+        for page in paginator.paginate():
+            log_groups.extend(page["logGroups"])  # type: ignore[arg-type]
+        return log_groups
+
+
+def main():
+    query: CloudWatchLogs = CloudWatchLogs()
+    result = query.list_log_groups()
+    print(result)

boto3_assist/cloudwatch/cloudwatch_query.py

@@ -0,0 +1,191 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+import os
+from datetime import datetime, timedelta, UTC
+from typing import Optional, Dict, Any, List
+from boto3_assist.cloudwatch.cloudwatch_connection import CloudWatchConnection
+from boto3_assist.cloudwatch.cloudwatch_logs import CloudWatchLogs
+
+
+class CloudWatchQuery(CloudWatchConnection):
+    """Query Cloud Watch"""
+
+    def __init__(
+        self,
+        *,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+        self.__cw_logs: CloudWatchLogs | None = None
+
+    @property
+    def cw_logs(self) -> CloudWatchLogs:
+        """CloudWatch Logs Connection"""
+        if self.__cw_logs is None:
+            self.__cw_logs = CloudWatchLogs(
+                aws_profile=self.aws_profile,
+                aws_region=self.aws_region,
+                aws_access_key_id=self.aws_access_key_id,
+                aws_secret_access_key=self.aws_secret_access_key,
+            )
+        return self.__cw_logs
+
+    def get_log_group_size(
+        self, log_group_name: str, start_time: datetime, end_time: datetime
+    ) -> Dict[str, Any]:
+        """
+        Get the log group size for a given period of time
+        Args:
+            log_group_name (str): _description_
+            start_time (datetime): _description_
+            end_time (datetime): _description_
+
+        Returns:
+            _type_: _description_
+        """
+        response = self.client.get_metric_data(
+            MetricDataQueries=[
+                {
+                    "Id": "storedBytes",
+                    "MetricStat": {
+                        "Metric": {
+                            "Namespace": "AWS/Logs",
+                            # "MetricName": "StoredBytes",
+                            "MetricName": "IncomingBytes",
+                            "Dimensions": [
+                                {"Name": "LogGroupName", "Value": log_group_name}
+                            ],
+                        },
+                        "Period": 86400,  # Daily data
+                        "Stat": "Sum",
+                    },
+                    "ReturnData": True,
+                },
+            ],
+            StartTime=start_time,
+            EndTime=end_time,
+        )
+
+        # Extract the total size in bytes for the period
+        size: float = 0.0
+        if response["MetricDataResults"]:
+            # Access the first MetricDataResult
+            metric_data_result = response["MetricDataResults"][0]
+            # Sum the values if they exist
+            size = (
+                sum(metric_data_result["Values"]) if metric_data_result["Values"] else 0
+            )
+        else:
+            size = 0
+
+        size_mb = size / (1024 * 1024)
+        size_gb = size_mb / 1024
+        resp: Dict[str, Any] = {
+            "LogGroupName": log_group_name,
+            "Size": {
+                "Bytes": size,
+                "MB": size_mb,
+                "GB": size_gb,
+            },
+            "StartDate": start_time.isoformat(),
+            "EndDate": end_time.isoformat(),
+        }
+
+        return resp
+
+    def get_log_sizes(
+        self,
+        start_date_time: datetime | None = None,
+        end_date_time: datetime | None = None,
+        days: int | None = 7,
+        top: int = 0,
+    ) -> List[Dict[str, Any]]:
+        """
+        Gets the log sizes for all log groups
+
+        Args:
+            start_date_time (datetime | None, optional): The Start Date. Defaults to None.
+                If None it's set to now in UTC time - the days field
+            end_date_time (datetime | None, optional): he Start Date. Defaults to None.
+                If None it's set to not in UTC time
+            days (int | None, optional): The days offset. Defaults to 7.
+            top (int, optional): If greater than zero it will return the top x after sorting
+                Defaults to 0.
+
+        Returns:
+            list: _description_
+        """
+        if not days:
+            days = 7
+        start_time = start_date_time or (datetime.now(UTC) - timedelta(days=days))
+        end_time = end_date_time or datetime.now(UTC)
+
+        # Step 1: List all log groups
+        log_groups = self.cw_logs.list_log_groups()
+        log_group_sizes = []
+
+        # Step 2: Get sizes for each log group
+        for log_group in log_groups:
+            log_group_name = log_group["logGroupName"]
+
+            size_info = self.get_log_group_size(log_group_name, start_time, end_time)
+            log_group_sizes.append(size_info)
+
+        # Step 3: Sort by size
+        # top_log_groups = sorted(log_group_sizes, key=lambda x: x[1], reverse=True)
+        top_log_groups = sorted(
+            log_group_sizes,
+            key=lambda x: x.get("Size", {}).get("Bytes", 0),
+            reverse=True,
+        )
+        if top and top > 0:
+            # find the top x if provided
+            top_log_groups = top_log_groups[:top]
+
+        return top_log_groups
+
+
+def main():
+    log_group = os.environ.get("LOG_GROUP_QUERY_SAMPLE", "<enter-log-group-here>")
+    start = datetime.now() - timedelta(days=7)  # Last 30 days
+    end = datetime.now()
+    cw_query: CloudWatchQuery = CloudWatchQuery()
+    result = cw_query.get_log_group_size(log_group, start, end)
+    print(result)
+
+    top = 25
+    days = 7
+    top_log_groups = cw_query.get_log_sizes(top=top, days=days)
+    print(f"Top {top} log groups by size for the last week:")
+
+    for top_log_group in top_log_groups:
+        log_group_name = top_log_group["LogGroupName"]
+        size_in_bytes = top_log_group.get("Size", {}).get("Bytes", 0)
+        size_in_megs = top_log_group.get("Size", {}).get("MB", 0)
+        size_in_gigs = top_log_group.get("Size", {}).get("GB", 0)
+        size: str = ""
+        if size_in_gigs > 1:
+            size = f"{size_in_gigs:.2f} GB"
+        elif size_in_megs > 1:
+            size = f"{size_in_megs:.2f} MB"
+        else:
+            size = f"{size_in_bytes} bytes"
+
+        print(f"{size}: {log_group_name}")
+
+
+if __name__ == "__main__":
+    main()

boto3_assist/connection.py

@@ -0,0 +1,101 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Optional
+
+from aws_lambda_powertools import Logger
+from boto3_assist.boto3session import Boto3SessionManager
+from boto3_assist.environment_services.environment_variables import (
+    EnvironmentVariables,
+)
+from boto3_assist.cloudwatch.cloudwatch_connection_tracker import (
+    CloudWatchConnectionTracker,
+)
+
+
+logger = Logger()
+tracker: CloudWatchConnectionTracker = CloudWatchConnectionTracker()
+
+
+class Connection:
+    """Boto 3 Connection"""
+
+    def __init__(
+        self,
+        *,
+        service_name: Optional[str] = None,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ) -> None:
+        self.aws_profile = aws_profile or EnvironmentVariables.AWS.profile()
+        self.aws_region = aws_region or EnvironmentVariables.AWS.region()
+
+        self.aws_access_key_id = (
+            aws_access_key_id or EnvironmentVariables.AWS.DynamoDB.aws_access_key_id()
+        )
+        self.aws_secret_access_key = (
+            aws_secret_access_key
+            or EnvironmentVariables.AWS.DynamoDB.aws_secret_access_key()
+        )
+        self.__session: Boto3SessionManager | None = None
+
+        self.__service_name: str | None = service_name
+        self.raise_on_error: bool = True
+
+    def setup(self, setup_source: Optional[str] = None) -> None:
+        """
+        Setup the environment.  Automatically called via init.
+        You can run setup at anytime with new parameters.
+        Args: setup_source: Optional[str] = None
+            Defines the source of the setup.  Useful for logging.
+        Returns: None
+        """
+
+        logger.debug(
+            {
+                "metric_filter": f"{self.service_name}_connection_setup",
+                "source": f"{self.service_name} Connection",
+                "aws_profile": self.aws_profile,
+                "aws_region": self.aws_region,
+                "setup_source": setup_source,
+            }
+        )
+
+        self.__session = Boto3SessionManager(
+            service_name=self.service_name,
+            aws_profile=self.aws_profile,
+            aws_region=self.aws_region,
+            aws_access_key_id=self.aws_access_key_id,
+            aws_secret_access_key=self.aws_secret_access_key,
+        )
+
+        tracker.increment_connection()
+
+        self.raise_on_error = EnvironmentVariables.AWS.DynamoDB.raise_on_error_setting()
+
+    @property
+    def service_name(self) -> str:
+        """Service Name"""
+        if self.__service_name is None:
+            raise RuntimeError("Service Name is not available")
+        return self.__service_name
+
+    @service_name.setter
+    def service_name(self, value: str):
+        logger.debug("Setting Service Name")
+        self.__service_name = value
+
+    @property
+    def session(self) -> Boto3SessionManager:
+        """Session"""
+        if self.__session is None:
+            self.setup(setup_source="session init")
+
+        if self.__session is None:
+            raise RuntimeError("Session is not available")
+        return self.__session

boto3_assist/connection_tracker.py

@@ -16,7 +16,7 @@ class ConnectionTracker:
 
     def __init__(self, service_name: str) -> None:
         self.__stack_trace_env_var: str = "BOTO3_ASSIST_CONNECTION_STACK_TRACE"
-        self.__connection_couter: int = 0
+        self.__connection_counter: int = 0
         self.__service_name: str = service_name
         self.__issue_stack_trace: bool | None = None
 
@@ -31,7 +31,7 @@ class ConnectionTracker:
 
     def increment_connection(self) -> None:
         """Increments the connection counter"""
-        self.__connection_couter += 1
+        self.__connection_counter += 1
 
         if self.connection_count > 1:
             service_message = ""
@@ -53,27 +53,27 @@ class ConnectionTracker:
 
             self.__log_warning(
                 f"{service_message}"
-                f"Your dynamodb connection count is {self.connection_count}. "
+                f"Your boto3 connection count is {self.connection_count}. "
                 "Under most circumstances you should be able to use the same connection "
                 "vs. creating a new one.  Connections are expensive in terms of time / latency. "
                 "If you are seeing perforance issues, check how and where you are creating your "
-                "connections.  You should be able to pass the .db connection to your other objects "
-                "and reuse your dynamodb boto connections."
+                "connections.  You should be able to pass the connection to your other objects "
+                "and reuse your boto3 connections."
                 f"{stack_trace_message}"
             )
 
     def decrement_connection(self) -> None:
         """Decrements the connection counter"""
-        self.__connection_couter -= 1
+        self.__connection_counter -= 1
 
     @property
     def connection_count(self) -> int:
         """Returns the current connection count"""
-        return self.__connection_couter
+        return self.__connection_counter
 
     def reset(self) -> None:
         """Resets the connection counter"""
-        self.__connection_couter = 0
+        self.__connection_counter = 0
 
     def __log_warning(self, message: str) -> None:
         """Logs a warning message"""