cmq 0.0.1__py3-none-any.whl

cmq/aws/aws.py

@@ -0,0 +1,256 @@
+import boto3
+
+from botocore.exceptions import ClientError
+from datetime import datetime, timedelta
+from functools import partial
+from matplotlib import pyplot as plt
+from typing import Any
+
+from cmq.base import Resource
+from cmq.iterator import UnknownSizeIterator, InfiniteAtpbar, Atpbar
+
+
+class AWSClientInterface:
+
+    def __init__(self):
+        self._service = ""
+        self._list_function = ""
+        self._list_key = ""
+        self._list_parameters = {}
+        self._list_paginated = True
+        self._includes = []
+
+    @property
+    def _desc(self) -> str:
+        return f"{self._service}_{self._resource}"
+
+    def __call__(self, **kwargs) -> "AWSClientInterface":
+        self._list_parameters = kwargs
+        return self
+
+    def get_client(self, context, service = None) -> boto3.client:
+        return context["aws_session"].client(service or self._service)
+
+    def get_parameters(self, context: dict) -> dict:
+        return self._list_parameters
+
+    def traverse(self, context: dict) -> None:
+        for resource in self.bar(context, self.get(context), desc=f"{self._desc} traverse"):
+            context.update({
+                f"{self._desc}": resource
+            })
+            self._traverse(context)
+
+    def get_paged_results(self, page) -> list:
+        return page.get(self._list_key, []) if self._list_key else self._get_results(page)
+
+    def _get_pages(self, context) -> Any:
+        client = self.get_client(context)
+        paginator = client.get_paginator(self._list_function)
+        return paginator.paginate(**self.get_parameters(context))
+
+    def _get(self, context) -> list:
+        resources = []
+        try:
+            if self._list_paginated:
+                resources = self.paginate(context)
+                resources = [r if isinstance(r, dict) else {"resource": r} for r in resources]
+            else:
+                client = self.get_client(context)
+                page = getattr(client, self._list_function)(**self.get_parameters(context))
+                resources = self.get_paged_results(page)
+
+            # Include additional information in the resources
+            for include in self._includes:
+                include(context, resources)
+
+            return resources
+        except Exception as ex:
+            return [{"error": str(ex)}]
+
+    def get(self, context):
+        resources = self._get(context)
+        return self._get_attr(self._exclude(resources))
+
+    def _get_results(self, page) -> list | None:
+        for value in page.values():
+            if isinstance(value, list):
+                return value
+        return None
+
+
+class TagResourceInterface:
+
+    def __init__(self):
+        self._tag_function = ""
+        self._tag_function_key = ""
+        self._tag_resource_key = ""
+
+    def tags(self) -> "TagResourceInterface":
+        self._includes.append(self._tag_resources)
+        return self
+
+    def _get_tag_resource_identifier(self, context, resource) -> str:
+        return resource[self._tag_resource_key]
+
+    def _tag_resources(self, context, resources):
+        if self._tag_function:
+            client = self.get_client(context)
+            for resource in self.bar(context, resources, "tags"):
+                if isinstance(resource, dict):
+                    try:
+                        tag_function = getattr(client, self._tag_function)
+                        tag_identifier = self._get_tag_resource_identifier(context, resource)
+                        result = tag_function(**{self._tag_function_key: tag_identifier})
+                        result_tags = self._get_tag_from_result(result)
+                        tags = self._format_tags(result_tags)
+                        resource.update(tags)
+                    except ClientError as ex:
+                        resource.update({"Tags": {"error": str(ex)}})
+
+    def _get_tag_from_result(self, result) -> list:
+        return self._get_results(result) or []
+
+    def _format_tags(self, tags) -> dict:
+        return {"Tags": {tag["Key"]: tag["Value"] for tag in tags}}
+
+
+class DescribeResourceInterface:
+
+    def __init__(self):
+        self._describe_function = ""
+        self._describe_function_key = ""
+        self._describe_resource_key = ""
+
+    def describe(self) -> "DescribeResourceInterface":
+        self._includes.append(self._describe_resources)
+        return self
+
+    def _get_describe_resource_identifier(self, context, resource) -> str:
+        return resource[self._describe_resource_key]
+
+    def _describe_resources(self, context, resources):
+        if self._describe_function:
+            client = self.get_client(context)
+            for resource in self.bar(context, resources, "describe"):
+                if isinstance(resource, dict):
+                    try:
+                        describe_function = getattr(client, self._describe_function)
+                        describe_identifier = self._get_describe_resource_identifier(context, resource)
+                        details = describe_function(**{self._describe_function_key: describe_identifier})
+                        resource.update({"Describe": details})
+                    except ClientError as ex:
+                        resource.update({"Describe": {"error": str(ex)}})
+
+
+class MetricResourceInterface:
+
+    def __init__(self):
+        self._metric_namespace = ""
+        self._metric_dimension_name = ""
+        self._metric_dimension_resource_key = ""
+
+    def _format_parameters(self,
+                           statistic: str,
+                           namespace: str, metric_name: str,
+                           dimensions: dict = {},
+                           period: int = 60*60, unit: str = "Count",
+                           start_time: datetime = None, end_time: datetime = None) -> dict:
+        # Set default parameter values
+        start_time = start_time or datetime.today() - timedelta(days=90)
+        end_time = end_time or datetime.today()
+        return {
+            "MetricDataQueries": [{
+                "Id": f"CMQ_{metric_name}",
+                "MetricStat": {
+                    "Metric": {
+                        "Namespace": namespace,
+                        "MetricName": metric_name,
+                        "Dimensions": [
+                            {"Name": k, "Value": v}
+                            for k, v in dimensions.items()
+                        ]
+                    },
+                    "Period": period,
+                    "Stat": statistic,
+                    "Unit": unit
+                },
+                "Label": unit,
+                "ReturnData": True,
+            }],
+            "StartTime": start_time,
+            "EndTime": end_time
+        }
+
+    def get_metric_data(self, results, context, **kwargs):
+        client = self.get_client(context, "cloudwatch")
+        dimensions = {self._metric_dimension_name: context[f"{self._desc}"][self._metric_dimension_resource_key]}
+        parameters = self._format_parameters(
+            namespace=self._metric_namespace,
+            dimensions=dimensions,
+            **kwargs
+        )
+        response = dimensions.copy()
+        paginator = client.get_paginator("get_metric_data")
+
+        for page in self.bar(context, paginator.paginate(**parameters), desc=f"metrics {dimensions[self._metric_dimension_name]}"):
+            for metric in page["MetricDataResults"]:
+                for key, value in metric.items():
+                    if key in response and isinstance(value, list):
+                        response[key].extend(value)
+                    else:
+                        response[key] = value
+        account_list = results.setdefault(context["session_name"], [])
+        account_list.append(response)
+
+    def metric(self, **kwargs) -> dict:
+        if not self._metric_namespace:
+            exit(f"Metric namespace not defined for {self._service}")
+
+        results: dict = {}
+        self._perform_action(partial(self.get_metric_data, results, **kwargs))
+        return results
+
+    def plot(self, unit_factor=1, **kwargs) -> None:
+        resources = self.metric(**kwargs)
+
+        plt.clf()
+        for metrics in resources.values():
+            for metric in metrics:
+                timestamps = metric["Timestamps"]
+                values = [value/unit_factor for value in metric["Values"]]
+                plt.plot(timestamps, values, label=metric[self._metric_dimension_resource_key])
+        plt.title(f"{self._resource} {kwargs['statistic']} {kwargs['metric_name']}")
+        plt.legend(loc='best')
+        plt.show()
+
+
+class AWSResource(TagResourceInterface, DescribeResourceInterface, MetricResourceInterface, AWSClientInterface, Resource):
+
+    def __init__(self, parent=None):
+        Resource.__init__(self, parent)
+        AWSClientInterface.__init__(self)
+        TagResourceInterface.__init__(self)
+        DescribeResourceInterface.__init__(self)
+        MetricResourceInterface.__init__(self)
+
+    def bar(self, context, resources, desc: str = "", leave=True):
+        if not self.enable_console():
+            return resources
+        desc = f"{context['session_name']: <20} {self._service} {desc}"
+        if not getattr(resources, "__len__", None):
+            resources = UnknownSizeIterator(resources)
+            return InfiniteAtpbar(resources, name=desc, len_=len(resources))
+        else:
+            return Atpbar(resources, name=desc, len_=len(resources))
+
+    def _get_pages(self, context) -> Any:
+        return self.bar(context, super()._get_pages(context), desc=f"{self._desc} get_pages")
+
+    def _list(self, results, context) -> None:
+        results.append(context[f"{self._desc}"])
+
+    def _dict(self, results, context) -> None:
+        account_list = results.setdefault(context["session_name"], [])
+        account_list.append(context.get(f"{self._desc}"))
+

cmq/aws/resource/address.py

@@ -0,0 +1,12 @@
+from cmq.aws.aws import AWSResource
+
+
+class address(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "ec2"
+        self._resource = "address"
+        self._list_function = "describe_addresses"
+        self._list_key = "Addresses"
+        self._list_paginated = False

cmq/aws/resource/alarm.py

@@ -0,0 +1,18 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+
+
+class alarm(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "cloudwatch"
+        self._resource = "alarm"
+        self._list_function = "describe_alarms"
+        self._list_key = "MetricAlarms"
+
+        self._tag_function = "list_tags_for_resource"
+        self._tag_function_key = "ResourceARN"
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource: dict[str, Any]) -> str:
+        return resource["AlarmArn"]

cmq/aws/resource/cloudformation.py

@@ -0,0 +1,10 @@
+from cmq.aws.aws import AWSResource
+
+
+class cloudformation(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "cloudformation"
+        self._resource = "stack"
+        self._list_function = "list_stacks"

cmq/aws/resource/cloudtrail.py

@@ -0,0 +1,111 @@
+import json
+from datetime import datetime, timedelta, timezone
+from cmq.aws.aws import AWSResource
+
+
+class cloudtrail(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "cloudtrail"
+        self._resource = "event"
+        self._list_function = "lookup_events"
+        self._transformers = {"CloudTrailEvent": json.loads}
+
+    def event_name(self, event_name: str) -> "cloudtrail":
+        """
+        Sets the event name for filtering events.
+
+        Args:
+            event_name (str): The event name to filter by.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        lookup = self._list_parameters.setdefault("LookupAttributes", [])
+        lookup.append({"AttributeKey": "EventName", "AttributeValue": event_name})
+        return self
+
+    def event_source(self, event_source: str) -> "cloudtrail":
+        """
+        Sets the event source for filtering events.
+
+        Args:
+            event_source (str): The event source to filter by.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        lookup = self._list_parameters.setdefault("LookupAttributes", [])
+        lookup.append({"AttributeKey": "EventSource", "AttributeValue": event_source})
+        return self
+
+    def resource_name(self, resource_name: str) -> "cloudtrail":
+        """
+        Sets the resource name for filtering events.
+
+        Args:
+            resource_name (str): The resource name to filter by.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        lookup = self._list_parameters.setdefault("LookupAttributes", [])
+        lookup.append({"AttributeKey": "ResourceName", "AttributeValue": resource_name})
+        return self
+
+    def user_name(self, user_name: str) -> "cloudtrail":
+        """
+        Sets the user name for filtering events.
+
+        Args:
+            user_name (str): The user name to filter by.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        lookup = self._list_parameters.setdefault("LookupAttributes", [])
+        lookup.append({"AttributeKey": "Username", "AttributeValue": user_name})
+        return self
+
+    def access_key(self, access_key: str) -> "cloudtrail":
+        """
+        Sets the access key for filtering events.
+
+        Args:
+            access_key (str): The access key to filter by.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        lookup = self._list_parameters.setdefault("LookupAttributes", [])
+        lookup.append({"AttributeKey": "AccessKeyId", "AttributeValue": access_key})
+        return self
+
+    def event_time(self, start_time: datetime, end_time: datetime) -> "cloudtrail":
+        """
+        Sets the event time range for filtering events.
+
+        Args:
+            start_time (datetime): The start time of the event range.
+            end_time (datetime): The end time of the event range.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        self._list_parameters.update({"StartTime": start_time, "EndTime": end_time})
+        return self
+
+    def last(self, **kwargs: dict[str, int]) -> "cloudtrail":
+        """
+        Sets the number of events to retrieve.
+
+        Args:
+            **kwargs: The keyword arguments for the timedelta.
+
+        Returns:
+            cloudtrail: The cloudtrail object for method chaining.
+        """
+        self._list_parameters.update({"StartTime": datetime.now(tz=timezone.utc) - timedelta(**kwargs),
+                                      "EndTime": datetime.now(tz=timezone.utc)})
+        return self

cmq/aws/resource/dynamodb.py

@@ -0,0 +1,26 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+
+
+class dynamodb(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "dynamodb"
+        self._resource = "dynamodb"
+        self._list_function = "list_tables"
+
+        self._describe_function = "describe_table"
+        self._describe_function_key = "TableName"
+        self._describe_resource_key = "resource"
+
+        self._tag_function = "list_tags_of_resource"
+        self._tag_function_key = "ResourceArn"
+        self._tag_resource_key = "resource"
+
+        self._metric_namespace = "AWS/DynamoDB"
+        self._metric_dimension_name = "TableName"
+        self._metric_dimension_resource_key = "resource"
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource) -> str:
+        return f"arn:aws:dynamodb:{context['aws_region']}:{context['aws_account']}:table/{resource['resource']}"

cmq/aws/resource/ec2.py

@@ -0,0 +1,18 @@
+from cmq.aws.aws import AWSResource
+
+
+class ec2(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "ec2"
+        self._resource = "instance"
+        self._list_function = "describe_instances"
+
+    def _get_results(self, page) -> list | None:
+        instances = []
+        reservations = page.get("Reservations", [])
+        for reservation in reservations:
+            for instance in reservation.get("Instances", []):
+                instances.append(instance)
+        return instances

cmq/aws/resource/elasticache.py

@@ -0,0 +1,26 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+from cmq.aws.resource.elasticache_parameter_group import elasticache_parameter_group
+
+
+class elasticache(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "elasticache"
+        self._resource = "elasticache"
+        self._list_function = "describe_cache_clusters"
+        self._list_key = "CacheClusters"
+
+        self._tag_function = "list_tags_for_resource"
+        self._tag_function_key = "ResourceName"
+
+        self._metric_namespace = "AWS/ElastiCache"
+        self._metric_dimension_name = "CacheClusterId"
+        self._metric_dimension_resource_key = "CacheClusterId"
+
+    def parameter_group(self) -> AWSResource:
+        return elasticache_parameter_group(self)
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource: dict[str, Any]) -> str:
+        return resource["ARN"]

cmq/aws/resource/elasticache_parameter_group.py

@@ -0,0 +1,11 @@
+from cmq.aws.aws import AWSResource
+
+
+class elasticache_parameter_group(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "elasticache"
+        self._resource = "elasticache_parameter_group"
+        self._list_function = "describe_cache_parameter_groups"
+        self._list_key = "CacheParameterGroups"

cmq/aws/resource/elasticache_replication_group.py

@@ -0,0 +1,17 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+
+
+class elasticache_replication_group(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "elasticache"
+        self._resource = "elasticache_replication_group"
+        self._list_function = "describe_replication_groups"
+        self._list_key = "ReplicationGroups"
+        self._tag_function = "list_tags_for_resource"
+        self._tag_function_key = "ResourceName"
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource: dict[str, Any]) -> str:
+        return resource["ARN"]

cmq/aws/resource/elasticache_subnet_group.py

@@ -0,0 +1,11 @@
+from cmq.aws.aws import AWSResource
+
+
+class elasticache_subnet_group(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "elasticache"
+        self._resource = "elasticache_subnet_group"
+        self._list_function = "describe_cache_subnet_groups"
+        self._list_key = "CacheSubnetGroups"

cmq/aws/resource/function.py

@@ -0,0 +1,28 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+
+
+class function(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "lambda"
+        self._resource = "function"
+        self._list_function = "list_functions"
+        self._list_key = "Functions"
+
+        self._tag_function = "list_tags"
+        self._tag_function_key = "Resource"
+
+        self._metric_namespace = "AWS/Lambda"
+        self._metric_dimension_name = "FunctionName"
+        self._metric_dimension_resource_key = "FunctionName"
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource: dict[str, Any]) -> str:
+        return resource["FunctionArn"]
+
+    def _get_tag_from_result(self, result):
+        return result.get("Tags", {})
+
+    def _format_tags(self, tags) -> dict:
+        return {"Tags": tags}

cmq/aws/resource/kinesis.py

@@ -0,0 +1,22 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+
+
+class kinesis(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "kinesis"
+        self._resource = "kinesis"
+        self._list_function = "list_streams"
+        self._list_key = "StreamSummaries"
+
+        self._tag_function = "list_tags_for_stream"
+        self._tag_function_key = "StreamARN"
+
+        self._metric_namespace = "AWS/Kinesis"
+        self._metric_dimension_name = "StreamName"
+        self._metric_dimension_resource_key = "StreamName"
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource: dict[str, Any]) -> str:
+        return resource["StreamARN"]

cmq/aws/resource/kms.py

@@ -0,0 +1,28 @@
+from typing import Any
+from cmq.aws.aws import AWSResource
+from cmq.aws.resource.kms_alias import kms_alias
+
+
+class kms(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "kms"
+        self._resource = "kms"
+        self._list_function = "list_keys"
+
+        self._tag_function = "list_resource_tags"
+        self._tag_function_key = "KeyId"
+
+        self._describe_function = "describe_key"
+        self._describe_function_key = "KeyId"
+        self._describe_resource_key = "KeyId"
+
+    def _get_tag_resource_identifier(self, context: dict[str, Any], resource) -> str:
+        return resource["KeyId"]
+
+    def _format_tags(self, tags) -> dict:
+        return {"Tags": {tag["TagKey"]: tag["TagValue"] for tag in tags}}
+
+    def aliases(self) -> AWSResource:
+        return kms_alias(self)

cmq/aws/resource/kms_alias.py

@@ -0,0 +1,16 @@
+from cmq.aws.aws import AWSResource
+
+
+class kms_alias(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "kms"
+        self._resource = "kms_alias"
+        self._list_function = "list_aliases"
+
+    def get_parameters(self, context: dict) -> dict:
+        parameters = self._list_parameters.copy()
+        if 'kms_kms' in context:
+            parameters["KeyId"] = context["kms_kms"]["KeyId"]
+        return parameters

cmq/aws/resource/log.py

@@ -0,0 +1,20 @@
+from cmq.aws.aws import AWSResource
+from cmq.aws.resource.log_stream import log_stream
+from cmq.aws.resource.log_event import log_event
+
+
+class log(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "logs"
+        self._resource = "log-group"
+        self._list_function = "describe_log_groups"
+
+    @property
+    def stream(self) -> log_stream:
+        return log_stream(self)
+
+    @property
+    def event(self, **kwargs) -> log_event:
+        return log_event(self).__call__(**kwargs)

cmq/aws/resource/log_event.py

@@ -0,0 +1,19 @@
+from cmq.aws.aws import AWSResource
+
+
+class log_event(AWSResource):
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._service = "logs"
+        self._resource = "log-event"
+        self._list_function = "filter_log_events"
+        self._list_paginated = False
+
+    def get_parameters(self, context: dict) -> dict:
+        parameters = self._list_parameters.copy()
+        if 'logs_log-group' in context:
+            parameters["logGroupName"] = context["logs_log-group"]["logGroupName"]
+        if 'logs_log-stream' in context:
+            parameters["logStreamNames"] = [context["logs_log-stream"]["logStreamName"]]
+        return parameters