cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/aws/ec2/auto_scaling_groups.py

@@ -5,20 +5,26 @@ from typing import Any
 import boto3
 import neo4j
 
-from .util import get_botocore_config
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ec2.auto_scaling_groups import AutoScalingGroupSchema
-from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
-from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema
+from cartography.models.aws.ec2.auto_scaling_groups import (
+    EC2InstanceAutoScalingGroupSchema,
+)
+from cartography.models.aws.ec2.auto_scaling_groups import (
+    EC2SubnetAutoScalingGroupSchema,
+)
 from cartography.models.aws.ec2.launch_configurations import LaunchConfigurationSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
+from .util import get_botocore_config
+
 logger = logging.getLogger(__name__)
 
 AsgData = namedtuple(
-    'AsgData', [
+    "AsgData",
+    [
         "group_list",
         "instance_list",
         "subnet_list",
@@ -28,46 +34,66 @@ AsgData = namedtuple(
 
 @timeit
 @aws_handle_regions
-def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> list[dict]:
-    client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
-    paginator = client.get_paginator('describe_auto_scaling_groups')
+def get_ec2_auto_scaling_groups(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict]:
+    client = boto3_session.client(
+        "autoscaling",
+        region_name=region,
+        config=get_botocore_config(),
+    )
+    paginator = client.get_paginator("describe_auto_scaling_groups")
     asgs: list[dict] = []
     for page in paginator.paginate():
-        asgs.extend(page['AutoScalingGroups'])
+        asgs.extend(page["AutoScalingGroups"])
     return asgs
 
 
 @timeit
 @aws_handle_regions
-def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> list[dict]:
-    client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
-    paginator = client.get_paginator('describe_launch_configurations')
+def get_launch_configurations(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict]:
+    client = boto3_session.client(
+        "autoscaling",
+        region_name=region,
+        config=get_botocore_config(),
+    )
+    paginator = client.get_paginator("describe_launch_configurations")
     lcs: list[dict] = []
     for page in paginator.paginate():
-        lcs.extend(page['LaunchConfigurations'])
+        lcs.extend(page["LaunchConfigurations"])
     return lcs
 
 
-def transform_launch_configurations(configurations: list[dict[str, Any]]) -> list[dict[str, Any]]:
+def transform_launch_configurations(
+    configurations: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
     transformed_configurations = []
     for config in configurations:
-        transformed_configurations.append({
-            'AssociatePublicIpAddress': config.get('AssociatePublicIpAddress'),
-            'LaunchConfigurationARN': config.get('LaunchConfigurationARN'),
-            'LaunchConfigurationName': config.get('LaunchConfigurationName'),
-            'CreatedTime': config.get('CreatedTime'),
-            'ImageId': config.get('ImageId'),
-            'KeyName': config.get('KeyName'),
-            'SecurityGroups': config.get('SecurityGroups'),
-            'InstanceType': config.get('InstanceType'),
-            'KernelId': config.get('KernelId'),
-            'RamdiskId': config.get('RamdiskId'),
-            'InstanceMonitoring': config.get('InstanceMonitoring', {}).get('Enabled'),
-            'SpotPrice': config.get('SpotPrice'),
-            'IamInstanceProfile': config.get('IamInstanceProfile'),
-            'EbsOptimized': config.get('EbsOptimized'),
-            'PlacementTenancy': config.get('PlacementTenancy'),
-        })
+        transformed_configurations.append(
+            {
+                "AssociatePublicIpAddress": config.get("AssociatePublicIpAddress"),
+                "LaunchConfigurationARN": config.get("LaunchConfigurationARN"),
+                "LaunchConfigurationName": config.get("LaunchConfigurationName"),
+                "CreatedTime": config.get("CreatedTime"),
+                "ImageId": config.get("ImageId"),
+                "KeyName": config.get("KeyName"),
+                "SecurityGroups": config.get("SecurityGroups"),
+                "InstanceType": config.get("InstanceType"),
+                "KernelId": config.get("KernelId"),
+                "RamdiskId": config.get("RamdiskId"),
+                "InstanceMonitoring": config.get("InstanceMonitoring", {}).get(
+                    "Enabled",
+                ),
+                "SpotPrice": config.get("SpotPrice"),
+                "IamInstanceProfile": config.get("IamInstanceProfile"),
+                "EbsOptimized": config.get("EbsOptimized"),
+                "PlacementTenancy": config.get("PlacementTenancy"),
+            },
+        )
     return transformed_configurations
 
 
@@ -76,42 +102,54 @@ def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
     related_vpcs = []
     related_instances = []
     for group in groups:
-        transformed_groups.append({
-            'AutoScalingGroupARN': group['AutoScalingGroupARN'],
-            'CapacityRebalance': group.get('CapacityRebalance'),
-            'CreatedTime': str(group.get('CreatedTime')),
-            'DefaultCooldown': group.get('DefaultCooldown'),
-            'DesiredCapacity': group.get('DesiredCapacity'),
-            'HealthCheckGracePeriod': group.get('HealthCheckGracePeriod'),
-            'HealthCheckType': group.get('HealthCheckType'),
-            'LaunchConfigurationName': group.get('LaunchConfigurationName'),
-            'LaunchTemplateName': group.get('LaunchTemplate', {}).get('LaunchTemplateName'),
-            'LaunchTemplateId': group.get('LaunchTemplate', {}).get('LaunchTemplateId'),
-            'LaunchTemplateVersion': group.get('LaunchTemplate', {}).get('Version'),
-            'MaxInstanceLifetime': group.get('MaxInstanceLifetime'),
-            'MaxSize': group.get('MaxSize'),
-            'MinSize': group.get('MinSize'),
-            'AutoScalingGroupName': group.get('AutoScalingGroupName'),
-            'NewInstancesProtectedFromScaleIn': group.get('NewInstancesProtectedFromScaleIn'),
-            'Status': group.get('Status'),
-        })
-
-        if group.get('VPCZoneIdentifier', None):
-            vpclist = group['VPCZoneIdentifier']
-            subnet_ids = vpclist.split(',') if ',' in vpclist else [vpclist]
+        transformed_groups.append(
+            {
+                "AutoScalingGroupARN": group["AutoScalingGroupARN"],
+                "CapacityRebalance": group.get("CapacityRebalance"),
+                "CreatedTime": str(group.get("CreatedTime")),
+                "DefaultCooldown": group.get("DefaultCooldown"),
+                "DesiredCapacity": group.get("DesiredCapacity"),
+                "HealthCheckGracePeriod": group.get("HealthCheckGracePeriod"),
+                "HealthCheckType": group.get("HealthCheckType"),
+                "LaunchConfigurationName": group.get("LaunchConfigurationName"),
+                "LaunchTemplateName": group.get("LaunchTemplate", {}).get(
+                    "LaunchTemplateName",
+                ),
+                "LaunchTemplateId": group.get("LaunchTemplate", {}).get(
+                    "LaunchTemplateId",
+                ),
+                "LaunchTemplateVersion": group.get("LaunchTemplate", {}).get("Version"),
+                "MaxInstanceLifetime": group.get("MaxInstanceLifetime"),
+                "MaxSize": group.get("MaxSize"),
+                "MinSize": group.get("MinSize"),
+                "AutoScalingGroupName": group.get("AutoScalingGroupName"),
+                "NewInstancesProtectedFromScaleIn": group.get(
+                    "NewInstancesProtectedFromScaleIn",
+                ),
+                "Status": group.get("Status"),
+            },
+        )
+
+        if group.get("VPCZoneIdentifier", None):
+            vpclist = group["VPCZoneIdentifier"]
+            subnet_ids = vpclist.split(",") if "," in vpclist else [vpclist]
             subnets = []
             for subnet_id in subnet_ids:
-                subnets.append({
-                    'VPCZoneIdentifier': subnet_id,
-                    'AutoScalingGroupARN': group['AutoScalingGroupARN'],
-                })
+                subnets.append(
+                    {
+                        "VPCZoneIdentifier": subnet_id,
+                        "AutoScalingGroupARN": group["AutoScalingGroupARN"],
+                    },
+                )
             related_vpcs.extend(subnets)
 
-        for instance_data in group.get('Instances', []):
-            related_instances.append({
-                'InstanceId': instance_data['InstanceId'],
-                'AutoScalingGroupARN': group['AutoScalingGroupARN'],
-            })
+        for instance_data in group.get("Instances", []):
+            related_instances.append(
+                {
+                    "InstanceId": instance_data["InstanceId"],
+                    "AutoScalingGroupARN": group["AutoScalingGroupARN"],
+                },
+            )
 
     return AsgData(
         group_list=transformed_groups,
@@ -122,7 +160,11 @@ def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
 
 @timeit
 def load_launch_configurations(
-    neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
     load(
         neo4j_session,
@@ -135,7 +177,11 @@ def load_launch_configurations(
 
 
 def load_groups(
-        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
     load(
         neo4j_session,
@@ -148,7 +194,11 @@ def load_groups(
 
 
 def load_asg_subnets(
-        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
     load(
         neo4j_session,
@@ -161,7 +211,11 @@ def load_asg_subnets(
 
 
 def load_asg_instances(
-        neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
     load(
         neo4j_session,
@@ -175,31 +229,80 @@ def load_asg_instances(
 
 @timeit
 def load_auto_scaling_groups(
-    neo4j_session: neo4j.Session, data: AsgData, region: str, current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: AsgData,
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
-    load_groups(neo4j_session, data.group_list, region, current_aws_account_id, update_tag)
-    load_asg_instances(neo4j_session, data.instance_list, region, current_aws_account_id, update_tag)
-    load_asg_subnets(neo4j_session, data.subnet_list, region, current_aws_account_id, update_tag)
+    load_groups(
+        neo4j_session,
+        data.group_list,
+        region,
+        current_aws_account_id,
+        update_tag,
+    )
+    load_asg_instances(
+        neo4j_session,
+        data.instance_list,
+        region,
+        current_aws_account_id,
+        update_tag,
+    )
+    load_asg_subnets(
+        neo4j_session,
+        data.subnet_list,
+        region,
+        current_aws_account_id,
+        update_tag,
+    )
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
     logger.debug("Running EC2 instance cleanup")
-    GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
-    GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
 def sync_ec2_auto_scaling_groups(
-        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: list[str],
-        current_aws_account_id: str, update_tag: int, common_job_parameters: dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
 ) -> None:
     for region in regions:
-        logger.debug("Syncing auto scaling groups for region '%s' in account '%s'.", region, current_aws_account_id)
+        logger.debug(
+            "Syncing auto scaling groups for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
         lc_data = get_launch_configurations(boto3_session, region)
         asg_data = get_ec2_auto_scaling_groups(boto3_session, region)
         lc_transformed = transform_launch_configurations(lc_data)
         asg_transformed = transform_auto_scaling_groups(asg_data)
-        load_launch_configurations(neo4j_session, lc_transformed, region, current_aws_account_id, update_tag)
-        load_auto_scaling_groups(neo4j_session, asg_transformed, region, current_aws_account_id, update_tag)
+        load_launch_configurations(
+            neo4j_session,
+            lc_transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        load_auto_scaling_groups(
+            neo4j_session,
+            asg_transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/elastic_ip_addresses.py

@@ -6,20 +6,28 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
-from .util import get_botocore_config
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
+from .util import get_botocore_config
+
 logger = logging.getLogger(__name__)
 
 
 @timeit
 @aws_handle_regions
-def get_elastic_ip_addresses(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+def get_elastic_ip_addresses(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict]:
+    client = boto3_session.client(
+        "ec2",
+        region_name=region,
+        config=get_botocore_config(),
+    )
     try:
-        addresses = client.describe_addresses()['Addresses']
+        addresses = client.describe_addresses()["Addresses"]
     except ClientError as e:
         logger.warning(f"Failed retrieve address for region - {region}. Error - {e}")
         raise
@@ -28,8 +36,11 @@ def get_elastic_ip_addresses(boto3_session: boto3.session.Session, region: str)
 
 @timeit
 def load_elastic_ip_addresses(
-    neo4j_session: neo4j.Session, elastic_ip_addresses: List[Dict], region: str,
-    current_aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    elastic_ip_addresses: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
     """
     Creates (:ElasticIpAddress)
@@ -37,7 +48,9 @@ def load_elastic_ip_addresses(
     (:EC2Instance)-[:ELASTIC_IP_ADDRESS]->(:ElasticIpAddress),
     (:NetworkInterface)-[:ELASTIC_IP_ADDRESS]->(:ElasticIpAddress),
     """
-    logger.info(f"Loading {len(elastic_ip_addresses)} Elastic IP Addresses in {region}.")
+    logger.info(
+        f"Loading {len(elastic_ip_addresses)} Elastic IP Addresses in {region}.",
+    )
     ingest_addresses = """
     UNWIND $elastic_ip_addresses as eia
         MERGE (address: ElasticIPAddress{id: eia.PublicIp})
@@ -80,9 +93,12 @@ def load_elastic_ip_addresses(
 
 
 @timeit
-def cleanup_elastic_ip_addresses(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+def cleanup_elastic_ip_addresses(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
     run_cleanup_job(
-        'aws_import_elastic_ip_addresses_cleanup.json',
+        "aws_import_elastic_ip_addresses_cleanup.json",
         neo4j_session,
         common_job_parameters,
     )
@@ -90,11 +106,23 @@ def cleanup_elastic_ip_addresses(neo4j_session: neo4j.Session, common_job_parame
 
 @timeit
 def sync_elastic_ip_addresses(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
-    current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing Elastic IP Addresses for region {region} in account {current_aws_account_id}.")
+        logger.info(
+            f"Syncing Elastic IP Addresses for region {region} in account {current_aws_account_id}.",
+        )
         addresses = get_elastic_ip_addresses(boto3_session, region)
-        load_elastic_ip_addresses(neo4j_session, addresses, region, current_aws_account_id, update_tag)
+        load_elastic_ip_addresses(
+            neo4j_session,
+            addresses,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup_elastic_ip_addresses(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/images.py

@@ -19,7 +19,11 @@ logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_images_in_use(neo4j_session: neo4j.Session, region: str, current_aws_account_id: str) -> List[str]:
+def get_images_in_use(
+    neo4j_session: neo4j.Session,
+    region: str,
+    current_aws_account_id: str,
+) -> List[str]:
     get_images_query = """
     CALL {
     MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
@@ -37,8 +41,10 @@ def get_images_in_use(neo4j_session: neo4j.Session, region: str, current_aws_acc
     RETURN DISTINCT image;
     """
     result = read_list_of_values_tx(
-        neo4j_session, get_images_query,
-        AWS_ACCOUNT_ID=current_aws_account_id, Region=region,
+        neo4j_session,
+        get_images_query,
+        AWS_ACCOUNT_ID=current_aws_account_id,
+        Region=region,
     )
     images = [str(image) for image in result]
     return images
@@ -46,40 +52,52 @@ def get_images_in_use(neo4j_session: neo4j.Session, region: str, current_aws_acc
 
 @timeit
 @aws_handle_regions
-def get_images(boto3_session: boto3.session.Session, region: str, image_ids: List[str]) -> List[Dict]:
-    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+def get_images(
+    boto3_session: boto3.session.Session,
+    region: str,
+    image_ids: List[str],
+) -> List[Dict]:
+    client = boto3_session.client(
+        "ec2",
+        region_name=region,
+        config=get_botocore_config(),
+    )
     images = []
     self_images = []
     try:
-        self_images = client.describe_images(Owners=['self'])['Images']
+        self_images = client.describe_images(Owners=["self"])["Images"]
     except ClientError as e:
-        logger.warning(f"Failed retrieve self owned images for region - {region}. Error - {e}")
+        logger.warning(
+            f"Failed retrieve self owned images for region - {region}. Error - {e}"
+        )
     images.extend(self_images)
     if image_ids:
-        self_image_ids = {image['ImageId'] for image in images}
+        self_image_ids = {image["ImageId"] for image in images}
         # Go one by one to avoid losing all images if one fails
         for image in image_ids:
             if image in self_image_ids:
                 continue
             try:
-                public_images = client.describe_images(ImageIds=[image])['Images']
+                public_images = client.describe_images(ImageIds=[image])["Images"]
                 images.extend(public_images)
             except ClientError as e:
-                logger.warning(f"Failed retrieve image id {image} for region - {region}. Error - {e}")
+                logger.warning(
+                    f"Failed retrieve image id {image} for region - {region}. Error - {e}"
+                )
     return images
 
 
 @timeit
 def load_images(
-        neo4j_session: neo4j.Session,
-        data: List[Dict[str, Any]],
-        region: str,
-        current_aws_account_id: str,
-        update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
 ) -> None:
     # AMI IDs are unique to each AWS Region. Hence we make an 'ID' string that is a combo of ImageId and region
     for image in data:
-        image['ID'] = image['ImageId'] + '|' + region
+        image["ID"] = image["ImageId"] + "|" + region
     load(
         neo4j_session,
         EC2ImageSchema(),
@@ -91,18 +109,29 @@ def load_images(
 
 
 @timeit
-def cleanup_images(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def cleanup_images(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
     cleanup_job = GraphJob.from_node_schema(EC2ImageSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
 
 
 @timeit
 def sync_ec2_images(
-        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
-        current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info("Syncing images for region '%s' in account '%s'.", region, current_aws_account_id)
+        logger.info(
+            "Syncing images for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
         images_in_use = get_images_in_use(neo4j_session, region, current_aws_account_id)
         data = get_images(boto3_session, region, images_in_use)
         load_images(neo4j_session, data, region, current_aws_account_id, update_tag)