cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/aws/efs.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_efs_mount_targets(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_mount_targets")
+    mountTargets = []
+    for page in paginator.paginate():
+        mountTargets.extend(page["MountTargets"])
+    return mountTargets
+
+
+@timeit
+def load_efs_mount_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} mount targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsMountTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        EfsMountTargetSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Efs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+        mountTargets = get_efs_mount_targets(boto3_session, region)
+        mount_target_data: List[Dict[str, Any]] = []
+        for mountTarget in mountTargets:
+            mount_target_data.append(mountTarget)
+
+        load_efs_mount_targets(
+            neo4j_session,
+            mount_target_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/eks.py

@@ -18,28 +18,32 @@ logger = logging.getLogger(__name__)
 @timeit
 @aws_handle_regions
 def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[str]:
-    client = boto3_session.client('eks', region_name=region)
+    client = boto3_session.client("eks", region_name=region)
     clusters: List[str] = []
-    paginator = client.get_paginator('list_clusters')
+    paginator = client.get_paginator("list_clusters")
     for page in paginator.paginate():
-        clusters.extend(page['clusters'])
+        clusters.extend(page["clusters"])
     return clusters
 
 
 @timeit
-def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_name: str) -> Dict:
-    client = boto3_session.client('eks', region_name=region)
+def get_eks_describe_cluster(
+    boto3_session: boto3.session.Session,
+    region: str,
+    cluster_name: str,
+) -> Dict:
+    client = boto3_session.client("eks", region_name=region)
     response = client.describe_cluster(name=cluster_name)
-    return response['cluster']
+    return response["cluster"]
 
 
 @timeit
 def load_eks_clusters(
-        neo4j_session: neo4j.Session,
-        cluster_data: List[Dict[str, Any]],
-        region: str,
-        current_aws_account_id: str,
-        aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    cluster_data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     load(
         neo4j_session,
@@ -57,50 +61,72 @@ def _process_logging(cluster: Dict) -> bool:
     at least one entry has audit logging set to Enabled.
     """
     logging: bool = False
-    cluster_logging: Any = cluster.get('logging', {}).get('clusterLogging')
+    cluster_logging: Any = cluster.get("logging", {}).get("clusterLogging")
     if cluster_logging:
-        logging = any(filter(lambda x: 'audit' in x['types'] and x['enabled'], cluster_logging))  # type: ignore
+        logging = any(filter(lambda x: "audit" in x["types"] and x["enabled"], cluster_logging))  # type: ignore
     return logging
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
     logger.info("Running EKS cluster cleanup")
-    GraphJob.from_node_schema(EKSClusterSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EKSClusterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 def transform(cluster_data: Dict[str, Any]) -> List[Dict[str, Any]]:
     transformed_list = []
     for cluster_name, cluster_dict in cluster_data.items():
         transformed_dict = cluster_dict.copy()
-        transformed_dict['ClusterLogging'] = _process_logging(transformed_dict)
-        transformed_dict['ClusterEndpointPublic'] = transformed_dict.get('resourcesVpcConfig', {}).get(
-            'endpointPublicAccess',
+        transformed_dict["ClusterLogging"] = _process_logging(transformed_dict)
+        transformed_dict["ClusterEndpointPublic"] = transformed_dict.get(
+            "resourcesVpcConfig",
+            {},
+        ).get(
+            "endpointPublicAccess",
         )
-        if 'createdAt' in transformed_dict:
-            transformed_dict['created_at'] = str(transformed_dict['createdAt'])
+        if "createdAt" in transformed_dict:
+            transformed_dict["created_at"] = str(transformed_dict["createdAt"])
         transformed_list.append(transformed_dict)
     return transformed_list
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        boto3_session: boto3.session.Session,
-        regions: List[str],
-        current_aws_account_id: str,
-        update_tag: int,
-        common_job_parameters: Dict[str, Any],
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
-        logger.info("Syncing EKS for region '%s' in account '%s'.", region, current_aws_account_id)
+        logger.info(
+            "Syncing EKS for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
 
         clusters: List[str] = get_eks_clusters(boto3_session, region)
         cluster_data = {}
         for cluster_name in clusters:
-            cluster_data[cluster_name] = get_eks_describe_cluster(boto3_session, region, cluster_name)
+            cluster_data[cluster_name] = get_eks_describe_cluster(
+                boto3_session,
+                region,
+                cluster_name,
+            )
         transformed_list = transform(cluster_data)
 
-        load_eks_clusters(neo4j_session, transformed_list, region, current_aws_account_id, update_tag)
+        load_eks_clusters(
+            neo4j_session,
+            transformed_list,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/elasticache.py

@@ -17,7 +17,7 @@ stat_handler = get_stats_client(__name__)
 
 
 def _get_topic(cluster: Dict) -> Dict:
-    return cluster['NotificationConfiguration']
+    return cluster["NotificationConfiguration"]
 
 
 def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
@@ -28,7 +28,7 @@ def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
     topics: List[Dict] = []
     for cluster in cluster_data:
         topic = _get_topic(cluster)
-        topic_arn = topic['TopicArn']
+        topic_arn = topic["TopicArn"]
         if topic_arn not in seen:
             seen.add(topic_arn)
             topics.append(topic)
@@ -37,20 +37,26 @@ def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
 
 @timeit
 @aws_handle_regions
-def get_elasticache_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_elasticache_clusters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict]:
     logger.debug(f"Getting ElastiCache Clusters in region '{region}'.")
-    client = boto3_session.client('elasticache', region_name=region)
-    paginator = client.get_paginator('describe_cache_clusters')
+    client = boto3_session.client("elasticache", region_name=region)
+    paginator = client.get_paginator("describe_cache_clusters")
     clusters: List[Dict] = []
     for page in paginator.paginate():
-        clusters.extend(page['CacheClusters'])
+        clusters.extend(page["CacheClusters"])
     return clusters
 
 
 @timeit
 def load_elasticache_clusters(
-    neo4j_session: neo4j.Session, clusters: List[Dict], region: str,
-    aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    clusters: List[Dict],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
 ) -> None:
     query = """
     UNWIND $clusters as elasticache_cluster
@@ -85,7 +91,9 @@ def load_elasticache_clusters(
         ON CREATE SET r2.firstseen = timestamp()
         SET r2.lastupdated = $aws_update_tag
     """
-    logger.info(f"Loading f{len(clusters)} ElastiCache clusters for region '{region}' into graph.")
+    logger.info(
+        f"Loading f{len(clusters)} ElastiCache clusters for region '{region}' into graph.",
+    )
     neo4j_session.run(
         query,
         clusters=clusters,
@@ -96,29 +104,45 @@ def load_elasticache_clusters(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, current_aws_account_id: str, update_tag: int) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    current_aws_account_id: str,
+    update_tag: int,
+) -> None:
     run_cleanup_job(
-        'aws_import_elasticache_cleanup.json',
+        "aws_import_elasticache_cleanup.json",
         neo4j_session,
-        {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_account_id},
+        {"UPDATE_TAG": update_tag, "AWS_ID": current_aws_account_id},
     )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing ElastiCache clusters for region '{region}' in account {current_aws_account_id}")
+        logger.info(
+            f"Syncing ElastiCache clusters for region '{region}' in account {current_aws_account_id}",
+        )
         clusters = get_elasticache_clusters(boto3_session, region)
-        load_elasticache_clusters(neo4j_session, clusters, region, current_aws_account_id, update_tag)
+        load_elasticache_clusters(
+            neo4j_session,
+            clusters,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, current_aws_account_id, update_tag)
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='AWSAccount',
+        group_type="AWSAccount",
         group_id=current_aws_account_id,
-        synced_type='ElasticacheCluster',
+        synced_type="ElasticacheCluster",
         update_tag=update_tag,
         stat_handler=stat_handler,
     )

cartography/intel/aws/elasticsearch.py

@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)
 def _get_botocore_config() -> botocore.config.Config:
     return botocore.config.Config(
         retries={
-            'max_attempts': 8,
+            "max_attempts": 8,
         },
     )
 
@@ -35,19 +35,26 @@ def _get_es_domains(client: botocore.client.BaseClient) -> List[Dict]:
     :return: list of ES domains
     """
     data = client.list_domain_names()
-    domain_names = [d['DomainName'] for d in data.get('DomainNames', [])]
+    domain_names = [d["DomainName"] for d in data.get("DomainNames", [])]
     # NOTE describe_elasticsearch_domains takes at most 5 domain names
-    domain_name_chunks = [domain_names[i:i + 5] for i in range(0, len(domain_names), 5)]
+    domain_name_chunks = [
+        domain_names[i : i + 5] for i in range(0, len(domain_names), 5)
+    ]
     domains: List[Dict] = []
     for domain_name_chunk in domain_name_chunks:
-        chunk_data = client.describe_elasticsearch_domains(DomainNames=domain_name_chunk)
-        domains.extend(chunk_data['DomainStatusList'])
+        chunk_data = client.describe_elasticsearch_domains(
+            DomainNames=domain_name_chunk,
+        )
+        domains.extend(chunk_data["DomainStatusList"])
     return domains
 
 
 @timeit
 def _load_es_domains(
-    neo4j_session: neo4j.Session, domain_list: List[Dict], aws_account_id: str, aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    domain_list: List[Dict],
+    aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     """
     Ingest Elastic Search domains
@@ -86,7 +93,7 @@ def _load_es_domains(
     # TODO this is a hacky workaround -- neo4j doesn't accept datetime objects and this section of the object
     # TODO contains one. we really shouldn't be sending the entire object to neo4j
     for d in domain_list:
-        del d['ServiceSoftwareOptions']
+        del d["ServiceSoftwareOptions"]
 
     neo4j_session.run(
         ingest_records,
@@ -104,7 +111,10 @@ def _load_es_domains(
 
 @timeit
 def _link_es_domains_to_dns(
-    neo4j_session: neo4j.Session, domain_id: str, domain_data: Dict, aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    domain_id: str,
+    domain_data: Dict,
+    aws_update_tag: int,
 ) -> None:
     """
     Link the ES domain to its DNS FQDN endpoint and create associated nodes in the graph
@@ -117,15 +127,24 @@ def _link_es_domains_to_dns(
     # TODO add support for endpoints to this method
     if domain_data.get("Endpoint"):
         ingest_dns_record_by_fqdn(
-            neo4j_session, aws_update_tag, domain_data["Endpoint"], domain_id,
-            record_label="ESDomain", dns_node_additional_label="AWSDNSRecord",
+            neo4j_session,
+            aws_update_tag,
+            domain_data["Endpoint"],
+            domain_id,
+            record_label="ESDomain",
+            dns_node_additional_label="AWSDNSRecord",
         )
     else:
         logger.debug(f"No es endpoint data for domain id {domain_id}")
 
 
 @timeit
-def _link_es_domain_vpc(neo4j_session: neo4j.Session, domain_id: str, domain_data: Dict, aws_update_tag: int) -> None:
+def _link_es_domain_vpc(
+    neo4j_session: neo4j.Session,
+    domain_id: str,
+    domain_data: Dict,
+    aws_update_tag: int,
+) -> None:
     """
     Link the ES domain to its DNS FQDN endpoint and create associated nodes in the graph
     if needed
@@ -177,7 +196,11 @@ def _link_es_domain_vpc(neo4j_session: neo4j.Session, domain_id: str, domain_dat
 
 
 @timeit
-def _process_access_policy(neo4j_session: neo4j.Session, domain_id: str, domain_data: Dict) -> None:
+def _process_access_policy(
+    neo4j_session: neo4j.Session,
+    domain_id: str,
+    domain_data: Dict,
+) -> None:
     """
     Link the ES domain to its DNS FQDN endpoint and create associated nodes in the graph
     if needed
@@ -186,12 +209,14 @@ def _process_access_policy(neo4j_session: neo4j.Session, domain_id: str, domain_
     :param domain_id: ES domain id
     :param domain_data: domain data
     """
-    tag_es = "MATCH (es:ESDomain{id: $DomainId}) SET es.exposed_internet = $InternetExposed"
+    tag_es = (
+        "MATCH (es:ESDomain{id: $DomainId}) SET es.exposed_internet = $InternetExposed"
+    )
 
     exposed_internet = False
 
     if domain_data.get("Endpoint") and domain_data.get("AccessPolicies"):
-        policy = Policy(json.loads(domain_data['AccessPolicies']))
+        policy = Policy(json.loads(domain_data["AccessPolicies"]))
         if policy.is_internet_accessible():
             exposed_internet = True
 
@@ -201,20 +226,32 @@ def _process_access_policy(neo4j_session: neo4j.Session, domain_id: str, domain_
 @timeit
 def cleanup(neo4j_session: neo4j.Session, update_tag: int, aws_account_id: int) -> None:
     run_cleanup_job(
-        'aws_import_es_cleanup.json',
+        "aws_import_es_cleanup.json",
         neo4j_session,
-        {'UPDATE_TAG': update_tag, 'AWS_ID': aws_account_id},
+        {"UPDATE_TAG": update_tag, "AWS_ID": aws_account_id},
     )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info("Syncing Elasticsearch Service for region '%s' in account '%s'.", region, current_aws_account_id)
-        client = boto3_session.client('es', region_name=region, config=_get_botocore_config())
+        logger.info(
+            "Syncing Elasticsearch Service for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
+        client = boto3_session.client(
+            "es",
+            region_name=region,
+            config=_get_botocore_config(),
+        )
         data = _get_es_domains(client)
         _load_es_domains(neo4j_session, data, current_aws_account_id, update_tag)
 

cartography/intel/aws/emr.py

@@ -24,40 +24,59 @@ DESCRIBE_SLEEP = 1
 
 @timeit
 @aws_handle_regions
-def get_emr_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
-    client = boto3_session.client('emr', region_name=region, config=get_botocore_config())
+def get_emr_clusters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "emr",
+        region_name=region,
+        config=get_botocore_config(),
+    )
     clusters: List[Dict[str, Any]] = []
-    paginator = client.get_paginator('list_clusters')
+    paginator = client.get_paginator("list_clusters")
     for page in paginator.paginate():
-        cluster = page['Clusters']
+        cluster = page["Clusters"]
         clusters.extend(cluster)
         time.sleep(LIST_SLEEP)
     return clusters
 
 
 @timeit
-def get_emr_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_id: str) -> Dict[str, Any]:
-    client = boto3_session.client('emr', region_name=region, config=get_botocore_config())
+def get_emr_describe_cluster(
+    boto3_session: boto3.session.Session,
+    region: str,
+    cluster_id: str,
+) -> Dict[str, Any]:
+    client = boto3_session.client(
+        "emr",
+        region_name=region,
+        config=get_botocore_config(),
+    )
     cluster_details: Dict[str, Any] = {}
     try:
         response = client.describe_cluster(ClusterId=cluster_id)
-        cluster_details = response['Cluster']
+        cluster_details = response["Cluster"]
     except botocore.exceptions.ClientError as e:
-        code = e.response['Error']['Code']
-        msg = e.response['Error']['Message']
-        logger.warning(f"Could not run EMR describe_cluster due to boto3 error {code}: {msg}. Skipping.")
+        code = e.response["Error"]["Code"]
+        msg = e.response["Error"]["Message"]
+        logger.warning(
+            f"Could not run EMR describe_cluster due to boto3 error {code}: {msg}. Skipping.",
+        )
     return cluster_details
 
 
 @timeit
 def load_emr_clusters(
-        neo4j_session: neo4j.Session,
-        cluster_data: List[Dict[str, Any]],
-        region: str,
-        current_aws_account_id: str,
-        aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    cluster_data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
-    logger.info(f"Loading EMR {len(cluster_data)} clusters for region '{region}' into graph.")
+    logger.info(
+        f"Loading EMR {len(cluster_data)} clusters for region '{region}' into graph.",
+    )
     load(
         neo4j_session,
         EMRClusterSchema(),
@@ -69,7 +88,10 @@ def load_emr_clusters(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
     logger.debug("Running EMR cleanup job.")
     cleanup_job = GraphJob.from_node_schema(EMRClusterSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
@@ -77,22 +99,38 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any])
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict[str, Any],
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing EMR for region '{region}' in account '{current_aws_account_id}'.")
+        logger.info(
+            f"Syncing EMR for region '{region}' in account '{current_aws_account_id}'.",
+        )
 
         clusters = get_emr_clusters(boto3_session, region)
 
         cluster_data: List[Dict[str, Any]] = []
         for cluster in clusters:
-            cluster_id = cluster['Id']
-            cluster_details = get_emr_describe_cluster(boto3_session, region, cluster_id)
+            cluster_id = cluster["Id"]
+            cluster_details = get_emr_describe_cluster(
+                boto3_session,
+                region,
+                cluster_id,
+            )
             if cluster_details:
                 cluster_data.append(cluster_details)
             time.sleep(DESCRIBE_SLEEP)
 
-        load_emr_clusters(neo4j_session, cluster_data, region, current_aws_account_id, update_tag)
+        load_emr_clusters(
+            neo4j_session,
+            cluster_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
     cleanup(neo4j_session, common_job_parameters)