cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/elasticsearch.py

@@ -8,6 +8,7 @@ import botocore.config
 import neo4j
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.dns import ingest_dns_record_by_fqdn
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
@@ -20,7 +21,7 @@ logger = logging.getLogger(__name__)
 def _get_botocore_config() -> botocore.config.Config:
     return botocore.config.Config(
         retries={
-            'max_attempts': 8,
+            "max_attempts": 8,
         },
     )
 
@@ -35,19 +36,26 @@ def _get_es_domains(client: botocore.client.BaseClient) -> List[Dict]:
     :return: list of ES domains
     """
     data = client.list_domain_names()
-    domain_names = [d['DomainName'] for d in data.get('DomainNames', [])]
+    domain_names = [d["DomainName"] for d in data.get("DomainNames", [])]
     # NOTE describe_elasticsearch_domains takes at most 5 domain names
-    domain_name_chunks = [domain_names[i:i + 5] for i in range(0, len(domain_names), 5)]
+    domain_name_chunks = [
+        domain_names[i : i + 5] for i in range(0, len(domain_names), 5)
+    ]
     domains: List[Dict] = []
     for domain_name_chunk in domain_name_chunks:
-        chunk_data = client.describe_elasticsearch_domains(DomainNames=domain_name_chunk)
-        domains.extend(chunk_data['DomainStatusList'])
+        chunk_data = client.describe_elasticsearch_domains(
+            DomainNames=domain_name_chunk,
+        )
+        domains.extend(chunk_data["DomainStatusList"])
     return domains
 
 
 @timeit
 def _load_es_domains(
-    neo4j_session: neo4j.Session, domain_list: List[Dict], aws_account_id: str, aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    domain_list: List[Dict],
+    aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     """
     Ingest Elastic Search domains
@@ -86,9 +94,10 @@ def _load_es_domains(
     # TODO this is a hacky workaround -- neo4j doesn't accept datetime objects and this section of the object
     # TODO contains one. we really shouldn't be sending the entire object to neo4j
     for d in domain_list:
-        del d['ServiceSoftwareOptions']
+        del d["ServiceSoftwareOptions"]
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_records,
         Records=domain_list,
         AWS_ACCOUNT_ID=aws_account_id,
@@ -104,7 +113,10 @@ def _load_es_domains(
 
 @timeit
 def _link_es_domains_to_dns(
-    neo4j_session: neo4j.Session, domain_id: str, domain_data: Dict, aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    domain_id: str,
+    domain_data: Dict,
+    aws_update_tag: int,
 ) -> None:
     """
     Link the ES domain to its DNS FQDN endpoint and create associated nodes in the graph
@@ -117,15 +129,24 @@ def _link_es_domains_to_dns(
     # TODO add support for endpoints to this method
     if domain_data.get("Endpoint"):
         ingest_dns_record_by_fqdn(
-            neo4j_session, aws_update_tag, domain_data["Endpoint"], domain_id,
-            record_label="ESDomain", dns_node_additional_label="AWSDNSRecord",
+            neo4j_session,
+            aws_update_tag,
+            domain_data["Endpoint"],
+            domain_id,
+            record_label="ESDomain",
+            dns_node_additional_label="AWSDNSRecord",
         )
     else:
         logger.debug(f"No es endpoint data for domain id {domain_id}")
 
 
 @timeit
-def _link_es_domain_vpc(neo4j_session: neo4j.Session, domain_id: str, domain_data: Dict, aws_update_tag: int) -> None:
+def _link_es_domain_vpc(
+    neo4j_session: neo4j.Session,
+    domain_id: str,
+    domain_data: Dict,
+    aws_update_tag: int,
+) -> None:
     """
     Link the ES domain to its DNS FQDN endpoint and create associated nodes in the graph
     if needed
@@ -160,7 +181,8 @@ def _link_es_domain_vpc(neo4j_session: neo4j.Session, domain_id: str, domain_dat
         groupList = vpc_data.get("SecurityGroupIds", [])
 
         if len(subnetList) > 0:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_subnet,
                 DomainId=domain_id,
                 SubnetList=subnetList,
@@ -168,7 +190,8 @@ def _link_es_domain_vpc(neo4j_session: neo4j.Session, domain_id: str, domain_dat
             )
 
         if len(groupList) > 0:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_sec_groups,
                 DomainId=domain_id,
                 SecGroupList=groupList,
@@ -177,7 +200,11 @@ def _link_es_domain_vpc(neo4j_session: neo4j.Session, domain_id: str, domain_dat
 
 
 @timeit
-def _process_access_policy(neo4j_session: neo4j.Session, domain_id: str, domain_data: Dict) -> None:
+def _process_access_policy(
+    neo4j_session: neo4j.Session,
+    domain_id: str,
+    domain_data: Dict,
+) -> None:
     """
     Link the ES domain to its DNS FQDN endpoint and create associated nodes in the graph
     if needed
@@ -186,35 +213,54 @@ def _process_access_policy(neo4j_session: neo4j.Session, domain_id: str, domain_
     :param domain_id: ES domain id
     :param domain_data: domain data
     """
-    tag_es = "MATCH (es:ESDomain{id: $DomainId}) SET es.exposed_internet = $InternetExposed"
+    tag_es = (
+        "MATCH (es:ESDomain{id: $DomainId}) SET es.exposed_internet = $InternetExposed"
+    )
 
     exposed_internet = False
 
     if domain_data.get("Endpoint") and domain_data.get("AccessPolicies"):
-        policy = Policy(json.loads(domain_data['AccessPolicies']))
+        policy = Policy(json.loads(domain_data["AccessPolicies"]))
         if policy.is_internet_accessible():
             exposed_internet = True
 
-    neo4j_session.run(tag_es, DomainId=domain_id, InternetExposed=exposed_internet)
+    run_write_query(
+        neo4j_session,
+        tag_es,
+        DomainId=domain_id,
+        InternetExposed=exposed_internet,
+    )
 
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, update_tag: int, aws_account_id: int) -> None:
     run_cleanup_job(
-        'aws_import_es_cleanup.json',
+        "aws_import_es_cleanup.json",
         neo4j_session,
-        {'UPDATE_TAG': update_tag, 'AWS_ID': aws_account_id},
+        {"UPDATE_TAG": update_tag, "AWS_ID": aws_account_id},
     )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     for region in regions:
-        logger.info("Syncing Elasticsearch Service for region '%s' in account '%s'.", region, current_aws_account_id)
-        client = boto3_session.client('es', region_name=region, config=_get_botocore_config())
+        logger.info(
+            "Syncing Elasticsearch Service for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
+        client = boto3_session.client(
+            "es",
+            region_name=region,
+            config=_get_botocore_config(),
+        )
         data = _get_es_domains(client)
         _load_es_domains(neo4j_session, data, current_aws_account_id, update_tag)
 

cartography/intel/aws/emr.py

@@ -24,40 +24,59 @@ DESCRIBE_SLEEP = 1
 
 @timeit
 @aws_handle_regions
-def get_emr_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
-    client = boto3_session.client('emr', region_name=region, config=get_botocore_config())
+def get_emr_clusters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "emr",
+        region_name=region,
+        config=get_botocore_config(),
+    )
     clusters: List[Dict[str, Any]] = []
-    paginator = client.get_paginator('list_clusters')
+    paginator = client.get_paginator("list_clusters")
     for page in paginator.paginate():
-        cluster = page['Clusters']
+        cluster = page["Clusters"]
         clusters.extend(cluster)
         time.sleep(LIST_SLEEP)
     return clusters
 
 
 @timeit
-def get_emr_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_id: str) -> Dict[str, Any]:
-    client = boto3_session.client('emr', region_name=region, config=get_botocore_config())
+def get_emr_describe_cluster(
+    boto3_session: boto3.session.Session,
+    region: str,
+    cluster_id: str,
+) -> Dict[str, Any]:
+    client = boto3_session.client(
+        "emr",
+        region_name=region,
+        config=get_botocore_config(),
+    )
     cluster_details: Dict[str, Any] = {}
     try:
         response = client.describe_cluster(ClusterId=cluster_id)
-        cluster_details = response['Cluster']
+        cluster_details = response["Cluster"]
     except botocore.exceptions.ClientError as e:
-        code = e.response['Error']['Code']
-        msg = e.response['Error']['Message']
-        logger.warning(f"Could not run EMR describe_cluster due to boto3 error {code}: {msg}. Skipping.")
+        code = e.response["Error"]["Code"]
+        msg = e.response["Error"]["Message"]
+        logger.warning(
+            f"Could not run EMR describe_cluster due to boto3 error {code}: {msg}. Skipping.",
+        )
     return cluster_details
 
 
 @timeit
 def load_emr_clusters(
-        neo4j_session: neo4j.Session,
-        cluster_data: List[Dict[str, Any]],
-        region: str,
-        current_aws_account_id: str,
-        aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    cluster_data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
-    logger.info(f"Loading EMR {len(cluster_data)} clusters for region '{region}' into graph.")
+    logger.info(
+        f"Loading EMR {len(cluster_data)} clusters for region '{region}' into graph.",
+    )
     load(
         neo4j_session,
         EMRClusterSchema(),
@@ -69,7 +88,10 @@ def load_emr_clusters(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
     logger.debug("Running EMR cleanup job.")
     cleanup_job = GraphJob.from_node_schema(EMRClusterSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
@@ -77,22 +99,38 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any])
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict[str, Any],
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing EMR for region '{region}' in account '{current_aws_account_id}'.")
+        logger.info(
+            f"Syncing EMR for region '{region}' in account '{current_aws_account_id}'.",
+        )
 
         clusters = get_emr_clusters(boto3_session, region)
 
         cluster_data: List[Dict[str, Any]] = []
         for cluster in clusters:
-            cluster_id = cluster['Id']
-            cluster_details = get_emr_describe_cluster(boto3_session, region, cluster_id)
+            cluster_id = cluster["Id"]
+            cluster_details = get_emr_describe_cluster(
+                boto3_session,
+                region,
+                cluster_id,
+            )
             if cluster_details:
                 cluster_data.append(cluster_details)
             time.sleep(DESCRIBE_SLEEP)
 
-        load_emr_clusters(neo4j_session, cluster_data, region, current_aws_account_id, update_tag)
+        load_emr_clusters(
+            neo4j_session,
+            cluster_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/eventbridge.py

@@ -0,0 +1,164 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
+from cartography.models.aws.eventbridge.target import EventBridgeTargetSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_eventbridge_rules(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_rules")
+    rules = []
+
+    for page in paginator.paginate():
+        rules.extend(page.get("Rules", []))
+
+    return rules
+
+
+@timeit
+@aws_handle_regions
+def get_eventbridge_targets(
+    boto3_session: boto3.Session, region: str, rules: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    targets = []
+    for rule in rules:
+        paginator = client.get_paginator("list_targets_by_rule")
+        for page in paginator.paginate(Rule=rule["Name"]):
+            for target in page.get("Targets", []):
+                target["RuleArn"] = rule["Arn"]
+                targets.append(target)
+    return targets
+
+
+def transform_eventbridge_targets(
+    targets: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform EventBridge target data for ingestion into Neo4j.
+    """
+    transformed_data = []
+    for target in targets:
+        transformed_target = {
+            "Id": target["Arn"],
+            "Arn": target["Arn"],
+            "RuleArn": target["RuleArn"],
+            "RoleArn": target.get("RoleArn"),
+            "Region": region,
+        }
+        transformed_data.append(transformed_target)
+    return transformed_data
+
+
+@timeit
+def load_eventbridge_rules(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} rules for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeRuleSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_eventbridge_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running EventBridge cleanup job.")
+    GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(EventBridgeTargetSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing EventBridge for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        rules = get_eventbridge_rules(boto3_session, region)
+        load_eventbridge_rules(
+            neo4j_session,
+            rules,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        targets = get_eventbridge_targets(boto3_session, region, rules)
+        transformed_targets = transform_eventbridge_targets(targets, region)
+        load_eventbridge_targets(
+            neo4j_session,
+            transformed_targets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/glue.py

@@ -0,0 +1,181 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.models.aws.glue.job import GlueJobSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_glue_connections(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_connections")
+    connections = []
+    for page in paginator.paginate():
+        connections.extend(page.get("ConnectionList", []))
+
+    return connections
+
+
+@timeit
+@aws_handle_regions
+def get_glue_jobs(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_jobs")
+    jobs = []
+    for page in paginator.paginate():
+        jobs.extend(page.get("Jobs", []))
+    return jobs
+
+
+def transform_glue_job(jobs: List[Dict[str, Any]], region: str) -> List[Dict[str, Any]]:
+    """
+    Transform Glue job data for ingestion
+    """
+    transformed_jobs = []
+    for job in jobs:
+        transformed_job = {
+            "Name": job["Name"],
+            "ProfileName": job.get("ProfileName"),
+            "JobMode": job.get("JobMode"),
+            "Connections": job.get("Connections", {}).get("Connections"),
+            "Region": region,
+            "Description": job.get("Description"),
+        }
+        transformed_jobs.append(transformed_job)
+    return transformed_jobs
+
+
+def transform_glue_connections(
+    connections: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Glue connection data for ingestion
+    """
+    transformed_connections = []
+    for connection in connections:
+        transformed_connection = {
+            "Name": connection["Name"],
+            "Description": connection.get("Description"),
+            "ConnectionType": connection.get("ConnectionType"),
+            "Status": connection.get("Status"),
+            "StatusReason": connection.get("StatusReason"),
+            "AuthenticationType": connection.get("AuthenticationConfiguration", {}).get(
+                "AuthenticationType"
+            ),
+            "SecretArn": connection.get("AuthenticationConfiguration", {}).get(
+                "SecretArn"
+            ),
+            "Region": region,
+        }
+        transformed_connections.append(transformed_connection)
+    return transformed_connections
+
+
+@timeit
+def load_glue_connections(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} connections for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueConnectionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_glue_jobs(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} jobs for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueJobSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Glue cleanup job.")
+    GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(GlueJobSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Glue for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        connections = get_glue_connections(boto3_session, region)
+        transformed_connections = transform_glue_connections(connections, region)
+        load_glue_connections(
+            neo4j_session,
+            transformed_connections,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        jobs = get_glue_jobs(boto3_session, region)
+        transformed_jobs = transform_glue_job(jobs, region)
+        load_glue_jobs(
+            neo4j_session,
+            transformed_jobs,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)