cartography 0.108.0rc1__py3-none-any.whl → 0.109.0__py3-none-any.whl

cartography/intel/aws/elasticache.py

@@ -1,118 +1,132 @@
 import logging
-from typing import Dict
-from typing import List
-from typing import Set
+from typing import Any
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.elasticache.cluster import ElasticacheClusterSchema
+from cartography.models.aws.elasticache.topic import ElasticacheTopicSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 stat_handler = get_stats_client(__name__)
 
 
-def _get_topic(cluster: Dict) -> Dict:
-    return cluster["NotificationConfiguration"]
-
-
-def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
-    """
-    Collect unique TopicArns from the cluster data
-    """
-    seen: Set[str] = set()
-    topics: List[Dict] = []
-    for cluster in cluster_data:
-        topic = _get_topic(cluster)
-        topic_arn = topic["TopicArn"]
-        if topic_arn not in seen:
-            seen.add(topic_arn)
-            topics.append(topic)
-    return topics
-
-
 @timeit
 @aws_handle_regions
 def get_elasticache_clusters(
     boto3_session: boto3.session.Session,
     region: str,
-) -> List[Dict]:
-    logger.debug(f"Getting ElastiCache Clusters in region '{region}'.")
+) -> list[dict[str, Any]]:
     client = boto3_session.client("elasticache", region_name=region)
     paginator = client.get_paginator("describe_cache_clusters")
-    clusters: List[Dict] = []
+    clusters: list[dict[str, Any]] = []
     for page in paginator.paginate():
-        clusters.extend(page["CacheClusters"])
+        clusters.extend(page.get("CacheClusters", []))
     return clusters
 
 
+def transform_elasticache_clusters(
+    clusters: list[dict[str, Any]], region: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cluster_data: list[dict[str, Any]] = []
+    topics: dict[str, dict[str, Any]] = {}
+
+    for cluster in clusters:
+        notification = cluster.get("NotificationConfiguration", {})
+        topic_arn = notification.get("TopicArn")
+        cluster_record = {
+            "ARN": cluster["ARN"],
+            "CacheClusterId": cluster["CacheClusterId"],
+            "CacheNodeType": cluster.get("CacheNodeType"),
+            "Engine": cluster.get("Engine"),
+            "EngineVersion": cluster.get("EngineVersion"),
+            "CacheClusterStatus": cluster.get("CacheClusterStatus"),
+            "NumCacheNodes": cluster.get("NumCacheNodes"),
+            "PreferredAvailabilityZone": cluster.get("PreferredAvailabilityZone"),
+            "PreferredMaintenanceWindow": cluster.get("PreferredMaintenanceWindow"),
+            "CacheClusterCreateTime": cluster.get("CacheClusterCreateTime"),
+            "CacheSubnetGroupName": cluster.get("CacheSubnetGroupName"),
+            "AutoMinorVersionUpgrade": cluster.get("AutoMinorVersionUpgrade"),
+            "ReplicationGroupId": cluster.get("ReplicationGroupId"),
+            "SnapshotRetentionLimit": cluster.get("SnapshotRetentionLimit"),
+            "SnapshotWindow": cluster.get("SnapshotWindow"),
+            "AuthTokenEnabled": cluster.get("AuthTokenEnabled"),
+            "TransitEncryptionEnabled": cluster.get("TransitEncryptionEnabled"),
+            "AtRestEncryptionEnabled": cluster.get("AtRestEncryptionEnabled"),
+            "TopicArn": topic_arn,
+            "Region": region,
+        }
+        cluster_data.append(cluster_record)
+
+        if topic_arn:
+            topics.setdefault(
+                topic_arn,
+                {
+                    "TopicArn": topic_arn,
+                    "TopicStatus": notification.get("TopicStatus"),
+                    "cluster_arns": [],
+                },
+            )["cluster_arns"].append(cluster["ARN"])
+
+    return cluster_data, list(topics.values())
+
+
 @timeit
 def load_elasticache_clusters(
     neo4j_session: neo4j.Session,
-    clusters: List[Dict],
+    clusters: list[dict[str, Any]],
     region: str,
     aws_account_id: str,
     update_tag: int,
 ) -> None:
-    query = """
-    UNWIND $clusters as elasticache_cluster
-        MERGE (cluster:ElasticacheCluster{id:elasticache_cluster.ARN})
-        ON CREATE SET cluster.firstseen = timestamp(),
-            cluster.arn = elasticache_cluster.ARN,
-            cluster.topic_arn = elasticache_cluster.NotificationConfiguration.TopicArn,
-            cluster.id = elasticache_cluster.CacheClusterId,
-            cluster.region = $region
-        SET cluster.lastupdated = $aws_update_tag
-
-        WITH cluster, elasticache_cluster
-        MATCH (owner:AWSAccount{id: $aws_account_id})
-        MERGE (owner)-[r3:RESOURCE]->(cluster)
-        ON CREATE SET r3.firstseen = timestamp()
-        SET r3.lastupdated = $aws_update_tag
-
-        WITH elasticache_cluster, owner
-        WHERE NOT elasticache_cluster.NotificationConfiguration IS NULL
-        MERGE (topic:ElasticacheTopic{id: elasticache_cluster.NotificationConfiguration.TopicArn})
-        ON CREATE SET topic.firstseen = timestamp(),
-            topic.arn = elasticache_cluster.NotificationConfiguration.TopicArn
-        SET topic.lastupdated = $aws_update_tag,
-            topic.status = elasticache_cluster.NotificationConfiguration.Status
-
-        MERGE (topic)-[r:CACHE_CLUSTER]->(cluster)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-        WITH cluster, topic
-
-        MERGE (owner)-[r2:RESOURCE]->(topic)
-        ON CREATE SET r2.firstseen = timestamp()
-        SET r2.lastupdated = $aws_update_tag
-    """
     logger.info(
-        f"Loading f{len(clusters)} ElastiCache clusters for region '{region}' into graph.",
+        f"Loading {len(clusters)} ElastiCache clusters for region '{region}' into graph."
     )
-    neo4j_session.run(
-        query,
-        clusters=clusters,
-        region=region,
-        aws_update_tag=update_tag,
-        aws_account_id=aws_account_id,
+    load(
+        neo4j_session,
+        ElasticacheClusterSchema(),
+        clusters,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
     )
 
 
 @timeit
-def cleanup(
+def load_elasticache_topics(
     neo4j_session: neo4j.Session,
-    current_aws_account_id: str,
+    topics: list[dict[str, Any]],
+    aws_account_id: str,
     update_tag: int,
 ) -> None:
-    run_cleanup_job(
-        "aws_import_elasticache_cleanup.json",
+    if not topics:
+        return
+    logger.info(f"Loading {len(topics)} ElastiCache topics into graph.")
+    load(
         neo4j_session,
-        {"UPDATE_TAG": update_tag, "AWS_ID": current_aws_account_id},
+        ElasticacheTopicSchema(),
+        topics,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(ElasticacheClusterSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(ElasticacheTopicSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -120,24 +134,33 @@ def cleanup(
 def sync(
     neo4j_session: neo4j.Session,
     boto3_session: boto3.session.Session,
-    regions: List[str],
+    regions: list[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
-            f"Syncing ElastiCache clusters for region '{region}' in account {current_aws_account_id}",
+            "Syncing ElastiCache clusters for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
         )
-        clusters = get_elasticache_clusters(boto3_session, region)
+        raw_clusters = get_elasticache_clusters(boto3_session, region)
+        cluster_data, topic_data = transform_elasticache_clusters(raw_clusters, region)
         load_elasticache_clusters(
             neo4j_session,
-            clusters,
+            cluster_data,
             region,
             current_aws_account_id,
             update_tag,
         )
-    cleanup(neo4j_session, current_aws_account_id, update_tag)
+        load_elasticache_topics(
+            neo4j_session,
+            topic_data,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/eventbridge.py

@@ -0,0 +1,91 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_eventbridge_rules(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_rules")
+    rules = []
+
+    for page in paginator.paginate():
+        rules.extend(page.get("Rules", []))
+
+    return rules
+
+
+@timeit
+def load_eventbridge_rules(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} rules for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeRuleSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running EventBridge cleanup job.")
+    GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing EventBridge for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        rules = get_eventbridge_rules(boto3_session, region)
+        load_eventbridge_rules(
+            neo4j_session,
+            rules,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/glue.py

@@ -0,0 +1,117 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_glue_connections(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_connections")
+    connections = []
+    for page in paginator.paginate():
+        connections.extend(page.get("ConnectionList", []))
+
+    return connections
+
+
+def transform_glue_connections(
+    connections: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Glue connection data for ingestion
+    """
+    transformed_connections = []
+    for connection in connections:
+        transformed_connection = {
+            "Name": connection["Name"],
+            "Description": connection.get("Description"),
+            "ConnectionType": connection.get("ConnectionType"),
+            "Status": connection.get("Status"),
+            "StatusReason": connection.get("StatusReason"),
+            "AuthenticationType": connection.get("AuthenticationConfiguration", {}).get(
+                "AuthenticationType"
+            ),
+            "SecretArn": connection.get("AuthenticationConfiguration", {}).get(
+                "SecretArn"
+            ),
+            "Region": region,
+        }
+        transformed_connections.append(transformed_connection)
+    return transformed_connections
+
+
+@timeit
+def load_glue_connections(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} connections for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueConnectionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Glue cleanup job.")
+    GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Glue for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        connections = get_glue_connections(boto3_session, region)
+        transformed_connections = transform_glue_connections(connections, region)
+        load_glue_connections(
+            neo4j_session,
+            transformed_connections,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/identitycenter.py

@@ -7,6 +7,7 @@ import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.identitycenter.awsidentitycenter import (
     AWSIdentityCenterInstanceSchema,
@@ -14,9 +15,11 @@ from cartography.models.aws.identitycenter.awsidentitycenter import (
 from cartography.models.aws.identitycenter.awspermissionset import (
     AWSPermissionSetSchema,
 )
+from cartography.models.aws.identitycenter.awspermissionset import (
+    RoleAssignmentAllowedByMatchLink,
+)
 from cartography.models.aws.identitycenter.awsssouser import AWSSSOUserSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -120,6 +123,8 @@ def load_permission_sets(
         InstanceArn=instance_arn,
         Region=region,
         AWS_ID=aws_account_id,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=aws_account_id,
     )
 
 
@@ -220,31 +225,64 @@ def get_role_assignments(
     return role_assignments
 
 
+@timeit
+def get_permset_roles(
+    neo4j_session: neo4j.Session,
+    role_assignments: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """
+    Enrich role assignments with exact role ARNs by querying existing permission set relationships.
+    Uses the ASSIGNED_TO_ROLE relationships created when permission sets were loaded.
+    """
+    # Get unique permission set ARNs from role assignments
+    permset_ids = list({ra["PermissionSetArn"] for ra in role_assignments})
+
+    query = """
+    MATCH (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-(permset:AWSPermissionSet)
+    WHERE permset.arn IN $PermSetIds
+    RETURN permset.arn AS PermissionSetArn, role.arn AS RoleArn
+    """
+    result = neo4j_session.run(query, PermSetIds=permset_ids)
+    permset_to_role = [record.data() for record in result]
+
+    # Create mapping from permission set ARN to role ARN
+    permset_to_role_map = {
+        entry["PermissionSetArn"]: entry["RoleArn"] for entry in permset_to_role
+    }
+
+    # Enrich role assignments with exact role ARNs
+    enriched_assignments = []
+    for assignment in role_assignments:
+        role_arn = permset_to_role_map.get(assignment["PermissionSetArn"])
+        enriched_assignments.append(
+            {
+                **assignment,
+                "RoleArn": role_arn,
+            }
+        )
+
+    return enriched_assignments
+
+
 @timeit
 def load_role_assignments(
     neo4j_session: neo4j.Session,
     role_assignments: List[Dict],
+    aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     """
-    Load role assignments into the graph
+    Load role assignments into the graph using MatchLink schema
     """
     logger.info(f"Loading {len(role_assignments)} role assignments")
-    if role_assignments:
-        neo4j_session.run(
-            """
-            UNWIND $role_assignments AS ra
-            MATCH (acc:AWSAccount{id:ra.AccountId}) -[:RESOURCE]->
-            (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-
-            (permset:AWSPermissionSet {id: ra.PermissionSetArn})
-            MATCH (sso:AWSSSOUser {id: ra.UserId})
-            MERGE (role)-[r:ALLOWED_BY]->(sso)
-            SET r.lastupdated = $aws_update_tag,
-            r.permission_set_arn = ra.PermissionSetArn
-            """,
-            role_assignments=role_assignments,
-            aws_update_tag=aws_update_tag,
-        )
+    load_matchlinks(
+        neo4j_session,
+        RoleAssignmentAllowedByMatchLink(),
+        role_assignments,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=aws_account_id,
+    )
 
 
 @timeit
@@ -262,11 +300,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSSSOUserSchema(), common_job_parameters).run(
         neo4j_session,
     )
-    run_cleanup_job(
-        "aws_import_identity_center_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
-    )
+
+    # Clean up role assignment MatchLinks
+    GraphJob.from_matchlink(
+        RoleAssignmentAllowedByMatchLink(),
+        "AWSAccount",
+        common_job_parameters["AWS_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(neo4j_session)
 
 
 @timeit
@@ -327,9 +368,16 @@ def sync_identity_center_instances(
                 instance_arn,
                 region,
             )
-            load_role_assignments(
+
+            # Enrich role assignments with exact role ARNs using permission set relationships
+            enriched_role_assignments = get_permset_roles(
                 neo4j_session,
                 role_assignments,
+            )
+            load_role_assignments(
+                neo4j_session,
+                enriched_role_assignments,
+                current_aws_account_id,
                 update_tag,
             )