cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/efs.py

@@ -0,0 +1,261 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.access_point import EfsAccessPointSchema
+from cartography.models.aws.efs.file_system import EfsFileSystemSchema
+from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_efs_file_systems(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_file_systems")
+    fileSystems = []
+    for page in paginator.paginate():
+        fileSystems.extend(page.get("FileSystems", []))
+
+    return fileSystems
+
+
+def transform_efs_file_systems(
+    fileSystems: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform SNS topic data for ingestion
+    """
+    transformed_file_systems = []
+    for file_system in fileSystems:
+        transformed_file_system = {
+            "FileSystemId": file_system["FileSystemId"],
+            "FileSystemArn": file_system["FileSystemArn"],
+            "Region": region,
+            "OwnerId": file_system.get("OwnerId"),
+            "CreationToken": file_system.get("CreationToken"),
+            "CreationTime": file_system.get("CreationTime"),
+            "LifeCycleState": file_system.get("LifeCycleState"),
+            "Name": file_system.get("Name"),
+            "NumberOfMountTargets": file_system.get("NumberOfMountTargets"),
+            "SizeInBytesValue": file_system.get("SizeInBytes", {}).get("Value"),
+            "SizeInBytesTimestamp": file_system.get("SizeInBytes", {}).get("Timestamp"),
+            "PerformanceMode": file_system.get("PerformanceMode"),
+            "Encrypted": file_system.get("Encrypted"),
+            "KmsKeyId": file_system.get("KmsKeyId"),
+            "ThroughputMode": file_system.get("ThroughputMode"),
+            "AvailabilityZoneName": file_system.get("AvailabilityZoneName"),
+            "AvailabilityZoneId": file_system.get("AvailabilityZoneId"),
+            "FileSystemProtection": file_system.get("FileSystemProtection", {}).get(
+                "ReplicationOverwriteProtection"
+            ),
+        }
+        transformed_file_systems.append(transformed_file_system)
+
+    return transformed_file_systems
+
+
+@timeit
+@aws_handle_regions
+def get_efs_mount_targets(
+    fileSystems: List[Dict[str, Any]], boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+    file_system_ids = []
+    for file_system in fileSystems:
+        file_system_ids.append(file_system["FileSystemId"])
+    paginator = client.get_paginator("describe_mount_targets")
+    mountTargets = []
+    for fs_id in file_system_ids:
+        for page in paginator.paginate(FileSystemId=fs_id):
+            mountTargets.extend(page.get("MountTargets", []))
+
+    return mountTargets
+
+
+@timeit
+@aws_handle_regions
+def get_efs_access_points(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+
+    paginator = client.get_paginator("describe_access_points")
+    accessPoints = []
+    for page in paginator.paginate():
+        accessPoints.extend(page.get("AccessPoints", []))
+
+    return accessPoints
+
+
+def transform_efs_access_points(
+    accessPoints: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Efs Access Points data for ingestion
+    """
+    transformed = []
+    for ap in accessPoints:
+        transformed.append(
+            {
+                "AccessPointArn": ap["AccessPointArn"],
+                "AccessPointId": ap["AccessPointId"],
+                "Region": region,
+                "FileSystemId": ap["FileSystemId"],
+                "Name": ap.get("Name"),
+                "LifeCycleState": ap.get("LifeCycleState"),
+                "OwnerId": ap.get("OwnerId"),
+                "Uid": ap.get("PosixUser", {}).get("Uid"),
+                "Gid": ap.get("PosixUser", {}).get("Gid"),
+                "RootDirectoryPath": ap.get("RootDirectory", {}).get("Path"),
+            }
+        )
+
+    return transformed
+
+
+@timeit
+def load_efs_mount_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} mount targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsMountTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_efs_file_systems(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} file systems for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsFileSystemSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_efs_access_points(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} access points for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsAccessPointSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(EfsMountTargetSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(EfsFileSystemSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(EfsAccessPointSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Efs for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        fileSystems = get_efs_file_systems(boto3_session, region)
+        tranformed_file_systems = transform_efs_file_systems(fileSystems, region)
+
+        load_efs_file_systems(
+            neo4j_session,
+            tranformed_file_systems,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        mountTargets = get_efs_mount_targets(fileSystems, boto3_session, region)
+
+        load_efs_mount_targets(
+            neo4j_session,
+            mountTargets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        accessPoints = get_efs_access_points(boto3_session, region)
+        accessPoints_transformed = transform_efs_access_points(accessPoints, region)
+
+        load_efs_access_points(
+            neo4j_session,
+            accessPoints_transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/eks.py

@@ -18,28 +18,32 @@ logger = logging.getLogger(__name__)
 @timeit
 @aws_handle_regions
 def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[str]:
-    client = boto3_session.client('eks', region_name=region)
+    client = boto3_session.client("eks", region_name=region)
     clusters: List[str] = []
-    paginator = client.get_paginator('list_clusters')
+    paginator = client.get_paginator("list_clusters")
     for page in paginator.paginate():
-        clusters.extend(page['clusters'])
+        clusters.extend(page["clusters"])
     return clusters
 
 
 @timeit
-def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str, cluster_name: str) -> Dict:
-    client = boto3_session.client('eks', region_name=region)
+def get_eks_describe_cluster(
+    boto3_session: boto3.session.Session,
+    region: str,
+    cluster_name: str,
+) -> Dict:
+    client = boto3_session.client("eks", region_name=region)
     response = client.describe_cluster(name=cluster_name)
-    return response['cluster']
+    return response["cluster"]
 
 
 @timeit
 def load_eks_clusters(
-        neo4j_session: neo4j.Session,
-        cluster_data: List[Dict[str, Any]],
-        region: str,
-        current_aws_account_id: str,
-        aws_update_tag: int,
+    neo4j_session: neo4j.Session,
+    cluster_data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
 ) -> None:
     load(
         neo4j_session,
@@ -57,50 +61,72 @@ def _process_logging(cluster: Dict) -> bool:
     at least one entry has audit logging set to Enabled.
     """
     logging: bool = False
-    cluster_logging: Any = cluster.get('logging', {}).get('clusterLogging')
+    cluster_logging: Any = cluster.get("logging", {}).get("clusterLogging")
     if cluster_logging:
-        logging = any(filter(lambda x: 'audit' in x['types'] and x['enabled'], cluster_logging))  # type: ignore
+        logging = any(filter(lambda x: "audit" in x["types"] and x["enabled"], cluster_logging))  # type: ignore
     return logging
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
     logger.info("Running EKS cluster cleanup")
-    GraphJob.from_node_schema(EKSClusterSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EKSClusterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 def transform(cluster_data: Dict[str, Any]) -> List[Dict[str, Any]]:
     transformed_list = []
     for cluster_name, cluster_dict in cluster_data.items():
         transformed_dict = cluster_dict.copy()
-        transformed_dict['ClusterLogging'] = _process_logging(transformed_dict)
-        transformed_dict['ClusterEndpointPublic'] = transformed_dict.get('resourcesVpcConfig', {}).get(
-            'endpointPublicAccess',
+        transformed_dict["ClusterLogging"] = _process_logging(transformed_dict)
+        transformed_dict["ClusterEndpointPublic"] = transformed_dict.get(
+            "resourcesVpcConfig",
+            {},
+        ).get(
+            "endpointPublicAccess",
         )
-        if 'createdAt' in transformed_dict:
-            transformed_dict['created_at'] = str(transformed_dict['createdAt'])
+        if "createdAt" in transformed_dict:
+            transformed_dict["created_at"] = str(transformed_dict["createdAt"])
         transformed_list.append(transformed_dict)
     return transformed_list
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        boto3_session: boto3.session.Session,
-        regions: List[str],
-        current_aws_account_id: str,
-        update_tag: int,
-        common_job_parameters: Dict[str, Any],
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
-        logger.info("Syncing EKS for region '%s' in account '%s'.", region, current_aws_account_id)
+        logger.info(
+            "Syncing EKS for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
 
         clusters: List[str] = get_eks_clusters(boto3_session, region)
         cluster_data = {}
         for cluster_name in clusters:
-            cluster_data[cluster_name] = get_eks_describe_cluster(boto3_session, region, cluster_name)
+            cluster_data[cluster_name] = get_eks_describe_cluster(
+                boto3_session,
+                region,
+                cluster_name,
+            )
         transformed_list = transform(cluster_data)
 
-        load_eks_clusters(neo4j_session, transformed_list, region, current_aws_account_id, update_tag)
+        load_eks_clusters(
+            neo4j_session,
+            transformed_list,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/elasticache.py

@@ -1,124 +1,171 @@
 import logging
-from typing import Dict
-from typing import List
-from typing import Set
+from typing import Any
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.elasticache.cluster import ElasticacheClusterSchema
+from cartography.models.aws.elasticache.topic import ElasticacheTopicSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 stat_handler = get_stats_client(__name__)
 
 
-def _get_topic(cluster: Dict) -> Dict:
-    return cluster['NotificationConfiguration']
-
-
-def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
-    """
-    Collect unique TopicArns from the cluster data
-    """
-    seen: Set[str] = set()
-    topics: List[Dict] = []
-    for cluster in cluster_data:
-        topic = _get_topic(cluster)
-        topic_arn = topic['TopicArn']
-        if topic_arn not in seen:
-            seen.add(topic_arn)
-            topics.append(topic)
-    return topics
-
-
 @timeit
 @aws_handle_regions
-def get_elasticache_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
-    logger.debug(f"Getting ElastiCache Clusters in region '{region}'.")
-    client = boto3_session.client('elasticache', region_name=region)
-    paginator = client.get_paginator('describe_cache_clusters')
-    clusters: List[Dict] = []
+def get_elasticache_clusters(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict[str, Any]]:
+    client = boto3_session.client("elasticache", region_name=region)
+    paginator = client.get_paginator("describe_cache_clusters")
+    clusters: list[dict[str, Any]] = []
     for page in paginator.paginate():
-        clusters.extend(page['CacheClusters'])
+        clusters.extend(page.get("CacheClusters", []))
     return clusters
 
 
+def transform_elasticache_clusters(
+    clusters: list[dict[str, Any]], region: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cluster_data: list[dict[str, Any]] = []
+    topics: dict[str, dict[str, Any]] = {}
+
+    for cluster in clusters:
+        notification = cluster.get("NotificationConfiguration", {})
+        topic_arn = notification.get("TopicArn")
+        cluster_record = {
+            "ARN": cluster["ARN"],
+            "CacheClusterId": cluster["CacheClusterId"],
+            "CacheNodeType": cluster.get("CacheNodeType"),
+            "Engine": cluster.get("Engine"),
+            "EngineVersion": cluster.get("EngineVersion"),
+            "CacheClusterStatus": cluster.get("CacheClusterStatus"),
+            "NumCacheNodes": cluster.get("NumCacheNodes"),
+            "PreferredAvailabilityZone": cluster.get("PreferredAvailabilityZone"),
+            "PreferredMaintenanceWindow": cluster.get("PreferredMaintenanceWindow"),
+            "CacheClusterCreateTime": cluster.get("CacheClusterCreateTime"),
+            "CacheSubnetGroupName": cluster.get("CacheSubnetGroupName"),
+            "AutoMinorVersionUpgrade": cluster.get("AutoMinorVersionUpgrade"),
+            "ReplicationGroupId": cluster.get("ReplicationGroupId"),
+            "SnapshotRetentionLimit": cluster.get("SnapshotRetentionLimit"),
+            "SnapshotWindow": cluster.get("SnapshotWindow"),
+            "AuthTokenEnabled": cluster.get("AuthTokenEnabled"),
+            "TransitEncryptionEnabled": cluster.get("TransitEncryptionEnabled"),
+            "AtRestEncryptionEnabled": cluster.get("AtRestEncryptionEnabled"),
+            "TopicArn": topic_arn,
+            "Region": region,
+        }
+        cluster_data.append(cluster_record)
+
+        if topic_arn:
+            topics.setdefault(
+                topic_arn,
+                {
+                    "TopicArn": topic_arn,
+                    "TopicStatus": notification.get("TopicStatus"),
+                    "cluster_arns": [],
+                },
+            )["cluster_arns"].append(cluster["ARN"])
+
+    return cluster_data, list(topics.values())
+
+
 @timeit
 def load_elasticache_clusters(
-    neo4j_session: neo4j.Session, clusters: List[Dict], region: str,
-    aws_account_id: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    clusters: list[dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
 ) -> None:
-    query = """
-    UNWIND $clusters as elasticache_cluster
-        MERGE (cluster:ElasticacheCluster{id:elasticache_cluster.ARN})
-        ON CREATE SET cluster.firstseen = timestamp(),
-            cluster.arn = elasticache_cluster.ARN,
-            cluster.topic_arn = elasticache_cluster.NotificationConfiguration.TopicArn,
-            cluster.id = elasticache_cluster.CacheClusterId,
-            cluster.region = $region
-        SET cluster.lastupdated = $aws_update_tag
-
-        WITH cluster, elasticache_cluster
-        MATCH (owner:AWSAccount{id: $aws_account_id})
-        MERGE (owner)-[r3:RESOURCE]->(cluster)
-        ON CREATE SET r3.firstseen = timestamp()
-        SET r3.lastupdated = $aws_update_tag
-
-        WITH elasticache_cluster, owner
-        WHERE NOT elasticache_cluster.NotificationConfiguration IS NULL
-        MERGE (topic:ElasticacheTopic{id: elasticache_cluster.NotificationConfiguration.TopicArn})
-        ON CREATE SET topic.firstseen = timestamp(),
-            topic.arn = elasticache_cluster.NotificationConfiguration.TopicArn
-        SET topic.lastupdated = $aws_update_tag,
-            topic.status = elasticache_cluster.NotificationConfiguration.Status
-
-        MERGE (topic)-[r:CACHE_CLUSTER]->(cluster)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-        WITH cluster, topic
-
-        MERGE (owner)-[r2:RESOURCE]->(topic)
-        ON CREATE SET r2.firstseen = timestamp()
-        SET r2.lastupdated = $aws_update_tag
-    """
-    logger.info(f"Loading f{len(clusters)} ElastiCache clusters for region '{region}' into graph.")
-    neo4j_session.run(
-        query,
-        clusters=clusters,
-        region=region,
-        aws_update_tag=update_tag,
-        aws_account_id=aws_account_id,
+    logger.info(
+        f"Loading {len(clusters)} ElastiCache clusters for region '{region}' into graph."
+    )
+    load(
+        neo4j_session,
+        ElasticacheClusterSchema(),
+        clusters,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
     )
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, current_aws_account_id: str, update_tag: int) -> None:
-    run_cleanup_job(
-        'aws_import_elasticache_cleanup.json',
+def load_elasticache_topics(
+    neo4j_session: neo4j.Session,
+    topics: list[dict[str, Any]],
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    if not topics:
+        return
+    logger.info(f"Loading {len(topics)} ElastiCache topics into graph.")
+    load(
         neo4j_session,
-        {'UPDATE_TAG': update_tag, 'AWS_ID': current_aws_account_id},
+        ElasticacheTopicSchema(),
+        topics,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(ElasticacheClusterSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(ElasticacheTopicSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
-    update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
 ) -> None:
     for region in regions:
-        logger.info(f"Syncing ElastiCache clusters for region '{region}' in account {current_aws_account_id}")
-        clusters = get_elasticache_clusters(boto3_session, region)
-        load_elasticache_clusters(neo4j_session, clusters, region, current_aws_account_id, update_tag)
-    cleanup(neo4j_session, current_aws_account_id, update_tag)
+        logger.info(
+            "Syncing ElastiCache clusters for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
+        raw_clusters = get_elasticache_clusters(boto3_session, region)
+        cluster_data, topic_data = transform_elasticache_clusters(raw_clusters, region)
+        load_elasticache_clusters(
+            neo4j_session,
+            cluster_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        load_elasticache_topics(
+            neo4j_session,
+            topic_data,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='AWSAccount',
+        group_type="AWSAccount",
         group_id=current_aws_account_id,
-        synced_type='ElasticacheCluster',
+        synced_type="ElasticacheCluster",
         update_tag=update_tag,
         stat_handler=stat_handler,
     )