cartography 0.107.0rc3__py3-none-any.whl → 0.108.0rc2__py3-none-any.whl

cartography/intel/aws/ec2/subnets.py

@@ -1,17 +1,17 @@
 import logging
-from typing import Dict
-from typing import List
+from typing import Any
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ec2.auto_scaling_groups import (
     EC2SubnetAutoScalingGroupSchema,
 )
 from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
+from cartography.models.aws.ec2.subnets import EC2SubnetSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 from .util import get_botocore_config
@@ -21,86 +21,53 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_subnet_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_subnet_data(
+    boto3_session: boto3.session.Session, region: str
+) -> list[dict[str, Any]]:
     client = boto3_session.client(
         "ec2",
         region_name=region,
         config=get_botocore_config(),
     )
     paginator = client.get_paginator("describe_subnets")
-    subnets: List[Dict] = []
+    subnets: list[dict[str, Any]] = []
     for page in paginator.paginate():
         subnets.extend(page["Subnets"])
     return subnets
 
 
+def transform_subnet_data(subnets: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Transform subnet data into a loadable format."""
+    transformed: list[dict[str, Any]] = []
+    for subnet in subnets:
+        transformed.append(subnet.copy())
+    return transformed
+
+
 @timeit
 def load_subnets(
     neo4j_session: neo4j.Session,
-    data: List[Dict],
+    data: list[dict[str, Any]],
     region: str,
     aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
-
-    ingest_subnets = """
-    UNWIND $subnets as subnet
-    MERGE (snet:EC2Subnet{subnetid: subnet.SubnetId})
-    ON CREATE SET snet.firstseen = timestamp()
-    SET snet.lastupdated = $aws_update_tag, snet.name = subnet.CidrBlock, snet.cidr_block = subnet.CidrBlock,
-    snet.available_ip_address_count = subnet.AvailableIpAddressCount, snet.default_for_az = subnet.DefaultForAz,
-    snet.map_customer_owned_ip_on_launch = subnet.MapCustomerOwnedIpOnLaunch,
-    snet.state = subnet.State, snet.assignipv6addressoncreation = subnet.AssignIpv6AddressOnCreation,
-    snet.map_public_ip_on_launch = subnet.MapPublicIpOnLaunch, snet.subnet_arn = subnet.SubnetArn,
-    snet.availability_zone = subnet.AvailabilityZone, snet.availability_zone_id = subnet.AvailabilityZoneId,
-    snet.subnet_id = subnet.SubnetId
-    """
-
-    ingest_subnet_vpc_relations = """
-    UNWIND $subnets as subnet
-    MATCH (snet:EC2Subnet{subnetid: subnet.SubnetId}), (vpc:AWSVpc{id: subnet.VpcId})
-    MERGE (snet)-[r:MEMBER_OF_AWS_VPC]->(vpc)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag
-    """
-
-    ingest_subnet_aws_account_relations = """
-    UNWIND $subnets as subnet
-    MATCH (snet:EC2Subnet{subnetid: subnet.SubnetId}), (aws:AWSAccount{id: $aws_account_id})
-    MERGE (aws)-[r:RESOURCE]->(snet)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $aws_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_subnets,
-        subnets=data,
-        aws_update_tag=aws_update_tag,
-        region=region,
-        aws_account_id=aws_account_id,
-    )
-    neo4j_session.run(
-        ingest_subnet_vpc_relations,
-        subnets=data,
-        aws_update_tag=aws_update_tag,
-        region=region,
-        aws_account_id=aws_account_id,
-    )
-    neo4j_session.run(
-        ingest_subnet_aws_account_relations,
-        subnets=data,
-        aws_update_tag=aws_update_tag,
-        region=region,
-        aws_account_id=aws_account_id,
+    load(
+        neo4j_session,
+        EC2SubnetSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=aws_update_tag,
     )
 
 
 @timeit
-def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        "aws_ingest_subnets_cleanup.json",
+def cleanup_subnets(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(
         neo4j_session,
-        common_job_parameters,
     )
     GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(
         neo4j_session,
@@ -115,10 +82,10 @@ def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -
 def sync_subnets(
     neo4j_session: neo4j.Session,
     boto3_session: boto3.session.Session,
-    regions: List[str],
+    regions: list[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
@@ -127,5 +94,12 @@ def sync_subnets(
             current_aws_account_id,
         )
         data = get_subnet_data(boto3_session, region)
-        load_subnets(neo4j_session, data, region, current_aws_account_id, update_tag)
+        transformed = transform_subnet_data(data)
+        load_subnets(
+            neo4j_session,
+            transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup_subnets(neo4j_session, common_job_parameters)

cartography/intel/aws/elasticache.py

@@ -1,118 +1,132 @@
 import logging
-from typing import Dict
-from typing import List
-from typing import Set
+from typing import Any
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.elasticache.cluster import ElasticacheClusterSchema
+from cartography.models.aws.elasticache.topic import ElasticacheTopicSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 stat_handler = get_stats_client(__name__)
 
 
-def _get_topic(cluster: Dict) -> Dict:
-    return cluster["NotificationConfiguration"]
-
-
-def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
-    """
-    Collect unique TopicArns from the cluster data
-    """
-    seen: Set[str] = set()
-    topics: List[Dict] = []
-    for cluster in cluster_data:
-        topic = _get_topic(cluster)
-        topic_arn = topic["TopicArn"]
-        if topic_arn not in seen:
-            seen.add(topic_arn)
-            topics.append(topic)
-    return topics
-
-
 @timeit
 @aws_handle_regions
 def get_elasticache_clusters(
     boto3_session: boto3.session.Session,
     region: str,
-) -> List[Dict]:
-    logger.debug(f"Getting ElastiCache Clusters in region '{region}'.")
+) -> list[dict[str, Any]]:
     client = boto3_session.client("elasticache", region_name=region)
     paginator = client.get_paginator("describe_cache_clusters")
-    clusters: List[Dict] = []
+    clusters: list[dict[str, Any]] = []
     for page in paginator.paginate():
-        clusters.extend(page["CacheClusters"])
+        clusters.extend(page.get("CacheClusters", []))
     return clusters
 
 
+def transform_elasticache_clusters(
+    clusters: list[dict[str, Any]], region: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    cluster_data: list[dict[str, Any]] = []
+    topics: dict[str, dict[str, Any]] = {}
+
+    for cluster in clusters:
+        notification = cluster.get("NotificationConfiguration", {})
+        topic_arn = notification.get("TopicArn")
+        cluster_record = {
+            "ARN": cluster["ARN"],
+            "CacheClusterId": cluster["CacheClusterId"],
+            "CacheNodeType": cluster.get("CacheNodeType"),
+            "Engine": cluster.get("Engine"),
+            "EngineVersion": cluster.get("EngineVersion"),
+            "CacheClusterStatus": cluster.get("CacheClusterStatus"),
+            "NumCacheNodes": cluster.get("NumCacheNodes"),
+            "PreferredAvailabilityZone": cluster.get("PreferredAvailabilityZone"),
+            "PreferredMaintenanceWindow": cluster.get("PreferredMaintenanceWindow"),
+            "CacheClusterCreateTime": cluster.get("CacheClusterCreateTime"),
+            "CacheSubnetGroupName": cluster.get("CacheSubnetGroupName"),
+            "AutoMinorVersionUpgrade": cluster.get("AutoMinorVersionUpgrade"),
+            "ReplicationGroupId": cluster.get("ReplicationGroupId"),
+            "SnapshotRetentionLimit": cluster.get("SnapshotRetentionLimit"),
+            "SnapshotWindow": cluster.get("SnapshotWindow"),
+            "AuthTokenEnabled": cluster.get("AuthTokenEnabled"),
+            "TransitEncryptionEnabled": cluster.get("TransitEncryptionEnabled"),
+            "AtRestEncryptionEnabled": cluster.get("AtRestEncryptionEnabled"),
+            "TopicArn": topic_arn,
+            "Region": region,
+        }
+        cluster_data.append(cluster_record)
+
+        if topic_arn:
+            topics.setdefault(
+                topic_arn,
+                {
+                    "TopicArn": topic_arn,
+                    "TopicStatus": notification.get("TopicStatus"),
+                    "cluster_arns": [],
+                },
+            )["cluster_arns"].append(cluster["ARN"])
+
+    return cluster_data, list(topics.values())
+
+
 @timeit
 def load_elasticache_clusters(
     neo4j_session: neo4j.Session,
-    clusters: List[Dict],
+    clusters: list[dict[str, Any]],
     region: str,
     aws_account_id: str,
     update_tag: int,
 ) -> None:
-    query = """
-    UNWIND $clusters as elasticache_cluster
-        MERGE (cluster:ElasticacheCluster{id:elasticache_cluster.ARN})
-        ON CREATE SET cluster.firstseen = timestamp(),
-            cluster.arn = elasticache_cluster.ARN,
-            cluster.topic_arn = elasticache_cluster.NotificationConfiguration.TopicArn,
-            cluster.id = elasticache_cluster.CacheClusterId,
-            cluster.region = $region
-        SET cluster.lastupdated = $aws_update_tag
-
-        WITH cluster, elasticache_cluster
-        MATCH (owner:AWSAccount{id: $aws_account_id})
-        MERGE (owner)-[r3:RESOURCE]->(cluster)
-        ON CREATE SET r3.firstseen = timestamp()
-        SET r3.lastupdated = $aws_update_tag
-
-        WITH elasticache_cluster, owner
-        WHERE NOT elasticache_cluster.NotificationConfiguration IS NULL
-        MERGE (topic:ElasticacheTopic{id: elasticache_cluster.NotificationConfiguration.TopicArn})
-        ON CREATE SET topic.firstseen = timestamp(),
-            topic.arn = elasticache_cluster.NotificationConfiguration.TopicArn
-        SET topic.lastupdated = $aws_update_tag,
-            topic.status = elasticache_cluster.NotificationConfiguration.Status
-
-        MERGE (topic)-[r:CACHE_CLUSTER]->(cluster)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-        WITH cluster, topic
-
-        MERGE (owner)-[r2:RESOURCE]->(topic)
-        ON CREATE SET r2.firstseen = timestamp()
-        SET r2.lastupdated = $aws_update_tag
-    """
     logger.info(
-        f"Loading f{len(clusters)} ElastiCache clusters for region '{region}' into graph.",
+        f"Loading {len(clusters)} ElastiCache clusters for region '{region}' into graph."
     )
-    neo4j_session.run(
-        query,
-        clusters=clusters,
-        region=region,
-        aws_update_tag=update_tag,
-        aws_account_id=aws_account_id,
+    load(
+        neo4j_session,
+        ElasticacheClusterSchema(),
+        clusters,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
     )
 
 
 @timeit
-def cleanup(
+def load_elasticache_topics(
     neo4j_session: neo4j.Session,
-    current_aws_account_id: str,
+    topics: list[dict[str, Any]],
+    aws_account_id: str,
     update_tag: int,
 ) -> None:
-    run_cleanup_job(
-        "aws_import_elasticache_cleanup.json",
+    if not topics:
+        return
+    logger.info(f"Loading {len(topics)} ElastiCache topics into graph.")
+    load(
         neo4j_session,
-        {"UPDATE_TAG": update_tag, "AWS_ID": current_aws_account_id},
+        ElasticacheTopicSchema(),
+        topics,
+        lastupdated=update_tag,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(ElasticacheClusterSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(ElasticacheTopicSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -120,24 +134,33 @@ def cleanup(
 def sync(
     neo4j_session: neo4j.Session,
     boto3_session: boto3.session.Session,
-    regions: List[str],
+    regions: list[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
-            f"Syncing ElastiCache clusters for region '{region}' in account {current_aws_account_id}",
+            "Syncing ElastiCache clusters for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
         )
-        clusters = get_elasticache_clusters(boto3_session, region)
+        raw_clusters = get_elasticache_clusters(boto3_session, region)
+        cluster_data, topic_data = transform_elasticache_clusters(raw_clusters, region)
         load_elasticache_clusters(
             neo4j_session,
-            clusters,
+            cluster_data,
             region,
             current_aws_account_id,
             update_tag,
         )
-    cleanup(neo4j_session, current_aws_account_id, update_tag)
+        load_elasticache_topics(
+            neo4j_session,
+            topic_data,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/guardduty.py

@@ -0,0 +1,275 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import boto3.session
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.guardduty.findings import GuardDutyFindingSchema
+from cartography.stats import get_stats_client
+from cartography.util import aws_handle_regions
+from cartography.util import aws_paginate
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+def _get_severity_range_for_threshold(
+    severity_threshold: str | None,
+) -> List[str] | None:
+    """
+    Convert severity threshold string to GuardDuty numeric severity range.
+
+    GuardDuty severity mappings:
+    - LOW: 1.0-3.9
+    - MEDIUM: 4.0-6.9
+    - HIGH: 7.0-8.9
+    - CRITICAL: 9.0-10.0
+
+    :param severity_threshold: Severity threshold (LOW, MEDIUM, HIGH, CRITICAL)
+    :return: List of numeric severity ranges to include, or None for no filtering
+    """
+    if not severity_threshold:
+        return None
+
+    threshold_upper = severity_threshold.upper().strip()
+
+    # Map threshold to numeric ranges - include threshold level and above
+    if threshold_upper == "LOW":
+        return ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]  # All severities
+    elif threshold_upper == "MEDIUM":
+        return ["4", "5", "6", "7", "8", "9", "10"]  # MEDIUM and above
+    elif threshold_upper == "HIGH":
+        return ["7", "8", "9", "10"]  # HIGH and CRITICAL only
+    elif threshold_upper == "CRITICAL":
+        return ["9", "10"]  # CRITICAL only
+    else:
+        return None
+
+
+@aws_handle_regions
+def get_detectors(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[str]:
+    """
+    Get GuardDuty detector IDs for all detectors in a region.
+    """
+    client = boto3_session.client("guardduty", region_name=region)
+
+    # Get all detector IDs in this region
+    detectors_response = client.list_detectors()
+    detector_ids = detectors_response.get("DetectorIds", [])
+
+    if not detector_ids:
+        logger.info(f"No GuardDuty detectors found in region {region}")
+        return []
+
+    logger.info(f"Found {len(detector_ids)} GuardDuty detectors in region {region}")
+    return detector_ids
+
+
+@aws_handle_regions
+@timeit
+def get_findings(
+    boto3_session: boto3.session.Session,
+    region: str,
+    detector_id: str,
+    severity_threshold: str | None = None,
+) -> List[Dict[str, Any]]:
+    """
+    Get GuardDuty findings for a specific detector.
+    Only fetches unarchived findings to avoid including closed/resolved findings.
+    Optionally filters by severity threshold.
+    """
+    client = boto3_session.client("guardduty", region_name=region)
+
+    # Build FindingCriteria - always exclude archived findings
+    criteria = {"service.archived": {"Equals": ["false"]}}
+
+    # Add severity filtering if threshold is provided
+    severity_range = _get_severity_range_for_threshold(severity_threshold)
+    if severity_range:
+        min_severity = min(
+            float(s) for s in severity_range
+        )  # get min severity from range
+        # I chose to ignore the type error here  because the AWS API has fields that require different types
+        criteria["severity"] = {"GreaterThanOrEqual": int(min_severity)}  # type: ignore
+
+    # Get all finding IDs for this detector with filtering
+    finding_ids = list(
+        aws_paginate(
+            client,
+            "list_findings",
+            "FindingIds",
+            DetectorId=detector_id,
+            FindingCriteria={"Criterion": criteria},
+        )
+    )
+
+    if not finding_ids:
+        logger.info(f"No findings found for detector {detector_id} in region {region}")
+        return []
+
+    findings_data = []
+
+    # Process findings in batches (GuardDuty API limit is 50)
+    batch_size = 50
+    for i in range(0, len(finding_ids), batch_size):
+        batch_ids = finding_ids[i : i + batch_size]
+
+        findings_response = client.get_findings(
+            DetectorId=detector_id, FindingIds=batch_ids
+        )
+
+        findings_batch = findings_response.get("Findings", [])
+        findings_data.extend(findings_batch)
+
+    logger.info(
+        f"Retrieved {len(findings_data)} findings for detector {detector_id} in region {region}"
+    )
+    return findings_data
+
+
+def transform_findings(findings: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Transform GuardDuty findings from API response to schema format."""
+    transformed: List[Dict[str, Any]] = []
+    for f in findings:
+        item: Dict[str, Any] = {
+            "id": f["Id"],
+            "arn": f.get("Arn"),
+            "type": f.get("Type"),
+            "severity": f.get("Severity"),
+            "title": f.get("Title"),
+            "description": f.get("Description"),
+            "confidence": f.get("Confidence"),
+            "eventfirstseen": f.get("EventFirstSeen"),
+            "eventlastseen": f.get("EventLastSeen"),
+            "accountid": f.get("AccountId"),
+            "region": f.get("Region"),
+            "detectorid": f.get("DetectorId"),
+            "archived": f.get("Archived"),
+        }
+
+        # Handle nested resource information
+        resource = f.get("Resource", {})
+        item["resource_type"] = resource.get("ResourceType")
+
+        # Extract resource ID based on resource type
+        if item["resource_type"] == "Instance":
+            details = resource.get("InstanceDetails", {})
+            item["resource_id"] = details.get("InstanceId")
+        elif item["resource_type"] == "S3Bucket":
+            buckets = resource.get("S3BucketDetails") or []
+            if buckets:
+                item["resource_id"] = buckets[0].get("Name")
+        else:
+            item["resource_id"] = None
+
+        transformed.append(item)
+
+    return transformed
+
+
+@timeit
+def load_guardduty_findings(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load GuardDuty findings information into the graph.
+    """
+    logger.info(
+        f"Loading {len(data)} GuardDuty findings for region {region} into graph."
+    )
+
+    load(
+        neo4j_session,
+        GuardDutyFindingSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup_guardduty(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run GuardDuty cleanup job.
+    """
+    logger.debug("Running GuardDuty cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        GuardDutyFindingSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Sync GuardDuty findings for all regions.
+    Severity threshold filter is obtained from common_job_parameters.
+    """
+    # Get severity threshold from common job parameters
+    severity_threshold = common_job_parameters.get("aws_guardduty_severity_threshold")
+    for region in regions:
+        logger.info(
+            f"Syncing GuardDuty findings for {region} in account {current_aws_account_id}"
+        )
+
+        # Get all detectors in the region
+        detector_ids = get_detectors(boto3_session, region)
+
+        if not detector_ids:
+            logger.info(f"No GuardDuty detectors found in region {region}, skipping.")
+            continue
+
+        all_findings = []
+
+        # Get findings for each detector
+        for detector_id in detector_ids:
+            findings = get_findings(
+                boto3_session, region, detector_id, severity_threshold
+            )
+            all_findings.extend(findings)
+
+        transformed_findings = transform_findings(all_findings)
+
+        load_guardduty_findings(
+            neo4j_session,
+            transformed_findings,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    # Cleanup and metadata update (outside region loop)
+    cleanup_guardduty(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="GuardDutyFinding",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/aws/resources.py

@@ -18,6 +18,7 @@ from . import eks
 from . import elasticache
 from . import elasticsearch
 from . import emr
+from . import guardduty
 from . import iam
 from . import identitycenter
 from . import inspector
@@ -111,5 +112,6 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "cloudtrail_management_events": cloudtrail_management_events.sync,
     "cloudwatch": cloudwatch.sync,
     "efs": efs.sync,
+    "guardduty": guardduty.sync,
     "codebuild": codebuild.sync,
 }