cartography 0.107.0rc2__py3-none-any.whl → 0.108.0rc1__py3-none-any.whl

cartography/intel/aws/ec2/snapshots.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 from typing import Dict
 from typing import List
 
@@ -6,8 +7,11 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import load
+from cartography.client.core.tx import read_list_of_values_tx
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.snapshots import EBSSnapshotSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -24,12 +28,13 @@ def get_snapshots_in_use(
     WHERE v.region = $Region
     RETURN v.snapshotid as snapshot
     """
-    results = neo4j_session.run(
+    results = read_list_of_values_tx(
+        neo4j_session,
         query,
         AWS_ACCOUNT_ID=current_aws_account_id,
         Region=region,
     )
-    return [r["snapshot"] for r in results if r["snapshot"]]
+    return [str(snapshot) for snapshot in results if snapshot]
 
 
 @timeit
@@ -45,7 +50,6 @@ def get_snapshots(
     for page in paginator.paginate(OwnerIds=["self"]):
         snapshots.extend(page["Snapshots"])
 
-    # fetch in-use snapshots not in self_owned snapshots
     self_owned_snapshot_ids = {s["SnapshotId"] for s in snapshots}
     other_snapshot_ids = set(in_use_snapshot_ids) - self_owned_snapshot_ids
     if other_snapshot_ids:
@@ -55,8 +59,7 @@ def get_snapshots(
         except ClientError as e:
             if e.response["Error"]["Code"] == "InvalidSnapshot.NotFound":
                 logger.warning(
-                    f"Failed to retrieve page of in-use, \
-                    not owned snapshots. Continuing anyway. Error - {e}",
+                    f"Failed to retrieve page of in-use, not owned snapshots. Continuing anyway. Error - {e}"
                 )
             else:
                 raise
@@ -64,93 +67,53 @@ def get_snapshots(
     return snapshots
 
 
+def transform_snapshots(snapshots: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    transformed: List[Dict[str, Any]] = []
+    for snap in snapshots:
+        transformed.append(
+            {
+                "SnapshotId": snap["SnapshotId"],
+                "Description": snap.get("Description"),
+                "Encrypted": snap.get("Encrypted"),
+                "Progress": snap.get("Progress"),
+                "StartTime": snap.get("StartTime"),
+                "State": snap.get("State"),
+                "StateMessage": snap.get("StateMessage"),
+                "VolumeId": snap.get("VolumeId"),
+                "VolumeSize": snap.get("VolumeSize"),
+                "OutpostArn": snap.get("OutpostArn"),
+                "DataEncryptionKeyId": snap.get("DataEncryptionKeyId"),
+                "KmsKeyId": snap.get("KmsKeyId"),
+            }
+        )
+    return transformed
+
+
 @timeit
 def load_snapshots(
     neo4j_session: neo4j.Session,
-    data: List[Dict],
+    data: List[Dict[str, Any]],
     region: str,
     current_aws_account_id: str,
     update_tag: int,
 ) -> None:
-    ingest_snapshots = """
-    UNWIND $snapshots_list as snapshot
-        MERGE (s:EBSSnapshot{id: snapshot.SnapshotId})
-        ON CREATE SET s.firstseen = timestamp()
-        SET s.lastupdated = $update_tag, s.description = snapshot.Description, s.encrypted = snapshot.Encrypted,
-        s.progress = snapshot.Progress, s.starttime = snapshot.StartTime, s.state = snapshot.State,
-        s.statemessage = snapshot.StateMessage, s.volumeid = snapshot.VolumeId, s.volumesize = snapshot.VolumeSize,
-        s.outpostarn = snapshot.OutpostArn, s.dataencryptionkeyid = snapshot.DataEncryptionKeyId,
-        s.kmskeyid = snapshot.KmsKeyId, s.region=$Region
-        WITH s
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(s)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    for snapshot in data:
-        snapshot["StartTime"] = str(snapshot["StartTime"])
-
-    neo4j_session.run(
-        ingest_snapshots,
-        snapshots_list=data,
-        AWS_ACCOUNT_ID=current_aws_account_id,
+    load(
+        neo4j_session,
+        EBSSnapshotSchema(),
+        data,
+        lastupdated=update_tag,
         Region=region,
-        update_tag=update_tag,
-    )
-
-
-@timeit
-def get_snapshot_volumes(snapshots: List[Dict]) -> List[Dict]:
-    snapshot_volumes: List[Dict] = []
-    for snapshot in snapshots:
-        if snapshot.get("VolumeId"):
-            snapshot_volumes.append(snapshot)
-
-    return snapshot_volumes
-
-
-@timeit
-def load_snapshot_volume_relations(
-    neo4j_session: neo4j.Session,
-    data: List[Dict],
-    current_aws_account_id: str,
-    update_tag: int,
-) -> None:
-    ingest_volumes = """
-    UNWIND $snapshot_volumes_list as volume
-        MERGE (v:EBSVolume{id: volume.VolumeId})
-        ON CREATE SET v.firstseen = timestamp()
-        SET v.lastupdated = $update_tag, v.snapshotid = volume.SnapshotId
-        WITH v, volume
-        MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (aa)-[r:RESOURCE]->(v)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-        WITH v, volume
-        MATCH (s:EBSSnapshot{id: volume.SnapshotId})
-        MERGE (s)-[r:CREATED_FROM]->(v)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag
-    """
-
-    neo4j_session.run(
-        ingest_volumes,
-        snapshot_volumes_list=data,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        update_tag=update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
 
 @timeit
 def cleanup_snapshots(
     neo4j_session: neo4j.Session,
-    common_job_parameters: Dict,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
-    run_cleanup_job(
-        "aws_import_snapshots_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    GraphJob.from_node_schema(EBSSnapshotSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -161,7 +124,7 @@ def sync_ebs_snapshots(
     regions: List[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
         logger.debug(
@@ -174,12 +137,12 @@ def sync_ebs_snapshots(
             region,
             current_aws_account_id,
         )
-        data = get_snapshots(boto3_session, region, snapshots_in_use)
-        load_snapshots(neo4j_session, data, region, current_aws_account_id, update_tag)
-        snapshot_volumes = get_snapshot_volumes(data)
-        load_snapshot_volume_relations(
+        raw_data = get_snapshots(boto3_session, region, snapshots_in_use)
+        transformed_data = transform_snapshots(raw_data)
+        load_snapshots(
             neo4j_session,
-            snapshot_volumes,
+            transformed_data,
+            region,
             current_aws_account_id,
             update_tag,
         )

cartography/intel/aws/ec2/subnets.py

@@ -53,7 +53,7 @@ def load_subnets(
     snet.state = subnet.State, snet.assignipv6addressoncreation = subnet.AssignIpv6AddressOnCreation,
     snet.map_public_ip_on_launch = subnet.MapPublicIpOnLaunch, snet.subnet_arn = subnet.SubnetArn,
     snet.availability_zone = subnet.AvailabilityZone, snet.availability_zone_id = subnet.AvailabilityZoneId,
-    snet.subnetid = subnet.SubnetId
+    snet.subnet_id = subnet.SubnetId
     """
 
     ingest_subnet_vpc_relations = """

cartography/intel/aws/ecs.py

@@ -169,6 +169,22 @@ def _get_containers_from_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, An
     return containers
 
 
+def transform_ecs_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Extract network interface ID from task attachments.
+    """
+    for task in tasks:
+        for attachment in task.get("attachments", []):
+            if attachment.get("type") == "ElasticNetworkInterface":
+                details = attachment.get("details", [])
+                for detail in details:
+                    if detail.get("name") == "networkInterfaceId":
+                        task["networkInterfaceId"] = detail.get("value")
+                        break
+                break
+    return tasks
+
+
 @timeit
 def load_ecs_clusters(
     neo4j_session: neo4j.Session,
@@ -407,6 +423,7 @@ def _sync_ecs_task_and_container_defns(
         boto3_session,
         region,
     )
+    tasks = transform_ecs_tasks(tasks)
     containers = _get_containers_from_tasks(tasks)
     load_ecs_tasks(
         neo4j_session,

cartography/intel/aws/guardduty.py

@@ -0,0 +1,275 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import boto3.session
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.guardduty.findings import GuardDutyFindingSchema
+from cartography.stats import get_stats_client
+from cartography.util import aws_handle_regions
+from cartography.util import aws_paginate
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+def _get_severity_range_for_threshold(
+    severity_threshold: str | None,
+) -> List[str] | None:
+    """
+    Convert severity threshold string to GuardDuty numeric severity range.
+
+    GuardDuty severity mappings:
+    - LOW: 1.0-3.9
+    - MEDIUM: 4.0-6.9
+    - HIGH: 7.0-8.9
+    - CRITICAL: 9.0-10.0
+
+    :param severity_threshold: Severity threshold (LOW, MEDIUM, HIGH, CRITICAL)
+    :return: List of numeric severity ranges to include, or None for no filtering
+    """
+    if not severity_threshold:
+        return None
+
+    threshold_upper = severity_threshold.upper().strip()
+
+    # Map threshold to numeric ranges - include threshold level and above
+    if threshold_upper == "LOW":
+        return ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]  # All severities
+    elif threshold_upper == "MEDIUM":
+        return ["4", "5", "6", "7", "8", "9", "10"]  # MEDIUM and above
+    elif threshold_upper == "HIGH":
+        return ["7", "8", "9", "10"]  # HIGH and CRITICAL only
+    elif threshold_upper == "CRITICAL":
+        return ["9", "10"]  # CRITICAL only
+    else:
+        return None
+
+
+@aws_handle_regions
+def get_detectors(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[str]:
+    """
+    Get GuardDuty detector IDs for all detectors in a region.
+    """
+    client = boto3_session.client("guardduty", region_name=region)
+
+    # Get all detector IDs in this region
+    detectors_response = client.list_detectors()
+    detector_ids = detectors_response.get("DetectorIds", [])
+
+    if not detector_ids:
+        logger.info(f"No GuardDuty detectors found in region {region}")
+        return []
+
+    logger.info(f"Found {len(detector_ids)} GuardDuty detectors in region {region}")
+    return detector_ids
+
+
+@aws_handle_regions
+@timeit
+def get_findings(
+    boto3_session: boto3.session.Session,
+    region: str,
+    detector_id: str,
+    severity_threshold: str | None = None,
+) -> List[Dict[str, Any]]:
+    """
+    Get GuardDuty findings for a specific detector.
+    Only fetches unarchived findings to avoid including closed/resolved findings.
+    Optionally filters by severity threshold.
+    """
+    client = boto3_session.client("guardduty", region_name=region)
+
+    # Build FindingCriteria - always exclude archived findings
+    criteria = {"service.archived": {"Equals": ["false"]}}
+
+    # Add severity filtering if threshold is provided
+    severity_range = _get_severity_range_for_threshold(severity_threshold)
+    if severity_range:
+        min_severity = min(
+            float(s) for s in severity_range
+        )  # get min severity from range
+        # I chose to ignore the type error here  because the AWS API has fields that require different types
+        criteria["severity"] = {"GreaterThanOrEqual": int(min_severity)}  # type: ignore
+
+    # Get all finding IDs for this detector with filtering
+    finding_ids = list(
+        aws_paginate(
+            client,
+            "list_findings",
+            "FindingIds",
+            DetectorId=detector_id,
+            FindingCriteria={"Criterion": criteria},
+        )
+    )
+
+    if not finding_ids:
+        logger.info(f"No findings found for detector {detector_id} in region {region}")
+        return []
+
+    findings_data = []
+
+    # Process findings in batches (GuardDuty API limit is 50)
+    batch_size = 50
+    for i in range(0, len(finding_ids), batch_size):
+        batch_ids = finding_ids[i : i + batch_size]
+
+        findings_response = client.get_findings(
+            DetectorId=detector_id, FindingIds=batch_ids
+        )
+
+        findings_batch = findings_response.get("Findings", [])
+        findings_data.extend(findings_batch)
+
+    logger.info(
+        f"Retrieved {len(findings_data)} findings for detector {detector_id} in region {region}"
+    )
+    return findings_data
+
+
+def transform_findings(findings: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Transform GuardDuty findings from API response to schema format."""
+    transformed: List[Dict[str, Any]] = []
+    for f in findings:
+        item: Dict[str, Any] = {
+            "id": f["Id"],
+            "arn": f.get("Arn"),
+            "type": f.get("Type"),
+            "severity": f.get("Severity"),
+            "title": f.get("Title"),
+            "description": f.get("Description"),
+            "confidence": f.get("Confidence"),
+            "eventfirstseen": f.get("EventFirstSeen"),
+            "eventlastseen": f.get("EventLastSeen"),
+            "accountid": f.get("AccountId"),
+            "region": f.get("Region"),
+            "detectorid": f.get("DetectorId"),
+            "archived": f.get("Archived"),
+        }
+
+        # Handle nested resource information
+        resource = f.get("Resource", {})
+        item["resource_type"] = resource.get("ResourceType")
+
+        # Extract resource ID based on resource type
+        if item["resource_type"] == "Instance":
+            details = resource.get("InstanceDetails", {})
+            item["resource_id"] = details.get("InstanceId")
+        elif item["resource_type"] == "S3Bucket":
+            buckets = resource.get("S3BucketDetails") or []
+            if buckets:
+                item["resource_id"] = buckets[0].get("Name")
+        else:
+            item["resource_id"] = None
+
+        transformed.append(item)
+
+    return transformed
+
+
+@timeit
+def load_guardduty_findings(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load GuardDuty findings information into the graph.
+    """
+    logger.info(
+        f"Loading {len(data)} GuardDuty findings for region {region} into graph."
+    )
+
+    load(
+        neo4j_session,
+        GuardDutyFindingSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def cleanup_guardduty(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run GuardDuty cleanup job.
+    """
+    logger.debug("Running GuardDuty cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        GuardDutyFindingSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Sync GuardDuty findings for all regions.
+    Severity threshold filter is obtained from common_job_parameters.
+    """
+    # Get severity threshold from common job parameters
+    severity_threshold = common_job_parameters.get("aws_guardduty_severity_threshold")
+    for region in regions:
+        logger.info(
+            f"Syncing GuardDuty findings for {region} in account {current_aws_account_id}"
+        )
+
+        # Get all detectors in the region
+        detector_ids = get_detectors(boto3_session, region)
+
+        if not detector_ids:
+            logger.info(f"No GuardDuty detectors found in region {region}, skipping.")
+            continue
+
+        all_findings = []
+
+        # Get findings for each detector
+        for detector_id in detector_ids:
+            findings = get_findings(
+                boto3_session, region, detector_id, severity_threshold
+            )
+            all_findings.extend(findings)
+
+        transformed_findings = transform_findings(all_findings)
+
+        load_guardduty_findings(
+            neo4j_session,
+            transformed_findings,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    # Cleanup and metadata update (outside region loop)
+    cleanup_guardduty(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="GuardDutyFinding",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/aws/resources.py

@@ -18,6 +18,7 @@ from . import eks
 from . import elasticache
 from . import elasticsearch
 from . import emr
+from . import guardduty
 from . import iam
 from . import identitycenter
 from . import inspector
@@ -111,5 +112,6 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "cloudtrail_management_events": cloudtrail_management_events.sync,
     "cloudwatch": cloudwatch.sync,
     "efs": efs.sync,
+    "guardduty": guardduty.sync,
     "codebuild": codebuild.sync,
 }