cartography 0.106.0rc2__py3-none-any.whl → 0.107.0rc2__py3-none-any.whl

cartography/intel/aws/cloudtrail_management_events.py

@@ -0,0 +1,364 @@
+import json
+import logging
+from datetime import datetime
+from datetime import timedelta
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudtrail.management_events import AssumedRoleMatchLink
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_assume_role_events(
+    boto3_session: boto3.Session, region: str, lookback_hours: int
+) -> List[Dict[str, Any]]:
+    """
+    Fetch CloudTrail AssumeRole events from the specified time period.
+
+    Focuses specifically on standard AssumeRole events, excluding SAML and WebIdentity variants.
+
+    :type boto3_session: boto3.Session
+    :param boto3_session: The boto3 session to use for API calls
+    :type region: str
+    :param region: The AWS region to fetch events from
+    :type lookback_hours: int
+    :param lookback_hours: Number of hours back to retrieve events from
+    :rtype: List[Dict[str, Any]]
+    :return: List of CloudTrail AssumeRole events
+    """
+    client = boto3_session.client(
+        "cloudtrail", region_name=region, config=get_botocore_config()
+    )
+
+    # Calculate time range
+    end_time = datetime.utcnow()
+    start_time = end_time - timedelta(hours=lookback_hours)
+
+    logger.info(
+        f"Fetching CloudTrail AssumeRole events for region '{region}' "
+        f"from {start_time} to {end_time} ({lookback_hours} hours)"
+    )
+
+    paginator = client.get_paginator("lookup_events")
+
+    page_iterator = paginator.paginate(
+        LookupAttributes=[
+            {"AttributeKey": "EventName", "AttributeValue": "AssumeRole"}
+        ],
+        StartTime=start_time,
+        EndTime=end_time,
+        PaginationConfig={
+            "MaxItems": 10000,  # Reasonable limit to prevent excessive API calls
+            "PageSize": 50,  # CloudTrail API limit per page
+        },
+    )
+
+    all_events = []
+    for page in page_iterator:
+        all_events.extend(page.get("Events", []))
+
+    logger.info(f"Retrieved {len(all_events)} AssumeRole events from region '{region}'")
+
+    return all_events
+
+
+@timeit
+def transform_assume_role_events_to_role_assumptions(
+    events: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform raw CloudTrail AssumeRole events into aggregated role assumption relationships.
+
+    Focuses specifically on standard AssumeRole events, providing optimized processing
+    for the most common role assumption scenario.
+
+    This function performs the complete transformation pipeline:
+    1. Extract role assumption events from CloudTrail AssumeRole data
+    2. Aggregate events by (source_principal, destination_principal) pairs
+    3. Return aggregated relationships ready for loading
+
+    :type events: List[Dict[str, Any]]
+    :param events: List of raw CloudTrail AssumeRole events from lookup_events API
+    :type region: str
+    :param region: The AWS region where events were retrieved from
+    :type current_aws_account_id: str
+    :param current_aws_account_id: The AWS account ID being synced
+    :rtype: List[Dict[str, Any]]
+    :return: List of aggregated role assumption relationships ready for loading
+    """
+    aggregated: Dict[tuple, Dict[str, Any]] = {}
+    logger.info(
+        f"Transforming {len(events)} CloudTrail AssumeRole events to role assumptions for region '{region}'"
+    )
+
+    for event in events:
+
+        cloudtrail_event = json.loads(event["CloudTrailEvent"])
+
+        if cloudtrail_event.get("userIdentity", {}).get("arn"):
+            source_principal = cloudtrail_event["userIdentity"]["arn"]
+            destination_principal = cloudtrail_event["requestParameters"]["roleArn"]
+        else:
+            logger.debug(
+                f"Skipping CloudTrail AssumeRole event due to missing UserIdentity.arn. Event: {event.get('EventId', 'unknown')}"
+            )
+            continue
+
+        destination_principal = cloudtrail_event["requestParameters"]["roleArn"]
+
+        normalized_source_principal = _convert_assumed_role_arn_to_role_arn(
+            source_principal
+        )
+        normalized_destination_principal = _convert_assumed_role_arn_to_role_arn(
+            destination_principal
+        )
+        event_time = event.get("EventTime")
+
+        key = (normalized_source_principal, normalized_destination_principal)
+
+        if key in aggregated:
+            aggregated[key]["times_used"] += 1
+            aggregated[key]["assume_role_count"] += 1  # All events are AssumeRole
+            # Handle None values safely for time comparisons
+            if event_time:
+                existing_first = aggregated[key]["first_seen_in_time_window"]
+                existing_last = aggregated[key]["last_used"]
+
+                if existing_first is None or event_time < existing_first:
+                    aggregated[key]["first_seen_in_time_window"] = event_time
+                if existing_last is None or event_time > existing_last:
+                    aggregated[key]["last_used"] = event_time
+        else:
+            aggregated[key] = {
+                "source_principal_arn": normalized_source_principal,
+                "destination_principal_arn": normalized_destination_principal,
+                "times_used": 1,
+                "first_seen_in_time_window": event_time,
+                "last_used": event_time,
+                "event_types": ["AssumeRole"],
+                "assume_role_count": 1,
+                "saml_count": 0,
+                "web_identity_count": 0,
+            }
+
+    return list(aggregated.values())
+
+
+@timeit
+def load_role_assumptions(
+    neo4j_session: neo4j.Session,
+    aggregated_role_assumptions: List[Dict[str, Any]],
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load aggregated role assumption relationships into Neo4j using MatchLink pattern.
+
+    Creates direct ASSUMED_ROLE relationships with aggregated properties:
+    (AWSUser|AWSRole|AWSPrincipal)-[:ASSUMED_ROLE {lastused, times_used, first_seen_in_time_window, last_seen}]->(AWSRole)
+
+    Assumes that both source principals and destination roles already exist in the graph.
+
+    :type neo4j_session: neo4j.Session
+    :param neo4j_session: The Neo4j session to use for database operations
+    :type aggregated_role_assumptions: List[Dict[str, Any]]
+    :param aggregated_role_assumptions: List of aggregated role assumption relationships from transform function
+    :type current_aws_account_id: str
+    :param current_aws_account_id: The AWS account ID being synced
+    :type aws_update_tag: int
+    :param aws_update_tag: Timestamp tag for tracking data freshness
+    :rtype: None
+    """
+    # Use MatchLink to create relationships between existing nodes
+    matchlink_schema = AssumedRoleMatchLink()
+
+    load_matchlinks(
+        neo4j_session,
+        matchlink_schema,
+        aggregated_role_assumptions,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=current_aws_account_id,
+    )
+
+    logger.info(
+        f"Successfully loaded {len(aggregated_role_assumptions)} role assumption relationships"
+    )
+
+
+def _convert_assumed_role_arn_to_role_arn(assumed_role_arn: str) -> str:
+    """
+    Convert an assumed role ARN to the original role ARN.
+
+    Example:
+    Input:  "arn:aws:sts::123456789012:assumed-role/MyRole/session-name"
+    Output: "arn:aws:iam::123456789012:role/MyRole"
+    """
+
+    # Split the ARN into parts
+    arn_parts = assumed_role_arn.split(":")
+    if len(arn_parts) >= 6 and arn_parts[2] == "sts" and "assumed-role" in arn_parts[5]:
+        # Extract account ID and role name
+        account_id = arn_parts[4]
+        resource_part = arn_parts[5]  # "assumed-role/MyRole/session-name"
+        role_name = resource_part.split("/")[1]  # Extract "MyRole"
+
+        # Construct the IAM role ARN
+        return f"arn:aws:iam::{account_id}:role/{role_name}"
+
+    # Return original ARN if conversion fails
+    return assumed_role_arn
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, current_aws_account_id: str, update_tag: int
+) -> None:
+    """
+    Run CloudTrail management events cleanup job to remove stale ASSUMED_ROLE relationships.
+
+    :type neo4j_session: neo4j.Session
+    :param neo4j_session: The Neo4j session to use for database operations
+    :type current_aws_account_id: str
+    :param current_aws_account_id: The AWS account ID being synced
+    :type update_tag: int
+    :param update_tag: Timestamp tag for tracking data freshness
+    :rtype: None
+    """
+    logger.info("Running CloudTrail management events cleanup job.")
+
+    matchlink_schema = AssumedRoleMatchLink()
+    cleanup_job = GraphJob.from_matchlink(
+        matchlink_schema,
+        "AWSAccount",
+        current_aws_account_id,
+        update_tag,
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync_assume_role_events(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
+    Sync CloudTrail management events to create ASSUMED_ROLE relationships.
+
+    This function orchestrates the complete process:
+    1. Fetch CloudTrail management events region by region
+    2. Transform events into role assumption records per region
+    3. Load role assumption relationships into Neo4j for each region
+    4. Run cleanup after processing all regions
+
+    The resulting graph contains direct relationships like:
+    (AWSUser|AWSRole|AWSPrincipal)-[:ASSUMED_ROLE {times_used, first_seen_in_time_window, last_used, lastused}]->(AWSRole)
+
+    :type neo4j_session: neo4j.Session
+    :param neo4j_session: The Neo4j session
+    :type boto3_session: boto3.Session
+    :param boto3_session: The boto3 session to use for API calls
+    :type regions: List[str]
+    :param regions: List of AWS regions to sync
+    :type current_aws_account_id: str
+    :param current_aws_account_id: The AWS account ID being synced
+    :type aws_update_tag: int
+    :param aws_update_tag: Timestamp tag for tracking data freshness
+    :rtype: None
+    """
+    # Extract lookback hours from common_job_parameters (set by CLI parameter)
+    lookback_hours = common_job_parameters.get(
+        "aws_cloudtrail_management_events_lookback_hours"
+    )
+
+    if not lookback_hours:
+        logger.info(
+            "CloudTrail management events sync skipped - no lookback period specified"
+        )
+        return
+
+    logger.info(
+        f"Syncing {len(regions)} regions with {lookback_hours} hour lookback period"
+    )
+
+    total_role_assumptions = 0
+
+    # Process events region by region
+    for region in regions:
+        logger.info(f"Processing CloudTrail events for region {region}")
+
+        # Process AssumeRole events specifically
+        logger.info(f"Fetching AssumeRole events specifically for region {region}")
+        assume_role_events = get_assume_role_events(
+            boto3_session=boto3_session,
+            region=region,
+            lookback_hours=lookback_hours,
+        )
+
+        # Transform AssumeRole events to role assumptions
+        assume_role_assumptions = transform_assume_role_events_to_role_assumptions(
+            events=assume_role_events,
+            region=region,
+            current_aws_account_id=current_aws_account_id,
+        )
+
+        # Load AssumeRole assumptions for this region
+        load_role_assumptions(
+            neo4j_session=neo4j_session,
+            aggregated_role_assumptions=assume_role_assumptions,
+            current_aws_account_id=current_aws_account_id,
+            aws_update_tag=update_tag,
+        )
+        total_role_assumptions += len(assume_role_assumptions)
+        logger.info(
+            f"Loaded {len(assume_role_assumptions)} AssumeRole assumptions for region {region}"
+        )
+
+    # Run cleanup for stale relationships after processing all regions
+    cleanup(neo4j_session, current_aws_account_id, update_tag)
+
+    logger.info(
+        f"CloudTrail management events sync completed successfully. "
+        f"Processed {total_role_assumptions} total role assumption events across {len(regions)} regions."
+    )
+
+
+# Main sync function for when we decide to add more event types
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
+    Main sync function for CloudTrail management events.
+    """
+    sync_assume_role_events(
+        neo4j_session=neo4j_session,
+        boto3_session=boto3_session,
+        regions=regions,
+        current_aws_account_id=current_aws_account_id,
+        update_tag=update_tag,
+        common_job_parameters=common_job_parameters,
+    )

cartography/intel/aws/codebuild.py

@@ -0,0 +1,132 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.codebuild.project import CodeBuildProjectSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_all_codebuild_projects(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+
+    client = boto3_session.client(
+        "codebuild", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_projects")
+
+    all_projects = []
+
+    for page in paginator.paginate():
+        project_names = page.get("projects", [])
+        if not project_names:
+            continue
+
+        # AWS batch_get_projects accepts up to 100 project names per call as per AWS documentation.
+        for i in range(0, len(project_names), 100):
+            batch = project_names[i : i + 100]
+            response = client.batch_get_projects(names=batch)
+            projects = response.get("projects", [])
+            all_projects.extend(projects)
+    return all_projects
+
+
+def transform_codebuild_projects(
+    projects: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CodeBuild project data for ingestion into Neo4j.
+
+    - Includes all environment variable names.
+    - Variables of type 'PLAINTEXT' retain their values.
+    - Other types (e.g., 'PARAMETER_STORE', 'SECRETS_MANAGER') have their values redacted.
+    """
+    transformed_codebuild_projects = []
+    for project in projects:
+        env_vars = project.get("environment", {}).get("environmentVariables", [])
+        env_var_strings = [
+            f"{var.get('name')}={var.get('value') if var.get('type') == 'PLAINTEXT' else '<REDACTED>'}"
+            for var in env_vars
+        ]
+        transformed_project = {
+            "arn": project["arn"],
+            "created": project.get("created"),
+            "environmentVariables": env_var_strings,
+            "sourceType": project.get("source", {}).get("type"),
+            "sourceLocation": project.get("source", {}).get("location"),
+        }
+        transformed_codebuild_projects.append(transformed_project)
+
+    return transformed_codebuild_projects
+
+
+@timeit
+def load_codebuild_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CodeBuild {len(data)} projects for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CodeBuildProjectSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CodeBuildProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CodeBuild for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        projects = get_all_codebuild_projects(boto3_session, region)
+        transformed_projects = transform_codebuild_projects(projects, region)
+
+        load_codebuild_projects(
+            neo4j_session,
+            transformed_projects,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/inspector.py

@@ -3,14 +3,17 @@ from typing import Any
 from typing import Dict
 from typing import Iterator
 from typing import List
+from typing import Set
 from typing import Tuple
 
 import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
+from cartography.models.aws.inspector.findings import InspectorFindingToPackageMatchLink
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
@@ -107,9 +110,10 @@ def get_inspector_findings(
 
 def transform_inspector_findings(
     results: List[Dict[str, Any]],
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, str]]]:
     findings_list: List[Dict] = []
-    packages: Dict[str, Any] = {}
+    packages_set: Set[frozenset] = set()
+    finding_to_package_map: List[Dict[str, str]] = []
 
     for f in results:
         finding: Dict = {}
@@ -163,55 +167,45 @@ def transform_inspector_findings(
                 "vendorUpdatedAt",
             )
 
-            new_packages = _process_packages(
-                f["packageVulnerabilityDetails"],
-                f["awsAccountId"],
-                f["findingArn"],
-            )
-            finding["vulnerablepackageids"] = list(new_packages.keys())
-            packages = {**packages, **new_packages}
-
+            packages = transform_inspector_packages(f["packageVulnerabilityDetails"])
+            finding["vulnerablepackageids"] = list(packages.keys())
+            for package_id, package in packages.items():
+                finding_to_package_map.append(
+                    {
+                        "findingarn": finding["id"],
+                        "packageid": package_id,
+                        "remediation": package.get("remediation"),
+                        "fixedInVersion": package.get("fixedInVersion"),
+                        "filePath": package.get("filePath"),
+                        "sourceLayerHash": package.get("sourceLayerHash"),
+                        "sourceLambdaLayerArn": package.get("sourceLambdaLayerArn"),
+                    }
+                )
+                packages_set.add(frozenset(package.items()))
         findings_list.append(finding)
-    packages_list = transform_inspector_packages(packages)
-    return findings_list, packages_list
-
+    packages_list = [dict(p) for p in packages_set]
+    return findings_list, packages_list, finding_to_package_map
 
-def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict[str, Any]]:
-    packages_list: List[Dict] = []
-    for package_id in packages.keys():
-        packages_list.append(packages[package_id])
 
-    return packages_list
-
-
-def _process_packages(
+def transform_inspector_packages(
     package_details: Dict[str, Any],
-    aws_account_id: str,
-    finding_arn: str,
 ) -> Dict[str, Any]:
     packages: Dict[str, Any] = {}
     for package in package_details["vulnerablePackages"]:
-        new_package = {}
-        new_package["id"] = (
-            f"{package.get('name', '')}|"
-            f"{package.get('arch', '')}|"
-            f"{package.get('version', '')}|"
-            f"{package.get('release', '')}|"
-            f"{package.get('epoch', '')}"
-        )
-        new_package["name"] = package.get("name")
-        new_package["arch"] = package.get("arch")
-        new_package["version"] = package.get("version")
-        new_package["release"] = package.get("release")
-        new_package["epoch"] = package.get("epoch")
-        new_package["manager"] = package.get("packageManager")
-        new_package["filepath"] = package.get("filePath")
-        new_package["fixedinversion"] = package.get("fixedInVersion")
-        new_package["sourcelayerhash"] = package.get("sourceLayerHash")
-        new_package["awsaccount"] = aws_account_id
-        new_package["findingarn"] = finding_arn
-
-        packages[new_package["id"]] = new_package
+        # Following RPM package naming convention for consistency
+        name = package["name"]  # Mandatory field
+        epoch = str(package.get("epoch", ""))
+        if epoch:
+            epoch = f"{epoch}:"
+        version = package["version"]  # Mandatory field
+        release = package.get("release", "")
+        if release:
+            release = f"-{release}"
+        arch = package.get("arch", "")
+        if arch:
+            arch = f".{arch}"
+        id = f"{name}|{epoch}{version}{release}{arch}"
+        packages[id] = {**package, "id": id}
 
     return packages
 
@@ -244,7 +238,6 @@ def load_inspector_findings(
 def load_inspector_packages(
     neo4j_session: neo4j.Session,
     packages: List[Dict[str, Any]],
-    region: str,
     aws_update_tag: int,
     current_aws_account_id: str,
 ) -> None:
@@ -252,12 +245,28 @@ def load_inspector_packages(
         neo4j_session,
         AWSInspectorPackageSchema(),
         packages,
-        Region=region,
         AWS_ID=current_aws_account_id,
         lastupdated=aws_update_tag,
     )
 
 
+@timeit
+def load_inspector_finding_to_package_match_links(
+    neo4j_session: neo4j.Session,
+    finding_to_package_map: List[Dict[str, str]],
+    aws_update_tag: int,
+    current_aws_account_id: str,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        InspectorFindingToPackageMatchLink(),
+        finding_to_package_map,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -270,6 +279,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_matchlink(
+        InspectorFindingToPackageMatchLink(),
+        "AWSAccount",
+        common_job_parameters["ACCOUNT_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(
+        neo4j_session,
+    )
 
 
 def _sync_findings_for_account(
@@ -288,7 +305,9 @@ def _sync_findings_for_account(
         logger.info(f"No findings to sync for account {account_id} in region {region}")
         return
     for f_batch in findings:
-        finding_data, package_data = transform_inspector_findings(f_batch)
+        finding_data, package_data, finding_to_package_map = (
+            transform_inspector_findings(f_batch)
+        )
         logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
         load_inspector_findings(
             neo4j_session,
@@ -301,7 +320,15 @@ def _sync_findings_for_account(
         load_inspector_packages(
             neo4j_session,
             package_data,
-            region,
+            update_tag,
+            current_aws_account_id,
+        )
+        logger.info(
+            f"Loading {len(finding_to_package_map)} finding to package relationships"
+        )
+        load_inspector_finding_to_package_match_links(
+            neo4j_session,
+            finding_to_package_map,
             update_tag,
             current_aws_account_id,
         )
@@ -337,5 +364,7 @@ def sync(
                 update_tag,
                 current_aws_account_id,
             )
+        common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
+        common_job_parameters["UPDATE_TAG"] = update_tag
 
     cleanup(neo4j_session, common_job_parameters)