cartography 0.106.0rc1__py3-none-any.whl → 0.107.0__py3-none-any.whl

cartography/intel/aws/codebuild.py

@@ -0,0 +1,132 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.codebuild.project import CodeBuildProjectSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_all_codebuild_projects(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+
+    client = boto3_session.client(
+        "codebuild", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_projects")
+
+    all_projects = []
+
+    for page in paginator.paginate():
+        project_names = page.get("projects", [])
+        if not project_names:
+            continue
+
+        # AWS batch_get_projects accepts up to 100 project names per call as per AWS documentation.
+        for i in range(0, len(project_names), 100):
+            batch = project_names[i : i + 100]
+            response = client.batch_get_projects(names=batch)
+            projects = response.get("projects", [])
+            all_projects.extend(projects)
+    return all_projects
+
+
+def transform_codebuild_projects(
+    projects: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CodeBuild project data for ingestion into Neo4j.
+
+    - Includes all environment variable names.
+    - Variables of type 'PLAINTEXT' retain their values.
+    - Other types (e.g., 'PARAMETER_STORE', 'SECRETS_MANAGER') have their values redacted.
+    """
+    transformed_codebuild_projects = []
+    for project in projects:
+        env_vars = project.get("environment", {}).get("environmentVariables", [])
+        env_var_strings = [
+            f"{var.get('name')}={var.get('value') if var.get('type') == 'PLAINTEXT' else '<REDACTED>'}"
+            for var in env_vars
+        ]
+        transformed_project = {
+            "arn": project["arn"],
+            "created": project.get("created"),
+            "environmentVariables": env_var_strings,
+            "sourceType": project.get("source", {}).get("type"),
+            "sourceLocation": project.get("source", {}).get("location"),
+        }
+        transformed_codebuild_projects.append(transformed_project)
+
+    return transformed_codebuild_projects
+
+
+@timeit
+def load_codebuild_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CodeBuild {len(data)} projects for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CodeBuildProjectSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CodeBuildProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CodeBuild for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        projects = get_all_codebuild_projects(boto3_session, region)
+        transformed_projects = transform_codebuild_projects(projects, region)
+
+        load_codebuild_projects(
+            neo4j_session,
+            transformed_projects,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/subnets.py

@@ -53,7 +53,7 @@ def load_subnets(
     snet.state = subnet.State, snet.assignipv6addressoncreation = subnet.AssignIpv6AddressOnCreation,
     snet.map_public_ip_on_launch = subnet.MapPublicIpOnLaunch, snet.subnet_arn = subnet.SubnetArn,
     snet.availability_zone = subnet.AvailabilityZone, snet.availability_zone_id = subnet.AvailabilityZoneId,
-    snet.subnetid = subnet.SubnetId
+    snet.subnet_id = subnet.SubnetId
     """
 
     ingest_subnet_vpc_relations = """

cartography/intel/aws/ecs.py

@@ -169,6 +169,22 @@ def _get_containers_from_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, An
     return containers
 
 
+def transform_ecs_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Extract network interface ID from task attachments.
+    """
+    for task in tasks:
+        for attachment in task.get("attachments", []):
+            if attachment.get("type") == "ElasticNetworkInterface":
+                details = attachment.get("details", [])
+                for detail in details:
+                    if detail.get("name") == "networkInterfaceId":
+                        task["networkInterfaceId"] = detail.get("value")
+                        break
+                break
+    return tasks
+
+
 @timeit
 def load_ecs_clusters(
     neo4j_session: neo4j.Session,
@@ -407,6 +423,7 @@ def _sync_ecs_task_and_container_defns(
         boto3_session,
         region,
     )
+    tasks = transform_ecs_tasks(tasks)
     containers = _get_containers_from_tasks(tasks)
     load_ecs_tasks(
         neo4j_session,

cartography/intel/aws/efs.py

@@ -9,6 +9,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.efs.access_point import EfsAccessPointSchema
 from cartography.models.aws.efs.file_system import EfsFileSystemSchema
 from cartography.models.aws.efs.mount_target import EfsMountTargetSchema
 from cartography.util import aws_handle_regions
@@ -44,6 +45,7 @@ def transform_efs_file_systems(
         transformed_file_system = {
             "FileSystemId": file_system["FileSystemId"],
             "FileSystemArn": file_system["FileSystemArn"],
+            "Region": region,
             "OwnerId": file_system.get("OwnerId"),
             "CreationToken": file_system.get("CreationToken"),
             "CreationTime": file_system.get("CreationTime"),
@@ -87,6 +89,49 @@ def get_efs_mount_targets(
     return mountTargets
 
 
+@timeit
+@aws_handle_regions
+def get_efs_access_points(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "efs", region_name=region, config=get_botocore_config()
+    )
+
+    paginator = client.get_paginator("describe_access_points")
+    accessPoints = []
+    for page in paginator.paginate():
+        accessPoints.extend(page.get("AccessPoints", []))
+
+    return accessPoints
+
+
+def transform_efs_access_points(
+    accessPoints: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Efs Access Points data for ingestion
+    """
+    transformed = []
+    for ap in accessPoints:
+        transformed.append(
+            {
+                "AccessPointArn": ap["AccessPointArn"],
+                "AccessPointId": ap["AccessPointId"],
+                "Region": region,
+                "FileSystemId": ap["FileSystemId"],
+                "Name": ap.get("Name"),
+                "LifeCycleState": ap.get("LifeCycleState"),
+                "OwnerId": ap.get("OwnerId"),
+                "Uid": ap.get("PosixUser", {}).get("Uid"),
+                "Gid": ap.get("PosixUser", {}).get("Gid"),
+                "RootDirectoryPath": ap.get("RootDirectory", {}).get("Path"),
+            }
+        )
+
+    return transformed
+
+
 @timeit
 def load_efs_mount_targets(
     neo4j_session: neo4j.Session,
@@ -129,6 +174,27 @@ def load_efs_file_systems(
     )
 
 
+@timeit
+def load_efs_access_points(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Efs {len(data)} access points for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EfsAccessPointSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -141,6 +207,9 @@ def cleanup(
     GraphJob.from_node_schema(EfsFileSystemSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(EfsAccessPointSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -178,4 +247,15 @@ def sync(
             update_tag,
         )
 
+        accessPoints = get_efs_access_points(boto3_session, region)
+        accessPoints_transformed = transform_efs_access_points(accessPoints, region)
+
+        load_efs_access_points(
+            neo4j_session,
+            accessPoints_transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/inspector.py

@@ -3,21 +3,22 @@ from typing import Any
 from typing import Dict
 from typing import Iterator
 from typing import List
+from typing import Set
 from typing import Tuple
 
 import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
+from cartography.models.aws.inspector.findings import InspectorFindingToPackageMatchLink
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
 from cartography.util import batch
 from cartography.util import timeit
-from cartography.util import to_asynchronous
-from cartography.util import to_synchronous
 
 logger = logging.getLogger(__name__)
 
@@ -109,9 +110,10 @@ def get_inspector_findings(
 
 def transform_inspector_findings(
     results: List[Dict[str, Any]],
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, str]]]:
     findings_list: List[Dict] = []
-    packages: Dict[str, Any] = {}
+    packages_set: Set[frozenset] = set()
+    finding_to_package_map: List[Dict[str, str]] = []
 
     for f in results:
         finding: Dict = {}
@@ -165,55 +167,45 @@ def transform_inspector_findings(
                 "vendorUpdatedAt",
             )
 
-            new_packages = _process_packages(
-                f["packageVulnerabilityDetails"],
-                f["awsAccountId"],
-                f["findingArn"],
-            )
-            finding["vulnerablepackageids"] = list(new_packages.keys())
-            packages = {**packages, **new_packages}
-
+            packages = transform_inspector_packages(f["packageVulnerabilityDetails"])
+            finding["vulnerablepackageids"] = list(packages.keys())
+            for package_id, package in packages.items():
+                finding_to_package_map.append(
+                    {
+                        "findingarn": finding["id"],
+                        "packageid": package_id,
+                        "remediation": package.get("remediation"),
+                        "fixedInVersion": package.get("fixedInVersion"),
+                        "filePath": package.get("filePath"),
+                        "sourceLayerHash": package.get("sourceLayerHash"),
+                        "sourceLambdaLayerArn": package.get("sourceLambdaLayerArn"),
+                    }
+                )
+                packages_set.add(frozenset(package.items()))
         findings_list.append(finding)
-    packages_list = transform_inspector_packages(packages)
-    return findings_list, packages_list
-
-
-def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict[str, Any]]:
-    packages_list: List[Dict] = []
-    for package_id in packages.keys():
-        packages_list.append(packages[package_id])
+    packages_list = [dict(p) for p in packages_set]
+    return findings_list, packages_list, finding_to_package_map
 
-    return packages_list
 
-
-def _process_packages(
+def transform_inspector_packages(
     package_details: Dict[str, Any],
-    aws_account_id: str,
-    finding_arn: str,
 ) -> Dict[str, Any]:
     packages: Dict[str, Any] = {}
     for package in package_details["vulnerablePackages"]:
-        new_package = {}
-        new_package["id"] = (
-            f"{package.get('name', '')}|"
-            f"{package.get('arch', '')}|"
-            f"{package.get('version', '')}|"
-            f"{package.get('release', '')}|"
-            f"{package.get('epoch', '')}"
-        )
-        new_package["name"] = package.get("name")
-        new_package["arch"] = package.get("arch")
-        new_package["version"] = package.get("version")
-        new_package["release"] = package.get("release")
-        new_package["epoch"] = package.get("epoch")
-        new_package["manager"] = package.get("packageManager")
-        new_package["filepath"] = package.get("filePath")
-        new_package["fixedinversion"] = package.get("fixedInVersion")
-        new_package["sourcelayerhash"] = package.get("sourceLayerHash")
-        new_package["awsaccount"] = aws_account_id
-        new_package["findingarn"] = finding_arn
-
-        packages[new_package["id"]] = new_package
+        # Following RPM package naming convention for consistency
+        name = package["name"]  # Mandatory field
+        epoch = str(package.get("epoch", ""))
+        if epoch:
+            epoch = f"{epoch}:"
+        version = package["version"]  # Mandatory field
+        release = package.get("release", "")
+        if release:
+            release = f"-{release}"
+        arch = package.get("arch", "")
+        if arch:
+            arch = f".{arch}"
+        id = f"{name}|{epoch}{version}{release}{arch}"
+        packages[id] = {**package, "id": id}
 
     return packages
 
@@ -246,7 +238,6 @@ def load_inspector_findings(
 def load_inspector_packages(
     neo4j_session: neo4j.Session,
     packages: List[Dict[str, Any]],
-    region: str,
     aws_update_tag: int,
     current_aws_account_id: str,
 ) -> None:
@@ -254,12 +245,28 @@ def load_inspector_packages(
         neo4j_session,
         AWSInspectorPackageSchema(),
         packages,
-        Region=region,
         AWS_ID=current_aws_account_id,
         lastupdated=aws_update_tag,
     )
 
 
+@timeit
+def load_inspector_finding_to_package_match_links(
+    neo4j_session: neo4j.Session,
+    finding_to_package_map: List[Dict[str, str]],
+    aws_update_tag: int,
+    current_aws_account_id: str,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        InspectorFindingToPackageMatchLink(),
+        finding_to_package_map,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -272,6 +279,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(
         neo4j_session,
     )
+    GraphJob.from_matchlink(
+        InspectorFindingToPackageMatchLink(),
+        "AWSAccount",
+        common_job_parameters["ACCOUNT_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(
+        neo4j_session,
+    )
 
 
 def _sync_findings_for_account(
@@ -290,7 +305,9 @@ def _sync_findings_for_account(
         logger.info(f"No findings to sync for account {account_id} in region {region}")
         return
     for f_batch in findings:
-        finding_data, package_data = transform_inspector_findings(f_batch)
+        finding_data, package_data, finding_to_package_map = (
+            transform_inspector_findings(f_batch)
+        )
         logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
         load_inspector_findings(
             neo4j_session,
@@ -303,7 +320,15 @@ def _sync_findings_for_account(
         load_inspector_packages(
             neo4j_session,
             package_data,
-            region,
+            update_tag,
+            current_aws_account_id,
+        )
+        logger.info(
+            f"Loading {len(finding_to_package_map)} finding to package relationships"
+        )
+        load_inspector_finding_to_package_match_links(
+            neo4j_session,
+            finding_to_package_map,
             update_tag,
             current_aws_account_id,
         )
@@ -329,10 +354,9 @@ def sync(
         member_accounts = get_member_accounts(boto3_session, region)
         # the current host account may not be considered a "member", but we still fetch its findings
         member_accounts.append(current_aws_account_id)
-
-        async def async_ingest_findings_for_account(account_id: str) -> None:
-            await to_asynchronous(
-                _sync_findings_for_account,
+        logger.info(f"Member accounts to be synced: {member_accounts}")
+        for account_id in member_accounts:
+            _sync_findings_for_account(
                 neo4j_session,
                 boto3_session,
                 region,
@@ -340,12 +364,7 @@ def sync(
                 update_tag,
                 current_aws_account_id,
             )
-
-        to_synchronous(
-            *[
-                async_ingest_findings_for_account(account_id)
-                for account_id in member_accounts
-            ]
-        )
+        common_job_parameters["ACCOUNT_ID"] = current_aws_account_id
+        common_job_parameters["UPDATE_TAG"] = update_tag
 
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -6,7 +6,9 @@ from cartography.intel.aws.ec2.route_tables import sync_route_tables
 from . import acm
 from . import apigateway
 from . import cloudtrail
+from . import cloudtrail_management_events
 from . import cloudwatch
+from . import codebuild
 from . import config
 from . import dynamodb
 from . import ecr
@@ -106,6 +108,8 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "config": config.sync,
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
+    "cloudtrail_management_events": cloudtrail_management_events.sync,
     "cloudwatch": cloudwatch.sync,
     "efs": efs.sync,
+    "codebuild": codebuild.sync,
 }

cartography/intel/aws/sns.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -9,6 +10,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.sns.topic import SNSTopicSchema
+from cartography.models.aws.sns.topic_subscription import SNSTopicSubscriptionSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
@@ -108,6 +110,48 @@ def load_sns_topics(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_subscriptions(
+    boto3_session: boto3.session.Session, region: str
+) -> List[Dict[str, Any]]:
+    """
+    Get all SNS Topics Subscriptions for a region.
+    """
+    client = boto3_session.client("sns", region_name=region)
+    paginator = client.get_paginator("list_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page.get("Subscriptions", []))
+
+    return subscriptions
+
+
+@timeit
+def load_sns_topic_subscription(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SNS Topic Subscription information into the graph
+    """
+    logger.info(
+        f"Loading {len(data)} SNS topic subscription for region {region} into graph."
+    )
+
+    load(
+        neo4j_session,
+        SNSTopicSubscriptionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
 @timeit
 def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     """
@@ -117,6 +161,11 @@ def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> No
     cleanup_job = GraphJob.from_node_schema(SNSTopicSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        SNSTopicSubscriptionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync(
@@ -128,7 +177,7 @@ def sync(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Sync SNS Topics for all regions
+    Sync SNS Topics and Subscriptions for all regions
     """
     for region in regions:
         logger.info(
@@ -153,9 +202,20 @@ def sync(
             update_tag,
         )
 
+        # Get and load subscriptions
+        subscriptions = get_subscriptions(boto3_session, region)
+
+        load_sns_topic_subscription(
+            neo4j_session,
+            subscriptions,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    # Cleanup and metadata update (outside region loop)
     cleanup_sns(neo4j_session, common_job_parameters)
 
-    # Record that we've synced this module
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",