cartography 0.104.0rc3__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/intel/aws/acm.py

@@ -0,0 +1,124 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.acm.certificate import ACMCertificateSchema
+from cartography.stats import get_stats_client
+from cartography.util import aws_handle_regions
+from cartography.util import merge_module_sync_metadata
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_acm_certificates(
+    boto3_session: boto3.session.Session, region: str
+) -> list[dict[str, Any]]:
+    client = boto3_session.client("acm", region_name=region)
+    paginator = client.get_paginator("list_certificates")
+    summaries: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        summaries.extend(page.get("CertificateSummaryList", []))
+
+    details: list[dict[str, Any]] = []
+    for summary in summaries:
+        arn = summary["CertificateArn"]
+        resp = client.describe_certificate(CertificateArn=arn)
+        details.append(resp["Certificate"])
+    return details
+
+
+def transform_acm_certificates(
+    certificates: list[dict[str, Any]], region: str
+) -> list[dict[str, Any]]:
+    transformed: list[dict[str, Any]] = []
+    for cert in certificates:
+        item: dict[str, Any] = {
+            "Arn": cert["CertificateArn"],
+            "DomainName": cert.get("DomainName"),
+            "Type": cert.get("Type"),
+            "Status": cert.get("Status"),
+            "KeyAlgorithm": cert.get("KeyAlgorithm"),
+            "SignatureAlgorithm": cert.get("SignatureAlgorithm"),
+            "NotBefore": cert.get("NotBefore"),
+            "NotAfter": cert.get("NotAfter"),
+            "InUseBy": cert.get("InUseBy", []),
+            "Region": region,
+        }
+        # Extract ELBV2 Listener ARNs for relationship creation
+        listener_arns = [a for a in item["InUseBy"] if ":listener/" in a]
+        if listener_arns:
+            item["ELBV2ListenerArns"] = listener_arns
+        transformed.append(item)
+    return transformed
+
+
+@timeit
+def load_acm_certificates(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} ACM certificates for region {region} into graph.")
+    load(
+        neo4j_session,
+        ACMCertificateSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup_acm_certificates(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    logger.debug("Running ACM certificate cleanup job.")
+    GraphJob.from_node_schema(ACMCertificateSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: list[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing ACM certificates for region {region} in account {current_aws_account_id}."
+        )
+        certs = get_acm_certificates(boto3_session, region)
+        transformed = transform_acm_certificates(certs, region)
+        load_acm_certificates(
+            neo4j_session,
+            transformed,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup_acm_certificates(neo4j_session, common_job_parameters)
+
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="AWSAccount",
+        group_id=current_aws_account_id,
+        synced_type="ACMCertificate",
+        update_tag=update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/aws/cloudtrail.py

@@ -4,7 +4,6 @@ from typing import Dict
 from typing import List
 
 import boto3
-import botocore.exceptions
 import neo4j
 
 from cartography.client.core.tx import load
@@ -25,10 +24,8 @@ def get_cloudtrail_trails(
     client = boto3_session.client(
         "cloudtrail", region_name=region, config=get_botocore_config()
     )
-    paginator = client.get_paginator("list_trails")
-    trails = []
-    for page in paginator.paginate():
-        trails.extend(page["Trails"])
+
+    trails = client.describe_trails()["trailList"]
 
     # CloudTrail multi-region trails are shown in list_trails,
     # but the get_trail call only works in the home region
@@ -36,28 +33,6 @@ def get_cloudtrail_trails(
     return trails_filtered
 
 
-@timeit
-def get_cloudtrail_trail(
-    boto3_session: boto3.Session,
-    region: str,
-    trail_name: str,
-) -> Dict[str, Any]:
-    client = boto3_session.client(
-        "cloudtrail", region_name=region, config=get_botocore_config()
-    )
-    trail_details: Dict[str, Any] = {}
-    try:
-        response = client.get_trail(Name=trail_name)
-        trail_details = response["Trail"]
-    except botocore.exceptions.ClientError as e:
-        code = e.response["Error"]["Code"]
-        msg = e.response["Error"]["Message"]
-        logger.warning(
-            f"Could not run CloudTrail get_trail due to boto3 error {code}: {msg}. Skipping.",
-        )
-    return trail_details
-
-
 @timeit
 def load_cloudtrail_trails(
     neo4j_session: neo4j.Session,
@@ -105,20 +80,10 @@ def sync(
             f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
         )
         trails = get_cloudtrail_trails(boto3_session, region)
-        trail_data: List[Dict[str, Any]] = []
-        for trail in trails:
-            trail_name = trail["Name"]
-            trail_details = get_cloudtrail_trail(
-                boto3_session,
-                region,
-                trail_name,
-            )
-            if trail_details:
-                trail_data.append(trail_details)
 
         load_cloudtrail_trails(
             neo4j_session,
-            trail_data,
+            trails,
             region,
             current_aws_account_id,
             update_tag,

cartography/intel/aws/codebuild.py

@@ -0,0 +1,132 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.codebuild.project import CodeBuildProjectSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_all_codebuild_projects(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+
+    client = boto3_session.client(
+        "codebuild", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_projects")
+
+    all_projects = []
+
+    for page in paginator.paginate():
+        project_names = page.get("projects", [])
+        if not project_names:
+            continue
+
+        # AWS batch_get_projects accepts up to 100 project names per call as per AWS documentation.
+        for i in range(0, len(project_names), 100):
+            batch = project_names[i : i + 100]
+            response = client.batch_get_projects(names=batch)
+            projects = response.get("projects", [])
+            all_projects.extend(projects)
+    return all_projects
+
+
+def transform_codebuild_projects(
+    projects: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CodeBuild project data for ingestion into Neo4j.
+
+    - Includes all environment variable names.
+    - Variables of type 'PLAINTEXT' retain their values.
+    - Other types (e.g., 'PARAMETER_STORE', 'SECRETS_MANAGER') have their values redacted.
+    """
+    transformed_codebuild_projects = []
+    for project in projects:
+        env_vars = project.get("environment", {}).get("environmentVariables", [])
+        env_var_strings = [
+            f"{var.get('name')}={var.get('value') if var.get('type') == 'PLAINTEXT' else '<REDACTED>'}"
+            for var in env_vars
+        ]
+        transformed_project = {
+            "arn": project["arn"],
+            "created": project.get("created"),
+            "environmentVariables": env_var_strings,
+            "sourceType": project.get("source", {}).get("type"),
+            "sourceLocation": project.get("source", {}).get("location"),
+        }
+        transformed_codebuild_projects.append(transformed_project)
+
+    return transformed_codebuild_projects
+
+
+@timeit
+def load_codebuild_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CodeBuild {len(data)} projects for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CodeBuildProjectSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CodeBuildProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CodeBuild for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        projects = get_all_codebuild_projects(boto3_session, region)
+        transformed_projects = transform_codebuild_projects(projects, region)
+
+        load_codebuild_projects(
+            neo4j_session,
+            transformed_projects,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ecr.py

@@ -107,9 +107,12 @@ def load_ecr_repositories(
 def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
     """
     Ensure that we only load ECRImage nodes to the graph if they have a defined imageDigest field.
+    Process repositories in a consistent order to handle overlapping image digests deterministically.
     """
     repo_images_list = []
-    for repo_uri, repo_images in repo_data.items():
+    # Sort repository URIs to ensure consistent processing order
+    for repo_uri in sorted(repo_data.keys()):
+        repo_images = repo_data[repo_uri]
         for img in repo_images:
             if "imageDigest" in img and img["imageDigest"]:
                 img["repo_uri"] = repo_uri
@@ -214,7 +217,9 @@ def _get_image_data(
         )
         image_data[repo["repositoryUri"]] = repo_image_obj
 
-    to_synchronous(*[async_get_images(repo) for repo in repositories])
+    # Sort repositories by name to ensure consistent processing order
+    sorted_repos = sorted(repositories, key=lambda x: x["repositoryName"])
+    to_synchronous(*[async_get_images(repo) for repo in sorted_repos])
 
     return image_data
 
@@ -237,6 +242,7 @@ def sync(
         image_data = {}
         repositories = get_ecr_repositories(boto3_session, region)
         image_data = _get_image_data(boto3_session, region, repositories)
+        # len here should be 1!
         load_ecr_repositories(
             neo4j_session,
             repositories,