cartography 0.117.0__py3-none-any.whl → 0.118.0__py3-none-any.whl

cartography/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.117.0'
-__version_tuple__ = version_tuple = (0, 117, 0)
+__version__ = version = '0.118.0'
+__version_tuple__ = version_tuple = (0, 118, 0)
 
 __commit_id__ = commit_id = None

cartography/cli.py

@@ -279,6 +279,17 @@ class CLI:
                 "Example: 'HIGH' will sync only HIGH and CRITICAL findings, filtering out LOW and MEDIUM severity findings."
             ),
         )
+        parser.add_argument(
+            "--experimental-aws-inspector-batch",
+            type=int,
+            default=1000,
+            help=(
+                "EXPERIMENTAL: This feature is experimental and may be removed in the future. "
+                "Batch size for AWS Inspector findings sync. Controls how many findings are fetched, processed and cleaned up at a time. "
+                "Default is 1000. Increase this value if you have a large number of findings and want to reduce API calls, "
+                "or decrease it if you're experiencing memory issues."
+            ),
+        )
         parser.add_argument(
             "--analysis-job-directory",
             type=str,

cartography/config.py

@@ -58,6 +58,9 @@ class Config:
     :type aws_guardduty_severity_threshold: str
     :param aws_guardduty_severity_threshold: GuardDuty severity threshold filter. Only findings at or above this
         severity level will be synced. Valid values: LOW, MEDIUM, HIGH, CRITICAL. Optional.
+    :type experimental_aws_inspector_batch: int
+    :param experimental_aws_inspector_batch: EXPERIMENTAL: Batch size for AWS Inspector findings sync. Controls how
+        many findings are fetched, processed and cleaned up at a time. Default is 1000. Optional.
     :type analysis_job_directory: str
     :param analysis_job_directory: Path to a directory tree containing analysis jobs to run. Optional.
     :type oci_sync_all_profiles: bool
@@ -195,6 +198,7 @@ class Config:
         aws_regions=None,
         aws_best_effort_mode=False,
         aws_cloudtrail_management_events_lookback_hours=None,
+        experimental_aws_inspector_batch=1000,
         azure_sync_all_subscriptions=False,
         azure_sp_auth=None,
         azure_tenant_id=None,
@@ -287,6 +291,7 @@ class Config:
         self.aws_cloudtrail_management_events_lookback_hours = (
             aws_cloudtrail_management_events_lookback_hours
         )
+        self.experimental_aws_inspector_batch = experimental_aws_inspector_batch
         self.azure_sync_all_subscriptions = azure_sync_all_subscriptions
         self.azure_sp_auth = azure_sp_auth
         self.azure_tenant_id = azure_tenant_id

cartography/graph/job.py

@@ -139,11 +139,13 @@ class GraphJob:
         cls,
         node_schema: CartographyNodeSchema,
         parameters: Dict[str, Any],
+        iterationsize: int = 100,
     ) -> "GraphJob":
         """
         Create a cleanup job from a CartographyNodeSchema object.
         For a given node, the fields used in the node_schema.sub_resource_relationship.target_node_node_matcher.keys()
         must be provided as keys and values in the params dict.
+        :param iterationsize: The number of items to process in each iteration. Defaults to 100.
         """
         queries: List[str] = build_cleanup_queries(node_schema)
 
@@ -165,7 +167,7 @@ class GraphJob:
                 query,
                 parameters=parameters,
                 iterative=True,
-                iterationsize=100,
+                iterationsize=iterationsize,
                 parent_job_name=node_schema.label,
                 parent_job_sequence_num=idx,
             )
@@ -185,6 +187,7 @@ class GraphJob:
         sub_resource_label: str,
         sub_resource_id: str,
         update_tag: int,
+        iterationsize: int = 100,
     ) -> "GraphJob":
         """
         Create a cleanup job from a CartographyRelSchema object (specifically, a MatchLink).
@@ -194,6 +197,7 @@ class GraphJob:
         - For a given rel_schema, the fields used in the rel_schema.properties._sub_resource_label.name and
         rel_schema.properties._sub_resource_id.name must be provided as keys and values in the params dict.
         - The rel_schema must have a source_node_matcher and target_node_matcher.
+        :param iterationsize: The number of items to process in each iteration. Defaults to 100.
         """
         cleanup_link_query = build_cleanup_query_for_matchlink(rel_schema)
         logger.debug(f"Cleanup query: {cleanup_link_query}")
@@ -208,7 +212,7 @@ class GraphJob:
             cleanup_link_query,
             parameters=parameters,
             iterative=True,
-            iterationsize=100,
+            iterationsize=iterationsize,
             parent_job_name=rel_schema.rel_label,
         )
 

cartography/graph/statement.py

@@ -52,6 +52,10 @@ class GraphStatement:
         self.parameters = parameters or {}
         self.iterative = iterative
         self.iterationsize = iterationsize
+        if iterationsize < 0:
+            raise ValueError(
+                f"iterationsize must be a positive integer, got {iterationsize}",
+            )
         self.parameters["LIMIT_SIZE"] = self.iterationsize
 
         self.parent_job_name = parent_job_name if parent_job_name else None

cartography/intel/aws/__init__.py

@@ -312,6 +312,7 @@ def start_aws_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
         "permission_relationships_file": config.permission_relationships_file,
         "aws_guardduty_severity_threshold": config.aws_guardduty_severity_threshold,
         "aws_cloudtrail_management_events_lookback_hours": config.aws_cloudtrail_management_events_lookback_hours,
+        "experimental_aws_inspector_batch": config.experimental_aws_inspector_batch,
     }
     try:
         boto3_session = boto3.Session()

cartography/intel/aws/apigateway.py

@@ -178,11 +178,24 @@ def get_rest_api_resources_methods_integrations(
                 method["apiId"] = api["id"]
                 method["httpMethod"] = http_method
                 methods.append(method)
-                integration = client.get_integration(
-                    restApiId=api["id"],
-                    resourceId=resource_id,
-                    httpMethod=http_method,
-                )
+                try:
+                    integration = client.get_integration(
+                        restApiId=api["id"],
+                        resourceId=resource_id,
+                        httpMethod=http_method,
+                    )
+                except ClientError as e:
+                    error_code = e.response.get("Error", {}).get("Code")
+                    if error_code == "NotFoundException":
+                        logger.warning(
+                            "No integration found for API %s resource %s method %s: %s",
+                            api["id"],
+                            resource_id,
+                            http_method,
+                            e,
+                        )
+                        continue
+                    raise
                 integration["resourceId"] = resource_id
                 integration["apiId"] = api["id"]
                 integration["integrationHttpMethod"] = integration.get("httpMethod")

cartography/intel/aws/ec2/elastic_ip_addresses.py

@@ -6,6 +6,7 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -83,7 +84,8 @@ def load_elastic_ip_addresses(
         SET r.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_addresses,
         elastic_ip_addresses=elastic_ip_addresses,
         Region=region,

cartography/intel/aws/ec2/internet_gateways.py

@@ -5,6 +5,7 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -63,13 +64,14 @@ def load_internet_gateways(
         SET r.lastupdated = $aws_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         query,
         internet_gateways=internet_gateways,
         region=region,
         aws_account_id=current_aws_account_id,
         aws_update_tag=update_tag,
-    ).consume()
+    )
 
 
 @timeit

cartography/intel/aws/ec2/load_balancer_v2s.py

@@ -6,6 +6,7 @@ import boto3
 import botocore
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -104,7 +105,8 @@ def load_load_balancer_v2s(
             logger.warning("Skipping load balancer entry with missing DNSName: %r", lb)
             continue
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_load_balancer_v2,
             ID=load_balancer_id,
             CREATED_TIME=str(lb["CreatedTime"]),
@@ -138,7 +140,8 @@ def load_load_balancer_v2s(
             SET r.lastupdated = $update_tag
             """
             for group in lb["SecurityGroups"]:
-                neo4j_session.run(
+                run_write_query(
+                    neo4j_session,
                     ingest_load_balancer_v2_security_group,
                     ID=load_balancer_id,
                     GROUP_ID=str(group),
@@ -182,7 +185,8 @@ def load_load_balancer_v2_subnets(
     SET r.lastupdated = $update_tag
     """
     for az in az_data:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_load_balancer_subnet,
             ID=load_balancer_id,
             SubnetId=az["SubnetId"],
@@ -219,7 +223,8 @@ def load_load_balancer_v2_target_groups(
             continue
 
         for instance in target_group["Targets"]:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_instances,
                 ID=load_balancer_id,
                 INSTANCE_ID=instance,
@@ -253,7 +258,8 @@ def load_load_balancer_v2_listeners(
         ON CREATE SET r.firstseen = timestamp()
         SET r.lastupdated = $update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_listener,
         LoadBalancerId=load_balancer_id,
         Listeners=listener_data,

cartography/intel/aws/ec2/network_interfaces.py

@@ -98,6 +98,10 @@ def transform_network_interface_data(
                 "SourceDestCheck": network_interface["SourceDestCheck"],
                 "Status": network_interface["Status"],
                 "SubnetId": network_interface["SubnetId"],
+                "AttachTime": network_interface.get("Attachment", {}).get("AttachTime"),
+                "DeviceIndex": network_interface.get("Attachment", {}).get(
+                    "DeviceIndex"
+                ),
                 "ElbV1Id": elb_v1_id,
                 "ElbV2Id": elb_v2_id,
             },

cartography/intel/aws/ec2/reserved_instances.py

@@ -6,6 +6,7 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -64,7 +65,8 @@ def load_reserved_instances(
         r_instance["Start"] = str(r_instance["Start"])
         r_instance["End"] = str(r_instance["End"])
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_reserved_instances,
         reserved_instances_list=data,
         AWS_ACCOUNT_ID=current_aws_account_id,

cartography/intel/aws/ec2/tgw.py

@@ -6,6 +6,7 @@ import boto3
 import botocore.exceptions
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -120,7 +121,8 @@ def load_transit_gateways(
     for tgw in data:
         tgw_id = tgw["TransitGatewayId"]
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_transit_gateway,
             TgwId=tgw_id,
             ARN=tgw["TransitGatewayArn"],
@@ -161,7 +163,8 @@ def _attach_shared_transit_gateway(
     """
 
     if tgw["OwnerId"] != current_aws_account_id:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_tgw,
             ARN=tgw["TransitGatewayArn"],
             TransitGatewayId=tgw["TransitGatewayId"],
@@ -202,7 +205,8 @@ def load_tgw_attachments(
     for tgwa in data:
         tgwa_id = tgwa["TransitGatewayAttachmentId"]
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_transit_gateway,
             TgwAttachmentId=tgwa_id,
             TransitGatewayId=tgwa["TransitGatewayId"],
@@ -261,7 +265,8 @@ def _attach_tgw_vpc_attachment_to_vpc_subnets(
     SET p.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         attach_vpc_tgw_attachment_to_vpc,
         VpcId=tgw_vpc_attachment["VpcId"],
         TgwAttachmentId=tgw_vpc_attachment["TransitGatewayAttachmentId"],
@@ -269,7 +274,8 @@ def _attach_tgw_vpc_attachment_to_vpc_subnets(
     )
 
     for subnet_id in tgw_vpc_attachment["SubnetIds"]:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_vpc_tgw_attachment_to_subnet,
             SubnetId=subnet_id,
             TgwAttachmentId=tgw_vpc_attachment["TransitGatewayAttachmentId"],

cartography/intel/aws/ec2/volumes.py

@@ -70,7 +70,7 @@ def transform_volumes(
 
         for attachment in active_attachments:
             vol_with_attachment = raw_vol.copy()
-            vol_with_attachment["InstanceId"] = attachment["InstanceId"]
+            vol_with_attachment["InstanceId"] = attachment.get("InstanceId")
             result.append(vol_with_attachment)
 
     return result

cartography/intel/aws/ecr.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import Any
 from typing import Dict
@@ -18,6 +19,12 @@ from cartography.util import to_synchronous
 
 logger = logging.getLogger(__name__)
 
+# Manifest list media types
+MANIFEST_LIST_MEDIA_TYPES = {
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.oci.image.index.v1+json",
+}
+
 
 @timeit
 @aws_handle_regions
@@ -34,6 +41,84 @@ def get_ecr_repositories(
     return ecr_repositories
 
 
+def _get_platform_specific_digests(
+    client: Any, repository_name: str, manifest_list_digest: str
+) -> tuple[List[Dict[str, Any]], set[str]]:
+    """
+    Fetch manifest list and extract platform-specific image digests and attestations.
+
+    Returns:
+        - List of all images (platform-specific + attestations) with digest, type, architecture, os, variant
+        - Set of ALL digests referenced in the manifest list
+    """
+    response = client.batch_get_image(
+        repositoryName=repository_name,
+        imageIds=[{"imageDigest": manifest_list_digest}],
+        acceptedMediaTypes=list(MANIFEST_LIST_MEDIA_TYPES),
+    )
+
+    if not response.get("images"):
+        raise ValueError(
+            f"No manifest list found for digest {manifest_list_digest} in repository {repository_name}"
+        )
+
+    # batch_get_image returns a single manifest list (hence [0])
+    # The manifests[] array inside contains all platform-specific images and attestations
+    manifest_json = json.loads(response["images"][0]["imageManifest"])
+    manifests = manifest_json.get("manifests", [])
+
+    if not manifests:
+        raise ValueError(
+            f"Manifest list {manifest_list_digest} has no manifests in repository {repository_name}"
+        )
+
+    all_images = []
+    all_referenced_digests = set()
+
+    for manifest_ref in manifests:
+        digest = manifest_ref.get("digest")
+        if not digest:
+            raise ValueError(
+                f"Manifest in list {manifest_list_digest} has no digest in repository {repository_name}"
+            )
+
+        all_referenced_digests.add(digest)
+
+        platform_info = manifest_ref.get("platform", {})
+        architecture = platform_info.get("architecture")
+        os_name = platform_info.get("os")
+
+        # Determine if this is an attestation
+        annotations = manifest_ref.get("annotations", {})
+        is_attestation = (
+            architecture == "unknown" and os_name == "unknown"
+        ) or annotations.get("vnd.docker.reference.type") == "attestation-manifest"
+
+        all_images.append(
+            {
+                "digest": digest,
+                "type": "attestation" if is_attestation else "image",
+                "architecture": architecture,
+                "os": os_name,
+                "variant": platform_info.get("variant"),
+                "attestation_type": (
+                    annotations.get("vnd.docker.reference.type")
+                    if is_attestation
+                    else None
+                ),
+                "attests_digest": (
+                    annotations.get("vnd.docker.reference.digest")
+                    if is_attestation
+                    else None
+                ),
+                "media_type": manifest_ref.get("mediaType"),
+                "artifact_media_type": manifest_ref.get("artifactType"),
+            }
+        )
+
+    return all_images, all_referenced_digests
+
+
 @timeit
 @aws_handle_regions
 def get_ecr_repository_images(
@@ -46,7 +131,11 @@ def get_ecr_repository_images(
     )
     client = boto3_session.client("ecr", region_name=region)
     list_paginator = client.get_paginator("list_images")
-    ecr_repository_images: List[Dict] = []
+
+    # First pass: Collect all image details and track manifest list referenced digests
+    all_image_details: List[Dict] = []
+    manifest_list_referenced_digests: set[str] = set()
+
     for page in list_paginator.paginate(repositoryName=repository_name):
         image_ids = page["imageIds"]
         if not image_ids:
@@ -58,14 +147,37 @@ def get_ecr_repository_images(
         for response in describe_response:
             image_details = response["imageDetails"]
             for detail in image_details:
-                tags = detail.get("imageTags") or []
-                if tags:
-                    for tag in tags:
-                        image_detail = {**detail, "imageTag": tag}
-                        image_detail.pop("imageTags", None)
-                        ecr_repository_images.append(image_detail)
-                else:
-                    ecr_repository_images.append({**detail})
+                # Check if this is a manifest list
+                media_type = detail.get("imageManifestMediaType")
+                if media_type in MANIFEST_LIST_MEDIA_TYPES:
+                    # Fetch all images from manifest list (platform-specific + attestations)
+                    manifest_list_digest = detail["imageDigest"]
+                    manifest_images, all_digests = _get_platform_specific_digests(
+                        client, repository_name, manifest_list_digest
+                    )
+                    detail["_manifest_images"] = manifest_images
+
+                    # Track ALL digests so we don't create ECRRepositoryImages for them
+                    manifest_list_referenced_digests.update(all_digests)
+
+                all_image_details.append(detail)
+
+    # Second pass: Only add images that should have ECRRepositoryImage nodes
+    ecr_repository_images: List[Dict] = []
+    for detail in all_image_details:
+        tags = detail.get("imageTags") or []
+        digest = detail.get("imageDigest")
+
+        if tags:
+            # Tagged images always get ECRRepositoryImage nodes (one per tag)
+            for tag in tags:
+                image_detail = {**detail, "imageTag": tag}
+                image_detail.pop("imageTags", None)
+                ecr_repository_images.append(image_detail)
+        elif digest not in manifest_list_referenced_digests:
+            # Untagged images only get nodes if they're NOT part of a manifest list
+            ecr_repository_images.append({**detail})
+
     return ecr_repository_images
 
 
@@ -91,52 +203,115 @@ def load_ecr_repositories(
 
 
 @timeit
-def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
+def transform_ecr_repository_images(repo_data: Dict) -> tuple[List[Dict], List[Dict]]:
     """
-    Ensure that we only load ECRImage nodes to the graph if they have a defined imageDigest field.
-    Process repositories in a consistent order to handle overlapping image digests deterministically.
+    Transform ECR repository images into repo image list and ECR image list.
+    For manifest lists, creates ECR images for manifest list, platform-specific images, and attestations.
+
+    Returns:
+        - repo_images_list: List of ECRRepositoryImage nodes with imageDigests field (one-to-many)
+        - ecr_images_list: List of ECRImage nodes with type, architecture, os, variant fields
     """
     repo_images_list = []
+    ecr_images_dict: Dict[str, Dict] = {}  # Deduplicate by digest
+
     # Sort repository URIs to ensure consistent processing order
     for repo_uri in sorted(repo_data.keys()):
         repo_images = repo_data[repo_uri]
         for img in repo_images:
             digest = img.get("imageDigest")
-            if digest:
-                tag = img.get("imageTag")
-                uri = repo_uri + (f":{tag}" if tag else "")
-                img["repo_uri"] = repo_uri
-                img["uri"] = uri
-                img["id"] = uri
-                repo_images_list.append(img)
-            else:
+            if not digest:
                 logger.warning(
                     "Repo %s has an image that has no imageDigest. Its tag is %s. Continuing on.",
                     repo_uri,
                     img.get("imageTag"),
                 )
+                continue
+
+            tag = img.get("imageTag")
+            uri = repo_uri + (f":{tag}" if tag else "")
+
+            # Build ECRRepositoryImage node
+            repo_image = {
+                **img,
+                "repo_uri": repo_uri,
+                "uri": uri,
+                "id": uri,
+            }
+
+            # Check if this is a manifest list with images
+            manifest_images = img.get("_manifest_images")
+            if manifest_images:
+                # For manifest list: include manifest list digest + all referenced digests
+                all_digests = [digest] + [m["digest"] for m in manifest_images]
+                repo_image["imageDigests"] = all_digests
+
+                # Create ECRImage for the manifest list itself
+                if digest not in ecr_images_dict:
+                    ecr_images_dict[digest] = {
+                        "imageDigest": digest,
+                        "type": "manifest_list",
+                        "architecture": None,
+                        "os": None,
+                        "variant": None,
+                    }
+
+                # Create ECRImage nodes for each image in the manifest list
+                for manifest_img in manifest_images:
+                    manifest_digest = manifest_img["digest"]
+                    if manifest_digest not in ecr_images_dict:
+                        ecr_images_dict[manifest_digest] = {
+                            "imageDigest": manifest_digest,
+                            "type": manifest_img.get("type"),
+                            "architecture": manifest_img.get("architecture"),
+                            "os": manifest_img.get("os"),
+                            "variant": manifest_img.get("variant"),
+                            "attestation_type": manifest_img.get("attestation_type"),
+                            "attests_digest": manifest_img.get("attests_digest"),
+                            "media_type": manifest_img.get("media_type"),
+                            "artifact_media_type": manifest_img.get(
+                                "artifact_media_type"
+                            ),
+                        }
+            else:
+                # Regular image: single digest
+                repo_image["imageDigests"] = [digest]
+
+                # Create ECRImage for regular image
+                if digest not in ecr_images_dict:
+                    ecr_images_dict[digest] = {
+                        "imageDigest": digest,
+                        "type": "image",
+                        "architecture": None,
+                        "os": None,
+                        "variant": None,
+                    }
+
+            # Remove internal field before returning
+            repo_image.pop("_manifest_images", None)
+            repo_images_list.append(repo_image)
 
-    return repo_images_list
+    ecr_images_list = list(ecr_images_dict.values())
+    return repo_images_list, ecr_images_list
 
 
 @timeit
 def load_ecr_repository_images(
     neo4j_session: neo4j.Session,
     repo_images_list: List[Dict],
+    ecr_images_list: List[Dict],
     region: str,
     current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     logger.info(
-        f"Loading {len(repo_images_list)} ECR repository images in {region} into graph.",
+        f"Loading {len(ecr_images_list)} ECR images and {len(repo_images_list)} ECR repository images in {region} into graph.",
     )
-    image_digests = {img["imageDigest"] for img in repo_images_list}
-    ecr_images = [{"imageDigest": d} for d in image_digests]
 
     load(
         neo4j_session,
         ECRImageSchema(),
-        ecr_images,
+        ecr_images_list,
         lastupdated=aws_update_tag,
         Region=region,
         AWS_ID=current_aws_account_id,
@@ -219,10 +394,11 @@ def sync(
             current_aws_account_id,
             update_tag,
         )
-        repo_images_list = transform_ecr_repository_images(image_data)
+        repo_images_list, ecr_images_list = transform_ecr_repository_images(image_data)
         load_ecr_repository_images(
             neo4j_session,
             repo_images_list,
+            ecr_images_list,
             region,
             current_aws_account_id,
             update_tag,