cartography 0.116.1__py3-none-any.whl → 0.118.0__py3-none-any.whl

cartography/intel/aws/ecr.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import Any
 from typing import Dict
@@ -18,6 +19,12 @@ from cartography.util import to_synchronous
 
 logger = logging.getLogger(__name__)
 
+# Manifest list media types
+MANIFEST_LIST_MEDIA_TYPES = {
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.oci.image.index.v1+json",
+}
+
 
 @timeit
 @aws_handle_regions
@@ -34,6 +41,84 @@ def get_ecr_repositories(
     return ecr_repositories
 
 
+def _get_platform_specific_digests(
+    client: Any, repository_name: str, manifest_list_digest: str
+) -> tuple[List[Dict[str, Any]], set[str]]:
+    """
+    Fetch manifest list and extract platform-specific image digests and attestations.
+
+    Returns:
+        - List of all images (platform-specific + attestations) with digest, type, architecture, os, variant
+        - Set of ALL digests referenced in the manifest list
+    """
+    response = client.batch_get_image(
+        repositoryName=repository_name,
+        imageIds=[{"imageDigest": manifest_list_digest}],
+        acceptedMediaTypes=list(MANIFEST_LIST_MEDIA_TYPES),
+    )
+
+    if not response.get("images"):
+        raise ValueError(
+            f"No manifest list found for digest {manifest_list_digest} in repository {repository_name}"
+        )
+
+    # batch_get_image returns a single manifest list (hence [0])
+    # The manifests[] array inside contains all platform-specific images and attestations
+    manifest_json = json.loads(response["images"][0]["imageManifest"])
+    manifests = manifest_json.get("manifests", [])
+
+    if not manifests:
+        raise ValueError(
+            f"Manifest list {manifest_list_digest} has no manifests in repository {repository_name}"
+        )
+
+    all_images = []
+    all_referenced_digests = set()
+
+    for manifest_ref in manifests:
+        digest = manifest_ref.get("digest")
+        if not digest:
+            raise ValueError(
+                f"Manifest in list {manifest_list_digest} has no digest in repository {repository_name}"
+            )
+
+        all_referenced_digests.add(digest)
+
+        platform_info = manifest_ref.get("platform", {})
+        architecture = platform_info.get("architecture")
+        os_name = platform_info.get("os")
+
+        # Determine if this is an attestation
+        annotations = manifest_ref.get("annotations", {})
+        is_attestation = (
+            architecture == "unknown" and os_name == "unknown"
+        ) or annotations.get("vnd.docker.reference.type") == "attestation-manifest"
+
+        all_images.append(
+            {
+                "digest": digest,
+                "type": "attestation" if is_attestation else "image",
+                "architecture": architecture,
+                "os": os_name,
+                "variant": platform_info.get("variant"),
+                "attestation_type": (
+                    annotations.get("vnd.docker.reference.type")
+                    if is_attestation
+                    else None
+                ),
+                "attests_digest": (
+                    annotations.get("vnd.docker.reference.digest")
+                    if is_attestation
+                    else None
+                ),
+                "media_type": manifest_ref.get("mediaType"),
+                "artifact_media_type": manifest_ref.get("artifactType"),
+            }
+        )
+
+    return all_images, all_referenced_digests
+
+
 @timeit
 @aws_handle_regions
 def get_ecr_repository_images(
@@ -46,7 +131,11 @@ def get_ecr_repository_images(
     )
     client = boto3_session.client("ecr", region_name=region)
     list_paginator = client.get_paginator("list_images")
-    ecr_repository_images: List[Dict] = []
+
+    # First pass: Collect all image details and track manifest list referenced digests
+    all_image_details: List[Dict] = []
+    manifest_list_referenced_digests: set[str] = set()
+
     for page in list_paginator.paginate(repositoryName=repository_name):
         image_ids = page["imageIds"]
         if not image_ids:
@@ -58,14 +147,37 @@ def get_ecr_repository_images(
         for response in describe_response:
             image_details = response["imageDetails"]
             for detail in image_details:
-                tags = detail.get("imageTags") or []
-                if tags:
-                    for tag in tags:
-                        image_detail = {**detail, "imageTag": tag}
-                        image_detail.pop("imageTags", None)
-                        ecr_repository_images.append(image_detail)
-                else:
-                    ecr_repository_images.append({**detail})
+                # Check if this is a manifest list
+                media_type = detail.get("imageManifestMediaType")
+                if media_type in MANIFEST_LIST_MEDIA_TYPES:
+                    # Fetch all images from manifest list (platform-specific + attestations)
+                    manifest_list_digest = detail["imageDigest"]
+                    manifest_images, all_digests = _get_platform_specific_digests(
+                        client, repository_name, manifest_list_digest
+                    )
+                    detail["_manifest_images"] = manifest_images
+
+                    # Track ALL digests so we don't create ECRRepositoryImages for them
+                    manifest_list_referenced_digests.update(all_digests)
+
+                all_image_details.append(detail)
+
+    # Second pass: Only add images that should have ECRRepositoryImage nodes
+    ecr_repository_images: List[Dict] = []
+    for detail in all_image_details:
+        tags = detail.get("imageTags") or []
+        digest = detail.get("imageDigest")
+
+        if tags:
+            # Tagged images always get ECRRepositoryImage nodes (one per tag)
+            for tag in tags:
+                image_detail = {**detail, "imageTag": tag}
+                image_detail.pop("imageTags", None)
+                ecr_repository_images.append(image_detail)
+        elif digest not in manifest_list_referenced_digests:
+            # Untagged images only get nodes if they're NOT part of a manifest list
+            ecr_repository_images.append({**detail})
+
     return ecr_repository_images
 
 
@@ -91,52 +203,115 @@ def load_ecr_repositories(
 
 
 @timeit
-def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
+def transform_ecr_repository_images(repo_data: Dict) -> tuple[List[Dict], List[Dict]]:
     """
-    Ensure that we only load ECRImage nodes to the graph if they have a defined imageDigest field.
-    Process repositories in a consistent order to handle overlapping image digests deterministically.
+    Transform ECR repository images into repo image list and ECR image list.
+    For manifest lists, creates ECR images for manifest list, platform-specific images, and attestations.
+
+    Returns:
+        - repo_images_list: List of ECRRepositoryImage nodes with imageDigests field (one-to-many)
+        - ecr_images_list: List of ECRImage nodes with type, architecture, os, variant fields
     """
     repo_images_list = []
+    ecr_images_dict: Dict[str, Dict] = {}  # Deduplicate by digest
+
     # Sort repository URIs to ensure consistent processing order
     for repo_uri in sorted(repo_data.keys()):
         repo_images = repo_data[repo_uri]
         for img in repo_images:
             digest = img.get("imageDigest")
-            if digest:
-                tag = img.get("imageTag")
-                uri = repo_uri + (f":{tag}" if tag else "")
-                img["repo_uri"] = repo_uri
-                img["uri"] = uri
-                img["id"] = uri
-                repo_images_list.append(img)
-            else:
+            if not digest:
                 logger.warning(
                     "Repo %s has an image that has no imageDigest. Its tag is %s. Continuing on.",
                     repo_uri,
                     img.get("imageTag"),
                 )
+                continue
+
+            tag = img.get("imageTag")
+            uri = repo_uri + (f":{tag}" if tag else "")
+
+            # Build ECRRepositoryImage node
+            repo_image = {
+                **img,
+                "repo_uri": repo_uri,
+                "uri": uri,
+                "id": uri,
+            }
+
+            # Check if this is a manifest list with images
+            manifest_images = img.get("_manifest_images")
+            if manifest_images:
+                # For manifest list: include manifest list digest + all referenced digests
+                all_digests = [digest] + [m["digest"] for m in manifest_images]
+                repo_image["imageDigests"] = all_digests
+
+                # Create ECRImage for the manifest list itself
+                if digest not in ecr_images_dict:
+                    ecr_images_dict[digest] = {
+                        "imageDigest": digest,
+                        "type": "manifest_list",
+                        "architecture": None,
+                        "os": None,
+                        "variant": None,
+                    }
+
+                # Create ECRImage nodes for each image in the manifest list
+                for manifest_img in manifest_images:
+                    manifest_digest = manifest_img["digest"]
+                    if manifest_digest not in ecr_images_dict:
+                        ecr_images_dict[manifest_digest] = {
+                            "imageDigest": manifest_digest,
+                            "type": manifest_img.get("type"),
+                            "architecture": manifest_img.get("architecture"),
+                            "os": manifest_img.get("os"),
+                            "variant": manifest_img.get("variant"),
+                            "attestation_type": manifest_img.get("attestation_type"),
+                            "attests_digest": manifest_img.get("attests_digest"),
+                            "media_type": manifest_img.get("media_type"),
+                            "artifact_media_type": manifest_img.get(
+                                "artifact_media_type"
+                            ),
+                        }
+            else:
+                # Regular image: single digest
+                repo_image["imageDigests"] = [digest]
+
+                # Create ECRImage for regular image
+                if digest not in ecr_images_dict:
+                    ecr_images_dict[digest] = {
+                        "imageDigest": digest,
+                        "type": "image",
+                        "architecture": None,
+                        "os": None,
+                        "variant": None,
+                    }
+
+            # Remove internal field before returning
+            repo_image.pop("_manifest_images", None)
+            repo_images_list.append(repo_image)
 
-    return repo_images_list
+    ecr_images_list = list(ecr_images_dict.values())
+    return repo_images_list, ecr_images_list
 
 
 @timeit
 def load_ecr_repository_images(
     neo4j_session: neo4j.Session,
     repo_images_list: List[Dict],
+    ecr_images_list: List[Dict],
     region: str,
     current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     logger.info(
-        f"Loading {len(repo_images_list)} ECR repository images in {region} into graph.",
+        f"Loading {len(ecr_images_list)} ECR images and {len(repo_images_list)} ECR repository images in {region} into graph.",
     )
-    image_digests = {img["imageDigest"] for img in repo_images_list}
-    ecr_images = [{"imageDigest": d} for d in image_digests]
 
     load(
         neo4j_session,
         ECRImageSchema(),
-        ecr_images,
+        ecr_images_list,
         lastupdated=aws_update_tag,
         Region=region,
         AWS_ID=current_aws_account_id,
@@ -219,10 +394,11 @@ def sync(
             current_aws_account_id,
             update_tag,
         )
-        repo_images_list = transform_ecr_repository_images(image_data)
+        repo_images_list, ecr_images_list = transform_ecr_repository_images(image_data)
         load_ecr_repository_images(
             neo4j_session,
             repo_images_list,
+            ecr_images_list,
             region,
             current_aws_account_id,
             update_tag,

cartography/intel/aws/ecr_image_layers.py

@@ -170,6 +170,111 @@ async def get_blob_json_via_presigned(
     return response.json()
 
 
+async def _extract_parent_image_from_attestation(
+    ecr_client: ECRClient,
+    repo_name: str,
+    attestation_manifest_digest: str,
+    http_client: httpx.AsyncClient,
+) -> Optional[dict[str, str]]:
+    """
+    Extract parent image information from an in-toto provenance attestation.
+
+    This function fetches an attestation manifest, downloads its in-toto layer,
+    and extracts the parent image reference from the SLSA provenance materials.
+
+    :param ecr_client: ECR client for fetching manifests and layers
+    :param repo_name: ECR repository name
+    :param attestation_manifest_digest: Digest of the attestation manifest
+    :param http_client: HTTP client for downloading blobs
+    :return: Dict with parent_image_uri and parent_image_digest, or None if no parent image found
+    """
+    try:
+        attestation_manifest, _ = await batch_get_manifest(
+            ecr_client,
+            repo_name,
+            attestation_manifest_digest,
+            [ECR_OCI_MANIFEST_MT, ECR_DOCKER_MANIFEST_MT],
+        )
+
+        if not attestation_manifest:
+            logger.debug(
+                "No attestation manifest found for digest %s in repo %s",
+                attestation_manifest_digest,
+                repo_name,
+            )
+            return None
+
+        # Get the in-toto layer from the attestation manifest
+        layers = attestation_manifest.get("layers", [])
+        intoto_layer = next(
+            (
+                layer
+                for layer in layers
+                if "in-toto" in layer.get("mediaType", "").lower()
+            ),
+            None,
+        )
+
+        if not intoto_layer:
+            logger.debug(
+                "No in-toto layer found in attestation manifest %s",
+                attestation_manifest_digest,
+            )
+            return None
+
+        # Download the in-toto attestation blob
+        intoto_digest = intoto_layer.get("digest")
+        if not intoto_digest:
+            logger.debug("No digest found for in-toto layer")
+            return None
+
+        attestation_blob = await get_blob_json_via_presigned(
+            ecr_client,
+            repo_name,
+            intoto_digest,
+            http_client,
+        )
+
+        if not attestation_blob:
+            logger.debug("Failed to download attestation blob")
+            return None
+
+        # Extract parent image from SLSA provenance materials
+        materials = attestation_blob.get("predicate", {}).get("materials", [])
+        for material in materials:
+            uri = material.get("uri", "")
+            uri_l = uri.lower()
+            # Look for container image URIs that are NOT the dockerfile itself
+            is_container_ref = (
+                uri_l.startswith("pkg:docker/")
+                or uri_l.startswith("pkg:oci/")
+                or uri_l.startswith("oci://")
+            )
+            if is_container_ref and "dockerfile" not in uri_l:
+                digest_obj = material.get("digest", {})
+                sha256_digest = digest_obj.get("sha256")
+                if sha256_digest:
+                    return {
+                        "parent_image_uri": uri,
+                        "parent_image_digest": f"sha256:{sha256_digest}",
+                    }
+
+        logger.debug(
+            "No parent image found in attestation materials for %s",
+            attestation_manifest_digest,
+        )
+        return None
+
+    except Exception as e:
+        logger.warning(
+            "Error extracting parent image from attestation %s in repo %s: %s",
+            attestation_manifest_digest,
+            repo_name,
+            e,
+        )
+        return None
+
+
 async def _diff_ids_for_manifest(
     ecr_client: ECRClient,
     repo_name: str,
@@ -228,6 +333,7 @@ async def _diff_ids_for_manifest(
 def transform_ecr_image_layers(
     image_layers_data: dict[str, dict[str, list[str]]],
     image_digest_map: dict[str, str],
+    image_attestation_map: Optional[dict[str, dict[str, str]]] = None,
 ) -> tuple[list[dict], list[dict]]:
     """
     Transform image layer data into format suitable for Neo4j ingestion.
@@ -235,8 +341,11 @@ def transform_ecr_image_layers(
 
     :param image_layers_data: Map of image URI to platform to diff_ids
     :param image_digest_map: Map of image URI to image digest
+    :param image_attestation_map: Map of image URI to attestation data (parent_image_uri, parent_image_digest)
     :return: List of layer objects ready for ingestion
     """
+    if image_attestation_map is None:
+        image_attestation_map = {}
     layers_by_diff_id: dict[str, dict[str, Any]] = {}
     memberships_by_digest: dict[str, dict[str, Any]] = {}
 
@@ -278,10 +387,20 @@ def transform_ecr_image_layers(
                     layer["tail_image_ids"].add(image_digest)
 
         if ordered_layers_for_image:
-            memberships_by_digest[image_digest] = {
+            membership: dict[str, Any] = {
                 "layer_diff_ids": ordered_layers_for_image,
             }
 
+            # Add attestation data if available for this image
+            if image_uri in image_attestation_map:
+                attestation = image_attestation_map[image_uri]
+                membership["parent_image_uri"] = attestation["parent_image_uri"]
+                membership["parent_image_digest"] = attestation["parent_image_digest"]
+                membership["from_attestation"] = True
+                membership["confidence"] = "explicit"
+
+            memberships_by_digest[image_digest] = membership
+
     # Convert sets back to lists for Neo4j ingestion
     layers = []
     for layer in layers_by_diff_id.values():
@@ -350,12 +469,18 @@ async def fetch_image_layers_async(
     ecr_client: ECRClient,
     repo_images_list: list[dict],
     max_concurrent: int = 200,
-) -> tuple[dict[str, dict[str, list[str]]], dict[str, str]]:
+) -> tuple[dict[str, dict[str, list[str]]], dict[str, str], dict[str, dict[str, str]]]:
     """
     Fetch image layers for ECR images in parallel with caching and non-blocking I/O.
+
+    Returns:
+        - image_layers_data: Map of image URI to platform to diff_ids
+        - image_digest_map: Map of image URI to image digest
+        - image_attestation_map: Map of image URI to attestation data (parent_image_uri, parent_image_digest)
     """
     image_layers_data: dict[str, dict[str, list[str]]] = {}
     image_digest_map: dict[str, str] = {}
+    image_attestation_map: dict[str, dict[str, str]] = {}
     semaphore = asyncio.Semaphore(max_concurrent)
 
     # Cache for manifest fetches keyed by (repo_name, imageDigest)
@@ -402,8 +527,8 @@ async def fetch_image_layers_async(
     async def fetch_single_image_layers(
         repo_image: dict,
         http_client: httpx.AsyncClient,
-    ) -> Optional[tuple[str, str, dict[str, list[str]]]]:
-        """Fetch layers for a single image."""
+    ) -> Optional[tuple[str, str, dict[str, list[str]], Optional[dict[str, str]]]]:
+        """Fetch layers for a single image and extract attestation if present."""
         async with semaphore:
             # Caller guarantees these fields exist in every repo_image
             uri = repo_image["uri"]
@@ -426,24 +551,37 @@ async def fetch_image_layers_async(
 
             manifest_media_type = (media_type or doc.get("mediaType", "")).lower()
             platform_layers: dict[str, list[str]] = {}
+            attestation_data: Optional[dict[str, str]] = None
 
             if doc.get("manifests") and manifest_media_type in INDEX_MEDIA_TYPES_LOWER:
 
                 async def _process_child_manifest(
                     manifest_ref: dict,
-                ) -> dict[str, list[str]]:
-                    # Skip attestation manifests - these aren't real images
+                ) -> tuple[dict[str, list[str]], Optional[dict[str, str]]]:
+                    # Check if this is an attestation manifest
                     if (
                         manifest_ref.get("annotations", {}).get(
                             "vnd.docker.reference.type"
                         )
                         == "attestation-manifest"
                     ):
-                        return {}
+                        # Extract base image from attestation
+                        child_digest = manifest_ref.get("digest")
+                        if child_digest:
+                            attestation_info = (
+                                await _extract_parent_image_from_attestation(
+                                    ecr_client,
+                                    repo_name,
+                                    child_digest,
+                                    http_client,
+                                )
+                            )
+                            return {}, attestation_info
+                        return {}, None
 
                     child_digest = manifest_ref.get("digest")
                     if not child_digest:
-                        return {}
+                        return {}, None
 
                     # Use optimized caching for child manifest
                     child_doc, _ = await _fetch_and_cache_manifest(
@@ -452,16 +590,17 @@ async def fetch_image_layers_async(
                         [ECR_OCI_MANIFEST_MT, ECR_DOCKER_MANIFEST_MT],
                     )
                     if not child_doc:
-                        return {}
+                        return {}, None
 
                     platform_hint = extract_platform_from_manifest(manifest_ref)
-                    return await _diff_ids_for_manifest(
+                    diff_map = await _diff_ids_for_manifest(
                         ecr_client,
                         repo_name,
                         child_doc,
                         http_client,
                         platform_hint,
                     )
+                    return diff_map, None
 
                 # Process all child manifests in parallel
                 child_tasks = [
@@ -474,8 +613,13 @@ async def fetch_image_layers_async(
 
                 # Merge results from successful child manifest processing
                 for result in child_results:
-                    if isinstance(result, dict):
-                        platform_layers.update(result)
+                    if isinstance(result, tuple) and len(result) == 2:
+                        layer_data, attest_data = result
+                        if layer_data:
+                            platform_layers.update(layer_data)
+                        if attest_data and not attestation_data:
+                            # Use first attestation found
+                            attestation_data = attest_data
             else:
                 diff_map = await _diff_ids_for_manifest(
                     ecr_client,
@@ -487,7 +631,7 @@ async def fetch_image_layers_async(
                 platform_layers.update(diff_map)
 
             if platform_layers:
-                return uri, digest, platform_layers
+                return uri, digest, platform_layers, attestation_data
 
             return None
 
@@ -507,7 +651,7 @@ async def fetch_image_layers_async(
         )
 
         if not tasks:
-            return image_layers_data, image_digest_map
+            return image_layers_data, image_digest_map, image_attestation_map
 
         progress_interval = max(1, min(100, total // 10 or 1))
         completed = 0
@@ -526,16 +670,22 @@ async def fetch_image_layers_async(
                 )
 
             if result:
-                uri, digest, layer_data = result
+                uri, digest, layer_data, attestation_data = result
                 if not digest:
                     raise ValueError(f"Empty digest returned for image {uri}")
                 image_layers_data[uri] = layer_data
                 image_digest_map[uri] = digest
+                if attestation_data:
+                    image_attestation_map[uri] = attestation_data
 
     logger.info(
         f"Successfully fetched layers for {len(image_layers_data)}/{len(repo_images_list)} images"
     )
-    return image_layers_data, image_digest_map
+    if image_attestation_map:
+        logger.info(
+            f"Found attestations with base image info for {len(image_attestation_map)} images"
+        )
+    return image_layers_data, image_digest_map, image_attestation_map
 
 
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict) -> None:
@@ -613,9 +763,11 @@ def sync(
                 f"Starting to fetch layers for {len(repo_images_list)} images..."
             )
 
-            async def _fetch_with_async_client() -> (
-                tuple[dict[str, dict[str, list[str]]], dict[str, str]]
-            ):
+            async def _fetch_with_async_client() -> tuple[
+                dict[str, dict[str, list[str]]],
+                dict[str, str],
+                dict[str, dict[str, str]],
+            ]:
                 # Use credentials from the existing boto3 session
                 credentials = boto3_session.get_credentials()
                 session = aioboto3.Session(
@@ -635,8 +787,8 @@ def sync(
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
 
-            image_layers_data, image_digest_map = loop.run_until_complete(
-                _fetch_with_async_client()
+            image_layers_data, image_digest_map, image_attestation_map = (
+                loop.run_until_complete(_fetch_with_async_client())
             )
 
             logger.info(
@@ -645,6 +797,7 @@ def sync(
             layers, memberships = transform_ecr_image_layers(
                 image_layers_data,
                 image_digest_map,
+                image_attestation_map,
             )
             load_ecr_image_layers(
                 neo4j_session,

cartography/intel/aws/elasticsearch.py

@@ -8,6 +8,7 @@ import botocore.config
 import neo4j
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import run_write_query
 from cartography.intel.dns import ingest_dns_record_by_fqdn
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
@@ -95,7 +96,8 @@ def _load_es_domains(
     for d in domain_list:
         del d["ServiceSoftwareOptions"]
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_records,
         Records=domain_list,
         AWS_ACCOUNT_ID=aws_account_id,
@@ -179,7 +181,8 @@ def _link_es_domain_vpc(
         groupList = vpc_data.get("SecurityGroupIds", [])
 
         if len(subnetList) > 0:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_subnet,
                 DomainId=domain_id,
                 SubnetList=subnetList,
@@ -187,7 +190,8 @@ def _link_es_domain_vpc(
             )
 
         if len(groupList) > 0:
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 ingest_sec_groups,
                 DomainId=domain_id,
                 SecGroupList=groupList,
@@ -220,7 +224,12 @@ def _process_access_policy(
         if policy.is_internet_accessible():
             exposed_internet = True
 
-    neo4j_session.run(tag_es, DomainId=domain_id, InternetExposed=exposed_internet)
+    run_write_query(
+        neo4j_session,
+        tag_es,
+        DomainId=domain_id,
+        InternetExposed=exposed_internet,
+    )
 
 
 @timeit