cartography 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

cartography/intel/trivy/__init__.py

@@ -11,8 +11,7 @@ from cartography.config import Config
 from cartography.intel.trivy.scanner import cleanup
 from cartography.intel.trivy.scanner import get_json_files_in_dir
 from cartography.intel.trivy.scanner import get_json_files_in_s3
-from cartography.intel.trivy.scanner import sync_single_image_from_file
-from cartography.intel.trivy.scanner import sync_single_image_from_s3
+from cartography.intel.trivy.scanner import sync_single_image
 from cartography.stats import get_stats_client
 from cartography.util import timeit
 
@@ -20,53 +19,93 @@ logger = logging.getLogger(__name__)
 stat_handler = get_stats_client("trivy.scanner")
 
 
-@timeit
-def get_scan_targets(
+def _get_scan_targets_and_aliases(
     neo4j_session: Session,
     account_ids: list[str] | None = None,
-) -> set[str]:
+) -> tuple[set[str], dict[str, str]]:
     """
-    Return list of ECR images from all accounts in the graph.
+    Return tag URIs and a mapping of digest-qualified URIs to tag URIs.
     """
     if not account_ids:
         aws_accounts = list_accounts(neo4j_session)
     else:
         aws_accounts = account_ids
 
-    ecr_images: set[str] = set()
+    image_uris: set[str] = set()
+    digest_aliases: dict[str, str] = {}
+
     for account_id in aws_accounts:
-        for _, _, image_uri, _, _ in get_ecr_images(neo4j_session, account_id):
-            ecr_images.add(image_uri)
+        for _, _, image_uri, _, digest in get_ecr_images(neo4j_session, account_id):
+            if not image_uri:
+                continue
+            image_uris.add(image_uri)
+            if digest:
+                # repo URI is everything before the trailing ":" (if present)
+                repo_uri = image_uri.rsplit(":", 1)[0]
+                digest_uri = f"{repo_uri}@{digest}"
+                digest_aliases[digest_uri] = image_uri
 
-    return ecr_images
+    return image_uris, digest_aliases
 
 
-def _get_intersection(
-    image_uris: set[str], json_files: set[str], trivy_s3_prefix: str
-) -> list[tuple[str, str]]:
+@timeit
+def get_scan_targets(
+    neo4j_session: Session,
+    account_ids: list[str] | None = None,
+) -> set[str]:
+    """
+    Return list of ECR images from all accounts in the graph.
     """
-    Get the intersection of ECR images in the graph and S3 scan results.
+    image_uris, _ = _get_scan_targets_and_aliases(neo4j_session, account_ids)
+    return image_uris
 
-    Args:
-        image_uris: Set of ECR images in the graph
-        json_files: Set of S3 object keys for JSON files
-        trivy_s3_prefix: S3 prefix path containing scan results
 
-    Returns:
-        List of tuples (image_uri, s3_object_key)
+def _prepare_trivy_data(
+    trivy_data: dict[str, Any],
+    image_uris: set[str],
+    digest_aliases: dict[str, str],
+    source: str,
+) -> tuple[dict[str, Any], str] | None:
     """
-    intersection = []
-    prefix_len = len(trivy_s3_prefix)
-    for s3_object_key in json_files:
-        # Sample key "123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
-        # Sample key "folder/derp/123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
-        # Remove the prefix and the .json suffix
-        image_uri = s3_object_key[prefix_len:-5]
+    Determine the tag URI that corresponds to this Trivy payload.
+
+    Returns (trivy_data, display_uri) if the payload can be linked to an image present
+    in the graph; otherwise returns None so the caller can skip ingestion.
+    """
+
+    artifact_name = (trivy_data.get("ArtifactName") or "").strip()
+    metadata = trivy_data.get("Metadata") or {}
+    candidates: list[str] = []
+
+    if artifact_name:
+        candidates.append(artifact_name)
 
-        if image_uri in image_uris:
-            intersection.append((image_uri, s3_object_key))
+    repo_tags = metadata.get("RepoTags", [])
+    repo_digests = metadata.get("RepoDigests", [])
+    stripped_tags_digests = [item.strip() for item in repo_tags + repo_digests]
+    candidates.extend(stripped_tags_digests)
 
-    return intersection
+    display_uri: str | None = None
+
+    for candidate in candidates:
+        if not candidate:
+            continue
+        if candidate in image_uris:
+            display_uri = candidate
+            break
+        alias = digest_aliases.get(candidate)
+        if alias:
+            display_uri = alias
+            break
+
+    if not display_uri:
+        logger.debug(
+            "Skipping Trivy results for %s because no matching image URI was found in the graph",
+            source,
+        )
+        return None
+
+    return trivy_data, display_uri
 
 
 @timeit
@@ -93,15 +132,12 @@ def sync_trivy_aws_ecr_from_s3(
         f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
     )
 
-    image_uris: set[str] = get_scan_targets(neo4j_session)
+    image_uris, digest_aliases = _get_scan_targets_and_aliases(neo4j_session)
     json_files: set[str] = get_json_files_in_s3(
         trivy_s3_bucket, trivy_s3_prefix, boto3_session
     )
-    intersection: list[tuple[str, str]] = _get_intersection(
-        image_uris, json_files, trivy_s3_prefix
-    )
 
-    if len(intersection) == 0:
+    if len(json_files) == 0:
         logger.error(
             f"Trivy sync was configured, but there are no ECR images with S3 json scan results in bucket "
             f"'{trivy_s3_bucket}' with prefix '{trivy_s3_prefix}'. "
@@ -110,18 +146,33 @@ def sync_trivy_aws_ecr_from_s3(
             f"`<image_uri>.json` and to be in the same bucket and prefix as the scan results. If the prefix is "
             "a folder, it MUST end with a trailing slash '/'. "
         )
-        logger.error(f"JSON files in S3: {json_files}")
         raise ValueError("No ECR images with S3 json scan results found.")
 
-    logger.info(f"Processing {len(intersection)} ECR images with S3 scan results")
-    for image_uri, s3_object_key in intersection:
-        sync_single_image_from_s3(
+    logger.info(f"Processing {len(json_files)} Trivy result files from S3")
+    s3_client = boto3_session.client("s3")
+    for s3_object_key in json_files:
+        logger.debug(
+            f"Reading scan results from S3: s3://{trivy_s3_bucket}/{s3_object_key}"
+        )
+        response = s3_client.get_object(Bucket=trivy_s3_bucket, Key=s3_object_key)
+        scan_data_json = response["Body"].read().decode("utf-8")
+        trivy_data = json.loads(scan_data_json)
+
+        prepared = _prepare_trivy_data(
+            trivy_data,
+            image_uris=image_uris,
+            digest_aliases=digest_aliases,
+            source=f"s3://{trivy_s3_bucket}/{s3_object_key}",
+        )
+        if prepared is None:
+            continue
+
+        prepared_data, display_uri = prepared
+        sync_single_image(
             neo4j_session,
-            image_uri,
+            prepared_data,
+            display_uri,
             update_tag,
-            trivy_s3_bucket,
-            s3_object_key,
-            boto3_session,
         )
 
     cleanup(neo4j_session, common_job_parameters)
@@ -137,7 +188,7 @@ def sync_trivy_aws_ecr_from_dir(
     """Sync Trivy scan results from local files for AWS ECR images."""
     logger.info(f"Using Trivy scan results from {results_dir}")
 
-    image_uris: set[str] = get_scan_targets(neo4j_session)
+    image_uris, digest_aliases = _get_scan_targets_and_aliases(neo4j_session)
     json_files: set[str] = get_json_files_in_dir(results_dir)
 
     if not json_files:
@@ -149,27 +200,27 @@ def sync_trivy_aws_ecr_from_dir(
     logger.info(f"Processing {len(json_files)} local Trivy result files")
 
     for file_path in json_files:
-        # First, check if the image exists in the graph before syncing
         try:
-            # Peek at the artifact name without processing the file
             with open(file_path, encoding="utf-8") as f:
                 trivy_data = json.load(f)
-                artifact_name = trivy_data.get("ArtifactName")
-
-            if artifact_name and artifact_name not in image_uris:
-                logger.debug(
-                    f"Skipping results for {artifact_name} since the image is not present in the graph"
-                )
-                continue
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to read Trivy data from {file_path}: {e}")
+            continue
 
-        except (json.JSONDecodeError, KeyError) as e:
-            logger.error(f"Failed to read artifact name from {file_path}: {e}")
+        prepared = _prepare_trivy_data(
+            trivy_data,
+            image_uris=image_uris,
+            digest_aliases=digest_aliases,
+            source=file_path,
+        )
+        if prepared is None:
             continue
 
-        # Now sync the file since we know the image exists in the graph
-        sync_single_image_from_file(
+        prepared_data, display_uri = prepared
+        sync_single_image(
             neo4j_session,
-            file_path,
+            prepared_data,
+            display_uri,
             update_tag,
         )
 

cartography/models/airbyte/user.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from cartography.models.core.common import PropertyRef
 from cartography.models.core.nodes import CartographyNodeProperties
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
@@ -98,6 +99,9 @@ class AirbyteUserToWorkspaceMemberRel(CartographyRelSchema):
 @dataclass(frozen=True)
 class AirbyteUserSchema(CartographyNodeSchema):
     label: str = "AirbyteUser"
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ["UserAccount"]
+    )  # UserAccount label is used for ontology mapping
     properties: AirbyteUserNodeProperties = AirbyteUserNodeProperties()
     sub_resource_relationship: AirbyteUserToOrganizationRel = (
         AirbyteUserToOrganizationRel()

cartography/models/anthropic/user.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from cartography.models.core.common import PropertyRef
 from cartography.models.core.nodes import CartographyNodeProperties
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
@@ -42,6 +43,9 @@ class AnthropicUserToOrganizationRel(CartographyRelSchema):
 @dataclass(frozen=True)
 class AnthropicUserSchema(CartographyNodeSchema):
     label: str = "AnthropicUser"
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ["UserAccount"]
+    )  # UserAccount label is used for ontology mapping
     properties: AnthropicUserNodeProperties = AnthropicUserNodeProperties()
     sub_resource_relationship: AnthropicUserToOrganizationRel = (
         AnthropicUserToOrganizationRel()

cartography/models/aws/ec2/networkinterfaces.py

@@ -47,6 +47,8 @@ class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties):
     # TODO: remove subnetid once we have migrated to subnet_id
     subnetid: PropertyRef = PropertyRef("SubnetId", extra_index=True)
     subnet_id: PropertyRef = PropertyRef("SubnetId", extra_index=True)
+    attach_time: PropertyRef = PropertyRef("AttachTime")
+    device_index: PropertyRef = PropertyRef("DeviceIndex")
 
 
 @dataclass(frozen=True)

cartography/models/aws/ecr/image.py

@@ -18,6 +18,15 @@ class ECRImageNodeProperties(CartographyNodeProperties):
     region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
     lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
     layer_diff_ids: PropertyRef = PropertyRef("layer_diff_ids")
+    type: PropertyRef = PropertyRef("type")
+    architecture: PropertyRef = PropertyRef("architecture")
+    os: PropertyRef = PropertyRef("os")
+    variant: PropertyRef = PropertyRef("variant")
+    attestation_type: PropertyRef = PropertyRef("attestation_type")
+    attests_digest: PropertyRef = PropertyRef("attests_digest")
+    media_type: PropertyRef = PropertyRef("media_type")
+    artifact_media_type: PropertyRef = PropertyRef("artifact_media_type")
+    child_image_digests: PropertyRef = PropertyRef("child_image_digests")
 
 
 @dataclass(frozen=True)
@@ -78,6 +87,50 @@ class ECRImageToParentImageRel(CartographyRelSchema):
     )
 
 
+@dataclass(frozen=True)
+class ECRImageContainsImageRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class ECRImageContainsImageRel(CartographyRelSchema):
+    """
+    Relationship from a manifest list ECRImage to platform-specific ECRImages it contains.
+    Only applies to ECRImage nodes with type="manifest_list".
+    """
+
+    target_node_label: str = "ECRImage"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"digest": PropertyRef("child_image_digests", one_to_many=True)},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "CONTAINS_IMAGE"
+    properties: ECRImageContainsImageRelProperties = (
+        ECRImageContainsImageRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class ECRImageAttestsRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class ECRImageAttestsRel(CartographyRelSchema):
+    """
+    Relationship from an attestation ECRImage to the ECRImage it attests/validates.
+    Only applies to ECRImage nodes with type="attestation".
+    """
+
+    target_node_label: str = "ECRImage"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"digest": PropertyRef("attests_digest")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "ATTESTS"
+    properties: ECRImageAttestsRelProperties = ECRImageAttestsRelProperties()
+
+
 @dataclass(frozen=True)
 class ECRImageSchema(CartographyNodeSchema):
     label: str = "ECRImage"
@@ -87,5 +140,7 @@ class ECRImageSchema(CartographyNodeSchema):
         [
             ECRImageHasLayerRel(),
             ECRImageToParentImageRel(),
+            ECRImageContainsImageRel(),
+            ECRImageAttestsRel(),
         ],
     )

cartography/models/aws/ecr/repository_image.py

@@ -71,7 +71,7 @@ class ECRRepositoryImageToECRImageRelProperties(CartographyRelProperties):
 class ECRRepositoryImageToECRImageRel(CartographyRelSchema):
     target_node_label: str = "ECRImage"
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {"id": PropertyRef("imageDigest")}
+        {"id": PropertyRef("imageDigests", one_to_many=True)}
     )
     direction: LinkDirection = LinkDirection.OUTWARD
     rel_label: str = "IMAGE"

cartography/models/aws/iam/group_membership.py

@@ -15,12 +15,13 @@ class AWSGroupToAWSUserRelProperties(CartographyRelProperties):
 
 @dataclass(frozen=True)
 class AWSGroupToAWSUserRel(CartographyRelSchema):
+    # AWSUser -MEMBER_AWS_GROUP-> AWSGroup
     target_node_label: str = "AWSUser"
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
         {
-            "arn": PropertyRef("user_arn"),
+            "arn": PropertyRef("user_arns", one_to_many=True),
         }
     )
-    direction: LinkDirection = LinkDirection.OUTWARD
+    direction: LinkDirection = LinkDirection.INWARD
     rel_label: str = "MEMBER_AWS_GROUP"
     properties: AWSGroupToAWSUserRelProperties = AWSGroupToAWSUserRelProperties()

cartography/models/aws/identitycenter/awsssouser.py

@@ -95,7 +95,9 @@ class AWSSSOUserToPermissionSetRel(CartographyRelSchema):
 class AWSSSOUserSchema(CartographyNodeSchema):
     label: str = "AWSSSOUser"
     properties: SSOUserProperties = SSOUserProperties()
-    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(["UserAccount"])
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ["UserAccount"]
+    )  # UserAccount label is used for ontology mapping
     sub_resource_relationship: AWSSSOUserToAWSAccountRel = AWSSSOUserToAWSAccountRel()
     other_relationships: OtherRelationships = OtherRelationships(
         [

cartography/models/bigfix/bigfix_computer.py

@@ -22,7 +22,7 @@ class BigfixComputerNodeProperties(CartographyNodeProperties):
     besrootserver: PropertyRef = PropertyRef("BESRootServer")
     bios: PropertyRef = PropertyRef("BIOS")
     computertype: PropertyRef = PropertyRef("ComputerType")
-    computername: PropertyRef = PropertyRef("ComputerName")
+    computername: PropertyRef = PropertyRef("ComputerName", extra_index=True)
     cpu: PropertyRef = PropertyRef("CPU")
     devicetype: PropertyRef = PropertyRef("DeviceType")
     distancetobesrelay: PropertyRef = PropertyRef("DistanceToBESRelay")

cartography/models/cloudflare/member.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from cartography.models.core.common import PropertyRef
 from cartography.models.core.nodes import CartographyNodeProperties
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
@@ -71,6 +72,9 @@ class CloudflareMemberToCloudflareRoleRel(CartographyRelSchema):
 @dataclass(frozen=True)
 class CloudflareMemberSchema(CartographyNodeSchema):
     label: str = "CloudflareMember"
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ["UserAccount"]
+    )  # UserAccount label is used for ontology mapping
     properties: CloudflareMemberNodeProperties = CloudflareMemberNodeProperties()
     sub_resource_relationship: CloudflareMemberToAccountRel = (
         CloudflareMemberToAccountRel()

cartography/models/crowdstrike/hosts.py

@@ -12,7 +12,7 @@ class CrowdstrikeHostNodeProperties(CartographyNodeProperties):
     instance_id: PropertyRef = PropertyRef("instance_id", extra_index=True)
     serial_number: PropertyRef = PropertyRef("serial_number", extra_index=True)
     status: PropertyRef = PropertyRef("status")
-    hostname: PropertyRef = PropertyRef("hostname")
+    hostname: PropertyRef = PropertyRef("hostname", extra_index=True)
     machine_domain: PropertyRef = PropertyRef("machine_domain")
     crowdstrike_first_seen: PropertyRef = PropertyRef("first_seen")
     crowdstrike_last_seen: PropertyRef = PropertyRef("last_seen")

cartography/models/duo/endpoint.py

@@ -21,7 +21,7 @@ class DuoEndpointNodeProperties(CartographyNodeProperties):
     device_id: PropertyRef = PropertyRef("device_id")
     device_identifier: PropertyRef = PropertyRef("device_identifier")
     device_identifier_type: PropertyRef = PropertyRef("device_identifier_type")
-    device_name: PropertyRef = PropertyRef("device_name")
+    device_name: PropertyRef = PropertyRef("device_name", extra_index=True)
     device_udid: PropertyRef = PropertyRef("device_udid")
     device_username: PropertyRef = PropertyRef("device_username")
     device_username_type: PropertyRef = PropertyRef("device_username_type")

cartography/models/duo/phone.py

@@ -22,8 +22,8 @@ class DuoPhoneNodeProperties(CartographyNodeProperties):
     fingerprint: PropertyRef = PropertyRef("fingerprint")
     last_seen: PropertyRef = PropertyRef("last_seen")
     model: PropertyRef = PropertyRef("model")
-    name: PropertyRef = PropertyRef("name")
-    phone_id: PropertyRef = PropertyRef("phone_id", extra_index=True)
+    name: PropertyRef = PropertyRef("name", extra_index=True)
+    phone_id: PropertyRef = PropertyRef("phone_id")
     platform: PropertyRef = PropertyRef("platform")
     postdelay: PropertyRef = PropertyRef("postdelay")
     predelay: PropertyRef = PropertyRef("predelay")

cartography/models/duo/user.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from cartography.models.core.common import PropertyRef
 from cartography.models.core.nodes import CartographyNodeProperties
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
@@ -149,6 +150,9 @@ class DuoUserToHumanRel(CartographyRelSchema):
 @dataclass(frozen=True)
 class DuoUserSchema(CartographyNodeSchema):
     label: str = "DuoUser"
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ["UserAccount"]
+    )  # UserAccount label is used for ontology mapping
     properties: DuoUserNodeProperties = DuoUserNodeProperties()
     sub_resource_relationship: DuoUserToDuoApiHostRel = DuoUserToDuoApiHostRel()
     other_relationships: OtherRelationships = OtherRelationships(

cartography/models/entra/user.py

@@ -85,5 +85,6 @@ class EntraUserSchema(CartographyNodeSchema):
     extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
         [
             "EntraIdentity",
-        ]
+            "UserAccount",
+        ]  # UserAccount label is used for ontology mapping
     )

cartography/models/github/users.py

@@ -33,6 +33,7 @@ from dataclasses import dataclass
 from cartography.models.core.common import PropertyRef
 from cartography.models.core.nodes import CartographyNodeProperties
 from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
 from cartography.models.core.relationships import CartographyRelProperties
 from cartography.models.core.relationships import CartographyRelSchema
 from cartography.models.core.relationships import LinkDirection
@@ -128,6 +129,9 @@ class GitHubOrganizationUserSchema(CartographyNodeSchema):
 @dataclass(frozen=True)
 class GitHubUnaffiliatedUserSchema(CartographyNodeSchema):
     label: str = "GitHubUser"
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
+        ["UserAccount"]
+    )  # UserAccount label is used for ontology mapping
     properties: GitHubUnaffiliatedUserNodeProperties = (
         GitHubUnaffiliatedUserNodeProperties()
     )