cartography 0.117.0__py3-none-any.whl → 0.118.0__py3-none-any.whl

cartography/intel/pagerduty/services.py

@@ -7,6 +7,7 @@ import dateutil.parser
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -96,7 +97,8 @@ def load_service_data(
             for team in service["teams"]:
                 team_relations.append({"service": service["id"], "team": team["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Services=data,
         update_tag=update_tag,
@@ -120,7 +122,8 @@ def _attach_teams(
         MERGE (t)-[r:ASSOCIATED_WITH]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -162,7 +165,8 @@ def load_integration_data(
         created_at = dateutil.parser.parse(integration["created_at"])
         integration["created_at"] = int(created_at.timestamp())
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Integrations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/teams.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -68,7 +69,8 @@ def load_team_data(
     """
     logger.info(f"Loading {len(data)} pagerduty teams.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Teams=data,
         update_tag=update_tag,
@@ -90,7 +92,8 @@ def load_team_relations(
         ON CREATE SET r.firstseen = timestamp()
         SET r.role = relation.role
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/users.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -57,7 +58,8 @@ def load_user_data(
     """
     logger.info(f"Loading {len(data)} pagerduty users.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Users=data,
         update_tag=update_tag,

cartography/intel/pagerduty/vendors.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,8 @@ def load_vendor_data(
     """
     logger.info(f"Loading {len(data)} pagerduty vendors.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Vendors=data,
         update_tag=update_tag,

cartography/intel/trivy/__init__.py

@@ -11,8 +11,7 @@ from cartography.config import Config
 from cartography.intel.trivy.scanner import cleanup
 from cartography.intel.trivy.scanner import get_json_files_in_dir
 from cartography.intel.trivy.scanner import get_json_files_in_s3
-from cartography.intel.trivy.scanner import sync_single_image_from_file
-from cartography.intel.trivy.scanner import sync_single_image_from_s3
+from cartography.intel.trivy.scanner import sync_single_image
 from cartography.stats import get_stats_client
 from cartography.util import timeit
 
@@ -20,53 +19,93 @@ logger = logging.getLogger(__name__)
 stat_handler = get_stats_client("trivy.scanner")
 
 
-@timeit
-def get_scan_targets(
+def _get_scan_targets_and_aliases(
     neo4j_session: Session,
     account_ids: list[str] | None = None,
-) -> set[str]:
+) -> tuple[set[str], dict[str, str]]:
     """
-    Return list of ECR images from all accounts in the graph.
+    Return tag URIs and a mapping of digest-qualified URIs to tag URIs.
     """
     if not account_ids:
         aws_accounts = list_accounts(neo4j_session)
     else:
         aws_accounts = account_ids
 
-    ecr_images: set[str] = set()
+    image_uris: set[str] = set()
+    digest_aliases: dict[str, str] = {}
+
     for account_id in aws_accounts:
-        for _, _, image_uri, _, _ in get_ecr_images(neo4j_session, account_id):
-            ecr_images.add(image_uri)
+        for _, _, image_uri, _, digest in get_ecr_images(neo4j_session, account_id):
+            if not image_uri:
+                continue
+            image_uris.add(image_uri)
+            if digest:
+                # repo URI is everything before the trailing ":" (if present)
+                repo_uri = image_uri.rsplit(":", 1)[0]
+                digest_uri = f"{repo_uri}@{digest}"
+                digest_aliases[digest_uri] = image_uri
 
-    return ecr_images
+    return image_uris, digest_aliases
 
 
-def _get_intersection(
-    image_uris: set[str], json_files: set[str], trivy_s3_prefix: str
-) -> list[tuple[str, str]]:
+@timeit
+def get_scan_targets(
+    neo4j_session: Session,
+    account_ids: list[str] | None = None,
+) -> set[str]:
+    """
+    Return list of ECR images from all accounts in the graph.
     """
-    Get the intersection of ECR images in the graph and S3 scan results.
+    image_uris, _ = _get_scan_targets_and_aliases(neo4j_session, account_ids)
+    return image_uris
 
-    Args:
-        image_uris: Set of ECR images in the graph
-        json_files: Set of S3 object keys for JSON files
-        trivy_s3_prefix: S3 prefix path containing scan results
 
-    Returns:
-        List of tuples (image_uri, s3_object_key)
+def _prepare_trivy_data(
+    trivy_data: dict[str, Any],
+    image_uris: set[str],
+    digest_aliases: dict[str, str],
+    source: str,
+) -> tuple[dict[str, Any], str] | None:
     """
-    intersection = []
-    prefix_len = len(trivy_s3_prefix)
-    for s3_object_key in json_files:
-        # Sample key "123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
-        # Sample key "folder/derp/123456789012.dkr.ecr.us-west-2.amazonaws.com/other-repo:v1.0.json"
-        # Remove the prefix and the .json suffix
-        image_uri = s3_object_key[prefix_len:-5]
+    Determine the tag URI that corresponds to this Trivy payload.
+
+    Returns (trivy_data, display_uri) if the payload can be linked to an image present
+    in the graph; otherwise returns None so the caller can skip ingestion.
+    """
+
+    artifact_name = (trivy_data.get("ArtifactName") or "").strip()
+    metadata = trivy_data.get("Metadata") or {}
+    candidates: list[str] = []
+
+    if artifact_name:
+        candidates.append(artifact_name)
 
-        if image_uri in image_uris:
-            intersection.append((image_uri, s3_object_key))
+    repo_tags = metadata.get("RepoTags", [])
+    repo_digests = metadata.get("RepoDigests", [])
+    stripped_tags_digests = [item.strip() for item in repo_tags + repo_digests]
+    candidates.extend(stripped_tags_digests)
 
-    return intersection
+    display_uri: str | None = None
+
+    for candidate in candidates:
+        if not candidate:
+            continue
+        if candidate in image_uris:
+            display_uri = candidate
+            break
+        alias = digest_aliases.get(candidate)
+        if alias:
+            display_uri = alias
+            break
+
+    if not display_uri:
+        logger.debug(
+            "Skipping Trivy results for %s because no matching image URI was found in the graph",
+            source,
+        )
+        return None
+
+    return trivy_data, display_uri
 
 
 @timeit
@@ -93,15 +132,12 @@ def sync_trivy_aws_ecr_from_s3(
         f"Using Trivy scan results from s3://{trivy_s3_bucket}/{trivy_s3_prefix}"
     )
 
-    image_uris: set[str] = get_scan_targets(neo4j_session)
+    image_uris, digest_aliases = _get_scan_targets_and_aliases(neo4j_session)
     json_files: set[str] = get_json_files_in_s3(
         trivy_s3_bucket, trivy_s3_prefix, boto3_session
     )
-    intersection: list[tuple[str, str]] = _get_intersection(
-        image_uris, json_files, trivy_s3_prefix
-    )
 
-    if len(intersection) == 0:
+    if len(json_files) == 0:
         logger.error(
             f"Trivy sync was configured, but there are no ECR images with S3 json scan results in bucket "
             f"'{trivy_s3_bucket}' with prefix '{trivy_s3_prefix}'. "
@@ -110,18 +146,33 @@ def sync_trivy_aws_ecr_from_s3(
             f"`<image_uri>.json` and to be in the same bucket and prefix as the scan results. If the prefix is "
             "a folder, it MUST end with a trailing slash '/'. "
         )
-        logger.error(f"JSON files in S3: {json_files}")
         raise ValueError("No ECR images with S3 json scan results found.")
 
-    logger.info(f"Processing {len(intersection)} ECR images with S3 scan results")
-    for image_uri, s3_object_key in intersection:
-        sync_single_image_from_s3(
+    logger.info(f"Processing {len(json_files)} Trivy result files from S3")
+    s3_client = boto3_session.client("s3")
+    for s3_object_key in json_files:
+        logger.debug(
+            f"Reading scan results from S3: s3://{trivy_s3_bucket}/{s3_object_key}"
+        )
+        response = s3_client.get_object(Bucket=trivy_s3_bucket, Key=s3_object_key)
+        scan_data_json = response["Body"].read().decode("utf-8")
+        trivy_data = json.loads(scan_data_json)
+
+        prepared = _prepare_trivy_data(
+            trivy_data,
+            image_uris=image_uris,
+            digest_aliases=digest_aliases,
+            source=f"s3://{trivy_s3_bucket}/{s3_object_key}",
+        )
+        if prepared is None:
+            continue
+
+        prepared_data, display_uri = prepared
+        sync_single_image(
             neo4j_session,
-            image_uri,
+            prepared_data,
+            display_uri,
             update_tag,
-            trivy_s3_bucket,
-            s3_object_key,
-            boto3_session,
         )
 
     cleanup(neo4j_session, common_job_parameters)
@@ -137,7 +188,7 @@ def sync_trivy_aws_ecr_from_dir(
     """Sync Trivy scan results from local files for AWS ECR images."""
     logger.info(f"Using Trivy scan results from {results_dir}")
 
-    image_uris: set[str] = get_scan_targets(neo4j_session)
+    image_uris, digest_aliases = _get_scan_targets_and_aliases(neo4j_session)
     json_files: set[str] = get_json_files_in_dir(results_dir)
 
     if not json_files:
@@ -149,27 +200,27 @@ def sync_trivy_aws_ecr_from_dir(
     logger.info(f"Processing {len(json_files)} local Trivy result files")
 
     for file_path in json_files:
-        # First, check if the image exists in the graph before syncing
         try:
-            # Peek at the artifact name without processing the file
             with open(file_path, encoding="utf-8") as f:
                 trivy_data = json.load(f)
-                artifact_name = trivy_data.get("ArtifactName")
-
-            if artifact_name and artifact_name not in image_uris:
-                logger.debug(
-                    f"Skipping results for {artifact_name} since the image is not present in the graph"
-                )
-                continue
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to read Trivy data from {file_path}: {e}")
+            continue
 
-        except (json.JSONDecodeError, KeyError) as e:
-            logger.error(f"Failed to read artifact name from {file_path}: {e}")
+        prepared = _prepare_trivy_data(
+            trivy_data,
+            image_uris=image_uris,
+            digest_aliases=digest_aliases,
+            source=file_path,
+        )
+        if prepared is None:
             continue
 
-        # Now sync the file since we know the image exists in the graph
-        sync_single_image_from_file(
+        prepared_data, display_uri = prepared
+        sync_single_image(
             neo4j_session,
-            file_path,
+            prepared_data,
+            display_uri,
             update_tag,
         )
 

cartography/models/aws/ec2/networkinterfaces.py

@@ -47,6 +47,8 @@ class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties):
     # TODO: remove subnetid once we have migrated to subnet_id
     subnetid: PropertyRef = PropertyRef("SubnetId", extra_index=True)
     subnet_id: PropertyRef = PropertyRef("SubnetId", extra_index=True)
+    attach_time: PropertyRef = PropertyRef("AttachTime")
+    device_index: PropertyRef = PropertyRef("DeviceIndex")
 
 
 @dataclass(frozen=True)

cartography/models/aws/ecr/image.py

@@ -18,6 +18,14 @@ class ECRImageNodeProperties(CartographyNodeProperties):
     region: PropertyRef = PropertyRef("Region", set_in_kwargs=True)
     lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
     layer_diff_ids: PropertyRef = PropertyRef("layer_diff_ids")
+    type: PropertyRef = PropertyRef("type")
+    architecture: PropertyRef = PropertyRef("architecture")
+    os: PropertyRef = PropertyRef("os")
+    variant: PropertyRef = PropertyRef("variant")
+    attestation_type: PropertyRef = PropertyRef("attestation_type")
+    attests_digest: PropertyRef = PropertyRef("attests_digest")
+    media_type: PropertyRef = PropertyRef("media_type")
+    artifact_media_type: PropertyRef = PropertyRef("artifact_media_type")
 
 
 @dataclass(frozen=True)

cartography/models/aws/ecr/repository_image.py

@@ -71,7 +71,7 @@ class ECRRepositoryImageToECRImageRelProperties(CartographyRelProperties):
 class ECRRepositoryImageToECRImageRel(CartographyRelSchema):
     target_node_label: str = "ECRImage"
     target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
-        {"id": PropertyRef("imageDigest")}
+        {"id": PropertyRef("imageDigests", one_to_many=True)}
     )
     direction: LinkDirection = LinkDirection.OUTWARD
     rel_label: str = "IMAGE"

cartography/sync.py

@@ -52,7 +52,7 @@ from cartography.util import STATUS_SUCCESS
 logger = logging.getLogger(__name__)
 
 
-TOP_LEVEL_MODULES = OrderedDict(
+TOP_LEVEL_MODULES: OrderedDict[str, Callable[..., None]] = OrderedDict(
     {  # preserve order so that the default sync always runs `analysis` at the very end
         "create-indexes": cartography.intel.create_indexes.run,
         "airbyte": cartography.intel.airbyte.start_airbyte_ingestion,

cartography/util.py

@@ -153,6 +153,9 @@ def merge_module_sync_metadata(
     :param synced_type: The sub-module's type
     :param update_tag: Timestamp used to determine data freshness
     """
+    # Import here to avoid circular import with cartography.client.core.tx
+    from cartography.client.core.tx import run_write_query
+
     template = Template(
         """
         MERGE (n:ModuleSyncMetadata{id:'${group_type}_${group_id}_${synced_type}'})
@@ -164,7 +167,8 @@ def merge_module_sync_metadata(
             n.lastupdated=$UPDATE_TAG
     """,
     )
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         template.safe_substitute(
             group_type=group_type,
             group_id=group_id,

{cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.117.0
+Version: 0.118.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License-Expression: Apache-2.0
@@ -59,7 +59,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: xmltodict
 Requires-Dist: duo-client
 Requires-Dist: cloudflare<5.0.0,>=4.1.0
-Requires-Dist: scaleway>=2.9.0
+Requires-Dist: scaleway>=2.10.0
 Requires-Dist: google-cloud-resource-manager>=1.14.2
 Requires-Dist: types-aiobotocore-ecr
 Requires-Dist: typer>=0.9.0
@@ -89,7 +89,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 ## Supported platforms
 - [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including image layers), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, Cognito, EC2, ECS, ECR (including multi-arch images, image layers, and attestations), EFS, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, Glue,  GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SNS, SQS, SSM, STS, Tags
 - [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
 - [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord