cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/redshift.py

@@ -5,6 +5,7 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -88,7 +89,8 @@ def load_redshift_cluster_data(
     SET r.lastupdated = $aws_update_tag
     """
     for cluster in clusters:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_cluster,
             Arn=cluster["arn"],
             AZ=cluster["AvailabilityZone"],
@@ -128,7 +130,8 @@ def _attach_ec2_security_groups(
     SET m.lastupdated = $aws_update_tag
     """
     for group in cluster.get("VpcSecurityGroups", []):
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_cluster_to_group,
             ClusterArn=cluster["arn"],
             GroupId=group["VpcSecurityGroupId"],
@@ -150,7 +153,8 @@ def _attach_iam_roles(
     SET s.lastupdated = $aws_update_tag
     """
     for role in cluster.get("IamRoles", []):
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_cluster_to_role,
             ClusterArn=cluster["arn"],
             RoleArn=role["IamRoleArn"],
@@ -172,7 +176,8 @@ def _attach_aws_vpc(
     SET m.lastupdated = $aws_update_tag
     """
     if cluster.get("VpcId"):
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             attach_cluster_to_vpc,
             ClusterArn=cluster["arn"],
             VpcId=cluster["VpcId"],

cartography/intel/aws/resourcegroupstaggingapi.py

@@ -1,5 +1,6 @@
 import logging
 from string import Template
+from typing import Any
 from typing import Dict
 from typing import List
 
@@ -56,6 +57,35 @@ def get_short_id_from_lb2_arn(alb_arn: str) -> str:
     return alb_arn.split("/")[-2]
 
 
+def get_resource_type_from_arn(arn: str) -> str:
+    """Return the resource type format expected by the Tagging API.
+
+    The Resource Groups Tagging API requires resource types in the form
+    ``service:resource``. Most ARNs embed the resource type in the fifth segment
+    after the service name. Load balancer ARNs add an extra ``app`` or ``net``
+    component that must be preserved. S3 and SQS ARNs only contain the service
+    name.  This helper extracts the appropriate string so that ARNs can be
+    grouped correctly for API calls.
+    """
+
+    parts = arn.split(":", 5)
+    service = parts[2]
+    if service in {"s3", "sqs"}:
+        return service
+
+    resource = parts[5]
+    if service == "elasticloadbalancing" and resource.startswith("loadbalancer/"):
+        segments = resource.split("/")
+        if len(segments) > 2 and segments[1] in {"app", "net"}:
+            resource_type = f"{segments[0]}/{segments[1]}"
+        else:
+            resource_type = segments[0]
+    else:
+        resource_type = resource.split("/")[0].split(":")[0]
+
+    return f"{service}:{resource_type}" if resource_type else service
+
+
 # We maintain a mapping from AWS resource types to their associated labels and unique identifiers.
 # label: the node label used in cartography for this resource type
 # property: the field of this node that uniquely identified this resource type
@@ -158,27 +188,27 @@ TAG_RESOURCE_TYPE_MAPPINGS: Dict = {
 @aws_handle_regions
 def get_tags(
     boto3_session: boto3.session.Session,
-    resource_type: str,
+    resource_types: list[str],
     region: str,
-) -> List[Dict]:
-    """
-    Create boto3 client and retrieve tag data.
-    """
-    # this is a temporary workaround to populate AWS tags for IAM roles.
-    # resourcegroupstaggingapi does not support IAM roles and no ETA is provided
-    # TODO: when resourcegroupstaggingapi supports iam:role, remove this condition block
-    if resource_type == "iam:role":
-        return get_role_tags(boto3_session)
+) -> list[dict[str, Any]]:
+    """Retrieve tag data for the provided resource types."""
+    resources: list[dict[str, Any]] = []
+
+    if "iam:role" in resource_types:
+        resources.extend(get_role_tags(boto3_session))
+        resource_types = [rt for rt in resource_types if rt != "iam:role"]
+
+    if not resource_types:
+        return resources
 
     client = boto3_session.client("resourcegroupstaggingapi", region_name=region)
     paginator = client.get_paginator("get_resources")
-    resources: List[Dict] = []
-    for page in paginator.paginate(
-        # Only ingest tags for resources that Cartography supports.
-        # This is just a starting list; there may be others supported by this API.
-        ResourceTypeFilters=[resource_type],
-    ):
-        resources.extend(page["ResourceTagMappingList"])
+
+    # Batch resource types into groups of 100
+    # (https://docs.aws.amazon.com/resourcegroupstagging/latest/APIReference/API_GetResources.html)
+    for resource_types_batch in batch(resource_types, size=100):
+        for page in paginator.paginate(ResourceTypeFilters=resource_types_batch):
+            resources.extend(page["ResourceTagMappingList"])
     return resources
 
 
@@ -210,6 +240,9 @@ def _load_tags_tx(
             r.firstseen = timestamp()
     """,
     )
+    if not tag_data:
+        return
+
     query = INGEST_TAG_TEMPLATE.safe_substitute(
         resource_label=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]["label"],
         property=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]["property"],
@@ -220,7 +253,7 @@ def _load_tags_tx(
         UpdateTag=aws_update_tag,
         Region=region,
         Account=current_aws_account_id,
-    )
+    ).consume()
 
 
 @timeit
@@ -262,6 +295,26 @@ def compute_resource_id(tag_mapping: Dict, resource_type: str) -> str:
     return resource_id
 
 
+def _group_tag_data_by_resource_type(
+    tag_data: List[Dict],
+    tag_resource_type_mappings: Dict,
+) -> Dict[str, List[Dict]]:
+    """Group raw tag data by the resource types Cartography supports."""
+
+    grouped: Dict[str, List[Dict]] = {rtype: [] for rtype in tag_resource_type_mappings}
+    for mapping in tag_data:
+        rtype = get_resource_type_from_arn(mapping["ResourceARN"])
+        if rtype in grouped:
+            grouped[rtype].append(mapping)
+        else:
+            logger.debug(
+                "Unknown tag resource type %s from ARN %s",
+                rtype,
+                mapping["ResourceARN"],
+            )
+    return grouped
+
+
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     run_cleanup_job(
@@ -285,8 +338,14 @@ def sync(
         logger.info(
             f"Syncing AWS tags for account {current_aws_account_id} and region {region}",
         )
+        all_tag_data = get_tags(
+            boto3_session, list(tag_resource_type_mappings.keys()), region
+        )
+        grouped = _group_tag_data_by_resource_type(
+            all_tag_data, tag_resource_type_mappings
+        )
         for resource_type in tag_resource_type_mappings.keys():
-            tag_data = get_tags(boto3_session, resource_type, region)
+            tag_data = grouped.get(resource_type, [])
             transform_tags(tag_data, resource_type)  # type: ignore
             logger.info(
                 f"Loading {len(tag_data)} tags for resource type {resource_type}",

cartography/intel/aws/resources.py

@@ -3,18 +3,27 @@ from typing import Dict
 
 from cartography.intel.aws.ec2.route_tables import sync_route_tables
 
+from . import acm
 from . import apigateway
+from . import apigatewayv2
 from . import cloudtrail
+from . import cloudtrail_management_events
 from . import cloudwatch
+from . import codebuild
+from . import cognito
 from . import config
 from . import dynamodb
 from . import ecr
+from . import ecr_image_layers
 from . import ecs
 from . import efs
 from . import eks
 from . import elasticache
 from . import elasticsearch
 from . import emr
+from . import eventbridge
+from . import glue
+from . import guardduty
 from . import iam
 from . import identitycenter
 from . import inspector
@@ -80,6 +89,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "ec2:volumes": sync_ebs_volumes,
     "ec2:snapshots": sync_ebs_snapshots,
     "ecr": ecr.sync,
+    "ecr:image_layers": ecr_image_layers.sync,
     "ecs": ecs.sync,
     "eks": eks.sync,
     "elasticache": elasticache.sync,
@@ -94,16 +104,24 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "permission_relationships": permission_relationships.sync,
     "resourcegroupstaggingapi": resourcegroupstaggingapi.sync,
     "apigateway": apigateway.sync,
+    "apigatewayv2": apigatewayv2.sync,
     "secretsmanager": secretsmanager.sync,
     "securityhub": securityhub.sync,
     "s3accountpublicaccessblock": s3accountpublicaccessblock.sync,
     "sns": sns.sync,
     "sqs": sqs.sync,
     "ssm": ssm.sync,
+    "acm:certificate": acm.sync,
     "inspector": inspector.sync,
     "config": config.sync,
     "identitycenter": identitycenter.sync_identity_center_instances,
     "cloudtrail": cloudtrail.sync,
+    "cloudtrail_management_events": cloudtrail_management_events.sync,
     "cloudwatch": cloudwatch.sync,
     "efs": efs.sync,
+    "guardduty": guardduty.sync,
+    "codebuild": codebuild.sync,
+    "cognito": cognito.sync,
+    "eventbridge": eventbridge.sync,
+    "glue": glue.sync,
 }