cartography 0.108.0rc2__py3-none-any.whl → 0.109.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.108.0rc2'
-__version_tuple__ = version_tuple = (0, 108, 0, 'rc2')
+__version__ = version = '0.109.0rc2'
+__version_tuple__ = version_tuple = (0, 109, 0, 'rc2')

cartography/data/indexes.cypher

@@ -29,14 +29,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambda) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaEventSourceMapping) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaFunctionAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:AWSLambdaLayer) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.id);
@@ -158,13 +150,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.arn);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSAlias) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);
@@ -259,8 +244,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:S3Bucket) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:SecretsManagerSecret) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:SecretsManagerSecret) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SecurityHub) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:SecurityHub) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.id);

cartography/data/jobs/cleanup/gcp_compute_vpc_cleanup.json

@@ -1,17 +1,5 @@
 {
   "statements": [
-    {
-      "query": "MATCH (n:GCPVpc) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
-      "iterative": true,
-      "iterationsize": 100,
-      "__comment__": "Delete GCP VPCs that no longer exist and detach them from all previously connected nodes."
-    },
-    {
-      "query": "MATCH (:GCPVpc)<-[r:RESOURCE]-(:GCPProject) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-      "iterative": true,
-      "iterationsize": 100,
-      "__comment__": "Remove GCP VPC-to-Project relationships that are out of date."
-    },
     {
       "query": "MATCH (:GCPInstance)-[r:MEMBER_OF_GCP_VPC]->(:GCPVpc) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
       "iterative": true,

cartography/intel/aws/cloudtrail_management_events.py

@@ -381,13 +381,15 @@ def transform_web_identity_role_events_to_role_assumptions(
 
             # Only process GitHub Actions events
             if "token.actions.githubusercontent.com" in identity_provider:
-                # GitHub repo fullname is directly in userName (e.g., "sublimagesec/sublimage")
-                github_repo = user_identity.get("userName", "")
-                if not github_repo:
+                # Extract GitHub repo fullname from userName format: "repo:{organization}/{repository}:{context}"
+                user_name = user_identity.get("userName", "")
+                if not user_name:
                     logger.debug(
                         f"Missing userName in GitHub WebIdentity event: {event.get('EventId', 'unknown')}"
                     )
                     continue
+
+                github_repo = _extract_github_repo_from_username(user_name)
                 key = (github_repo, destination_principal)
 
                 if key in github_aggregated:
@@ -572,6 +574,37 @@ def _convert_assumed_role_arn_to_role_arn(assumed_role_arn: str) -> str:
     return assumed_role_arn
 
 
+def _extract_github_repo_from_username(user_name: str) -> str:
+    """
+    Extract GitHub repository fullname from CloudTrail userName field.
+
+    GitHub Actions CloudTrail events have userName in the format:
+    "repo:{organization}/{repository}:{context}"
+    """
+    if not user_name:
+        return ""
+
+    parts = user_name.split(":")
+
+    # Need at least 3 parts: ["repo", "{organization}/{repository}", "{context}"]
+    if len(parts) < 3 or parts[0] != "repo":
+        return ""
+
+    # Extract "{organization}/{repository}"
+    repo_fullname = parts[1]
+
+    # Validate it looks like "{organization}/{repository}" format
+    if repo_fullname.count("/") != 1:
+        return ""
+
+    # Ensure both organization and repo exist
+    owner, repo = repo_fullname.split("/")
+    if not owner or not repo:
+        return ""
+
+    return repo_fullname
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session, current_aws_account_id: str, update_tag: int

cartography/intel/aws/ecr.py

@@ -6,9 +6,12 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ecr.image import ECRImageSchema
+from cartography.models.aws.ecr.repository import ECRRepositorySchema
+from cartography.models.aws.ecr.repository_image import ECRRepositoryImageSchema
 from cartography.util import aws_handle_regions
-from cartography.util import batch
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 from cartography.util import to_asynchronous
 from cartography.util import to_synchronous
@@ -74,33 +77,17 @@ def load_ecr_repositories(
     current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
-    query = """
-    UNWIND $Repositories as ecr_repo
-        MERGE (repo:ECRRepository{id: ecr_repo.repositoryArn})
-        ON CREATE SET repo.firstseen = timestamp(),
-            repo.arn = ecr_repo.repositoryArn,
-            repo.name = ecr_repo.repositoryName,
-            repo.region = $Region,
-            repo.created_at = ecr_repo.createdAt
-        SET repo.lastupdated = $aws_update_tag,
-            repo.uri = ecr_repo.repositoryUri
-        WITH repo
-
-        MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (owner)-[r:RESOURCE]->(repo)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
     logger.info(
         f"Loading {len(repos)} ECR repositories for region {region} into graph.",
     )
-    neo4j_session.run(
-        query,
-        Repositories=repos,
+    load(
+        neo4j_session,
+        ECRRepositorySchema(),
+        repos,
+        lastupdated=aws_update_tag,
         Region=region,
-        aws_update_tag=aws_update_tag,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-    ).consume()  # See issue #440
+        AWS_ID=current_aws_account_id,
+    )
 
 
 @timeit
@@ -114,8 +101,13 @@ def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
     for repo_uri in sorted(repo_data.keys()):
         repo_images = repo_data[repo_uri]
         for img in repo_images:
-            if "imageDigest" in img and img["imageDigest"]:
+            digest = img.get("imageDigest")
+            if digest:
+                tag = img.get("imageTag")
+                uri = repo_uri + (f":{tag}" if tag else "")
                 img["repo_uri"] = repo_uri
+                img["uri"] = uri
+                img["id"] = uri
                 repo_images_list.append(img)
             else:
                 logger.warning(
@@ -127,74 +119,51 @@ def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
     return repo_images_list
 
 
-def _load_ecr_repo_img_tx(
-    tx: neo4j.Transaction,
-    repo_images_list: List[Dict],
-    aws_update_tag: int,
-    region: str,
-) -> None:
-    query = """
-    UNWIND $RepoList as repo_img
-        MERGE (ri:ECRRepositoryImage{id: repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')})
-        ON CREATE SET ri.firstseen = timestamp()
-        SET ri.lastupdated = $aws_update_tag,
-            ri.tag = repo_img.imageTag,
-            ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, ''),
-            ri.image_size_bytes = repo_img.imageSizeInBytes,
-            ri.image_pushed_at = repo_img.imagePushedAt,
-            ri.image_manifest_media_type = repo_img.imageManifestMediaType,
-            ri.artifact_media_type = repo_img.artifactMediaType,
-            ri.last_recorded_pull_time = repo_img.lastRecordedPullTime
-        WITH ri, repo_img
-
-        MERGE (img:ECRImage{id: repo_img.imageDigest})
-        ON CREATE SET img.firstseen = timestamp(),
-            img.digest = repo_img.imageDigest
-        SET img.lastupdated = $aws_update_tag,
-            img.region = $Region
-        WITH ri, img, repo_img
-
-        MERGE (ri)-[r1:IMAGE]->(img)
-        ON CREATE SET r1.firstseen = timestamp()
-        SET r1.lastupdated = $aws_update_tag
-        WITH ri, repo_img
-
-        MATCH (repo:ECRRepository{uri: repo_img.repo_uri})
-        MERGE (repo)-[r2:REPO_IMAGE]->(ri)
-        ON CREATE SET r2.firstseen = timestamp()
-        SET r2.lastupdated = $aws_update_tag
-    """
-    tx.run(
-        query,
-        RepoList=repo_images_list,
-        Region=region,
-        aws_update_tag=aws_update_tag,
-    )
-
-
 @timeit
 def load_ecr_repository_images(
     neo4j_session: neo4j.Session,
     repo_images_list: List[Dict],
     region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     logger.info(
         f"Loading {len(repo_images_list)} ECR repository images in {region} into graph.",
     )
-    for repo_image_batch in batch(repo_images_list, size=10000):
-        neo4j_session.write_transaction(
-            _load_ecr_repo_img_tx,
-            repo_image_batch,
-            aws_update_tag,
-            region,
-        )
+    image_digests = {img["imageDigest"] for img in repo_images_list}
+    ecr_images = [{"imageDigest": d} for d in image_digests]
+
+    load(
+        neo4j_session,
+        ECRImageSchema(),
+        ecr_images,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+    load(
+        neo4j_session,
+        ECRRepositoryImageSchema(),
+        repo_images_list,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
 
 
 @timeit
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     logger.debug("Running ECR cleanup job.")
-    run_cleanup_job("aws_import_ecr_cleanup.json", neo4j_session, common_job_parameters)
+    GraphJob.from_node_schema(ECRRepositorySchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(ECRRepositoryImageSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(ECRImageSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 def _get_image_data(
@@ -251,5 +220,11 @@ def sync(
             update_tag,
         )
         repo_images_list = transform_ecr_repository_images(image_data)
-        load_ecr_repository_images(neo4j_session, repo_images_list, region, update_tag)
+        load_ecr_repository_images(
+            neo4j_session,
+            repo_images_list,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/glue.py

@@ -0,0 +1,117 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_glue_connections(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_connections")
+    connections = []
+    for page in paginator.paginate():
+        connections.extend(page.get("ConnectionList", []))
+
+    return connections
+
+
+def transform_glue_connections(
+    connections: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform Glue connection data for ingestion
+    """
+    transformed_connections = []
+    for connection in connections:
+        transformed_connection = {
+            "Name": connection["Name"],
+            "Description": connection.get("Description"),
+            "ConnectionType": connection.get("ConnectionType"),
+            "Status": connection.get("Status"),
+            "StatusReason": connection.get("StatusReason"),
+            "AuthenticationType": connection.get("AuthenticationConfiguration", {}).get(
+                "AuthenticationType"
+            ),
+            "SecretArn": connection.get("AuthenticationConfiguration", {}).get(
+                "SecretArn"
+            ),
+            "Region": region,
+        }
+        transformed_connections.append(transformed_connection)
+    return transformed_connections
+
+
+@timeit
+def load_glue_connections(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} connections for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueConnectionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Glue cleanup job.")
+    GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Glue for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        connections = get_glue_connections(boto3_session, region)
+        transformed_connections = transform_glue_connections(connections, region)
+        load_glue_connections(
+            neo4j_session,
+            transformed_connections,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/identitycenter.py

@@ -7,6 +7,7 @@ import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
 from cartography.graph.job import GraphJob
 from cartography.models.aws.identitycenter.awsidentitycenter import (
     AWSIdentityCenterInstanceSchema,
@@ -14,9 +15,11 @@ from cartography.models.aws.identitycenter.awsidentitycenter import (
 from cartography.models.aws.identitycenter.awspermissionset import (
     AWSPermissionSetSchema,
 )
+from cartography.models.aws.identitycenter.awspermissionset import (
+    RoleAssignmentAllowedByMatchLink,
+)
 from cartography.models.aws.identitycenter.awsssouser import AWSSSOUserSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -120,6 +123,8 @@ def load_permission_sets(
         InstanceArn=instance_arn,
         Region=region,
         AWS_ID=aws_account_id,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=aws_account_id,
     )
 
 
@@ -220,31 +225,64 @@ def get_role_assignments(
     return role_assignments
 
 
+@timeit
+def get_permset_roles(
+    neo4j_session: neo4j.Session,
+    role_assignments: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """
+    Enrich role assignments with exact role ARNs by querying existing permission set relationships.
+    Uses the ASSIGNED_TO_ROLE relationships created when permission sets were loaded.
+    """
+    # Get unique permission set ARNs from role assignments
+    permset_ids = list({ra["PermissionSetArn"] for ra in role_assignments})
+
+    query = """
+    MATCH (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-(permset:AWSPermissionSet)
+    WHERE permset.arn IN $PermSetIds
+    RETURN permset.arn AS PermissionSetArn, role.arn AS RoleArn
+    """
+    result = neo4j_session.run(query, PermSetIds=permset_ids)
+    permset_to_role = [record.data() for record in result]
+
+    # Create mapping from permission set ARN to role ARN
+    permset_to_role_map = {
+        entry["PermissionSetArn"]: entry["RoleArn"] for entry in permset_to_role
+    }
+
+    # Enrich role assignments with exact role ARNs
+    enriched_assignments = []
+    for assignment in role_assignments:
+        role_arn = permset_to_role_map.get(assignment["PermissionSetArn"])
+        enriched_assignments.append(
+            {
+                **assignment,
+                "RoleArn": role_arn,
+            }
+        )
+
+    return enriched_assignments
+
+
 @timeit
 def load_role_assignments(
     neo4j_session: neo4j.Session,
     role_assignments: List[Dict],
+    aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
     """
-    Load role assignments into the graph
+    Load role assignments into the graph using MatchLink schema
     """
     logger.info(f"Loading {len(role_assignments)} role assignments")
-    if role_assignments:
-        neo4j_session.run(
-            """
-            UNWIND $role_assignments AS ra
-            MATCH (acc:AWSAccount{id:ra.AccountId}) -[:RESOURCE]->
-            (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-
-            (permset:AWSPermissionSet {id: ra.PermissionSetArn})
-            MATCH (sso:AWSSSOUser {id: ra.UserId})
-            MERGE (role)-[r:ALLOWED_BY]->(sso)
-            SET r.lastupdated = $aws_update_tag,
-            r.permission_set_arn = ra.PermissionSetArn
-            """,
-            role_assignments=role_assignments,
-            aws_update_tag=aws_update_tag,
-        )
+    load_matchlinks(
+        neo4j_session,
+        RoleAssignmentAllowedByMatchLink(),
+        role_assignments,
+        lastupdated=aws_update_tag,
+        _sub_resource_label="AWSAccount",
+        _sub_resource_id=aws_account_id,
+    )
 
 
 @timeit
@@ -262,11 +300,14 @@ def cleanup(
     GraphJob.from_node_schema(AWSSSOUserSchema(), common_job_parameters).run(
         neo4j_session,
     )
-    run_cleanup_job(
-        "aws_import_identity_center_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
-    )
+
+    # Clean up role assignment MatchLinks
+    GraphJob.from_matchlink(
+        RoleAssignmentAllowedByMatchLink(),
+        "AWSAccount",
+        common_job_parameters["AWS_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(neo4j_session)
 
 
 @timeit
@@ -327,9 +368,16 @@ def sync_identity_center_instances(
                 instance_arn,
                 region,
             )
-            load_role_assignments(
+
+            # Enrich role assignments with exact role ARNs using permission set relationships
+            enriched_role_assignments = get_permset_roles(
                 neo4j_session,
                 role_assignments,
+            )
+            load_role_assignments(
+                neo4j_session,
+                enriched_role_assignments,
+                current_aws_account_id,
                 update_tag,
             )