cartography 0.110.0rc1__py3-none-any.whl → 0.110.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.110.0rc1'
-__version_tuple__ = version_tuple = (0, 110, 0, 'rc1')
+__version__ = version = '0.110.0rc2'
+__version_tuple__ = version_tuple = (0, 110, 0, 'rc2')

cartography/cli.py

@@ -254,14 +254,6 @@ class CLI:
                 "The name of environment variable containing Entra Client Secret for Service Principal Authentication."
             ),
         )
-        parser.add_argument(
-            "--entra-best-effort-mode",
-            action="store_true",
-            help=(
-                "Enable Entra ID sync best effort mode. This will allow cartography to continue "
-                "syncing other Entra ID entities and delay raising an exception until the end of the sync."
-            ),
-        )
         parser.add_argument(
             "--aws-requested-syncs",
             type=str,

cartography/config.py

@@ -51,9 +51,6 @@ class Config:
     :param entra_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
     :type entra_client_secret: str
     :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
-    :type entra_best_effort_mode: bool
-    :param entra_best_effort_mode: If True, Entra ID sync will continue on errors and raise an aggregated
-        exception at the end of the sync. If False (default), exceptions will be raised immediately.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
     :type aws_guardduty_severity_threshold: str
@@ -167,8 +164,6 @@ class Config:
     :param sentinelone_api_url: SentinelOne API URL. Optional.
     :type sentinelone_api_token: string
     :param sentinelone_api_token: SentinelOne API token for authentication. Optional.
-    :type sentinelone_api_token_env_var: string
-    :param sentinelone_api_token_env_var: The name of an environment variable containing the SentinelOne API token. Optional.
     :type sentinelone_account_ids: list[str]
     :param sentinelone_account_ids: List of SentinelOne account IDs to sync. Optional.
     """
@@ -194,7 +189,6 @@ class Config:
         entra_tenant_id=None,
         entra_client_id=None,
         entra_client_secret=None,
-        entra_best_effort_mode=False,
         aws_requested_syncs=None,
         aws_guardduty_severity_threshold=None,
         analysis_job_directory=None,
@@ -257,7 +251,6 @@ class Config:
         scaleway_org=None,
         sentinelone_api_url=None,
         sentinelone_api_token=None,
-        sentinelone_api_token_env_var=None,
         sentinelone_account_ids=None,
     ):
         self.neo4j_uri = neo4j_uri
@@ -281,7 +274,6 @@ class Config:
         self.entra_tenant_id = entra_tenant_id
         self.entra_client_id = entra_client_id
         self.entra_client_secret = entra_client_secret
-        self.entra_best_effort_mode = entra_best_effort_mode
         self.aws_requested_syncs = aws_requested_syncs
         self.aws_guardduty_severity_threshold = aws_guardduty_severity_threshold
         self.analysis_job_directory = analysis_job_directory
@@ -344,5 +336,4 @@ class Config:
         self.scaleway_org = scaleway_org
         self.sentinelone_api_url = sentinelone_api_url
         self.sentinelone_api_token = sentinelone_api_token
-        self.sentinelone_api_token_env_var = sentinelone_api_token_env_var
         self.sentinelone_account_ids = sentinelone_account_ids

cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json

@@ -22,8 +22,8 @@
             "iterative": false
         },
         {
-            "__comment__": "Attach EC2KeyPairs with matching fingerprints to eachother and set duplicate_keyfingerprint = True",
-            "query": "MATCH (k1:EC2KeyPair), (k2:EC2KeyPair) WHERE k1.id <> k2.id AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG return COUNT(*) as TotalCompleted",
+            "__comment__": "Attach EC2KeyPairs with matching fingerprints to each other and set duplicate_keyfingerprint = True. Use id(k1) < id(k2) to avoid Cartesian product warning and ensure O(1) comparison.",
+            "query": "MATCH (k1:EC2KeyPair) MATCH (k2:EC2KeyPair) WHERE id(k1) < id(k2) AND k1.keyfingerprint = k2.keyfingerprint SET k1.duplicate_keyfingerprint = True, k2.duplicate_keyfingerprint = True MERGE (k1)-[r:MATCHING_FINGERPRINT]-(k2) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG RETURN COUNT(*) as TotalCompleted",
             "iterative": false
         }
     ]

cartography/intel/aws/cognito.py

@@ -0,0 +1,201 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cognito.identity_pool import CognitoIdentityPoolSchema
+from cartography.models.aws.cognito.user_pool import CognitoUserPoolSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pools(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_identity_pools")
+
+    all_identity_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        identity_pools = page.get("IdentityPools", [])
+        all_identity_pools.extend(identity_pools)
+    return all_identity_pools
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pool_roles(
+    boto3_session: boto3.Session, identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    all_identity_pool_details = []
+    for identity_pool in identity_pools:
+        response = client.get_identity_pool_roles(
+            IdentityPoolId=identity_pool["IdentityPoolId"]
+        )
+        all_identity_pool_details.append(response)
+    return all_identity_pool_details
+
+
+@timeit
+@aws_handle_regions
+def get_user_pools(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-idp", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_user_pools")
+    all_user_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        user_pools = page.get("UserPools", [])
+        all_user_pools.extend(user_pools)
+    return all_user_pools
+
+
+def transform_identity_pools(
+    identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_identity_pools = []
+    for pool in identity_pools:
+        transformed_pool = {
+            "IdentityPoolId": pool["IdentityPoolId"],
+            "Region": region,
+            "Roles": list(pool.get("Roles", {}).values()),
+        }
+        transformed_identity_pools.append(transformed_pool)
+    return transformed_identity_pools
+
+
+def transform_user_pools(
+    user_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_user_pools = []
+    for pool in user_pools:
+        transformed_pool = {
+            "Id": pool["Id"],
+            "Region": region,
+            "Name": pool["Name"],
+            "Status": pool.get("Status"),
+        }
+        transformed_user_pools.append(transformed_pool)
+    return transformed_user_pools
+
+
+@timeit
+def load_identity_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito Identity Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoIdentityPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_user_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito User Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoUserPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CognitoIdentityPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(CognitoUserPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Cognito Identity Pools for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        identity_pools = get_identity_pools(boto3_session, region)
+        if not identity_pools:
+            logger.info(
+                f"No Cognito Identity Pools found in region '{region}'. Skipping sync."
+            )
+        else:
+            identity_pool_roles = get_identity_pool_roles(
+                boto3_session, identity_pools, region
+            )
+            transformed_identity_pools = transform_identity_pools(
+                identity_pool_roles, region
+            )
+
+            load_identity_pools(
+                neo4j_session,
+                transformed_identity_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            user_pools = get_user_pools(boto3_session, region)
+            transformed_user_pools = transform_user_pools(user_pools, region)
+
+            load_user_pools(
+                neo4j_session,
+                transformed_user_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ecs.py

@@ -171,9 +171,15 @@ def _get_containers_from_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, An
 
 def transform_ecs_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """
-    Extract network interface ID from task attachments.
+    Extract network interface ID from task attachments and service name from group.
     """
     for task in tasks:
+        # Extract serviceName from group field
+        group = task.get("group")
+        if group and group.startswith("service:"):
+            task["serviceName"] = group.split("service:", 1)[1]
+
+        # Extract network interface ID from task attachments
         for attachment in task.get("attachments", []):
             if attachment.get("type") == "ElasticNetworkInterface":
                 details = attachment.get("details", [])

cartography/intel/aws/glue.py

@@ -10,6 +10,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
 from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.models.aws.glue.job import GlueJobSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -32,6 +33,37 @@ def get_glue_connections(
     return connections
 
 
+@timeit
+@aws_handle_regions
+def get_glue_jobs(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_jobs")
+    jobs = []
+    for page in paginator.paginate():
+        jobs.extend(page.get("Jobs", []))
+    return jobs
+
+
+def transform_glue_job(jobs: List[Dict[str, Any]], region: str) -> List[Dict[str, Any]]:
+    """
+    Transform Glue job data for ingestion
+    """
+    transformed_jobs = []
+    for job in jobs:
+        transformed_job = {
+            "Name": job["Name"],
+            "ProfileName": job.get("ProfileName"),
+            "JobMode": job.get("JobMode"),
+            "Connections": job.get("Connections", {}).get("Connections"),
+            "Region": region,
+            "Description": job.get("Description"),
+        }
+        transformed_jobs.append(transformed_job)
+    return transformed_jobs
+
+
 def transform_glue_connections(
     connections: List[Dict[str, Any]], region: str
 ) -> List[Dict[str, Any]]:
@@ -79,6 +111,27 @@ def load_glue_connections(
     )
 
 
+@timeit
+def load_glue_jobs(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} jobs for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueJobSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -88,6 +141,7 @@ def cleanup(
     GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(GlueJobSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit
@@ -114,4 +168,14 @@ def sync(
             update_tag,
         )
 
+        jobs = get_glue_jobs(boto3_session, region)
+        transformed_jobs = transform_glue_job(jobs, region)
+        load_glue_jobs(
+            neo4j_session,
+            transformed_jobs,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/kms.py

@@ -76,8 +76,8 @@ def get_policy(key: Dict, client: botocore.client.BaseClient) -> Any:
     try:
         policy = client.get_key_policy(KeyId=key["KeyId"], PolicyName="default")
     except ClientError as e:
-        policy = None
         if e.response["Error"]["Code"] == "AccessDeniedException":
+            policy = None
             logger.warning(
                 f"kms:get_key_policy on key id {key['KeyId']} failed with AccessDeniedException; continuing sync.",
                 exc_info=True,
@@ -187,6 +187,18 @@ def transform_kms_key_policies(
     policy_data = {}
 
     for key_id, policy, *_ in policy_alias_grants_data:
+        # Handle keys with null policy (access denied)
+        if policy is None:
+            logger.info(
+                f"Skipping KMS key {key_id} policy due to AccessDenied; policy analysis properties will be null"
+            )
+            policy_data[key_id] = {
+                "kms_key": key_id,
+                "anonymous_access": None,
+                "anonymous_actions": None,
+            }
+            continue
+
         parsed_policy = parse_policy(key_id, policy)
         policy_data[key_id] = parsed_policy
 

cartography/intel/aws/rds.py

@@ -9,6 +9,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.rds.cluster import RDSClusterSchema
+from cartography.models.aws.rds.event_subscription import RDSEventSubscriptionSchema
 from cartography.models.aws.rds.instance import RDSInstanceSchema
 from cartography.models.aws.rds.snapshot import RDSSnapshotSchema
 from cartography.models.aws.rds.subnet_group import DBSubnetGroupSchema
@@ -136,6 +137,38 @@ def load_rds_snapshots(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_rds_event_subscription_data(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("rds", region_name=region)
+    paginator = client.get_paginator("describe_event_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page["EventSubscriptionsList"])
+    return subscriptions
+
+
+@timeit
+def load_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RDSEventSubscriptionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 def _validate_rds_endpoint(rds: Dict) -> Dict:
     """
     Get Endpoint from RDS data structure.  Log to debug if an Endpoint field does not exist.
@@ -292,6 +325,28 @@ def transform_rds_instances(
     return instances
 
 
+def transform_rds_event_subscriptions(data: List[Dict]) -> List[Dict]:
+    subscriptions = []
+    for subscription in data:
+        transformed = {
+            "CustSubscriptionId": subscription.get("CustSubscriptionId"),
+            "EventSubscriptionArn": subscription.get("EventSubscriptionArn"),
+            "CustomerAwsId": subscription.get("CustomerAwsId"),
+            "SnsTopicArn": subscription.get("SnsTopicArn"),
+            "SourceType": subscription.get("SourceType"),
+            "Status": subscription.get("Status"),
+            "Enabled": subscription.get("Enabled"),
+            "SubscriptionCreationTime": dict_value_to_str(
+                subscription, "SubscriptionCreationTime"
+            ),
+            "event_categories": subscription.get("EventCategoriesList") or None,
+            "source_ids": subscription.get("SourceIdsList") or None,
+            "lastupdated": None,  # This will be set by the loader
+        }
+        subscriptions.append(transformed)
+    return subscriptions
+
+
 def transform_rds_subnet_groups(
     data: List[Dict], region: str, current_aws_account_id: str
 ) -> List[Dict]:
@@ -412,6 +467,20 @@ def cleanup_rds_snapshots(
     )
 
 
+@timeit
+def cleanup_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Remove RDS event subscriptions that weren't updated in this sync run
+    """
+    logger.debug("Running RDS event subscriptions cleanup job")
+    GraphJob.from_node_schema(RDSEventSubscriptionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
 @timeit
 def sync_rds_clusters(
     neo4j_session: neo4j.Session,
@@ -498,6 +567,32 @@ def sync_rds_snapshots(
     cleanup_rds_snapshots(neo4j_session, common_job_parameters)
 
 
+@timeit
+def sync_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Grab RDS event subscription data from AWS, ingest to neo4j, and run the cleanup job.
+    """
+    for region in regions:
+        logger.info(
+            "Syncing RDS event subscriptions for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
+        data = get_rds_event_subscription_data(boto3_session, region)
+        transformed = transform_rds_event_subscriptions(data)
+        load_rds_event_subscriptions(
+            neo4j_session, transformed, region, current_aws_account_id, update_tag
+        )
+    cleanup_rds_event_subscriptions(neo4j_session, common_job_parameters)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -531,6 +626,16 @@ def sync(
         update_tag,
         common_job_parameters,
     )
+
+    sync_rds_event_subscriptions(
+        neo4j_session,
+        boto3_session,
+        regions,
+        current_aws_account_id,
+        update_tag,
+        common_job_parameters,
+    )
+
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/resources.py

@@ -9,6 +9,7 @@ from . import cloudtrail
 from . import cloudtrail_management_events
 from . import cloudwatch
 from . import codebuild
+from . import cognito
 from . import config
 from . import dynamodb
 from . import ecr
@@ -116,6 +117,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "efs": efs.sync,
     "guardduty": guardduty.sync,
     "codebuild": codebuild.sync,
+    "cognito": cognito.sync,
     "eventbridge": eventbridge.sync,
     "glue": glue.sync,
 }

cartography/intel/aws/route53.py

@@ -398,7 +398,9 @@ def link_sub_zones(
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(z:AWSDNSZone)
         <-[:MEMBER_OF_DNS_ZONE]-(record:DNSRecord{type:"NS"})
         -[:DNS_POINTS_TO]->(ns:NameServer)<-[:NAMESERVER]-(z2:AWSDNSZone)
-    WHERE record.name=z2.name AND NOT z=z2
+        WHERE record.name = z2.name AND
+        z2.name ENDS WITH '.' + z.name AND
+        NOT z = z2
     RETURN z.id as zone_id, z2.id as subzone_id
     """
     zone_to_subzone = neo4j_session.read_transaction(