cartography 0.110.0rc1__py3-none-any.whl → 0.111.0__py3-none-any.whl

cartography/intel/aws/glue.py

@@ -10,6 +10,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
 from cartography.models.aws.glue.connection import GlueConnectionSchema
+from cartography.models.aws.glue.job import GlueJobSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -32,6 +33,37 @@ def get_glue_connections(
     return connections
 
 
+@timeit
+@aws_handle_regions
+def get_glue_jobs(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "glue", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("get_jobs")
+    jobs = []
+    for page in paginator.paginate():
+        jobs.extend(page.get("Jobs", []))
+    return jobs
+
+
+def transform_glue_job(jobs: List[Dict[str, Any]], region: str) -> List[Dict[str, Any]]:
+    """
+    Transform Glue job data for ingestion
+    """
+    transformed_jobs = []
+    for job in jobs:
+        transformed_job = {
+            "Name": job["Name"],
+            "ProfileName": job.get("ProfileName"),
+            "JobMode": job.get("JobMode"),
+            "Connections": job.get("Connections", {}).get("Connections"),
+            "Region": region,
+            "Description": job.get("Description"),
+        }
+        transformed_jobs.append(transformed_job)
+    return transformed_jobs
+
+
 def transform_glue_connections(
     connections: List[Dict[str, Any]], region: str
 ) -> List[Dict[str, Any]]:
@@ -79,6 +111,27 @@ def load_glue_connections(
     )
 
 
+@timeit
+def load_glue_jobs(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Glue {len(data)} jobs for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        GlueJobSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -88,6 +141,7 @@ def cleanup(
     GraphJob.from_node_schema(GlueConnectionSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(GlueJobSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit
@@ -114,4 +168,14 @@ def sync(
             update_tag,
         )
 
+        jobs = get_glue_jobs(boto3_session, region)
+        transformed_jobs = transform_glue_job(jobs, region)
+        load_glue_jobs(
+            neo4j_session,
+            transformed_jobs,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/kms.py

@@ -76,8 +76,8 @@ def get_policy(key: Dict, client: botocore.client.BaseClient) -> Any:
     try:
         policy = client.get_key_policy(KeyId=key["KeyId"], PolicyName="default")
     except ClientError as e:
-        policy = None
         if e.response["Error"]["Code"] == "AccessDeniedException":
+            policy = None
             logger.warning(
                 f"kms:get_key_policy on key id {key['KeyId']} failed with AccessDeniedException; continuing sync.",
                 exc_info=True,
@@ -187,6 +187,18 @@ def transform_kms_key_policies(
     policy_data = {}
 
     for key_id, policy, *_ in policy_alias_grants_data:
+        # Handle keys with null policy (access denied)
+        if policy is None:
+            logger.info(
+                f"Skipping KMS key {key_id} policy due to AccessDenied; policy analysis properties will be null"
+            )
+            policy_data[key_id] = {
+                "kms_key": key_id,
+                "anonymous_access": None,
+                "anonymous_actions": None,
+            }
+            continue
+
         parsed_policy = parse_policy(key_id, policy)
         policy_data[key_id] = parsed_policy
 

cartography/intel/aws/rds.py

@@ -9,6 +9,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.rds.cluster import RDSClusterSchema
+from cartography.models.aws.rds.event_subscription import RDSEventSubscriptionSchema
 from cartography.models.aws.rds.instance import RDSInstanceSchema
 from cartography.models.aws.rds.snapshot import RDSSnapshotSchema
 from cartography.models.aws.rds.subnet_group import DBSubnetGroupSchema
@@ -136,6 +137,38 @@ def load_rds_snapshots(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_rds_event_subscription_data(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client("rds", region_name=region)
+    paginator = client.get_paginator("describe_event_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page["EventSubscriptionsList"])
+    return subscriptions
+
+
+@timeit
+def load_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RDSEventSubscriptionSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 def _validate_rds_endpoint(rds: Dict) -> Dict:
     """
     Get Endpoint from RDS data structure.  Log to debug if an Endpoint field does not exist.
@@ -292,6 +325,28 @@ def transform_rds_instances(
     return instances
 
 
+def transform_rds_event_subscriptions(data: List[Dict]) -> List[Dict]:
+    subscriptions = []
+    for subscription in data:
+        transformed = {
+            "CustSubscriptionId": subscription.get("CustSubscriptionId"),
+            "EventSubscriptionArn": subscription.get("EventSubscriptionArn"),
+            "CustomerAwsId": subscription.get("CustomerAwsId"),
+            "SnsTopicArn": subscription.get("SnsTopicArn"),
+            "SourceType": subscription.get("SourceType"),
+            "Status": subscription.get("Status"),
+            "Enabled": subscription.get("Enabled"),
+            "SubscriptionCreationTime": dict_value_to_str(
+                subscription, "SubscriptionCreationTime"
+            ),
+            "event_categories": subscription.get("EventCategoriesList") or None,
+            "source_ids": subscription.get("SourceIdsList") or None,
+            "lastupdated": None,  # This will be set by the loader
+        }
+        subscriptions.append(transformed)
+    return subscriptions
+
+
 def transform_rds_subnet_groups(
     data: List[Dict], region: str, current_aws_account_id: str
 ) -> List[Dict]:
@@ -412,6 +467,20 @@ def cleanup_rds_snapshots(
     )
 
 
+@timeit
+def cleanup_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Remove RDS event subscriptions that weren't updated in this sync run
+    """
+    logger.debug("Running RDS event subscriptions cleanup job")
+    GraphJob.from_node_schema(RDSEventSubscriptionSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
 @timeit
 def sync_rds_clusters(
     neo4j_session: neo4j.Session,
@@ -498,6 +567,32 @@ def sync_rds_snapshots(
     cleanup_rds_snapshots(neo4j_session, common_job_parameters)
 
 
+@timeit
+def sync_rds_event_subscriptions(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Grab RDS event subscription data from AWS, ingest to neo4j, and run the cleanup job.
+    """
+    for region in regions:
+        logger.info(
+            "Syncing RDS event subscriptions for region '%s' in account '%s'.",
+            region,
+            current_aws_account_id,
+        )
+        data = get_rds_event_subscription_data(boto3_session, region)
+        transformed = transform_rds_event_subscriptions(data)
+        load_rds_event_subscriptions(
+            neo4j_session, transformed, region, current_aws_account_id, update_tag
+        )
+    cleanup_rds_event_subscriptions(neo4j_session, common_job_parameters)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -531,6 +626,16 @@ def sync(
         update_tag,
         common_job_parameters,
     )
+
+    sync_rds_event_subscriptions(
+        neo4j_session,
+        boto3_session,
+        regions,
+        current_aws_account_id,
+        update_tag,
+        common_job_parameters,
+    )
+
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/resources.py

@@ -9,6 +9,7 @@ from . import cloudtrail
 from . import cloudtrail_management_events
 from . import cloudwatch
 from . import codebuild
+from . import cognito
 from . import config
 from . import dynamodb
 from . import ecr
@@ -116,6 +117,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     "efs": efs.sync,
     "guardduty": guardduty.sync,
     "codebuild": codebuild.sync,
+    "cognito": cognito.sync,
     "eventbridge": eventbridge.sync,
     "glue": glue.sync,
 }

cartography/intel/aws/route53.py

@@ -398,7 +398,9 @@ def link_sub_zones(
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(z:AWSDNSZone)
         <-[:MEMBER_OF_DNS_ZONE]-(record:DNSRecord{type:"NS"})
         -[:DNS_POINTS_TO]->(ns:NameServer)<-[:NAMESERVER]-(z2:AWSDNSZone)
-    WHERE record.name=z2.name AND NOT z=z2
+        WHERE record.name = z2.name AND
+        z2.name ENDS WITH '.' + z.name AND
+        NOT z = z2
     RETURN z.id as zone_id, z2.id as subzone_id
     """
     zone_to_subzone = neo4j_session.read_transaction(

cartography/intel/aws/s3.py

@@ -71,6 +71,7 @@ def get_s3_bucket_details(
         Dict[str, Any],
         Dict[str, Any],
         Dict[str, Any],
+        Dict[str, Any],
     ]
 
     async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail:
@@ -88,6 +89,7 @@ def get_s3_bucket_details(
             versioning,
             public_access_block,
             bucket_ownership_controls,
+            bucket_logging,
         ) = await asyncio.gather(
             to_asynchronous(get_acl, bucket, client),
             to_asynchronous(get_policy, bucket, client),
@@ -95,6 +97,7 @@ def get_s3_bucket_details(
             to_asynchronous(get_versioning, bucket, client),
             to_asynchronous(get_public_access_block, bucket, client),
             to_asynchronous(get_bucket_ownership_controls, bucket, client),
+            to_asynchronous(get_bucket_logging, bucket, client),
         )
         return (
             bucket["Name"],
@@ -104,6 +107,7 @@ def get_s3_bucket_details(
             versioning,
             public_access_block,
             bucket_ownership_controls,
+            bucket_logging,
         )
 
     bucket_details = to_synchronous(
@@ -241,6 +245,29 @@ def get_bucket_ownership_controls(
     return bucket_ownership_controls
 
 
+@timeit
+@aws_handle_regions
+def get_bucket_logging(
+    bucket: Dict, client: botocore.client.BaseClient
+) -> Optional[Dict]:
+    """
+    Gets the S3 bucket logging status configuration.
+    """
+    bucket_logging = None
+    try:
+        bucket_logging = client.get_bucket_logging(Bucket=bucket["Name"])
+    except ClientError as e:
+        if _is_common_exception(e, bucket):
+            pass
+        else:
+            raise
+    except EndpointConnectionError:
+        logger.warning(
+            f"Failed to retrieve S3 bucket logging status for {bucket['Name']} - Could not connect to the endpoint URL",
+        )
+    return bucket_logging
+
+
 @timeit
 def _is_common_exception(e: Exception, bucket: Dict) -> bool:
     error_msg = "Failed to retrieve S3 bucket detail"
@@ -319,6 +346,7 @@ def _load_s3_acls(
         "aws_s3acl_analysis.json",
         neo4j_session,
         {"AWS_ID": aws_account_id},
+        package="cartography.data.jobs.scoped_analysis",
     )
 
 
@@ -479,6 +507,30 @@ def _load_bucket_ownership_controls(
     )
 
 
+@timeit
+def _load_bucket_logging(
+    neo4j_session: neo4j.Session,
+    bucket_logging_configs: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 bucket logging status configuration into neo4j.
+    """
+    # Load basic logging status
+    ingest_bucket_logging = """
+    UNWIND $bucket_logging_configs AS bucket_logging
+    MATCH (bucket:S3Bucket{name: bucket_logging.bucket})
+    SET bucket.logging_enabled = bucket_logging.logging_enabled,
+        bucket.logging_target_bucket = bucket_logging.target_bucket,
+        bucket.lastupdated = $update_tag
+    """
+    neo4j_session.run(
+        ingest_bucket_logging,
+        bucket_logging_configs=bucket_logging_configs,
+        update_tag=update_tag,
+    )
+
+
 def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
     set_defaults = """
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL
@@ -516,6 +568,7 @@ def load_s3_details(
     versioning_configs: List[Dict] = []
     public_access_block_configs: List[Dict] = []
     bucket_ownership_controls_configs: List[Dict] = []
+    bucket_logging_configs: List[Dict] = []
     for (
         bucket,
         acl,
@@ -524,6 +577,7 @@ def load_s3_details(
         versioning,
         public_access_block,
         bucket_ownership_controls,
+        bucket_logging,
     ) in s3_details_iter:
         parsed_acls = parse_acl(acl, bucket, aws_account_id)
         if parsed_acls is not None:
@@ -551,6 +605,9 @@ def load_s3_details(
         )
         if parsed_bucket_ownership_controls is not None:
             bucket_ownership_controls_configs.append(parsed_bucket_ownership_controls)
+        parsed_bucket_logging = parse_bucket_logging(bucket, bucket_logging)
+        if parsed_bucket_logging is not None:
+            bucket_logging_configs.append(parsed_bucket_logging)
 
     # cleanup existing policy properties set on S3 Buckets
     run_cleanup_job(
@@ -569,6 +626,7 @@ def load_s3_details(
     _load_bucket_ownership_controls(
         neo4j_session, bucket_ownership_controls_configs, update_tag
     )
+    _load_bucket_logging(neo4j_session, bucket_logging_configs, update_tag)
     _set_default_values(neo4j_session, aws_account_id)
 
 
@@ -851,6 +909,52 @@ def parse_bucket_ownership_controls(
     }
 
 
+def parse_bucket_logging(bucket: str, bucket_logging: Optional[Dict]) -> Optional[Dict]:
+    """Parses the S3 bucket logging status configuration and returns a dict of the relevant data"""
+    # Logging status object JSON looks like:
+    # {
+    #     'LoggingEnabled': {
+    #         'TargetBucket': 'string',
+    #         'TargetGrants': [
+    #             {
+    #                 'Grantee': {
+    #                     'DisplayName': 'string',
+    #                     'EmailAddress': 'string',
+    #                     'ID': 'string',
+    #                     'Type': 'CanonicalUser'|'AmazonCustomerByEmail'|'Group',
+    #                     'URI': 'string'
+    #                 },
+    #                 'Permission': 'FULL_CONTROL'|'READ'|'WRITE'
+    #             },
+    #         ],
+    #         'TargetPrefix': 'string',
+    #         'TargetObjectKeyFormat': {
+    #             'SimplePrefix': {},
+    #             'PartitionedPrefix': {
+    #                 'PartitionDateSource': 'EventTime'|'DeliveryTime'
+    #             }
+    #         }
+    #     }
+    # }
+    # Or empty dict {} if logging is not enabled
+    if bucket_logging is None:
+        return None
+
+    logging_config = bucket_logging.get("LoggingEnabled", {})
+    if not logging_config:
+        return {
+            "bucket": bucket,
+            "logging_enabled": False,
+            "target_bucket": None,
+        }
+
+    return {
+        "bucket": bucket,
+        "logging_enabled": True,
+        "target_bucket": logging_config.get("TargetBucket"),
+    }
+
+
 @timeit
 def parse_notification_configuration(
     bucket: str, notification_config: Optional[Dict]

cartography/intel/entra/__init__.py

@@ -1,14 +1,13 @@
 import asyncio
-import datetime
 import logging
-from traceback import TracebackException
-from typing import Awaitable
-from typing import Callable
 
 import neo4j
 
 from cartography.config import Config
-from cartography.intel.entra.resources import RESOURCE_FUNCTIONS
+from cartography.intel.entra.applications import sync_entra_applications
+from cartography.intel.entra.groups import sync_entra_groups
+from cartography.intel.entra.ou import sync_entra_ous
+from cartography.intel.entra.users import sync_entra_users
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -40,46 +39,45 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     }
 
     async def main() -> None:
-        failed_stages = []
-        exception_tracebacks = []
+        # Run user sync
+        await sync_entra_users(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-        async def run_stage(name: str, func: Callable[..., Awaitable[None]]) -> None:
-            try:
-                await func(
-                    neo4j_session,
-                    config.entra_tenant_id,
-                    config.entra_client_id,
-                    config.entra_client_secret,
-                    config.update_tag,
-                    common_job_parameters,
-                )
-            except Exception as e:
-                if config.entra_best_effort_mode:
-                    timestamp = datetime.datetime.now()
-                    failed_stages.append(name)
-                    exception_traceback = TracebackException.from_exception(e)
-                    traceback_string = "".join(exception_traceback.format())
-                    exception_tracebacks.append(
-                        f"{timestamp} - Exception for stage {name}\n{traceback_string}"
-                    )
-                    logger.warning(
-                        f"Caught exception syncing {name}. entra-best-effort-mode is on so we are continuing "
-                        "on to the next Entra sync. All exceptions will be aggregated and re-logged at the end of the sync.",
-                        exc_info=True,
-                    )
-                else:
-                    logger.error("Error during Entra sync", exc_info=True)
-                    raise
+        # Run group sync
+        await sync_entra_groups(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-        for name, func in RESOURCE_FUNCTIONS:
-            await run_stage(name, func)
+        # Run OU sync
+        await sync_entra_ous(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-        if failed_stages:
-            logger.error(
-                f"Entra sync failed for the following stages: {', '.join(failed_stages)}. "
-                "See the logs for more details.",
-            )
-            raise Exception("\n".join(exception_tracebacks))
+        # Run application sync
+        await sync_entra_applications(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
 
-    # Execute all syncs in sequence
+    # Execute both syncs in sequence
     asyncio.run(main())

cartography/intel/entra/applications.py

@@ -172,11 +172,12 @@ async def get_app_role_assignments(
             )
             continue
         except Exception as e:
+            # Only catch truly unexpected errors - these should be rare
             logger.error(
                 f"Unexpected error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}",
                 exc_info=True,
             )
-            raise
+            continue
 
     logger.info(f"Retrieved {len(assignments)} app role assignments total")
     return assignments

cartography/intel/entra/ou.py

@@ -43,7 +43,7 @@ async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
                 current_request = None
         except Exception as e:
             logger.error(f"Failed to retrieve administrative units: {str(e)}")
-            raise
+            current_request = None
 
     return all_units
 

cartography/intel/github/__init__.py

@@ -3,7 +3,6 @@ import json
 import logging
 
 import neo4j
-from requests import exceptions
 
 import cartography.intel.github.repos
 import cartography.intel.github.teams
@@ -34,27 +33,24 @@ def start_github_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
     }
     # run sync for the provided github tokens
     for auth_data in auth_tokens["organization"]:
-        try:
-            cartography.intel.github.users.sync(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-            cartography.intel.github.repos.sync(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-            cartography.intel.github.teams.sync_github_teams(
-                neo4j_session,
-                common_job_parameters,
-                auth_data["token"],
-                auth_data["url"],
-                auth_data["name"],
-            )
-        except exceptions.RequestException as e:
-            logger.error("Could not complete request to the GitHub API: %s", e)
+        cartography.intel.github.users.sync(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+        cartography.intel.github.repos.sync(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )
+        cartography.intel.github.teams.sync_github_teams(
+            neo4j_session,
+            common_job_parameters,
+            auth_data["token"],
+            auth_data["url"],
+            auth_data["name"],
+        )