cartography 0.104.0rc3__py3-none-any.whl → 0.106.0rc1__py3-none-any.whl

cartography/intel/aws/s3.py

@@ -17,6 +17,7 @@ from botocore.exceptions import EndpointConnectionError
 from policyuniverse.policy import Policy
 
 from cartography.stats import get_stats_client
+from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
 from cartography.util import run_analysis_job
 from cartography.util import run_cleanup_job
@@ -54,7 +55,7 @@ def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
 def get_s3_bucket_details(
     boto3_session: boto3.session.Session,
     bucket_data: Dict,
-) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict], None, None]:
+) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict, Dict], None, None]:
     """
     Iterates over all S3 buckets. Yields bucket name (string), S3 bucket policies (JSON), ACLs (JSON),
     default encryption policy (JSON), Versioning (JSON), and Public Access Block (JSON)
@@ -69,6 +70,7 @@ def get_s3_bucket_details(
         Dict[str, Any],
         Dict[str, Any],
         Dict[str, Any],
+        Dict[str, Any],
     ]
 
     async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail:
@@ -85,14 +87,24 @@ def get_s3_bucket_details(
             encryption,
             versioning,
             public_access_block,
+            bucket_ownership_controls,
         ) = await asyncio.gather(
             to_asynchronous(get_acl, bucket, client),
             to_asynchronous(get_policy, bucket, client),
             to_asynchronous(get_encryption, bucket, client),
             to_asynchronous(get_versioning, bucket, client),
             to_asynchronous(get_public_access_block, bucket, client),
+            to_asynchronous(get_bucket_ownership_controls, bucket, client),
+        )
+        return (
+            bucket["Name"],
+            acl,
+            policy,
+            encryption,
+            versioning,
+            public_access_block,
+            bucket_ownership_controls,
         )
-        return bucket["Name"], acl, policy, encryption, versioning, public_access_block
 
     bucket_details = to_synchronous(
         *[_get_bucket_detail(bucket) for bucket in bucket_data["Buckets"]],
@@ -204,6 +216,31 @@ def get_public_access_block(
     return public_access_block
 
 
+@timeit
+def get_bucket_ownership_controls(
+    bucket: Dict, client: botocore.client.BaseClient
+) -> Optional[Dict]:
+    """
+    Gets the S3 object ownership controls configuration.
+    """
+    bucket_ownership_controls = None
+    try:
+        bucket_ownership_controls = client.get_bucket_ownership_controls(
+            Bucket=bucket["Name"]
+        )
+    except ClientError as e:
+        if _is_common_exception(e, bucket):
+            pass
+        else:
+            raise
+    except EndpointConnectionError:
+        logger.warning(
+            f"Failed to retrieve S3 bucket ownership controls for {bucket['Name']}"
+            " - Could not connect to the endpoint URL",
+        )
+    return bucket_ownership_controls
+
+
 @timeit
 def _is_common_exception(e: Exception, bucket: Dict) -> bool:
     error_msg = "Failed to retrieve S3 bucket detail"
@@ -240,6 +277,11 @@ def _is_common_exception(e: Exception, bucket: Dict) -> bool:
             f"{error_msg} for {bucket['Name']} - IllegalLocationConstraintException",
         )
         return True
+    elif "OwnershipControlsNotFoundError" in e.args[0]:
+        logger.warning(
+            f"{error_msg} for {bucket['Name']} - OwnershipControlsNotFoundError"
+        )
+        return True
     return False
 
 
@@ -414,6 +456,29 @@ def _load_s3_public_access_block(
     )
 
 
+@timeit
+def _load_bucket_ownership_controls(
+    neo4j_session: neo4j.Session,
+    bucket_ownership_controls_configs: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 BucketOwnershipControls results into neo4j.
+    """
+    ingest_bucket_ownership_controls = """
+    UNWIND $bucket_ownership_controls_configs AS bucket_ownership_controls
+    MATCH (s:S3Bucket) where s.name = bucket_ownership_controls.bucket
+    SET s.object_ownership = bucket_ownership_controls.object_ownership,
+        s.lastupdated = $UpdateTag
+    """
+
+    neo4j_session.run(
+        ingest_bucket_ownership_controls,
+        bucket_ownership_controls_configs=bucket_ownership_controls_configs,
+        UpdateTag=update_tag,
+    )
+
+
 def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
     set_defaults = """
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL
@@ -450,6 +515,7 @@ def load_s3_details(
     encryption_configs: List[Dict] = []
     versioning_configs: List[Dict] = []
     public_access_block_configs: List[Dict] = []
+    bucket_ownership_controls_configs: List[Dict] = []
     for (
         bucket,
         acl,
@@ -457,6 +523,7 @@ def load_s3_details(
         encryption,
         versioning,
         public_access_block,
+        bucket_ownership_controls,
     ) in s3_details_iter:
         parsed_acls = parse_acl(acl, bucket, aws_account_id)
         if parsed_acls is not None:
@@ -479,6 +546,11 @@ def load_s3_details(
         )
         if parsed_public_access_block is not None:
             public_access_block_configs.append(parsed_public_access_block)
+        parsed_bucket_ownership_controls = parse_bucket_ownership_controls(
+            bucket, bucket_ownership_controls
+        )
+        if parsed_bucket_ownership_controls is not None:
+            bucket_ownership_controls_configs.append(parsed_bucket_ownership_controls)
 
     # cleanup existing policy properties set on S3 Buckets
     run_cleanup_job(
@@ -494,6 +566,9 @@ def load_s3_details(
     _load_s3_encryption(neo4j_session, encryption_configs, update_tag)
     _load_s3_versioning(neo4j_session, versioning_configs, update_tag)
     _load_s3_public_access_block(neo4j_session, public_access_block_configs, update_tag)
+    _load_bucket_ownership_controls(
+        neo4j_session, bucket_ownership_controls_configs, update_tag
+    )
     _set_default_values(neo4j_session, aws_account_id)
 
 
@@ -751,6 +826,76 @@ def parse_public_access_block(
     }
 
 
+@timeit
+def parse_bucket_ownership_controls(
+    bucket: str, bucket_ownership_controls: Optional[Dict]
+) -> Optional[Dict]:
+    """Parses the S3 bucket ownership controls object and returns a dict of the relevant data"""
+    # Versioning object JSON looks like:
+    # {
+    #     'OwnershipControls': {
+    #         'Rules': [
+    #             {
+    #                 'ObjectOwnership': 'BucketOwnerPreferred'|'ObjectWriter'|'BucketOwnerEnforced'
+    #             },
+    #         ]
+    #     }
+    # }
+    if bucket_ownership_controls is None:
+        return None
+    return {
+        "bucket": bucket,
+        "object_ownership": bucket_ownership_controls.get("OwnershipControls", {})
+        .get("Rules", [{}])[0]
+        .get("ObjectOwnership"),
+    }
+
+
+@timeit
+def parse_notification_configuration(
+    bucket: str, notification_config: Optional[Dict]
+) -> List[Dict]:
+    """
+    Parse S3 bucket notification configuration to extract SNS topic notifications.
+    Returns a list of notification configurations.
+    """
+    if not notification_config or "TopicConfigurations" not in notification_config:
+        return []
+
+    notifications = []
+    for topic_config in notification_config.get("TopicConfigurations", []):
+        notification = {
+            "bucket": bucket,
+            "TopicArn": topic_config["TopicArn"],
+        }
+        notifications.append(notification)
+    return notifications
+
+
+@timeit
+def _load_s3_notifications(
+    neo4j_session: neo4j.Session,
+    notifications: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 bucket to SNS topic notification relationships into neo4j.
+    """
+    ingest_notifications = """
+    UNWIND $notifications AS notification
+    MATCH (bucket:S3Bucket{name: notification.bucket})
+    MATCH (topic:SNSTopic{arn: notification.TopicArn})
+    MERGE (bucket)-[r:NOTIFIES]->(topic)
+    ON CREATE SET r.firstseen = timestamp()
+    SET r.lastupdated = $UpdateTag
+    """
+    neo4j_session.run(
+        ingest_notifications,
+        notifications=notifications,
+        UpdateTag=update_tag,
+    )
+
+
 @timeit
 def load_s3_buckets(
     neo4j_session: neo4j.Session,
@@ -811,6 +956,43 @@ def cleanup_s3_bucket_acl_and_policy(
     )
 
 
+@timeit
+@aws_handle_regions
+def _sync_s3_notifications(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    bucket_data: Dict,
+    update_tag: int,
+) -> None:
+    """
+    Sync S3 bucket notification configurations to Neo4j.
+    """
+    logger.info("Syncing S3 bucket notifications")
+    s3_client = boto3_session.client("s3")
+    notifications = []
+
+    for bucket in bucket_data["Buckets"]:
+        try:
+            notification_config = s3_client.get_bucket_notification_configuration(
+                Bucket=bucket["Name"]
+            )
+            parsed_notifications = parse_notification_configuration(
+                bucket["Name"], notification_config
+            )
+            notifications.extend(parsed_notifications)
+            logger.debug(
+                f"Found {len(parsed_notifications)} notifications for bucket {bucket['Name']}"
+            )
+        except ClientError as e:
+            logger.warning(
+                f"Failed to retrieve notification configuration for bucket {bucket['Name']}: {e}"
+            )
+            continue
+
+    logger.info(f"Loading {len(notifications)} S3 bucket notifications into Neo4j")
+    _load_s3_notifications(neo4j_session, notifications, update_tag)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -820,9 +1002,16 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
-    logger.info("Syncing S3 for account '%s'.", current_aws_account_id)
-    bucket_data = get_s3_bucket_list(boto3_session)
+    """
+    Sync S3 buckets and their configurations to Neo4j.
+    This includes:
+    1. Basic bucket information
+    2. ACLs and policies
+    3. Notification configurations
+    """
+    logger.info("Syncing S3 for account '%s'", current_aws_account_id)
 
+    bucket_data = get_s3_bucket_list(boto3_session)
     load_s3_buckets(neo4j_session, bucket_data, current_aws_account_id, update_tag)
     cleanup_s3_buckets(neo4j_session, common_job_parameters)
 
@@ -835,6 +1024,8 @@ def sync(
     )
     cleanup_s3_bucket_acl_and_policy(neo4j_session, common_job_parameters)
 
+    _sync_s3_notifications(neo4j_session, boto3_session, bucket_data, update_tag)
+
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/sqs.py

@@ -9,8 +9,10 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.sqs.queue import SQSQueueSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -33,9 +35,7 @@ def get_sqs_queue_attributes(
     boto3_session: boto3.session.Session,
     queue_urls: List[str],
 ) -> List[Tuple[str, Any]]:
-    """
-    Iterates over all SQS queues. Returns a dict with url as key, and attributes as value.
-    """
+    """Iterates over all SQS queues and returns a list of (url, attributes)."""
     client = boto3_session.client("sqs")
 
     queue_attributes = []
@@ -58,108 +58,53 @@ def get_sqs_queue_attributes(
     return queue_attributes
 
 
-@timeit
-def load_sqs_queues(
-    neo4j_session: neo4j.Session,
-    data: List[Tuple[str, Any]],
-    region: str,
-    current_aws_account_id: str,
-    aws_update_tag: int,
-) -> None:
-    ingest_queues = """
-    UNWIND $Queues as sqs_queue
-        MERGE (queue:SQSQueue{id: sqs_queue.QueueArn})
-        ON CREATE SET queue.firstseen = timestamp(), queue.url = sqs_queue.url
-        SET queue.name = sqs_queue.name, queue.region = $Region, queue.arn = sqs_queue.QueueArn,
-            queue.created_timestamp = sqs_queue.CreatedTimestamp, queue.delay_seconds = sqs_queue.DelaySeconds,
-            queue.last_modified_timestamp = sqs_queue.LastModifiedTimestamp,
-            queue.maximum_message_size = sqs_queue.MaximumMessageSize,
-            queue.message_retention_period = sqs_queue.MessageRetentionPeriod,
-            queue.policy = sqs_queue.Policy, queue.arn = sqs_queue.QueueArn,
-            queue.receive_message_wait_time_seconds = sqs_queue.ReceiveMessageWaitTimeSeconds,
-            queue.redrive_policy_dead_letter_target_arn = sqs_queue.RedrivePolicy.deadLetterTargetArn,
-            queue.redrive_policy_max_receive_count = sqs_queue.RedrivePolicy.maxReceiveCount,
-            queue.visibility_timeout = sqs_queue.VisibilityTimeout,
-            queue.kms_master_key_id = sqs_queue.KmsMasterKeyId,
-            queue.kms_data_key_reuse_period_seconds = sqs_queue.KmsDataKeyReusePeriodSeconds,
-            queue.fifo_queue = sqs_queue.FifoQueue,
-            queue.content_based_deduplication = sqs_queue.ContentBasedDeduplication,
-            queue.deduplication_scope = sqs_queue.DeduplicationScope,
-            queue.fifo_throughput_limit = sqs_queue.FifoThroughputLimit,
-            queue.lastupdated = $aws_update_tag
-        WITH queue
-        MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (owner)-[r:RESOURCE]->(queue)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    dead_letter_queues: List[Dict] = []
-    queues: List[Dict] = []
-    for url, queue in data:
+def transform_sqs_queues(data: List[Tuple[str, Any]]) -> List[Dict[str, Any]]:
+    transformed: List[Dict[str, Any]] = []
+    for url, attrs in data:
+        queue = dict(attrs)
         queue["url"] = url
-        queue["name"] = queue["QueueArn"].split(":")[-1]
-        queue["CreatedTimestamp"] = int(queue["CreatedTimestamp"])
-        queue["LastModifiedTimestamp"] = int(queue["LastModifiedTimestamp"])
-        redrive_policy = queue.get("RedrivePolicy")
+        queue["name"] = attrs["QueueArn"].split(":")[-1]
+        queue["CreatedTimestamp"] = int(attrs.get("CreatedTimestamp", 0))
+        queue["LastModifiedTimestamp"] = int(attrs.get("LastModifiedTimestamp", 0))
+        redrive_policy = attrs.get("RedrivePolicy")
         if redrive_policy:
             try:
                 rp = json.loads(redrive_policy)
             except TypeError:
                 rp = {}
-            queue["RedrivePolicy"] = rp
-            dead_letter_arn = rp.get("deadLetterTargetArn")
-            if dead_letter_arn:
-                dead_letter_queues.append(
-                    {
-                        "arn": queue["QueueArn"],
-                        "dead_letter_arn": dead_letter_arn,
-                    },
-                )
-        queues.append(queue)
-
-    neo4j_session.run(
-        ingest_queues,
-        Queues=queues,
-        Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
-    )
-
-    _attach_dead_letter_queues(neo4j_session, dead_letter_queues, aws_update_tag)
+            queue["redrive_policy_dead_letter_target_arn"] = rp.get(
+                "deadLetterTargetArn"
+            )
+            queue["redrive_policy_max_receive_count"] = rp.get("maxReceiveCount")
+        transformed.append(queue)
+    return transformed
 
 
 @timeit
-def _attach_dead_letter_queues(
+def load_sqs_queues(
     neo4j_session: neo4j.Session,
-    data: List[Dict[str, str]],
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
-    """
-    Attach deadletter queues to their queues.
-    """
-    attach_deadletter_to_queue = """
-    UNWIND $Relations as relation
-        MATCH (queue:SQSQueue{id: relation.arn}), (deadletter:SQSQueue{id: relation.dead_letter_arn})
-        MERGE (queue)-[r:HAS_DEADLETTER_QUEUE]->(deadletter)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    neo4j_session.run(
-        attach_deadletter_to_queue,
-        Relations=data,
-        aws_update_tag=aws_update_tag,
+    load(
+        neo4j_session,
+        SQSQueueSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
     )
 
 
 @timeit
 def cleanup_sqs_queues(
     neo4j_session: neo4j.Session,
-    common_job_parameters: Dict,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
-    run_cleanup_job(
-        "aws_import_sqs_queues_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    GraphJob.from_node_schema(SQSQueueSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -170,7 +115,7 @@ def sync(
     regions: List[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
@@ -179,12 +124,13 @@ def sync(
             current_aws_account_id,
         )
         queue_urls = get_sqs_queue_list(boto3_session, region)
-        if len(queue_urls) == 0:
+        if not queue_urls:
             continue
         queue_attributes = get_sqs_queue_attributes(boto3_session, queue_urls)
+        transformed = transform_sqs_queues(queue_attributes)
         load_sqs_queues(
             neo4j_session,
-            queue_attributes,
+            transformed,
             region,
             current_aws_account_id,
             update_tag,

cartography/intel/entra/__init__.py

@@ -4,6 +4,8 @@ import logging
 import neo4j
 
 from cartography.config import Config
+from cartography.intel.entra.applications import sync_entra_applications
+from cartography.intel.entra.groups import sync_entra_groups
 from cartography.intel.entra.ou import sync_entra_ous
 from cartography.intel.entra.users import sync_entra_users
 from cartography.util import timeit
@@ -47,6 +49,16 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
             common_job_parameters,
         )
 
+        # Run group sync
+        await sync_entra_groups(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
+
         # Run OU sync
         await sync_entra_ous(
             neo4j_session,
@@ -57,5 +69,15 @@ def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
             common_job_parameters,
         )
 
+        # Run application sync
+        await sync_entra_applications(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        )
+
     # Execute both syncs in sequence
     asyncio.run(main())