cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/s3.py

@@ -16,7 +16,9 @@ from botocore.exceptions import ClientError
 from botocore.exceptions import EndpointConnectionError
 from policyuniverse.policy import Policy
 
+from cartography.client.core.tx import run_write_query
 from cartography.stats import get_stats_client
+from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
 from cartography.util import run_analysis_job
 from cartography.util import run_cleanup_job
@@ -54,7 +56,7 @@ def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
 def get_s3_bucket_details(
     boto3_session: boto3.session.Session,
     bucket_data: Dict,
-) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict], None, None]:
+) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict, Dict], None, None]:
     """
     Iterates over all S3 buckets. Yields bucket name (string), S3 bucket policies (JSON), ACLs (JSON),
     default encryption policy (JSON), Versioning (JSON), and Public Access Block (JSON)
@@ -69,6 +71,8 @@ def get_s3_bucket_details(
         Dict[str, Any],
         Dict[str, Any],
         Dict[str, Any],
+        Dict[str, Any],
+        Dict[str, Any],
     ]
 
     async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail:
@@ -85,14 +89,27 @@ def get_s3_bucket_details(
             encryption,
             versioning,
             public_access_block,
+            bucket_ownership_controls,
+            bucket_logging,
         ) = await asyncio.gather(
             to_asynchronous(get_acl, bucket, client),
             to_asynchronous(get_policy, bucket, client),
             to_asynchronous(get_encryption, bucket, client),
             to_asynchronous(get_versioning, bucket, client),
             to_asynchronous(get_public_access_block, bucket, client),
+            to_asynchronous(get_bucket_ownership_controls, bucket, client),
+            to_asynchronous(get_bucket_logging, bucket, client),
+        )
+        return (
+            bucket["Name"],
+            acl,
+            policy,
+            encryption,
+            versioning,
+            public_access_block,
+            bucket_ownership_controls,
+            bucket_logging,
         )
-        return bucket["Name"], acl, policy, encryption, versioning, public_access_block
 
     bucket_details = to_synchronous(
         *[_get_bucket_detail(bucket) for bucket in bucket_data["Buckets"]],
@@ -204,6 +221,54 @@ def get_public_access_block(
     return public_access_block
 
 
+@timeit
+def get_bucket_ownership_controls(
+    bucket: Dict, client: botocore.client.BaseClient
+) -> Optional[Dict]:
+    """
+    Gets the S3 object ownership controls configuration.
+    """
+    bucket_ownership_controls = None
+    try:
+        bucket_ownership_controls = client.get_bucket_ownership_controls(
+            Bucket=bucket["Name"]
+        )
+    except ClientError as e:
+        if _is_common_exception(e, bucket):
+            pass
+        else:
+            raise
+    except EndpointConnectionError:
+        logger.warning(
+            f"Failed to retrieve S3 bucket ownership controls for {bucket['Name']}"
+            " - Could not connect to the endpoint URL",
+        )
+    return bucket_ownership_controls
+
+
+@timeit
+@aws_handle_regions
+def get_bucket_logging(
+    bucket: Dict, client: botocore.client.BaseClient
+) -> Optional[Dict]:
+    """
+    Gets the S3 bucket logging status configuration.
+    """
+    bucket_logging = None
+    try:
+        bucket_logging = client.get_bucket_logging(Bucket=bucket["Name"])
+    except ClientError as e:
+        if _is_common_exception(e, bucket):
+            pass
+        else:
+            raise
+    except EndpointConnectionError:
+        logger.warning(
+            f"Failed to retrieve S3 bucket logging status for {bucket['Name']} - Could not connect to the endpoint URL",
+        )
+    return bucket_logging
+
+
 @timeit
 def _is_common_exception(e: Exception, bucket: Dict) -> bool:
     error_msg = "Failed to retrieve S3 bucket detail"
@@ -240,6 +305,11 @@ def _is_common_exception(e: Exception, bucket: Dict) -> bool:
             f"{error_msg} for {bucket['Name']} - IllegalLocationConstraintException",
         )
         return True
+    elif "OwnershipControlsNotFoundError" in e.args[0]:
+        logger.warning(
+            f"{error_msg} for {bucket['Name']} - OwnershipControlsNotFoundError"
+        )
+        return True
     return False
 
 
@@ -265,7 +335,8 @@ def _load_s3_acls(
     SET r.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_acls,
         acls=acls,
         UpdateTag=update_tag,
@@ -277,6 +348,7 @@ def _load_s3_acls(
         "aws_s3acl_analysis.json",
         neo4j_session,
         {"AWS_ID": aws_account_id},
+        package="cartography.data.jobs.scoped_analysis",
     )
 
 
@@ -298,7 +370,8 @@ def _load_s3_policies(
     s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policies,
         policies=policies,
         UpdateTag=update_tag,
@@ -331,11 +404,12 @@ def _load_s3_policy_statements(
         MERGE (bucket)-[r:POLICY_STATEMENT]->(statement)
         SET r.lastupdated = $UpdateTag
         """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_policy_statement,
         Statements=statements,
         UpdateTag=update_tag,
-    ).consume()
+    )
 
 
 @timeit
@@ -357,7 +431,8 @@ def _load_s3_encryption(
     s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_encryption,
         encryption_configs=encryption_configs,
         UpdateTag=update_tag,
@@ -381,7 +456,8 @@ def _load_s3_versioning(
         s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_versioning,
         versioning_configs=versioning_configs,
         UpdateTag=update_tag,
@@ -407,19 +483,70 @@ def _load_s3_public_access_block(
         s.lastupdated = $UpdateTag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_public_access_block,
         public_access_block_configs=public_access_block_configs,
         UpdateTag=update_tag,
     )
 
 
+@timeit
+def _load_bucket_ownership_controls(
+    neo4j_session: neo4j.Session,
+    bucket_ownership_controls_configs: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 BucketOwnershipControls results into neo4j.
+    """
+    ingest_bucket_ownership_controls = """
+    UNWIND $bucket_ownership_controls_configs AS bucket_ownership_controls
+    MATCH (s:S3Bucket) where s.name = bucket_ownership_controls.bucket
+    SET s.object_ownership = bucket_ownership_controls.object_ownership,
+        s.lastupdated = $UpdateTag
+    """
+
+    run_write_query(
+        neo4j_session,
+        ingest_bucket_ownership_controls,
+        bucket_ownership_controls_configs=bucket_ownership_controls_configs,
+        UpdateTag=update_tag,
+    )
+
+
+@timeit
+def _load_bucket_logging(
+    neo4j_session: neo4j.Session,
+    bucket_logging_configs: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 bucket logging status configuration into neo4j.
+    """
+    # Load basic logging status
+    ingest_bucket_logging = """
+    UNWIND $bucket_logging_configs AS bucket_logging
+    MATCH (bucket:S3Bucket{name: bucket_logging.bucket})
+    SET bucket.logging_enabled = bucket_logging.logging_enabled,
+        bucket.logging_target_bucket = bucket_logging.target_bucket,
+        bucket.lastupdated = $update_tag
+    """
+    run_write_query(
+        neo4j_session,
+        ingest_bucket_logging,
+        bucket_logging_configs=bucket_logging_configs,
+        update_tag=update_tag,
+    )
+
+
 def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
     set_defaults = """
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL
     SET s.anonymous_access = false, s.anonymous_actions = []
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         set_defaults,
         AWS_ID=aws_account_id,
     )
@@ -428,7 +555,8 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No
     MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.default_encryption IS NULL
     SET s.default_encryption = false
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         set_encryption_defaults,
         AWS_ID=aws_account_id,
     )
@@ -450,6 +578,8 @@ def load_s3_details(
     encryption_configs: List[Dict] = []
     versioning_configs: List[Dict] = []
     public_access_block_configs: List[Dict] = []
+    bucket_ownership_controls_configs: List[Dict] = []
+    bucket_logging_configs: List[Dict] = []
     for (
         bucket,
         acl,
@@ -457,6 +587,8 @@ def load_s3_details(
         encryption,
         versioning,
         public_access_block,
+        bucket_ownership_controls,
+        bucket_logging,
     ) in s3_details_iter:
         parsed_acls = parse_acl(acl, bucket, aws_account_id)
         if parsed_acls is not None:
@@ -479,6 +611,14 @@ def load_s3_details(
         )
         if parsed_public_access_block is not None:
             public_access_block_configs.append(parsed_public_access_block)
+        parsed_bucket_ownership_controls = parse_bucket_ownership_controls(
+            bucket, bucket_ownership_controls
+        )
+        if parsed_bucket_ownership_controls is not None:
+            bucket_ownership_controls_configs.append(parsed_bucket_ownership_controls)
+        parsed_bucket_logging = parse_bucket_logging(bucket, bucket_logging)
+        if parsed_bucket_logging is not None:
+            bucket_logging_configs.append(parsed_bucket_logging)
 
     # cleanup existing policy properties set on S3 Buckets
     run_cleanup_job(
@@ -494,6 +634,10 @@ def load_s3_details(
     _load_s3_encryption(neo4j_session, encryption_configs, update_tag)
     _load_s3_versioning(neo4j_session, versioning_configs, update_tag)
     _load_s3_public_access_block(neo4j_session, public_access_block_configs, update_tag)
+    _load_bucket_ownership_controls(
+        neo4j_session, bucket_ownership_controls_configs, update_tag
+    )
+    _load_bucket_logging(neo4j_session, bucket_logging_configs, update_tag)
     _set_default_values(neo4j_session, aws_account_id)
 
 
@@ -751,6 +895,123 @@ def parse_public_access_block(
     }
 
 
+@timeit
+def parse_bucket_ownership_controls(
+    bucket: str, bucket_ownership_controls: Optional[Dict]
+) -> Optional[Dict]:
+    """Parses the S3 bucket ownership controls object and returns a dict of the relevant data"""
+    # Versioning object JSON looks like:
+    # {
+    #     'OwnershipControls': {
+    #         'Rules': [
+    #             {
+    #                 'ObjectOwnership': 'BucketOwnerPreferred'|'ObjectWriter'|'BucketOwnerEnforced'
+    #             },
+    #         ]
+    #     }
+    # }
+    if bucket_ownership_controls is None:
+        return None
+    return {
+        "bucket": bucket,
+        "object_ownership": bucket_ownership_controls.get("OwnershipControls", {})
+        .get("Rules", [{}])[0]
+        .get("ObjectOwnership"),
+    }
+
+
+def parse_bucket_logging(bucket: str, bucket_logging: Optional[Dict]) -> Optional[Dict]:
+    """Parses the S3 bucket logging status configuration and returns a dict of the relevant data"""
+    # Logging status object JSON looks like:
+    # {
+    #     'LoggingEnabled': {
+    #         'TargetBucket': 'string',
+    #         'TargetGrants': [
+    #             {
+    #                 'Grantee': {
+    #                     'DisplayName': 'string',
+    #                     'EmailAddress': 'string',
+    #                     'ID': 'string',
+    #                     'Type': 'CanonicalUser'|'AmazonCustomerByEmail'|'Group',
+    #                     'URI': 'string'
+    #                 },
+    #                 'Permission': 'FULL_CONTROL'|'READ'|'WRITE'
+    #             },
+    #         ],
+    #         'TargetPrefix': 'string',
+    #         'TargetObjectKeyFormat': {
+    #             'SimplePrefix': {},
+    #             'PartitionedPrefix': {
+    #                 'PartitionDateSource': 'EventTime'|'DeliveryTime'
+    #             }
+    #         }
+    #     }
+    # }
+    # Or empty dict {} if logging is not enabled
+    if bucket_logging is None:
+        return None
+
+    logging_config = bucket_logging.get("LoggingEnabled", {})
+    if not logging_config:
+        return {
+            "bucket": bucket,
+            "logging_enabled": False,
+            "target_bucket": None,
+        }
+
+    return {
+        "bucket": bucket,
+        "logging_enabled": True,
+        "target_bucket": logging_config.get("TargetBucket"),
+    }
+
+
+@timeit
+def parse_notification_configuration(
+    bucket: str, notification_config: Optional[Dict]
+) -> List[Dict]:
+    """
+    Parse S3 bucket notification configuration to extract SNS topic notifications.
+    Returns a list of notification configurations.
+    """
+    if not notification_config or "TopicConfigurations" not in notification_config:
+        return []
+
+    notifications = []
+    for topic_config in notification_config.get("TopicConfigurations", []):
+        notification = {
+            "bucket": bucket,
+            "TopicArn": topic_config["TopicArn"],
+        }
+        notifications.append(notification)
+    return notifications
+
+
+@timeit
+def _load_s3_notifications(
+    neo4j_session: neo4j.Session,
+    notifications: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Ingest S3 bucket to SNS topic notification relationships into neo4j.
+    """
+    ingest_notifications = """
+    UNWIND $notifications AS notification
+    MATCH (bucket:S3Bucket{name: notification.bucket})
+    MATCH (topic:SNSTopic{arn: notification.TopicArn})
+    MERGE (bucket)-[r:NOTIFIES]->(topic)
+    ON CREATE SET r.firstseen = timestamp()
+    SET r.lastupdated = $UpdateTag
+    """
+    run_write_query(
+        neo4j_session,
+        ingest_notifications,
+        notifications=notifications,
+        UpdateTag=update_tag,
+    )
+
+
 @timeit
 def load_s3_buckets(
     neo4j_session: neo4j.Session,
@@ -776,7 +1037,8 @@ def load_s3_buckets(
 
     for bucket in data["Buckets"]:
         arn = "arn:aws:s3:::" + bucket["Name"]
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_bucket,
             BucketName=bucket["Name"],
             BucketRegion=bucket["Region"],
@@ -811,6 +1073,43 @@ def cleanup_s3_bucket_acl_and_policy(
     )
 
 
+@timeit
+@aws_handle_regions
+def _sync_s3_notifications(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    bucket_data: Dict,
+    update_tag: int,
+) -> None:
+    """
+    Sync S3 bucket notification configurations to Neo4j.
+    """
+    logger.info("Syncing S3 bucket notifications")
+    s3_client = boto3_session.client("s3")
+    notifications = []
+
+    for bucket in bucket_data["Buckets"]:
+        try:
+            notification_config = s3_client.get_bucket_notification_configuration(
+                Bucket=bucket["Name"]
+            )
+            parsed_notifications = parse_notification_configuration(
+                bucket["Name"], notification_config
+            )
+            notifications.extend(parsed_notifications)
+            logger.debug(
+                f"Found {len(parsed_notifications)} notifications for bucket {bucket['Name']}"
+            )
+        except ClientError as e:
+            logger.warning(
+                f"Failed to retrieve notification configuration for bucket {bucket['Name']}: {e}"
+            )
+            continue
+
+    logger.info(f"Loading {len(notifications)} S3 bucket notifications into Neo4j")
+    _load_s3_notifications(neo4j_session, notifications, update_tag)
+
+
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
@@ -820,9 +1119,16 @@ def sync(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
-    logger.info("Syncing S3 for account '%s'.", current_aws_account_id)
-    bucket_data = get_s3_bucket_list(boto3_session)
+    """
+    Sync S3 buckets and their configurations to Neo4j.
+    This includes:
+    1. Basic bucket information
+    2. ACLs and policies
+    3. Notification configurations
+    """
+    logger.info("Syncing S3 for account '%s'", current_aws_account_id)
 
+    bucket_data = get_s3_bucket_list(boto3_session)
     load_s3_buckets(neo4j_session, bucket_data, current_aws_account_id, update_tag)
     cleanup_s3_buckets(neo4j_session, common_job_parameters)
 
@@ -835,6 +1141,8 @@ def sync(
     )
     cleanup_s3_bucket_acl_and_policy(neo4j_session, common_job_parameters)
 
+    _sync_s3_notifications(neo4j_session, boto3_session, bucket_data, update_tag)
+
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",