cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/secretsmanager.py

@@ -7,6 +7,7 @@ import neo4j
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
+from cartography.models.aws.secretsmanager.secret import SecretsManagerSecretSchema
 from cartography.models.aws.secretsmanager.secret_version import (
     SecretsManagerSecretVersionSchema,
 )
@@ -14,7 +15,6 @@ from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import dict_date_to_epoch
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -32,6 +32,37 @@ def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[D
     return secrets
 
 
+def transform_secrets(
+    secrets: List[Dict],
+) -> List[Dict]:
+    """
+    Transform AWS Secrets Manager Secrets to match the data model.
+    """
+    transformed_data = []
+    for secret in secrets:
+        # Start with a copy of the original secret data
+        transformed = dict(secret)
+
+        # Convert date fields to epoch timestamps
+        transformed["CreatedDate"] = dict_date_to_epoch(secret, "CreatedDate")
+        transformed["LastRotatedDate"] = dict_date_to_epoch(secret, "LastRotatedDate")
+        transformed["LastChangedDate"] = dict_date_to_epoch(secret, "LastChangedDate")
+        transformed["LastAccessedDate"] = dict_date_to_epoch(secret, "LastAccessedDate")
+        transformed["DeletedDate"] = dict_date_to_epoch(secret, "DeletedDate")
+
+        # Flatten nested RotationRules.AutomaticallyAfterDays property
+        if "RotationRules" in secret and secret["RotationRules"]:
+            rotation_rules = secret["RotationRules"]
+            if "AutomaticallyAfterDays" in rotation_rules:
+                transformed["RotationRulesAutomaticallyAfterDays"] = rotation_rules[
+                    "AutomaticallyAfterDays"
+                ]
+
+        transformed_data.append(transformed)
+
+    return transformed_data
+
+
 @timeit
 def load_secrets(
     neo4j_session: neo4j.Session,
@@ -40,48 +71,33 @@ def load_secrets(
     current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
-    ingest_secrets = """
-    UNWIND $Secrets as secret
-        MERGE (s:SecretsManagerSecret{id: secret.ARN})
-        ON CREATE SET s.firstseen = timestamp()
-        SET s.name = secret.Name, s.arn = secret.ARN, s.description = secret.Description,
-            s.kms_key_id = secret.KmsKeyId, s.rotation_enabled = secret.RotationEnabled,
-            s.rotation_lambda_arn = secret.RotationLambdaARN,
-            s.rotation_rules_automatically_after_days = secret.RotationRules.AutomaticallyAfterDays,
-            s.last_rotated_date = secret.LastRotatedDate, s.last_changed_date = secret.LastChangedDate,
-            s.last_accessed_date = secret.LastAccessedDate, s.deleted_date = secret.DeletedDate,
-            s.owning_service = secret.OwningService, s.created_date = secret.CreatedDate,
-            s.primary_region = secret.PrimaryRegion, s.region = $Region,
-            s.lastupdated = $aws_update_tag
-        WITH s
-        MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (owner)-[r:RESOURCE]->(s)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    for secret in data:
-        secret["LastRotatedDate"] = dict_date_to_epoch(secret, "LastRotatedDate")
-        secret["LastChangedDate"] = dict_date_to_epoch(secret, "LastChangedDate")
-        secret["LastAccessedDate"] = dict_date_to_epoch(secret, "LastAccessedDate")
-        secret["DeletedDate"] = dict_date_to_epoch(secret, "DeletedDate")
-        secret["CreatedDate"] = dict_date_to_epoch(secret, "CreatedDate")
-
-    neo4j_session.run(
-        ingest_secrets,
-        Secrets=data,
+    """
+    Load transformed secrets into Neo4j using the data model.
+    Expects data to already be transformed by transform_secrets().
+    """
+    logger.info(f"Loading {len(data)} Secrets for region {region} into graph.")
+
+    # Load using the schema-based approach
+    load(
+        neo4j_session,
+        SecretsManagerSecretSchema(),
+        data,
+        lastupdated=aws_update_tag,
         Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
+        AWS_ID=current_aws_account_id,
     )
 
 
 @timeit
 def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        "aws_import_secrets_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    """
+    Run Secrets cleanup job using the data model.
+    """
+    logger.debug("Running Secrets cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(
+        SecretsManagerSecretSchema(), common_job_parameters
     )
+    cleanup_job.run(neo4j_session)
 
 
 @timeit
@@ -91,24 +107,36 @@ def get_secret_versions(
 ) -> List[Dict]:
     """
     Get all versions of a secret from AWS Secrets Manager.
+
+    Note: list_secret_version_ids is not paginatable through boto3's paginator,
+    so we implement manual pagination.
     """
     client = boto3_session.client("secretsmanager", region_name=region)
-    paginator = client.get_paginator("list_secret_version_ids")
+    next_token = None
     versions = []
 
-    for page in paginator.paginate(SecretId=secret_arn, IncludeDeprecated=True):
-        for version in page["Versions"]:
+    while True:
+        params = {"SecretId": secret_arn, "IncludeDeprecated": True}
+        if next_token:
+            params["NextToken"] = next_token
+
+        response = client.list_secret_version_ids(**params)
+
+        for version in response.get("Versions", []):
             version["SecretId"] = secret_arn
             version["ARN"] = f"{secret_arn}:version:{version['VersionId']}"
-        versions.extend(page["Versions"])
+
+        versions.extend(response.get("Versions", []))
+
+        next_token = response.get("NextToken")
+        if not next_token:
+            break
 
     return versions
 
 
 def transform_secret_versions(
     versions: List[Dict],
-    region: str,
-    aws_account_id: str,
 ) -> List[Dict]:
     """
     Transform AWS Secrets Manager Secret Versions to match the data model.
@@ -119,7 +147,7 @@ def transform_secret_versions(
             "ARN": version["ARN"],
             "SecretId": version["SecretId"],
             "VersionId": version["VersionId"],
-            "VersionStages": version["VersionStages"],
+            "VersionStages": version.get("VersionStages"),
             "CreatedDate": dict_date_to_epoch(version, "CreatedDate"),
         }
 
@@ -189,7 +217,15 @@ def sync(
         )
         secrets = get_secret_list(boto3_session, region)
 
-        load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
+        transformed_secrets = transform_secrets(secrets)
+
+        load_secrets(
+            neo4j_session,
+            transformed_secrets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
 
         all_versions = []
         for secret in secrets:
@@ -202,11 +238,7 @@ def sync(
             )
             all_versions.extend(versions)
 
-        transformed_data = transform_secret_versions(
-            all_versions,
-            region,
-            current_aws_account_id,
-        )
+        transformed_data = transform_secret_versions(all_versions)
 
         load_secret_versions(
             neo4j_session,

cartography/intel/aws/securityhub.py

@@ -6,6 +6,7 @@ import boto3
 import neo4j
 from dateutil import parser
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -50,7 +51,8 @@ def load_hub(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $aws_update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_hub,
         Hub=data,
         AWS_ACCOUNT_ID=current_aws_account_id,

cartography/intel/aws/sns.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -9,6 +10,7 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.models.aws.sns.topic import SNSTopicSchema
+from cartography.models.aws.sns.topic_subscription import SNSTopicSubscriptionSchema
 from cartography.stats import get_stats_client
 from cartography.util import aws_handle_regions
 from cartography.util import merge_module_sync_metadata
@@ -108,6 +110,48 @@ def load_sns_topics(
     )
 
 
+@timeit
+@aws_handle_regions
+def get_subscriptions(
+    boto3_session: boto3.session.Session, region: str
+) -> List[Dict[str, Any]]:
+    """
+    Get all SNS Topics Subscriptions for a region.
+    """
+    client = boto3_session.client("sns", region_name=region)
+    paginator = client.get_paginator("list_subscriptions")
+    subscriptions = []
+    for page in paginator.paginate():
+        subscriptions.extend(page.get("Subscriptions", []))
+
+    return subscriptions
+
+
+@timeit
+def load_sns_topic_subscription(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load SNS Topic Subscription information into the graph
+    """
+    logger.info(
+        f"Loading {len(data)} SNS topic subscription for region {region} into graph."
+    )
+
+    load(
+        neo4j_session,
+        SNSTopicSubscriptionSchema(),
+        data,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
 @timeit
 def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     """
@@ -117,6 +161,11 @@ def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> No
     cleanup_job = GraphJob.from_node_schema(SNSTopicSchema(), common_job_parameters)
     cleanup_job.run(neo4j_session)
 
+    cleanup_job = GraphJob.from_node_schema(
+        SNSTopicSubscriptionSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
 
 @timeit
 def sync(
@@ -128,7 +177,7 @@ def sync(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Sync SNS Topics for all regions
+    Sync SNS Topics and Subscriptions for all regions
     """
     for region in regions:
         logger.info(
@@ -153,9 +202,20 @@ def sync(
             update_tag,
         )
 
+        # Get and load subscriptions
+        subscriptions = get_subscriptions(boto3_session, region)
+
+        load_sns_topic_subscription(
+            neo4j_session,
+            subscriptions,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    # Cleanup and metadata update (outside region loop)
     cleanup_sns(neo4j_session, common_job_parameters)
 
-    # Record that we've synced this module
     merge_module_sync_metadata(
         neo4j_session,
         group_type="AWSAccount",

cartography/intel/aws/sqs.py

@@ -9,8 +9,10 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.sqs.queue import SQSQueueSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -33,9 +35,7 @@ def get_sqs_queue_attributes(
     boto3_session: boto3.session.Session,
     queue_urls: List[str],
 ) -> List[Tuple[str, Any]]:
-    """
-    Iterates over all SQS queues. Returns a dict with url as key, and attributes as value.
-    """
+    """Iterates over all SQS queues and returns a list of (url, attributes)."""
     client = boto3_session.client("sqs")
 
     queue_attributes = []
@@ -58,108 +58,53 @@ def get_sqs_queue_attributes(
     return queue_attributes
 
 
-@timeit
-def load_sqs_queues(
-    neo4j_session: neo4j.Session,
-    data: List[Tuple[str, Any]],
-    region: str,
-    current_aws_account_id: str,
-    aws_update_tag: int,
-) -> None:
-    ingest_queues = """
-    UNWIND $Queues as sqs_queue
-        MERGE (queue:SQSQueue{id: sqs_queue.QueueArn})
-        ON CREATE SET queue.firstseen = timestamp(), queue.url = sqs_queue.url
-        SET queue.name = sqs_queue.name, queue.region = $Region, queue.arn = sqs_queue.QueueArn,
-            queue.created_timestamp = sqs_queue.CreatedTimestamp, queue.delay_seconds = sqs_queue.DelaySeconds,
-            queue.last_modified_timestamp = sqs_queue.LastModifiedTimestamp,
-            queue.maximum_message_size = sqs_queue.MaximumMessageSize,
-            queue.message_retention_period = sqs_queue.MessageRetentionPeriod,
-            queue.policy = sqs_queue.Policy, queue.arn = sqs_queue.QueueArn,
-            queue.receive_message_wait_time_seconds = sqs_queue.ReceiveMessageWaitTimeSeconds,
-            queue.redrive_policy_dead_letter_target_arn = sqs_queue.RedrivePolicy.deadLetterTargetArn,
-            queue.redrive_policy_max_receive_count = sqs_queue.RedrivePolicy.maxReceiveCount,
-            queue.visibility_timeout = sqs_queue.VisibilityTimeout,
-            queue.kms_master_key_id = sqs_queue.KmsMasterKeyId,
-            queue.kms_data_key_reuse_period_seconds = sqs_queue.KmsDataKeyReusePeriodSeconds,
-            queue.fifo_queue = sqs_queue.FifoQueue,
-            queue.content_based_deduplication = sqs_queue.ContentBasedDeduplication,
-            queue.deduplication_scope = sqs_queue.DeduplicationScope,
-            queue.fifo_throughput_limit = sqs_queue.FifoThroughputLimit,
-            queue.lastupdated = $aws_update_tag
-        WITH queue
-        MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
-        MERGE (owner)-[r:RESOURCE]->(queue)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    dead_letter_queues: List[Dict] = []
-    queues: List[Dict] = []
-    for url, queue in data:
+def transform_sqs_queues(data: List[Tuple[str, Any]]) -> List[Dict[str, Any]]:
+    transformed: List[Dict[str, Any]] = []
+    for url, attrs in data:
+        queue = dict(attrs)
         queue["url"] = url
-        queue["name"] = queue["QueueArn"].split(":")[-1]
-        queue["CreatedTimestamp"] = int(queue["CreatedTimestamp"])
-        queue["LastModifiedTimestamp"] = int(queue["LastModifiedTimestamp"])
-        redrive_policy = queue.get("RedrivePolicy")
+        queue["name"] = attrs["QueueArn"].split(":")[-1]
+        queue["CreatedTimestamp"] = int(attrs.get("CreatedTimestamp", 0))
+        queue["LastModifiedTimestamp"] = int(attrs.get("LastModifiedTimestamp", 0))
+        redrive_policy = attrs.get("RedrivePolicy")
         if redrive_policy:
             try:
                 rp = json.loads(redrive_policy)
             except TypeError:
                 rp = {}
-            queue["RedrivePolicy"] = rp
-            dead_letter_arn = rp.get("deadLetterTargetArn")
-            if dead_letter_arn:
-                dead_letter_queues.append(
-                    {
-                        "arn": queue["QueueArn"],
-                        "dead_letter_arn": dead_letter_arn,
-                    },
-                )
-        queues.append(queue)
-
-    neo4j_session.run(
-        ingest_queues,
-        Queues=queues,
-        Region=region,
-        AWS_ACCOUNT_ID=current_aws_account_id,
-        aws_update_tag=aws_update_tag,
-    )
-
-    _attach_dead_letter_queues(neo4j_session, dead_letter_queues, aws_update_tag)
+            queue["redrive_policy_dead_letter_target_arn"] = rp.get(
+                "deadLetterTargetArn"
+            )
+            queue["redrive_policy_max_receive_count"] = rp.get("maxReceiveCount")
+        transformed.append(queue)
+    return transformed
 
 
 @timeit
-def _attach_dead_letter_queues(
+def load_sqs_queues(
     neo4j_session: neo4j.Session,
-    data: List[Dict[str, str]],
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
     aws_update_tag: int,
 ) -> None:
-    """
-    Attach deadletter queues to their queues.
-    """
-    attach_deadletter_to_queue = """
-    UNWIND $Relations as relation
-        MATCH (queue:SQSQueue{id: relation.arn}), (deadletter:SQSQueue{id: relation.dead_letter_arn})
-        MERGE (queue)-[r:HAS_DEADLETTER_QUEUE]->(deadletter)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $aws_update_tag
-    """
-    neo4j_session.run(
-        attach_deadletter_to_queue,
-        Relations=data,
-        aws_update_tag=aws_update_tag,
+    load(
+        neo4j_session,
+        SQSQueueSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
     )
 
 
 @timeit
 def cleanup_sqs_queues(
     neo4j_session: neo4j.Session,
-    common_job_parameters: Dict,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
-    run_cleanup_job(
-        "aws_import_sqs_queues_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    GraphJob.from_node_schema(SQSQueueSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -170,7 +115,7 @@ def sync(
     regions: List[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: Dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
@@ -179,12 +124,13 @@ def sync(
             current_aws_account_id,
         )
         queue_urls = get_sqs_queue_list(boto3_session, region)
-        if len(queue_urls) == 0:
+        if not queue_urls:
             continue
         queue_attributes = get_sqs_queue_attributes(boto3_session, queue_urls)
+        transformed = transform_sqs_queues(queue_attributes)
         load_sqs_queues(
             neo4j_session,
-            queue_attributes,
+            transformed,
             region,
             current_aws_account_id,
             update_tag,

cartography/intel/aws/ssm.py

@@ -9,6 +9,7 @@ import boto3
 import neo4j
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import read_list_of_values_tx
 from cartography.graph.job import GraphJob
 from cartography.models.aws.ssm.instance_information import SSMInstanceInformationSchema
 from cartography.models.aws.ssm.instance_patch import SSMInstancePatchSchema
@@ -31,15 +32,12 @@ def get_instance_ids(
     WHERE i.region = $Region
     RETURN i.id
     """
-    results = neo4j_session.run(
+    return neo4j_session.execute_read(
+        read_list_of_values_tx,
         get_instances_query,
         AWS_ACCOUNT_ID=current_aws_account_id,
         Region=region,
     )
-    instance_ids = []
-    for r in results:
-        instance_ids.append(r["i.id"])
-    return instance_ids
 
 
 @timeit