cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/aws/cloudwatch.py

@@ -9,7 +9,11 @@ import neo4j
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cloudwatch.log_metric_filter import (
+    CloudWatchLogMetricFilterSchema,
+)
 from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
+from cartography.models.aws.cloudwatch.metric_alarm import CloudWatchMetricAlarmSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -31,6 +35,84 @@ def get_cloudwatch_log_groups(
     return logGroups
 
 
+@timeit
+@aws_handle_regions
+def get_cloudwatch_log_metric_filters(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    logs_client = boto3_session.client(
+        "logs", region_name=region, config=get_botocore_config()
+    )
+    paginator = logs_client.get_paginator("describe_metric_filters")
+    metric_filters = []
+
+    for page in paginator.paginate():
+        metric_filters.extend(page.get("metricFilters", []))
+
+    return metric_filters
+
+
+def transform_metric_filters(
+    metric_filters: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudWatch log metric filter data for ingestion into Neo4j.
+    Ensures that the 'id' field is a unique combination of logGroupName and filterName.
+    """
+    transformed_filters = []
+    for filter in metric_filters:
+        transformed_filter = {
+            "id": f"{filter['logGroupName']}:{filter['filterName']}",
+            "arn": f"{filter['logGroupName']}:{filter['filterName']}",
+            "filterName": filter["filterName"],
+            "filterPattern": filter.get("filterPattern"),
+            "logGroupName": filter["logGroupName"],
+            "metricName": filter["metricTransformations"][0]["metricName"],
+            "metricNamespace": filter["metricTransformations"][0]["metricNamespace"],
+            "metricValue": filter["metricTransformations"][0]["metricValue"],
+            "Region": region,
+        }
+        transformed_filters.append(transformed_filter)
+    return transformed_filters
+
+
+@timeit
+@aws_handle_regions
+def get_cloudwatch_metric_alarms(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cloudwatch", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("describe_alarms")
+    alarms = []
+    for page in paginator.paginate():
+        alarms.extend(page["MetricAlarms"])
+    return alarms
+
+
+def transform_metric_alarms(
+    metric_alarms: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudWatch metric alarm data for ingestion into Neo4j.
+    """
+    transformed_alarms = []
+    for alarm in metric_alarms:
+        transformed_alarm = {
+            "AlarmArn": alarm["AlarmArn"],
+            "AlarmName": alarm.get("AlarmName"),
+            "AlarmDescription": alarm.get("AlarmDescription"),
+            "StateValue": alarm.get("StateValue"),
+            "StateReason": alarm.get("StateReason"),
+            "ActionsEnabled": alarm.get("ActionsEnabled"),
+            "ComparisonOperator": alarm.get("ComparisonOperator"),
+            "Region": region,
+        }
+        transformed_alarms.append(transformed_alarm)
+    return transformed_alarms
+
+
 @timeit
 def load_cloudwatch_log_groups(
     neo4j_session: neo4j.Session,
@@ -52,6 +134,48 @@ def load_cloudwatch_log_groups(
     )
 
 
+@timeit
+def load_cloudwatch_log_metric_filters(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} log metric filters for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchLogMetricFilterSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_cloudwatch_metric_alarms(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CloudWatch {len(data)} metric alarms for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CloudWatchMetricAlarmSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -62,6 +186,12 @@ def cleanup(
         CloudWatchLogGroupSchema(), common_job_parameters
     )
     cleanup_job.run(neo4j_session)
+    GraphJob.from_node_schema(
+        CloudWatchLogMetricFilterSchema(), common_job_parameters
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(CloudWatchMetricAlarmSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -78,16 +208,32 @@ def sync(
             f"Syncing CloudWatch for region '{region}' in account '{current_aws_account_id}'.",
         )
         logGroups = get_cloudwatch_log_groups(boto3_session, region)
-        group_data: List[Dict[str, Any]] = []
-        for logGroup in logGroups:
-            group_data.append(logGroup)
 
         load_cloudwatch_log_groups(
             neo4j_session,
-            group_data,
+            logGroups,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        metric_filters = get_cloudwatch_log_metric_filters(boto3_session, region)
+        transformed_filters = transform_metric_filters(metric_filters, region)
+        load_cloudwatch_log_metric_filters(
+            neo4j_session,
+            transformed_filters,
             region,
             current_aws_account_id,
             update_tag,
         )
 
+        metric_alarms = get_cloudwatch_metric_alarms(boto3_session, region)
+        transformed_alarms = transform_metric_alarms(metric_alarms, region)
+        load_cloudwatch_metric_alarms(
+            neo4j_session,
+            transformed_alarms,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/codebuild.py

@@ -0,0 +1,132 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.codebuild.project import CodeBuildProjectSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_all_codebuild_projects(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+
+    client = boto3_session.client(
+        "codebuild", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_projects")
+
+    all_projects = []
+
+    for page in paginator.paginate():
+        project_names = page.get("projects", [])
+        if not project_names:
+            continue
+
+        # AWS batch_get_projects accepts up to 100 project names per call as per AWS documentation.
+        for i in range(0, len(project_names), 100):
+            batch = project_names[i : i + 100]
+            response = client.batch_get_projects(names=batch)
+            projects = response.get("projects", [])
+            all_projects.extend(projects)
+    return all_projects
+
+
+def transform_codebuild_projects(
+    projects: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CodeBuild project data for ingestion into Neo4j.
+
+    - Includes all environment variable names.
+    - Variables of type 'PLAINTEXT' retain their values.
+    - Other types (e.g., 'PARAMETER_STORE', 'SECRETS_MANAGER') have their values redacted.
+    """
+    transformed_codebuild_projects = []
+    for project in projects:
+        env_vars = project.get("environment", {}).get("environmentVariables", [])
+        env_var_strings = [
+            f"{var.get('name')}={var.get('value') if var.get('type') == 'PLAINTEXT' else '<REDACTED>'}"
+            for var in env_vars
+        ]
+        transformed_project = {
+            "arn": project["arn"],
+            "created": project.get("created"),
+            "environmentVariables": env_var_strings,
+            "sourceType": project.get("source", {}).get("type"),
+            "sourceLocation": project.get("source", {}).get("location"),
+        }
+        transformed_codebuild_projects.append(transformed_project)
+
+    return transformed_codebuild_projects
+
+
+@timeit
+def load_codebuild_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading CodeBuild {len(data)} projects for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CodeBuildProjectSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CodeBuildProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing CodeBuild for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        projects = get_all_codebuild_projects(boto3_session, region)
+        transformed_projects = transform_codebuild_projects(projects, region)
+
+        load_codebuild_projects(
+            neo4j_session,
+            transformed_projects,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/cognito.py

@@ -0,0 +1,201 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cognito.identity_pool import CognitoIdentityPoolSchema
+from cartography.models.aws.cognito.user_pool import CognitoUserPoolSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pools(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_identity_pools")
+
+    all_identity_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        identity_pools = page.get("IdentityPools", [])
+        all_identity_pools.extend(identity_pools)
+    return all_identity_pools
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pool_roles(
+    boto3_session: boto3.Session, identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    all_identity_pool_details = []
+    for identity_pool in identity_pools:
+        response = client.get_identity_pool_roles(
+            IdentityPoolId=identity_pool["IdentityPoolId"]
+        )
+        all_identity_pool_details.append(response)
+    return all_identity_pool_details
+
+
+@timeit
+@aws_handle_regions
+def get_user_pools(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-idp", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_user_pools")
+    all_user_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        user_pools = page.get("UserPools", [])
+        all_user_pools.extend(user_pools)
+    return all_user_pools
+
+
+def transform_identity_pools(
+    identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_identity_pools = []
+    for pool in identity_pools:
+        transformed_pool = {
+            "IdentityPoolId": pool["IdentityPoolId"],
+            "Region": region,
+            "Roles": list(pool.get("Roles", {}).values()),
+        }
+        transformed_identity_pools.append(transformed_pool)
+    return transformed_identity_pools
+
+
+def transform_user_pools(
+    user_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_user_pools = []
+    for pool in user_pools:
+        transformed_pool = {
+            "Id": pool["Id"],
+            "Region": region,
+            "Name": pool["Name"],
+            "Status": pool.get("Status"),
+        }
+        transformed_user_pools.append(transformed_pool)
+    return transformed_user_pools
+
+
+@timeit
+def load_identity_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito Identity Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoIdentityPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_user_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito User Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoUserPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CognitoIdentityPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(CognitoUserPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Cognito Identity Pools for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        identity_pools = get_identity_pools(boto3_session, region)
+        if not identity_pools:
+            logger.info(
+                f"No Cognito Identity Pools found in region '{region}'. Skipping sync."
+            )
+        else:
+            identity_pool_roles = get_identity_pool_roles(
+                boto3_session, identity_pools, region
+            )
+            transformed_identity_pools = transform_identity_pools(
+                identity_pool_roles, region
+            )
+
+            load_identity_pools(
+                neo4j_session,
+                transformed_identity_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            user_pools = get_user_pools(boto3_session, region)
+            transformed_user_pools = transform_user_pools(user_pools, region)
+
+            load_user_pools(
+                neo4j_session,
+                transformed_user_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/config.py

@@ -5,6 +5,7 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -80,7 +81,8 @@ def load_configuration_recorders(
     for recorder in data:
         recorder["_id"] = f'{recorder["name"]}:{current_aws_account_id}:{region}'
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_configuration_recorders,
         Recorders=data,
         Region=region,
@@ -120,7 +122,8 @@ def load_delivery_channels(
     for channel in data:
         channel["_id"] = f'{channel["name"]}:{current_aws_account_id}:{region}'
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_delivery_channels,
         Channels=data,
         Region=region,
@@ -167,7 +170,8 @@ def load_config_rules(
             for detail in rule["Source"]["SourceDetails"]:
                 details.append(f"{detail}")
         rule["_source_details"] = details
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_config_rules,
         Rules=data,
         Region=region,

cartography/intel/aws/ec2/elastic_ip_addresses.py

@@ -6,6 +6,7 @@ import boto3
 import neo4j
 from botocore.exceptions import ClientError
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -83,7 +84,8 @@ def load_elastic_ip_addresses(
         SET r.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_addresses,
         elastic_ip_addresses=elastic_ip_addresses,
         Region=region,

cartography/intel/aws/ec2/instances.py

@@ -4,6 +4,7 @@ from collections import namedtuple
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 
 import boto3
 import neo4j
@@ -44,6 +45,25 @@ Ec2Data = namedtuple(
 )
 
 
+def _get_eks_cluster_name(tags: List[Dict[str, str]]) -> Optional[str]:
+    for tag in tags:
+        key = tag.get("Key")
+        value = tag.get("Value")
+
+        if key == "eks:cluster-name" and value:
+            return value
+
+        if key == "alpha.eksctl.io/cluster-name" and value:
+            return value
+
+        if key and key.startswith("kubernetes.io/cluster/"):
+            cluster_name = key.split("kubernetes.io/cluster/")[-1]
+            if cluster_name:
+                return cluster_name
+
+    return None
+
+
 @timeit
 @aws_handle_regions
 def get_ec2_instances(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
@@ -87,6 +107,7 @@ def transform_ec2_instances(
             launch_time_unix = (
                 str(time.mktime(launch_time.timetuple())) if launch_time else None
             )
+            eks_cluster_name = _get_eks_cluster_name(instance.get("Tags", []))
             instance_list.append(
                 {
                     "InstanceId": instance_id,
@@ -118,6 +139,7 @@ def transform_ec2_instances(
                     "HibernationOptions": instance.get("HibernationOptions", {}).get(
                         "Configured",
                     ),
+                    "EksClusterName": eks_cluster_name,
                 },
             )
 
@@ -163,7 +185,9 @@ def transform_ec2_instances(
                             "MacAddress": network_interface["MacAddress"],
                             "Description": network_interface["Description"],
                             "PrivateDnsName": network_interface.get("PrivateDnsName"),
-                            "PrivateIpAddress": network_interface["PrivateIpAddress"],
+                            "PrivateIpAddress": network_interface.get(
+                                "PrivateIpAddress"
+                            ),
                             "InstanceId": instance_id,
                             "SubnetId": subnet_id,
                             "GroupId": security_group["GroupId"],

cartography/intel/aws/ec2/internet_gateways.py

@@ -5,6 +5,7 @@ from typing import List
 import boto3
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -63,13 +64,14 @@ def load_internet_gateways(
         SET r.lastupdated = $aws_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         query,
         internet_gateways=internet_gateways,
         region=region,
         aws_account_id=current_aws_account_id,
         aws_update_tag=update_tag,
-    ).consume()
+    )
 
 
 @timeit