cartography 0.110.0rc1__py3-none-any.whl → 0.111.0__py3-none-any.whl

cartography/intel/aws/cognito.py

@@ -0,0 +1,201 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.cognito.identity_pool import CognitoIdentityPoolSchema
+from cartography.models.aws.cognito.user_pool import CognitoUserPoolSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pools(
+    boto3_session: boto3.Session, region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_identity_pools")
+
+    all_identity_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        identity_pools = page.get("IdentityPools", [])
+        all_identity_pools.extend(identity_pools)
+    return all_identity_pools
+
+
+@timeit
+@aws_handle_regions
+def get_identity_pool_roles(
+    boto3_session: boto3.Session, identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-identity", region_name=region, config=get_botocore_config()
+    )
+    all_identity_pool_details = []
+    for identity_pool in identity_pools:
+        response = client.get_identity_pool_roles(
+            IdentityPoolId=identity_pool["IdentityPoolId"]
+        )
+        all_identity_pool_details.append(response)
+    return all_identity_pool_details
+
+
+@timeit
+@aws_handle_regions
+def get_user_pools(boto3_session: boto3.Session, region: str) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "cognito-idp", region_name=region, config=get_botocore_config()
+    )
+    paginator = client.get_paginator("list_user_pools")
+    all_user_pools = []
+
+    for page in paginator.paginate(MaxResults=50):
+        user_pools = page.get("UserPools", [])
+        all_user_pools.extend(user_pools)
+    return all_user_pools
+
+
+def transform_identity_pools(
+    identity_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_identity_pools = []
+    for pool in identity_pools:
+        transformed_pool = {
+            "IdentityPoolId": pool["IdentityPoolId"],
+            "Region": region,
+            "Roles": list(pool.get("Roles", {}).values()),
+        }
+        transformed_identity_pools.append(transformed_pool)
+    return transformed_identity_pools
+
+
+def transform_user_pools(
+    user_pools: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    transformed_user_pools = []
+    for pool in user_pools:
+        transformed_pool = {
+            "Id": pool["Id"],
+            "Region": region,
+            "Name": pool["Name"],
+            "Status": pool.get("Status"),
+        }
+        transformed_user_pools.append(transformed_pool)
+    return transformed_user_pools
+
+
+@timeit
+def load_identity_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito Identity Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoIdentityPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def load_user_pools(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading Cognito User Pools {len(data)} for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        CognitoUserPoolSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    logger.debug("Running Efs cleanup job.")
+    GraphJob.from_node_schema(CognitoIdentityPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(CognitoUserPoolSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(
+            f"Syncing Cognito Identity Pools for region '{region}' in account '{current_aws_account_id}'.",
+        )
+
+        identity_pools = get_identity_pools(boto3_session, region)
+        if not identity_pools:
+            logger.info(
+                f"No Cognito Identity Pools found in region '{region}'. Skipping sync."
+            )
+        else:
+            identity_pool_roles = get_identity_pool_roles(
+                boto3_session, identity_pools, region
+            )
+            transformed_identity_pools = transform_identity_pools(
+                identity_pool_roles, region
+            )
+
+            load_identity_pools(
+                neo4j_session,
+                transformed_identity_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            user_pools = get_user_pools(boto3_session, region)
+            transformed_user_pools = transform_user_pools(user_pools, region)
+
+            load_user_pools(
+                neo4j_session,
+                transformed_user_pools,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/vpc.py

@@ -1,13 +1,15 @@
 import logging
-from string import Template
-from typing import Dict
-from typing import List
+from typing import Any
 
 import boto3
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.vpc import AWSVpcSchema
+from cartography.models.aws.ec2.vpc_cidr import AWSIPv4CidrBlockSchema
+from cartography.models.aws.ec2.vpc_cidr import AWSIPv6CidrBlockSchema
 from cartography.util import aws_handle_regions
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 from .util import get_botocore_config
@@ -17,87 +19,78 @@ logger = logging.getLogger(__name__)
 
 @timeit
 @aws_handle_regions
-def get_ec2_vpcs(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+def get_ec2_vpcs(
+    boto3_session: boto3.session.Session,
+    region: str,
+) -> list[dict[str, Any]]:
     client = boto3_session.client(
         "ec2",
         region_name=region,
         config=get_botocore_config(),
     )
-    return client.describe_vpcs()["Vpcs"]
-
-
-def _get_cidr_association_statement(block_type: str) -> str:
-    INGEST_CIDR_TEMPLATE = Template(
-        """
-    MATCH (vpc:AWSVpc{id: $VpcId})
-    WITH vpc
-    UNWIND $CidrBlock as block_data
-        MERGE (new_block:$block_label{id: $VpcId + '|' + block_data.$block_cidr})
-        ON CREATE SET new_block.firstseen = timestamp()
-        SET new_block.association_id = block_data.AssociationId,
-        new_block.cidr_block = block_data.$block_cidr,
-        new_block.block_state = block_data.$state_name.State,
-        new_block.block_state_message = block_data.$state_name.StatusMessage,
-        new_block.lastupdated = $update_tag
-        WITH vpc, new_block
-        MERGE (vpc)-[r:BLOCK_ASSOCIATION]->(new_block)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $update_tag""",
-    )
-
-    BLOCK_CIDR = "CidrBlock"
-    STATE_NAME = "CidrBlockState"
-
-    # base label type. We add the AWS ipv4 or 6 depending on block type
-    BLOCK_TYPE = "AWSCidrBlock"
-
-    if block_type == "ipv6":
-        BLOCK_CIDR = "Ipv6" + BLOCK_CIDR
-        STATE_NAME = "Ipv6" + STATE_NAME
-        BLOCK_TYPE = BLOCK_TYPE + ":AWSIpv6CidrBlock"
-    elif block_type == "ipv4":
-        BLOCK_TYPE = BLOCK_TYPE + ":AWSIpv4CidrBlock"
-    else:
-        raise ValueError(f"Unsupported block type specified - {block_type}")
-
-    return INGEST_CIDR_TEMPLATE.safe_substitute(
-        block_label=BLOCK_TYPE,
-        block_cidr=BLOCK_CIDR,
-        state_name=STATE_NAME,
-    )
-
-
-@timeit
-def load_cidr_association_set(
-    neo4j_session: neo4j.Session,
-    vpc_id: str,
-    vpc_data: Dict,
-    block_type: str,
-    update_tag: int,
-) -> None:
-    ingest_statement = _get_cidr_association_statement(block_type)
-
-    if block_type == "ipv6":
-        data = vpc_data.get("Ipv6CidrBlockAssociationSet", [])
-    else:
-        data = vpc_data.get("CidrBlockAssociationSet", [])
-
-    neo4j_session.run(
-        ingest_statement,
-        VpcId=vpc_id,
-        CidrBlock=data,
-        update_tag=update_tag,
-    )
+    return client.describe_vpcs().get("Vpcs", [])
+
+
+def transform_vpc_data(
+    vpc_list: list[dict[str, Any]], region: str
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+
+    vpc_data: list[dict[str, Any]] = []
+    ipv4_cidr_blocks: list[dict[str, Any]] = []
+    ipv6_cidr_blocks: list[dict[str, Any]] = []
+
+    for vpc in vpc_list:
+        vpc_record = {
+            "VpcId": vpc.get("VpcId"),
+            "InstanceTenancy": vpc.get("InstanceTenancy"),
+            "State": vpc.get("State"),
+            "IsDefault": vpc.get("IsDefault"),
+            "PrimaryCIDRBlock": vpc.get("CidrBlock"),
+            "DhcpOptionsId": vpc.get("DhcpOptionsId"),
+            "lastupdated": vpc.get("lastupdated"),
+        }
+        vpc_data.append(vpc_record)
+
+        ipv4_associations = vpc.get("CidrBlockAssociationSet", [])
+        for association in ipv4_associations:
+            ipv4_block = {
+                "Id": vpc["VpcId"] + "|" + association.get("CidrBlock"),
+                "VpcId": vpc["VpcId"],
+                "AssociationId": association.get("AssociationId"),
+                "CidrBlock": association.get("CidrBlock"),
+                "BlockState": association.get("CidrBlockState", {}).get("State"),
+                "BlockStateMessage": association.get("CidrBlockState", {}).get(
+                    "StatusMessage"
+                ),
+            }
+            ipv4_cidr_blocks.append(ipv4_block)
+
+        ipv6_associations = vpc.get("Ipv6CidrBlockAssociationSet", [])
+        for association in ipv6_associations:
+            ipv6_block = {
+                "Id": vpc["VpcId"] + "|" + association.get("Ipv6CidrBlock"),
+                "VpcId": vpc["VpcId"],
+                "AssociationId": association.get("AssociationId"),
+                "CidrBlock": association.get("Ipv6CidrBlock"),
+                "BlockState": association.get("Ipv6CidrBlockState", {}).get("State"),
+                "BlockStateMessage": association.get("Ipv6CidrBlockState", {}).get(
+                    "StatusMessage"
+                ),
+            }
+            ipv6_cidr_blocks.append(ipv6_block)
+
+    return vpc_data, ipv4_cidr_blocks, ipv6_cidr_blocks
 
 
 @timeit
 def load_ec2_vpcs(
     neo4j_session: neo4j.Session,
-    data: List[Dict],
+    vpcs: list[dict[str, Any]],
     region: str,
-    current_aws_account_id: str,
+    aws_account_id: str,
     update_tag: int,
 ) -> None:
+    logger.info(f"Loading {len(vpcs)} EC2 VPCs for region '{region}' into graph.")
     # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-vpcs.html
     # {
     #     "Vpcs": [
@@ -126,69 +119,69 @@ def load_ec2_vpcs(
     #         }
     #     ]
     # }
+    load(
+        neo4j_session,
+        AWSVpcSchema(),
+        vpcs,
+        lastupdated=update_tag,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
 
-    ingest_vpc = """
-    MERGE (new_vpc:AWSVpc{id: $VpcId})
-    ON CREATE SET new_vpc.firstseen = timestamp(), new_vpc.vpcid =$VpcId
-    SET new_vpc.instance_tenancy = $InstanceTenancy,
-    new_vpc.state = $State,
-    new_vpc.is_default = $IsDefault,
-    new_vpc.primary_cidr_block = $PrimaryCIDRBlock,
-    new_vpc.dhcp_options_id = $DhcpOptionsId,
-    new_vpc.region = $Region,
-    new_vpc.lastupdated = $update_tag
-    WITH new_vpc
-    MATCH (awsAccount:AWSAccount{id: $AWS_ACCOUNT_ID})
-    MERGE (awsAccount)-[r:RESOURCE]->(new_vpc)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag"""
-
-    for vpc in data:
-        vpc_id = vpc["VpcId"]  # fail if not present
-
-        neo4j_session.run(
-            ingest_vpc,
-            VpcId=vpc_id,
-            InstanceTenancy=vpc.get("InstanceTenancy", None),
-            State=vpc.get("State", None),
-            IsDefault=vpc.get("IsDefault", None),
-            PrimaryCIDRBlock=vpc.get("CidrBlock", None),
-            DhcpOptionsId=vpc.get("DhcpOptionsId", None),
-            Region=region,
-            AWS_ACCOUNT_ID=current_aws_account_id,
-            update_tag=update_tag,
-        )
 
-        load_cidr_association_set(
-            neo4j_session,
-            vpc_id=vpc_id,
-            block_type="ipv4",
-            vpc_data=vpc,
-            update_tag=update_tag,
-        )
+@timeit
+def load_ipv4_cidr_blocks(
+    neo4j_session: neo4j.Session,
+    ipv4_cidr_blocks: list[dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AWSIPv4CidrBlockSchema(),
+        ipv4_cidr_blocks,
+        lastupdated=update_tag,
+    )
 
-        load_cidr_association_set(
-            neo4j_session,
-            vpc_id=vpc_id,
-            block_type="ipv6",
-            vpc_data=vpc,
-            update_tag=update_tag,
-        )
+
+@timeit
+def load_ipv6_cidr_blocks(
+    neo4j_session: neo4j.Session,
+    ipv6_cidr_blocks: list[dict[str, Any]],
+    region: str,
+    aws_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AWSIPv6CidrBlockSchema(),
+        ipv6_cidr_blocks,
+        lastupdated=update_tag,
+    )
 
 
 @timeit
-def cleanup_ec2_vpcs(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job("aws_import_vpc_cleanup.json", neo4j_session, common_job_parameters)
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AWSIPv6CidrBlockSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AWSIPv4CidrBlockSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AWSVpcSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit
 def sync_vpc(
     neo4j_session: neo4j.Session,
     boto3_session: boto3.session.Session,
-    regions: List[str],
+    regions: list[str],
     current_aws_account_id: str,
     update_tag: int,
-    common_job_parameters: Dict,
+    common_job_parameters: dict[str, Any],
 ) -> None:
     for region in regions:
         logger.info(
@@ -196,6 +189,29 @@ def sync_vpc(
             region,
             current_aws_account_id,
         )
-        data = get_ec2_vpcs(boto3_session, region)
-        load_ec2_vpcs(neo4j_session, data, region, current_aws_account_id, update_tag)
-    cleanup_ec2_vpcs(neo4j_session, common_job_parameters)
+        raw_vpc_data = get_ec2_vpcs(boto3_session, region)
+        vpc_data, ipv4_cidr_blocks, ipv6_cidr_blocks = transform_vpc_data(
+            raw_vpc_data, region
+        )
+        load_ec2_vpcs(
+            neo4j_session,
+            vpc_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        load_ipv4_cidr_blocks(
+            neo4j_session,
+            ipv4_cidr_blocks,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        load_ipv6_cidr_blocks(
+            neo4j_session,
+            ipv6_cidr_blocks,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/ecs.py

@@ -171,9 +171,15 @@ def _get_containers_from_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, An
 
 def transform_ecs_tasks(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """
-    Extract network interface ID from task attachments.
+    Extract network interface ID from task attachments and service name from group.
     """
     for task in tasks:
+        # Extract serviceName from group field
+        group = task.get("group")
+        if group and group.startswith("service:"):
+            task["serviceName"] = group.split("service:", 1)[1]
+
+        # Extract network interface ID from task attachments
         for attachment in task.get("attachments", []):
             if attachment.get("type") == "ElasticNetworkInterface":
                 details = attachment.get("details", [])

cartography/intel/aws/eventbridge.py

@@ -10,6 +10,7 @@ from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
 from cartography.intel.aws.ec2.util import get_botocore_config
 from cartography.models.aws.eventbridge.rule import EventBridgeRuleSchema
+from cartography.models.aws.eventbridge.target import EventBridgeTargetSchema
 from cartography.util import aws_handle_regions
 from cartography.util import timeit
 
@@ -33,6 +34,44 @@ def get_eventbridge_rules(
     return rules
 
 
+@timeit
+@aws_handle_regions
+def get_eventbridge_targets(
+    boto3_session: boto3.Session, region: str, rules: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    client = boto3_session.client(
+        "events", region_name=region, config=get_botocore_config()
+    )
+    targets = []
+    for rule in rules:
+        paginator = client.get_paginator("list_targets_by_rule")
+        for page in paginator.paginate(Rule=rule["Name"]):
+            for target in page.get("Targets", []):
+                target["RuleArn"] = rule["Arn"]
+                targets.append(target)
+    return targets
+
+
+def transform_eventbridge_targets(
+    targets: List[Dict[str, Any]],
+    region: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform EventBridge target data for ingestion into Neo4j.
+    """
+    transformed_data = []
+    for target in targets:
+        transformed_target = {
+            "Id": target["Arn"],
+            "Arn": target["Arn"],
+            "RuleArn": target["RuleArn"],
+            "RoleArn": target.get("RoleArn"),
+            "Region": region,
+        }
+        transformed_data.append(transformed_target)
+    return transformed_data
+
+
 @timeit
 def load_eventbridge_rules(
     neo4j_session: neo4j.Session,
@@ -54,6 +93,27 @@ def load_eventbridge_rules(
     )
 
 
+@timeit
+def load_eventbridge_targets(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    logger.info(
+        f"Loading EventBridge {len(data)} targets for region '{region}' into graph.",
+    )
+    load(
+        neo4j_session,
+        EventBridgeTargetSchema(),
+        data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
 @timeit
 def cleanup(
     neo4j_session: neo4j.Session,
@@ -63,6 +123,9 @@ def cleanup(
     GraphJob.from_node_schema(EventBridgeRuleSchema(), common_job_parameters).run(
         neo4j_session
     )
+    GraphJob.from_node_schema(EventBridgeTargetSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -88,4 +151,14 @@ def sync(
             update_tag,
         )
 
+        targets = get_eventbridge_targets(boto3_session, region, rules)
+        transformed_targets = transform_eventbridge_targets(targets, region)
+        load_eventbridge_targets(
+            neo4j_session,
+            transformed_targets,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
     cleanup(neo4j_session, common_job_parameters)