cartography 0.95.0rc1__py3-none-any.whl → 0.96.0__py3-none-any.whl

cartography/intel/aws/ec2/network_acls.py

@@ -0,0 +1,209 @@
+import logging
+from collections import namedtuple
+from typing import Any
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclEgressRuleSchema
+from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclInboundRuleSchema
+from cartography.models.aws.ec2.network_acls import EC2NetworkAclSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+Ec2AclObjects = namedtuple(
+    "Ec2AclObjects", [
+        'network_acls',
+        'inbound_rules',
+        'outbound_rules',
+    ],
+)
+
+
+@timeit
+@aws_handle_regions
+def get_network_acl_data(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_network_acls')
+    acls = []
+    for page in paginator.paginate():
+        acls.extend(page['NetworkAcls'])
+    return acls
+
+
+def transform_network_acl_data(
+        data_list: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+) -> Ec2AclObjects:
+    network_acls = []
+    inbound_rules = []
+    outbound_rules = []
+
+    for network_acl in data_list:
+        network_acl_id = network_acl['NetworkAclId']
+        base_network_acl = {
+            'Id': network_acl_id,
+            'Arn': f'arn:aws:ec2:{region}:{current_aws_account_id}:network-acl/{network_acl_id}',
+            'IsDefault': network_acl['IsDefault'],
+            'VpcId': network_acl['VpcId'],
+            'OwnerId': network_acl['OwnerId'],
+        }
+        if network_acl.get('Associations') and network_acl['Associations']:
+            # Include subnet associations in the data object if they exist
+            for association in network_acl['Associations']:
+                base_network_acl['NetworkAclAssociationId'] = association['NetworkAclAssociationId']
+                base_network_acl['SubnetId'] = association['SubnetId']
+                network_acls.append(base_network_acl)
+        else:
+            # Otherwise if there's no associations then don't include that in the data object
+            network_acls.append(base_network_acl)
+
+        if network_acl.get("Entries"):
+            for rule in network_acl["Entries"]:
+                direction = 'egress' if rule['Egress'] else 'inbound'
+                transformed_rule = {
+                    'Id': f"{network_acl['NetworkAclId']}/{direction}/{rule['RuleNumber']}",
+                    'CidrBlock': rule.get('CidrBlock'),
+                    'Ipv6CidrBlock': rule.get('Ipv6CidrBlock'),
+                    'Egress': rule['Egress'],
+                    'Protocol': rule['Protocol'],
+                    'RuleAction': rule['RuleAction'],
+                    'RuleNumber': rule['RuleNumber'],
+                    # Add pointer back to the nacl to create an edge
+                    'NetworkAclId': network_acl_id,
+                    'FromPort': rule.get('PortRange', {}).get('FromPort'),
+                    'ToPort': rule.get('PortRange', {}).get('ToPort'),
+                }
+                if transformed_rule['Egress']:
+                    outbound_rules.append(transformed_rule)
+                else:
+                    inbound_rules.append(transformed_rule)
+    return Ec2AclObjects(
+        network_acls=network_acls,
+        inbound_rules=inbound_rules,
+        outbound_rules=outbound_rules,
+    )
+
+
+@timeit
+def load_all_nacl_data(
+        neo4j_session: neo4j.Session,
+        ec2_acl_objects: Ec2AclObjects,
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load_network_acls(
+        neo4j_session,
+        ec2_acl_objects.network_acls,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+    load_network_acl_inbound_rules(
+        neo4j_session,
+        ec2_acl_objects.inbound_rules,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+    load_network_acl_egress_rules(
+        neo4j_session,
+        ec2_acl_objects.outbound_rules,
+        region,
+        aws_account_id,
+        update_tag,
+    )
+
+
+@timeit
+def load_network_acls(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acls in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_network_acl_inbound_rules(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acl inbound rules in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclInboundRuleSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_network_acl_egress_rules(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        aws_account_id: str,
+        update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(data)} network acl egress rules in {region}.")
+    load(
+        neo4j_session,
+        EC2NetworkAclEgressRuleSchema(),
+        data,
+        Region=region,
+        AWS_ID=aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup_network_acls(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    GraphJob.from_node_schema(EC2NetworkAclSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2NetworkAclInboundRuleSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2NetworkAclEgressRuleSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_network_acls(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(f"Syncing EC2 network ACLs for region '{region}' in account '{current_aws_account_id}'.")
+        data = get_network_acl_data(boto3_session, region)
+        ec2_acl_data = transform_network_acl_data(data, region, current_aws_account_id)
+        load_all_nacl_data(
+            neo4j_session,
+            ec2_acl_data,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+        cleanup_network_acls(neo4j_session, common_job_parameters)

cartography/intel/aws/ec2/subnets.py

@@ -7,6 +7,7 @@ import neo4j
 
 from .util import get_botocore_config
 from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema
 from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
 from cartography.util import aws_handle_regions
 from cartography.util import run_cleanup_job
@@ -79,6 +80,7 @@ def load_subnets(
 def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
     run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters)
     GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(EC2SubnetAutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
 
 
 @timeit

cartography/intel/aws/iam.py

@@ -539,11 +539,12 @@ def _transform_policy_statements(statements: Any, policy_id: str) -> List[Dict]:
     if not isinstance(statements, list):
         statements = [statements]
     for stmt in statements:
-        if "Sid" not in stmt:
+        if "Sid" in stmt and stmt["Sid"]:
+            statement_id = stmt["Sid"]
+        else:
             statement_id = count
             count += 1
-        else:
-            statement_id = stmt["Sid"]
+
         stmt["id"] = f"{policy_id}/statement/{statement_id}"
         if "Resource" in stmt:
             stmt["Resource"] = ensure_list(stmt["Resource"])

cartography/intel/aws/identitycenter.py

@@ -0,0 +1,307 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.identitycenter.awsidentitycenter import AWSIdentityCenterInstanceSchema
+from cartography.models.aws.identitycenter.awspermissionset import AWSPermissionSetSchema
+from cartography.models.aws.identitycenter.awsssouser import AWSSSOUserSchema
+from cartography.util import aws_handle_regions
+from cartography.util import run_cleanup_job
+from cartography.util import timeit
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_identity_center_instances(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
+    """
+    Get all AWS IAM Identity Center instances in the current region
+    """
+    client = boto3_session.client('sso-admin', region_name=region)
+    instances = []
+
+    paginator = client.get_paginator('list_instances')
+    for page in paginator.paginate():
+        instances.extend(page.get('Instances', []))
+
+    return instances
+
+
+@timeit
+def load_identity_center_instances(
+    neo4j_session: neo4j.Session,
+    instance_data: List[Dict],
+    region: str,
+    current_aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load Identity Center instances into the graph
+    """
+    logger.info(f"Loading {len(instance_data)} Identity Center instances for region {region}")
+    load(
+        neo4j_session,
+        AWSIdentityCenterInstanceSchema(),
+        instance_data,
+        lastupdated=aws_update_tag,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+    )
+
+
+@timeit
+def get_permission_sets(boto3_session: boto3.session.Session, instance_arn: str, region: str) -> List[Dict]:
+    """
+    Get all permission sets for a given Identity Center instance
+    """
+    client = boto3_session.client('sso-admin', region_name=region)
+    permission_sets = []
+
+    paginator = client.get_paginator('list_permission_sets')
+    for page in paginator.paginate(InstanceArn=instance_arn):
+        # Get detailed info for each permission set
+        for arn in page.get('PermissionSets', []):
+            details = client.describe_permission_set(
+                InstanceArn=instance_arn,
+                PermissionSetArn=arn,
+            )
+            permission_set = details.get('PermissionSet', {})
+            if permission_set:
+                permission_set['RoleHint'] = (
+                    f":role/aws-reserved/sso.amazonaws.com/AWSReservedSSO_{permission_set.get('Name')}"
+                )
+                permission_sets.append(permission_set)
+
+    return permission_sets
+
+
+@timeit
+def get_permission_set_roles(
+    boto3_session: boto3.session.Session,
+    instance_arn: str,
+    permission_set_arn: str,
+    region: str,
+) -> List[Dict]:
+    """
+    Get all accounts associated with a given permission set
+    """
+    client = boto3_session.client('sso-admin', region_name=region)
+    accounts = []
+
+    paginator = client.get_paginator('list_accounts_for_provisioned_permission_set')
+    for page in paginator.paginate(InstanceArn=instance_arn, PermissionSetArn=permission_set_arn):
+        accounts.extend(page.get('AccountIds', []))
+
+    return accounts
+
+
+@timeit
+def load_permission_sets(
+    neo4j_session: neo4j.Session,
+    permission_sets: List[Dict],
+    instance_arn: str,
+    region: str,
+    aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load Identity Center permission sets into the graph
+    """
+    logger.info(f"Loading {len(permission_sets)} permission sets for instance {instance_arn} in region {region}")
+
+    load(
+        neo4j_session,
+        AWSPermissionSetSchema(),
+        permission_sets,
+        lastupdated=aws_update_tag,
+        InstanceArn=instance_arn,
+        Region=region,
+        AWS_ID=aws_account_id,
+    )
+
+
+@timeit
+def get_sso_users(
+    boto3_session: boto3.session.Session,
+    identity_store_id: str,
+    region: str,
+) -> List[Dict]:
+    """
+    Get all SSO users for a given Identity Store
+    """
+    client = boto3_session.client('identitystore', region_name=region)
+    users = []
+
+    paginator = client.get_paginator('list_users')
+    for page in paginator.paginate(IdentityStoreId=identity_store_id):
+        user_page = page.get('Users', [])
+        for user in user_page:
+            if user.get('ExternalIds', None):
+                user['ExternalId'] = user.get('ExternalIds')[0].get('Id')
+            users.append(user)
+
+    return users
+
+
+@timeit
+def load_sso_users(
+    neo4j_session: neo4j.Session,
+    users: List[Dict],
+    identity_store_id: str,
+    region: str,
+    aws_account_id: str,
+    aws_update_tag: int,
+) -> None:
+    """
+    Load SSO users into the graph
+    """
+    logger.info(f"Loading {len(users)} SSO users for identity store {identity_store_id} in region {region}")
+
+    load(
+        neo4j_session,
+        AWSSSOUserSchema(),
+        users,
+        lastupdated=aws_update_tag,
+        IdentityStoreId=identity_store_id,
+        AWS_ID=aws_account_id,
+        Region=region,
+    )
+
+
+@timeit
+def get_role_assignments(
+    boto3_session: boto3.session.Session,
+    users: List[Dict],
+    instance_arn: str,
+    region: str,
+) -> List[Dict]:
+    """
+    Get role assignments for SSO users
+    """
+
+    logger.info(f"Getting role assignments for {len(users)} users")
+    client = boto3_session.client('sso-admin', region_name=region)
+    role_assignments = []
+
+    for user in users:
+        user_id = user['UserId']
+        paginator = client.get_paginator('list_account_assignments_for_principal')
+        for page in paginator.paginate(InstanceArn=instance_arn, PrincipalId=user_id, PrincipalType='USER'):
+            for assignment in page.get('AccountAssignments', []):
+                role_assignments.append({
+                    'UserId': user_id,
+                    'PermissionSetArn': assignment.get('PermissionSetArn'),
+                    'AccountId': assignment.get('AccountId'),
+                })
+
+    return role_assignments
+
+
+@timeit
+def load_role_assignments(
+    neo4j_session: neo4j.Session,
+    role_assignments: List[Dict],
+    aws_update_tag: int,
+) -> None:
+    """
+    Load role assignments into the graph
+    """
+    logger.info(f"Loading {len(role_assignments)} role assignments")
+    if role_assignments:
+        neo4j_session.run(
+            """
+            UNWIND $role_assignments AS ra
+            MATCH (acc:AWSAccount{id:ra.AccountId}) -[:RESOURCE]->
+            (role:AWSRole)<-[:ASSIGNED_TO_ROLE]-
+            (permset:AWSPermissionSet {id: ra.PermissionSetArn})
+            MATCH (sso:AWSSSOUser {id: ra.UserId})
+            MERGE (role)-[r:ALLOWED_BY]->(sso)
+            SET r.lastupdated = $aws_update_tag,
+            r.permission_set_arn = ra.PermissionSetArn
+            """,
+            role_assignments=role_assignments,
+            aws_update_tag=aws_update_tag,
+        )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    GraphJob.from_node_schema(AWSIdentityCenterInstanceSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(AWSPermissionSetSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(AWSSSOUserSchema(), common_job_parameters).run(neo4j_session)
+    run_cleanup_job(
+        'aws_import_identity_center_cleanup.json',
+        neo4j_session,
+        common_job_parameters,
+    )
+
+
+@timeit
+def sync_identity_center_instances(
+    neo4j_session: neo4j.Session,
+    boto3_session: boto3.session.Session,
+    regions: List[str],
+    current_aws_account_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    Sync Identity Center instances, their permission sets, and SSO users
+    """
+    logger.info(f"Syncing Identity Center instances for regions {regions}")
+    for region in regions:
+        logger.info(f"Syncing Identity Center instances for region {region}")
+        instances = get_identity_center_instances(boto3_session, region)
+        load_identity_center_instances(
+            neo4j_session,
+            instances,
+            region,
+            current_aws_account_id,
+            update_tag,
+        )
+
+        # For each instance, get and load its permission sets and SSO users
+        for instance in instances:
+            instance_arn = instance['InstanceArn']
+            identity_store_id = instance['IdentityStoreId']
+
+            permission_sets = get_permission_sets(boto3_session, instance_arn, region)
+
+            load_permission_sets(
+                neo4j_session,
+                permission_sets,
+                instance_arn,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            users = get_sso_users(boto3_session, identity_store_id, region)
+            load_sso_users(
+                neo4j_session,
+                users,
+                identity_store_id,
+                region,
+                current_aws_account_id,
+                update_tag,
+            )
+
+            # Get and load role assignments
+            role_assignments = get_role_assignments(
+                boto3_session,
+                users,
+                instance_arn,
+                region,
+            )
+            load_role_assignments(
+                neo4j_session,
+                role_assignments,
+                update_tag,
+            )
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -10,6 +10,7 @@ from . import elasticache
 from . import elasticsearch
 from . import emr
 from . import iam
+from . import identitycenter
 from . import inspector
 from . import kms
 from . import lambda_function
@@ -32,6 +33,7 @@ from .ec2.key_pairs import sync_ec2_key_pairs
 from .ec2.launch_templates import sync_ec2_launch_templates
 from .ec2.load_balancer_v2s import sync_load_balancer_v2s
 from .ec2.load_balancers import sync_load_balancers
+from .ec2.network_acls import sync_network_acls
 from .ec2.network_interfaces import sync_network_interfaces
 from .ec2.reserved_instances import sync_ec2_reserved_instances
 from .ec2.security_groups import sync_ec2_security_groupinfo
@@ -55,6 +57,7 @@ RESOURCE_FUNCTIONS: Dict = {
     'ec2:keypair': sync_ec2_key_pairs,
     'ec2:load_balancer': sync_load_balancers,
     'ec2:load_balancer_v2': sync_load_balancer_v2s,
+    'ec2:network_acls': sync_network_acls,
     'ec2:network_interface': sync_network_interfaces,
     'ec2:security_group': sync_ec2_security_groupinfo,
     'ec2:subnet': sync_subnets,
@@ -86,4 +89,5 @@ RESOURCE_FUNCTIONS: Dict = {
     'ssm': ssm.sync,
     'inspector': inspector.sync,
     'config': config.sync,
+    'identitycenter': identitycenter.sync_identity_center_instances,
 }

cartography/intel/cve/__init__.py

@@ -25,7 +25,7 @@ def start_cve_ingestion(
     """
     if not config.cve_enabled:
         return
-    cve_api_key = config.cve_api_key if config.cve_api_key else None
+    cve_api_key: str | None = config.cve_api_key if config.cve_api_key else None
 
     # sync CVE year archives, if not yet synced
     existing_years = feed.get_cve_sync_metadata(neo4j_session)

cartography/intel/cve/feed.py

@@ -22,9 +22,9 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
-MAX_RETRIES = 3
-# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
-CONNECT_AND_READ_TIMEOUT = (60, 60)
+MAX_RETRIES = 8
+# Connect and read timeouts of 120 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+CONNECT_AND_READ_TIMEOUT = (30, 120)
 CVE_FEED_ID = "NIST_NVD"
 BATCH_SIZE_DAYS = 120
 RESULTS_PER_PAGE = 2000
@@ -68,7 +68,7 @@ def _map_cve_dict(cve_dict: Dict[Any, Any], data: Dict[Any, Any]) -> None:
     cve_dict["startIndex"] = data["startIndex"]
 
 
-def _call_cves_api(url: str, api_key: str, params: Dict[str, Any]) -> Dict[Any, Any]:
+def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dict[Any, Any]:
     totalResults = 0
     sleep_time = DEFAULT_SLEEP_TIME
     retries = 0
@@ -98,6 +98,9 @@ def _call_cves_api(url: str, api_key: str, params: Dict[str, Any]) -> Dict[Any,
             retries += 1
             if retries >= MAX_RETRIES:
                 raise
+            # Exponential backoff
+            sleep_time *= 2
+            time.sleep(sleep_time)
             continue
         data = res.json()
         _map_cve_dict(results, data)
@@ -114,7 +117,7 @@ def get_cves_in_batches(
     start_date: datetime,
     end_date: datetime,
     date_param_names: Dict[str, str],
-    api_key: str,
+    api_key: str | None,
 ) -> Dict[Any, Any]:
     cves: Dict[Any, Any] = dict()
     current_start_date: datetime = start_date
@@ -153,7 +156,7 @@ def get_cves_in_batches(
 
 
 def get_modified_cves(
-    nist_cve_url: str, last_modified_date: str, api_key: str,
+    nist_cve_url: str, last_modified_date: str, api_key: str | None,
 ) -> Dict[Any, Any]:
     cves = dict()
     end_date = datetime.now(tz=timezone.utc)
@@ -171,7 +174,7 @@ def get_modified_cves(
 
 
 def get_published_cves_per_year(
-    nist_cve_url: str, year: str, api_key: str,
+    nist_cve_url: str, year: str, api_key: str | None,
 ) -> Dict[Any, Any]:
     cves = {}
     start_of_year = datetime.strptime(f"{year}-01-01", "%Y-%m-%d")