cartography 0.101.1rc1__py3-none-any.whl → 0.102.0__py3-none-any.whl

cartography/intel/aws/ec2/route_tables.py

@@ -0,0 +1,287 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.ec2.route_table_associations import RouteTableAssociationSchema
+from cartography.models.aws.ec2.route_tables import RouteTableSchema
+from cartography.models.aws.ec2.routes import RouteSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+def _get_route_id_and_target(route_table_id: str, route: dict[str, Any]) -> tuple[str, str | None]:
+    """
+    Generate a unique identifier for an AWS EC2 route and return the target of the route
+    regardless of its type.
+
+    Args:
+        route_table_id: The ID of the route table this route belongs to
+        route: The route data from AWS API
+
+    Returns:
+        A tuple containing the unique identifier for the route and the target of the route
+    """
+    route_target_keys = [
+        'DestinationCidrBlock',
+        'DestinationIpv6CidrBlock',
+        'GatewayId',
+        'InstanceId',
+        'NatGatewayId',
+        'TransitGatewayId',
+        'LocalGatewayId',
+        'CarrierGatewayId',
+        'NetworkInterfaceId',
+        'VpcPeeringConnectionId',
+        'EgressOnlyInternetGatewayId',
+        'CoreNetworkArn',
+    ]
+
+    # Start with the route table ID
+    parts = [route_table_id]
+    target = None
+    found_target = False
+
+    for key in route_target_keys:
+        # Each route is a "union"-like data structure, so only one of the keys will be present.
+        if key in route:
+            parts.append(route[key])
+            target = route[key]
+            found_target = True
+            break
+
+    if not found_target:
+        logger.warning(
+            f"No target found for route in {route_table_id}. Please review the route and file an issue to "
+            "https://github.com/cartography-cncf/cartography/issues sharing what the route table looks like "
+            "so that we can update the available keys.",
+        )
+
+    return '|'.join(parts), target
+
+
+@timeit
+@aws_handle_regions
+def get_route_tables(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_route_tables')
+    route_tables: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        route_tables.extend(page['RouteTables'])
+    return route_tables
+
+
+def _transform_route_table_associations(
+        route_table_id: str,
+        associations: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], bool]:
+    """
+    Transform route table association data into a format suitable for cartography ingestion.
+
+    Args:
+        route_table_id: The ID of the route table
+        associations: List of association data from AWS API
+
+    Returns:
+        1. List of transformed association data
+        2. Boolean indicating if the association is the main association, meaning that the route table is the main
+        route table for the VPC
+    """
+    transformed = []
+    is_main = False
+    for association in associations:
+        if association.get('SubnetId'):
+            target = association['SubnetId']
+        elif association.get('GatewayId'):
+            target = association['GatewayId']
+        else:
+            is_main = True
+            target = 'main'
+
+        transformed_association = {
+            'id': association['RouteTableAssociationId'],
+            'route_table_id': route_table_id,
+            'subnet_id': association.get('SubnetId'),
+            'gateway_id': association.get('GatewayId'),
+            'main': association.get('Main', False),
+            'association_state': association.get('AssociationState', {}).get('State'),
+            'association_state_message': association.get('AssociationState', {}).get('Message'),
+            '_target': target,
+        }
+        transformed.append(transformed_association)
+    return transformed, is_main
+
+
+def _transform_route_table_routes(route_table_id: str, routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform route table route data into a format suitable for cartography ingestion.
+
+    Args:
+        route_table_id: The ID of the route table
+        routes: List of route data from AWS API
+
+    Returns:
+        List of transformed route data
+    """
+    transformed = []
+    for route in routes:
+        route_id, target = _get_route_id_and_target(route_table_id, route)
+
+        transformed_route = {
+            'id': route_id,
+            'route_table_id': route_table_id,
+            'destination_cidr_block': route.get('DestinationCidrBlock'),
+            'destination_ipv6_cidr_block': route.get('DestinationIpv6CidrBlock'),
+            'gateway_id': route.get('GatewayId'),
+            'instance_id': route.get('InstanceId'),
+            'instance_owner_id': route.get('InstanceOwnerId'),
+            'nat_gateway_id': route.get('NatGatewayId'),
+            'transit_gateway_id': route.get('TransitGatewayId'),
+            'local_gateway_id': route.get('LocalGatewayId'),
+            'carrier_gateway_id': route.get('CarrierGatewayId'),
+            'network_interface_id': route.get('NetworkInterfaceId'),
+            'vpc_peering_connection_id': route.get('VpcPeeringConnectionId'),
+            'state': route.get('State'),
+            'origin': route.get('Origin'),
+            'core_network_arn': route.get('CoreNetworkArn'),
+            'destination_prefix_list_id': route.get('DestinationPrefixListId'),
+            'egress_only_internet_gateway_id': route.get('EgressOnlyInternetGatewayId'),
+            '_target': target,
+        }
+        transformed.append(transformed_route)
+    return transformed
+
+
+def transform_route_table_data(
+        route_tables: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+    """
+    Transform route table data into a format suitable for cartography ingestion.
+
+    Args:
+        route_tables: List of route table data from AWS API
+
+    Returns:
+        Tuple of (transformed route table data, transformed association data, transformed route data)
+    """
+    transformed_tables = []
+    association_data = []
+    route_data = []
+
+    for rt in route_tables:
+        route_table_id = rt['RouteTableId']
+
+        # Transform routes
+        current_routes = []
+        if rt.get('Routes'):
+            current_routes = _transform_route_table_routes(route_table_id, rt['Routes'])
+            route_data.extend(current_routes)
+
+        # If the rt has a association marked with main=True, then it is the main route table for the VPC.
+        is_main = False
+        # Transform associations
+        if rt.get('Associations'):
+            associations, is_main = _transform_route_table_associations(route_table_id, rt['Associations'])
+            association_data.extend(associations)
+
+        transformed_rt = {
+            'id': route_table_id,
+            'route_table_id': route_table_id,
+            'owner_id': rt.get('OwnerId'),
+            'vpc_id': rt.get('VpcId'),
+            'VpnGatewayIds': [vgw['GatewayId'] for vgw in rt.get('PropagatingVgws', [])],
+            'RouteTableAssociationIds': [assoc['RouteTableAssociationId'] for assoc in rt.get('Associations', [])],
+            'RouteIds': [route['id'] for route in current_routes],
+            'tags': rt.get('Tags', []),
+            'main': is_main,
+        }
+        transformed_tables.append(transformed_rt)
+
+    return transformed_tables, association_data, route_data
+
+
+@timeit
+def load_route_tables(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RouteTableSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_route_table_associations(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RouteTableAssociationSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_routes(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RouteSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running EC2 route tables cleanup")
+    GraphJob.from_node_schema(RouteTableSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(RouteSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(RouteTableAssociationSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_route_tables(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info("Syncing EC2 route tables for region '%s' in account '%s'.", region, current_aws_account_id)
+        route_tables = get_route_tables(boto3_session, region)
+        transformed_tables, association_data, route_data = transform_route_table_data(route_tables)
+        load_routes(neo4j_session, route_data, region, current_aws_account_id, update_tag)
+        load_route_table_associations(neo4j_session, association_data, region, current_aws_account_id, update_tag)
+        load_route_tables(neo4j_session, transformed_tables, region, current_aws_account_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -45,6 +45,7 @@ from .ec2.volumes import sync_ebs_volumes
 from .ec2.vpc import sync_vpc
 from .ec2.vpc_peerings import sync_vpc_peerings
 from .iam_instance_profiles import sync_iam_instance_profiles
+from cartography.intel.aws.ec2.route_tables import sync_route_tables
 
 RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     'iam': iam.sync,
@@ -62,6 +63,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     'ec2:load_balancer_v2': sync_load_balancer_v2s,
     'ec2:network_acls': sync_network_acls,
     'ec2:network_interface': sync_network_interfaces,
+    'ec2:route_table': sync_route_tables,
     'ec2:security_group': sync_ec2_security_groupinfo,
     'ec2:subnet': sync_subnets,
     'ec2:tgw': sync_transit_gateways,

cartography/intel/aws/util/common.py

@@ -1,7 +1,10 @@
+import logging
 from typing import List
 
 from cartography.intel.aws.resources import RESOURCE_FUNCTIONS
 
+logger = logging.getLogger(__name__)
+
 
 def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str]:
     validated_resources: List[str] = []
@@ -19,3 +22,27 @@ def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str
                 f'Our full list of valid values is: {valid_syncs}.',
             )
     return validated_resources
+
+
+def parse_and_validate_aws_regions(aws_regions: str) -> list[str]:
+    """
+    Parse and validate a comma-separated string of AWS regions.
+    :param aws_regions: Comma-separated string of AWS regions
+    :return: A validated list of AWS regions
+    """
+    validated_regions: List[str] = []
+    for region in aws_regions.split(','):
+        region = region.strip()
+        if region:
+            validated_regions.append(region)
+        else:
+            logger.warning(
+                f'Unable to parse string "{region}". Please check the value you passed to `aws-regions`. '
+                f'You specified "{aws_regions}". Continuing on with sync.',
+            )
+
+    if not validated_regions:
+        raise ValueError(
+            f'`aws-regions` was set but no regions were specified. You provided this string: "{aws_regions}"',
+        )
+    return validated_regions

cartography/intel/crowdstrike/__init__.py

@@ -1,11 +1,14 @@
 import logging
+from typing import Any
 
 import neo4j
 
 from cartography.config import Config
+from cartography.graph.job import GraphJob
 from cartography.intel.crowdstrike.endpoints import sync_hosts
 from cartography.intel.crowdstrike.spotlight import sync_vulnerabilities
 from cartography.intel.crowdstrike.util import get_authorization
+from cartography.models.crowdstrike.hosts import CrowdstrikeHostSchema
 from cartography.stats import get_stats_client
 from cartography.util import merge_module_sync_metadata
 from cartography.util import run_cleanup_job
@@ -50,11 +53,7 @@ def start_crowdstrike_ingestion(
         config.update_tag,
         authorization,
     )
-    run_cleanup_job(
-        "crowdstrike_import_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
-    )
+    cleanup(neo4j_session, common_job_parameters)
 
     group_id = "public"
     if config.crowdstrike_api_url:
@@ -67,3 +66,16 @@ def start_crowdstrike_ingestion(
         update_tag=config.update_tag,
         stat_handler=stat_handler,
     )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.info("Running Crowdstrike cleanup")
+    GraphJob.from_node_schema(CrowdstrikeHostSchema(), common_job_parameters).run(neo4j_session)
+
+    # Cleanup other crowdstrike assets not handled by the data model
+    run_cleanup_job(
+        "crowdstrike_import_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )

cartography/intel/crowdstrike/endpoints.py

@@ -6,6 +6,8 @@ import neo4j
 from falconpy.hosts import Hosts
 from falconpy.oauth2 import OAuth2
 
+from cartography.client.core.tx import load
+from cartography.models.crowdstrike.hosts import CrowdstrikeHostSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -24,55 +26,21 @@ def sync_hosts(
         load_host_data(neo4j_session, host_data, update_tag)
 
 
+@timeit
 def load_host_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
-    Transform and load scan information
-    """
-    ingestion_cypher_query = """
-    UNWIND $Hosts AS host
-        MERGE (h:CrowdstrikeHost{id: host.device_id})
-        ON CREATE SET h.cid = host.cid,
-            h.instance_id = host.instance_id,
-            h.serial_number = host.serial_number,
-            h.firstseen = timestamp()
-        SET h.status = host.status,
-            h.hostname = host.hostname,
-            h.machine_domain = host.machine_domain,
-            h.crowdstrike_first_seen = host.first_seen,
-            h.crowdstrike_last_seen = host.last_seen,
-            h.local_ip = host.local_ip,
-            h.external_ip = host.external_ip,
-            h.cpu_signature = host.cpu_signature,
-            h.bios_manufacturer = host.bios_manufacturer,
-            h.bios_version = host.bios_version,
-            h.mac_address = host.mac_address,
-            h.os_version = host.os_version,
-            h.os_build = host.os_build,
-            h.platform_id = host.platform_id,
-            h.platform_name = host.platform_name,
-            h.service_provider = host.service_provider,
-            h.service_provider_account_id = host.service_provider_account_id,
-            h.agent_version = host.agent_version,
-            h.system_manufacturer = host.system_manufacturer,
-            h.system_product_name = host.system_product_name,
-            h.product_type = host.product_type,
-            h.product_type_desc = host.product_type_desc,
-            h.provision_status = host.provision_status,
-            h.reduced_functionality_mode = host.reduced_functionality_mode,
-            h.kernel_version = host.kernel_version,
-            h.major_version = host.major_version,
-            h.minor_version = host.minor_version,
-            h.tags = host.tags,
-            h.modified_timestamp = host.modified_timestamp,
-            h.lastupdated = $update_tag
+    Load Crowdstrike host data into Neo4j.
     """
     logger.info(f"Loading {len(data)} crowdstrike hosts.")
-    neo4j_session.run(
-        ingestion_cypher_query,
-        Hosts=data,
-        update_tag=update_tag,
+    load(
+        neo4j_session,
+        CrowdstrikeHostSchema(),
+        data,
+        lastupdated=update_tag,
     )
 
 

cartography/intel/entra/__init__.py

@@ -0,0 +1,43 @@
+import asyncio
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.entra.users import sync_entra_users
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Entra data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.entra_tenant_id or not config.entra_client_id or not config.entra_client_secret:
+        logger.info(
+            'Entra import is not configured - skipping this module. '
+            'See docs to configure.',
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "TENANT_ID": config.entra_tenant_id,
+    }
+
+    asyncio.run(
+        sync_entra_users(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        ),
+    )