cartography 0.101.1rc2__py3-none-any.whl → 0.102.0rc2__py3-none-any.whl

cartography/intel/aws/ec2/route_tables.py

@@ -0,0 +1,287 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.aws.ec2.util import get_botocore_config
+from cartography.models.aws.ec2.route_table_associations import RouteTableAssociationSchema
+from cartography.models.aws.ec2.route_tables import RouteTableSchema
+from cartography.models.aws.ec2.routes import RouteSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+def _get_route_id_and_target(route_table_id: str, route: dict[str, Any]) -> tuple[str, str | None]:
+    """
+    Generate a unique identifier for an AWS EC2 route and return the target of the route
+    regardless of its type.
+
+    Args:
+        route_table_id: The ID of the route table this route belongs to
+        route: The route data from AWS API
+
+    Returns:
+        A tuple containing the unique identifier for the route and the target of the route
+    """
+    route_target_keys = [
+        'DestinationCidrBlock',
+        'DestinationIpv6CidrBlock',
+        'GatewayId',
+        'InstanceId',
+        'NatGatewayId',
+        'TransitGatewayId',
+        'LocalGatewayId',
+        'CarrierGatewayId',
+        'NetworkInterfaceId',
+        'VpcPeeringConnectionId',
+        'EgressOnlyInternetGatewayId',
+        'CoreNetworkArn',
+    ]
+
+    # Start with the route table ID
+    parts = [route_table_id]
+    target = None
+    found_target = False
+
+    for key in route_target_keys:
+        # Each route is a "union"-like data structure, so only one of the keys will be present.
+        if key in route:
+            parts.append(route[key])
+            target = route[key]
+            found_target = True
+            break
+
+    if not found_target:
+        logger.warning(
+            f"No target found for route in {route_table_id}. Please review the route and file an issue to "
+            "https://github.com/cartography-cncf/cartography/issues sharing what the route table looks like "
+            "so that we can update the available keys.",
+        )
+
+    return '|'.join(parts), target
+
+
+@timeit
+@aws_handle_regions
+def get_route_tables(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_route_tables')
+    route_tables: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        route_tables.extend(page['RouteTables'])
+    return route_tables
+
+
+def _transform_route_table_associations(
+        route_table_id: str,
+        associations: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], bool]:
+    """
+    Transform route table association data into a format suitable for cartography ingestion.
+
+    Args:
+        route_table_id: The ID of the route table
+        associations: List of association data from AWS API
+
+    Returns:
+        1. List of transformed association data
+        2. Boolean indicating if the association is the main association, meaning that the route table is the main
+        route table for the VPC
+    """
+    transformed = []
+    is_main = False
+    for association in associations:
+        if association.get('SubnetId'):
+            target = association['SubnetId']
+        elif association.get('GatewayId'):
+            target = association['GatewayId']
+        else:
+            is_main = True
+            target = 'main'
+
+        transformed_association = {
+            'id': association['RouteTableAssociationId'],
+            'route_table_id': route_table_id,
+            'subnet_id': association.get('SubnetId'),
+            'gateway_id': association.get('GatewayId'),
+            'main': association.get('Main', False),
+            'association_state': association.get('AssociationState', {}).get('State'),
+            'association_state_message': association.get('AssociationState', {}).get('Message'),
+            '_target': target,
+        }
+        transformed.append(transformed_association)
+    return transformed, is_main
+
+
+def _transform_route_table_routes(route_table_id: str, routes: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Transform route table route data into a format suitable for cartography ingestion.
+
+    Args:
+        route_table_id: The ID of the route table
+        routes: List of route data from AWS API
+
+    Returns:
+        List of transformed route data
+    """
+    transformed = []
+    for route in routes:
+        route_id, target = _get_route_id_and_target(route_table_id, route)
+
+        transformed_route = {
+            'id': route_id,
+            'route_table_id': route_table_id,
+            'destination_cidr_block': route.get('DestinationCidrBlock'),
+            'destination_ipv6_cidr_block': route.get('DestinationIpv6CidrBlock'),
+            'gateway_id': route.get('GatewayId'),
+            'instance_id': route.get('InstanceId'),
+            'instance_owner_id': route.get('InstanceOwnerId'),
+            'nat_gateway_id': route.get('NatGatewayId'),
+            'transit_gateway_id': route.get('TransitGatewayId'),
+            'local_gateway_id': route.get('LocalGatewayId'),
+            'carrier_gateway_id': route.get('CarrierGatewayId'),
+            'network_interface_id': route.get('NetworkInterfaceId'),
+            'vpc_peering_connection_id': route.get('VpcPeeringConnectionId'),
+            'state': route.get('State'),
+            'origin': route.get('Origin'),
+            'core_network_arn': route.get('CoreNetworkArn'),
+            'destination_prefix_list_id': route.get('DestinationPrefixListId'),
+            'egress_only_internet_gateway_id': route.get('EgressOnlyInternetGatewayId'),
+            '_target': target,
+        }
+        transformed.append(transformed_route)
+    return transformed
+
+
+def transform_route_table_data(
+        route_tables: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+    """
+    Transform route table data into a format suitable for cartography ingestion.
+
+    Args:
+        route_tables: List of route table data from AWS API
+
+    Returns:
+        Tuple of (transformed route table data, transformed association data, transformed route data)
+    """
+    transformed_tables = []
+    association_data = []
+    route_data = []
+
+    for rt in route_tables:
+        route_table_id = rt['RouteTableId']
+
+        # Transform routes
+        current_routes = []
+        if rt.get('Routes'):
+            current_routes = _transform_route_table_routes(route_table_id, rt['Routes'])
+            route_data.extend(current_routes)
+
+        # If the rt has a association marked with main=True, then it is the main route table for the VPC.
+        is_main = False
+        # Transform associations
+        if rt.get('Associations'):
+            associations, is_main = _transform_route_table_associations(route_table_id, rt['Associations'])
+            association_data.extend(associations)
+
+        transformed_rt = {
+            'id': route_table_id,
+            'route_table_id': route_table_id,
+            'owner_id': rt.get('OwnerId'),
+            'vpc_id': rt.get('VpcId'),
+            'VpnGatewayIds': [vgw['GatewayId'] for vgw in rt.get('PropagatingVgws', [])],
+            'RouteTableAssociationIds': [assoc['RouteTableAssociationId'] for assoc in rt.get('Associations', [])],
+            'RouteIds': [route['id'] for route in current_routes],
+            'tags': rt.get('Tags', []),
+            'main': is_main,
+        }
+        transformed_tables.append(transformed_rt)
+
+    return transformed_tables, association_data, route_data
+
+
+@timeit
+def load_route_tables(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RouteTableSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_route_table_associations(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RouteTableAssociationSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_routes(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        RouteSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.debug("Running EC2 route tables cleanup")
+    GraphJob.from_node_schema(RouteTableSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(RouteSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(RouteTableAssociationSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync_route_tables(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info("Syncing EC2 route tables for region '%s' in account '%s'.", region, current_aws_account_id)
+        route_tables = get_route_tables(boto3_session, region)
+        transformed_tables, association_data, route_data = transform_route_table_data(route_tables)
+        load_routes(neo4j_session, route_data, region, current_aws_account_id, update_tag)
+        load_route_table_associations(neo4j_session, association_data, region, current_aws_account_id, update_tag)
+        load_route_tables(neo4j_session, transformed_tables, region, current_aws_account_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -45,6 +45,7 @@ from .ec2.volumes import sync_ebs_volumes
 from .ec2.vpc import sync_vpc
 from .ec2.vpc_peerings import sync_vpc_peerings
 from .iam_instance_profiles import sync_iam_instance_profiles
+from cartography.intel.aws.ec2.route_tables import sync_route_tables
 
 RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     'iam': iam.sync,
@@ -62,6 +63,7 @@ RESOURCE_FUNCTIONS: Dict[str, Callable[..., None]] = {
     'ec2:load_balancer_v2': sync_load_balancer_v2s,
     'ec2:network_acls': sync_network_acls,
     'ec2:network_interface': sync_network_interfaces,
+    'ec2:route_table': sync_route_tables,
     'ec2:security_group': sync_ec2_security_groupinfo,
     'ec2:subnet': sync_subnets,
     'ec2:tgw': sync_transit_gateways,

cartography/intel/entra/__init__.py

@@ -0,0 +1,43 @@
+import asyncio
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.entra.users import sync_entra_users
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_entra_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Entra data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.entra_tenant_id or not config.entra_client_id or not config.entra_client_secret:
+        logger.info(
+            'Entra import is not configured - skipping this module. '
+            'See docs to configure.',
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "TENANT_ID": config.entra_tenant_id,
+    }
+
+    asyncio.run(
+        sync_entra_users(
+            neo4j_session,
+            config.entra_tenant_id,
+            config.entra_client_id,
+            config.entra_client_secret,
+            config.update_tag,
+            common_job_parameters,
+        ),
+    )

cartography/intel/entra/users.py

@@ -0,0 +1,205 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.identity import ClientSecretCredential
+from msgraph import GraphServiceClient
+from msgraph.generated.models.organization import Organization
+from msgraph.generated.models.user import User
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.entra.tenant import EntraTenantSchema
+from cartography.models.entra.user import EntraUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+async def get_tenant(client: GraphServiceClient) -> Organization:
+    """
+    Get tenant information from Microsoft Graph API
+    """
+    org = await client.organization.get()
+    return org.value[0]  # Get the first (and typically only) tenant
+
+
+@timeit
+async def get_users(client: GraphServiceClient) -> list[User]:
+    """
+    Get all users from Microsoft Graph API with pagination support
+    """
+    all_users: list[User] = []
+    request_configuration = client.users.UsersRequestBuilderGetRequestConfiguration(
+        query_parameters=client.users.UsersRequestBuilderGetQueryParameters(
+            # Request more items per page to reduce number of API calls
+            top=999,
+        ),
+    )
+
+    page = await client.users.get(request_configuration=request_configuration)
+    while page:
+        all_users.extend(page.value)
+        if not page.odata_next_link:
+            break
+        page = await client.users.with_url(page.odata_next_link).get()
+
+    return all_users
+
+
+@timeit
+def transform_users(users: list[User]) -> list[dict[str, Any]]:
+    """
+    Transform the API response into the format expected by our schema
+    """
+    result: list[dict[str, Any]] = []
+    for user in users:
+        transformed_user = {
+            'id': user.id,
+            'user_principal_name': user.user_principal_name,
+            'display_name': user.display_name,
+            'given_name': user.given_name,
+            'surname': user.surname,
+            'mail': user.mail,
+            'other_mails': user.other_mails,
+            'preferred_language': user.preferred_language,
+            'preferred_name': user.preferred_name,
+            'state': user.state,
+            'usage_location': user.usage_location,
+            'user_type': user.user_type,
+            'show_in_address_list': user.show_in_address_list,
+            'sign_in_sessions_valid_from_date_time': user.sign_in_sessions_valid_from_date_time,
+            'security_identifier': user.on_premises_security_identifier,
+            'account_enabled': user.account_enabled,
+            'age_group': user.age_group,
+            'business_phones': user.business_phones,
+            'city': user.city,
+            'company_name': user.company_name,
+            'consent_provided_for_minor': user.consent_provided_for_minor,
+            'country': user.country,
+            'created_date_time': user.created_date_time,
+            'creation_type': user.creation_type,
+            'deleted_date_time': user.deleted_date_time,
+            'department': user.department,
+            'employee_id': user.employee_id,
+            'employee_type': user.employee_type,
+            'external_user_state': user.external_user_state,
+            'external_user_state_change_date_time': user.external_user_state_change_date_time,
+            'hire_date': user.hire_date,
+            'is_management_restricted': user.is_management_restricted,
+            'is_resource_account': user.is_resource_account,
+            'job_title': user.job_title,
+            'last_password_change_date_time': user.last_password_change_date_time,
+            'mail_nickname': user.mail_nickname,
+            'office_location': user.office_location,
+            'on_premises_distinguished_name': user.on_premises_distinguished_name,
+            'on_premises_domain_name': user.on_premises_domain_name,
+            'on_premises_immutable_id': user.on_premises_immutable_id,
+            'on_premises_last_sync_date_time': user.on_premises_last_sync_date_time,
+            'on_premises_sam_account_name': user.on_premises_sam_account_name,
+            'on_premises_security_identifier': user.on_premises_security_identifier,
+            'on_premises_sync_enabled': user.on_premises_sync_enabled,
+            'on_premises_user_principal_name': user.on_premises_user_principal_name,
+        }
+        result.append(transformed_user)
+    return result
+
+
+@timeit
+def transform_tenant(tenant: Organization, tenant_id: str) -> dict[str, Any]:
+    """
+    Transform the tenant data into the format expected by our schema
+    """
+    return {
+        'id': tenant_id,
+        'created_date_time': tenant.created_date_time,
+        'default_usage_location': tenant.default_usage_location,
+        'deleted_date_time': tenant.deleted_date_time,
+        'display_name': tenant.display_name,
+        'marketing_notification_emails': tenant.marketing_notification_emails,
+        'mobile_device_management_authority': tenant.mobile_device_management_authority,
+        'on_premises_last_sync_date_time': tenant.on_premises_last_sync_date_time,
+        'on_premises_sync_enabled': tenant.on_premises_sync_enabled,
+        'partner_tenant_type': tenant.partner_tenant_type,
+        'postal_code': tenant.postal_code,
+        'preferred_language': tenant.preferred_language,
+        'state': tenant.state,
+        'street': tenant.street,
+        'tenant_type': tenant.tenant_type,
+    }
+
+
+@timeit
+def load_tenant(
+    neo4j_session: neo4j.Session,
+    tenant: dict[str, Any],
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        EntraTenantSchema(),
+        [tenant],
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    users: list[dict[str, Any]],
+    tenant_id: str,
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(users)} Entra users")
+    load(
+        neo4j_session,
+        EntraUserSchema(),
+        users,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    GraphJob.from_node_schema(EntraUserSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+async def sync_entra_users(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync Entra users and tenant information
+    :param neo4j_session: Neo4J session for database interface
+    :param tenant_id: Entra tenant ID
+    :param client_id: Entra application client ID
+    :param client_secret: Entra application client secret
+    :param update_tag: Timestamp used to determine data freshness
+    :param common_job_parameters: dict of other job parameters to carry to sub-jobs
+    :return: None
+    """
+    # Initialize Graph client
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+    client = GraphServiceClient(credential, scopes=['https://graph.microsoft.com/.default'])
+
+    # Get tenant information
+    tenant = await get_tenant(client)
+    users = await get_users(client)
+
+    transformed_users = transform_users(users)
+    transformed_tenant = transform_tenant(tenant, tenant_id)
+
+    load_tenant(neo4j_session, transformed_tenant, update_tag)
+    load_users(neo4j_session, transformed_users, tenant_id, update_tag)
+
+    cleanup(neo4j_session, common_job_parameters)

cartography/models/aws/ec2/load_balancer_listeners.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class ELBListenerNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('id')
+    port: PropertyRef = PropertyRef('port')
+    protocol: PropertyRef = PropertyRef('protocol')
+    instance_port: PropertyRef = PropertyRef('instance_port')
+    instance_protocol: PropertyRef = PropertyRef('instance_protocol')
+    policy_names: PropertyRef = PropertyRef('policy_names')
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class ELBListenerToLoadBalancerRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class ELBListenerToLoadBalancer(CartographyRelSchema):
+    target_node_label: str = 'LoadBalancer'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('LoadBalancerId')},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "ELB_LISTENER"
+    properties: ELBListenerToLoadBalancerRelProperties = ELBListenerToLoadBalancerRelProperties()
+
+
+@dataclass(frozen=True)
+class ELBListenerToAWSAccountRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class ELBListenerToAWSAccount(CartographyRelSchema):
+    target_node_label: str = 'AWSAccount'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: ELBListenerToAWSAccountRelProperties = ELBListenerToAWSAccountRelProperties()
+
+
+@dataclass(frozen=True)
+class ELBListenerSchema(CartographyNodeSchema):
+    label: str = 'ELBListener'
+    properties: ELBListenerNodeProperties = ELBListenerNodeProperties()
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(['Endpoint'])
+    sub_resource_relationship: ELBListenerToAWSAccount = ELBListenerToAWSAccount()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            ELBListenerToLoadBalancer(),
+        ],
+    )