cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/entra/groups.py

@@ -0,0 +1,198 @@
+import logging
+from typing import Any
+from typing import AsyncGenerator
+from typing import Generator
+
+import neo4j
+from azure.identity import ClientSecretCredential
+from msgraph import GraphServiceClient
+from msgraph.generated.models.directory_object import DirectoryObject
+from msgraph.generated.models.group import Group
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.entra.group import EntraGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+async def get_entra_groups(client: GraphServiceClient) -> AsyncGenerator[Group, None]:
+    """Get all groups from Microsoft Graph API with pagination using a generator."""
+    request_configuration = client.groups.GroupsRequestBuilderGetRequestConfiguration(
+        query_parameters=client.groups.GroupsRequestBuilderGetQueryParameters(top=999)
+    )
+    page = await client.groups.get(request_configuration=request_configuration)
+    while page:
+        if page.value:
+            for group in page.value:
+                yield group
+        if not page.odata_next_link:
+            break
+        page = await client.groups.with_url(page.odata_next_link).get()
+
+
+@timeit
+async def get_group_members(
+    client: GraphServiceClient, group_id: str
+) -> tuple[list[str], list[str]]:
+    """Get member user IDs and subgroup IDs for a given group."""
+    user_ids: list[str] = []
+    group_ids: list[str] = []
+    request_builder = client.groups.by_group_id(group_id).members
+    page = await request_builder.get()
+    while page:
+        if page.value:
+            for obj in page.value:
+                if isinstance(obj, DirectoryObject):
+                    odata_type = getattr(obj, "odata_type", "")
+                    if odata_type == "#microsoft.graph.user":
+                        user_ids.append(obj.id)
+                    elif odata_type == "#microsoft.graph.group":
+                        group_ids.append(obj.id)
+        if not page.odata_next_link:
+            break
+        page = await request_builder.with_url(page.odata_next_link).get()
+    return user_ids, group_ids
+
+
+@timeit
+async def get_group_owners(client: GraphServiceClient, group_id: str) -> list[str]:
+    """Get owner user IDs for a given group."""
+    owner_ids: list[str] = []
+    request_builder = client.groups.by_group_id(group_id).owners
+    page = await request_builder.get()
+    while page:
+        if page.value:
+            for obj in page.value:
+                odata_type = getattr(obj, "odata_type", "")
+                if odata_type == "#microsoft.graph.user":
+                    owner_ids.append(obj.id)
+        if not page.odata_next_link:
+            break
+        page = await request_builder.with_url(page.odata_next_link).get()
+    return owner_ids
+
+
+def transform_groups(
+    groups: list[Group],
+    user_member_map: dict[str, list[str]],
+    group_member_map: dict[str, list[str]],
+    group_owner_map: dict[str, list[str]],
+) -> Generator[dict[str, Any], None, None]:
+    """Transform API responses into dictionaries for ingestion using a generator."""
+    for g in groups:
+        yield {
+            "id": g.id,
+            "display_name": g.display_name,
+            "description": g.description,
+            "mail": g.mail,
+            "mail_nickname": g.mail_nickname,
+            "mail_enabled": g.mail_enabled,
+            "security_enabled": g.security_enabled,
+            "group_types": g.group_types,
+            "visibility": g.visibility,
+            "is_assignable_to_role": g.is_assignable_to_role,
+            "created_date_time": g.created_date_time,
+            "deleted_date_time": g.deleted_date_time,
+            "member_ids": user_member_map.get(g.id, []),
+            "member_group_ids": group_member_map.get(g.id, []),
+            "owner_ids": group_owner_map.get(g.id, []),
+        }
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    groups: list[dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    logger.info(f"Loading {len(groups)} Entra groups")
+    load(
+        neo4j_session,
+        EntraGroupSchema(),
+        groups,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def cleanup_groups(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(EntraGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+async def sync_entra_groups(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """Sync Entra groups."""
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
+    )
+    client = GraphServiceClient(
+        credential, scopes=["https://graph.microsoft.com/.default"]
+    )
+
+    # Collect groups in batches to avoid loading all at once
+    groups_batch = []
+    batch_size = 100  # Process groups in batches
+
+    user_member_map: dict[str, list[str]] = {}
+    group_member_map: dict[str, list[str]] = {}
+    group_owner_map: dict[str, list[str]] = {}
+
+    # First pass: collect groups and their owners/members
+    async for group in get_entra_groups(client):
+        groups_batch.append(group)
+
+        # Fetch owners and members for this group
+        owners = await get_group_owners(client, group.id)
+        group_owner_map[group.id] = owners
+
+        try:
+            users, subgroups = await get_group_members(client, group.id)
+            user_member_map[group.id] = users
+            group_member_map[group.id] = subgroups
+        except Exception as e:
+            logger.error(f"Failed to fetch members for group {group.id}: {e}")
+            user_member_map[group.id] = []
+            group_member_map[group.id] = []
+
+        # Process batch when it reaches the size limit
+        if len(groups_batch) >= batch_size:
+            transformed_groups = list(
+                transform_groups(
+                    groups_batch, user_member_map, group_member_map, group_owner_map
+                )
+            )
+            load_groups(neo4j_session, transformed_groups, update_tag, tenant_id)
+
+            # Clear the batch and maps for processed groups
+            for g in groups_batch:
+                user_member_map.pop(g.id, None)
+                group_member_map.pop(g.id, None)
+                group_owner_map.pop(g.id, None)
+            groups_batch.clear()
+
+    # Process any remaining groups
+    if groups_batch:
+        transformed_groups = list(
+            transform_groups(
+                groups_batch, user_member_map, group_member_map, group_owner_map
+            )
+        )
+        load_groups(neo4j_session, transformed_groups, update_tag, tenant_id)
+
+    cleanup_groups(neo4j_session, common_job_parameters)

cartography/intel/entra/ou.py

@@ -1,6 +1,8 @@
 # cartography/intel/entra/ou.py
 import logging
 from typing import Any
+from typing import AsyncGenerator
+from typing import Generator
 
 import neo4j
 from azure.identity import ClientSecretCredential
@@ -9,7 +11,6 @@ from msgraph.generated.models.administrative_unit import AdministrativeUnit
 
 from cartography.client.core.tx import load
 from cartography.graph.job import GraphJob
-from cartography.intel.entra.users import load_tenant
 from cartography.models.entra.ou import EntraOUSchema
 from cartography.util import timeit
 
@@ -17,30 +18,44 @@ logger = logging.getLogger(__name__)
 
 
 @timeit
-async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
+async def get_entra_ous(
+    client: GraphServiceClient,
+) -> AsyncGenerator[AdministrativeUnit, None]:
     """
-    Get all OUs from Microsoft Graph API with pagination support
+    Get all OUs from Microsoft Graph API with pagination support using a generator
     """
-    all_units: list[AdministrativeUnit] = []
-    request = client.directory.administrative_units.request()
-
-    while request:
-        response = await request.get()
-        all_units.extend(response.value)
-        request = response.odata_next_link if response.odata_next_link else None
-
-    return all_units
+    # Initialize first page request
+    current_request = client.directory.administrative_units
+
+    while current_request:
+        try:
+            response = await current_request.get()
+            if response and response.value:
+                for unit in response.value:
+                    yield unit
+
+                # Handle next page using OData link
+                if response.odata_next_link:
+                    current_request = client.directory.administrative_units.with_url(
+                        response.odata_next_link
+                    )
+                else:
+                    current_request = None
+            else:
+                current_request = None
+        except Exception as e:
+            logger.error(f"Failed to retrieve administrative units: {str(e)}")
+            current_request = None
 
 
 def transform_ous(
     units: list[AdministrativeUnit], tenant_id: str
-) -> list[dict[str, Any]]:
+) -> Generator[dict[str, Any], None, None]:
     """
-    Transform the API response into the format expected by our schema
+    Transform the API response into the format expected by our schema using a generator
     """
-    result: list[dict[str, Any]] = []
     for unit in units:
-        transformed_unit = {
+        yield {
             "id": unit.id,
             "display_name": unit.display_name,
             "description": unit.description,
@@ -50,8 +65,6 @@ def transform_ous(
             "deleted_date_time": unit.deleted_date_time,
             "tenant_id": tenant_id,
         }
-        result.append(transformed_unit)
-    return result
 
 
 @timeit
@@ -100,13 +113,24 @@ async def sync_entra_ous(
         credential, scopes=["https://graph.microsoft.com/.default"]
     )
 
-    # Get OUs
-    units = await get_entra_ous(client)
-    transformed_units = transform_ous(units, tenant_id)
+    # Process OUs in batches
+    batch_size = 100  # OUs are typically fewer than users/groups
+    units_batch = []
+
+    async for unit in get_entra_ous(client):
+        units_batch.append(unit)
+
+        if len(units_batch) >= batch_size:
+            transformed_units = list(transform_ous(units_batch, tenant_id))
+            load_ous(
+                neo4j_session, transformed_units, update_tag, common_job_parameters
+            )
+            units_batch.clear()
 
-    # Load data
-    load_tenant(neo4j_session, {"id": tenant_id}, update_tag)
-    load_ous(neo4j_session, transformed_units, update_tag, common_job_parameters)
+    # Process any remaining OUs
+    if units_batch:
+        transformed_units = list(transform_ous(units_batch, tenant_id))
+        load_ous(neo4j_session, transformed_units, update_tag, common_job_parameters)
 
     # Cleanup stale data
     cleanup_ous(neo4j_session, common_job_parameters)

cartography/intel/entra/service_principals.py

@@ -0,0 +1,217 @@
+import logging
+import re
+from typing import Any
+from typing import AsyncGenerator
+
+import neo4j
+from azure.identity import ClientSecretCredential
+from msgraph import GraphServiceClient
+from msgraph.generated.models.service_principal import ServicePrincipal
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.entra.service_principal import EntraServicePrincipalSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+SERVICE_PRINCIPALS_PAGE_SIZE = 999
+
+
+@timeit
+async def get_entra_service_principals(
+    client: GraphServiceClient,
+) -> AsyncGenerator[ServicePrincipal, None]:
+    """
+    Gets Entra service principals using the Microsoft Graph API with a generator.
+
+    :param client: GraphServiceClient
+    :return: Generator of raw ServicePrincipal objects from Microsoft Graph
+    """
+    count = 0
+    # Get all service principals with pagination
+    request_configuration = client.service_principals.ServicePrincipalsRequestBuilderGetRequestConfiguration(
+        query_parameters=client.service_principals.ServicePrincipalsRequestBuilderGetQueryParameters(
+            top=SERVICE_PRINCIPALS_PAGE_SIZE
+        )
+    )
+    page = await client.service_principals.get(
+        request_configuration=request_configuration
+    )
+
+    while page:
+        if page.value:
+            for spn in page.value:
+                count += 1
+                yield spn
+
+        if not page.odata_next_link:
+            break
+        page = await client.service_principals.with_url(page.odata_next_link).get()
+
+    logger.info(f"Retrieved {count} Entra service principals total")
+
+
+async def get_service_principal_by_app_id(
+    client: GraphServiceClient, app_id: str
+) -> ServicePrincipal | None:
+    """
+    Gets a service principal by app_id using the Microsoft Graph API.
+    This function is extracted from the original app_role_assignments logic.
+
+    :param client: GraphServiceClient
+    :param app_id: Application ID to search for
+    :return: ServicePrincipal object or None if not found
+    """
+    service_principals_page = await client.service_principals.get(
+        request_configuration=client.service_principals.ServicePrincipalsRequestBuilderGetRequestConfiguration(
+            query_parameters=client.service_principals.ServicePrincipalsRequestBuilderGetQueryParameters(
+                filter=f"appId eq '{app_id}'"
+            )
+        )
+    )
+
+    if not service_principals_page or not service_principals_page.value:
+        logger.warning(
+            f"No service principal found for application {app_id}. Continuing."
+        )
+        return None
+
+    return service_principals_page.value[0]
+
+
+def transform_service_principals(
+    service_principals: list[ServicePrincipal],
+) -> list[dict[str, Any]]:
+    result = []
+    for spn in service_principals:
+        aws_identity_center_instance_id = None
+        match = re.search(r"d-[a-z0-9]{10}", spn.login_url or "")
+        aws_identity_center_instance_id = match.group(0) if match else None
+        transformed = {
+            "id": spn.id,
+            "app_id": spn.app_id,
+            "account_enabled": spn.account_enabled,
+            # uuid.UUID to string
+            "app_owner_organization_id": (
+                str(spn.app_owner_organization_id)
+                if spn.app_owner_organization_id
+                else None
+            ),
+            "aws_identity_center_instance_id": aws_identity_center_instance_id,
+            "display_name": spn.display_name,
+            "login_url": spn.login_url,
+            "preferred_single_sign_on_mode": spn.preferred_single_sign_on_mode,
+            "preferred_token_signing_key_thumbprint": spn.preferred_token_signing_key_thumbprint,
+            "reply_urls": spn.reply_urls,
+            "service_principal_type": spn.service_principal_type,
+            "sign_in_audience": spn.sign_in_audience,
+            "tags": spn.tags,
+            # uuid.UUID to string
+            "token_encryption_key_id": (
+                str(spn.token_encryption_key_id)
+                if spn.token_encryption_key_id
+                else None
+            ),
+        }
+        result.append(transformed)
+    return result
+
+
+@timeit
+def load_service_principals(
+    neo4j_session: neo4j.Session,
+    service_principal_data: list[dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    load(
+        neo4j_session,
+        EntraServicePrincipalSchema(),
+        service_principal_data,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def cleanup_service_principals(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    """
+    Delete Entra service principals from the graph if they were not updated in the last sync.
+
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    GraphJob.from_node_schema(EntraServicePrincipalSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+async def sync_service_principals(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync Entra service principals to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param tenant_id: Entra tenant ID
+    :param client_id: Azure application client ID
+    :param client_secret: Azure application client secret
+    :param update_tag: Update tag for tracking data freshness
+    :param common_job_parameters: Common job parameters for cleanup
+    """
+    # Create credentials and client
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+
+    client = GraphServiceClient(
+        credential,
+        scopes=["https://graph.microsoft.com/.default"],
+    )
+    service_principals_batch = []
+    batch_size = 50  # Batch size for service principals
+    total_count = 0
+
+    # Stream service principals and process in batches
+    async for spn in get_entra_service_principals(client):
+        service_principals_batch.append(spn)
+        total_count += 1
+
+        # Transform and load service principals in batches
+        if len(service_principals_batch) >= batch_size:
+            transformed_service_principals = transform_service_principals(
+                service_principals_batch
+            )
+            load_service_principals(
+                neo4j_session, transformed_service_principals, update_tag, tenant_id
+            )
+            logger.info(
+                f"Loaded batch of {len(service_principals_batch)} service principals (total: {total_count})"
+            )
+            service_principals_batch.clear()
+            transformed_service_principals.clear()
+
+    # Process remaining service principals
+    if service_principals_batch:
+        transformed_service_principals = transform_service_principals(
+            service_principals_batch
+        )
+        load_service_principals(
+            neo4j_session, transformed_service_principals, update_tag, tenant_id
+        )
+        service_principals_batch.clear()
+        transformed_service_principals.clear()
+
+    logger.info(f"Completed loading {total_count} service principals")
+    cleanup_service_principals(neo4j_session, common_job_parameters)