cartography 0.105.0__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/intel/entra/applications.py

@@ -0,0 +1,366 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import httpx
+import neo4j
+from azure.identity import ClientSecretCredential
+from kiota_abstractions.api_error import APIError
+from msgraph.graph_service_client import GraphServiceClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.entra.users import load_tenant
+from cartography.models.entra.app_role_assignment import EntraAppRoleAssignmentSchema
+from cartography.models.entra.application import EntraApplicationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# Configurable constants for API pagination
+# Microsoft Graph API recommends page sizes up to 999 for most resources
+# Set to 999 by default, but can be adjusted if needed
+#
+# Adjust these values if:
+# - You have performance issues (decrease values)
+# - You want to minimize API calls (increase values up to 999)
+# - You're hitting rate limits (decrease values)
+APPLICATIONS_PAGE_SIZE = 999
+APP_ROLE_ASSIGNMENTS_PAGE_SIZE = (
+    999  # Currently not used, but reserved for future pagination improvements
+)
+
+# Warning thresholds for potential data completeness issues
+# Log warnings when individual users/groups have more assignments than this threshold
+HIGH_ASSIGNMENT_COUNT_THRESHOLD = 100
+
+
+@timeit
+async def get_entra_applications(client: GraphServiceClient) -> List[Any]:
+    """
+    Gets Entra applications using the Microsoft Graph API.
+
+    :param client: GraphServiceClient
+    :return: List of raw Application objects from Microsoft Graph
+    """
+    applications = []
+
+    # Get all applications with pagination
+    request_configuration = client.applications.ApplicationsRequestBuilderGetRequestConfiguration(
+        query_parameters=client.applications.ApplicationsRequestBuilderGetQueryParameters(
+            top=APPLICATIONS_PAGE_SIZE
+        )
+    )
+    page = await client.applications.get(request_configuration=request_configuration)
+
+    while page:
+        if page.value:
+            applications.extend(page.value)
+
+        if not page.odata_next_link:
+            break
+        page = await client.applications.with_url(page.odata_next_link).get()
+
+    logger.info(f"Retrieved {len(applications)} Entra applications total")
+    return applications
+
+
+@timeit
+async def get_app_role_assignments(
+    client: GraphServiceClient, applications: List[Any]
+) -> List[Any]:
+    """
+    Gets app role assignments efficiently by querying each application's service principal.
+
+    :param client: GraphServiceClient
+    :param applications: List of Application objects (from get_entra_applications)
+    :return: List of raw app role assignment objects from Microsoft Graph
+    """
+    assignments = []
+
+    for app in applications:
+        if not app.app_id:
+            logger.warning(f"Application {app.id} has no app_id, skipping")
+            continue
+
+        try:
+            # First, get the service principal for this application
+            # The service principal represents the app in the directory
+            service_principals_page = await client.service_principals.get(
+                request_configuration=client.service_principals.ServicePrincipalsRequestBuilderGetRequestConfiguration(
+                    query_parameters=client.service_principals.ServicePrincipalsRequestBuilderGetQueryParameters(
+                        filter=f"appId eq '{app.app_id}'"
+                    )
+                )
+            )
+
+            if not service_principals_page or not service_principals_page.value:
+                logger.debug(
+                    f"No service principal found for application {app.app_id} ({app.display_name})"
+                )
+                continue
+
+            service_principal = service_principals_page.value[0]
+
+            # Ensure service principal has an ID
+            if not service_principal.id:
+                logger.warning(
+                    f"Service principal for application {app.app_id} ({app.display_name}) has no ID, skipping"
+                )
+                continue
+
+            # Get all assignments for this service principal (users, groups, service principals)
+            assignments_page = await client.service_principals.by_service_principal_id(
+                service_principal.id
+            ).app_role_assigned_to.get()
+
+            app_assignments = []
+            while assignments_page:
+                if assignments_page.value:
+                    # Add application context to each assignment
+                    for assignment in assignments_page.value:
+                        # Add the application app_id to the assignment for relationship matching
+                        assignment.application_app_id = app.app_id
+                    app_assignments.extend(assignments_page.value)
+
+                if not assignments_page.odata_next_link:
+                    break
+                assignments_page = await client.service_principals.with_url(
+                    assignments_page.odata_next_link
+                ).get()
+
+            # Log warning if a single application has many assignments (potential pagination issues)
+            if len(app_assignments) >= HIGH_ASSIGNMENT_COUNT_THRESHOLD:
+                logger.warning(
+                    f"Application {app.display_name} ({app.app_id}) has {len(app_assignments)} role assignments. "
+                    f"If this seems unexpectedly high, there may be pagination limits affecting data completeness."
+                )
+
+            assignments.extend(app_assignments)
+            logger.debug(
+                f"Retrieved {len(app_assignments)} assignments for application {app.display_name}"
+            )
+
+        except APIError as e:
+            # Handle Microsoft Graph API errors (403 Forbidden, 404 Not Found, etc.)
+            if e.response_status_code == 403:
+                logger.warning(
+                    f"Access denied when fetching app role assignments for application {app.app_id} ({app.display_name}). "
+                    f"This application may not have sufficient permissions or may not exist."
+                )
+            elif e.response_status_code == 404:
+                logger.warning(
+                    f"Application {app.app_id} ({app.display_name}) not found when fetching app role assignments. "
+                    f"Application may have been deleted or does not exist."
+                )
+            elif e.response_status_code == 429:
+                logger.warning(
+                    f"Rate limit hit when fetching app role assignments for application {app.app_id} ({app.display_name}). "
+                    f"Consider reducing APPLICATIONS_PAGE_SIZE or implementing retry logic."
+                )
+            else:
+                logger.warning(
+                    f"Microsoft Graph API error when fetching app role assignments for application {app.app_id} ({app.display_name}): "
+                    f"Status {e.response_status_code}, Error: {str(e)}"
+                )
+            continue
+        except (httpx.TimeoutException, httpx.ConnectError, httpx.NetworkError) as e:
+            # Handle network-related errors
+            logger.warning(
+                f"Network error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}"
+            )
+            continue
+        except Exception as e:
+            # Only catch truly unexpected errors - these should be rare
+            logger.error(
+                f"Unexpected error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}",
+                exc_info=True,
+            )
+            continue
+
+    logger.info(f"Retrieved {len(assignments)} app role assignments total")
+    return assignments
+
+
+def transform_applications(applications: List[Any]) -> List[Dict[str, Any]]:
+    """
+    Transform application data for graph loading.
+
+    :param applications: Raw Application objects from Microsoft Graph API
+    :return: Transformed application data for graph loading
+    """
+    result = []
+    for app in applications:
+        transformed = {
+            "id": app.id,
+            "app_id": app.app_id,
+            "display_name": app.display_name,
+            "publisher_domain": getattr(app, "publisher_domain", None),
+            "sign_in_audience": app.sign_in_audience,
+        }
+        result.append(transformed)
+    return result
+
+
+def transform_app_role_assignments(
+    assignments: List[Any],
+) -> List[Dict[str, Any]]:
+    """
+    Transform app role assignment data for graph loading.
+
+    :param assignments: Raw app role assignment objects from Microsoft Graph API
+    :return: Transformed assignment data for graph loading
+    """
+    result = []
+    for assignment in assignments:
+        transformed = {
+            "id": assignment.id,
+            "app_role_id": (
+                str(assignment.app_role_id) if assignment.app_role_id else None
+            ),
+            "created_date_time": assignment.created_date_time,
+            "principal_id": (
+                str(assignment.principal_id) if assignment.principal_id else None
+            ),
+            "principal_display_name": assignment.principal_display_name,
+            "principal_type": assignment.principal_type,
+            "resource_display_name": assignment.resource_display_name,
+            "resource_id": (
+                str(assignment.resource_id) if assignment.resource_id else None
+            ),
+            "application_app_id": getattr(assignment, "application_app_id", None),
+        }
+        result.append(transformed)
+    return result
+
+
+@timeit
+def load_applications(
+    neo4j_session: neo4j.Session,
+    applications_data: List[Dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    """
+    Load Entra applications to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param applications_data: Application data to load
+    :param update_tag: Update tag for tracking data freshness
+    :param tenant_id: Entra tenant ID
+    """
+    load(
+        neo4j_session,
+        EntraApplicationSchema(),
+        applications_data,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def load_app_role_assignments(
+    neo4j_session: neo4j.Session,
+    assignments_data: List[Dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    """
+    Load Entra app role assignments to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param assignments_data: Assignment data to load
+    :param update_tag: Update tag for tracking data freshness
+    :param tenant_id: Entra tenant ID
+    """
+    load(
+        neo4j_session,
+        EntraAppRoleAssignmentSchema(),
+        assignments_data,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def cleanup_applications(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    """
+    Delete Entra applications and their relationships from the graph if they were not updated in the last sync.
+
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    GraphJob.from_node_schema(EntraApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def cleanup_app_role_assignments(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    """
+    Delete Entra app role assignments and their relationships from the graph if they were not updated in the last sync.
+
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    GraphJob.from_node_schema(
+        EntraAppRoleAssignmentSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+async def sync_entra_applications(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
+    Sync Entra applications and their app role assignments to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param tenant_id: Entra tenant ID
+    :param client_id: Azure application client ID
+    :param client_secret: Azure application client secret
+    :param update_tag: Update tag for tracking data freshness
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    # Create credentials and client
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+
+    client = GraphServiceClient(
+        credential,
+        scopes=["https://graph.microsoft.com/.default"],
+    )
+
+    # Load tenant (prerequisite)
+    load_tenant(neo4j_session, {"id": tenant_id}, update_tag)
+
+    # Get and transform applications data
+    applications_data = await get_entra_applications(client)
+    transformed_applications = transform_applications(applications_data)
+
+    # Get and transform app role assignments data
+    assignments_data = await get_app_role_assignments(client, applications_data)
+    transformed_assignments = transform_app_role_assignments(assignments_data)
+
+    # Load applications and assignments
+    load_applications(neo4j_session, transformed_applications, update_tag, tenant_id)
+    load_app_role_assignments(
+        neo4j_session, transformed_assignments, update_tag, tenant_id
+    )
+
+    # Cleanup stale data
+    cleanup_applications(neo4j_session, common_job_parameters)
+    cleanup_app_role_assignments(neo4j_session, common_job_parameters)

cartography/intel/entra/users.py

@@ -15,6 +15,51 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# NOTE:
+# Microsoft Graph imposes limits on the length of the $select clause as well as
+# the number of properties that can be selected in a single request.  In
+# practice we have seen 400 Bad Request responses that bubble up as
+# `Microsoft.SharePoint.Client.InvalidClientQueryException` once that limit is
+# breached (Graph internally rewrites the next-link using a SharePoint style
+# `id in (…)` filter which is then rejected).
+#
+# To avoid tripping this bug we only request a *core* subset of user attributes
+# that are most commonly used in downstream analysis.  The transform() function
+# tolerates missing attributes (the generated MS Graph SDK simply returns
+# `None` for properties that are not present in the payload), so fetching fewer
+# fields is safe – we merely get more `null` values in the graph.
+#
+# If you need additional attributes in the future, append them here but keep the
+# total character count of the comma-separated list comfortably below 500 and
+# stay within the official v1.0 contract (beta-only fields cause similar
+# failures). 20–25 fields is a good rule-of-thumb.
+#
+# References:
+#   • https://learn.microsoft.com/graph/query-parameters#select-parameter
+#   • https://learn.microsoft.com/graph/api/user-list?view=graph-rest-1.0
+#
+USER_SELECT_FIELDS = [
+    "id",
+    "userPrincipalName",
+    "displayName",
+    "givenName",
+    "surname",
+    "mail",
+    "mobilePhone",
+    "businessPhones",
+    "jobTitle",
+    "department",
+    "officeLocation",
+    "city",
+    "country",
+    "companyName",
+    "preferredLanguage",
+    "employeeId",
+    "employeeType",
+    "accountEnabled",
+    "ageGroup",
+]
+
 
 @timeit
 async def get_tenant(client: GraphServiceClient) -> Organization:
@@ -27,14 +72,20 @@ async def get_tenant(client: GraphServiceClient) -> Organization:
 
 @timeit
 async def get_users(client: GraphServiceClient) -> list[User]:
+    """Fetch all users with their manager reference in as few requests as possible.
+
+    We leverage `$expand=manager($select=id)` so the manager's *id* is hydrated
+    alongside every user record.  This avoids making a second round-trip per
+    user – vastly reducing latency and eliminating the noisy 404s that occur
+    when a user has no manager assigned.
     """
-    Get all users from Microsoft Graph API with pagination support
-    """
+
     all_users: list[User] = []
     request_configuration = client.users.UsersRequestBuilderGetRequestConfiguration(
         query_parameters=client.users.UsersRequestBuilderGetQueryParameters(
-            # Request more items per page to reduce number of API calls
             top=999,
+            select=USER_SELECT_FIELDS,
+            expand=["manager($select=id)"],
         ),
     )
 
@@ -43,18 +94,32 @@ async def get_users(client: GraphServiceClient) -> list[User]:
         all_users.extend(page.value)
         if not page.odata_next_link:
             break
-        page = await client.users.with_url(page.odata_next_link).get()
+
+        try:
+            page = await client.users.with_url(page.odata_next_link).get()
+        except Exception as e:
+            logger.error(
+                "Failed to fetch next page of Entra ID users – stopping pagination early: %s",
+                e,
+            )
+            break
 
     return all_users
 
 
 @timeit
+# The manager reference is now embedded in the user objects courtesy of the
+# `$expand` we added above, so we no longer need a separate `manager_map`.
 def transform_users(users: list[User]) -> list[dict[str, Any]]:
-    """
-    Transform the API response into the format expected by our schema
-    """
+    """Convert MS Graph SDK `User` models into dicts matching our schema."""
+
     result: list[dict[str, Any]] = []
     for user in users:
+        manager_id: str | None = None
+        if getattr(user, "manager", None) is not None:
+            # The SDK materialises `manager` as a DirectoryObject (or subclass)
+            manager_id = getattr(user.manager, "id", None)
+
         transformed_user = {
             "id": user.id,
             "user_principal_name": user.user_principal_name,
@@ -62,47 +127,24 @@ def transform_users(users: list[User]) -> list[dict[str, Any]]:
             "given_name": user.given_name,
             "surname": user.surname,
             "mail": user.mail,
-            "other_mails": user.other_mails,
-            "preferred_language": user.preferred_language,
-            "preferred_name": user.preferred_name,
-            "state": user.state,
-            "usage_location": user.usage_location,
-            "user_type": user.user_type,
-            "show_in_address_list": user.show_in_address_list,
-            "sign_in_sessions_valid_from_date_time": user.sign_in_sessions_valid_from_date_time,
-            "security_identifier": user.on_premises_security_identifier,
-            "account_enabled": user.account_enabled,
-            "age_group": user.age_group,
+            "mobile_phone": user.mobile_phone,
             "business_phones": user.business_phones,
+            "job_title": user.job_title,
+            "department": user.department,
+            "office_location": user.office_location,
             "city": user.city,
-            "company_name": user.company_name,
-            "consent_provided_for_minor": user.consent_provided_for_minor,
+            "state": user.state,
             "country": user.country,
-            "created_date_time": user.created_date_time,
-            "creation_type": user.creation_type,
-            "deleted_date_time": user.deleted_date_time,
-            "department": user.department,
+            "company_name": user.company_name,
+            "preferred_language": user.preferred_language,
             "employee_id": user.employee_id,
             "employee_type": user.employee_type,
-            "external_user_state": user.external_user_state,
-            "external_user_state_change_date_time": user.external_user_state_change_date_time,
-            "hire_date": user.hire_date,
-            "is_management_restricted": user.is_management_restricted,
-            "is_resource_account": user.is_resource_account,
-            "job_title": user.job_title,
-            "last_password_change_date_time": user.last_password_change_date_time,
-            "mail_nickname": user.mail_nickname,
-            "office_location": user.office_location,
-            "on_premises_distinguished_name": user.on_premises_distinguished_name,
-            "on_premises_domain_name": user.on_premises_domain_name,
-            "on_premises_immutable_id": user.on_premises_immutable_id,
-            "on_premises_last_sync_date_time": user.on_premises_last_sync_date_time,
-            "on_premises_sam_account_name": user.on_premises_sam_account_name,
-            "on_premises_security_identifier": user.on_premises_security_identifier,
-            "on_premises_sync_enabled": user.on_premises_sync_enabled,
-            "on_premises_user_principal_name": user.on_premises_user_principal_name,
+            "account_enabled": user.account_enabled,
+            "age_group": user.age_group,
+            "manager_id": manager_id,
         }
         result.append(transformed_user)
+
     return result
 
 
@@ -198,7 +240,7 @@ async def sync_entra_users(
         credential, scopes=["https://graph.microsoft.com/.default"]
     )
 
-    # Get tenant information
+    # Fetch tenant and users (with manager reference already populated by `$expand`)
     tenant = await get_tenant(client)
     users = await get_users(client)
 

cartography/intel/kubernetes/__init__.py

@@ -3,12 +3,12 @@ import logging
 from neo4j import Session
 
 from cartography.config import Config
+from cartography.intel.kubernetes.clusters import sync_kubernetes_cluster
 from cartography.intel.kubernetes.namespaces import sync_namespaces
 from cartography.intel.kubernetes.pods import sync_pods
 from cartography.intel.kubernetes.secrets import sync_secrets
 from cartography.intel.kubernetes.services import sync_services
 from cartography.intel.kubernetes.util import get_k8s_clients
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -16,26 +16,42 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def start_k8s_ingestion(session: Session, config: Config) -> None:
+    if not config.update_tag:
+        logger.error("Cartography update tag not provided.")
+        return
 
-    common_job_parameters = {"UPDATE_TAG": config.update_tag}
     if not config.k8s_kubeconfig:
-        logger.error("kubeconfig not found.")
+        logger.error("Kubernetes kubeconfig not provided.")
         return
 
+    common_job_parameters = {"UPDATE_TAG": config.update_tag}
+
     for client in get_k8s_clients(config.k8s_kubeconfig):
         logger.info(f"Syncing data for k8s cluster {client.name}...")
         try:
-            cluster = sync_namespaces(session, client, config.update_tag)
-            pods = sync_pods(session, client, config.update_tag, cluster)
-            sync_services(session, client, config.update_tag, cluster, pods)
-            sync_secrets(session, client, config.update_tag, cluster)
+            cluster_info = sync_kubernetes_cluster(
+                session,
+                client,
+                config.update_tag,
+                common_job_parameters,
+            )
+            common_job_parameters["CLUSTER_ID"] = cluster_info.get("id")
+
+            sync_namespaces(session, client, config.update_tag, common_job_parameters)
+            all_pods = sync_pods(
+                session,
+                client,
+                config.update_tag,
+                common_job_parameters,
+            )
+            sync_secrets(session, client, config.update_tag, common_job_parameters)
+            sync_services(
+                session,
+                client,
+                all_pods,
+                config.update_tag,
+                common_job_parameters,
+            )
         except Exception:
             logger.exception(f"Failed to sync data for k8s cluster {client.name}...")
             raise
-
-    run_cleanup_job(
-        "kubernetes_import_cleanup.json",
-        session,
-        common_job_parameters,
-        package="cartography.data.jobs.cleanup",
-    )