cartography 0.105.0__py3-none-any.whl → 0.106.0rc2__py3-none-any.whl

cartography/intel/entra/applications.py

@@ -0,0 +1,366 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import httpx
+import neo4j
+from azure.identity import ClientSecretCredential
+from kiota_abstractions.api_error import APIError
+from msgraph.graph_service_client import GraphServiceClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.entra.users import load_tenant
+from cartography.models.entra.app_role_assignment import EntraAppRoleAssignmentSchema
+from cartography.models.entra.application import EntraApplicationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# Configurable constants for API pagination
+# Microsoft Graph API recommends page sizes up to 999 for most resources
+# Set to 999 by default, but can be adjusted if needed
+#
+# Adjust these values if:
+# - You have performance issues (decrease values)
+# - You want to minimize API calls (increase values up to 999)
+# - You're hitting rate limits (decrease values)
+APPLICATIONS_PAGE_SIZE = 999
+APP_ROLE_ASSIGNMENTS_PAGE_SIZE = (
+    999  # Currently not used, but reserved for future pagination improvements
+)
+
+# Warning thresholds for potential data completeness issues
+# Log warnings when individual users/groups have more assignments than this threshold
+HIGH_ASSIGNMENT_COUNT_THRESHOLD = 100
+
+
+@timeit
+async def get_entra_applications(client: GraphServiceClient) -> List[Any]:
+    """
+    Gets Entra applications using the Microsoft Graph API.
+
+    :param client: GraphServiceClient
+    :return: List of raw Application objects from Microsoft Graph
+    """
+    applications = []
+
+    # Get all applications with pagination
+    request_configuration = client.applications.ApplicationsRequestBuilderGetRequestConfiguration(
+        query_parameters=client.applications.ApplicationsRequestBuilderGetQueryParameters(
+            top=APPLICATIONS_PAGE_SIZE
+        )
+    )
+    page = await client.applications.get(request_configuration=request_configuration)
+
+    while page:
+        if page.value:
+            applications.extend(page.value)
+
+        if not page.odata_next_link:
+            break
+        page = await client.applications.with_url(page.odata_next_link).get()
+
+    logger.info(f"Retrieved {len(applications)} Entra applications total")
+    return applications
+
+
+@timeit
+async def get_app_role_assignments(
+    client: GraphServiceClient, applications: List[Any]
+) -> List[Any]:
+    """
+    Gets app role assignments efficiently by querying each application's service principal.
+
+    :param client: GraphServiceClient
+    :param applications: List of Application objects (from get_entra_applications)
+    :return: List of raw app role assignment objects from Microsoft Graph
+    """
+    assignments = []
+
+    for app in applications:
+        if not app.app_id:
+            logger.warning(f"Application {app.id} has no app_id, skipping")
+            continue
+
+        try:
+            # First, get the service principal for this application
+            # The service principal represents the app in the directory
+            service_principals_page = await client.service_principals.get(
+                request_configuration=client.service_principals.ServicePrincipalsRequestBuilderGetRequestConfiguration(
+                    query_parameters=client.service_principals.ServicePrincipalsRequestBuilderGetQueryParameters(
+                        filter=f"appId eq '{app.app_id}'"
+                    )
+                )
+            )
+
+            if not service_principals_page or not service_principals_page.value:
+                logger.debug(
+                    f"No service principal found for application {app.app_id} ({app.display_name})"
+                )
+                continue
+
+            service_principal = service_principals_page.value[0]
+
+            # Ensure service principal has an ID
+            if not service_principal.id:
+                logger.warning(
+                    f"Service principal for application {app.app_id} ({app.display_name}) has no ID, skipping"
+                )
+                continue
+
+            # Get all assignments for this service principal (users, groups, service principals)
+            assignments_page = await client.service_principals.by_service_principal_id(
+                service_principal.id
+            ).app_role_assigned_to.get()
+
+            app_assignments = []
+            while assignments_page:
+                if assignments_page.value:
+                    # Add application context to each assignment
+                    for assignment in assignments_page.value:
+                        # Add the application app_id to the assignment for relationship matching
+                        assignment.application_app_id = app.app_id
+                    app_assignments.extend(assignments_page.value)
+
+                if not assignments_page.odata_next_link:
+                    break
+                assignments_page = await client.service_principals.with_url(
+                    assignments_page.odata_next_link
+                ).get()
+
+            # Log warning if a single application has many assignments (potential pagination issues)
+            if len(app_assignments) >= HIGH_ASSIGNMENT_COUNT_THRESHOLD:
+                logger.warning(
+                    f"Application {app.display_name} ({app.app_id}) has {len(app_assignments)} role assignments. "
+                    f"If this seems unexpectedly high, there may be pagination limits affecting data completeness."
+                )
+
+            assignments.extend(app_assignments)
+            logger.debug(
+                f"Retrieved {len(app_assignments)} assignments for application {app.display_name}"
+            )
+
+        except APIError as e:
+            # Handle Microsoft Graph API errors (403 Forbidden, 404 Not Found, etc.)
+            if e.response_status_code == 403:
+                logger.warning(
+                    f"Access denied when fetching app role assignments for application {app.app_id} ({app.display_name}). "
+                    f"This application may not have sufficient permissions or may not exist."
+                )
+            elif e.response_status_code == 404:
+                logger.warning(
+                    f"Application {app.app_id} ({app.display_name}) not found when fetching app role assignments. "
+                    f"Application may have been deleted or does not exist."
+                )
+            elif e.response_status_code == 429:
+                logger.warning(
+                    f"Rate limit hit when fetching app role assignments for application {app.app_id} ({app.display_name}). "
+                    f"Consider reducing APPLICATIONS_PAGE_SIZE or implementing retry logic."
+                )
+            else:
+                logger.warning(
+                    f"Microsoft Graph API error when fetching app role assignments for application {app.app_id} ({app.display_name}): "
+                    f"Status {e.response_status_code}, Error: {str(e)}"
+                )
+            continue
+        except (httpx.TimeoutException, httpx.ConnectError, httpx.NetworkError) as e:
+            # Handle network-related errors
+            logger.warning(
+                f"Network error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}"
+            )
+            continue
+        except Exception as e:
+            # Only catch truly unexpected errors - these should be rare
+            logger.error(
+                f"Unexpected error when fetching app role assignments for application {app.app_id} ({app.display_name}): {e}",
+                exc_info=True,
+            )
+            continue
+
+    logger.info(f"Retrieved {len(assignments)} app role assignments total")
+    return assignments
+
+
+def transform_applications(applications: List[Any]) -> List[Dict[str, Any]]:
+    """
+    Transform application data for graph loading.
+
+    :param applications: Raw Application objects from Microsoft Graph API
+    :return: Transformed application data for graph loading
+    """
+    result = []
+    for app in applications:
+        transformed = {
+            "id": app.id,
+            "app_id": app.app_id,
+            "display_name": app.display_name,
+            "publisher_domain": getattr(app, "publisher_domain", None),
+            "sign_in_audience": app.sign_in_audience,
+        }
+        result.append(transformed)
+    return result
+
+
+def transform_app_role_assignments(
+    assignments: List[Any],
+) -> List[Dict[str, Any]]:
+    """
+    Transform app role assignment data for graph loading.
+
+    :param assignments: Raw app role assignment objects from Microsoft Graph API
+    :return: Transformed assignment data for graph loading
+    """
+    result = []
+    for assignment in assignments:
+        transformed = {
+            "id": assignment.id,
+            "app_role_id": (
+                str(assignment.app_role_id) if assignment.app_role_id else None
+            ),
+            "created_date_time": assignment.created_date_time,
+            "principal_id": (
+                str(assignment.principal_id) if assignment.principal_id else None
+            ),
+            "principal_display_name": assignment.principal_display_name,
+            "principal_type": assignment.principal_type,
+            "resource_display_name": assignment.resource_display_name,
+            "resource_id": (
+                str(assignment.resource_id) if assignment.resource_id else None
+            ),
+            "application_app_id": getattr(assignment, "application_app_id", None),
+        }
+        result.append(transformed)
+    return result
+
+
+@timeit
+def load_applications(
+    neo4j_session: neo4j.Session,
+    applications_data: List[Dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    """
+    Load Entra applications to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param applications_data: Application data to load
+    :param update_tag: Update tag for tracking data freshness
+    :param tenant_id: Entra tenant ID
+    """
+    load(
+        neo4j_session,
+        EntraApplicationSchema(),
+        applications_data,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def load_app_role_assignments(
+    neo4j_session: neo4j.Session,
+    assignments_data: List[Dict[str, Any]],
+    update_tag: int,
+    tenant_id: str,
+) -> None:
+    """
+    Load Entra app role assignments to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param assignments_data: Assignment data to load
+    :param update_tag: Update tag for tracking data freshness
+    :param tenant_id: Entra tenant ID
+    """
+    load(
+        neo4j_session,
+        EntraAppRoleAssignmentSchema(),
+        assignments_data,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+@timeit
+def cleanup_applications(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    """
+    Delete Entra applications and their relationships from the graph if they were not updated in the last sync.
+
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    GraphJob.from_node_schema(EntraApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def cleanup_app_role_assignments(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    """
+    Delete Entra app role assignments and their relationships from the graph if they were not updated in the last sync.
+
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    GraphJob.from_node_schema(
+        EntraAppRoleAssignmentSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+async def sync_entra_applications(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
+    Sync Entra applications and their app role assignments to the graph.
+
+    :param neo4j_session: Neo4j session
+    :param tenant_id: Entra tenant ID
+    :param client_id: Azure application client ID
+    :param client_secret: Azure application client secret
+    :param update_tag: Update tag for tracking data freshness
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG and TENANT_ID
+    """
+    # Create credentials and client
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id,
+        client_id=client_id,
+        client_secret=client_secret,
+    )
+
+    client = GraphServiceClient(
+        credential,
+        scopes=["https://graph.microsoft.com/.default"],
+    )
+
+    # Load tenant (prerequisite)
+    load_tenant(neo4j_session, {"id": tenant_id}, update_tag)
+
+    # Get and transform applications data
+    applications_data = await get_entra_applications(client)
+    transformed_applications = transform_applications(applications_data)
+
+    # Get and transform app role assignments data
+    assignments_data = await get_app_role_assignments(client, applications_data)
+    transformed_assignments = transform_app_role_assignments(assignments_data)
+
+    # Load applications and assignments
+    load_applications(neo4j_session, transformed_applications, update_tag, tenant_id)
+    load_app_role_assignments(
+        neo4j_session, transformed_assignments, update_tag, tenant_id
+    )
+
+    # Cleanup stale data
+    cleanup_applications(neo4j_session, common_job_parameters)
+    cleanup_app_role_assignments(neo4j_session, common_job_parameters)

cartography/intel/kubernetes/__init__.py

@@ -3,12 +3,12 @@ import logging
 from neo4j import Session
 
 from cartography.config import Config
+from cartography.intel.kubernetes.clusters import sync_kubernetes_cluster
 from cartography.intel.kubernetes.namespaces import sync_namespaces
 from cartography.intel.kubernetes.pods import sync_pods
 from cartography.intel.kubernetes.secrets import sync_secrets
 from cartography.intel.kubernetes.services import sync_services
 from cartography.intel.kubernetes.util import get_k8s_clients
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -16,26 +16,42 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def start_k8s_ingestion(session: Session, config: Config) -> None:
+    if not config.update_tag:
+        logger.error("Cartography update tag not provided.")
+        return
 
-    common_job_parameters = {"UPDATE_TAG": config.update_tag}
     if not config.k8s_kubeconfig:
-        logger.error("kubeconfig not found.")
+        logger.error("Kubernetes kubeconfig not provided.")
         return
 
+    common_job_parameters = {"UPDATE_TAG": config.update_tag}
+
     for client in get_k8s_clients(config.k8s_kubeconfig):
         logger.info(f"Syncing data for k8s cluster {client.name}...")
         try:
-            cluster = sync_namespaces(session, client, config.update_tag)
-            pods = sync_pods(session, client, config.update_tag, cluster)
-            sync_services(session, client, config.update_tag, cluster, pods)
-            sync_secrets(session, client, config.update_tag, cluster)
+            cluster_info = sync_kubernetes_cluster(
+                session,
+                client,
+                config.update_tag,
+                common_job_parameters,
+            )
+            common_job_parameters["CLUSTER_ID"] = cluster_info.get("id")
+
+            sync_namespaces(session, client, config.update_tag, common_job_parameters)
+            all_pods = sync_pods(
+                session,
+                client,
+                config.update_tag,
+                common_job_parameters,
+            )
+            sync_secrets(session, client, config.update_tag, common_job_parameters)
+            sync_services(
+                session,
+                client,
+                all_pods,
+                config.update_tag,
+                common_job_parameters,
+            )
         except Exception:
             logger.exception(f"Failed to sync data for k8s cluster {client.name}...")
             raise
-
-    run_cleanup_job(
-        "kubernetes_import_cleanup.json",
-        session,
-        common_job_parameters,
-        package="cartography.data.jobs.cleanup",
-    )

cartography/intel/kubernetes/clusters.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+from kubernetes.client.models import V1Namespace
+from kubernetes.client.models import VersionInfo
+
+from cartography.client.core.tx import load
+from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.clusters import KubernetesClusterSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_kubernetes_cluster_namespace(client: K8sClient) -> V1Namespace:
+    return client.core.read_namespace("kube-system")
+
+
+@timeit
+def get_kubernetes_cluster_version(client: K8sClient) -> VersionInfo:
+    return client.version.get_code()
+
+
+def transform_kubernetes_cluster(
+    client: K8sClient,
+    namespace: V1Namespace,
+    version: VersionInfo,
+) -> list[dict[str, Any]]:
+    cluster = {
+        "id": namespace.metadata.uid,
+        "creation_timestamp": get_epoch(namespace.metadata.creation_timestamp),
+        "external_id": client.external_id,
+        "name": client.name,
+        "git_version": version.git_version,
+        "version_major": version.major,
+        "version_minor": version.minor,
+        "go_version": version.go_version,
+        "compiler": version.compiler,
+        "platform": version.platform,
+    }
+
+    return [cluster]
+
+
+def load_kubernetes_cluster(
+    neo4j_session: neo4j.Session,
+    cluster_data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading '{}' Kubernetes cluster into graph".format(cluster_data[0].get("name"))
+    )
+    load(
+        neo4j_session,
+        KubernetesClusterSchema(),
+        cluster_data,
+        lastupdated=update_tag,
+    )
+
+
+# cleaning up the kubernetes cluster node is currently not supported
+# def cleanup(
+#     neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+# ) -> None:
+#     logger.debug("Running cleanup job for KubernetesCluster")
+#     run_cleanup_job(
+#         "kubernetes_cluster_cleanup.json", neo4j_session, common_job_parameters
+#     )
+
+
+@timeit
+def sync_kubernetes_cluster(
+    neo4j_session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> dict[str, Any]:
+    namespace = get_kubernetes_cluster_namespace(client)
+    version = get_kubernetes_cluster_version(client)
+    cluster_info = transform_kubernetes_cluster(client, namespace, version)
+
+    load_kubernetes_cluster(neo4j_session, cluster_info, update_tag)
+    return cluster_info[0]

cartography/intel/kubernetes/namespaces.py

@@ -1,82 +1,84 @@
 import logging
-from typing import Dict
-from typing import List
-from typing import Tuple
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1Namespace
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
-from cartography.stats import get_stats_client
-from cartography.util import merge_module_sync_metadata
+from cartography.models.kubernetes.namespaces import KubernetesNamespaceSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
-stat_handler = get_stats_client(__name__)
 
 
 @timeit
-def sync_namespaces(session: Session, client: K8sClient, update_tag: int) -> Dict:
-    cluster, namespaces = get_namespaces(client)
-    load_namespaces(session, cluster, namespaces, update_tag)
-    merge_module_sync_metadata(
-        session,
-        group_type="KubernetesCluster",
-        group_id=cluster["uid"],
-        synced_type="KubernetesCluster",
-        update_tag=update_tag,
-        stat_handler=stat_handler,
-    )
-    return cluster
+def get_namespaces(client: K8sClient) -> list[V1Namespace]:
+    items = k8s_paginate(client.core.list_namespace)
+    return items
 
 
-@timeit
-def get_namespaces(client: K8sClient) -> Tuple[Dict, List[Dict]]:
-    cluster = dict()
-    namespaces = list()
-    for namespace in client.core.list_namespace().items:
-        namespaces.append(
+def transform_namespaces(namespaces: list[V1Namespace]) -> list[dict[str, Any]]:
+    transformed_namespaces = []
+    for namespace in namespaces:
+        transformed_namespaces.append(
             {
                 "uid": namespace.metadata.uid,
                 "name": namespace.metadata.name,
                 "creation_timestamp": get_epoch(namespace.metadata.creation_timestamp),
                 "deletion_timestamp": get_epoch(namespace.metadata.deletion_timestamp),
-            },
+                "status_phase": namespace.status.phase if namespace.status else None,
+            }
         )
-        if namespace.metadata.name == "kube-system":
-            cluster = {"uid": namespace.metadata.uid, "name": client.name}
-    return cluster, namespaces
+    return transformed_namespaces
 
 
 def load_namespaces(
-    session: Session,
-    cluster: Dict,
-    data: List[Dict],
+    session: neo4j.Session,
+    namespaces: list[dict[str, Any]],
     update_tag: int,
+    cluster_name: str,
+    cluster_id: str,
+) -> None:
+    logger.info(f"Loading {len(namespaces)} kubernetes namespaces.")
+    load(
+        session,
+        KubernetesNamespaceSchema(),
+        namespaces,
+        lastupdated=update_tag,
+        cluster_name=cluster_name,
+        CLUSTER_ID=cluster_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
 ) -> None:
-    ingestion_cypher_query = """
-    MERGE (cluster:KubernetesCluster {id: $cluster_id})
-    ON CREATE SET cluster.firstseen = timestamp()
-    SET cluster.name = $cluster_name,
-        cluster.lastupdated = $update_tag
-    WITH cluster
-    UNWIND $namespaces as namespace
-        MERGE (space:KubernetesNamespace {id: namespace.uid})
-        ON CREATE SET space.firstseen = timestamp()
-        SET space.lastupdated = $update_tag,
-            space.name = namespace.name,
-            space.created_at = namespace.creation_timestamp,
-            space.deleted_at = namespace.deletion_timestamp
-        WITH cluster, space
-        MERGE (cluster)-[rel1:HAS_NAMESPACE]->(space)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes namespaces.")
-    session.run(
-        ingestion_cypher_query,
-        namespaces=data,
-        cluster_id=cluster["uid"],
-        cluster_name=cluster["name"],
-        update_tag=update_tag,
+    logger.debug("Running cleanup job for KubernetesNamespace")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesNamespaceSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync_namespaces(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    namespaces = get_namespaces(client)
+    transformed_namespaces = transform_namespaces(namespaces)
+    cluster_id: str = common_job_parameters["CLUSTER_ID"]
+    load_namespaces(
+        session,
+        transformed_namespaces,
+        update_tag,
+        client.name,
+        cluster_id,
     )
+    cleanup(session, common_job_parameters)