cartography 0.104.0rc3__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/intel/entra/users.py

@@ -15,6 +15,51 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# NOTE:
+# Microsoft Graph imposes limits on the length of the $select clause as well as
+# the number of properties that can be selected in a single request.  In
+# practice we have seen 400 Bad Request responses that bubble up as
+# `Microsoft.SharePoint.Client.InvalidClientQueryException` once that limit is
+# breached (Graph internally rewrites the next-link using a SharePoint style
+# `id in (…)` filter which is then rejected).
+#
+# To avoid tripping this bug we only request a *core* subset of user attributes
+# that are most commonly used in downstream analysis.  The transform() function
+# tolerates missing attributes (the generated MS Graph SDK simply returns
+# `None` for properties that are not present in the payload), so fetching fewer
+# fields is safe – we merely get more `null` values in the graph.
+#
+# If you need additional attributes in the future, append them here but keep the
+# total character count of the comma-separated list comfortably below 500 and
+# stay within the official v1.0 contract (beta-only fields cause similar
+# failures). 20–25 fields is a good rule-of-thumb.
+#
+# References:
+#   • https://learn.microsoft.com/graph/query-parameters#select-parameter
+#   • https://learn.microsoft.com/graph/api/user-list?view=graph-rest-1.0
+#
+USER_SELECT_FIELDS = [
+    "id",
+    "userPrincipalName",
+    "displayName",
+    "givenName",
+    "surname",
+    "mail",
+    "mobilePhone",
+    "businessPhones",
+    "jobTitle",
+    "department",
+    "officeLocation",
+    "city",
+    "country",
+    "companyName",
+    "preferredLanguage",
+    "employeeId",
+    "employeeType",
+    "accountEnabled",
+    "ageGroup",
+]
+
 
 @timeit
 async def get_tenant(client: GraphServiceClient) -> Organization:
@@ -27,14 +72,20 @@ async def get_tenant(client: GraphServiceClient) -> Organization:
 
 @timeit
 async def get_users(client: GraphServiceClient) -> list[User]:
+    """Fetch all users with their manager reference in as few requests as possible.
+
+    We leverage `$expand=manager($select=id)` so the manager's *id* is hydrated
+    alongside every user record.  This avoids making a second round-trip per
+    user – vastly reducing latency and eliminating the noisy 404s that occur
+    when a user has no manager assigned.
     """
-    Get all users from Microsoft Graph API with pagination support
-    """
+
     all_users: list[User] = []
     request_configuration = client.users.UsersRequestBuilderGetRequestConfiguration(
         query_parameters=client.users.UsersRequestBuilderGetQueryParameters(
-            # Request more items per page to reduce number of API calls
             top=999,
+            select=USER_SELECT_FIELDS,
+            expand=["manager($select=id)"],
         ),
     )
 
@@ -43,18 +94,32 @@ async def get_users(client: GraphServiceClient) -> list[User]:
         all_users.extend(page.value)
         if not page.odata_next_link:
             break
-        page = await client.users.with_url(page.odata_next_link).get()
+
+        try:
+            page = await client.users.with_url(page.odata_next_link).get()
+        except Exception as e:
+            logger.error(
+                "Failed to fetch next page of Entra ID users – stopping pagination early: %s",
+                e,
+            )
+            break
 
     return all_users
 
 
 @timeit
+# The manager reference is now embedded in the user objects courtesy of the
+# `$expand` we added above, so we no longer need a separate `manager_map`.
 def transform_users(users: list[User]) -> list[dict[str, Any]]:
-    """
-    Transform the API response into the format expected by our schema
-    """
+    """Convert MS Graph SDK `User` models into dicts matching our schema."""
+
     result: list[dict[str, Any]] = []
     for user in users:
+        manager_id: str | None = None
+        if getattr(user, "manager", None) is not None:
+            # The SDK materialises `manager` as a DirectoryObject (or subclass)
+            manager_id = getattr(user.manager, "id", None)
+
         transformed_user = {
             "id": user.id,
             "user_principal_name": user.user_principal_name,
@@ -62,47 +127,24 @@ def transform_users(users: list[User]) -> list[dict[str, Any]]:
             "given_name": user.given_name,
             "surname": user.surname,
             "mail": user.mail,
-            "other_mails": user.other_mails,
-            "preferred_language": user.preferred_language,
-            "preferred_name": user.preferred_name,
-            "state": user.state,
-            "usage_location": user.usage_location,
-            "user_type": user.user_type,
-            "show_in_address_list": user.show_in_address_list,
-            "sign_in_sessions_valid_from_date_time": user.sign_in_sessions_valid_from_date_time,
-            "security_identifier": user.on_premises_security_identifier,
-            "account_enabled": user.account_enabled,
-            "age_group": user.age_group,
+            "mobile_phone": user.mobile_phone,
             "business_phones": user.business_phones,
+            "job_title": user.job_title,
+            "department": user.department,
+            "office_location": user.office_location,
             "city": user.city,
-            "company_name": user.company_name,
-            "consent_provided_for_minor": user.consent_provided_for_minor,
+            "state": user.state,
             "country": user.country,
-            "created_date_time": user.created_date_time,
-            "creation_type": user.creation_type,
-            "deleted_date_time": user.deleted_date_time,
-            "department": user.department,
+            "company_name": user.company_name,
+            "preferred_language": user.preferred_language,
             "employee_id": user.employee_id,
             "employee_type": user.employee_type,
-            "external_user_state": user.external_user_state,
-            "external_user_state_change_date_time": user.external_user_state_change_date_time,
-            "hire_date": user.hire_date,
-            "is_management_restricted": user.is_management_restricted,
-            "is_resource_account": user.is_resource_account,
-            "job_title": user.job_title,
-            "last_password_change_date_time": user.last_password_change_date_time,
-            "mail_nickname": user.mail_nickname,
-            "office_location": user.office_location,
-            "on_premises_distinguished_name": user.on_premises_distinguished_name,
-            "on_premises_domain_name": user.on_premises_domain_name,
-            "on_premises_immutable_id": user.on_premises_immutable_id,
-            "on_premises_last_sync_date_time": user.on_premises_last_sync_date_time,
-            "on_premises_sam_account_name": user.on_premises_sam_account_name,
-            "on_premises_security_identifier": user.on_premises_security_identifier,
-            "on_premises_sync_enabled": user.on_premises_sync_enabled,
-            "on_premises_user_principal_name": user.on_premises_user_principal_name,
+            "account_enabled": user.account_enabled,
+            "age_group": user.age_group,
+            "manager_id": manager_id,
         }
         result.append(transformed_user)
+
     return result
 
 
@@ -198,7 +240,7 @@ async def sync_entra_users(
         credential, scopes=["https://graph.microsoft.com/.default"]
     )
 
-    # Get tenant information
+    # Fetch tenant and users (with manager reference already populated by `$expand`)
     tenant = await get_tenant(client)
     users = await get_users(client)
 

cartography/intel/kubernetes/__init__.py

@@ -3,12 +3,12 @@ import logging
 from neo4j import Session
 
 from cartography.config import Config
+from cartography.intel.kubernetes.clusters import sync_kubernetes_cluster
 from cartography.intel.kubernetes.namespaces import sync_namespaces
 from cartography.intel.kubernetes.pods import sync_pods
 from cartography.intel.kubernetes.secrets import sync_secrets
 from cartography.intel.kubernetes.services import sync_services
 from cartography.intel.kubernetes.util import get_k8s_clients
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -16,26 +16,42 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def start_k8s_ingestion(session: Session, config: Config) -> None:
+    if not config.update_tag:
+        logger.error("Cartography update tag not provided.")
+        return
 
-    common_job_parameters = {"UPDATE_TAG": config.update_tag}
     if not config.k8s_kubeconfig:
-        logger.error("kubeconfig not found.")
+        logger.error("Kubernetes kubeconfig not provided.")
         return
 
+    common_job_parameters = {"UPDATE_TAG": config.update_tag}
+
     for client in get_k8s_clients(config.k8s_kubeconfig):
         logger.info(f"Syncing data for k8s cluster {client.name}...")
         try:
-            cluster = sync_namespaces(session, client, config.update_tag)
-            pods = sync_pods(session, client, config.update_tag, cluster)
-            sync_services(session, client, config.update_tag, cluster, pods)
-            sync_secrets(session, client, config.update_tag, cluster)
+            cluster_info = sync_kubernetes_cluster(
+                session,
+                client,
+                config.update_tag,
+                common_job_parameters,
+            )
+            common_job_parameters["CLUSTER_ID"] = cluster_info.get("id")
+
+            sync_namespaces(session, client, config.update_tag, common_job_parameters)
+            all_pods = sync_pods(
+                session,
+                client,
+                config.update_tag,
+                common_job_parameters,
+            )
+            sync_secrets(session, client, config.update_tag, common_job_parameters)
+            sync_services(
+                session,
+                client,
+                all_pods,
+                config.update_tag,
+                common_job_parameters,
+            )
         except Exception:
             logger.exception(f"Failed to sync data for k8s cluster {client.name}...")
             raise
-
-    run_cleanup_job(
-        "kubernetes_import_cleanup.json",
-        session,
-        common_job_parameters,
-        package="cartography.data.jobs.cleanup",
-    )

cartography/intel/kubernetes/clusters.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+from kubernetes.client.models import V1Namespace
+from kubernetes.client.models import VersionInfo
+
+from cartography.client.core.tx import load
+from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import K8sClient
+from cartography.models.kubernetes.clusters import KubernetesClusterSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_kubernetes_cluster_namespace(client: K8sClient) -> V1Namespace:
+    return client.core.read_namespace("kube-system")
+
+
+@timeit
+def get_kubernetes_cluster_version(client: K8sClient) -> VersionInfo:
+    return client.version.get_code()
+
+
+def transform_kubernetes_cluster(
+    client: K8sClient,
+    namespace: V1Namespace,
+    version: VersionInfo,
+) -> list[dict[str, Any]]:
+    cluster = {
+        "id": namespace.metadata.uid,
+        "creation_timestamp": get_epoch(namespace.metadata.creation_timestamp),
+        "external_id": client.external_id,
+        "name": client.name,
+        "git_version": version.git_version,
+        "version_major": version.major,
+        "version_minor": version.minor,
+        "go_version": version.go_version,
+        "compiler": version.compiler,
+        "platform": version.platform,
+    }
+
+    return [cluster]
+
+
+def load_kubernetes_cluster(
+    neo4j_session: neo4j.Session,
+    cluster_data: list[dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading '{}' Kubernetes cluster into graph".format(cluster_data[0].get("name"))
+    )
+    load(
+        neo4j_session,
+        KubernetesClusterSchema(),
+        cluster_data,
+        lastupdated=update_tag,
+    )
+
+
+# cleaning up the kubernetes cluster node is currently not supported
+# def cleanup(
+#     neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+# ) -> None:
+#     logger.debug("Running cleanup job for KubernetesCluster")
+#     run_cleanup_job(
+#         "kubernetes_cluster_cleanup.json", neo4j_session, common_job_parameters
+#     )
+
+
+@timeit
+def sync_kubernetes_cluster(
+    neo4j_session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> dict[str, Any]:
+    namespace = get_kubernetes_cluster_namespace(client)
+    version = get_kubernetes_cluster_version(client)
+    cluster_info = transform_kubernetes_cluster(client, namespace, version)
+
+    load_kubernetes_cluster(neo4j_session, cluster_info, update_tag)
+    return cluster_info[0]

cartography/intel/kubernetes/namespaces.py

@@ -1,82 +1,84 @@
 import logging
-from typing import Dict
-from typing import List
-from typing import Tuple
+from typing import Any
 
-from neo4j import Session
+import neo4j
+from kubernetes.client.models import V1Namespace
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.kubernetes.util import get_epoch
+from cartography.intel.kubernetes.util import k8s_paginate
 from cartography.intel.kubernetes.util import K8sClient
-from cartography.stats import get_stats_client
-from cartography.util import merge_module_sync_metadata
+from cartography.models.kubernetes.namespaces import KubernetesNamespaceSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
-stat_handler = get_stats_client(__name__)
 
 
 @timeit
-def sync_namespaces(session: Session, client: K8sClient, update_tag: int) -> Dict:
-    cluster, namespaces = get_namespaces(client)
-    load_namespaces(session, cluster, namespaces, update_tag)
-    merge_module_sync_metadata(
-        session,
-        group_type="KubernetesCluster",
-        group_id=cluster["uid"],
-        synced_type="KubernetesCluster",
-        update_tag=update_tag,
-        stat_handler=stat_handler,
-    )
-    return cluster
+def get_namespaces(client: K8sClient) -> list[V1Namespace]:
+    items = k8s_paginate(client.core.list_namespace)
+    return items
 
 
-@timeit
-def get_namespaces(client: K8sClient) -> Tuple[Dict, List[Dict]]:
-    cluster = dict()
-    namespaces = list()
-    for namespace in client.core.list_namespace().items:
-        namespaces.append(
+def transform_namespaces(namespaces: list[V1Namespace]) -> list[dict[str, Any]]:
+    transformed_namespaces = []
+    for namespace in namespaces:
+        transformed_namespaces.append(
             {
                 "uid": namespace.metadata.uid,
                 "name": namespace.metadata.name,
                 "creation_timestamp": get_epoch(namespace.metadata.creation_timestamp),
                 "deletion_timestamp": get_epoch(namespace.metadata.deletion_timestamp),
-            },
+                "status_phase": namespace.status.phase if namespace.status else None,
+            }
         )
-        if namespace.metadata.name == "kube-system":
-            cluster = {"uid": namespace.metadata.uid, "name": client.name}
-    return cluster, namespaces
+    return transformed_namespaces
 
 
 def load_namespaces(
-    session: Session,
-    cluster: Dict,
-    data: List[Dict],
+    session: neo4j.Session,
+    namespaces: list[dict[str, Any]],
     update_tag: int,
+    cluster_name: str,
+    cluster_id: str,
+) -> None:
+    logger.info(f"Loading {len(namespaces)} kubernetes namespaces.")
+    load(
+        session,
+        KubernetesNamespaceSchema(),
+        namespaces,
+        lastupdated=update_tag,
+        cluster_name=cluster_name,
+        CLUSTER_ID=cluster_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
 ) -> None:
-    ingestion_cypher_query = """
-    MERGE (cluster:KubernetesCluster {id: $cluster_id})
-    ON CREATE SET cluster.firstseen = timestamp()
-    SET cluster.name = $cluster_name,
-        cluster.lastupdated = $update_tag
-    WITH cluster
-    UNWIND $namespaces as namespace
-        MERGE (space:KubernetesNamespace {id: namespace.uid})
-        ON CREATE SET space.firstseen = timestamp()
-        SET space.lastupdated = $update_tag,
-            space.name = namespace.name,
-            space.created_at = namespace.creation_timestamp,
-            space.deleted_at = namespace.deletion_timestamp
-        WITH cluster, space
-        MERGE (cluster)-[rel1:HAS_NAMESPACE]->(space)
-        ON CREATE SET rel1.firstseen = timestamp()
-        SET rel1.lastupdated = $update_tag
-    """
-    logger.info(f"Loading {len(data)} kubernetes namespaces.")
-    session.run(
-        ingestion_cypher_query,
-        namespaces=data,
-        cluster_id=cluster["uid"],
-        cluster_name=cluster["name"],
-        update_tag=update_tag,
+    logger.debug("Running cleanup job for KubernetesNamespace")
+    cleanup_job = GraphJob.from_node_schema(
+        KubernetesNamespaceSchema(), common_job_parameters
+    )
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync_namespaces(
+    session: neo4j.Session,
+    client: K8sClient,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    namespaces = get_namespaces(client)
+    transformed_namespaces = transform_namespaces(namespaces)
+    cluster_id: str = common_job_parameters["CLUSTER_ID"]
+    load_namespaces(
+        session,
+        transformed_namespaces,
+        update_tag,
+        client.name,
+        cluster_id,
     )
+    cleanup(session, common_job_parameters)