cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/azure/__init__.py

@@ -1,15 +1,32 @@
 import logging
 from typing import Dict
 from typing import List
-from typing import Optional
 
 import neo4j
 
 from cartography.config import Config
 from cartography.util import timeit
 
+from . import aks
+from . import app_service
 from . import compute
+from . import container_instances
 from . import cosmosdb
+from . import data_factory
+from . import data_factory_dataset
+from . import data_factory_linked_service
+from . import data_factory_pipeline
+from . import data_lake
+from . import event_grid
+from . import functions
+from . import load_balancers
+from . import logic_apps
+from . import monitor
+from . import network
+from . import permission_relationships
+from . import rbac
+from . import resource_groups
+from . import security_center
 from . import sql
 from . import storage
 from . import subscription
@@ -27,30 +44,162 @@ def _sync_one_subscription(
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    container_instances.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
     compute.sync(
         neo4j_session,
-        credentials.arm_credentials,
+        credentials.credential,
         subscription_id,
         update_tag,
         common_job_parameters,
     )
     cosmosdb.sync(
         neo4j_session,
-        credentials.arm_credentials,
+        credentials.credential,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    app_service.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    functions.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    event_grid.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    logic_apps.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    rbac.sync(
+        neo4j_session,
+        credentials,
         subscription_id,
         update_tag,
         common_job_parameters,
     )
     sql.sync(
         neo4j_session,
-        credentials.arm_credentials,
+        credentials.credential,
         subscription_id,
         update_tag,
         common_job_parameters,
     )
     storage.sync(
         neo4j_session,
-        credentials.arm_credentials,
+        credentials.credential,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    resource_groups.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    aks.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    factories_raw = data_factory.sync_data_factories(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    linked_services_by_factory = (
+        data_factory_linked_service.sync_data_factory_linked_services(
+            neo4j_session,
+            credentials,
+            factories_raw,
+            subscription_id,
+            update_tag,
+            common_job_parameters,
+        )
+    )
+    datasets_by_factory = data_factory_dataset.sync_data_factory_datasets(
+        neo4j_session,
+        credentials,
+        factories_raw,
+        linked_services_by_factory,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    data_factory_pipeline.sync_data_factory_pipelines(
+        neo4j_session,
+        credentials,
+        factories_raw,
+        datasets_by_factory,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    data_lake.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    network.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    load_balancers.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    monitor.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    security_center.sync(
+        neo4j_session,
+        credentials,
+        subscription_id,
+        update_tag,
+        common_job_parameters,
+    )
+    permission_relationships.sync(
+        neo4j_session,
         subscription_id,
         update_tag,
         common_job_parameters,
@@ -59,13 +208,12 @@ def _sync_one_subscription(
 
 def _sync_tenant(
     neo4j_session: neo4j.Session,
-    tenant_id: str,
-    current_user: Optional[str],
+    credentials: Credentials,
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
-    logger.info("Syncing Azure Tenant: %s", tenant_id)
-    tenant.sync(neo4j_session, tenant_id, current_user, update_tag, common_job_parameters)  # type: ignore
+    logger.info("Syncing Azure Tenant: %s", credentials.tenant_id)
+    tenant.sync(neo4j_session, credentials.tenant_id, None, update_tag, common_job_parameters)  # type: ignore
 
 
 def _sync_multiple_subscriptions(
@@ -106,56 +254,62 @@ def start_azure_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
         "permission_relationships_file": config.permission_relationships_file,
+        "azure_permission_relationships_file": config.azure_permission_relationships_file,
     }
 
-    try:
-        if config.azure_sp_auth:
-            credentials = Authenticator().authenticate_sp(
-                config.azure_tenant_id,
-                config.azure_client_id,
-                config.azure_client_secret,
+    if config.azure_sp_auth:
+        if not (
+            config.azure_tenant_id
+            and config.azure_client_id
+            and config.azure_client_secret
+        ):
+            raise ValueError(
+                "Azure Service Principal authentication requested, but tenant ID, client ID, "
+                "and client secret were not all provided.",
             )
-        else:
-            credentials = Authenticator().authenticate_cli()
-
-    except Exception as e:
-        logger.error(
-            (
-                "Unable to authenticate with Azure Service Principal, an error occurred: %s."
-                "Make sure your Azure Service Principal details are provided correctly."
-            ),
-            e,
+
+        credentials = Authenticator().authenticate_sp(
+            config.azure_tenant_id,
+            config.azure_client_id,
+            config.azure_client_secret,
+        )
+    else:
+        credentials = Authenticator().authenticate_cli()
+
+    if not credentials:
+        raise RuntimeError(
+            "Azure authentication failed. Ensure Azure CLI login or Service Principal credentials are configured.",
         )
-        return
+
+    common_job_parameters["TENANT_ID"] = credentials.tenant_id
 
     _sync_tenant(
         neo4j_session,
-        credentials.get_tenant_id(),
-        credentials.get_current_user(),
+        credentials,
         config.update_tag,
         common_job_parameters,
     )
+    if credentials.tenant_id:
+        if config.azure_sync_all_subscriptions:
+            subscriptions = subscription.get_all_azure_subscriptions(credentials)
 
-    if config.azure_sync_all_subscriptions:
-        subscriptions = subscription.get_all_azure_subscriptions(credentials)
+        else:
+            sub_id_to_sync = config.azure_subscription_id or credentials.subscription_id
+            subscriptions = subscription.get_current_azure_subscription(
+                credentials,
+                sub_id_to_sync,
+            )
 
-    else:
-        subscriptions = subscription.get_current_azure_subscription(
-            credentials,
-            credentials.subscription_id,
-        )
+        if not subscriptions:
+            raise RuntimeError(
+                "No Azure subscriptions found. Ensure the credentials have access to at least one subscription.",
+            )
 
-    if not subscriptions:
-        logger.warning(
-            "No valid Azure credentials are found. No Azure subscriptions can be synced. Exiting Azure sync stage.",
+        _sync_multiple_subscriptions(
+            neo4j_session,
+            credentials,
+            credentials.tenant_id,
+            subscriptions,
+            config.update_tag,
+            common_job_parameters,
         )
-        return
-
-    _sync_multiple_subscriptions(
-        neo4j_session,
-        credentials,
-        credentials.get_tenant_id(),
-        subscriptions,
-        config.update_tag,
-        common_job_parameters,
-    )

cartography/intel/azure/aks.py

@@ -0,0 +1,175 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.core.exceptions import ClientAuthenticationError
+from azure.core.exceptions import HttpResponseError
+from azure.mgmt.containerservice import ContainerServiceClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.aks_cluster import AzureKubernetesClusterSchema
+from cartography.models.azure.aks_nodepool import AzureKubernetesNodePoolSchema
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+def _get_resource_group_from_id(resource_id: str) -> str:
+    """
+    Helper function to parse the resource group name from a full resource ID string.
+    """
+    # CORRECTED: Convert to lowercase to handle case inconsistencies from the API.
+    parts = resource_id.lower().split("/")
+    try:
+        # The resource group name is always the string after 'resourcegroups'
+        rg_index = parts.index("resourcegroups")
+        return parts[rg_index + 1]
+    except (ValueError, IndexError):
+        logger.warning(
+            f"Could not parse resource group name from resource ID: {resource_id}"
+        )
+        return ""
+
+
+@timeit
+def get_aks_clusters(
+    client: ContainerServiceClient, subscription_id: str
+) -> list[dict]:
+    try:
+        return [cluster.as_dict() for cluster in client.managed_clusters.list()]
+    except (ClientAuthenticationError, HttpResponseError) as e:
+        logger.warning(
+            f"Failed to get AKS clusters for subscription {subscription_id}: {str(e)}"
+        )
+        return []
+
+
+@timeit
+def get_agent_pools(
+    client: ContainerServiceClient,
+    cluster_name: str,
+    resource_group_name: str,
+) -> list[dict]:
+    try:
+        return [
+            pool.as_dict()
+            for pool in client.agent_pools.list(resource_group_name, cluster_name)
+        ]
+    except (ClientAuthenticationError, HttpResponseError) as e:
+        logger.warning(
+            f"Failed to get agent pools for cluster {cluster_name}: {str(e)}"
+        )
+        return []
+
+
+@timeit
+def transform_aks_clusters(clusters: list[dict]) -> list[dict]:
+    transformed_clusters: list[dict[str, Any]] = []
+    for cluster in clusters:
+        transformed_cluster = {
+            "id": cluster.get("id"),
+            "name": cluster.get("name"),
+            "location": cluster.get("location"),
+            "provisioning_state": cluster.get("properties", {}).get(
+                "provisioning_state"
+            ),
+            "kubernetes_version": cluster.get("properties", {}).get(
+                "kubernetes_version"
+            ),
+            "fqdn": cluster.get("properties", {}).get("fqdn"),
+        }
+        transformed_clusters.append(transformed_cluster)
+    return transformed_clusters
+
+
+@timeit
+def transform_agent_pools(agent_pools: list[dict]) -> list[dict]:
+    transformed_pools: list[dict[str, Any]] = []
+    for pool in agent_pools:
+        transformed_pool = {
+            "id": pool.get("id"),
+            "name": pool.get("name"),
+            "provisioning_state": pool.get("properties", {}).get("provisioning_state"),
+            "vm_size": pool.get("properties", {}).get("vm_size"),
+            "os_type": pool.get("properties", {}).get("os_type"),
+            "count": pool.get("properties", {}).get("count"),
+        }
+        transformed_pools.append(transformed_pool)
+    return transformed_pools
+
+
+@timeit
+def load_aks_clusters(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureKubernetesClusterSchema(),
+        data,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def load_agent_pools(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    cluster_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureKubernetesNodePoolSchema(),
+        data,
+        lastupdated=update_tag,
+        CLUSTER_ID=cluster_id,
+    )
+
+
+@timeit
+def cleanup_clusters(neo4j_session: neo4j.Session, common_job_parameters: dict) -> None:
+    GraphJob.from_node_schema(
+        AzureKubernetesClusterSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    logger.info(f"Syncing Azure Kubernetes Service for subscription {subscription_id}.")
+    client = ContainerServiceClient(credentials.credential, subscription_id)
+
+    clusters = get_aks_clusters(client, subscription_id)
+    transformed_clusters = transform_aks_clusters(clusters)
+    load_aks_clusters(neo4j_session, transformed_clusters, subscription_id, update_tag)
+
+    for cluster in clusters:
+        cluster_id = cluster.get("id")
+        if not cluster_id:
+            continue
+
+        resource_group_name = _get_resource_group_from_id(cluster_id)
+        if resource_group_name:
+            agent_pools = get_agent_pools(client, cluster["name"], resource_group_name)
+            transformed_pools = transform_agent_pools(agent_pools)
+            load_agent_pools(neo4j_session, transformed_pools, cluster_id, update_tag)
+
+            pool_cleanup_params = common_job_parameters.copy()
+            pool_cleanup_params["CLUSTER_ID"] = cluster_id
+            GraphJob.from_node_schema(
+                AzureKubernetesNodePoolSchema(), pool_cleanup_params
+            ).run(neo4j_session)
+
+    cleanup_clusters(neo4j_session, common_job_parameters)

cartography/intel/azure/app_service.py

@@ -0,0 +1,105 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from azure.core.exceptions import ClientAuthenticationError
+from azure.core.exceptions import HttpResponseError
+from azure.mgmt.web import WebSiteManagementClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.app_service import AzureAppServiceSchema
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_app_services(credentials: Credentials, subscription_id: str) -> List[Dict]:
+    """
+    Get a list of App Services from the given Azure subscription.
+    """
+    try:
+        client = WebSiteManagementClient(credentials.credential, subscription_id)
+        # NOTE: This is the same API call as Functions. We get all web apps
+        # and then filter them in the transform stage.
+        return [app.as_dict() for app in client.web_apps.list()]
+    except (ClientAuthenticationError, HttpResponseError) as e:
+        logger.warning(
+            f"Failed to get app services for subscription {subscription_id}: {str(e)}"
+        )
+        return []
+
+
+@timeit
+def transform_app_services(app_services_response: List[Dict]) -> List[Dict]:
+    """
+    Transform the raw API response to the dictionary structure that the model expects.
+    """
+    transformed_apps: List[Dict[str, Any]] = []
+    for app in app_services_response:
+        if "functionapp" not in app.get("kind", ""):
+            transformed_app = {
+                "id": app.get("id"),
+                "name": app.get("name"),
+                "kind": app.get("kind"),
+                "location": app.get("location"),
+                "state": app.get("state"),
+                "default_host_name": app.get("default_host_name"),
+                "https_only": app.get("https_only"),
+            }
+            transformed_apps.append(transformed_app)
+    return transformed_apps
+
+
+@timeit
+def load_app_services(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load the transformed Azure App Service data to Neo4j.
+    """
+    load(
+        neo4j_session,
+        AzureAppServiceSchema(),
+        data,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def cleanup_app_services(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run the cleanup job for Azure App Services.
+    """
+    GraphJob.from_node_schema(AzureAppServiceSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    The main sync function for Azure App Services.
+    """
+    logger.info(f"Syncing Azure App Services for subscription {subscription_id}.")
+    raw_apps = get_app_services(credentials, subscription_id)
+    transformed_apps = transform_app_services(raw_apps)
+    load_app_services(neo4j_session, transformed_apps, subscription_id, update_tag)
+    cleanup_app_services(neo4j_session, common_job_parameters)