cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/azure/functions.py

@@ -0,0 +1,124 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from azure.core.exceptions import ClientAuthenticationError
+from azure.core.exceptions import HttpResponseError
+from azure.mgmt.web import WebSiteManagementClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.function_app import AzureFunctionAppSchema
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_function_apps(credentials: Credentials, subscription_id: str) -> List[Dict]:
+    """
+    Get a list of Function Apps from the given Azure subscription.
+    """
+    try:
+        client = WebSiteManagementClient(credentials.credential, subscription_id)
+        # Note: Function Apps are a type of Web App, so we list all web apps
+        # and then filter them in the transform stage.
+        return [app.as_dict() for app in client.web_apps.list()]
+
+    except ClientAuthenticationError as e:
+        logger.warning(
+            (
+                "Failed to authenticate to get function apps for subscription '%s'. "
+                "Please check your credentials. Error: %s"
+            ),
+            subscription_id,
+            e,
+        )
+        return []
+
+    except HttpResponseError as e:
+        logger.warning(
+            (
+                "Failed to get function apps for subscription '%s' due to an API error. "
+                "Status code: %s. Message: %s"
+            ),
+            subscription_id,
+            e.status_code,
+            str(e),
+        )
+        return []
+
+
+@timeit
+def transform_function_apps(function_apps_response: List[Dict]) -> List[Dict]:
+    """
+    Transform the raw API response to the dictionary structure that the model expects.
+    """
+    transformed_apps: List[Dict[str, Any]] = []
+    for app in function_apps_response:
+        # We only want to ingest resources that are explicitly function apps.
+        if "functionapp" in app.get("kind", ""):
+            transformed_app = {
+                "id": app.get("id"),
+                "name": app.get("name"),
+                "kind": app.get("kind"),
+                "location": app.get("location"),
+                "state": app.get("state"),
+                "default_host_name": app.get("default_host_name"),
+                "https_only": app.get("https_only"),
+            }
+            transformed_apps.append(transformed_app)
+    return transformed_apps
+
+
+@timeit
+def load_function_apps(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load the transformed Azure Function App data to Neo4j.
+    """
+    load(
+        neo4j_session,
+        AzureFunctionAppSchema(),
+        data,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def cleanup_function_apps(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Run the cleanup job for Azure Function Apps.
+    """
+    GraphJob.from_node_schema(AzureFunctionAppSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """
+    The main sync function for Azure Function Apps.
+    """
+    logger.info(f"Syncing Azure Function Apps for subscription {subscription_id}.")
+    raw_apps = get_function_apps(credentials, subscription_id)
+    transformed_apps = transform_function_apps(raw_apps)
+    load_function_apps(neo4j_session, transformed_apps, subscription_id, update_tag)
+    cleanup_function_apps(neo4j_session, common_job_parameters)

cartography/intel/azure/load_balancers.py

@@ -0,0 +1,263 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.mgmt.network import NetworkManagementClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.load_balancer.load_balancer import AzureLoadBalancerSchema
+from cartography.models.azure.load_balancer.load_balancer_backend_pool import (
+    AzureLoadBalancerBackendPoolSchema,
+)
+from cartography.models.azure.load_balancer.load_balancer_frontend_ip import (
+    AzureLoadBalancerFrontendIPSchema,
+)
+from cartography.models.azure.load_balancer.load_balancer_inbound_nat_rule import (
+    AzureLoadBalancerInboundNatRuleSchema,
+)
+from cartography.models.azure.load_balancer.load_balancer_rule import (
+    AzureLoadBalancerRuleSchema,
+)
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+def _get_resource_group_from_id(resource_id: str) -> str:
+    parts = resource_id.lower().split("/")
+    rg_index = parts.index("resourcegroups")
+    return parts[rg_index + 1]
+
+
+@timeit
+def get_load_balancers(client: NetworkManagementClient) -> list[dict]:
+    """
+    Get a list of all Load Balancers in a subscription.
+    """
+    return [lb.as_dict() for lb in client.load_balancers.list_all()]
+
+
+def transform_load_balancers(load_balancers: list[dict]) -> list[dict]:
+    transformed: list[dict[str, Any]] = []
+    for lb in load_balancers:
+        transformed.append(
+            {
+                "id": lb.get("id"),
+                "name": lb.get("name"),
+                "location": lb.get("location"),
+                "sku_name": lb.get("sku", {}).get("name"),
+            }
+        )
+    return transformed
+
+
+def transform_frontend_ips(load_balancer: dict) -> list[dict]:
+    transformed: list[dict[str, Any]] = []
+    for config in load_balancer.get("frontend_ip_configurations", []):
+        transformed.append(
+            {
+                "id": config.get("id"),
+                "name": config.get("name"),
+                "private_ip_address": config.get("properties", {}).get(
+                    "private_ip_address"
+                ),
+                "public_ip_address_id": config.get("properties", {})
+                .get("public_ip_address", {})
+                .get("id"),
+            }
+        )
+    return transformed
+
+
+def transform_backend_pools(load_balancer: dict) -> list[dict]:
+    transformed: list[dict[str, Any]] = []
+    for pool in load_balancer.get("backend_address_pools", []):
+        transformed.append(
+            {
+                "id": pool.get("id"),
+                "name": pool.get("name"),
+            }
+        )
+    return transformed
+
+
+def transform_rules(load_balancer: dict) -> list[dict]:
+    transformed: list[dict[str, Any]] = []
+    for rule in load_balancer.get("load_balancing_rules", []):
+        transformed.append(
+            {
+                "id": rule.get("id"),
+                "name": rule.get("name"),
+                "protocol": rule.get("properties", {}).get("protocol"),
+                "frontend_port": rule.get("properties", {}).get("frontend_port"),
+                "backend_port": rule.get("properties", {}).get("backend_port"),
+                "FRONTEND_IP_ID": rule.get("frontend_ip_configuration", {}).get("id"),
+                "BACKEND_POOL_ID": rule.get("backend_address_pool", {}).get("id"),
+            }
+        )
+    return transformed
+
+
+def transform_inbound_nat_rules(load_balancer: dict) -> list[dict]:
+    transformed: list[dict[str, Any]] = []
+    for rule in load_balancer.get("inbound_nat_rules", []):
+        transformed.append(
+            {
+                "id": rule.get("id"),
+                "name": rule.get("name"),
+                "protocol": rule.get("properties", {}).get("protocol"),
+                "frontend_port": rule.get("properties", {}).get("frontend_port"),
+                "backend_port": rule.get("properties", {}).get("backend_port"),
+            }
+        )
+    return transformed
+
+
+@timeit
+def load_load_balancers(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureLoadBalancerSchema(),
+        data,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def load_frontend_ips(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    lb_id: str,
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureLoadBalancerFrontendIPSchema(),
+        data,
+        lastupdated=update_tag,
+        LOAD_BALANCER_ID=lb_id,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def load_backend_pools(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    lb_id: str,
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureLoadBalancerBackendPoolSchema(),
+        data,
+        lastupdated=update_tag,
+        LOAD_BALANCER_ID=lb_id,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def load_rules(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    lb_id: str,
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureLoadBalancerRuleSchema(),
+        data,
+        lastupdated=update_tag,
+        LOAD_BALANCER_ID=lb_id,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def load_inbound_nat_rules(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    lb_id: str,
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureLoadBalancerInboundNatRuleSchema(),
+        data,
+        lastupdated=update_tag,
+        LOAD_BALANCER_ID=lb_id,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    logger.info(f"Syncing Azure Load Balancers for subscription {subscription_id}.")
+    client = NetworkManagementClient(credentials.credential, subscription_id)
+
+    load_balancers = get_load_balancers(client)
+    transformed_lbs = transform_load_balancers(load_balancers)
+    load_load_balancers(neo4j_session, transformed_lbs, subscription_id, update_tag)
+
+    for lb in load_balancers:
+        lb_id = lb["id"]
+
+        frontend_ips = transform_frontend_ips(lb)
+        load_frontend_ips(
+            neo4j_session, frontend_ips, lb_id, subscription_id, update_tag
+        )
+
+        backend_pools = transform_backend_pools(lb)
+        load_backend_pools(
+            neo4j_session, backend_pools, lb_id, subscription_id, update_tag
+        )
+
+        rules = transform_rules(lb)
+        load_rules(neo4j_session, rules, lb_id, subscription_id, update_tag)
+
+        inbound_nat_rules = transform_inbound_nat_rules(lb)
+        load_inbound_nat_rules(
+            neo4j_session, inbound_nat_rules, lb_id, subscription_id, update_tag
+        )
+
+        # TODO: Implement relationships from Backend Pools and Inbound NAT Rules to Network Interfaces (NICs).
+
+        # Scoped cleanup for child components
+        cleanup_params = common_job_parameters.copy()
+        cleanup_params["LOAD_BALANCER_ID"] = lb_id
+        GraphJob.from_node_schema(
+            AzureLoadBalancerFrontendIPSchema(), cleanup_params
+        ).run(neo4j_session)
+        GraphJob.from_node_schema(
+            AzureLoadBalancerBackendPoolSchema(), cleanup_params
+        ).run(neo4j_session)
+        GraphJob.from_node_schema(AzureLoadBalancerRuleSchema(), cleanup_params).run(
+            neo4j_session
+        )
+        GraphJob.from_node_schema(
+            AzureLoadBalancerInboundNatRuleSchema(), cleanup_params
+        ).run(neo4j_session)
+
+    GraphJob.from_node_schema(AzureLoadBalancerSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/azure/logic_apps.py

@@ -0,0 +1,101 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.core.exceptions import ClientAuthenticationError
+from azure.core.exceptions import HttpResponseError
+from azure.mgmt.logic import LogicManagementClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.logic_apps import AzureLogicAppSchema
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_logic_apps(credentials: Credentials, subscription_id: str) -> list[dict]:
+    """
+    Get a list of Logic Apps from the given Azure subscription.
+    """
+    try:
+        client = LogicManagementClient(credentials.credential, subscription_id)
+        # NOTE: The resource for a Logic App is called a "Workflow" in the SDK.
+        return [w.as_dict() for w in client.workflows.list_by_subscription()]
+    except (ClientAuthenticationError, HttpResponseError) as e:
+        logger.warning(
+            f"Failed to get logic apps for subscription {subscription_id}: {str(e)}"
+        )
+        return []
+
+
+def transform_logic_apps(logic_apps_response: list[dict]) -> list[dict]:
+    """
+    Transform the raw API response to the dictionary structure that the model expects.
+    """
+    transformed_apps: list[dict[str, Any]] = []
+    for app in logic_apps_response:
+        transformed_app = {
+            "id": app.get("id"),
+            "name": app.get("name"),
+            "location": app.get("location"),
+            "state": app.get("properties", {}).get("state"),
+            "created_time": app.get("properties", {}).get("created_time"),
+            "changed_time": app.get("properties", {}).get("changed_time"),
+            "version": app.get("properties", {}).get("version"),
+            "access_endpoint": app.get("properties", {}).get("access_endpoint"),
+        }
+        transformed_apps.append(transformed_app)
+    return transformed_apps
+
+
+@timeit
+def load_logic_apps(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load the transformed Azure Logic App data to Neo4j.
+    """
+    load(
+        neo4j_session,
+        AzureLogicAppSchema(),
+        data,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def cleanup_logic_apps(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    """
+    Run the cleanup job for Azure Logic Apps.
+    """
+    GraphJob.from_node_schema(AzureLogicAppSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    """
+    The main sync function for Azure Logic Apps.
+    """
+    logger.info(f"Syncing Azure Logic Apps for subscription {subscription_id}.")
+    raw_apps = get_logic_apps(credentials, subscription_id)
+    transformed_apps = transform_logic_apps(raw_apps)
+    load_logic_apps(neo4j_session, transformed_apps, subscription_id, update_tag)
+    cleanup_logic_apps(neo4j_session, common_job_parameters)

cartography/intel/azure/monitor.py

@@ -0,0 +1,105 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.core.exceptions import HttpResponseError
+from azure.mgmt.monitor import MonitorManagementClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.monitor import AzureMonitorMetricAlertSchema
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_metric_alerts(client: MonitorManagementClient) -> list[dict]:
+    """
+    Get a list of Metric Alerts from the given Azure subscription.
+    """
+    try:
+        return [
+            alert.as_dict() for alert in client.metric_alerts.list_by_subscription()
+        ]
+    except HttpResponseError:
+        logger.warning(
+            "Failed to get Azure Monitor Metric Alerts due to a transient error.",
+            exc_info=True,
+        )
+        return []
+
+
+def transform_metric_alerts(metric_alerts: list[dict]) -> list[dict]:
+    """
+    Transform the raw API response to the dictionary structure that the model expects.
+    """
+    transformed_alerts: list[dict[str, Any]] = []
+    for alert in metric_alerts:
+        transformed_alert = {
+            "id": alert.get("id"),
+            "name": alert.get("name"),
+            "location": alert.get("location"),
+            "description": alert.get("description"),
+            "severity": alert.get("severity"),
+            "enabled": alert.get("enabled"),
+            "window_size": str(alert.get("window_size")),
+            "evaluation_frequency": str(alert.get("evaluation_frequency")),
+            "last_updated_time": alert.get("properties", {}).get("last_updated_time"),
+        }
+        transformed_alerts.append(transformed_alert)
+    return transformed_alerts
+
+
+@timeit
+def load_metric_alerts(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    subscription_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load the transformed Azure Monitor Metric Alert data to Neo4j.
+    """
+    load(
+        neo4j_session,
+        AzureMonitorMetricAlertSchema(),
+        data,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
+    )
+
+
+@timeit
+def cleanup_metric_alerts(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    """
+    Run the cleanup job for Azure Monitor Metric Alerts.
+    """
+    GraphJob.from_node_schema(
+        AzureMonitorMetricAlertSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    """
+    The main sync function for Azure Monitor.
+    """
+    logger.info(
+        f"Syncing Azure Monitor Metric Alerts for subscription {subscription_id}."
+    )
+    client = MonitorManagementClient(credentials.credential, subscription_id)
+    raw_alerts = get_metric_alerts(client)
+    transformed_alerts = transform_metric_alerts(raw_alerts)
+    load_metric_alerts(neo4j_session, transformed_alerts, subscription_id, update_tag)
+    cleanup_metric_alerts(neo4j_session, common_job_parameters)