cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/azure/subscription.py

@@ -7,7 +7,9 @@ import neo4j
 from azure.core.exceptions import HttpResponseError
 from azure.mgmt.resource import SubscriptionClient
 
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.subscription import AzureSubscriptionSchema
 from cartography.util import timeit
 
 from .util.credentials import Credentials
@@ -18,19 +20,16 @@ logger = logging.getLogger(__name__)
 def get_all_azure_subscriptions(credentials: Credentials) -> List[Dict]:
     try:
         # Create the client
-        client = SubscriptionClient(credentials.arm_credentials)
+        client = SubscriptionClient(credentials.credential)
 
         # Get all the accessible subscriptions
         subs = list(client.subscriptions.list())
 
     except HttpResponseError as e:
-        logger.error(
-            f"failed to fetch subscriptions for the credentials \
-            The provided credentials do not have access to any subscriptions - \
-            {e}",
-        )
-
-        return []
+        raise RuntimeError(
+            f"Failed to fetch subscriptions for the credentials. "
+            f"The provided credentials do not have access to any subscriptions: {e}",
+        ) from e
 
     subscriptions = []
     for sub in subs:
@@ -52,19 +51,16 @@ def get_current_azure_subscription(
 ) -> List[Dict]:
     try:
         # Create the client
-        client = SubscriptionClient(credentials.arm_credentials)
+        client = SubscriptionClient(credentials.credential)
 
         # Get all the accessible subscriptions
         sub = client.subscriptions.get(subscription_id)
 
     except HttpResponseError as e:
-        logger.error(
-            f"failed to fetch subscription for the credentials \
-            The provided credentials do not have access to this subscription: {subscription_id} - \
-            {e}",
-        )
-
-        return []
+        raise RuntimeError(
+            f"Failed to fetch subscription for the credentials. "
+            f"The provided credentials do not have access to this subscription: {subscription_id}: {e}",
+        ) from e
 
     return [
         {
@@ -82,36 +78,18 @@ def load_azure_subscriptions(
     subscriptions: List[Dict],
     update_tag: int,
 ) -> None:
-    query = """
-    MERGE (at:AzureTenant{id: $TENANT_ID})
-    ON CREATE SET at.firstseen = timestamp()
-    SET at.lastupdated = $update_tag
-    WITH at
-    MERGE (as:AzureSubscription{id: $SUBSCRIPTION_ID})
-    ON CREATE SET as.firstseen = timestamp(), as.path = $SUBSCRIPTION_PATH
-    SET as.lastupdated = $update_tag, as.name = $SUBSCRIPTION_NAME, as.state = $SUBSCRIPTION_STATE
-    WITH as, at
-    MERGE (at)-[r:RESOURCE]->(as)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag;
-    """
-    for sub in subscriptions:
-        neo4j_session.run(
-            query,
-            TENANT_ID=tenant_id,
-            SUBSCRIPTION_ID=sub["subscriptionId"],
-            SUBSCRIPTION_PATH=sub["id"],
-            SUBSCRIPTION_NAME=sub["displayName"],
-            SUBSCRIPTION_STATE=sub["state"],
-            update_tag=update_tag,
-        )
+    load(
+        neo4j_session,
+        AzureSubscriptionSchema(),
+        subscriptions,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
 
 
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job(
-        "azure_subscriptions_cleanup.json",
+    GraphJob.from_node_schema(AzureSubscriptionSchema(), common_job_parameters).run(
         neo4j_session,
-        common_job_parameters,
     )
 
 

cartography/intel/azure/tenant.py

@@ -1,58 +1,67 @@
 import logging
 from typing import Dict
+from typing import Optional
 
 import neo4j
 
-from cartography.util import run_cleanup_job
-from cartography.util import timeit
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.principal import AzurePrincipalSchema
 
-from .util.credentials import Credentials
+# Import the new, separated schemas
+from cartography.models.azure.tenant import AzureTenantSchema
+from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
-def get_tenant_id(credentials: Credentials) -> str:
-    return credentials.get_tenant_id()
-
-
+@timeit
 def load_azure_tenant(
     neo4j_session: neo4j.Session,
     tenant_id: str,
-    current_user: str,
+    current_user: Optional[str],
     update_tag: int,
 ) -> None:
-    query = """
-    MERGE (at:AzureTenant{id: $TENANT_ID})
-    ON CREATE SET at.firstseen = timestamp()
-    SET at.lastupdated = $update_tag
-    WITH at
-    MERGE (ap:AzurePrincipal{id: $CURRENT_USER})
-    ON CREATE SET ap.email = $CURRENT_USER, ap.firstseen = timestamp()
-    SET ap.lastupdated = $update_tag
-    WITH at, ap
-    MERGE (at)-[r:RESOURCE]->(ap)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag;
     """
-    neo4j_session.run(
-        query,
-        TENANT_ID=tenant_id,
-        CURRENT_USER=current_user,
-        update_tag=update_tag,
-    )
+    Ingest the Azure Tenant and, if available, the Azure Principal into Neo4j.
+    """
+    tenant_data = {"id": tenant_id}
+    load(neo4j_session, AzureTenantSchema(), [tenant_data], lastupdated=update_tag)
 
+    if current_user:
+        principal_data = {"id": current_user}
+        load(
+            neo4j_session,
+            AzurePrincipalSchema(),
+            [principal_data],
+            lastupdated=update_tag,
+            TENANT_ID=tenant_id,
+        )
 
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job("azure_tenant_cleanup.json", neo4j_session, common_job_parameters)
+
+@timeit
+def cleanup_azure_tenant(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Delete stale Azure Tenants and Principals.
+    """
+    GraphJob.from_node_schema(AzureTenantSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AzurePrincipalSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
 def sync(
     neo4j_session: neo4j.Session,
     tenant_id: str,
-    current_user: str,
+    current_user: Optional[str],
     update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    logger.info("Syncing Azure tenant '%s'.", tenant_id)
     load_azure_tenant(neo4j_session, tenant_id, current_user, update_tag)
-    cleanup(neo4j_session, common_job_parameters)
+    cleanup_azure_tenant(neo4j_session, common_job_parameters)

cartography/intel/azure/util/common.py

@@ -0,0 +1,13 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_resource_group_from_id(resource_id: str) -> str:
+    """
+    Helper function to parse the resource group name from a full resource ID string.
+    e.g. /subscriptions/sub_id/resourceGroups/rg_name/providers/...
+    """
+    parts = resource_id.lower().split("/")
+    rg_index = parts.index("resourcegroups")
+    return parts[rg_index + 1]

cartography/intel/azure/util/credentials.py

@@ -1,222 +1,97 @@
 import logging
-from datetime import datetime
-from datetime import timedelta
 from typing import Any
 from typing import Optional
 
-import adal
-import requests
-from azure.common.credentials import get_azure_cli_credentials
-from azure.common.credentials import get_cli_profile
-from azure.core.exceptions import HttpResponseError
+import jwt
+from azure.identity import AzureCliCredential
 from azure.identity import ClientSecretCredential
-from msrestazure.azure_active_directory import AADTokenCredentials
+from azure.mgmt.resource import SubscriptionClient
 
 logger = logging.getLogger(__name__)
-AUTHORITY_HOST_URI = "https://login.microsoftonline.com"
+
+
+def _get_tenant_id_from_token(credential: Any) -> str:
+    """
+    A helper function to get the tenant ID from the claims in an access token.
+    """
+    token = credential.get_token("https://management.azure.com/.default")
+    decoded_token = jwt.decode(token.token, options={"verify_signature": False})
+    return decoded_token.get("tid", "")
 
 
 class Credentials:
+    """
+    A simple data container for the credential object and its associated IDs.
+    """
 
     def __init__(
         self,
-        arm_credentials: Any,
-        aad_graph_credentials: Any,
+        credential: Any,
         tenant_id: Optional[str] = None,
         subscription_id: Optional[str] = None,
-        context: Optional[adal.AuthenticationContext] = None,
-        current_user: Optional[str] = None,
     ) -> None:
-        self.arm_credentials = arm_credentials  # Azure Resource Manager API credentials
-        self.aad_graph_credentials = (
-            aad_graph_credentials  # Azure AD Graph API credentials
-        )
+        self.credential = credential
         self.tenant_id = tenant_id
         self.subscription_id = subscription_id
-        self.context = context
-        self.current_user = current_user
-
-    def get_current_user(self) -> Optional[str]:
-        return self.current_user
-
-    def get_tenant_id(self) -> Any:
-        if self.tenant_id:
-            return self.tenant_id
-        elif "tenant_id" in self.aad_graph_credentials.token:
-            return self.aad_graph_credentials.token["tenant_id"]
-        else:
-            # This is a last resort, e.g. for MSI authentication
-            try:
-                h = {
-                    "Authorization": "Bearer {}".format(
-                        self.arm_credentials.token["access_token"],
-                    ),
-                }
-                r = requests.get(
-                    "https://management.azure.com/tenants?api-version=2020-01-01",
-                    headers=h,
-                )
-                r2 = r.json()
-                return r2.get("value")[0].get("tenantId")
-            except requests.ConnectionError as e:
-                logger.error(f"Unable to infer tenant ID: {e}")
-                return None
-
-    def get_credentials(self, resource: str) -> Any:
-        if resource == "arm":
-            self.arm_credentials = self.get_fresh_credentials(self.arm_credentials)
-            return self.arm_credentials
-        elif resource == "aad_graph":
-            self.aad_graph_credentials = self.get_fresh_credentials(
-                self.aad_graph_credentials,
-            )
-            return self.aad_graph_credentials
-        else:
-            raise Exception("Invalid credentials resource type")
-
-    def get_fresh_credentials(self, credentials: Any) -> Any:
-        """
-        Check if credentials are outdated and if so refresh them.
-        """
-        if self.context and hasattr(credentials, "token"):
-            expiration_datetime = datetime.fromtimestamp(
-                credentials.token["expires_on"],
-            )
-            current_datetime = datetime.now()
-            expiration_delta = expiration_datetime - current_datetime
-            if expiration_delta < timedelta(minutes=5):
-                return self.refresh_credential(credentials)
-        return credentials
-
-    def refresh_credential(self, credentials: Any) -> Any:
-        """
-        Refresh credentials
-        """
-        logger.debug("Refreshing credentials")
-        authority_uri = AUTHORITY_HOST_URI + "/" + self.get_tenant_id()
-        if self.context:
-            existing_cache = self.context.cache
-            context = adal.AuthenticationContext(authority_uri, cache=existing_cache)
-
-        else:
-            context = adal.AuthenticationContext(authority_uri)
-
-        new_token = context.acquire_token(
-            credentials.token["resource"],
-            credentials.token["user_id"],
-            credentials.token["_client_id"],
-        )
-
-        new_credentials = AADTokenCredentials(
-            new_token,
-            credentials.token.get("_client_id"),
-        )
-        return new_credentials
 
 
 class Authenticator:
 
-    def authenticate_cli(self) -> Credentials:
+    def authenticate_cli(self) -> Optional[Credentials]:
         """
-        Implements authentication for the Azure provider
+        Implements authentication using the Azure CLI with the modern library.
         """
+        logging.getLogger("urllib3").setLevel(logging.ERROR)
+        logging.getLogger(
+            "azure.core.pipeline.policies.http_logging_policy",
+        ).setLevel(logging.ERROR)
         try:
+            credential = AzureCliCredential()
 
-            # Set logging level to error for libraries as otherwise generates a lot of warnings
-            logging.getLogger("adal-python").setLevel(logging.ERROR)
-            logging.getLogger("msrest").setLevel(logging.ERROR)
-            logging.getLogger("msrestazure.azure_active_directory").setLevel(
-                logging.ERROR,
-            )
-            logging.getLogger("urllib3").setLevel(logging.ERROR)
-            logging.getLogger(
-                "azure.core.pipeline.policies.http_logging_policy",
-            ).setLevel(logging.ERROR)
+            subscription_client = SubscriptionClient(credential)
+            subscription = next(subscription_client.subscriptions.list())
+            subscription_id = subscription.subscription_id
 
-            arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(
-                with_tenant=True,
-            )
-            aad_graph_credentials, placeholder_1, placeholder_2 = (
-                get_azure_cli_credentials(
-                    with_tenant=True,
-                    resource="https://graph.windows.net",
-                )
-            )
-
-            profile = get_cli_profile()
+            tenant_id = _get_tenant_id_from_token(credential)
 
             return Credentials(
-                arm_credentials,
-                aad_graph_credentials,
+                credential=credential,
                 tenant_id=tenant_id,
-                current_user=profile.get_current_account_user(),
                 subscription_id=subscription_id,
             )
-
-        except HttpResponseError as e:
-            if (
-                ", AdalError: Unsupported wstrust endpoint version. "
-                "Current supported version is wstrust2005 or wstrust13." in e.args
-            ):
-                logger.error(
-                    f"You are likely authenticating with a Microsoft Account. \
-                    This authentication mode only supports Azure Active Directory principal authentication.\
-                    {e}",
-                )
-
-            raise e
+        except Exception as e:
+            logger.error(
+                f"Failed to authenticate with Azure CLI. Have you run 'az login'? Details: {e}"
+            )
+            return None
 
     def authenticate_sp(
         self,
         tenant_id: Optional[str] = None,
         client_id: Optional[str] = None,
         client_secret: Optional[str] = None,
-    ) -> Credentials:
+        subscription_id: Optional[str] = None,
+    ) -> Optional[Credentials]:
         """
-        Implements authentication for the Azure provider
+        Implements authentication using a Service Principal with the modern library.
         """
         try:
-
-            # Set logging level to error for libraries as otherwise generates a lot of warnings
-            logging.getLogger("adal-python").setLevel(logging.ERROR)
-            logging.getLogger("msrest").setLevel(logging.ERROR)
-            logging.getLogger("msrestazure.azure_active_directory").setLevel(
-                logging.ERROR,
-            )
-            logging.getLogger("urllib3").setLevel(logging.ERROR)
-            logging.getLogger(
-                "azure.core.pipeline.policies.http_logging_policy",
-            ).setLevel(logging.ERROR)
-
-            arm_credentials = ClientSecretCredential(
-                client_id=client_id,
-                client_secret=client_secret,
-                tenant_id=tenant_id,
-            )
-
-            aad_graph_credentials = ClientSecretCredential(
+            credential = ClientSecretCredential(
                 client_id=client_id,
                 client_secret=client_secret,
                 tenant_id=tenant_id,
-                resource="https://graph.windows.net",
             )
-
             return Credentials(
-                arm_credentials,
-                aad_graph_credentials,
+                credential=credential,
                 tenant_id=tenant_id,
-                current_user=client_id,
+                subscription_id=subscription_id,
             )
-
-        except HttpResponseError as e:
-            if (
-                ", AdalError: Unsupported wstrust endpoint version. "
-                "Current supported version is wstrust2005 or wstrust13." in e.args
-            ):
-                logger.error(
-                    f"You are likely authenticating with a Microsoft Account. \
-                    This authentication mode only supports Azure Active Directory principal authentication.\
-                    {e}",
-                )
-
-            raise e
+        except Exception as e:
+            logger.error(
+                (
+                    "Failed to authenticate with Service Principal. "
+                    "Please ensure the tenant ID, client ID, and client secret are correct. Details: %s"
+                ),
+                e,
+            )
+            return None

cartography/intel/azure/util/tag.py

@@ -0,0 +1,41 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def transform_tags(
+    resource_list: list[dict], subscription_id: str, resource_id_field: str = "id"
+) -> list[dict]:
+    """
+    Transforms tags from a list of Azure resources into a standardized list of tag dictionaries.
+
+    Args:
+        resource_list: List of Azure resources with tags
+        subscription_id: The Azure subscription ID to scope the tags
+        resource_id_field: Field name containing the resource ID (default: "id")
+
+    Returns:
+        List of tag dictionaries with subscription-scoped IDs
+    """
+    tags_list = []
+    for resource in resource_list:
+        resource_id = resource.get(resource_id_field)
+        tags = resource.get("tags")
+
+        if not resource_id or not tags:
+            continue
+
+        for key, value in tags.items():
+            # Generate the deterministic ID scoped to subscription: "{subscription_id}|{key}:{value}"
+            tag_id = f"{subscription_id}|{key}:{value}"
+            tags_list.append(
+                {
+                    "id": tag_id,
+                    "key": key,
+                    "value": value,
+                    "subscription_id": subscription_id,
+                    "resource_id": resource_id,
+                }
+            )
+
+    return tags_list

cartography/intel/create_indexes.py

@@ -3,6 +3,7 @@ from typing import List
 
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.config import Config
 from cartography.util import load_resource_binary
 
@@ -23,4 +24,4 @@ def run(neo4j_session: neo4j.Session, config: Config) -> None:
     logger.info("Creating indexes for cartography node types.")
     for statement in get_index_statements():
         logger.debug("Executing statement: %s", statement)
-        neo4j_session.run(statement)
+        run_write_query(neo4j_session, statement)

cartography/intel/crowdstrike/spotlight.py

@@ -6,6 +6,7 @@ import neo4j
 from falconpy.oauth2 import OAuth2
 from falconpy.spotlight_vulnerabilities import Spotlight_Vulnerabilities
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -79,7 +80,8 @@ def load_vulnerability_data(
             cves.append(cve)
         vuln["host_info_local_ip"] = item.get("host_info", {}).get("local_ip")
         vulns.append(vuln)
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Vulnerabilities=vulns,
         update_tag=update_tag,
@@ -106,7 +108,8 @@ def _load_cves(neo4j_session: neo4j.Session, data: List[Dict], update_tag: int)
         ON CREATE SET hc.firstseen = timestamp()
         SET hc.lastupdated = $update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         cves=data,
         update_tag=update_tag,

cartography/intel/dns.py

@@ -8,6 +8,7 @@ import dns.rdatatype
 import dns.resolver
 import neo4j
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -104,7 +105,8 @@ def _link_ip_to_A_record(
     SET r.lastupdated = $update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest,
         ParentId=parent_record,
         IP_LIST=ip_list,
@@ -151,7 +153,8 @@ def ingest_dns_record(
 
     record_id = f"{name}+{type}"
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         template.safe_substitute(
             record_label=record_label,
             dns_node_additional_label=dns_node_additional_label,