cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/azure/subscription.py

@@ -7,104 +7,98 @@ import neo4j
 from azure.core.exceptions import HttpResponseError
 from azure.mgmt.resource import SubscriptionClient
 
-from .util.credentials import Credentials
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.subscription import AzureSubscriptionSchema
 from cartography.util import timeit
 
+from .util.credentials import Credentials
+
 logger = logging.getLogger(__name__)
 
 
 def get_all_azure_subscriptions(credentials: Credentials) -> List[Dict]:
     try:
         # Create the client
-        client = SubscriptionClient(credentials.arm_credentials)
+        client = SubscriptionClient(credentials.credential)
 
         # Get all the accessible subscriptions
         subs = list(client.subscriptions.list())
 
     except HttpResponseError as e:
-        logger.error(
-            f'failed to fetch subscriptions for the credentials \
-            The provided credentials do not have access to any subscriptions - \
-            {e}',
-        )
-
-        return []
+        raise RuntimeError(
+            f"Failed to fetch subscriptions for the credentials. "
+            f"The provided credentials do not have access to any subscriptions: {e}",
+        ) from e
 
     subscriptions = []
     for sub in subs:
-        subscriptions.append({
-            'id': sub.id,
-            'subscriptionId': sub.subscription_id,
-            'displayName': sub.display_name,
-            'state': sub.state,
-        })
+        subscriptions.append(
+            {
+                "id": sub.id,
+                "subscriptionId": sub.subscription_id,
+                "displayName": sub.display_name,
+                "state": sub.state,
+            },
+        )
 
     return subscriptions
 
 
-def get_current_azure_subscription(credentials: Credentials, subscription_id: Optional[str]) -> List[Dict]:
+def get_current_azure_subscription(
+    credentials: Credentials,
+    subscription_id: Optional[str],
+) -> List[Dict]:
     try:
         # Create the client
-        client = SubscriptionClient(credentials.arm_credentials)
+        client = SubscriptionClient(credentials.credential)
 
         # Get all the accessible subscriptions
         sub = client.subscriptions.get(subscription_id)
 
     except HttpResponseError as e:
-        logger.error(
-            f'failed to fetch subscription for the credentials \
-            The provided credentials do not have access to this subscription: {subscription_id} - \
-            {e}',
-        )
-
-        return []
+        raise RuntimeError(
+            f"Failed to fetch subscription for the credentials. "
+            f"The provided credentials do not have access to this subscription: {subscription_id}: {e}",
+        ) from e
 
     return [
         {
-            'id': sub.id,
-            'subscriptionId': sub.subscription_id,
-            'displayName': sub.display_name,
-            'state': sub.state,
+            "id": sub.id,
+            "subscriptionId": sub.subscription_id,
+            "displayName": sub.display_name,
+            "state": sub.state,
         },
     ]
 
 
 def load_azure_subscriptions(
-    neo4j_session: neo4j.Session, tenant_id: str, subscriptions: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    subscriptions: List[Dict],
+    update_tag: int,
 ) -> None:
-    query = """
-    MERGE (at:AzureTenant{id: $TENANT_ID})
-    ON CREATE SET at.firstseen = timestamp()
-    SET at.lastupdated = $update_tag
-    WITH at
-    MERGE (as:AzureSubscription{id: $SUBSCRIPTION_ID})
-    ON CREATE SET as.firstseen = timestamp(), as.path = $SUBSCRIPTION_PATH
-    SET as.lastupdated = $update_tag, as.name = $SUBSCRIPTION_NAME, as.state = $SUBSCRIPTION_STATE
-    WITH as, at
-    MERGE (at)-[r:RESOURCE]->(as)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag;
-    """
-    for sub in subscriptions:
-        neo4j_session.run(
-            query,
-            TENANT_ID=tenant_id,
-            SUBSCRIPTION_ID=sub['subscriptionId'],
-            SUBSCRIPTION_PATH=sub['id'],
-            SUBSCRIPTION_NAME=sub['displayName'],
-            SUBSCRIPTION_STATE=sub['state'],
-            update_tag=update_tag,
-        )
+    load(
+        neo4j_session,
+        AzureSubscriptionSchema(),
+        subscriptions,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
 
 
 def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('azure_subscriptions_cleanup.json', neo4j_session, common_job_parameters)
+    GraphJob.from_node_schema(AzureSubscriptionSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, tenant_id: str, subscriptions: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    subscriptions: List[Dict],
+    update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     load_azure_subscriptions(neo4j_session, tenant_id, subscriptions, update_tag)

cartography/intel/azure/tenant.py

@@ -1,49 +1,67 @@
 import logging
 from typing import Dict
+from typing import Optional
 
 import neo4j
 
-from .util.credentials import Credentials
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.principal import AzurePrincipalSchema
+
+# Import the new, separated schemas
+from cartography.models.azure.tenant import AzureTenantSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
-def get_tenant_id(credentials: Credentials) -> str:
-    return credentials.get_tenant_id()
+@timeit
+def load_azure_tenant(
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    current_user: Optional[str],
+    update_tag: int,
+) -> None:
+    """
+    Ingest the Azure Tenant and, if available, the Azure Principal into Neo4j.
+    """
+    tenant_data = {"id": tenant_id}
+    load(neo4j_session, AzureTenantSchema(), [tenant_data], lastupdated=update_tag)
+
+    if current_user:
+        principal_data = {"id": current_user}
+        load(
+            neo4j_session,
+            AzurePrincipalSchema(),
+            [principal_data],
+            lastupdated=update_tag,
+            TENANT_ID=tenant_id,
+        )
 
 
-def load_azure_tenant(neo4j_session: neo4j.Session, tenant_id: str, current_user: str, update_tag: int) -> None:
-    query = """
-    MERGE (at:AzureTenant{id: $TENANT_ID})
-    ON CREATE SET at.firstseen = timestamp()
-    SET at.lastupdated = $update_tag
-    WITH at
-    MERGE (ap:AzurePrincipal{id: $CURRENT_USER})
-    ON CREATE SET ap.email = $CURRENT_USER, ap.firstseen = timestamp()
-    SET ap.lastupdated = $update_tag
-    WITH at, ap
-    MERGE (at)-[r:RESOURCE]->(ap)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $update_tag;
+@timeit
+def cleanup_azure_tenant(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict
+) -> None:
+    """
+    Delete stale Azure Tenants and Principals.
     """
-    neo4j_session.run(
-        query,
-        TENANT_ID=tenant_id,
-        CURRENT_USER=current_user,
-        update_tag=update_tag,
+    GraphJob.from_node_schema(AzureTenantSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AzurePrincipalSchema(), common_job_parameters).run(
+        neo4j_session
     )
-
-
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    run_cleanup_job('azure_tenant_cleanup.json', neo4j_session, common_job_parameters)
 
 
 @timeit
 def sync(
-    neo4j_session: neo4j.Session, tenant_id: str, current_user: str, update_tag: int,
+    neo4j_session: neo4j.Session,
+    tenant_id: str,
+    current_user: Optional[str],
+    update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
+    logger.info("Syncing Azure tenant '%s'.", tenant_id)
     load_azure_tenant(neo4j_session, tenant_id, current_user, update_tag)
-    cleanup(neo4j_session, common_job_parameters)
+    cleanup_azure_tenant(neo4j_session, common_job_parameters)

cartography/intel/azure/util/common.py

@@ -0,0 +1,13 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_resource_group_from_id(resource_id: str) -> str:
+    """
+    Helper function to parse the resource group name from a full resource ID string.
+    e.g. /subscriptions/sub_id/resourceGroups/rg_name/providers/...
+    """
+    parts = resource_id.lower().split("/")
+    rg_index = parts.index("resourcegroups")
+    return parts[rg_index + 1]

cartography/intel/azure/util/credentials.py

@@ -1,182 +1,97 @@
 import logging
-from datetime import datetime
-from datetime import timedelta
 from typing import Any
 from typing import Optional
 
-import adal
-import requests
-from azure.common.credentials import get_azure_cli_credentials
-from azure.common.credentials import get_cli_profile
-from azure.core.exceptions import HttpResponseError
+import jwt
+from azure.identity import AzureCliCredential
 from azure.identity import ClientSecretCredential
-from msrestazure.azure_active_directory import AADTokenCredentials
+from azure.mgmt.resource import SubscriptionClient
 
 logger = logging.getLogger(__name__)
-AUTHORITY_HOST_URI = 'https://login.microsoftonline.com'
+
+
+def _get_tenant_id_from_token(credential: Any) -> str:
+    """
+    A helper function to get the tenant ID from the claims in an access token.
+    """
+    token = credential.get_token("https://management.azure.com/.default")
+    decoded_token = jwt.decode(token.token, options={"verify_signature": False})
+    return decoded_token.get("tid", "")
 
 
 class Credentials:
+    """
+    A simple data container for the credential object and its associated IDs.
+    """
 
     def __init__(
-            self,
-            arm_credentials: Any,
-            aad_graph_credentials: Any,
-            tenant_id: Optional[str] = None,
-            subscription_id: Optional[str] = None,
-            context: Optional[adal.AuthenticationContext] = None,
-            current_user: Optional[str] = None,
+        self,
+        credential: Any,
+        tenant_id: Optional[str] = None,
+        subscription_id: Optional[str] = None,
     ) -> None:
-        self.arm_credentials = arm_credentials  # Azure Resource Manager API credentials
-        self.aad_graph_credentials = aad_graph_credentials  # Azure AD Graph API credentials
+        self.credential = credential
         self.tenant_id = tenant_id
         self.subscription_id = subscription_id
-        self.context = context
-        self.current_user = current_user
-
-    def get_current_user(self) -> Optional[str]:
-        return self.current_user
-
-    def get_tenant_id(self) -> Any:
-        if self.tenant_id:
-            return self.tenant_id
-        elif 'tenant_id' in self.aad_graph_credentials.token:
-            return self.aad_graph_credentials.token['tenant_id']
-        else:
-            # This is a last resort, e.g. for MSI authentication
-            try:
-                h = {'Authorization': 'Bearer {}'.format(self.arm_credentials.token['access_token'])}
-                r = requests.get('https://management.azure.com/tenants?api-version=2020-01-01', headers=h)
-                r2 = r.json()
-                return r2.get('value')[0].get('tenantId')
-            except requests.ConnectionError as e:
-                logger.error(f'Unable to infer tenant ID: {e}')
-                return None
-
-    def get_credentials(self, resource: str) -> Any:
-        if resource == 'arm':
-            self.arm_credentials = self.get_fresh_credentials(self.arm_credentials)
-            return self.arm_credentials
-        elif resource == 'aad_graph':
-            self.aad_graph_credentials = self.get_fresh_credentials(self.aad_graph_credentials)
-            return self.aad_graph_credentials
-        else:
-            raise Exception('Invalid credentials resource type')
-
-    def get_fresh_credentials(self, credentials: Any) -> Any:
-        """
-        Check if credentials are outdated and if so refresh them.
-        """
-        if self.context and hasattr(credentials, 'token'):
-            expiration_datetime = datetime.fromtimestamp(credentials.token['expires_on'])
-            current_datetime = datetime.now()
-            expiration_delta = expiration_datetime - current_datetime
-            if expiration_delta < timedelta(minutes=5):
-                return self.refresh_credential(credentials)
-        return credentials
-
-    def refresh_credential(self, credentials: Any) -> Any:
-        """
-        Refresh credentials
-        """
-        logger.debug('Refreshing credentials')
-        authority_uri = AUTHORITY_HOST_URI + '/' + self.get_tenant_id()
-        if self.context:
-            existing_cache = self.context.cache
-            context = adal.AuthenticationContext(authority_uri, cache=existing_cache)
-
-        else:
-            context = adal.AuthenticationContext(authority_uri)
-
-        new_token = context.acquire_token(
-            credentials.token['resource'], credentials.token['user_id'], credentials.token['_client_id'],
-        )
-
-        new_credentials = AADTokenCredentials(new_token, credentials.token.get('_client_id'))
-        return new_credentials
 
 
 class Authenticator:
 
-    def authenticate_cli(self) -> Credentials:
+    def authenticate_cli(self) -> Optional[Credentials]:
         """
-        Implements authentication for the Azure provider
+        Implements authentication using the Azure CLI with the modern library.
         """
+        logging.getLogger("urllib3").setLevel(logging.ERROR)
+        logging.getLogger(
+            "azure.core.pipeline.policies.http_logging_policy",
+        ).setLevel(logging.ERROR)
         try:
+            credential = AzureCliCredential()
 
-            # Set logging level to error for libraries as otherwise generates a lot of warnings
-            logging.getLogger('adal-python').setLevel(logging.ERROR)
-            logging.getLogger('msrest').setLevel(logging.ERROR)
-            logging.getLogger('msrestazure.azure_active_directory').setLevel(logging.ERROR)
-            logging.getLogger('urllib3').setLevel(logging.ERROR)
-            logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR)
-
-            arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True)
-            aad_graph_credentials, placeholder_1, placeholder_2 = get_azure_cli_credentials(
-                with_tenant=True, resource='https://graph.windows.net',
-            )
+            subscription_client = SubscriptionClient(credential)
+            subscription = next(subscription_client.subscriptions.list())
+            subscription_id = subscription.subscription_id
 
-            profile = get_cli_profile()
+            tenant_id = _get_tenant_id_from_token(credential)
 
             return Credentials(
-                arm_credentials, aad_graph_credentials, tenant_id=tenant_id,
-                current_user=profile.get_current_account_user(), subscription_id=subscription_id,
+                credential=credential,
+                tenant_id=tenant_id,
+                subscription_id=subscription_id,
             )
-
-        except HttpResponseError as e:
-            if ', AdalError: Unsupported wstrust endpoint version. ' \
-                    'Current supported version is wstrust2005 or wstrust13.' in e.args:
-                logger.error(
-                    f'You are likely authenticating with a Microsoft Account. \
-                    This authentication mode only supports Azure Active Directory principal authentication.\
-                    {e}',
-                )
-
-            raise e
+        except Exception as e:
+            logger.error(
+                f"Failed to authenticate with Azure CLI. Have you run 'az login'? Details: {e}"
+            )
+            return None
 
     def authenticate_sp(
-            self,
-            tenant_id: Optional[str] = None,
-            client_id: Optional[str] = None,
-            client_secret: Optional[str] = None,
-    ) -> Credentials:
+        self,
+        tenant_id: Optional[str] = None,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        subscription_id: Optional[str] = None,
+    ) -> Optional[Credentials]:
         """
-        Implements authentication for the Azure provider
+        Implements authentication using a Service Principal with the modern library.
         """
         try:
-
-            # Set logging level to error for libraries as otherwise generates a lot of warnings
-            logging.getLogger('adal-python').setLevel(logging.ERROR)
-            logging.getLogger('msrest').setLevel(logging.ERROR)
-            logging.getLogger('msrestazure.azure_active_directory').setLevel(logging.ERROR)
-            logging.getLogger('urllib3').setLevel(logging.ERROR)
-            logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR)
-
-            arm_credentials = ClientSecretCredential(
+            credential = ClientSecretCredential(
                 client_id=client_id,
                 client_secret=client_secret,
                 tenant_id=tenant_id,
             )
-
-            aad_graph_credentials = ClientSecretCredential(
-                client_id=client_id,
-                client_secret=client_secret,
+            return Credentials(
+                credential=credential,
                 tenant_id=tenant_id,
-                resource='https://graph.windows.net',
+                subscription_id=subscription_id,
             )
-
-            return Credentials(
-                arm_credentials, aad_graph_credentials, tenant_id=tenant_id,
-                current_user=client_id,
+        except Exception as e:
+            logger.error(
+                (
+                    "Failed to authenticate with Service Principal. "
+                    "Please ensure the tenant ID, client ID, and client secret are correct. Details: %s"
+                ),
+                e,
             )
-
-        except HttpResponseError as e:
-            if ', AdalError: Unsupported wstrust endpoint version. ' \
-                    'Current supported version is wstrust2005 or wstrust13.' in e.args:
-                logger.error(
-                    f'You are likely authenticating with a Microsoft Account. \
-                    This authentication mode only supports Azure Active Directory principal authentication.\
-                    {e}',
-                )
-
-            raise e
+            return None

cartography/intel/azure/util/tag.py

@@ -0,0 +1,41 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def transform_tags(
+    resource_list: list[dict], subscription_id: str, resource_id_field: str = "id"
+) -> list[dict]:
+    """
+    Transforms tags from a list of Azure resources into a standardized list of tag dictionaries.
+
+    Args:
+        resource_list: List of Azure resources with tags
+        subscription_id: The Azure subscription ID to scope the tags
+        resource_id_field: Field name containing the resource ID (default: "id")
+
+    Returns:
+        List of tag dictionaries with subscription-scoped IDs
+    """
+    tags_list = []
+    for resource in resource_list:
+        resource_id = resource.get(resource_id_field)
+        tags = resource.get("tags")
+
+        if not resource_id or not tags:
+            continue
+
+        for key, value in tags.items():
+            # Generate the deterministic ID scoped to subscription: "{subscription_id}|{key}:{value}"
+            tag_id = f"{subscription_id}|{key}:{value}"
+            tags_list.append(
+                {
+                    "id": tag_id,
+                    "key": key,
+                    "value": value,
+                    "subscription_id": subscription_id,
+                    "resource_id": resource_id,
+                }
+            )
+
+    return tags_list

cartography/intel/bigfix/__init__.py

@@ -20,8 +20,8 @@ def start_bigfix_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
 
     if not config.bigfix_username or not config.bigfix_password:
         logger.info(
-            'BigFix import is not configured - skipping this module. '
-            'See docs to configure.',
+            "BigFix import is not configured - skipping this module. "
+            "See docs to configure.",
         )
         return