cartography 0.116.1__py3-none-any.whl → 0.118.0__py3-none-any.whl

cartography/intel/azure/cosmosdb.py

@@ -12,6 +12,7 @@ from azure.core.exceptions import HttpResponseError
 from azure.core.exceptions import ResourceNotFoundError
 from azure.mgmt.cosmosdb import CosmosDBManagementClient
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -132,7 +133,8 @@ def load_database_account_data(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_database_account,
         database_accounts_list=database_account_list,
         AZURE_SUBSCRIPTION_ID=subscription_id,
@@ -218,7 +220,8 @@ def _load_database_account_write_locations(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_write_location,
             write_locations_list=write_locations,
             DatabaseAccountId=database_account_id,
@@ -259,7 +262,8 @@ def _load_database_account_read_locations(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_read_location,
             read_locations_list=read_locations,
             DatabaseAccountId=database_account_id,
@@ -297,7 +301,8 @@ def _load_database_account_associated_locations(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_associated_location,
             associated_locations_list=associated_locations,
             DatabaseAccountId=database_account_id,
@@ -348,7 +353,8 @@ def _load_cosmosdb_cors_policy(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_cors_policy,
             cors_policies_list=cors_policies,
             DatabaseAccountId=database_account_id,
@@ -386,7 +392,8 @@ def _load_cosmosdb_failover_policies(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_failover_policies,
             failover_policies_list=failover_policies,
             DatabaseAccountId=database_account_id,
@@ -429,7 +436,8 @@ def _load_cosmosdb_private_endpoint_connections(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_private_endpoint_connections,
             private_endpoint_connections_list=private_endpoint_connections,
             DatabaseAccountId=database_account_id,
@@ -466,7 +474,8 @@ def _load_cosmosdb_virtual_network_rules(
         SET r.lastupdated = $azure_update_tag
         """
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             ingest_virtual_network_rules,
             virtual_network_rules_list=virtual_network_rules,
             DatabaseAccountId=database_account_id,
@@ -822,7 +831,8 @@ def _load_sql_databases(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_sql_databases,
         sql_databases_list=sql_databases,
         azure_update_tag=update_tag,
@@ -854,7 +864,8 @@ def _load_cassandra_keyspaces(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_cassandra_keyspaces,
         cassandra_keyspaces_list=cassandra_keyspaces,
         azure_update_tag=update_tag,
@@ -886,7 +897,8 @@ def _load_mongodb_databases(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_mongodb_databases,
         mongodb_databases_list=mongodb_databases,
         azure_update_tag=update_tag,
@@ -918,7 +930,8 @@ def _load_table_resources(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_tables,
         table_resources_list=table_resources,
         azure_update_tag=update_tag,
@@ -1046,7 +1059,8 @@ def _load_sql_containers(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_containers,
         sql_containers_list=containers,
         azure_update_tag=update_tag,
@@ -1175,7 +1189,8 @@ def _load_cassandra_tables(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_cassandra_tables,
         cassandra_tables_list=cassandra_tables,
         azure_update_tag=update_tag,
@@ -1299,7 +1314,8 @@ def _load_collections(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_collections,
         mongodb_collections_list=collections,
         azure_update_tag=update_tag,

cartography/intel/azure/data_lake.py

@@ -0,0 +1,124 @@
+import logging
+from typing import Any
+
+import neo4j
+from azure.core.exceptions import ClientAuthenticationError
+from azure.core.exceptions import HttpResponseError
+from azure.mgmt.storage import StorageManagementClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.data_lake_filesystem import AzureDataLakeFileSystemSchema
+from cartography.util import timeit
+
+from .util.credentials import Credentials
+
+logger = logging.getLogger(__name__)
+
+
+def _get_resource_group_from_id(resource_id: str) -> str:
+    """
+    Helper function to parse the resource group name from a full resource ID string.
+    """
+    parts = resource_id.lower().split("/")
+    rg_index = parts.index("resourcegroups")
+    return parts[rg_index + 1]
+
+
+@timeit
+def get_datalake_accounts(credentials: Credentials, subscription_id: str) -> list[dict]:
+    try:
+        client = StorageManagementClient(credentials.credential, subscription_id)
+        storage_accounts = [sa.as_dict() for sa in client.storage_accounts.list()]
+        return [sa for sa in storage_accounts if sa.get("is_hns_enabled")]
+    except (ClientAuthenticationError, HttpResponseError) as e:
+        logger.warning(f"Failed to get Storage Accounts for Data Lake sync: {str(e)}")
+        return []
+
+
+@timeit
+def get_filesystems_for_account(
+    client: StorageManagementClient,
+    account: dict,
+) -> list[dict]:
+    resource_group_name = _get_resource_group_from_id(account["id"])
+    try:
+        return [
+            c.as_dict()
+            for c in client.blob_containers.list(
+                resource_group_name,
+                account["name"],
+            )
+        ]
+    except (ClientAuthenticationError, HttpResponseError) as e:
+        logger.warning(
+            f"Failed to get containers for storage account {account['name']}: {str(e)}",
+        )
+        return []
+
+
+@timeit
+def transform_datalake_filesystems(filesystems_response: list[dict]) -> list[dict]:
+    transformed_filesystems: list[dict[str, Any]] = []
+    for fs in filesystems_response:
+        transformed_filesystem = {
+            "id": fs.get("id"),
+            "name": fs.get("name"),
+            "public_access": fs.get("properties", {}).get("public_access"),
+            "last_modified_time": fs.get("properties", {}).get("last_modified_time"),
+            "has_immutability_policy": fs.get("properties", {}).get(
+                "has_immutability_policy",
+            ),
+            "has_legal_hold": fs.get("properties", {}).get("has_legal_hold"),
+        }
+        transformed_filesystems.append(transformed_filesystem)
+    return transformed_filesystems
+
+
+@timeit
+def load_datalake_filesystems(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    storage_account_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        AzureDataLakeFileSystemSchema(),
+        data,
+        lastupdated=update_tag,
+        STORAGE_ACCOUNT_ID=storage_account_id,
+    )
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    credentials: Credentials,
+    subscription_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    logger.info(
+        f"Syncing Azure Data Lake File Systems for subscription {subscription_id}.",
+    )
+    client = StorageManagementClient(credentials.credential, subscription_id)
+
+    datalake_accounts = get_datalake_accounts(credentials, subscription_id)
+    for account in datalake_accounts:
+        account_id = account["id"]
+        raw_filesystems = get_filesystems_for_account(client, account)
+        transformed_filesystems = transform_datalake_filesystems(raw_filesystems)
+
+        load_datalake_filesystems(
+            neo4j_session,
+            transformed_filesystems,
+            account_id,
+            update_tag,
+        )
+
+        cleanup_params = common_job_parameters.copy()
+        cleanup_params["STORAGE_ACCOUNT_ID"] = account_id
+        GraphJob.from_node_schema(AzureDataLakeFileSystemSchema(), cleanup_params).run(
+            neo4j_session,
+        )

cartography/intel/azure/sql.py

@@ -14,6 +14,7 @@ from azure.mgmt.sql.models import SecurityAlertPolicyName
 from azure.mgmt.sql.models import TransparentDataEncryptionName
 from msrestazure.azure_exceptions import CloudError
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -85,7 +86,8 @@ def load_server_data(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_server,
         server_list=server_list,
         AZURE_SUBSCRIPTION_ID=subscription_id,
@@ -504,7 +506,8 @@ def _load_server_dns_aliases(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_dns_aliases,
         dns_aliases_list=dns_aliases,
         azure_update_tag=update_tag,
@@ -535,7 +538,8 @@ def _load_server_ad_admins(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_ad_admins,
         ad_admins_list=ad_admins,
         azure_update_tag=update_tag,
@@ -567,7 +571,8 @@ def _load_recoverable_databases(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_recoverable_databases,
         recoverable_databases_list=recoverable_databases,
         azure_update_tag=update_tag,
@@ -603,7 +608,8 @@ def _load_restorable_dropped_databases(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_restorable_dropped_databases,
         restorable_dropped_databases_list=restorable_dropped_databases,
         azure_update_tag=update_tag,
@@ -634,7 +640,8 @@ def _load_failover_groups(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_failover_groups,
         failover_groups_list=failover_groups,
         azure_update_tag=update_tag,
@@ -669,7 +676,8 @@ def _load_elastic_pools(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_elastic_pools,
         elastic_pools_list=elastic_pools,
         azure_update_tag=update_tag,
@@ -710,7 +718,8 @@ def _load_databases(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_databases,
         databases_list=databases,
         azure_update_tag=update_tag,
@@ -982,7 +991,8 @@ def _load_replication_links(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_replication_links,
         replication_links_list=replication_links,
         azure_update_tag=update_tag,
@@ -1021,7 +1031,8 @@ def _load_db_threat_detection_policies(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_threat_detection_policies,
         threat_detection_policies_list=threat_detection_policies,
         azure_update_tag=update_tag,
@@ -1054,7 +1065,8 @@ def _load_restore_points(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_restore_points,
         restore_points_list=restore_points,
         azure_update_tag=update_tag,
@@ -1085,7 +1097,8 @@ def _load_transparent_data_encryptions(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_data_encryptions,
         transparent_data_encryptions_list=encryptions_list,
         azure_update_tag=update_tag,

cartography/intel/azure/storage.py

@@ -11,6 +11,7 @@ from azure.core.exceptions import HttpResponseError
 from azure.core.exceptions import ResourceNotFoundError
 from azure.mgmt.storage import StorageManagementClient
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -99,7 +100,8 @@ def load_storage_account_data(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_storage_account,
         storage_accounts_list=storage_account_list,
         AZURE_SUBSCRIPTION_ID=subscription_id,
@@ -395,7 +397,8 @@ def _load_queue_services(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_queue_services,
         queue_services_list=queue_services,
         azure_update_tag=update_tag,
@@ -424,7 +427,8 @@ def _load_table_services(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_table_services,
         table_services_list=table_services,
         azure_update_tag=update_tag,
@@ -453,7 +457,8 @@ def _load_file_services(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_file_services,
         file_services_list=file_services,
         azure_update_tag=update_tag,
@@ -482,7 +487,8 @@ def _load_blob_services(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_blob_services,
         blob_services_list=blob_services,
         azure_update_tag=update_tag,
@@ -595,7 +601,8 @@ def _load_queues(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_queues,
         queues_list=queues,
         azure_update_tag=update_tag,
@@ -709,7 +716,8 @@ def _load_tables(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_tables,
         tables_list=tables,
         azure_update_tag=update_tag,
@@ -833,7 +841,8 @@ def _load_shares(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_shares,
         shares_list=shares,
         azure_update_tag=update_tag,
@@ -964,7 +973,8 @@ def _load_blob_containers(
     SET r.lastupdated = $azure_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingest_blob_containers,
         blob_containers_list=blob_containers,
         azure_update_tag=update_tag,

cartography/intel/azure/subscription.py

@@ -7,6 +7,7 @@ import neo4j
 from azure.core.exceptions import HttpResponseError
 from azure.mgmt.resource import SubscriptionClient
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
@@ -96,7 +97,8 @@ def load_azure_subscriptions(
     SET r.lastupdated = $update_tag;
     """
     for sub in subscriptions:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             TENANT_ID=tenant_id,
             SUBSCRIPTION_ID=sub["subscriptionId"],

cartography/intel/crowdstrike/spotlight.py

@@ -6,6 +6,7 @@ import neo4j
 from falconpy.oauth2 import OAuth2
 from falconpy.spotlight_vulnerabilities import Spotlight_Vulnerabilities
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -79,7 +80,8 @@ def load_vulnerability_data(
             cves.append(cve)
         vuln["host_info_local_ip"] = item.get("host_info", {}).get("local_ip")
         vulns.append(vuln)
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Vulnerabilities=vulns,
         update_tag=update_tag,
@@ -106,7 +108,8 @@ def _load_cves(neo4j_session: neo4j.Session, data: List[Dict], update_tag: int)
         ON CREATE SET hc.firstseen = timestamp()
         SET hc.lastupdated = $update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         cves=data,
         update_tag=update_tag,

cartography/intel/entra/app_role_assignments.py

@@ -119,10 +119,17 @@ async def get_app_role_assignments_for_app(
         # Clear previous page before fetching next
         assignments_page.value = None
 
-        # Fetch next page
+        # Fetch next page using the SAME request builder to preserve response typing
+        # Using the root service_principals builder here can return ServicePrincipal objects,
+        # which lack AppRoleAssignment fields like principal_id. Stay on the
+        # app_role_assigned_to builder to ensure AppRoleAssignmentCollectionResponse typing.
         logger.debug(f"Fetching page {page_count + 1} of assignments for {app_id}")
         next_page_url = assignments_page.odata_next_link
-        assignments_page = await client.service_principals.with_url(next_page_url).get()
+        assignments_page = await (
+            client.service_principals.by_service_principal_id(service_principal_id)
+            .app_role_assigned_to.with_url(next_page_url)
+            .get()
+        )
 
     logger.info(
         f"Successfully retrieved {assignment_count} assignments for application {app_id} (pages: {page_count})"

cartography/intel/gcp/__init__.py

@@ -3,9 +3,11 @@ import logging
 from collections import namedtuple
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Set
 
 import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
 from googleapiclient.discovery import HttpError
 from googleapiclient.discovery import Resource
 
@@ -82,6 +84,7 @@ def _sync_project_resources(
     projects: List[Dict],
     gcp_update_tag: int,
     common_job_parameters: Dict,
+    credentials: Optional[GoogleCredentials] = None,
 ) -> None:
     """
     Syncs GCP service-specific resources (Compute, Storage, GKE, DNS, IAM) for each project.
@@ -97,12 +100,13 @@ def _sync_project_resources(
         project_id = project["projectId"]
         common_job_parameters["PROJECT_ID"] = project_id
         enabled_services = _services_enabled_on_project(
-            build_client("serviceusage", "v1"), project_id
+            build_client("serviceusage", "v1", credentials=credentials),
+            project_id,
         )
 
         if service_names.compute in enabled_services:
             logger.info("Syncing GCP project %s for Compute.", project_id)
-            compute_cred = build_client("compute", "v1")
+            compute_cred = build_client("compute", "v1", credentials=credentials)
             compute.sync(
                 neo4j_session,
                 compute_cred,
@@ -113,7 +117,7 @@ def _sync_project_resources(
 
         if service_names.storage in enabled_services:
             logger.info("Syncing GCP project %s for Storage.", project_id)
-            storage_cred = build_client("storage", "v1")
+            storage_cred = build_client("storage", "v1", credentials=credentials)
             storage.sync_gcp_buckets(
                 neo4j_session,
                 storage_cred,
@@ -124,7 +128,7 @@ def _sync_project_resources(
 
         if service_names.gke in enabled_services:
             logger.info("Syncing GCP project %s for GKE.", project_id)
-            container_cred = build_client("container", "v1")
+            container_cred = build_client("container", "v1", credentials=credentials)
             gke.sync_gke_clusters(
                 neo4j_session,
                 container_cred,
@@ -135,7 +139,7 @@ def _sync_project_resources(
 
         if service_names.dns in enabled_services:
             logger.info("Syncing GCP project %s for DNS.", project_id)
-            dns_cred = build_client("dns", "v1")
+            dns_cred = build_client("dns", "v1", credentials=credentials)
             dns.sync(
                 neo4j_session,
                 dns_cred,
@@ -146,7 +150,7 @@ def _sync_project_resources(
 
         if service_names.iam in enabled_services:
             logger.info("Syncing GCP project %s for IAM.", project_id)
-            iam_cred = build_client("iam", "v1")
+            iam_cred = build_client("iam", "v1", credentials=credentials)
             iam.sync(
                 neo4j_session,
                 iam_cred,
@@ -159,7 +163,11 @@ def _sync_project_resources(
 
 
 @timeit
-def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+def start_gcp_ingestion(
+    neo4j_session: neo4j.Session,
+    config: Config,
+    credentials: Optional[GoogleCredentials] = None,
+) -> None:
     """
     Starts the GCP ingestion process by initializing Google Application Default Credentials, creating the necessary
     resource objects, listing all GCP organizations and projects available to the GCP identity, and supplying that
@@ -188,7 +196,10 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     # This ensures children are cleaned up before their parents.
 
     orgs = sync_gcp_organizations(
-        neo4j_session, config.update_tag, common_job_parameters
+        neo4j_session,
+        config.update_tag,
+        common_job_parameters,
+        credentials=credentials,
     )
 
     # Track org cleanup jobs to run at the very end
@@ -210,6 +221,7 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
             config.update_tag,
             common_job_parameters,
             org_resource_name,
+            credentials=credentials,
         )
 
         # Sync projects under org and each folder
@@ -219,11 +231,16 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
             folders,
             config.update_tag,
             common_job_parameters,
+            credentials=credentials,
         )
 
         # Ingest per-project resources (these run their own cleanup immediately since they're leaf nodes)
         _sync_project_resources(
-            neo4j_session, projects, config.update_tag, common_job_parameters
+            neo4j_session,
+            projects,
+            config.update_tag,
+            common_job_parameters,
+            credentials=credentials,
         )
 
         # Clean up projects and folders for this org (children before parents)