cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl

cartography/intel/entra/users.py

@@ -56,51 +56,51 @@ def transform_users(users: list[User]) -> list[dict[str, Any]]:
     result: list[dict[str, Any]] = []
     for user in users:
         transformed_user = {
-            'id': user.id,
-            'user_principal_name': user.user_principal_name,
-            'display_name': user.display_name,
-            'given_name': user.given_name,
-            'surname': user.surname,
-            'mail': user.mail,
-            'other_mails': user.other_mails,
-            'preferred_language': user.preferred_language,
-            'preferred_name': user.preferred_name,
-            'state': user.state,
-            'usage_location': user.usage_location,
-            'user_type': user.user_type,
-            'show_in_address_list': user.show_in_address_list,
-            'sign_in_sessions_valid_from_date_time': user.sign_in_sessions_valid_from_date_time,
-            'security_identifier': user.on_premises_security_identifier,
-            'account_enabled': user.account_enabled,
-            'age_group': user.age_group,
-            'business_phones': user.business_phones,
-            'city': user.city,
-            'company_name': user.company_name,
-            'consent_provided_for_minor': user.consent_provided_for_minor,
-            'country': user.country,
-            'created_date_time': user.created_date_time,
-            'creation_type': user.creation_type,
-            'deleted_date_time': user.deleted_date_time,
-            'department': user.department,
-            'employee_id': user.employee_id,
-            'employee_type': user.employee_type,
-            'external_user_state': user.external_user_state,
-            'external_user_state_change_date_time': user.external_user_state_change_date_time,
-            'hire_date': user.hire_date,
-            'is_management_restricted': user.is_management_restricted,
-            'is_resource_account': user.is_resource_account,
-            'job_title': user.job_title,
-            'last_password_change_date_time': user.last_password_change_date_time,
-            'mail_nickname': user.mail_nickname,
-            'office_location': user.office_location,
-            'on_premises_distinguished_name': user.on_premises_distinguished_name,
-            'on_premises_domain_name': user.on_premises_domain_name,
-            'on_premises_immutable_id': user.on_premises_immutable_id,
-            'on_premises_last_sync_date_time': user.on_premises_last_sync_date_time,
-            'on_premises_sam_account_name': user.on_premises_sam_account_name,
-            'on_premises_security_identifier': user.on_premises_security_identifier,
-            'on_premises_sync_enabled': user.on_premises_sync_enabled,
-            'on_premises_user_principal_name': user.on_premises_user_principal_name,
+            "id": user.id,
+            "user_principal_name": user.user_principal_name,
+            "display_name": user.display_name,
+            "given_name": user.given_name,
+            "surname": user.surname,
+            "mail": user.mail,
+            "other_mails": user.other_mails,
+            "preferred_language": user.preferred_language,
+            "preferred_name": user.preferred_name,
+            "state": user.state,
+            "usage_location": user.usage_location,
+            "user_type": user.user_type,
+            "show_in_address_list": user.show_in_address_list,
+            "sign_in_sessions_valid_from_date_time": user.sign_in_sessions_valid_from_date_time,
+            "security_identifier": user.on_premises_security_identifier,
+            "account_enabled": user.account_enabled,
+            "age_group": user.age_group,
+            "business_phones": user.business_phones,
+            "city": user.city,
+            "company_name": user.company_name,
+            "consent_provided_for_minor": user.consent_provided_for_minor,
+            "country": user.country,
+            "created_date_time": user.created_date_time,
+            "creation_type": user.creation_type,
+            "deleted_date_time": user.deleted_date_time,
+            "department": user.department,
+            "employee_id": user.employee_id,
+            "employee_type": user.employee_type,
+            "external_user_state": user.external_user_state,
+            "external_user_state_change_date_time": user.external_user_state_change_date_time,
+            "hire_date": user.hire_date,
+            "is_management_restricted": user.is_management_restricted,
+            "is_resource_account": user.is_resource_account,
+            "job_title": user.job_title,
+            "last_password_change_date_time": user.last_password_change_date_time,
+            "mail_nickname": user.mail_nickname,
+            "office_location": user.office_location,
+            "on_premises_distinguished_name": user.on_premises_distinguished_name,
+            "on_premises_domain_name": user.on_premises_domain_name,
+            "on_premises_immutable_id": user.on_premises_immutable_id,
+            "on_premises_last_sync_date_time": user.on_premises_last_sync_date_time,
+            "on_premises_sam_account_name": user.on_premises_sam_account_name,
+            "on_premises_security_identifier": user.on_premises_security_identifier,
+            "on_premises_sync_enabled": user.on_premises_sync_enabled,
+            "on_premises_user_principal_name": user.on_premises_user_principal_name,
         }
         result.append(transformed_user)
     return result
@@ -112,21 +112,21 @@ def transform_tenant(tenant: Organization, tenant_id: str) -> dict[str, Any]:
     Transform the tenant data into the format expected by our schema
     """
     return {
-        'id': tenant_id,
-        'created_date_time': tenant.created_date_time,
-        'default_usage_location': tenant.default_usage_location,
-        'deleted_date_time': tenant.deleted_date_time,
-        'display_name': tenant.display_name,
-        'marketing_notification_emails': tenant.marketing_notification_emails,
-        'mobile_device_management_authority': tenant.mobile_device_management_authority,
-        'on_premises_last_sync_date_time': tenant.on_premises_last_sync_date_time,
-        'on_premises_sync_enabled': tenant.on_premises_sync_enabled,
-        'partner_tenant_type': tenant.partner_tenant_type,
-        'postal_code': tenant.postal_code,
-        'preferred_language': tenant.preferred_language,
-        'state': tenant.state,
-        'street': tenant.street,
-        'tenant_type': tenant.tenant_type,
+        "id": tenant_id,
+        "created_date_time": tenant.created_date_time,
+        "default_usage_location": tenant.default_usage_location,
+        "deleted_date_time": tenant.deleted_date_time,
+        "display_name": tenant.display_name,
+        "marketing_notification_emails": tenant.marketing_notification_emails,
+        "mobile_device_management_authority": tenant.mobile_device_management_authority,
+        "on_premises_last_sync_date_time": tenant.on_premises_last_sync_date_time,
+        "on_premises_sync_enabled": tenant.on_premises_sync_enabled,
+        "partner_tenant_type": tenant.partner_tenant_type,
+        "postal_code": tenant.postal_code,
+        "preferred_language": tenant.preferred_language,
+        "state": tenant.state,
+        "street": tenant.street,
+        "tenant_type": tenant.tenant_type,
     }
 
 
@@ -161,8 +161,12 @@ def load_users(
     )
 
 
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
-    GraphJob.from_node_schema(EntraUserSchema(), common_job_parameters).run(neo4j_session)
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(EntraUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )
 
 
 @timeit
@@ -190,7 +194,9 @@ async def sync_entra_users(
         client_id=client_id,
         client_secret=client_secret,
     )
-    client = GraphServiceClient(credential, scopes=['https://graph.microsoft.com/.default'])
+    client = GraphServiceClient(
+        credential, scopes=["https://graph.microsoft.com/.default"]
+    )
 
     # Get tenant information
     tenant = await get_tenant(client)

cartography/intel/gcp/__init__.py

@@ -24,17 +24,19 @@ from cartography.util import run_analysis_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
-Resources = namedtuple('Resources', 'compute container crm_v1 crm_v2 dns storage serviceusage iam')
+Resources = namedtuple(
+    "Resources", "compute container crm_v1 crm_v2 dns storage serviceusage iam"
+)
 
 # Mapping of service short names to their full names as in docs. See https://developers.google.com/apis-explorer,
 # and https://cloud.google.com/service-usage/docs/reference/rest/v1/services#ServiceConfig
-Services = namedtuple('Services', 'compute storage gke dns iam')
+Services = namedtuple("Services", "compute storage gke dns iam")
 service_names = Services(
-    compute='compute.googleapis.com',
-    storage='storage.googleapis.com',
-    gke='container.googleapis.com',
-    dns='dns.googleapis.com',
-    iam='iam.googleapis.com',
+    compute="compute.googleapis.com",
+    storage="storage.googleapis.com",
+    gke="container.googleapis.com",
+    dns="dns.googleapis.com",
+    iam="iam.googleapis.com",
 )
 
 
@@ -47,7 +49,12 @@ def _get_crm_resource_v1(credentials: GoogleCredentials) -> Resource:
     """
     # cache_discovery=False to suppress extra warnings.
     # See https://github.com/googleapis/google-api-python-client/issues/299#issuecomment-268915510 and related issues
-    return googleapiclient.discovery.build('cloudresourcemanager', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "cloudresourcemanager",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_crm_resource_v2(credentials: GoogleCredentials) -> Resource:
@@ -57,7 +64,12 @@ def _get_crm_resource_v2(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A CRM v2 resource object
     """
-    return googleapiclient.discovery.build('cloudresourcemanager', 'v2', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "cloudresourcemanager",
+        "v2",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_compute_resource(credentials: GoogleCredentials) -> Resource:
@@ -67,7 +79,12 @@ def _get_compute_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A Compute resource object
     """
-    return googleapiclient.discovery.build('compute', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "compute",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_storage_resource(credentials: GoogleCredentials) -> Resource:
@@ -79,7 +96,12 @@ def _get_storage_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A Storage resource object
     """
-    return googleapiclient.discovery.build('storage', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "storage",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_container_resource(credentials: GoogleCredentials) -> Resource:
@@ -90,7 +112,12 @@ def _get_container_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A Container resource object
     """
-    return googleapiclient.discovery.build('container', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "container",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_dns_resource(credentials: GoogleCredentials) -> Resource:
@@ -101,7 +128,12 @@ def _get_dns_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A DNS resource object
     """
-    return googleapiclient.discovery.build('dns', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "dns",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
@@ -112,14 +144,21 @@ def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A serviceusage resource object
     """
-    return googleapiclient.discovery.build('serviceusage', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "serviceusage",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_iam_resource(credentials: GoogleCredentials) -> Resource:
     """
     Instantiates a Google IAM resource object to call the IAM API.
     """
-    return googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "iam", "v1", credentials=credentials, cache_discovery=False
+    )
 
 
 def _initialize_resources(credentials: GoogleCredentials) -> Resource:
@@ -150,16 +189,22 @@ def _services_enabled_on_project(serviceusage: Resource, project_id: str) -> Set
     :return: A set of services that are enabled on the project
     """
     try:
-        req = serviceusage.services().list(parent=f'projects/{project_id}', filter='state:ENABLED')
+        req = serviceusage.services().list(
+            parent=f"projects/{project_id}",
+            filter="state:ENABLED",
+        )
         services = set()
         while req is not None:
             res = req.execute()
-            if 'services' in res:
-                services.update({svc['config']['name'] for svc in res['services']})
-            req = serviceusage.services().list_next(previous_request=req, previous_response=res)
+            if "services" in res:
+                services.update({svc["config"]["name"] for svc in res["services"]})
+            req = serviceusage.services().list_next(
+                previous_request=req,
+                previous_response=res,
+            )
         return services
     except googleapiclient.discovery.HttpError as http_error:
-        http_error = json.loads(http_error.content.decode('utf-8'))
+        http_error = json.loads(http_error.content.decode("utf-8"))
         # This is set to log-level `info` because Google creates many projects under the hood that cartography cannot
         # audit (e.g. adding a script to a Google spreadsheet causes a project to get created) and we don't need to emit
         # a warning for these projects.
@@ -172,7 +217,10 @@ def _services_enabled_on_project(serviceusage: Resource, project_id: str) -> Set
 
 
 def _sync_single_project_compute(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -189,11 +237,20 @@ def _sync_single_project_compute(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     compute_cred = _get_compute_resource(get_gcp_credentials())
     if service_names.compute in enabled_services:
-        compute.sync(neo4j_session, compute_cred, project_id, gcp_update_tag, common_job_parameters)
+        compute.sync(
+            neo4j_session,
+            compute_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_storage(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -210,11 +267,20 @@ def _sync_single_project_storage(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     storage_cred = _get_storage_resource(get_gcp_credentials())
     if service_names.storage in enabled_services:
-        storage.sync_gcp_buckets(neo4j_session, storage_cred, project_id, gcp_update_tag, common_job_parameters)
+        storage.sync_gcp_buckets(
+            neo4j_session,
+            storage_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_gke(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -231,11 +297,20 @@ def _sync_single_project_gke(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     container_cred = _get_container_resource(get_gcp_credentials())
     if service_names.gke in enabled_services:
-        gke.sync_gke_clusters(neo4j_session, container_cred, project_id, gcp_update_tag, common_job_parameters)
+        gke.sync_gke_clusters(
+            neo4j_session,
+            container_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_dns(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -252,7 +327,13 @@ def _sync_single_project_dns(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     dns_cred = _get_dns_resource(get_gcp_credentials())
     if service_names.dns in enabled_services:
-        dns.sync(neo4j_session, dns_cred, project_id, gcp_update_tag, common_job_parameters)
+        dns.sync(
+            neo4j_session,
+            dns_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_iam(
@@ -276,12 +357,17 @@ def _sync_single_project_iam(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     iam_cred = _get_iam_resource(get_gcp_credentials())
     if service_names.iam in enabled_services:
-        iam.sync(neo4j_session, iam_cred, project_id, gcp_update_tag, common_job_parameters)
+        iam.sync(
+            neo4j_session, iam_cred, project_id, gcp_update_tag, common_job_parameters
+        )
 
 
 def _sync_multiple_projects(
-    neo4j_session: neo4j.Session, resources: Resource, projects: List[Dict],
-    gcp_update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    projects: List[Dict],
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     """
     Handles graph sync for multiple GCP projects.
@@ -296,36 +382,67 @@ def _sync_multiple_projects(
     :return: Nothing
     """
     logger.info("Syncing %d GCP projects.", len(projects))
-    crm.sync_gcp_projects(neo4j_session, projects, gcp_update_tag, common_job_parameters)
+    crm.sync_gcp_projects(
+        neo4j_session,
+        projects,
+        gcp_update_tag,
+        common_job_parameters,
+    )
     # Compute data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for Compute.", project_id)
-        _sync_single_project_compute(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_compute(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # Storage data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for Storage", project_id)
-        _sync_single_project_storage(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_storage(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # GKE data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for GKE", project_id)
-        _sync_single_project_gke(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_gke(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # DNS data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for DNS", project_id)
-        _sync_single_project_dns(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_dns(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # IAM data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for IAM", project_id)
-        _sync_single_project_iam(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_iam(
+            neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters
+        )
 
 
 @timeit
@@ -341,7 +458,10 @@ def get_gcp_credentials() -> Optional[GoogleCredentials]:
         credentials, project_id = default()
         return credentials
     except DefaultCredentialsError as e:
-        logger.debug("Error occurred calling GoogleCredentials.get_application_default().", exc_info=True)
+        logger.debug(
+            "Error occurred calling GoogleCredentials.get_application_default().",
+            exc_info=True,
+        )
         logger.error(
             (
                 "Unable to initialize Google Compute Platform creds. If you don't have GCP data or don't want to load "
@@ -376,27 +496,43 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     resources = _initialize_resources(credentials)
 
     # If we don't have perms to pull Orgs or Folders from GCP, we will skip safely
-    crm.sync_gcp_organizations(neo4j_session, resources.crm_v1, config.update_tag, common_job_parameters)
-    crm.sync_gcp_folders(neo4j_session, resources.crm_v2, config.update_tag, common_job_parameters)
+    crm.sync_gcp_organizations(
+        neo4j_session,
+        resources.crm_v1,
+        config.update_tag,
+        common_job_parameters,
+    )
+    crm.sync_gcp_folders(
+        neo4j_session,
+        resources.crm_v2,
+        config.update_tag,
+        common_job_parameters,
+    )
 
     projects = crm.get_gcp_projects(resources.crm_v1)
 
-    _sync_multiple_projects(neo4j_session, resources, projects, config.update_tag, common_job_parameters)
+    _sync_multiple_projects(
+        neo4j_session,
+        resources,
+        projects,
+        config.update_tag,
+        common_job_parameters,
+    )
 
     run_analysis_job(
-        'gcp_compute_asset_inet_exposure.json',
+        "gcp_compute_asset_inet_exposure.json",
         neo4j_session,
         common_job_parameters,
     )
 
     run_analysis_job(
-        'gcp_gke_asset_exposure.json',
+        "gcp_gke_asset_exposure.json",
         neo4j_session,
         common_job_parameters,
     )
 
     run_analysis_job(
-        'gcp_gke_basic_auth.json',
+        "gcp_gke_basic_auth.json",
         neo4j_session,
         common_job_parameters,
     )