cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/intel/gcp/__init__.py

@@ -24,17 +24,19 @@ from cartography.util import run_analysis_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
-Resources = namedtuple('Resources', 'compute container crm_v1 crm_v2 dns storage serviceusage iam')
+Resources = namedtuple(
+    "Resources", "compute container crm_v1 crm_v2 dns storage serviceusage iam"
+)
 
 # Mapping of service short names to their full names as in docs. See https://developers.google.com/apis-explorer,
 # and https://cloud.google.com/service-usage/docs/reference/rest/v1/services#ServiceConfig
-Services = namedtuple('Services', 'compute storage gke dns iam')
+Services = namedtuple("Services", "compute storage gke dns iam")
 service_names = Services(
-    compute='compute.googleapis.com',
-    storage='storage.googleapis.com',
-    gke='container.googleapis.com',
-    dns='dns.googleapis.com',
-    iam='iam.googleapis.com',
+    compute="compute.googleapis.com",
+    storage="storage.googleapis.com",
+    gke="container.googleapis.com",
+    dns="dns.googleapis.com",
+    iam="iam.googleapis.com",
 )
 
 
@@ -47,7 +49,12 @@ def _get_crm_resource_v1(credentials: GoogleCredentials) -> Resource:
     """
     # cache_discovery=False to suppress extra warnings.
     # See https://github.com/googleapis/google-api-python-client/issues/299#issuecomment-268915510 and related issues
-    return googleapiclient.discovery.build('cloudresourcemanager', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "cloudresourcemanager",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_crm_resource_v2(credentials: GoogleCredentials) -> Resource:
@@ -57,7 +64,12 @@ def _get_crm_resource_v2(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A CRM v2 resource object
     """
-    return googleapiclient.discovery.build('cloudresourcemanager', 'v2', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "cloudresourcemanager",
+        "v2",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_compute_resource(credentials: GoogleCredentials) -> Resource:
@@ -67,7 +79,12 @@ def _get_compute_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A Compute resource object
     """
-    return googleapiclient.discovery.build('compute', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "compute",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_storage_resource(credentials: GoogleCredentials) -> Resource:
@@ -79,7 +96,12 @@ def _get_storage_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A Storage resource object
     """
-    return googleapiclient.discovery.build('storage', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "storage",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_container_resource(credentials: GoogleCredentials) -> Resource:
@@ -90,7 +112,12 @@ def _get_container_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A Container resource object
     """
-    return googleapiclient.discovery.build('container', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "container",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_dns_resource(credentials: GoogleCredentials) -> Resource:
@@ -101,7 +128,12 @@ def _get_dns_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A DNS resource object
     """
-    return googleapiclient.discovery.build('dns', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "dns",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
@@ -112,14 +144,21 @@ def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
     :param credentials: The GoogleCredentials object
     :return: A serviceusage resource object
     """
-    return googleapiclient.discovery.build('serviceusage', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "serviceusage",
+        "v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
 def _get_iam_resource(credentials: GoogleCredentials) -> Resource:
     """
     Instantiates a Google IAM resource object to call the IAM API.
     """
-    return googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "iam", "v1", credentials=credentials, cache_discovery=False
+    )
 
 
 def _initialize_resources(credentials: GoogleCredentials) -> Resource:
@@ -150,16 +189,22 @@ def _services_enabled_on_project(serviceusage: Resource, project_id: str) -> Set
     :return: A set of services that are enabled on the project
     """
     try:
-        req = serviceusage.services().list(parent=f'projects/{project_id}', filter='state:ENABLED')
+        req = serviceusage.services().list(
+            parent=f"projects/{project_id}",
+            filter="state:ENABLED",
+        )
         services = set()
         while req is not None:
             res = req.execute()
-            if 'services' in res:
-                services.update({svc['config']['name'] for svc in res['services']})
-            req = serviceusage.services().list_next(previous_request=req, previous_response=res)
+            if "services" in res:
+                services.update({svc["config"]["name"] for svc in res["services"]})
+            req = serviceusage.services().list_next(
+                previous_request=req,
+                previous_response=res,
+            )
         return services
     except googleapiclient.discovery.HttpError as http_error:
-        http_error = json.loads(http_error.content.decode('utf-8'))
+        http_error = json.loads(http_error.content.decode("utf-8"))
         # This is set to log-level `info` because Google creates many projects under the hood that cartography cannot
         # audit (e.g. adding a script to a Google spreadsheet causes a project to get created) and we don't need to emit
         # a warning for these projects.
@@ -172,7 +217,10 @@ def _services_enabled_on_project(serviceusage: Resource, project_id: str) -> Set
 
 
 def _sync_single_project_compute(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -189,11 +237,20 @@ def _sync_single_project_compute(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     compute_cred = _get_compute_resource(get_gcp_credentials())
     if service_names.compute in enabled_services:
-        compute.sync(neo4j_session, compute_cred, project_id, gcp_update_tag, common_job_parameters)
+        compute.sync(
+            neo4j_session,
+            compute_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_storage(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -210,11 +267,20 @@ def _sync_single_project_storage(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     storage_cred = _get_storage_resource(get_gcp_credentials())
     if service_names.storage in enabled_services:
-        storage.sync_gcp_buckets(neo4j_session, storage_cred, project_id, gcp_update_tag, common_job_parameters)
+        storage.sync_gcp_buckets(
+            neo4j_session,
+            storage_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_gke(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -231,11 +297,20 @@ def _sync_single_project_gke(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     container_cred = _get_container_resource(get_gcp_credentials())
     if service_names.gke in enabled_services:
-        gke.sync_gke_clusters(neo4j_session, container_cred, project_id, gcp_update_tag, common_job_parameters)
+        gke.sync_gke_clusters(
+            neo4j_session,
+            container_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_dns(
-    neo4j_session: neo4j.Session, resources: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -252,7 +327,13 @@ def _sync_single_project_dns(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     dns_cred = _get_dns_resource(get_gcp_credentials())
     if service_names.dns in enabled_services:
-        dns.sync(neo4j_session, dns_cred, project_id, gcp_update_tag, common_job_parameters)
+        dns.sync(
+            neo4j_session,
+            dns_cred,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
 
 def _sync_single_project_iam(
@@ -276,12 +357,17 @@ def _sync_single_project_iam(
     enabled_services = _services_enabled_on_project(resources.serviceusage, project_id)
     iam_cred = _get_iam_resource(get_gcp_credentials())
     if service_names.iam in enabled_services:
-        iam.sync(neo4j_session, iam_cred, project_id, gcp_update_tag, common_job_parameters)
+        iam.sync(
+            neo4j_session, iam_cred, project_id, gcp_update_tag, common_job_parameters
+        )
 
 
 def _sync_multiple_projects(
-    neo4j_session: neo4j.Session, resources: Resource, projects: List[Dict],
-    gcp_update_tag: int, common_job_parameters: Dict,
+    neo4j_session: neo4j.Session,
+    resources: Resource,
+    projects: List[Dict],
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
 ) -> None:
     """
     Handles graph sync for multiple GCP projects.
@@ -296,36 +382,67 @@ def _sync_multiple_projects(
     :return: Nothing
     """
     logger.info("Syncing %d GCP projects.", len(projects))
-    crm.sync_gcp_projects(neo4j_session, projects, gcp_update_tag, common_job_parameters)
+    crm.sync_gcp_projects(
+        neo4j_session,
+        projects,
+        gcp_update_tag,
+        common_job_parameters,
+    )
     # Compute data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for Compute.", project_id)
-        _sync_single_project_compute(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_compute(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # Storage data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for Storage", project_id)
-        _sync_single_project_storage(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_storage(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # GKE data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for GKE", project_id)
-        _sync_single_project_gke(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_gke(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # DNS data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for DNS", project_id)
-        _sync_single_project_dns(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_dns(
+            neo4j_session,
+            resources,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
 
     # IAM data sync
     for project in projects:
-        project_id = project['projectId']
+        project_id = project["projectId"]
         logger.info("Syncing GCP project %s for IAM", project_id)
-        _sync_single_project_iam(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
+        _sync_single_project_iam(
+            neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters
+        )
 
 
 @timeit
@@ -341,7 +458,10 @@ def get_gcp_credentials() -> Optional[GoogleCredentials]:
         credentials, project_id = default()
         return credentials
     except DefaultCredentialsError as e:
-        logger.debug("Error occurred calling GoogleCredentials.get_application_default().", exc_info=True)
+        logger.debug(
+            "Error occurred calling GoogleCredentials.get_application_default().",
+            exc_info=True,
+        )
         logger.error(
             (
                 "Unable to initialize Google Compute Platform creds. If you don't have GCP data or don't want to load "
@@ -376,27 +496,43 @@ def start_gcp_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     resources = _initialize_resources(credentials)
 
     # If we don't have perms to pull Orgs or Folders from GCP, we will skip safely
-    crm.sync_gcp_organizations(neo4j_session, resources.crm_v1, config.update_tag, common_job_parameters)
-    crm.sync_gcp_folders(neo4j_session, resources.crm_v2, config.update_tag, common_job_parameters)
+    crm.sync_gcp_organizations(
+        neo4j_session,
+        resources.crm_v1,
+        config.update_tag,
+        common_job_parameters,
+    )
+    crm.sync_gcp_folders(
+        neo4j_session,
+        resources.crm_v2,
+        config.update_tag,
+        common_job_parameters,
+    )
 
     projects = crm.get_gcp_projects(resources.crm_v1)
 
-    _sync_multiple_projects(neo4j_session, resources, projects, config.update_tag, common_job_parameters)
+    _sync_multiple_projects(
+        neo4j_session,
+        resources,
+        projects,
+        config.update_tag,
+        common_job_parameters,
+    )
 
     run_analysis_job(
-        'gcp_compute_asset_inet_exposure.json',
+        "gcp_compute_asset_inet_exposure.json",
         neo4j_session,
         common_job_parameters,
     )
 
     run_analysis_job(
-        'gcp_gke_asset_exposure.json',
+        "gcp_gke_asset_exposure.json",
         neo4j_session,
         common_job_parameters,
     )
 
     run_analysis_job(
-        'gcp_gke_basic_auth.json',
+        "gcp_gke_basic_auth.json",
         neo4j_session,
         common_job_parameters,
     )