cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/intel/gcp/iam.py

@@ -20,7 +20,9 @@ DESCRIBE_SLEEP = 1
 
 
 @timeit
-def get_gcp_service_accounts(iam_client: Resource, project_id: str) -> List[Dict[str, Any]]:
+def get_gcp_service_accounts(
+    iam_client: Resource, project_id: str
+) -> List[Dict[str, Any]]:
     """
     Retrieve a list of GCP service accounts for a given project.
 
@@ -30,19 +32,29 @@ def get_gcp_service_accounts(iam_client: Resource, project_id: str) -> List[Dict
     """
     service_accounts: List[Dict[str, Any]] = []
     try:
-        request = iam_client.projects().serviceAccounts().list(
-            name=f'projects/{project_id}',
+        request = (
+            iam_client.projects()
+            .serviceAccounts()
+            .list(
+                name=f"projects/{project_id}",
+            )
         )
         while request is not None:
             response = request.execute()
-            if 'accounts' in response:
-                service_accounts.extend(response['accounts'])
-            request = iam_client.projects().serviceAccounts().list_next(
-                previous_request=request,
-                previous_response=response,
+            if "accounts" in response:
+                service_accounts.extend(response["accounts"])
+            request = (
+                iam_client.projects()
+                .serviceAccounts()
+                .list_next(
+                    previous_request=request,
+                    previous_response=response,
+                )
             )
     except Exception as e:
-        logger.warning(f"Error retrieving service accounts for project {project_id}: {e}")
+        logger.warning(
+            f"Error retrieving service accounts for project {project_id}: {e}"
+        )
     return service_accounts
 
 
@@ -59,17 +71,17 @@ def get_gcp_roles(iam_client: Resource, project_id: str) -> List[Dict]:
         roles = []
 
         # Get custom roles
-        custom_req = iam_client.projects().roles().list(parent=f'projects/{project_id}')
+        custom_req = iam_client.projects().roles().list(parent=f"projects/{project_id}")
         while custom_req is not None:
             resp = custom_req.execute()
-            roles.extend(resp.get('roles', []))
+            roles.extend(resp.get("roles", []))
             custom_req = iam_client.projects().roles().list_next(custom_req, resp)
 
         # Get predefined roles
-        predefined_req = iam_client.roles().list(view='FULL')
+        predefined_req = iam_client.roles().list(view="FULL")
         while predefined_req is not None:
             resp = predefined_req.execute()
-            roles.extend(resp.get('roles', []))
+            roles.extend(resp.get("roles", []))
             predefined_req = iam_client.roles().list_next(predefined_req, resp)
 
         return roles
@@ -93,17 +105,19 @@ def load_gcp_service_accounts(
     :param project_id: The GCP Project ID associated with the service accounts.
     :param gcp_update_tag: The timestamp of the current sync run.
     """
-    logger.debug(f"Loading {len(service_accounts)} service accounts for project {project_id}")
+    logger.debug(
+        f"Loading {len(service_accounts)} service accounts for project {project_id}"
+    )
     transformed_service_accounts = []
     for sa in service_accounts:
         transformed_sa = {
-            'id': sa['uniqueId'],
-            'email': sa.get('email'),
-            'displayName': sa.get('displayName'),
-            'oauth2ClientId': sa.get('oauth2ClientId'),
-            'uniqueId': sa.get('uniqueId'),
-            'disabled': sa.get('disabled', False),
-            'projectId': project_id,
+            "id": sa["uniqueId"],
+            "email": sa.get("email"),
+            "displayName": sa.get("displayName"),
+            "oauth2ClientId": sa.get("oauth2ClientId"),
+            "uniqueId": sa.get("uniqueId"),
+            "disabled": sa.get("disabled", False),
+            "projectId": project_id,
         }
         transformed_service_accounts.append(transformed_sa)
 
@@ -113,7 +127,7 @@ def load_gcp_service_accounts(
         transformed_service_accounts,
         lastupdated=gcp_update_tag,
         projectId=project_id,
-        additional_labels=['GCPPrincipal'],
+        additional_labels=["GCPPrincipal"],
     )
 
 
@@ -135,25 +149,25 @@ def load_gcp_roles(
     logger.debug(f"Loading {len(roles)} roles for project {project_id}")
     transformed_roles = []
     for role in roles:
-        role_name = role['name']
-        if role_name.startswith('roles/'):
-            if role_name in ['roles/owner', 'roles/editor', 'roles/viewer']:
-                role_type = 'BASIC'
+        role_name = role["name"]
+        if role_name.startswith("roles/"):
+            if role_name in ["roles/owner", "roles/editor", "roles/viewer"]:
+                role_type = "BASIC"
             else:
-                role_type = 'PREDEFINED'
+                role_type = "PREDEFINED"
         else:
-            role_type = 'CUSTOM'
+            role_type = "CUSTOM"
 
         transformed_role = {
-            'id': role_name,
-            'name': role_name,
-            'title': role.get('title'),
-            'description': role.get('description'),
-            'deleted': role.get('deleted', False),
-            'etag': role.get('etag'),
-            'includedPermissions': role.get('includedPermissions', []),
-            'roleType': role_type,
-            'projectId': project_id,
+            "id": role_name,
+            "name": role_name,
+            "title": role.get("title"),
+            "description": role.get("description"),
+            "deleted": role.get("deleted", False),
+            "etag": role.get("etag"),
+            "includedPermissions": role.get("includedPermissions", []),
+            "roleType": role_type,
+            "projectId": project_id,
         }
         transformed_roles.append(transformed_role)
 
@@ -167,7 +181,9 @@ def load_gcp_roles(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
     """
     Run cleanup jobs for GCP IAM data in Neo4j.
 
@@ -177,7 +193,7 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any])
     logger.debug("Running GCP IAM cleanup job")
     job_params = {
         **common_job_parameters,
-        'projectId': common_job_parameters.get('PROJECT_ID'),
+        "projectId": common_job_parameters.get("PROJECT_ID"),
     }
 
     cleanup_jobs = [
@@ -210,8 +226,12 @@ def sync(
 
     # Get and load service accounts
     service_accounts = get_gcp_service_accounts(iam_client, project_id)
-    logger.info(f"Found {len(service_accounts)} service accounts in project {project_id}")
-    load_gcp_service_accounts(neo4j_session, service_accounts, project_id, gcp_update_tag)
+    logger.info(
+        f"Found {len(service_accounts)} service accounts in project {project_id}"
+    )
+    load_gcp_service_accounts(
+        neo4j_session, service_accounts, project_id, gcp_update_tag
+    )
 
     # Get and load roles
     roles = get_gcp_roles(iam_client, project_id)

cartography/intel/gcp/storage.py

@@ -33,7 +33,7 @@ def get_gcp_buckets(storage: Resource, project_id: str) -> Dict:
         return res
     except HttpError as e:
         reason = compute._get_error_reason(e)
-        if reason == 'invalid':
+        if reason == "invalid":
             logger.warning(
                 (
                     "The project %s is invalid - returned a 400 invalid error."
@@ -43,12 +43,15 @@ def get_gcp_buckets(storage: Resource, project_id: str) -> Dict:
                 e,
             )
             return {}
-        elif reason == 'forbidden':
+        elif reason == "forbidden":
             logger.warning(
                 (
                     "You do not have storage.bucket.list access to the project %s. "
                     "Full details: %s"
-                ), project_id, e, )
+                ),
+                project_id,
+                e,
+            )
             return {}
         else:
             raise
@@ -56,7 +59,7 @@ def get_gcp_buckets(storage: Resource, project_id: str) -> Dict:
 
 @timeit
 def transform_gcp_buckets(bucket_res: Dict) -> List[Dict]:
-    '''
+    """
     Transform the GCP Storage Bucket response object for Neo4j ingestion
 
     :type bucket_res: The GCP storage resource object (https://cloud.google.com/storage/docs/json_api/v1/buckets)
@@ -64,40 +67,52 @@ def transform_gcp_buckets(bucket_res: Dict) -> List[Dict]:
 
     :rtype: list
     :return: List of buckets ready for ingestion to Neo4j
-    '''
+    """
 
     bucket_list = []
-    for b in bucket_res.get('items', []):
+    for b in bucket_res.get("items", []):
         bucket = {}
-        bucket['etag'] = b.get('etag')
-        bucket['iam_config_bucket_policy_only'] = \
-            b.get('iamConfiguration', {}).get('bucketPolicyOnly', {}).get('enabled', None)
-        bucket['id'] = b['id']
-        bucket['labels'] = [(key, val) for (key, val) in b.get('labels', {}).items()]
-        bucket['owner_entity'] = b.get('owner', {}).get('entity')
-        bucket['owner_entity_id'] = b.get('owner', {}).get('entityId')
-        bucket['kind'] = b.get('kind')
-        bucket['location'] = b.get('location')
-        bucket['location_type'] = b.get('locationType')
-        bucket['meta_generation'] = b.get('metageneration', None)
-        bucket['project_number'] = b['projectNumber']
-        bucket['self_link'] = b.get('selfLink')
-        bucket['storage_class'] = b.get('storageClass')
-        bucket['time_created'] = b.get('timeCreated')
-        bucket['updated'] = b.get('updated')
-        bucket['versioning_enabled'] = b.get('versioning', {}).get('enabled', None)
-        bucket['default_event_based_hold'] = b.get('defaultEventBasedHold', None)
-        bucket['retention_period'] = b.get('retentionPolicy', {}).get('retentionPeriod', None)
-        bucket['default_kms_key_name'] = b.get('encryption', {}).get('defaultKmsKeyName')
-        bucket['log_bucket'] = b.get('logging', {}).get('logBucket')
-        bucket['requester_pays'] = b.get('billing', {}).get('requesterPays', None)
+        bucket["etag"] = b.get("etag")
+        bucket["iam_config_bucket_policy_only"] = (
+            b.get("iamConfiguration", {})
+            .get("bucketPolicyOnly", {})
+            .get("enabled", None)
+        )
+        bucket["id"] = b["id"]
+        bucket["labels"] = [(key, val) for (key, val) in b.get("labels", {}).items()]
+        bucket["owner_entity"] = b.get("owner", {}).get("entity")
+        bucket["owner_entity_id"] = b.get("owner", {}).get("entityId")
+        bucket["kind"] = b.get("kind")
+        bucket["location"] = b.get("location")
+        bucket["location_type"] = b.get("locationType")
+        bucket["meta_generation"] = b.get("metageneration", None)
+        bucket["project_number"] = b["projectNumber"]
+        bucket["self_link"] = b.get("selfLink")
+        bucket["storage_class"] = b.get("storageClass")
+        bucket["time_created"] = b.get("timeCreated")
+        bucket["updated"] = b.get("updated")
+        bucket["versioning_enabled"] = b.get("versioning", {}).get("enabled", None)
+        bucket["default_event_based_hold"] = b.get("defaultEventBasedHold", None)
+        bucket["retention_period"] = b.get("retentionPolicy", {}).get(
+            "retentionPeriod",
+            None,
+        )
+        bucket["default_kms_key_name"] = b.get("encryption", {}).get(
+            "defaultKmsKeyName",
+        )
+        bucket["log_bucket"] = b.get("logging", {}).get("logBucket")
+        bucket["requester_pays"] = b.get("billing", {}).get("requesterPays", None)
         bucket_list.append(bucket)
     return bucket_list
 
 
 @timeit
-def load_gcp_buckets(neo4j_session: neo4j.Session, buckets: List[Dict], gcp_update_tag: int) -> None:
-    '''
+def load_gcp_buckets(
+    neo4j_session: neo4j.Session,
+    buckets: List[Dict],
+    gcp_update_tag: int,
+) -> None:
+    """
     Ingest GCP Storage Buckets to Neo4j
 
     :type neo4j_session: Neo4j session object
@@ -111,7 +126,7 @@ def load_gcp_buckets(neo4j_session: neo4j.Session, buckets: List[Dict], gcp_upda
 
     :rtype: NoneType
     :return: Nothing
-    '''
+    """
 
     query = """
     MERGE(p:GCPProject{projectnumber:$ProjectNumber})
@@ -146,30 +161,34 @@ def load_gcp_buckets(neo4j_session: neo4j.Session, buckets: List[Dict], gcp_upda
     for bucket in buckets:
         neo4j_session.run(
             query,
-            ProjectNumber=bucket['project_number'],
-            BucketId=bucket['id'],
-            SelfLink=bucket['self_link'],
-            Kind=bucket['kind'],
-            Location=bucket['location'],
-            LocationType=bucket['location_type'],
-            MetaGeneration=bucket['meta_generation'],
-            StorageClass=bucket['storage_class'],
-            TimeCreated=bucket['time_created'],
-            RetentionPeriod=bucket['retention_period'],
-            IamConfigBucketPolicyOnly=bucket['iam_config_bucket_policy_only'],
-            OwnerEntity=bucket['owner_entity'],
-            OwnerEntityId=bucket['owner_entity_id'],
-            VersioningEnabled=bucket['versioning_enabled'],
-            LogBucket=bucket['log_bucket'],
-            RequesterPays=bucket['requester_pays'],
-            DefaultKmsKeyName=bucket['default_kms_key_name'],
+            ProjectNumber=bucket["project_number"],
+            BucketId=bucket["id"],
+            SelfLink=bucket["self_link"],
+            Kind=bucket["kind"],
+            Location=bucket["location"],
+            LocationType=bucket["location_type"],
+            MetaGeneration=bucket["meta_generation"],
+            StorageClass=bucket["storage_class"],
+            TimeCreated=bucket["time_created"],
+            RetentionPeriod=bucket["retention_period"],
+            IamConfigBucketPolicyOnly=bucket["iam_config_bucket_policy_only"],
+            OwnerEntity=bucket["owner_entity"],
+            OwnerEntityId=bucket["owner_entity_id"],
+            VersioningEnabled=bucket["versioning_enabled"],
+            LogBucket=bucket["log_bucket"],
+            RequesterPays=bucket["requester_pays"],
+            DefaultKmsKeyName=bucket["default_kms_key_name"],
             gcp_update_tag=gcp_update_tag,
         )
         _attach_gcp_bucket_labels(neo4j_session, bucket, gcp_update_tag)
 
 
 @timeit
-def _attach_gcp_bucket_labels(neo4j_session: neo4j.Session, bucket: Resource, gcp_update_tag: int) -> None:
+def _attach_gcp_bucket_labels(
+    neo4j_session: neo4j.Session,
+    bucket: Resource,
+    gcp_update_tag: int,
+) -> None:
     """
     Attach GCP bucket labels to the bucket.
     :param neo4j_session: The neo4j session
@@ -189,19 +208,22 @@ def _attach_gcp_bucket_labels(neo4j_session: neo4j.Session, bucket: Resource, gc
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = $gcp_update_tag
     """
-    for (key, val) in bucket.get('labels', []):
+    for key, val in bucket.get("labels", []):
         neo4j_session.run(
             query,
             BucketLabelId=f"GCPBucket_{key}",
             Key=key,
             Value=val,
-            BucketId=bucket['id'],
+            BucketId=bucket["id"],
             gcp_update_tag=gcp_update_tag,
         )
 
 
 @timeit
-def cleanup_gcp_buckets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+def cleanup_gcp_buckets(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
     """
     Delete out-of-date GCP Storage Bucket nodes and relationships
 
@@ -214,12 +236,19 @@ def cleanup_gcp_buckets(neo4j_session: neo4j.Session, common_job_parameters: Dic
     :rtype: NoneType
     :return: Nothing
     """
-    run_cleanup_job('gcp_storage_bucket_cleanup.json', neo4j_session, common_job_parameters)
+    run_cleanup_job(
+        "gcp_storage_bucket_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
 
 
 @timeit
 def sync_gcp_buckets(
-    neo4j_session: neo4j.Session, storage: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    storage: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -247,5 +276,5 @@ def sync_gcp_buckets(
     storage_res = get_gcp_buckets(storage, project_id)
     bucket_list = transform_gcp_buckets(storage_res)
     load_gcp_buckets(neo4j_session, bucket_list, gcp_update_tag)
-    # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
+    # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
     cleanup_gcp_buckets(neo4j_session, common_job_parameters)

cartography/intel/github/__init__.py

@@ -23,7 +23,9 @@ def start_github_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
     :return: None
     """
     if not config.github_config:
-        logger.info('GitHub import is not configured - skipping this module. See docs to configure.')
+        logger.info(
+            "GitHub import is not configured - skipping this module. See docs to configure.",
+        )
         return
 
     auth_tokens = json.loads(base64.b64decode(config.github_config).decode())
@@ -31,28 +33,28 @@ def start_github_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
         "UPDATE_TAG": config.update_tag,
     }
     # run sync for the provided github tokens
-    for auth_data in auth_tokens['organization']:
+    for auth_data in auth_tokens["organization"]:
         try:
             cartography.intel.github.users.sync(
                 neo4j_session,
                 common_job_parameters,
-                auth_data['token'],
-                auth_data['url'],
-                auth_data['name'],
+                auth_data["token"],
+                auth_data["url"],
+                auth_data["name"],
             )
             cartography.intel.github.repos.sync(
                 neo4j_session,
                 common_job_parameters,
-                auth_data['token'],
-                auth_data['url'],
-                auth_data['name'],
+                auth_data["token"],
+                auth_data["url"],
+                auth_data["name"],
             )
             cartography.intel.github.teams.sync_github_teams(
                 neo4j_session,
                 common_job_parameters,
-                auth_data['token'],
-                auth_data['url'],
-                auth_data['name'],
+                auth_data["token"],
+                auth_data["url"],
+                auth_data["name"],
             )
         except exceptions.RequestException as e:
             logger.error("Could not complete request to the GitHub API: %s", e)