cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/gcp/gke.py

@@ -1,13 +1,18 @@
 import json
 import logging
+from typing import Any
 from typing import Dict
+from typing import List
 
 import neo4j
 from googleapiclient.discovery import HttpError
 from googleapiclient.discovery import Resource
 
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.gke import GCPGKEClusterSchema
 from cartography.util import timeit
+
 logger = logging.getLogger(__name__)
 
 
@@ -26,16 +31,21 @@ def get_gke_clusters(container: Resource, project_id: str) -> Dict:
     :return: Cluster response object
     """
     try:
-        req = container.projects().zones().clusters().list(projectId=project_id, zone='-')
+        req = (
+            container.projects().zones().clusters().list(projectId=project_id, zone="-")
+        )
         res = req.execute()
         return res
     except HttpError as e:
-        err = json.loads(e.content.decode('utf-8'))['error']
-        if err['status'] == 'PERMISSION_DENIED':
+        err = json.loads(e.content.decode("utf-8"))["error"]
+        if err["status"] == "PERMISSION_DENIED":
             logger.warning(
                 (
                     "Could not retrieve GKE clusters on project %s due to permissions issue. Code: %s, Message: %s"
-                ), project_id, err['code'], err['message'],
+                ),
+                project_id,
+                err["code"],
+                err["message"],
             )
             return {}
         else:
@@ -43,94 +53,27 @@ def get_gke_clusters(container: Resource, project_id: str) -> Dict:
 
 
 @timeit
-def load_gke_clusters(neo4j_session: neo4j.Session, cluster_resp: Dict, project_id: str, gcp_update_tag: int) -> None:
+def load_gke_clusters(
+    neo4j_session: neo4j.Session,
+    cluster_resp: Dict,
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
     """
-    Ingest GCP GKE Clusters to Neo4j
-
-    :type neo4j_session: Neo4j session object
-    :param neo4j session: The Neo4j session object
-
-    :type cluster_resp: Dict
-    :param cluster_resp: A cluster response object from the GKE API
-
-    :type gcp_update_tag: timestamp
-    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
-
-    :rtype: NoneType
-    :return: Nothing
+    Ingest GCP GKE clusters using the data model loader.
     """
+    clusters: List[Dict[str, Any]] = transform_gke_clusters(cluster_resp)
 
-    query = """
-    MERGE(cluster:GKECluster{id:$ClusterSelfLink})
-    ON CREATE SET
-        cluster.firstseen = timestamp(),
-        cluster.created_at = $ClusterCreateTime
-    SET
-        cluster.name = $ClusterName,
-        cluster.self_link = $ClusterSelfLink,
-        cluster.description = $ClusterDescription,
-        cluster.logging_service = $ClusterLoggingService,
-        cluster.monitoring_service = $ClusterMonitoringService,
-        cluster.network = $ClusterNetwork,
-        cluster.subnetwork = $ClusterSubnetwork,
-        cluster.cluster_ipv4cidr = $ClusterIPv4Cidr,
-        cluster.zone = $ClusterZone,
-        cluster.location = $ClusterLocation,
-        cluster.endpoint = $ClusterEndpoint,
-        cluster.initial_version = $ClusterInitialVersion,
-        cluster.current_master_version = $ClusterMasterVersion,
-        cluster.status = $ClusterStatus,
-        cluster.services_ipv4cidr = $ClusterServicesIPv4Cidr,
-        cluster.database_encryption = $ClusterDatabaseEncryption,
-        cluster.network_policy = $ClusterNetworkPolicy,
-        cluster.master_authorized_networks = $ClusterMasterAuthorizedNetworks,
-        cluster.legacy_abac = $ClusterAbac,
-        cluster.shielded_nodes = $ClusterShieldedNodes,
-        cluster.private_nodes = $ClusterPrivateNodes,
-        cluster.private_endpoint_enabled = $ClusterPrivateEndpointEnabled,
-        cluster.private_endpoint = $ClusterPrivateEndpoint,
-        cluster.public_endpoint = $ClusterPublicEndpoint,
-        cluster.masterauth_username = $ClusterMasterUsername,
-        cluster.masterauth_password = $ClusterMasterPassword
-    WITH cluster
-    MATCH (owner:GCPProject{id:$ProjectId})
-    MERGE (owner)-[r:RESOURCE]->(cluster)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $gcp_update_tag
-    """
-    for cluster in cluster_resp.get('clusters', []):
-        neo4j_session.run(
-            query,
-            ProjectId=project_id,
-            ClusterSelfLink=cluster['selfLink'],
-            ClusterCreateTime=cluster['createTime'],
-            ClusterName=cluster['name'],
-            ClusterDescription=cluster.get('description'),
-            ClusterLoggingService=cluster.get('loggingService'),
-            ClusterMonitoringService=cluster.get('monitoringService'),
-            ClusterNetwork=cluster.get('network'),
-            ClusterSubnetwork=cluster.get('subnetwork'),
-            ClusterIPv4Cidr=cluster.get('clusterIpv4Cidr'),
-            ClusterZone=cluster.get('zone'),
-            ClusterLocation=cluster.get('location'),
-            ClusterEndpoint=cluster.get('endpoint'),
-            ClusterInitialVersion=cluster.get('initialClusterVersion'),
-            ClusterMasterVersion=cluster.get('currentMasterVersion'),
-            ClusterStatus=cluster.get('status'),
-            ClusterServicesIPv4Cidr=cluster.get('servicesIpv4Cidr'),
-            ClusterDatabaseEncryption=cluster.get('databaseEncryption', {}).get('state'),
-            ClusterNetworkPolicy=_process_network_policy(cluster),
-            ClusterMasterAuthorizedNetworks=cluster.get('masterAuthorizedNetworksConfig', {}).get('enabled'),
-            ClusterAbac=cluster.get('legacyAbac', {}).get('enabled'),
-            ClusterShieldedNodes=cluster.get('shieldedNodes', {}).get('enabled'),
-            ClusterPrivateNodes=cluster.get('privateClusterConfig', {}).get('enablePrivateNodes'),
-            ClusterPrivateEndpointEnabled=cluster.get('privateClusterConfig', {}).get('enablePrivateEndpoint'),
-            ClusterPrivateEndpoint=cluster.get('privateClusterConfig', {}).get('privateEndpoint'),
-            ClusterPublicEndpoint=cluster.get('privateClusterConfig', {}).get('publicEndpoint'),
-            ClusterMasterUsername=cluster.get('masterAuth', {}).get('username'),
-            ClusterMasterPassword=cluster.get('masterAuth', {}).get('password'),
-            gcp_update_tag=gcp_update_tag,
-        )
+    if not clusters:
+        return
+
+    load(
+        neo4j_session,
+        GCPGKEClusterSchema(),
+        clusters,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
 
 
 def _process_network_policy(cluster: Dict) -> bool:
@@ -138,33 +81,32 @@ def _process_network_policy(cluster: Dict) -> bool:
     Parse cluster.networkPolicy to verify if
     the provider has been enabled.
     """
-    provider = cluster.get('networkPolicy', {}).get('provider')
-    enabled = cluster.get('networkPolicy', {}).get('enabled')
+    provider = cluster.get("networkPolicy", {}).get("provider")
+    enabled = cluster.get("networkPolicy", {}).get("enabled")
     if provider and enabled is True:
         return provider
     return False
 
 
 @timeit
-def cleanup_gke_clusters(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+def cleanup_gke_clusters(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+) -> None:
     """
-    Delete out-of-date GCP GKE Clusters nodes and relationships
-
-    :type neo4j_session: The Neo4j session object
-    :param neo4j_session: The Neo4j session
-
-    :type common_job_parameters: dict
-    :param common_job_parameters: Dictionary of other job parameters to pass to Neo4j
-
-    :rtype: NoneType
-    :return: Nothing
+    Scoped cleanup for GKE clusters based on the project sub-resource relationship.
     """
-    run_cleanup_job('gcp_gke_cluster_cleanup.json', neo4j_session, common_job_parameters)
+    GraphJob.from_node_schema(GCPGKEClusterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
 def sync_gke_clusters(
-    neo4j_session: neo4j.Session, container: Resource, project_id: str, gcp_update_tag: int,
+    neo4j_session: neo4j.Session,
+    container: Resource,
+    project_id: str,
+    gcp_update_tag: int,
     common_job_parameters: Dict,
 ) -> None:
     """
@@ -188,8 +130,59 @@ def sync_gke_clusters(
     :rtype: NoneType
     :return: Nothing
     """
-    logger.info("Syncing Compute objects for project %s.", project_id)
+    logger.info("Syncing GKE clusters for project %s.", project_id)
     gke_res = get_gke_clusters(container, project_id)
     load_gke_clusters(neo4j_session, gke_res, project_id, gcp_update_tag)
-    # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
     cleanup_gke_clusters(neo4j_session, common_job_parameters)
+
+
+def transform_gke_clusters(api_result: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """
+    Transform GKE API response into a list of dicts suitable for the data model loader.
+    """
+    result: List[Dict[str, Any]] = []
+    for c in api_result.get("clusters", []):
+        transformed: Dict[str, Any] = {
+            # Required fields
+            "id": c["selfLink"],
+            "self_link": c["selfLink"],
+            "name": c["name"],
+            "created_at": c.get("createTime"),
+            # Optional fields
+            "description": c.get("description"),
+            "logging_service": c.get("loggingService"),
+            "monitoring_service": c.get("monitoringService"),
+            "network": c.get("network"),
+            "subnetwork": c.get("subnetwork"),
+            "cluster_ipv4cidr": c.get("clusterIpv4Cidr"),
+            "zone": c.get("zone"),
+            "location": c.get("location"),
+            "endpoint": c.get("endpoint"),
+            "initial_version": c.get("initialClusterVersion"),
+            "current_master_version": c.get("currentMasterVersion"),
+            "status": c.get("status"),
+            "services_ipv4cidr": c.get("servicesIpv4Cidr"),
+            "database_encryption": (c.get("databaseEncryption", {}) or {}).get("state"),
+            "network_policy": _process_network_policy(c),
+            "master_authorized_networks": (
+                c.get("masterAuthorizedNetworksConfig", {}) or {}
+            ).get("enabled"),
+            "legacy_abac": (c.get("legacyAbac", {}) or {}).get("enabled"),
+            "shielded_nodes": (c.get("shieldedNodes", {}) or {}).get("enabled"),
+            "private_nodes": (c.get("privateClusterConfig", {}) or {}).get(
+                "enablePrivateNodes"
+            ),
+            "private_endpoint_enabled": (c.get("privateClusterConfig", {}) or {}).get(
+                "enablePrivateEndpoint"
+            ),
+            "private_endpoint": (c.get("privateClusterConfig", {}) or {}).get(
+                "privateEndpoint"
+            ),
+            "public_endpoint": (c.get("privateClusterConfig", {}) or {}).get(
+                "publicEndpoint"
+            ),
+            "masterauth_username": (c.get("masterAuth", {}) or {}).get("username"),
+            "masterauth_password": (c.get("masterAuth", {}) or {}).get("password"),
+        }
+        result.append(transformed)
+    return result

cartography/intel/gcp/iam.py

@@ -0,0 +1,262 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.iam import GCPRoleSchema
+from cartography.models.gcp.iam import GCPServiceAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# GCP API can be subject to rate limiting, so add small delays between calls
+LIST_SLEEP = 1
+DESCRIBE_SLEEP = 1
+
+
+@timeit
+def get_gcp_service_accounts(
+    iam_client: Resource, project_id: str
+) -> List[Dict[str, Any]]:
+    """
+    Retrieve a list of GCP service accounts for a given project.
+
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :param project_id: The GCP Project ID to retrieve service accounts from.
+    :return: A list of dictionaries representing GCP service accounts.
+    """
+    service_accounts: List[Dict[str, Any]] = []
+    request = (
+        iam_client.projects()
+        .serviceAccounts()
+        .list(
+            name=f"projects/{project_id}",
+        )
+    )
+    while request is not None:
+        response = request.execute()
+        if "accounts" in response:
+            service_accounts.extend(response["accounts"])
+        request = (
+            iam_client.projects()
+            .serviceAccounts()
+            .list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        )
+    return service_accounts
+
+
+@timeit
+def get_gcp_predefined_roles(iam_client: Resource) -> List[Dict]:
+    """
+    Retrieve all predefined (Google-managed) IAM roles.
+
+    Predefined roles are global and not project-specific, so they can be fetched once
+    and reused across all target projects. This is useful for the CAI fallback where
+    the target project may not have the IAM API enabled.
+
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :return: A list of dictionaries representing GCP predefined roles.
+    """
+    roles: List[Dict] = []
+    predefined_req = iam_client.roles().list(view="FULL")
+    while predefined_req is not None:
+        resp = predefined_req.execute()
+        roles.extend(resp.get("roles", []))
+        predefined_req = iam_client.roles().list_next(predefined_req, resp)
+    return roles
+
+
+@timeit
+def get_gcp_roles(iam_client: Resource, project_id: str) -> List[Dict]:
+    """
+    Retrieve custom and predefined roles from GCP for a given project.
+
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :param project_id: The GCP Project ID to retrieve roles from.
+    :return: A list of dictionaries representing GCP roles.
+    """
+    roles = []
+
+    # Get custom roles
+    custom_req = iam_client.projects().roles().list(parent=f"projects/{project_id}")
+    while custom_req is not None:
+        resp = custom_req.execute()
+        roles.extend(resp.get("roles", []))
+        custom_req = iam_client.projects().roles().list_next(custom_req, resp)
+
+    # Get predefined roles (global, not project-specific)
+    roles.extend(get_gcp_predefined_roles(iam_client))
+
+    return roles
+
+
+def transform_gcp_service_accounts(
+    raw_accounts: List[Dict[str, Any]],
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform raw GCP service accounts into loader-friendly dicts.
+    """
+    result: List[Dict[str, Any]] = []
+    for sa in raw_accounts:
+        result.append(
+            {
+                "id": sa["uniqueId"],
+                "email": sa.get("email"),
+                "displayName": sa.get("displayName"),
+                "oauth2ClientId": sa.get("oauth2ClientId"),
+                "uniqueId": sa.get("uniqueId"),
+                "disabled": sa.get("disabled", False),
+                "projectId": project_id,
+            },
+        )
+    return result
+
+
+@timeit
+def load_gcp_service_accounts(
+    neo4j_session: neo4j.Session,
+    service_accounts: List[Dict[str, Any]],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """
+    Load GCP service account data into Neo4j.
+    """
+    logger.debug(
+        f"Loading {len(service_accounts)} service accounts for project {project_id}"
+    )
+
+    load(
+        neo4j_session,
+        GCPServiceAccountSchema(),
+        service_accounts,
+        lastupdated=gcp_update_tag,
+        projectId=project_id,
+    )
+
+
+def transform_gcp_roles(
+    raw_roles: List[Dict[str, Any]],
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    """
+    Transform raw GCP roles into loader-friendly dicts.
+    """
+    result: List[Dict[str, Any]] = []
+    for role in raw_roles:
+        role_name = role["name"]
+        if role_name.startswith("roles/"):
+            role_type = (
+                "BASIC"
+                if role_name in ["roles/owner", "roles/editor", "roles/viewer"]
+                else "PREDEFINED"
+            )
+        else:
+            role_type = "CUSTOM"
+
+        result.append(
+            {
+                "id": role_name,
+                "name": role_name,
+                "title": role.get("title"),
+                "description": role.get("description"),
+                "deleted": role.get("deleted", False),
+                "etag": role.get("etag"),
+                "includedPermissions": role.get("includedPermissions", []),
+                "roleType": role_type,
+                "projectId": project_id,
+            },
+        )
+    return result
+
+
+@timeit
+def load_gcp_roles(
+    neo4j_session: neo4j.Session,
+    roles: List[Dict[str, Any]],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """
+    Load GCP role data into Neo4j.
+    """
+    logger.debug(f"Loading {len(roles)} roles for project {project_id}")
+
+    load(
+        neo4j_session,
+        GCPRoleSchema(),
+        roles,
+        lastupdated=gcp_update_tag,
+        projectId=project_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    """
+    Run cleanup jobs for GCP IAM data in Neo4j.
+
+    :param neo4j_session: The Neo4j session.
+    :param common_job_parameters: Common job parameters for cleanup.
+    """
+    logger.debug("Running GCP IAM cleanup job")
+    job_params = {
+        **common_job_parameters,
+        "projectId": common_job_parameters.get("PROJECT_ID"),
+    }
+
+    cleanup_jobs = [
+        GraphJob.from_node_schema(GCPServiceAccountSchema(), job_params),
+        GraphJob.from_node_schema(GCPRoleSchema(), job_params),
+    ]
+
+    for cleanup_job in cleanup_jobs:
+        cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    iam_client: Resource,
+    project_id: str,
+    gcp_update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
+    Sync GCP IAM resources for a given project.
+
+    :param neo4j_session: The Neo4j session.
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :param project_id: The GCP Project ID to sync.
+    :param gcp_update_tag: The timestamp of the current sync run.
+    :param common_job_parameters: Common job parameters for the sync.
+    """
+    logger.info(f"Syncing GCP IAM for project {project_id}")
+
+    service_accounts_raw = get_gcp_service_accounts(iam_client, project_id)
+    logger.info(
+        f"Found {len(service_accounts_raw)} service accounts in project {project_id}"
+    )
+    service_accounts = transform_gcp_service_accounts(service_accounts_raw, project_id)
+    load_gcp_service_accounts(
+        neo4j_session, service_accounts, project_id, gcp_update_tag
+    )
+
+    roles_raw = get_gcp_roles(iam_client, project_id)
+    logger.info(f"Found {len(roles_raw)} roles in project {project_id}")
+    roles = transform_gcp_roles(roles_raw, project_id)
+    load_gcp_roles(neo4j_session, roles, project_id, gcp_update_tag)
+
+    # Run cleanup
+    cleanup(neo4j_session, common_job_parameters)