cartography 0.112.0__py3-none-any.whl → 0.114.0__py3-none-any.whl

cartography/intel/gcp/clients.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Optional
+
+import googleapiclient.discovery
+import httplib2
+from google.auth import default
+from google.auth.credentials import Credentials as GoogleCredentials
+from google.auth.exceptions import DefaultCredentialsError
+from google_auth_httplib2 import AuthorizedHttp
+from googleapiclient.discovery import Resource
+
+logger = logging.getLogger(__name__)
+
+# Default HTTP timeout (seconds) for Google API clients built via discovery.build
+_GCP_HTTP_TIMEOUT = 120
+
+
+def _authorized_http_with_timeout(
+    credentials: GoogleCredentials,
+    timeout: int = _GCP_HTTP_TIMEOUT,
+) -> AuthorizedHttp:
+    """
+    Build an AuthorizedHttp with a per-request timeout, avoiding global socket timeouts.
+    """
+    return AuthorizedHttp(credentials, http=httplib2.Http(timeout=timeout))
+
+
+def build_client(service: str, version: str = "v1") -> Resource:
+    credentials = get_gcp_credentials()
+    if credentials is None:
+        raise RuntimeError("GCP credentials are not available; cannot build client.")
+    client = googleapiclient.discovery.build(
+        service,
+        version,
+        http=_authorized_http_with_timeout(credentials),
+        cache_discovery=False,
+    )
+    return client
+
+
+def get_gcp_credentials() -> Optional[GoogleCredentials]:
+    """
+    Gets access tokens for GCP API access.
+    """
+    try:
+        # Explicitly use Application Default Credentials with the cloud-platform scope.
+        credentials, _ = default(
+            scopes=["https://www.googleapis.com/auth/cloud-platform"],
+        )
+        return credentials
+    except DefaultCredentialsError as e:
+        logger.debug(
+            "Error occurred calling google.auth.default().",
+            exc_info=True,
+        )
+        logger.error(
+            (
+                "Unable to initialize Google Compute Platform creds. If you don't have GCP data or don't want to load "
+                "GCP data then you can ignore this message. Otherwise, the error code is: %s "
+                "Make sure your GCP credentials are configured correctly, your credentials file (if any) is valid, and "
+                "that the identity you are authenticating to has the securityReviewer role attached."
+            ),
+            e,
+        )
+    return None

cartography/intel/gcp/compute.py

@@ -656,51 +656,25 @@ def load_gcp_subnets(
     neo4j_session: neo4j.Session,
     subnets: List[Dict],
     gcp_update_tag: int,
+    project_id: str,
 ) -> None:
     """
-    Ingest GCP subnet data to Neo4j
+    Ingest GCP subnet data to Neo4j using the data model
     :param neo4j_session: The Neo4j session
     :param subnets: List of the subnets
     :param gcp_update_tag: The timestamp to set these Neo4j nodes with
+    :param project_id: The project ID
     :return: Nothing
     """
-    query = """
-    MERGE(vpc:GCPVpc{id:$VpcPartialUri})
-    ON CREATE SET vpc.firstseen = timestamp(),
-    vpc.partial_uri = $VpcPartialUri
+    from cartography.models.gcp.compute.subnet import GCPSubnetSchema
 
-    MERGE(subnet:GCPSubnet{id:$PartialUri})
-    ON CREATE SET subnet.firstseen = timestamp(),
-    subnet.partial_uri = $PartialUri
-    SET subnet.self_link = $SubnetSelfLink,
-    subnet.project_id = $ProjectId,
-    subnet.name = $SubnetName,
-    subnet.region = $Region,
-    subnet.gateway_address = $GatewayAddress,
-    subnet.ip_cidr_range = $IpCidrRange,
-    subnet.private_ip_google_access = $PrivateIpGoogleAccess,
-    subnet.vpc_partial_uri = $VpcPartialUri,
-    subnet.lastupdated = $gcp_update_tag
-
-    MERGE (vpc)-[r:RESOURCE]->(subnet)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $gcp_update_tag
-    """
-    for s in subnets:
-        neo4j_session.run(
-            query,
-            VpcPartialUri=s["vpc_partial_uri"],
-            VpcSelfLink=s["vpc_self_link"],
-            PartialUri=s["partial_uri"],
-            SubnetSelfLink=s["self_link"],
-            ProjectId=s["project_id"],
-            SubnetName=s["name"],
-            Region=s["region"],
-            GatewayAddress=s["gateway_address"],
-            IpCidrRange=s["ip_cidr_range"],
-            PrivateIpGoogleAccess=s["private_ip_google_access"],
-            gcp_update_tag=gcp_update_tag,
-        )
+    load(
+        neo4j_session,
+        GCPSubnetSchema(),
+        subnets,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
 
 
 @timeit
@@ -981,7 +955,7 @@ def _attach_gcp_vpc(
     """
     query = """
     MATCH (i:GCPInstance{id:$InstanceId})-[:NETWORK_INTERFACE]->(nic:GCPNetworkInterface)
-          -[p:PART_OF_SUBNET]->(sn:GCPSubnet)<-[r:RESOURCE]-(vpc:GCPVpc)
+          -[p:PART_OF_SUBNET]->(sn:GCPSubnet)<-[r:HAS]-(vpc:GCPVpc)
     MERGE (i)-[m:MEMBER_OF_GCP_VPC]->(vpc)
     ON CREATE SET m.firstseen = timestamp()
     SET m.lastupdated = $gcp_update_tag
@@ -1185,15 +1159,15 @@ def cleanup_gcp_subnets(
     common_job_parameters: Dict,
 ) -> None:
     """
-    Delete out-of-date GCP VPC subnet nodes and relationships
+    Delete out-of-date GCP VPC subnet nodes and relationships using data model
     :param neo4j_session: The Neo4j session
     :param common_job_parameters: dict of other job parameters to pass to Neo4j
     :return: Nothing
     """
-    run_cleanup_job(
-        "gcp_compute_vpc_subnet_cleanup.json",
-        neo4j_session,
-        common_job_parameters,
+    from cartography.models.gcp.compute.subnet import GCPSubnetSchema
+
+    GraphJob.from_node_schema(GCPSubnetSchema(), common_job_parameters).run(
+        neo4j_session
     )
 
 
@@ -1296,7 +1270,7 @@ def sync_gcp_subnets(
     for r in regions:
         subnet_res = get_gcp_subnets(project_id, r, compute)
         subnets = transform_gcp_subnets(subnet_res)
-        load_gcp_subnets(neo4j_session, subnets, gcp_update_tag)
+        load_gcp_subnets(neo4j_session, subnets, gcp_update_tag, project_id)
         # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
         cleanup_gcp_subnets(neo4j_session, common_job_parameters)
 

cartography/intel/gcp/crm/folders.py

@@ -0,0 +1,108 @@
+import logging
+from typing import Dict
+from typing import List
+
+import neo4j
+from google.cloud import resourcemanager_v3
+
+from cartography.client.core.tx import load
+from cartography.models.gcp.crm.folders import GCPFolderSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_gcp_folders(org_resource_name: str) -> List[Dict]:
+    """
+    Return a list of all descendant GCP folders under the specified organization by traversing the folder tree.
+
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :return: List of folder dicts with 'name' field containing full resource names (e.g., "folders/123456")
+    """
+    results: List[Dict] = []
+    client = resourcemanager_v3.FoldersClient()
+    # BFS over folders starting at the org root
+    queue: List[str] = [org_resource_name]
+    seen: set[str] = set()
+    while queue:
+        parent = queue.pop(0)
+        if parent in seen:
+            continue
+        seen.add(parent)
+
+        for folder in client.list_folders(parent=parent):
+            results.append(
+                {
+                    "name": folder.name,
+                    "parent": parent,
+                    "displayName": folder.display_name,
+                    "lifecycleState": folder.state.name,
+                }
+            )
+            if folder.name:
+                queue.append(folder.name)
+    return results
+
+
+@timeit
+def transform_gcp_folders(data: List[Dict]) -> List[Dict]:
+    """
+    Transform GCP folder data to add parent_org or parent_folder fields based on parent type.
+
+    :param data: List of folder dicts
+    :return: List of transformed folder dicts with parent_org and parent_folder fields
+    """
+    for folder in data:
+        folder["parent_org"] = None
+        folder["parent_folder"] = None
+
+        if folder["parent"].startswith("organizations"):
+            folder["parent_org"] = folder["parent"]
+        elif folder["parent"].startswith("folders"):
+            folder["parent_folder"] = folder["parent"]
+        else:
+            logger.warning(
+                f"Folder {folder['name']} has unexpected parent type: {folder['parent']}"
+            )
+
+    return data
+
+
+@timeit
+def load_gcp_folders(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    gcp_update_tag: int,
+    org_resource_name: str,
+) -> None:
+    """
+    Load GCP folders into the graph.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    """
+    transformed_data = transform_gcp_folders(data)
+    load(
+        neo4j_session,
+        GCPFolderSchema(),
+        transformed_data,
+        lastupdated=gcp_update_tag,
+        ORG_RESOURCE_NAME=org_resource_name,
+    )
+
+
+@timeit
+def sync_gcp_folders(
+    neo4j_session: neo4j.Session,
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+    org_resource_name: str,
+) -> List[Dict]:
+    """
+    Get GCP folder data using the CRM v2 resource object and load the data to Neo4j.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :return: List of folders synced
+    """
+    logger.debug("Syncing GCP folders")
+    folders = get_gcp_folders(org_resource_name)
+    load_gcp_folders(neo4j_session, folders, gcp_update_tag, org_resource_name)
+    return folders

cartography/intel/gcp/crm/orgs.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Dict
+from typing import List
+
+import neo4j
+from google.cloud import resourcemanager_v3
+
+from cartography.client.core.tx import load
+from cartography.models.gcp.crm.organizations import GCPOrganizationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_gcp_organizations() -> List[Dict]:
+    """
+    Return list of GCP organizations that the authenticated principal can access using the high-level client.
+    Returns empty list on error.
+    :return: List of org dicts with keys: name, displayName, lifecycleState.
+    """
+    client = resourcemanager_v3.OrganizationsClient()
+    orgs = []
+    for org in client.search_organizations():
+        orgs.append(
+            {
+                "name": org.name,
+                "displayName": org.display_name,
+                "lifecycleState": org.state.name,
+            }
+        )
+    return orgs
+
+
+@timeit
+def load_gcp_organizations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    gcp_update_tag: int,
+) -> None:
+    for org in data:
+        org["id"] = org["name"]
+
+    load(
+        neo4j_session,
+        GCPOrganizationSchema(),
+        data,
+        lastupdated=gcp_update_tag,
+    )
+
+
+@timeit
+def sync_gcp_organizations(
+    neo4j_session: neo4j.Session,
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> List[Dict]:
+    """
+    Get GCP organization data using the CRM v1 resource object and load the data to Neo4j.
+    Returns the list of organizations synced.
+    """
+    logger.debug("Syncing GCP organizations")
+    data = get_gcp_organizations()
+    load_gcp_organizations(neo4j_session, data, gcp_update_tag)
+    return data

cartography/intel/gcp/crm/projects.py

@@ -0,0 +1,109 @@
+import logging
+from typing import Dict
+from typing import List
+
+import neo4j
+from google.cloud import resourcemanager_v3
+
+from cartography.client.core.tx import load
+from cartography.models.gcp.crm.projects import GCPProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_gcp_projects(org_resource_name: str, folders: List[Dict]) -> List[Dict]:
+    """
+    Return list of ACTIVE GCP projects under the specified organization
+    and within the specified folders.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :param folders: List of folder dictionaries containing 'name' field with full resource names
+    """
+    folder_names = [folder["name"] for folder in folders] if folders else []
+    # Build list of parent resources to check (org and all folders)
+    parents = set([org_resource_name] + folder_names)
+    results: List[Dict] = []
+    for parent in parents:
+        client = resourcemanager_v3.ProjectsClient()
+        for proj in client.list_projects(parent=parent):
+            # list_projects returns ACTIVE projects by default
+            name_field = proj.name  # "projects/<number>"
+            project_number = name_field.split("/")[-1] if name_field else None
+            project_parent = proj.parent
+            results.append(
+                {
+                    "projectId": getattr(proj, "project_id", None),
+                    "projectNumber": project_number,
+                    "name": getattr(proj, "display_name", None),
+                    "lifecycleState": proj.state.name,
+                    "parent": project_parent,
+                }
+            )
+    return results
+
+
+@timeit
+def transform_gcp_projects(data: List[Dict]) -> List[Dict]:
+    """
+    Transform GCP project data to add parent_org or parent_folder fields based on parent type.
+
+    :param data: List of project dicts
+    :return: List of transformed project dicts with parent_org and parent_folder fields
+    """
+    for project in data:
+        project["parent_org"] = None
+        project["parent_folder"] = None
+
+        # Set parent fields based on parent type
+        if project["parent"].startswith("organizations"):
+            project["parent_org"] = project["parent"]
+        elif project["parent"].startswith("folders"):
+            project["parent_folder"] = project["parent"]
+        else:
+            logger.warning(
+                f"Project {project['projectId']} has unexpected parent type: {project['parent']}"
+            )
+
+    return data
+
+
+@timeit
+def load_gcp_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    gcp_update_tag: int,
+    org_resource_name: str,
+) -> None:
+    """
+    Load GCP projects into the graph.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    """
+    transformed_data = transform_gcp_projects(data)
+    load(
+        neo4j_session,
+        GCPProjectSchema(),
+        transformed_data,
+        lastupdated=gcp_update_tag,
+        ORG_RESOURCE_NAME=org_resource_name,
+    )
+
+
+@timeit
+def sync_gcp_projects(
+    neo4j_session: neo4j.Session,
+    org_resource_name: str,
+    folders: List[Dict],
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> List[Dict]:
+    """
+    Get and sync GCP project data to Neo4j.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :param folders: List of folder dictionaries containing 'name' field with full resource names
+    :return: List of projects synced
+    """
+    logger.debug("Syncing GCP projects")
+    projects = get_gcp_projects(org_resource_name, folders)
+    load_gcp_projects(neo4j_session, projects, gcp_update_tag, org_resource_name)
+    return projects