cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/gcp/crm/folders.py

@@ -0,0 +1,114 @@
+import logging
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
+from google.cloud import resourcemanager_v3
+
+from cartography.client.core.tx import load
+from cartography.models.gcp.crm.folders import GCPFolderSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_gcp_folders(
+    org_resource_name: str,
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
+    """
+    Return a list of all descendant GCP folders under the specified organization by traversing the folder tree.
+
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :return: List of folder dicts with 'name' field containing full resource names (e.g., "folders/123456")
+    """
+    results: List[Dict] = []
+    client = resourcemanager_v3.FoldersClient(credentials=credentials)
+    # BFS over folders starting at the org root
+    queue: List[str] = [org_resource_name]
+    seen: set[str] = set()
+    while queue:
+        parent = queue.pop(0)
+        if parent in seen:
+            continue
+        seen.add(parent)
+
+        for folder in client.list_folders(parent=parent):
+            results.append(
+                {
+                    "name": folder.name,
+                    "parent": parent,
+                    "displayName": folder.display_name,
+                    "lifecycleState": folder.state.name,
+                }
+            )
+            if folder.name:
+                queue.append(folder.name)
+    return results
+
+
+@timeit
+def transform_gcp_folders(data: List[Dict]) -> List[Dict]:
+    """
+    Transform GCP folder data to add parent_org or parent_folder fields based on parent type.
+
+    :param data: List of folder dicts
+    :return: List of transformed folder dicts with parent_org and parent_folder fields
+    """
+    for folder in data:
+        folder["parent_org"] = None
+        folder["parent_folder"] = None
+
+        if folder["parent"].startswith("organizations"):
+            folder["parent_org"] = folder["parent"]
+        elif folder["parent"].startswith("folders"):
+            folder["parent_folder"] = folder["parent"]
+        else:
+            logger.warning(
+                f"Folder {folder['name']} has unexpected parent type: {folder['parent']}"
+            )
+
+    return data
+
+
+@timeit
+def load_gcp_folders(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    gcp_update_tag: int,
+    org_resource_name: str,
+) -> None:
+    """
+    Load GCP folders into the graph.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    """
+    transformed_data = transform_gcp_folders(data)
+    load(
+        neo4j_session,
+        GCPFolderSchema(),
+        transformed_data,
+        lastupdated=gcp_update_tag,
+        ORG_RESOURCE_NAME=org_resource_name,
+    )
+
+
+@timeit
+def sync_gcp_folders(
+    neo4j_session: neo4j.Session,
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+    org_resource_name: str,
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
+    """
+    Get GCP folder data using the CRM v2 resource object and load the data to Neo4j.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :return: List of folders synced
+    """
+    logger.debug("Syncing GCP folders")
+    folders = get_gcp_folders(org_resource_name, credentials=credentials)
+    load_gcp_folders(neo4j_session, folders, gcp_update_tag, org_resource_name)
+    return folders

cartography/intel/gcp/crm/orgs.py

@@ -0,0 +1,70 @@
+import logging
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
+from google.cloud import resourcemanager_v3
+
+from cartography.client.core.tx import load
+from cartography.models.gcp.crm.organizations import GCPOrganizationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_gcp_organizations(
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
+    """
+    Return list of GCP organizations that the authenticated principal can access using the high-level client.
+    Returns empty list on error.
+    :return: List of org dicts with keys: name, displayName, lifecycleState.
+    """
+    client = resourcemanager_v3.OrganizationsClient(credentials=credentials)
+    orgs = []
+    for org in client.search_organizations():
+        orgs.append(
+            {
+                "name": org.name,
+                "displayName": org.display_name,
+                "lifecycleState": org.state.name,
+            }
+        )
+    return orgs
+
+
+@timeit
+def load_gcp_organizations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    gcp_update_tag: int,
+) -> None:
+    for org in data:
+        org["id"] = org["name"]
+
+    load(
+        neo4j_session,
+        GCPOrganizationSchema(),
+        data,
+        lastupdated=gcp_update_tag,
+    )
+
+
+@timeit
+def sync_gcp_organizations(
+    neo4j_session: neo4j.Session,
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
+    """
+    Get GCP organization data using the CRM v1 resource object and load the data to Neo4j.
+    Returns the list of organizations synced.
+    """
+    logger.debug("Syncing GCP organizations")
+    data = get_gcp_organizations(credentials=credentials)
+    load_gcp_organizations(neo4j_session, data, gcp_update_tag)
+    return data

cartography/intel/gcp/crm/projects.py

@@ -0,0 +1,120 @@
+import logging
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
+from google.cloud import resourcemanager_v3
+
+from cartography.client.core.tx import load
+from cartography.models.gcp.crm.projects import GCPProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_gcp_projects(
+    org_resource_name: str,
+    folders: List[Dict],
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
+    """
+    Return list of ACTIVE GCP projects under the specified organization
+    and within the specified folders.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :param folders: List of folder dictionaries containing 'name' field with full resource names
+    """
+    folder_names = [folder["name"] for folder in folders] if folders else []
+    # Build list of parent resources to check (org and all folders)
+    parents = set([org_resource_name] + folder_names)
+    results: List[Dict] = []
+    for parent in parents:
+        client = resourcemanager_v3.ProjectsClient(credentials=credentials)
+        for proj in client.list_projects(parent=parent):
+            # list_projects returns ACTIVE projects by default
+            name_field = proj.name  # "projects/<number>"
+            project_number = name_field.split("/")[-1] if name_field else None
+            project_parent = proj.parent
+            results.append(
+                {
+                    "projectId": getattr(proj, "project_id", None),
+                    "projectNumber": project_number,
+                    "name": getattr(proj, "display_name", None),
+                    "lifecycleState": proj.state.name,
+                    "parent": project_parent,
+                }
+            )
+    return results
+
+
+@timeit
+def transform_gcp_projects(data: List[Dict]) -> List[Dict]:
+    """
+    Transform GCP project data to add parent_org or parent_folder fields based on parent type.
+
+    :param data: List of project dicts
+    :return: List of transformed project dicts with parent_org and parent_folder fields
+    """
+    for project in data:
+        project["parent_org"] = None
+        project["parent_folder"] = None
+
+        # Set parent fields based on parent type
+        if project["parent"].startswith("organizations"):
+            project["parent_org"] = project["parent"]
+        elif project["parent"].startswith("folders"):
+            project["parent_folder"] = project["parent"]
+        else:
+            logger.warning(
+                f"Project {project['projectId']} has unexpected parent type: {project['parent']}"
+            )
+
+    return data
+
+
+@timeit
+def load_gcp_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    gcp_update_tag: int,
+    org_resource_name: str,
+) -> None:
+    """
+    Load GCP projects into the graph.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    """
+    transformed_data = transform_gcp_projects(data)
+    load(
+        neo4j_session,
+        GCPProjectSchema(),
+        transformed_data,
+        lastupdated=gcp_update_tag,
+        ORG_RESOURCE_NAME=org_resource_name,
+    )
+
+
+@timeit
+def sync_gcp_projects(
+    neo4j_session: neo4j.Session,
+    org_resource_name: str,
+    folders: List[Dict],
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
+    """
+    Get and sync GCP project data to Neo4j.
+    :param org_resource_name: Full organization resource name (e.g., "organizations/123456789012")
+    :param folders: List of folder dictionaries containing 'name' field with full resource names
+    :return: List of projects synced
+    """
+    logger.debug("Syncing GCP projects")
+    projects = get_gcp_projects(
+        org_resource_name,
+        folders,
+        credentials=credentials,
+    )
+    load_gcp_projects(neo4j_session, projects, gcp_update_tag, org_resource_name)
+    return projects

cartography/intel/gcp/dns.py

@@ -7,235 +7,190 @@ import neo4j
 from googleapiclient.discovery import HttpError
 from googleapiclient.discovery import Resource
 
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.dns import GCPDNSZoneSchema
+from cartography.models.gcp.dns import GCPRecordSetSchema
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_dns_zones(dns: Resource, project_id: str) -> List[Resource]:
-    """
-    Returns a list of DNS zones within the given project.
-
-    :type dns: The GCP DNS resource object
-    :param dns: The DNS resource object created by googleapiclient.discovery.build()
-
-    :type project_id: str
-    :param project_id: Current Google Project Id
-
-    :rtype: list
-    :return: List of DNS zones
-    """
+def get_dns_zones(dns: Resource, project_id: str) -> List[Dict]:
+    """Returns a list of DNS zones within the given project."""
     try:
-        zones = []
+        zones: List[Dict] = []
         request = dns.managedZones().list(project=project_id)
         while request is not None:
             response = request.execute()
-            for managed_zone in response['managedZones']:
+            for managed_zone in response["managedZones"]:
                 zones.append(managed_zone)
-            request = dns.managedZones().list_next(previous_request=request, previous_response=response)
+            request = dns.managedZones().list_next(
+                previous_request=request,
+                previous_response=response,
+            )
         return zones
     except HttpError as e:
-        err = json.loads(e.content.decode('utf-8'))['error']
-        if err.get('status', '') == 'PERMISSION_DENIED' or err.get('message', '') == 'Forbidden':
+        err = json.loads(e.content.decode("utf-8"))["error"]
+        if (
+            err.get("status", "") == "PERMISSION_DENIED"
+            or err.get("message", "") == "Forbidden"
+        ):
             logger.warning(
                 (
-                    "Could not retrieve DNS zones on project %s due to permissions issues. Code: %s, Message: %s"
-                ), project_id, err['code'], err['message'],
+                    "Could not retrieve DNS zones on project %s due to permissions issues. "
+                    "Code: %s, Message: %s"
+                ),
+                project_id,
+                err["code"],
+                err["message"],
             )
             return []
-        else:
-            raise
+        raise
 
 
 @timeit
-def get_dns_rrs(dns: Resource, dns_zones: List[Dict], project_id: str) -> List[Resource]:
-    """
-    Returns a list of DNS Resource Record Sets within the given project.
-
-    :type dns: The GCP DNS resource object
-    :param dns: The DNS resource object created by googleapiclient.discovery.build()
-
-    :type dns_zones: list
-    :param dns_zones: List of DNS zones for the project
-
-    :type project_id: str
-    :param project_id: Current Google Project Id
-
-    :rtype: list
-    :return: List of Resource Record Sets
-    """
+def get_dns_rrs(dns: Resource, dns_zones: List[Dict], project_id: str) -> List[Dict]:
+    """Returns a list of DNS Resource Record Sets within the given project."""
     try:
-        rrs: List[Resource] = []
+        rrs: List[Dict] = []
         for zone in dns_zones:
-            request = dns.resourceRecordSets().list(project=project_id, managedZone=zone['id'])
+            request = dns.resourceRecordSets().list(
+                project=project_id,
+                managedZone=zone["id"],
+            )
             while request is not None:
                 response = request.execute()
-                for resource_record_set in response['rrsets']:
-                    resource_record_set['zone'] = zone['id']
+                for resource_record_set in response["rrsets"]:
+                    resource_record_set["zone"] = zone["id"]
                     rrs.append(resource_record_set)
-                request = dns.resourceRecordSets().list_next(previous_request=request, previous_response=response)
+                request = dns.resourceRecordSets().list_next(
+                    previous_request=request,
+                    previous_response=response,
+                )
         return rrs
     except HttpError as e:
-        err = json.loads(e.content.decode('utf-8'))['error']
-        if err.get('status', '') == 'PERMISSION_DENIED' or err.get('message', '') == 'Forbidden':
+        err = json.loads(e.content.decode("utf-8"))["error"]
+        if (
+            err.get("status", "") == "PERMISSION_DENIED"
+            or err.get("message", "") == "Forbidden"
+        ):
             logger.warning(
                 (
-                    "Could not retrieve DNS RRS on project %s due to permissions issues. Code: %s, Message: %s"
-                ), project_id, err['code'], err['message'],
+                    "Could not retrieve DNS RRS on project %s due to permissions issues. "
+                    "Code: %s, Message: %s"
+                ),
+                project_id,
+                err["code"],
+                err["message"],
             )
             return []
-        else:
-            raise
-        raise e
+        raise
 
 
 @timeit
-def load_dns_zones(neo4j_session: neo4j.Session, dns_zones: List[Dict], project_id: str, gcp_update_tag: int) -> None:
-    """
-    Ingest GCP DNS Zones into Neo4j
-
-    :type neo4j_session: Neo4j session object
-    :param neo4j session: The Neo4j session object
-
-    :type dns_resp: Dict
-    :param dns_resp: A DNS response object from the GKE API
-
-    :type project_id: str
-    :param project_id: Current Google Project Id
-
-    :type gcp_update_tag: timestamp
-    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
-
-    :rtype: NoneType
-    :return: Nothing
-    """
-
-    ingest_records = """
-    UNWIND $records as record
-    MERGE(zone:GCPDNSZone{id:record.id})
-    ON CREATE SET
-        zone.firstseen = timestamp(),
-        zone.created_at = record.creationTime
-    SET
-        zone.name = record.name,
-        zone.dns_name = record.dnsName,
-        zone.description = record.description,
-        zone.visibility = record.visibility,
-        zone.kind = record.kind,
-        zone.nameservers = record.nameServers,
-        zone.lastupdated = $gcp_update_tag
-    WITH zone
-    MATCH (owner:GCPProject{id:$ProjectId})
-    MERGE (owner)-[r:RESOURCE]->(zone)
-    ON CREATE SET
-        r.firstseen = timestamp(),
-        r.lastupdated = $gcp_update_tag
-    """
-    neo4j_session.run(
-        ingest_records,
-        records=dns_zones,
-        ProjectId=project_id,
-        gcp_update_tag=gcp_update_tag,
-    )
+def transform_dns_zones(dns_zones: List[Dict]) -> List[Dict]:
+    """Transform raw DNS zone responses into Neo4j-ready dicts."""
+    zones: List[Dict] = []
+    for z in dns_zones:
+        zones.append(
+            {
+                "id": z["id"],
+                "name": z.get("name"),
+                "dns_name": z.get("dnsName"),
+                "description": z.get("description"),
+                "visibility": z.get("visibility"),
+                "kind": z.get("kind"),
+                "nameservers": z.get("nameServers"),
+                "created_at": z.get("creationTime"),
+            }
+        )
+    return zones
 
 
 @timeit
-def load_rrs(neo4j_session: neo4j.Session, dns_rrs: List[Resource], project_id: str, gcp_update_tag: int) -> None:
-    """
-    Ingest GCP RRS into Neo4j
-
-    :type neo4j_session: Neo4j session object
-    :param neo4j session: The Neo4j session object
-
-    :type dns_rrs: list
-    :param dns_rrs: A list of RRS
-
-    :type project_id: str
-    :param project_id: Current Google Project Id
-
-    :type gcp_update_tag: timestamp
-    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
-
-    :rtype: NoneType
-    :return: Nothing
-    """
-
-    ingest_records = """
-    UNWIND $records as record
-    MERGE(rrs:GCPRecordSet{id:record.name})
-    ON CREATE SET
-        rrs.firstseen = timestamp()
-    SET
-        rrs.name = record.name,
-        rrs.type = record.type,
-        rrs.ttl = record.ttl,
-        rrs.data = record.rrdatas,
-        rrs.lastupdated = $gcp_update_tag
-    WITH rrs, record
-    MATCH (zone:GCPDNSZone{id:record.zone})
-    MERGE (zone)-[r:HAS_RECORD]->(rrs)
-    ON CREATE SET
-        r.firstseen = timestamp(),
-        r.lastupdated = $gcp_update_tag
-    """
-    neo4j_session.run(
-        ingest_records,
-        records=dns_rrs,
-        gcp_update_tag=gcp_update_tag,
-    )
+def transform_dns_rrs(dns_rrs: List[Dict]) -> List[Dict]:
+    """Transform raw DNS record set responses into Neo4j-ready dicts."""
+    records: List[Dict] = []
+    for r in dns_rrs:
+        records.append(
+            {
+                # Compose a unique ID to avoid collisions across types and zones
+                "id": f"{r['name']}|{r.get('type')}|{r.get('zone')}",
+                "name": r["name"],
+                "type": r.get("type"),
+                "ttl": r.get("ttl"),
+                "data": r.get("rrdatas"),
+                "zone_id": r.get("zone"),
+            }
+        )
+    return records
 
 
 @timeit
-def cleanup_dns_records(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
-    """
-    Delete out-of-date GCP DNS Zones and RRS nodes and relationships
-
-    :type neo4j_session: The Neo4j session object
-    :param neo4j_session: The Neo4j session
+def load_dns_zones(
+    neo4j_session: neo4j.Session,
+    dns_zones: List[Dict],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """Ingest GCP DNS Zones into Neo4j."""
+    load(
+        neo4j_session,
+        GCPDNSZoneSchema(),
+        dns_zones,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
 
-    :type common_job_parameters: dict
-    :param common_job_parameters: Dictionary of other job parameters to pass to Neo4j
 
-    :rtype: NoneType
-    :return: Nothing
-    """
-    run_cleanup_job('gcp_dns_cleanup.json', neo4j_session, common_job_parameters)
+@timeit
+def load_rrs(
+    neo4j_session: neo4j.Session,
+    dns_rrs: List[Dict],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """Ingest GCP DNS Resource Record Sets into Neo4j."""
+    load(
+        neo4j_session,
+        GCPRecordSetSchema(),
+        dns_rrs,
+        lastupdated=gcp_update_tag,
+        PROJECT_ID=project_id,
+    )
 
 
 @timeit
-def sync(
-    neo4j_session: neo4j.Session, dns: Resource, project_id: str, gcp_update_tag: int,
+def cleanup_dns_records(
+    neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    """
-    Get GCP DNS Zones and Resource Record Sets using the DNS resource object, ingest to Neo4j, and clean up old data.
-
-    :type neo4j_session: The Neo4j session object
-    :param neo4j_session: The Neo4j session
-
-    :type dns: The DNS resource object created by googleapiclient.discovery.build()
-    :param dns: The GCP DNS resource object
-
-    :type project_id: str
-    :param project_id: The project ID of the corresponding project
-
-    :type gcp_update_tag: timestamp
-    :param gcp_update_tag: The timestamp value to set our new Neo4j nodes with
+    """Delete out-of-date GCP DNS Zones and Record Sets nodes and relationships."""
+    # Record sets depend on zones, so clean them up first.
+    GraphJob.from_node_schema(GCPRecordSetSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(GCPDNSZoneSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
-    :type common_job_parameters: dict
-    :param common_job_parameters: Dictionary of other job parameters to pass to Neo4j
 
-    :rtype: NoneType
-    :return: Nothing
-    """
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    dns: Resource,
+    project_id: str,
+    gcp_update_tag: int,
+    common_job_parameters: Dict,
+) -> None:
+    """Get GCP DNS Zones and Record Sets, load them into Neo4j, and clean up old data."""
     logger.info("Syncing DNS records for project %s.", project_id)
-    # DNS ZONES
-    dns_zones = get_dns_zones(dns, project_id)
+    dns_zones_resp = get_dns_zones(dns, project_id)
+    dns_zones = transform_dns_zones(dns_zones_resp)
     load_dns_zones(neo4j_session, dns_zones, project_id, gcp_update_tag)
-    # RECORD SETS
-    dns_rrs = get_dns_rrs(dns, dns_zones, project_id)
+    dns_rrs_resp = get_dns_rrs(dns, dns_zones_resp, project_id)
+    dns_rrs = transform_dns_rrs(dns_rrs_resp)
     load_rrs(neo4j_session, dns_rrs, project_id, gcp_update_tag)
-    # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
     cleanup_dns_records(neo4j_session, common_job_parameters)