cartography 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

cartography/intel/gcp/compute.py

@@ -15,6 +15,7 @@ from googleapiclient.discovery import Resource
 from googleapiclient.errors import HttpError
 
 from cartography.client.core.tx import load
+from cartography.client.core.tx import run_write_query
 from cartography.graph.job import GraphJob
 from cartography.models.gcp.compute.vpc import GCPVpcSchema
 from cartography.util import run_cleanup_job
@@ -619,7 +620,8 @@ def load_gcp_instances(
     SET r.lastupdated = $gcp_update_tag
     """
     for instance in data:
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             ProjectId=instance["project_id"],
             PartialUri=instance["partial_uri"],
@@ -714,7 +716,8 @@ def load_gcp_forwarding_rules(
         network = fwd.get("network", None)
         subnetwork = fwd.get("subnetwork", None)
 
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             PartialUri=fwd["partial_uri"],
             IPAddress=fwd["ip_address"],
@@ -760,7 +763,8 @@ def _attach_fwd_rule_to_subnet(
         SET p.lastupdated = $gcp_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         query,
         PartialUri=fwd["partial_uri"],
         SubNetworkPartialUri=fwd.get("subnetwork_partial_uri", None),
@@ -787,7 +791,8 @@ def _attach_fwd_rule_to_vpc(
         SET r.lastupdated = $gcp_update_tag
     """
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         query,
         PartialUri=fwd["partial_uri"],
         NetworkPartialUri=fwd.get("network_partial_uri", None),
@@ -831,7 +836,8 @@ def _attach_instance_tags(
     for tag in instance.get("tags", {}).get("items", []):
         for nic in instance.get("networkInterfaces", []):
             tag_id = _create_gcp_network_tag_id(nic["vpc_partial_uri"], tag)
-            neo4j_session.run(
+            run_write_query(
+                neo4j_session,
                 query,
                 InstanceId=instance["partial_uri"],
                 TagId=tag_id,
@@ -880,7 +886,8 @@ def _attach_gcp_nics(
     for nic in instance.get("networkInterfaces", []):
         # Make an ID for GCPNetworkInterface nodes because GCP doesn't define one but we need to uniquely identify them
         nic_id = f"{instance['partial_uri']}/networkinterfaces/{nic['name']}"
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             InstanceId=instance["partial_uri"],
             NicId=nic_id,
@@ -926,7 +933,8 @@ def _attach_gcp_nic_access_configs(
     for ac in nic.get("accessConfigs", []):
         # Make an ID for GCPNicAccessConfig nodes because GCP doesn't define one but we need to uniquely identify them
         access_config_id = f"{nic_id}/accessconfigs/{ac['type']}"
-        neo4j_session.run(
+        run_write_query(
+            neo4j_session,
             query,
             NicId=nic_id,
             AccessConfigId=access_config_id,
@@ -960,7 +968,8 @@ def _attach_gcp_vpc(
     ON CREATE SET m.firstseen = timestamp()
     SET m.lastupdated = $gcp_update_tag
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         query,
         InstanceId=instance_id,
         gcp_update_tag=gcp_update_tag,
@@ -974,10 +983,22 @@ def load_gcp_ingress_firewalls(
     gcp_update_tag: int,
 ) -> None:
     """
-    Load the firewall list to Neo4j
+    Load the firewall list to Neo4j.
     :param fw_list: The transformed list of firewalls
     :return: Nothing
     """
+    neo4j_session.execute_write(
+        _load_gcp_ingress_firewalls_tx,
+        fw_list,
+        gcp_update_tag,
+    )
+
+
+def _load_gcp_ingress_firewalls_tx(
+    tx: neo4j.Transaction,
+    fw_list: List[Resource],
+    gcp_update_tag: int,
+) -> None:
     query = """
     MERGE (fw:GCPFirewall{id:$FwPartialUri})
     ON CREATE SET fw.firstseen = timestamp(),
@@ -1000,7 +1021,7 @@ def load_gcp_ingress_firewalls(
     SET r.lastupdated = $gcp_update_tag
     """
     for fw in fw_list:
-        neo4j_session.run(
+        tx.run(
             query,
             FwPartialUri=fw["id"],
             Direction=fw["direction"],
@@ -1011,20 +1032,20 @@ def load_gcp_ingress_firewalls(
             VpcPartialUri=fw["vpc_partial_uri"],
             HasTargetServiceAccounts=fw["has_target_service_accounts"],
             gcp_update_tag=gcp_update_tag,
-        )
-        _attach_firewall_rules(neo4j_session, fw, gcp_update_tag)
-        _attach_target_tags(neo4j_session, fw, gcp_update_tag)
+        ).consume()
+        _attach_firewall_rules(tx, fw, gcp_update_tag)
+        _attach_target_tags(tx, fw, gcp_update_tag)
 
 
 @timeit
 def _attach_firewall_rules(
-    neo4j_session: neo4j.Session,
+    tx: neo4j.Transaction,
     fw: Resource,
     gcp_update_tag: int,
 ) -> None:
     """
     Attach the allow_rules to the Firewall object
-    :param neo4j_session: The Neo4j session
+    :param tx: The Neo4j transaction
     :param fw: The Firewall object
     :param gcp_update_tag: The timestamp
     :return: Nothing
@@ -1065,7 +1086,7 @@ def _attach_firewall_rules(
             # If sourceRanges is not specified then the rule must specify sourceTags.
             # Since an IP range cannot have a tag applied to it, it is ok if we don't ingest this rule.
             for ip_range in fw.get("sourceRanges", []):
-                neo4j_session.run(
+                tx.run(
                     template.safe_substitute(fw_rule_relationship_label=label),
                     FwPartialUri=fw["id"],
                     RuleId=rule["ruleid"],
@@ -1074,18 +1095,18 @@ def _attach_firewall_rules(
                     ToPort=rule.get("toport"),
                     Range=ip_range,
                     gcp_update_tag=gcp_update_tag,
-                )
+                ).consume()
 
 
 @timeit
 def _attach_target_tags(
-    neo4j_session: neo4j.Session,
+    tx: neo4j.Transaction,
     fw: Resource,
     gcp_update_tag: int,
 ) -> None:
     """
     Attach target tags to the firewall object
-    :param neo4j_session: The neo4j session
+    :param tx: The neo4j transaction
     :param fw: The firewall object
     :param gcp_update_tag: The timestamp
     :return: Nothing
@@ -1105,13 +1126,13 @@ def _attach_target_tags(
     """
     for tag in fw.get("targetTags", []):
         tag_id = _create_gcp_network_tag_id(fw["vpc_partial_uri"], tag)
-        neo4j_session.run(
+        tx.run(
             query,
             FwPartialUri=fw["id"],
             TagId=tag_id,
             TagValue=tag,
             gcp_update_tag=gcp_update_tag,
-        )
+        ).consume()
 
 
 @timeit

cartography/intel/gcp/crm/folders.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Dict
 from typing import List
+from typing import Optional
 
 import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
 from google.cloud import resourcemanager_v3
 
 from cartography.client.core.tx import load
@@ -13,7 +15,10 @@ logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_gcp_folders(org_resource_name: str) -> List[Dict]:
+def get_gcp_folders(
+    org_resource_name: str,
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
     """
     Return a list of all descendant GCP folders under the specified organization by traversing the folder tree.
 
@@ -21,7 +26,7 @@ def get_gcp_folders(org_resource_name: str) -> List[Dict]:
     :return: List of folder dicts with 'name' field containing full resource names (e.g., "folders/123456")
     """
     results: List[Dict] = []
-    client = resourcemanager_v3.FoldersClient()
+    client = resourcemanager_v3.FoldersClient(credentials=credentials)
     # BFS over folders starting at the org root
     queue: List[str] = [org_resource_name]
     seen: set[str] = set()
@@ -96,6 +101,7 @@ def sync_gcp_folders(
     gcp_update_tag: int,
     common_job_parameters: Dict,
     org_resource_name: str,
+    credentials: Optional[GoogleCredentials] = None,
 ) -> List[Dict]:
     """
     Get GCP folder data using the CRM v2 resource object and load the data to Neo4j.
@@ -103,6 +109,6 @@ def sync_gcp_folders(
     :return: List of folders synced
     """
     logger.debug("Syncing GCP folders")
-    folders = get_gcp_folders(org_resource_name)
+    folders = get_gcp_folders(org_resource_name, credentials=credentials)
     load_gcp_folders(neo4j_session, folders, gcp_update_tag, org_resource_name)
     return folders

cartography/intel/gcp/crm/orgs.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Dict
 from typing import List
+from typing import Optional
 
 import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
 from google.cloud import resourcemanager_v3
 
 from cartography.client.core.tx import load
@@ -13,13 +15,15 @@ logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_gcp_organizations() -> List[Dict]:
+def get_gcp_organizations(
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
     """
     Return list of GCP organizations that the authenticated principal can access using the high-level client.
     Returns empty list on error.
     :return: List of org dicts with keys: name, displayName, lifecycleState.
     """
-    client = resourcemanager_v3.OrganizationsClient()
+    client = resourcemanager_v3.OrganizationsClient(credentials=credentials)
     orgs = []
     for org in client.search_organizations():
         orgs.append(
@@ -54,12 +58,13 @@ def sync_gcp_organizations(
     neo4j_session: neo4j.Session,
     gcp_update_tag: int,
     common_job_parameters: Dict,
+    credentials: Optional[GoogleCredentials] = None,
 ) -> List[Dict]:
     """
     Get GCP organization data using the CRM v1 resource object and load the data to Neo4j.
     Returns the list of organizations synced.
     """
     logger.debug("Syncing GCP organizations")
-    data = get_gcp_organizations()
+    data = get_gcp_organizations(credentials=credentials)
     load_gcp_organizations(neo4j_session, data, gcp_update_tag)
     return data

cartography/intel/gcp/crm/projects.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Dict
 from typing import List
+from typing import Optional
 
 import neo4j
+from google.auth.credentials import Credentials as GoogleCredentials
 from google.cloud import resourcemanager_v3
 
 from cartography.client.core.tx import load
@@ -13,7 +15,11 @@ logger = logging.getLogger(__name__)
 
 
 @timeit
-def get_gcp_projects(org_resource_name: str, folders: List[Dict]) -> List[Dict]:
+def get_gcp_projects(
+    org_resource_name: str,
+    folders: List[Dict],
+    credentials: Optional[GoogleCredentials] = None,
+) -> List[Dict]:
     """
     Return list of ACTIVE GCP projects under the specified organization
     and within the specified folders.
@@ -25,7 +31,7 @@ def get_gcp_projects(org_resource_name: str, folders: List[Dict]) -> List[Dict]:
     parents = set([org_resource_name] + folder_names)
     results: List[Dict] = []
     for parent in parents:
-        client = resourcemanager_v3.ProjectsClient()
+        client = resourcemanager_v3.ProjectsClient(credentials=credentials)
         for proj in client.list_projects(parent=parent):
             # list_projects returns ACTIVE projects by default
             name_field = proj.name  # "projects/<number>"
@@ -96,6 +102,7 @@ def sync_gcp_projects(
     folders: List[Dict],
     gcp_update_tag: int,
     common_job_parameters: Dict,
+    credentials: Optional[GoogleCredentials] = None,
 ) -> List[Dict]:
     """
     Get and sync GCP project data to Neo4j.
@@ -104,6 +111,10 @@ def sync_gcp_projects(
     :return: List of projects synced
     """
     logger.debug("Syncing GCP projects")
-    projects = get_gcp_projects(org_resource_name, folders)
+    projects = get_gcp_projects(
+        org_resource_name,
+        folders,
+        credentials=credentials,
+    )
     load_gcp_projects(neo4j_session, projects, gcp_update_tag, org_resource_name)
     return projects

cartography/intel/github/repos.py

@@ -4,6 +4,7 @@ from collections import defaultdict
 from collections import namedtuple
 from string import Template
 from typing import Any
+from typing import cast
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -157,12 +158,19 @@ def _get_repo_collaborators_inner_func(
     org: str,
     api_url: str,
     token: str,
-    repo_raw_data: list[dict[str, Any]],
+    repo_raw_data: list[dict[str, Any] | None],
     affiliation: str,
 ) -> dict[str, list[UserAffiliationAndRepoPermission]]:
     result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
 
     for repo in repo_raw_data:
+        # GitHub can return null repo entries. See issues #1334 and #1404.
+        if repo is None:
+            logger.info(
+                "Skipping null repository entry while fetching %s collaborators.",
+                affiliation,
+            )
+            continue
         repo_name = repo["name"]
         repo_url = repo["url"]
 
@@ -212,7 +220,7 @@ def _get_repo_collaborators_inner_func(
 
 
 def _get_repo_collaborators_for_multiple_repos(
-    repo_raw_data: list[dict[str, Any]],
+    repo_raw_data: list[dict[str, Any] | None],
     affiliation: str,
     org: str,
     api_url: str,
@@ -279,7 +287,7 @@ def _get_repo_collaborators(
 
 
 @timeit
-def get(token: str, api_url: str, organization: str) -> List[Dict]:
+def get(token: str, api_url: str, organization: str) -> List[Optional[Dict]]:
     """
     Retrieve a list of repos from a Github organization as described in
     https://docs.github.com/en/graphql/reference/objects#repository.
@@ -287,6 +295,8 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
     :param api_url: The Github v4 API endpoint as string.
     :param organization: The name of the target Github organization as string.
     :return: A list of dicts representing repos. See tests.data.github.repos for data shape.
+        Note: The list may contain None entries per GraphQL spec when resolvers error
+        (permissions, rate limits, transient issues). See issues #1334 and #1404.
     """
     # TODO: link the Github organization to the repositories
     repos, _ = fetch_all(
@@ -297,11 +307,15 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
         "repositories",
         count=50,
     )
-    return repos.nodes
+    # Cast is needed because GitHub's GraphQL RepositoryConnection.nodes is typed [Repository] (not [Repository!])
+    # per GraphQL spec, allowing null entries when resolvers error (permissions, rate limits, transient issues).
+    # See https://github.com/cartography-cncf/cartography/issues/1334
+    # and https://github.com/cartography-cncf/cartography/issues/1404
+    return cast(List[Optional[Dict]], repos.nodes)
 
 
 def transform(
-    repos_json: List[Dict],
+    repos_json: List[Optional[Dict]],
     direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
     outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
 ) -> Dict:
@@ -340,6 +354,10 @@ def transform(
     transformed_dependencies: List[Dict] = []
     transformed_manifests: List[Dict] = []
     for repo_object in repos_json:
+        # GitHub can return null repo entries. See issues #1334 and #1404.
+        if repo_object is None:
+            logger.debug("Skipping null repository entry during transformation.")
+            continue
         _transform_repo_languages(
             repo_object["url"],
             repo_object,

cartography/intel/gsuite/__init__.py

@@ -16,7 +16,8 @@ from google.oauth2.service_account import Credentials as ServiceAccountCredentia
 from googleapiclient.discovery import Resource
 
 from cartography.config import Config
-from cartography.intel.gsuite import api
+from cartography.intel.gsuite import groups
+from cartography.intel.gsuite import users
 from cartography.util import timeit
 
 OAUTH_SCOPES = [
@@ -148,15 +149,18 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
             return
 
     resources = _initialize_resources(creds)
-    api.sync_gsuite_users(
-        neo4j_session,
-        resources.admin,
-        config.update_tag,
-        common_job_parameters,
-    )
-    api.sync_gsuite_groups(
+    customer_ids = users.sync_gsuite_users(
         neo4j_session,
         resources.admin,
         config.update_tag,
         common_job_parameters,
     )
+    for customer_id in customer_ids:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["CUSTOMER_ID"] = customer_id
+        groups.sync_gsuite_groups(
+            neo4j_session,
+            resources.admin,
+            config.update_tag,
+            scoped_job_parameters,
+        )