cartography 0.118.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

cartography/intel/aws/inspector.py

@@ -7,6 +7,7 @@ from typing import Set
 from typing import Tuple
 
 import boto3
+import botocore
 import neo4j
 
 from cartography.client.core.tx import load
@@ -17,7 +18,9 @@ from cartography.models.aws.inspector.findings import InspectorFindingToPackageM
 from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
 from cartography.util import aws_handle_regions
 from cartography.util import aws_paginate
+from cartography.util import AWS_REGION_ACCESS_DENIED_ERROR_CODES
 from cartography.util import batch
+from cartography.util import is_service_control_policy_explicit_deny
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -70,7 +73,6 @@ def get_member_accounts(
 
 
 @timeit
-@aws_handle_regions
 def get_inspector_findings(
     session: boto3.session.Session,
     region: str,
@@ -83,6 +85,10 @@ def get_inspector_findings(
     only fetch those in ACTIVE or SUPPRESSED statuses.
     Run the query in batches and return an iterator to fetch the results.
     """
+    # Note: We can't use @aws_handle_regions decorator here because this function returns a generator.
+    # The decorator would only catch exceptions during function call, not during iteration.
+    # Instead, we rely on aws_handle_regions being applied at get_member_accounts level,
+    # and the paginate operation itself will raise errors that bubble up naturally.
     client = session.client("inspector2", region_name=region)
     logger.info(
         f"Getting findings in batches of {batch_size} for account {account_id} in region {region}"
@@ -308,38 +314,64 @@ def _sync_findings_for_account(
     """
     Syncs the findings for a given account in a given region.
     """
-    findings = get_inspector_findings(boto3_session, region, account_id, batch_size)
-    if not findings:
-        logger.info(f"No findings to sync for account {account_id} in region {region}")
-        return
-    for f_batch in findings:
-        finding_data, package_data, finding_to_package_map = (
-            transform_inspector_findings(f_batch)
-        )
-        logger.info(f"Loading {len(finding_data)} findings from account {account_id}")
-        load_inspector_findings(
-            neo4j_session,
-            finding_data,
-            region,
-            update_tag,
-            current_aws_account_id,
-        )
-        logger.info(f"Loading {len(package_data)} packages")
-        load_inspector_packages(
-            neo4j_session,
-            package_data,
-            update_tag,
-            current_aws_account_id,
-        )
-        logger.info(
-            f"Loading {len(finding_to_package_map)} finding to package relationships"
-        )
-        load_inspector_finding_to_package_match_links(
-            neo4j_session,
-            finding_to_package_map,
-            update_tag,
-            current_aws_account_id,
-        )
+    try:
+        findings = get_inspector_findings(boto3_session, region, account_id, batch_size)
+        if not findings:
+            logger.info(
+                f"No findings to sync for account {account_id} in region {region}"
+            )
+            return
+        for f_batch in findings:
+            finding_data, package_data, finding_to_package_map = (
+                transform_inspector_findings(f_batch)
+            )
+            logger.info(
+                f"Loading {len(finding_data)} findings from account {account_id}"
+            )
+            load_inspector_findings(
+                neo4j_session,
+                finding_data,
+                region,
+                update_tag,
+                current_aws_account_id,
+            )
+            logger.info(f"Loading {len(package_data)} packages")
+            load_inspector_packages(
+                neo4j_session,
+                package_data,
+                update_tag,
+                current_aws_account_id,
+            )
+            logger.info(
+                f"Loading {len(finding_to_package_map)} finding to package relationships"
+            )
+            load_inspector_finding_to_package_match_links(
+                neo4j_session,
+                finding_to_package_map,
+                update_tag,
+                current_aws_account_id,
+            )
+    except botocore.exceptions.ClientError as e:
+        error_code = e.response.get("Error", {}).get("Code")
+        # Handle the same error codes as aws_handle_regions decorator
+        if error_code in AWS_REGION_ACCESS_DENIED_ERROR_CODES:
+            error_message = e.response.get("Error", {}).get("Message")
+            if is_service_control_policy_explicit_deny(e):
+                logger.warning(
+                    "Service control policy denied access to Inspector findings for account %s in region %s: %s",
+                    account_id,
+                    region,
+                    error_message,
+                )
+            else:
+                logger.warning(
+                    "Access denied to Inspector findings for account %s in region %s. Skipping...",
+                    account_id,
+                    region,
+                )
+            return
+        else:
+            raise
 
 
 @timeit

cartography/intel/aws/resourcegroupstaggingapi.py

@@ -253,7 +253,7 @@ def _load_tags_tx(
         UpdateTag=aws_update_tag,
         Region=region,
         Account=current_aws_account_id,
-    )
+    ).consume()
 
 
 @timeit

cartography/intel/gcp/compute.py

@@ -1032,7 +1032,7 @@ def _load_gcp_ingress_firewalls_tx(
             VpcPartialUri=fw["vpc_partial_uri"],
             HasTargetServiceAccounts=fw["has_target_service_accounts"],
             gcp_update_tag=gcp_update_tag,
-        )
+        ).consume()
         _attach_firewall_rules(tx, fw, gcp_update_tag)
         _attach_target_tags(tx, fw, gcp_update_tag)
 
@@ -1095,7 +1095,7 @@ def _attach_firewall_rules(
                     ToPort=rule.get("toport"),
                     Range=ip_range,
                     gcp_update_tag=gcp_update_tag,
-                )
+                ).consume()
 
 
 @timeit
@@ -1132,7 +1132,7 @@ def _attach_target_tags(
             TagId=tag_id,
             TagValue=tag,
             gcp_update_tag=gcp_update_tag,
-        )
+        ).consume()
 
 
 @timeit

cartography/intel/github/repos.py

@@ -4,6 +4,7 @@ from collections import defaultdict
 from collections import namedtuple
 from string import Template
 from typing import Any
+from typing import cast
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -157,12 +158,19 @@ def _get_repo_collaborators_inner_func(
     org: str,
     api_url: str,
     token: str,
-    repo_raw_data: list[dict[str, Any]],
+    repo_raw_data: list[dict[str, Any] | None],
     affiliation: str,
 ) -> dict[str, list[UserAffiliationAndRepoPermission]]:
     result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
 
     for repo in repo_raw_data:
+        # GitHub can return null repo entries. See issues #1334 and #1404.
+        if repo is None:
+            logger.info(
+                "Skipping null repository entry while fetching %s collaborators.",
+                affiliation,
+            )
+            continue
         repo_name = repo["name"]
         repo_url = repo["url"]
 
@@ -212,7 +220,7 @@ def _get_repo_collaborators_inner_func(
 
 
 def _get_repo_collaborators_for_multiple_repos(
-    repo_raw_data: list[dict[str, Any]],
+    repo_raw_data: list[dict[str, Any] | None],
     affiliation: str,
     org: str,
     api_url: str,
@@ -279,7 +287,7 @@ def _get_repo_collaborators(
 
 
 @timeit
-def get(token: str, api_url: str, organization: str) -> List[Dict]:
+def get(token: str, api_url: str, organization: str) -> List[Optional[Dict]]:
     """
     Retrieve a list of repos from a Github organization as described in
     https://docs.github.com/en/graphql/reference/objects#repository.
@@ -287,6 +295,8 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
     :param api_url: The Github v4 API endpoint as string.
     :param organization: The name of the target Github organization as string.
     :return: A list of dicts representing repos. See tests.data.github.repos for data shape.
+        Note: The list may contain None entries per GraphQL spec when resolvers error
+        (permissions, rate limits, transient issues). See issues #1334 and #1404.
     """
     # TODO: link the Github organization to the repositories
     repos, _ = fetch_all(
@@ -297,11 +307,15 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
         "repositories",
         count=50,
     )
-    return repos.nodes
+    # Cast is needed because GitHub's GraphQL RepositoryConnection.nodes is typed [Repository] (not [Repository!])
+    # per GraphQL spec, allowing null entries when resolvers error (permissions, rate limits, transient issues).
+    # See https://github.com/cartography-cncf/cartography/issues/1334
+    # and https://github.com/cartography-cncf/cartography/issues/1404
+    return cast(List[Optional[Dict]], repos.nodes)
 
 
 def transform(
-    repos_json: List[Dict],
+    repos_json: List[Optional[Dict]],
     direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
     outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
 ) -> Dict:
@@ -340,6 +354,10 @@ def transform(
     transformed_dependencies: List[Dict] = []
     transformed_manifests: List[Dict] = []
     for repo_object in repos_json:
+        # GitHub can return null repo entries. See issues #1334 and #1404.
+        if repo_object is None:
+            logger.debug("Skipping null repository entry during transformation.")
+            continue
         _transform_repo_languages(
             repo_object["url"],
             repo_object,

cartography/intel/gsuite/__init__.py

@@ -16,7 +16,8 @@ from google.oauth2.service_account import Credentials as ServiceAccountCredentia
 from googleapiclient.discovery import Resource
 
 from cartography.config import Config
-from cartography.intel.gsuite import api
+from cartography.intel.gsuite import groups
+from cartography.intel.gsuite import users
 from cartography.util import timeit
 
 OAUTH_SCOPES = [
@@ -148,15 +149,18 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
             return
 
     resources = _initialize_resources(creds)
-    api.sync_gsuite_users(
-        neo4j_session,
-        resources.admin,
-        config.update_tag,
-        common_job_parameters,
-    )
-    api.sync_gsuite_groups(
+    customer_ids = users.sync_gsuite_users(
         neo4j_session,
         resources.admin,
         config.update_tag,
         common_job_parameters,
     )
+    for customer_id in customer_ids:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["CUSTOMER_ID"] = customer_id
+        groups.sync_gsuite_groups(
+            neo4j_session,
+            resources.admin,
+            config.update_tag,
+            scoped_job_parameters,
+        )

cartography/intel/gsuite/groups.py

@@ -0,0 +1,291 @@
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
+
+from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.models.gsuite.group import GSuiteGroupSchema
+from cartography.models.gsuite.group import GSuiteGroupToGroupMemberRel
+from cartography.models.gsuite.group import GSuiteGroupToGroupOwnerRel
+from cartography.models.gsuite.tenant import GSuiteTenantSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_all_groups(
+    admin: Resource, customer_id: str = "my_customer"
+) -> list[dict[str, Any]]:
+    """
+    Return list of Google Groups in your organization
+    Returns empty list if we are unable to enumerate the groups for any reasons
+
+    googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
+
+    :param admin: google's apiclient discovery resource object.  From googleapiclient.discovery.build
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :return: list of Google groups in domain
+    """
+    request = admin.groups().list(
+        customer=customer_id,
+        maxResults=20,
+        orderBy="email",
+    )
+    response_objects = []
+    while request is not None:
+        try:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            response_objects.extend(resp.get("groups", []))
+            request = admin.groups().list_next(request, resp)
+        except HttpError as e:
+            if (
+                e.resp.status == 403
+                and "Request had insufficient authentication scopes" in str(e)
+            ):
+                logger.error(
+                    "Missing required GSuite scopes. If using the gcloud CLI, "
+                    "run: gcloud auth application-default login --scopes="
+                    '"https://www.googleapis.com/auth/admin.directory.user.readonly,'
+                    "https://www.googleapis.com/auth/admin.directory.group.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.group.member.readonly,"
+                    'https://www.googleapis.com/auth/cloud-platform"'
+                )
+            raise
+    return response_objects
+
+
+@timeit
+def get_members_for_groups(
+    admin: Resource, groups_email: list[str]
+) -> dict[str, list[dict[str, Any]]]:
+    """Get all members for given groups emails
+
+    Args:
+        admin (Resource): google's apiclient discovery resource object.  From googleapiclient.discovery.build
+        See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+        groups_email (list[str]): List of group email addresses to get members for
+
+
+    :return: list of dictionaries representing Users or Groups grouped by group email
+    """
+    results: dict[str, list[dict]] = {}
+    for group_email in groups_email:
+        request = admin.members().list(
+            groupKey=group_email,
+            maxResults=500,
+        )
+        members: list[dict] = []
+        while request is not None:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            members = members + resp.get("members", [])
+            request = admin.members().list_next(request, resp)
+        results[group_email] = members
+
+    return results
+
+
+@timeit
+def transform_groups(
+    groups: list[dict], group_memberships: dict[str, list[dict[str, Any]]]
+) -> tuple[list[dict], list[dict], list[dict]]:
+    """Strips list of API response objects to return list of group objects only and lists of subgroup relationships
+
+    :param groups: Raw groups from Google API
+    :param group_memberships: Group memberships data
+    :return: tuple of (groups, group_member_relationships, group_owner_relationships)
+    """
+    transformed_groups: list[dict] = []
+    group_member_relationships: list[dict] = []
+    group_owner_relationships: list[dict] = []
+
+    for group in groups:
+        group_id = group["id"]
+        group_email = group["email"]
+        group["member_ids"] = []
+        group["owner_ids"] = []
+
+        for member in group_memberships.get(group_email, []):
+            if member["type"] == "GROUP":
+                # Create group-to-group relationships
+                relationship_data = {
+                    "parent_group_id": group_id,
+                    "subgroup_id": member.get("id"),
+                    "role": member.get("role"),
+                }
+
+                if member.get("role") == "OWNER":
+                    group_owner_relationships.append(relationship_data)
+                else:
+                    group_member_relationships.append(relationship_data)
+                continue
+
+            # Handle user memberships
+            if member.get("role") == "OWNER":
+                group["owner_ids"].append(member.get("id"))
+            group["member_ids"].append(member.get("id"))
+
+        transformed_groups.append(group)
+
+    return transformed_groups, group_member_relationships, group_owner_relationships
+
+
+@timeit
+def load_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    groups: list[dict],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite groups using the modern data model
+    """
+    logger.info("Ingesting %d gsuite groups", len(groups))
+
+    # Load tenant first if it doesn't exist
+    tenant_data = [{"id": customer_id}]
+    load(
+        neo4j_session,
+        GSuiteTenantSchema(),
+        tenant_data,
+        lastupdated=gsuite_update_tag,
+    )
+
+    # Load groups with relationship to tenant
+    load(
+        neo4j_session,
+        GSuiteGroupSchema(),
+        groups,
+        lastupdated=gsuite_update_tag,
+        CUSTOMER_ID=customer_id,
+    )
+
+
+@timeit
+def load_gsuite_group_to_group_relationships(
+    neo4j_session: neo4j.Session,
+    group_member_relationships: list[dict],
+    group_owner_relationships: list[dict],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Load GSuite group-to-group relationships using MatchLinks
+    """
+    logger.info(
+        "Ingesting %d group member relationships", len(group_member_relationships)
+    )
+    logger.info(
+        "Ingesting %d group owner relationships", len(group_owner_relationships)
+    )
+
+    # Load group member relationships (Group -> Group MEMBER)
+    if group_member_relationships:
+        load_matchlinks(
+            neo4j_session,
+            GSuiteGroupToGroupMemberRel(),
+            group_member_relationships,
+            lastupdated=gsuite_update_tag,
+            _sub_resource_label="GSuiteTenant",
+            _sub_resource_id=customer_id,
+        )
+
+    # Load group owner relationships (Group -> Group OWNER)
+    if group_owner_relationships:
+        load_matchlinks(
+            neo4j_session,
+            GSuiteGroupToGroupOwnerRel(),
+            group_owner_relationships,
+            lastupdated=gsuite_update_tag,
+            _sub_resource_label="GSuiteTenant",
+            _sub_resource_id=customer_id,
+        )
+
+
+@timeit
+def cleanup_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+    customer_id: str,
+    gsuite_update_tag: int,
+) -> None:
+    """
+    Clean up GSuite groups and group-to-group relationships using the modern data model
+    """
+    logger.debug("Running GSuite groups cleanup job")
+
+    # Cleanup group nodes
+    GraphJob.from_node_schema(GSuiteGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+    # Cleanup group-to-group member relationships
+    GraphJob.from_matchlink(
+        GSuiteGroupToGroupMemberRel(),
+        "GSuiteTenant",
+        customer_id,
+        gsuite_update_tag,
+    ).run(neo4j_session)
+
+    # Cleanup group-to-group owner relationships
+    GraphJob.from_matchlink(
+        GSuiteGroupToGroupOwnerRel(),
+        "GSuiteTenant",
+        customer_id,
+        gsuite_update_tag,
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_gsuite_groups(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    gsuite_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    GET GSuite group objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: Nothing
+    """
+    logger.debug("Syncing GSuite Groups")
+
+    customer_id = common_job_parameters.get(
+        "CUSTOMER_ID", "my_customer"
+    )  # Default to "my_customer" for backward compatibility
+
+    # 1. GET - Fetch data from API
+    resp_objs = get_all_groups(admin, customer_id)
+    group_members = get_members_for_groups(admin, [resp["email"] for resp in resp_objs])
+
+    # 2. TRANSFORM - Shape data for ingestion
+    groups, group_member_relationships, group_owner_relationships = transform_groups(
+        resp_objs, group_members
+    )
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_gsuite_groups(neo4j_session, groups, customer_id, gsuite_update_tag)
+
+    # Load group-to-group relationships after groups are loaded
+    load_gsuite_group_to_group_relationships(
+        neo4j_session,
+        group_member_relationships,
+        group_owner_relationships,
+        customer_id,
+        gsuite_update_tag,
+    )
+
+    # 4. CLEANUP - Remove stale data
+    cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
+    cleanup_gsuite_groups(neo4j_session, cleanup_params, customer_id, gsuite_update_tag)