cartography 0.100.0rc1__tar.gz → 0.100.0rc3__tar.gz

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/.github/workflows/ossf-scorecard.yml

@@ -30,7 +30,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -49,7 +49,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: SARIF file
           path: results.sarif
@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9
+        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         with:
           sarif_file: results.sarif

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/.github/workflows/publish-to-ghcr-and-pypi.yml

@@ -53,12 +53,12 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/${{ github.repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
@@ -83,7 +83,7 @@ jobs:
           sleep 10
 
       - name: Build and push
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           file: Dockerfile
           context: .

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/.github/workflows/sphinx.yml

@@ -33,7 +33,7 @@ jobs:
       # https://docs.github.com/en/actions/guides/building-and-testing-python#caching-dependencies
       # ^-- How to set up caching for pip on Ubuntu
       - name: Cache pip
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/.github/workflows/test_suite.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           python-version: "3.10"
       # Cache our pip dir for efficiency; see https://medium.com/ai2-blog/python-caching-in-github-actions-e9452698e98d.
-      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/pip
           key: ${{ hashFiles('setup.py') }}-${{ hashFiles('pyproject.toml') }}
@@ -74,7 +74,7 @@ jobs:
         with:
           python-version: "3.10"
       # Cache our pip dir for efficiency; see https://medium.com/ai2-blog/python-caching-in-github-actions-e9452698e98d.
-      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+      - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/pip
           key: ${{ hashFiles('setup.py') }}-${{ hashFiles('pyproject.toml') }}
@@ -97,15 +97,15 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/${{ github.repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           file: Dockerfile
           push: false # only build the image, don't push it anywhere

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/Dockerfile

@@ -1,6 +1,6 @@
 # This is a thin distribution of the cartography software.
 # It is published at ghcr.io.
-FROM python:3.10-slim@sha256:66aad90b231f011cb80e1966e03526a7175f0586724981969b23903abac19081
+FROM python:3.10-slim@sha256:f680fc3f447366d9be2ae53dc7a6447fe9b33311af209225783932704f0cb4e7
 
 # Default to ''. Overridden with a specific version specifier e.g. '==0.98.0' by build args or from GitHub actions.
 ARG VERSION_SPECIFIER
@@ -14,7 +14,7 @@ WORKDIR /var/cartography
 ENV HOME=/var/cartography
 
 # Install cartography at the given version specifier. Can be ''.
-RUN pip install cartography${VERSION_SPECIFIER}
+RUN pip install --no-cache-dir cartography${VERSION_SPECIFIER}
 
 USER ${uid}:${gid}
 

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cartography
-Version: 0.100.0rc1
+Version: 0.100.0rc3
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2
@@ -59,10 +59,10 @@ Requires-Dist: moto; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: pytest>=6.2.4; extra == "dev"
 Requires-Dist: pytest-mock; extra == "dev"
-Requires-Dist: pytest-cov==2.10.0; extra == "dev"
+Requires-Dist: pytest-cov==6.0.0; extra == "dev"
 Requires-Dist: pytest-rerunfailures; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: types-requests<2.32.0.20241017; extra == "dev"
+Requires-Dist: types-requests<2.32.0.20250302; extra == "dev"
 
 ![Cartography](docs/root/images/logo-horizontal.png)
 

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.100.0rc1'
+__version__ = version = '0.100.0rc3'
 __version_tuple__ = version_tuple = (0, 100, 0)

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/cli.py

@@ -439,9 +439,10 @@ class CLI:
             '--gsuite-auth-method',
             type=str,
             default='delegated',
-            choices=['delegated', 'oauth'],
+            choices=['delegated', 'oauth', 'default'],
             help=(
-                'The method used by GSuite to authenticate. delegated is the legacy one.'
+                'GSuite authentication method. Can be "delegated" for service account or "oauth" for OAuth. '
+                '"Default" best if using gcloud CLI.'
             ),
         )
         parser.add_argument(

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/intel/aws/ecr.py

@@ -33,10 +33,21 @@ def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> L
 def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]:
     logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region)
     client = boto3_session.client('ecr', region_name=region)
-    paginator = client.get_paginator('list_images')
+    list_paginator = client.get_paginator('list_images')
     ecr_repository_images: List[Dict] = []
-    for page in paginator.paginate(repositoryName=repository_name):
-        ecr_repository_images.extend(page['imageIds'])
+    for page in list_paginator.paginate(repositoryName=repository_name):
+        image_ids = page['imageIds']
+        if not image_ids:
+            continue
+        describe_paginator = client.get_paginator('describe_images')
+        describe_response = describe_paginator.paginate(repositoryName=repository_name, imageIds=image_ids)
+        for response in describe_response:
+            image_details = response['imageDetails']
+            image_details = [
+                {**detail, 'imageTag': detail['imageTags'][0]} if detail.get('imageTags') else detail
+                for detail in image_details
+            ]
+            ecr_repository_images.extend(image_details)
     return ecr_repository_images
 
 
@@ -103,7 +114,12 @@ def _load_ecr_repo_img_tx(
         ON CREATE SET ri.firstseen = timestamp()
         SET ri.lastupdated = $aws_update_tag,
             ri.tag = repo_img.imageTag,
-            ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')
+            ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, ''),
+            ri.image_size_bytes = repo_img.imageSizeInBytes,
+            ri.image_pushed_at = repo_img.imagePushedAt,
+            ri.image_manifest_media_type = repo_img.imageManifestMediaType,
+            ri.artifact_media_type = repo_img.artifactMediaType,
+            ri.last_recorded_pull_time = repo_img.lastRecordedPullTime
         WITH ri, repo_img
 
         MERGE (img:ECRImage{id: repo_img.imageDigest})

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/intel/cve/feed.py

@@ -212,19 +212,19 @@ def transform_cves(cve_json: Dict[Any, Any]) -> List[Dict[Any, Any]]:
             if cvss31:
                 cvss31.update(cvss31["cvssData"])
                 cvss31.pop("cvssData")
-                cve["vectorString"] = cvss31["vectorString"]
-                cve["attackVector"] = cvss31["attackVector"]
-                cve["attackComplexity"] = cvss31["attackComplexity"]
-                cve["privilegesRequired"] = cvss31["privilegesRequired"]
-                cve["userInteraction"] = cvss31["userInteraction"]
-                cve["scope"] = cvss31["scope"]
-                cve["confidentialityImpact"] = cvss31["confidentialityImpact"]
-                cve["integrityImpact"] = cvss31["integrityImpact"]
-                cve["availabilityImpact"] = cvss31["availabilityImpact"]
-                cve["baseScore"] = cvss31["baseScore"]
-                cve["baseSeverity"] = cvss31["baseSeverity"]
-                cve["exploitabilityScore"] = cvss31["exploitabilityScore"]
-                cve["impactScore"] = cvss31["impactScore"]
+                cve["vectorString"] = cvss31.get("vectorString")
+                cve["attackVector"] = cvss31.get("attackVector")
+                cve["attackComplexity"] = cvss31.get("attackComplexity")
+                cve["privilegesRequired"] = cvss31.get("privilegesRequired")
+                cve["userInteraction"] = cvss31.get("userInteraction")
+                cve["scope"] = cvss31.get("scope")
+                cve["confidentialityImpact"] = cvss31.get("confidentialityImpact")
+                cve["integrityImpact"] = cvss31.get("integrityImpact")
+                cve["availabilityImpact"] = cvss31.get("availabilityImpact")
+                cve["baseScore"] = cvss31.get("baseScore")
+                cve["baseSeverity"] = cvss31.get("baseSeverity")
+                cve["exploitabilityScore"] = cvss31.get("exploitabilityScore")
+                cve["impactScore"] = cvss31.get("impactScore")
         except Exception:
             logger.error("Failed to transform CVE data {data}")
             raise

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/intel/gcp/__init__.py

@@ -17,21 +17,23 @@ from cartography.intel.gcp import compute
 from cartography.intel.gcp import crm
 from cartography.intel.gcp import dns
 from cartography.intel.gcp import gke
+from cartography.intel.gcp import iam
 from cartography.intel.gcp import storage
 from cartography.util import run_analysis_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
-Resources = namedtuple('Resources', 'compute container crm_v1 crm_v2 dns storage serviceusage')
+Resources = namedtuple('Resources', 'compute container crm_v1 crm_v2 dns storage serviceusage iam')
 
 # Mapping of service short names to their full names as in docs. See https://developers.google.com/apis-explorer,
 # and https://cloud.google.com/service-usage/docs/reference/rest/v1/services#ServiceConfig
-Services = namedtuple('Services', 'compute storage gke dns')
+Services = namedtuple('Services', 'compute storage gke dns iam')
 service_names = Services(
     compute='compute.googleapis.com',
     storage='storage.googleapis.com',
     gke='container.googleapis.com',
     dns='dns.googleapis.com',
+    iam='iam.googleapis.com',
 )
 
 
@@ -112,6 +114,13 @@ def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
     return googleapiclient.discovery.build('serviceusage', 'v1', credentials=credentials, cache_discovery=False)
 
 
+def _get_iam_resource(credentials: GoogleCredentials) -> Resource:
+    """
+    Instantiates a Google IAM resource object to call the IAM API.
+    """
+    return googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False)
+
+
 def _initialize_resources(credentials: GoogleCredentials) -> Resource:
     """
     Create namedtuple of all resource objects necessary for GCP data gathering.
@@ -126,6 +135,7 @@ def _initialize_resources(credentials: GoogleCredentials) -> Resource:
         container=None,
         dns=None,
         storage=None,
+        iam=_get_iam_resource(credentials),
     )
 
 
@@ -286,6 +296,18 @@ def _sync_multiple_projects(
         logger.info("Syncing GCP project %s for DNS", project_id)
         _sync_single_project_dns(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
 
+    # IAM data sync
+    for project in projects:
+        project_id = project['projectId']
+        logger.info("Syncing GCP project %s for IAM", project_id)
+        iam.sync(
+            neo4j_session,
+            resources.iam,
+            project_id,
+            gcp_update_tag,
+            common_job_parameters,
+        )
+
 
 @timeit
 def get_gcp_credentials() -> GoogleCredentials:

cartography-0.100.0rc3/cartography/intel/gcp/iam.py

@@ -0,0 +1,222 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.iam import GCPRoleSchema
+from cartography.models.gcp.iam import GCPServiceAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+# GCP API can be subject to rate limiting, so add small delays between calls
+LIST_SLEEP = 1
+DESCRIBE_SLEEP = 1
+
+
+@timeit
+def get_gcp_service_accounts(iam_client: Resource, project_id: str) -> List[Dict[str, Any]]:
+    """
+    Retrieve a list of GCP service accounts for a given project.
+
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :param project_id: The GCP Project ID to retrieve service accounts from.
+    :return: A list of dictionaries representing GCP service accounts.
+    """
+    service_accounts: List[Dict[str, Any]] = []
+    try:
+        request = iam_client.projects().serviceAccounts().list(
+            name=f'projects/{project_id}',
+        )
+        while request is not None:
+            response = request.execute()
+            if 'accounts' in response:
+                service_accounts.extend(response['accounts'])
+            request = iam_client.projects().serviceAccounts().list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+    except Exception as e:
+        logger.warning(f"Error retrieving service accounts for project {project_id}: {e}")
+    return service_accounts
+
+
+@timeit
+def get_gcp_roles(iam_client: Resource, project_id: str) -> List[Dict]:
+    """
+    Retrieve custom and predefined roles from GCP for a given project.
+
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :param project_id: The GCP Project ID to retrieve roles from.
+    :return: A list of dictionaries representing GCP roles.
+    """
+    try:
+        roles = []
+
+        # Get custom roles
+        custom_req = iam_client.projects().roles().list(parent=f'projects/{project_id}')
+        while custom_req is not None:
+            resp = custom_req.execute()
+            roles.extend(resp.get('roles', []))
+            custom_req = iam_client.projects().roles().list_next(custom_req, resp)
+
+        # Get predefined roles
+        predefined_req = iam_client.roles().list(view='FULL')
+        while predefined_req is not None:
+            resp = predefined_req.execute()
+            roles.extend(resp.get('roles', []))
+            predefined_req = iam_client.roles().list_next(predefined_req, resp)
+
+        return roles
+    except Exception as e:
+        logger.warning(f"Error getting GCP roles - {e}")
+        return []
+
+
+@timeit
+def load_gcp_service_accounts(
+    neo4j_session: neo4j.Session,
+    service_accounts: List[Dict[str, Any]],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """
+    Load GCP service account data into Neo4j.
+
+    :param neo4j_session: The Neo4j session.
+    :param service_accounts: A list of service account data to load.
+    :param project_id: The GCP Project ID associated with the service accounts.
+    :param gcp_update_tag: The timestamp of the current sync run.
+    """
+    logger.debug(f"Loading {len(service_accounts)} service accounts for project {project_id}")
+    transformed_service_accounts = []
+    for sa in service_accounts:
+        transformed_sa = {
+            'id': sa['uniqueId'],
+            'email': sa.get('email'),
+            'displayName': sa.get('displayName'),
+            'oauth2ClientId': sa.get('oauth2ClientId'),
+            'uniqueId': sa.get('uniqueId'),
+            'disabled': sa.get('disabled', False),
+            'projectId': project_id,
+        }
+        transformed_service_accounts.append(transformed_sa)
+
+    load(
+        neo4j_session,
+        GCPServiceAccountSchema(),
+        transformed_service_accounts,
+        lastupdated=gcp_update_tag,
+        projectId=project_id,
+        additional_labels=['GCPPrincipal'],
+    )
+
+
+@timeit
+def load_gcp_roles(
+    neo4j_session: neo4j.Session,
+    roles: List[Dict[str, Any]],
+    project_id: str,
+    gcp_update_tag: int,
+) -> None:
+    """
+    Load GCP role data into Neo4j.
+
+    :param neo4j_session: The Neo4j session.
+    :param roles: A list of role data to load.
+    :param project_id: The GCP Project ID associated with the roles.
+    :param gcp_update_tag: The timestamp of the current sync run.
+    """
+    logger.debug(f"Loading {len(roles)} roles for project {project_id}")
+    transformed_roles = []
+    for role in roles:
+        role_name = role['name']
+        if role_name.startswith('roles/'):
+            if role_name in ['roles/owner', 'roles/editor', 'roles/viewer']:
+                role_type = 'BASIC'
+            else:
+                role_type = 'PREDEFINED'
+        else:
+            role_type = 'CUSTOM'
+
+        transformed_role = {
+            'id': role_name,
+            'name': role_name,
+            'title': role.get('title'),
+            'description': role.get('description'),
+            'deleted': role.get('deleted', False),
+            'etag': role.get('etag'),
+            'includedPermissions': role.get('includedPermissions', []),
+            'roleType': role_type,
+            'projectId': project_id,
+        }
+        transformed_roles.append(transformed_role)
+
+    load(
+        neo4j_session,
+        GCPRoleSchema(),
+        transformed_roles,
+        lastupdated=gcp_update_tag,
+        projectId=project_id,
+    )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    """
+    Run cleanup jobs for GCP IAM data in Neo4j.
+
+    :param neo4j_session: The Neo4j session.
+    :param common_job_parameters: Common job parameters for cleanup.
+    """
+    logger.debug("Running GCP IAM cleanup job")
+    job_params = {
+        **common_job_parameters,
+        'projectId': common_job_parameters.get('PROJECT_ID'),
+    }
+
+    cleanup_jobs = [
+        GraphJob.from_node_schema(GCPServiceAccountSchema(), job_params),
+        GraphJob.from_node_schema(GCPRoleSchema(), job_params),
+    ]
+
+    for cleanup_job in cleanup_jobs:
+        cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    iam_client: Resource,
+    project_id: str,
+    gcp_update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    """
+    Sync GCP IAM resources for a given project.
+
+    :param neo4j_session: The Neo4j session.
+    :param iam_client: The IAM resource object created by googleapiclient.discovery.build().
+    :param project_id: The GCP Project ID to sync.
+    :param gcp_update_tag: The timestamp of the current sync run.
+    :param common_job_parameters: Common job parameters for the sync.
+    """
+    logger.info(f"Syncing GCP IAM for project {project_id}")
+
+    # Get and load service accounts
+    service_accounts = get_gcp_service_accounts(iam_client, project_id)
+    logger.info(f"Found {len(service_accounts)} service accounts in project {project_id}")
+    load_gcp_service_accounts(neo4j_session, service_accounts, project_id, gcp_update_tag)
+
+    # Get and load roles
+    roles = get_gcp_roles(iam_client, project_id)
+    logger.info(f"Found {len(roles)} roles in project {project_id}")
+    load_gcp_roles(neo4j_session, roles, project_id, gcp_update_tag)
+
+    # Run cleanup
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/intel/gsuite/__init__.py

@@ -6,6 +6,7 @@ from collections import namedtuple
 
 import googleapiclient.discovery
 import neo4j
+from google.auth import default
 from google.auth.exceptions import DefaultCredentialsError
 from google.auth.transport.requests import Request
 from google.oauth2 import credentials
@@ -18,10 +19,11 @@ from cartography.config import Config
 from cartography.intel.gsuite import api
 from cartography.util import timeit
 
-OAUTH_SCOPE = [
+OAUTH_SCOPES = [
     'https://www.googleapis.com/auth/admin.directory.user.readonly',
     'https://www.googleapis.com/auth/admin.directory.group.readonly',
     'https://www.googleapis.com/auth/admin.directory.group.member',
+    'https://www.googleapis.com/auth/cloud-platform',
 ]
 
 logger = logging.getLogger(__name__)
@@ -70,7 +72,7 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
         try:
             creds = service_account.Credentials.from_service_account_file(
                 config.gsuite_config,
-                scopes=OAUTH_SCOPE,
+                scopes=OAUTH_SCOPES,
             )
             creds = creds.with_subject(os.environ.get('GSUITE_DELEGATED_ADMIN'))
 
@@ -96,10 +98,10 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
                 refresh_token=auth_tokens['refresh_token'],
                 expiry=None,
                 token_uri=auth_tokens['token_uri'],
-                scopes=OAUTH_SCOPE,
+                scopes=OAUTH_SCOPES,
             )
             creds.refresh(Request())
-            creds = creds.create_scoped(OAUTH_SCOPE)
+            creds = creds.create_scoped(OAUTH_SCOPES)
         except DefaultCredentialsError as e:
             logger.error(
                 (
@@ -111,6 +113,21 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
                 e,
             )
             return
+    elif config.gsuite_auth_method == 'default':
+        logger.info('Attempting to authenticate to GSuite using default credentials')
+        try:
+            creds, _ = default(scopes=OAUTH_SCOPES)
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize GSuite creds using default credentials. If you don't have GSuite data or "
+                    "don't want to load GSuite data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure you have valid application default credentials configured. "
+                    "For more details see README"
+                ),
+                e,
+            )
+            return
 
     resources = _initialize_resources(creds)
     api.sync_gsuite_users(neo4j_session, resources.admin, config.update_tag, common_job_parameters)

{cartography-0.100.0rc1 → cartography-0.100.0rc3}/cartography/intel/gsuite/api.py

@@ -4,6 +4,7 @@ from typing import List
 
 import neo4j
 from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
 
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
@@ -30,9 +31,21 @@ def get_all_groups(admin: Resource) -> List[Dict]:
     request = admin.groups().list(customer='my_customer', maxResults=20, orderBy='email')
     response_objects = []
     while request is not None:
-        resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
-        response_objects.append(resp)
-        request = admin.groups().list_next(request, resp)
+        try:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            response_objects.append(resp)
+            request = admin.groups().list_next(request, resp)
+        except HttpError as e:
+            if e.resp.status == 403 and "Request had insufficient authentication scopes" in str(e):
+                logger.error(
+                    "Missing required GSuite scopes. If using the gcloud CLI, ",
+                    "run: gcloud auth application-default login --scopes="
+                    '"https://www.googleapis.com/auth/admin.directory.user.readonly,'
+                    'https://www.googleapis.com/auth/admin.directory.group.readonly,'
+                    'https://www.googleapis.com/auth/admin.directory.group.member.readonly,'
+                    'https://www.googleapis.com/auth/cloud-platform"',
+                )
+            raise
     return response_objects
 
 
@@ -140,6 +153,7 @@ def load_gsuite_groups(neo4j_session: neo4j.Session, groups: List[Dict], gsuite_
         g.etag = group.etag,
         g.kind = group.kind,
         g.name = group.name,
+        g:GCPPrincipal,
         g.lastupdated = $UpdateTag
     """
     logger.info(f'Ingesting {len(groups)} gsuite groups')
@@ -179,6 +193,7 @@ def load_gsuite_users(neo4j_session: neo4j.Session, users: List[Dict], gsuite_up
         u.suspended = user.suspended,
         u.thumbnail_photo_etag = user.thumbnailPhotoEtag,
         u.thumbnail_photo_url = user.thumbnailPhotoUrl,
+        u:GCPPrincipal,
         u.lastupdated = $UpdateTag
     """
     logger.info(f'Ingesting {len(users)} gsuite users')