cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/googleworkspace/oauth_apps.py

@@ -0,0 +1,259 @@
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
+
+from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.models.googleworkspace.oauth_app import GoogleWorkspaceOAuthAppSchema
+from cartography.models.googleworkspace.oauth_app import (
+    GoogleWorkspaceUserToOAuthAppRel,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_oauth_tokens_for_user(admin: Resource, user_id: str) -> list[dict]:
+    """
+    Get OAuth tokens for a specific user
+    https://developers.google.com/workspace/admin/directory/reference/rest/v1/tokens/list
+
+    :param admin: apiclient discovery resource object
+    :param user_id: User's ID
+    :return: list of OAuth tokens for the user
+    """
+    try:
+        request = admin.tokens().list(userKey=user_id)
+        resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+        tokens = resp.get("items", [])
+
+        # Add user_id to each token for relationship mapping
+        for token in tokens:
+            token["user_id"] = user_id
+
+        return tokens
+    except HttpError as e:
+        if (
+            e.resp.status == 403
+            and "Request had insufficient authentication scopes" in str(e)
+        ):
+            logger.error(
+                "Missing required Google Workspace scopes. If using the gcloud CLI, "
+                "run: gcloud auth application-default login --scopes="
+                "https://www.googleapis.com/auth/admin.directory.customer.readonly,"
+                "https://www.googleapis.com/auth/admin.directory.user.readonly,"
+                "https://www.googleapis.com/auth/admin.directory.user.security,"
+                "https://www.googleapis.com/auth/cloud-identity.devices.readonly,"
+                "https://www.googleapis.com/auth/cloud-identity.groups.readonly,"
+                "https://www.googleapis.com/auth/cloud-platform"
+            )
+        elif e.resp.status == 404:
+            # User has no OAuth tokens, this is normal
+            return []
+        else:
+            logger.warning(f"Error fetching OAuth tokens for user: {e}")
+        return []
+
+
+@timeit
+def get_all_oauth_tokens(admin: Resource, user_ids: list[str]) -> list[dict]:
+    """
+    Get OAuth tokens for all users in the organization
+
+    :param admin: apiclient discovery resource object
+    :param user_ids: List of user IDs
+    :return: list of all OAuth tokens across all users
+    """
+    all_tokens = []
+
+    for user_id in user_ids:
+        tokens = get_oauth_tokens_for_user(admin, user_id)
+        all_tokens.extend(tokens)
+
+    logger.debug(f"Retrieved {len(all_tokens)} OAuth tokens for {len(user_ids)} users")
+    return all_tokens
+
+
+@timeit
+def transform_oauth_apps_and_authorizations(
+    tokens: list[dict],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    """
+    Transform OAuth token objects to create app nodes and user authorization relationships
+
+    :param tokens: Raw API response token objects
+    :return: tuple of (apps, authorizations)
+        - apps: list of unique OAuth app dictionaries
+        - authorizations: list of user->app authorization relationships with scopes
+    """
+    # Group tokens by client_id to create unique apps
+    apps_map: dict[str, dict[str, Any]] = {}
+    authorizations: list[dict[str, Any]] = []
+
+    for token in tokens:
+        client_id = token.get("clientId")
+        user_id = token.get("user_id")
+        scopes = token.get("scopes", [])
+
+        if not client_id or not user_id:
+            logger.warning("Skipping token due to missing client_id or user_id")
+            continue
+
+        # Create or update app entry
+        if client_id not in apps_map:
+            apps_map[client_id] = {
+                "client_id": client_id,
+                "display_text": token.get("displayText"),
+                "anonymous": token.get("anonymous", False),
+                "native_app": token.get("nativeApp", False),
+            }
+
+        # Create authorization relationship
+        authorizations.append(
+            {
+                "user_id": user_id,
+                "client_id": client_id,
+                "scopes": scopes,
+            }
+        )
+
+    apps = list(apps_map.values())
+    logger.info(
+        f"Transformed {len(apps)} unique OAuth apps with {len(authorizations)} authorizations"
+    )
+    return apps, authorizations
+
+
+@timeit
+def load_googleworkspace_oauth_apps(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    googleworkspace_update_tag: int,
+    customer_id: str,
+) -> None:
+    """
+    Load Google Workspace OAuth apps
+    """
+    logger.info(
+        "Ingesting %s Google Workspace OAuth apps for customer %s",
+        len(data),
+        customer_id,
+    )
+    load(
+        neo4j_session,
+        GoogleWorkspaceOAuthAppSchema(),
+        data,
+        lastupdated=googleworkspace_update_tag,
+        CUSTOMER_ID=customer_id,
+    )
+
+
+@timeit
+def load_user_to_app_authorizations(
+    neo4j_session: neo4j.Session,
+    authorizations: list[dict[str, Any]],
+    googleworkspace_update_tag: int,
+    customer_id: str,
+) -> None:
+    """
+    Load user to OAuth app authorization relationships using MatchLinks
+    """
+    logger.info(
+        "Creating %s user to OAuth app authorization relationships",
+        len(authorizations),
+    )
+
+    load_matchlinks(
+        neo4j_session,
+        GoogleWorkspaceUserToOAuthAppRel(),
+        authorizations,
+        lastupdated=googleworkspace_update_tag,
+        _sub_resource_label="GoogleWorkspaceTenant",
+        _sub_resource_id=customer_id,
+    )
+
+
+@timeit
+def cleanup_googleworkspace_oauth_apps(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Clean up Google Workspace OAuth apps
+    """
+    logger.debug("Running Google Workspace OAuth apps cleanup job")
+    GraphJob.from_node_schema(
+        GoogleWorkspaceOAuthAppSchema(), common_job_parameters
+    ).run(neo4j_session)
+
+
+@timeit
+def cleanup_user_to_app_authorizations(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Clean up stale user to OAuth app authorization relationships using MatchLink cleanup
+    """
+    logger.debug("Running user to OAuth app authorization relationships cleanup")
+
+    GraphJob.from_matchlink(
+        GoogleWorkspaceUserToOAuthAppRel(),
+        "GoogleWorkspaceTenant",
+        common_job_parameters["CUSTOMER_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(neo4j_session)
+
+
+@timeit
+def sync_googleworkspace_oauth_apps(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    user_ids: list[str],
+    googleworkspace_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    GET Google Workspace OAuth app objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    :param user_ids: List of user IDs to fetch tokens for
+    :param googleworkspace_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: Nothing
+    """
+    logger.debug("Syncing Google Workspace OAuth Apps")
+
+    # 1. GET - Fetch data from API
+    tokens = get_all_oauth_tokens(admin, user_ids)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    apps, authorizations = transform_oauth_apps_and_authorizations(tokens)
+
+    # 3. LOAD - Ingest apps to Neo4j using data model
+    load_googleworkspace_oauth_apps(
+        neo4j_session,
+        apps,
+        googleworkspace_update_tag,
+        common_job_parameters["CUSTOMER_ID"],
+    )
+
+    # 4. LOAD - Create user to app authorization relationships
+    load_user_to_app_authorizations(
+        neo4j_session,
+        authorizations,
+        googleworkspace_update_tag,
+        common_job_parameters["CUSTOMER_ID"],
+    )
+
+    # 5. CLEANUP - Remove stale data
+    cleanup_googleworkspace_oauth_apps(neo4j_session, common_job_parameters)
+    cleanup_user_to_app_authorizations(neo4j_session, common_job_parameters)

cartography/intel/googleworkspace/tenant.py

@@ -0,0 +1,85 @@
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
+
+from cartography.client.core.tx import load
+from cartography.models.googleworkspace.tenant import GoogleWorkspaceTenantSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_tenant(admin: Resource) -> dict[str, Any]:
+    """
+    Return the Google Workspace tenant information
+    https://developers.google.com/workspace/admin/directory/reference/rest/v1/customers/get
+
+    :param admin: apiclient discovery resource object
+    :return: Google Workspace tenant information
+    """
+    try:
+        req = admin.customers().get(
+            customerKey="my_customer",
+        )
+        resp = req.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+        return resp
+    except HttpError as e:
+        if (
+            e.resp.status == 403
+            and "Request had insufficient authentication scopes" in str(e)
+        ):
+            logger.error(
+                "Missing required Google Workspace scopes. If using the gcloud CLI, "
+                "run: gcloud auth application-default login --scopes="
+                "https://www.googleapis.com/auth/admin.directory.customer.readonly,"
+                "https://www.googleapis.com/auth/admin.directory.user.readonly,"
+                "https://www.googleapis.com/auth/admin.directory.user.security,"
+                "https://www.googleapis.com/auth/cloud-identity.devices.readonly,"
+                "https://www.googleapis.com/auth/cloud-identity.groups.readonly,"
+                "https://www.googleapis.com/auth/cloud-platform"
+            )
+        raise
+
+
+@timeit
+def load_googleworkspace_tenant(
+    neo4j_session: neo4j.Session,
+    tenant_data: dict[str, Any],
+    googleworkspace_update_tag: int,
+) -> None:
+    """
+    Load Google Workspace tenant
+    """
+    logger.info("Ingesting %s Google Workspace tenant", tenant_data["id"])
+    load(
+        neo4j_session,
+        GoogleWorkspaceTenantSchema(),
+        [tenant_data],
+        lastupdated=googleworkspace_update_tag,
+    )
+
+
+@timeit
+def sync_googleworkspace_tenant(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    googleworkspace_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> list[str]:
+    """Sync Google Workspace tenant data."""
+    logger.debug("Syncing Google Workspace Tenant data")
+
+    # GET - Fetch data from API
+    resp_obj = get_tenant(admin)
+
+    # LOAD - Ingest to Neo4j using data model
+    load_googleworkspace_tenant(neo4j_session, resp_obj, googleworkspace_update_tag)
+
+    # Return the customer ID
+    return resp_obj["id"]

cartography/intel/googleworkspace/users.py

@@ -0,0 +1,138 @@
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.googleworkspace.user import GoogleWorkspaceUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_all_users(admin: Resource) -> list[dict]:
+    """
+    Return list of Google Users in your organization
+    Returns empty list if we are unable to enumerate the users for any reasons
+    https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
+
+    :param admin: apiclient discovery resource object
+    :return: list of Google users in domain
+    see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
+    """
+    request = admin.users().list(
+        customer="my_customer",
+        maxResults=500,
+        orderBy="email",
+    )
+    response_objects = []
+    while request is not None:
+        resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+        response_objects.append(resp)
+        request = admin.users().list_next(request, resp)
+    return response_objects
+
+
+@timeit
+def transform_users(response_objects: list[dict]) -> list[dict[str, Any]]:
+    """Transform list of API response objects to return list of user objects with flattened structure grouped by customerId
+    :param response_objects: Raw API response objects
+    :return: list of dictionary objects for data model consumption
+    """
+    results = []
+    for response_object in response_objects:
+        for user in response_object["users"]:
+            # Flatten the nested name structure
+            transformed_user = user.copy()
+            if "name" in user and isinstance(user["name"], dict):
+                transformed_user["name"] = user["name"].get("fullName")
+                transformed_user["family_name"] = user["name"].get("familyName")
+                transformed_user["given_name"] = user["name"].get("givenName")
+            for org in user.get("organizations", []):
+                if org.get("primary"):
+                    transformed_user["organization_name"] = org.get("name")
+                    transformed_user["organization_title"] = org.get("title")
+                    transformed_user["organization_department"] = org.get("department")
+            results.append(transformed_user)
+    return results
+
+
+@timeit
+def load_googleworkspace_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    googleworkspace_update_tag: int,
+    customer_id: str,
+) -> None:
+    """
+    Load Google Workspace users
+    """
+    logger.info(
+        "Ingesting %s Google Workspace users for customer %s", len(data), customer_id
+    )
+    # Load users with relationship to tenant
+    load(
+        neo4j_session,
+        GoogleWorkspaceUserSchema(),
+        data,
+        lastupdated=googleworkspace_update_tag,
+        CUSTOMER_ID=customer_id,
+    )
+
+
+@timeit
+def cleanup_googleworkspace_users(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Clean up Google Workspace users
+    """
+    logger.debug("Running Google Workspace users cleanup job")
+    GraphJob.from_node_schema(GoogleWorkspaceUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_googleworkspace_users(
+    neo4j_session: neo4j.Session,
+    admin: Resource,
+    googleworkspace_update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> list[str]:
+    """
+    GET Google Workspace user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
+
+    :param neo4j_session: The Neo4j session
+    :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
+    See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
+    :param googleworkspace_update_tag: The timestamp value to set our new Neo4j nodes with
+    :param common_job_parameters: Parameters to carry to the Neo4j jobs
+    :return: List of user IDs
+    """
+    logger.debug("Syncing Google Workspace Users")
+
+    # 1. GET - Fetch data from API
+    resp_objs = get_all_users(admin)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    raw_users = transform_users(resp_objs)
+
+    # 3. LOAD - Ingest to Neo4j using data model
+    load_googleworkspace_users(
+        neo4j_session,
+        raw_users,
+        googleworkspace_update_tag,
+        common_job_parameters["CUSTOMER_ID"],
+    )
+
+    # 4. CLEANUP - Remove stale data
+    cleanup_googleworkspace_users(neo4j_session, common_job_parameters)
+
+    return [user["id"] for user in raw_users]

cartography/intel/gsuite/__init__.py

@@ -5,42 +5,56 @@ import os
 from collections import namedtuple
 
 import googleapiclient.discovery
-import httplib2
 import neo4j
+from google.auth import default
+from google.auth.exceptions import DefaultCredentialsError
+from google.auth.transport.requests import Request
+from google.oauth2 import credentials
+from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials as OAuth2Credentials
+from google.oauth2.service_account import Credentials as ServiceAccountCredentials
 from googleapiclient.discovery import Resource
-from oauth2client.client import ApplicationDefaultCredentialsError
-from oauth2client.client import GoogleCredentials
 
 from cartography.config import Config
-from cartography.intel.gsuite import api
+from cartography.intel.gsuite import groups
+from cartography.intel.gsuite import users
 from cartography.util import timeit
 
-OAUTH_SCOPE = [
-    'https://www.googleapis.com/auth/admin.directory.user.readonly',
-    'https://www.googleapis.com/auth/admin.directory.group.readonly',
-    'https://www.googleapis.com/auth/admin.directory.group.member',
+OAUTH_SCOPES = [
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+    "https://www.googleapis.com/auth/admin.directory.group.readonly",
+    "https://www.googleapis.com/auth/admin.directory.group.member",
 ]
 
 logger = logging.getLogger(__name__)
 
-Resources = namedtuple('Resources', 'admin')
+Resources = namedtuple("Resources", "admin")
 
 
-def _get_admin_resource(credentials: GoogleCredentials) -> Resource:
+def _get_admin_resource(
+    credentials: OAuth2Credentials | ServiceAccountCredentials,
+) -> Resource:
     """
     Instantiates a Google API resource object to call the Google API.
     Used to pull users and groups.  See https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
 
-    :param credentials: The GoogleCredentials object
+    :param credentials: The credentials object
     :return: An admin api resource object
     """
-    return googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
+    return googleapiclient.discovery.build(
+        "admin",
+        "directory_v1",
+        credentials=credentials,
+        cache_discovery=False,
+    )
 
 
-def _initialize_resources(credentials: GoogleCredentials) -> Resources:
+def _initialize_resources(
+    credentials: OAuth2Credentials | ServiceAccountCredentials,
+) -> Resources:
     """
     Create namedtuple of all resource objects necessary for Google API data gathering.
-    :param credentials: The GoogleCredentials object
+    :param credentials: The credentials object
     :return: namedtuple of all resource objects
     """
     return Resources(
@@ -61,51 +75,96 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
         "UPDATE_TAG": config.update_tag,
     }
 
-    if config.gsuite_auth_method == 'delegated':  # Legacy delegated method
-        logger.info('Attempting to authenticate to GSuite using legacy delegated method')
+    creds: OAuth2Credentials | ServiceAccountCredentials
+    if config.gsuite_auth_method == "delegated":  # Legacy delegated method
+        if config.gsuite_config is None or not os.path.isfile(config.gsuite_config):
+            logger.warning(
+                (
+                    "The GSuite config file is not set or is not a valid file."
+                    "Skipping GSuite ingestion."
+                ),
+            )
+            return
+        logger.info(
+            "Attempting to authenticate to GSuite using legacy delegated method"
+        )
         try:
-            credentials = GoogleCredentials.from_stream(config.gsuite_config)
-            credentials = credentials.create_scoped(OAUTH_SCOPE)
-            credentials = credentials.create_delegated(os.environ.get('GSUITE_DELEGATED_ADMIN'))
+            creds = service_account.Credentials.from_service_account_file(
+                config.gsuite_config,
+                scopes=OAUTH_SCOPES,
+            )
+            creds = creds.with_subject(os.environ.get("GSUITE_DELEGATED_ADMIN"))
 
-        except ApplicationDefaultCredentialsError as e:
+        except DefaultCredentialsError as e:
             logger.error(
                 (
                     "Unable to initialize GSuite creds. If you don't have GSuite data or don't want to load "
-                    'Gsuite data then you can ignore this message. Otherwise, the error code is: %s '
-                    'Make sure your GSuite credentials file (if any) is valid. '
-                    'For more details see README'
+                    "Gsuite data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure your GSuite credentials file (if any) is valid. "
+                    "For more details see README"
                 ),
                 e,
             )
             return
-    elif config.gsuite_auth_method == 'oauth':
+    elif config.gsuite_auth_method == "oauth":
         auth_tokens = json.loads(str(base64.b64decode(config.gsuite_config).decode()))
-        logger.info('Attempting to authenticate to GSuite using OAuth')
+        logger.info("Attempting to authenticate to GSuite using OAuth")
         try:
-            credentials = GoogleCredentials(
-                None,
-                auth_tokens['client_id'],
-                auth_tokens['client_secret'],
-                auth_tokens['refresh_token'],
-                None,
-                auth_tokens['token_uri'],
-                'Cartography',
+            creds = credentials.Credentials(
+                token=None,
+                client_id=auth_tokens["client_id"],
+                client_secret=auth_tokens["client_secret"],
+                refresh_token=auth_tokens["refresh_token"],
+                expiry=None,
+                token_uri=auth_tokens["token_uri"],
+                scopes=OAUTH_SCOPES,
             )
-            credentials.refresh(httplib2.Http())
-            credentials = credentials.create_scoped(OAUTH_SCOPE)
-        except ApplicationDefaultCredentialsError as e:
+            creds.refresh(Request())
+        except DefaultCredentialsError as e:
             logger.error(
                 (
                     "Unable to initialize GSuite creds. If you don't have GSuite data or don't want to load "
-                    'Gsuite data then you can ignore this message. Otherwise, the error code is: %s '
-                    'Make sure your GSuite credentials are configured correctly, your credentials are valid. '
-                    'For more details see README'
+                    "Gsuite data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure your GSuite credentials are configured correctly, your credentials are valid. "
+                    "For more details see README"
+                ),
+                e,
+            )
+            return
+    elif config.gsuite_auth_method == "default":
+        logger.info("Attempting to authenticate to GSuite using default credentials")
+        try:
+            creds, _ = default(scopes=OAUTH_SCOPES)
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize GSuite creds using default credentials. If you don't have GSuite data or "
+                    "don't want to load GSuite data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure you have valid application default credentials configured. "
+                    "For more details see README"
                 ),
                 e,
             )
             return
 
-    resources = _initialize_resources(credentials)
-    api.sync_gsuite_users(neo4j_session, resources.admin, config.update_tag, common_job_parameters)
-    api.sync_gsuite_groups(neo4j_session, resources.admin, config.update_tag, common_job_parameters)
+    logger.warning(
+        "The GSuite module is deprecated and will no longer receive updates. It will be completely removed in the 1.0.0 release. "
+        "For migration, please refer to the Google Workspace module configuration."
+    )
+
+    resources = _initialize_resources(creds)
+    customer_ids = users.sync_gsuite_users(
+        neo4j_session,
+        resources.admin,
+        config.update_tag,
+        common_job_parameters,
+    )
+    for customer_id in customer_ids:
+        scoped_job_parameters = common_job_parameters.copy()
+        scoped_job_parameters["CUSTOMER_ID"] = customer_id
+        groups.sync_gsuite_groups(
+            neo4j_session,
+            resources.admin,
+            config.update_tag,
+            scoped_job_parameters,
+        )