cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/github/teams.py

@@ -84,7 +84,7 @@ def _get_teams_repos_inner_func(
     repo_urls: list[str],
     repo_permissions: list[str],
 ) -> None:
-    logger.info(f"Loading team repos for {team_name}.")
+    logger.info(f"Retrieving team repos for {team_name}.")
     team_repos = _get_team_repos(org, api_url, token, team_name)
 
     # The `or []` is because `.nodes` can be None. See:
@@ -192,7 +192,7 @@ def _get_teams_users_inner_func(
     user_urls: List[str],
     user_roles: List[str],
 ) -> None:
-    logger.info(f"Loading team users for {team_name}.")
+    logger.info(f"Retrieving team users for {team_name}.")
     team_users = _get_team_users(org, api_url, token, team_name)
     # The `or []` is because `.nodes` can be None. See:
     # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
@@ -299,7 +299,7 @@ def _get_child_teams_inner_func(
     team_name: str,
     team_urls: List[str],
 ) -> None:
-    logger.info(f"Loading child teams for {team_name}.")
+    logger.info(f"Retrieving child teams for {team_name}.")
     child_teams = _get_child_teams(org, api_url, token, team_name)
     # The `or []` is because `.nodes` can be None. See:
     # https://docs.github.com/en/graphql/reference/objects#teammemberconnection

cartography/intel/github/users.py

@@ -37,6 +37,7 @@ GITHUB_ORG_USERS_PAGINATED_GRAPHQL = """
                         isSiteAdmin
                         email
                         company
+                        organizationVerifiedDomainEmails(login: $login)
                     }
                     role
                 }
@@ -64,6 +65,7 @@ GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL = """
                         isSiteAdmin
                         email
                         company
+                        organizationVerifiedDomainEmails(login: $login)
                     }
                     organizationRole
                 }
@@ -154,6 +156,9 @@ def transform_users(
     for user in user_data:
         # all members get the 'MEMBER_OF' relationship
         processed_user = deepcopy(user["node"])
+        # GitHub returns empty strings for email addresses, so we convert them to None
+        if "email" in processed_user and not processed_user["email"]:
+            processed_user["email"] = None
         processed_user["hasTwoFactorEnabled"] = user["hasTwoFactorEnabled"]
         processed_user["MEMBER_OF"] = org_data["url"]
         # admins get a second relationship expressing them as such

cartography/intel/github/util.py

@@ -157,6 +157,18 @@ def fetch_all(
             retry += 1
             exc = err
         except requests.exceptions.HTTPError as err:
+            if (
+                err.response is not None
+                and err.response.status_code == 502
+                and kwargs.get("count")
+                and kwargs["count"] > 1
+            ):
+                kwargs["count"] = max(1, kwargs["count"] // 2)
+                logger.warning(
+                    "GitHub: Received 502 response. Reducing page size to %s and retrying.",
+                    kwargs["count"],
+                )
+                continue
             retry += 1
             exc = err
         except requests.exceptions.ChunkedEncodingError as err:

cartography/intel/googleworkspace/__init__.py

@@ -0,0 +1,193 @@
+import base64
+import json
+import logging
+import os
+from collections import namedtuple
+
+import googleapiclient.discovery
+import neo4j
+from google.auth import default
+from google.auth.exceptions import DefaultCredentialsError
+from google.auth.transport.requests import Request
+from google.oauth2 import credentials
+from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials as OAuth2Credentials
+from google.oauth2.service_account import Credentials as ServiceAccountCredentials
+
+from cartography.config import Config
+from cartography.intel.googleworkspace import devices
+from cartography.intel.googleworkspace import groups
+from cartography.intel.googleworkspace import oauth_apps
+from cartography.intel.googleworkspace import tenant
+from cartography.intel.googleworkspace import users
+from cartography.util import timeit
+
+OAUTH_SCOPES = [
+    "https://www.googleapis.com/auth/admin.directory.customer.readonly",
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+    "https://www.googleapis.com/auth/admin.directory.user.security",
+    "https://www.googleapis.com/auth/cloud-identity.devices.readonly",
+    "https://www.googleapis.com/auth/cloud-identity.groups.readonly",
+]
+
+logger = logging.getLogger(__name__)
+
+Resources = namedtuple("Resources", ["admin", "cloudidentity"])
+
+
+def _initialize_resources(
+    creds: OAuth2Credentials | ServiceAccountCredentials,
+) -> Resources:
+    """
+    Create namedtuple of all resource objects necessary for Google API data gathering.
+    :param credentials: The credentials object
+    :return: namedtuple of all resource objects
+    """
+
+    return Resources(
+        googleapiclient.discovery.build(
+            "admin",
+            "directory_v1",
+            credentials=creds,
+            cache_discovery=False,
+        ),
+        googleapiclient.discovery.build(
+            "cloudidentity",
+            "v1",
+            credentials=creds,
+            cache_discovery=False,
+        ),
+    )
+
+
+@timeit
+def start_googleworkspace_ingestion(
+    neo4j_session: neo4j.Session, config: Config
+) -> None:
+    """
+    Starts the Google Workspace ingestion process by initializing
+
+    :param neo4j_session: The Neo4j session
+    :param config: A `cartography.config` object
+    :return: Nothing
+    """
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    creds: OAuth2Credentials | ServiceAccountCredentials
+    if config.googleworkspace_auth_method == "delegated":  # Legacy delegated method
+        if config.googleworkspace_config is None or not os.path.isfile(
+            config.googleworkspace_config
+        ):
+            logger.warning(
+                (
+                    "The Google Workspace config file is not set or is not a valid file."
+                    "Skipping Google Workspace ingestion."
+                ),
+            )
+            return
+        logger.info(
+            "Attempting to authenticate to Google Workspace using legacy delegated method"
+        )
+        try:
+            creds = service_account.Credentials.from_service_account_file(
+                config.googleworkspace_config,
+                scopes=OAUTH_SCOPES,
+            )
+            creds = creds.with_subject(os.environ.get("GOOGLE_DELEGATED_ADMIN"))
+
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize Google Workspace creds. If you don't have Google Workspace data or don't want to load "
+                    "Google Workspace data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure your Google Workspace credentials file (if any) is valid. "
+                    "For more details see documentation."
+                ),
+                e,
+            )
+            return
+    elif config.googleworkspace_auth_method == "oauth":
+        auth_tokens = json.loads(
+            str(base64.b64decode(config.googleworkspace_config).decode())
+        )
+        logger.info("Attempting to authenticate to Google Workspace using OAuth")
+        try:
+            creds = credentials.Credentials(
+                token=None,
+                client_id=auth_tokens["client_id"],
+                client_secret=auth_tokens["client_secret"],
+                refresh_token=auth_tokens["refresh_token"],
+                expiry=None,
+                token_uri=auth_tokens["token_uri"],
+                scopes=OAUTH_SCOPES,
+            )
+            creds.refresh(Request())
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize Google Workspace creds. If you don't have Google Workspace data or don't want to load "
+                    "Google Workspace data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure your Google Workspace credentials are configured correctly, your credentials are valid. "
+                    "For more details see documentation."
+                ),
+                e,
+            )
+            return
+    elif config.googleworkspace_auth_method == "default":
+        logger.info(
+            "Attempting to authenticate to Google Workspace using default credentials"
+        )
+        try:
+            creds, _ = default(scopes=OAUTH_SCOPES)
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize Google Workspace creds using default credentials. If you don't have Google Workspace data or "
+                    "don't want to load Google Workspace data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure you have valid application default credentials configured. "
+                    "For more details see documentation."
+                ),
+                e,
+            )
+            return
+
+    resources = _initialize_resources(creds)
+    customer_id = tenant.sync_googleworkspace_tenant(
+        neo4j_session,
+        resources.admin,
+        config.update_tag,
+        common_job_parameters,
+    )
+    common_job_parameters["CUSTOMER_ID"] = customer_id
+
+    # Sync users and get the list of transformed users for OAuth token sync
+    user_ids = users.sync_googleworkspace_users(
+        neo4j_session,
+        resources.admin,
+        config.update_tag,
+        common_job_parameters,
+    )
+
+    # Sync OAuth apps for all users
+    oauth_apps.sync_googleworkspace_oauth_apps(
+        neo4j_session,
+        resources.admin,
+        user_ids,
+        config.update_tag,
+        common_job_parameters,
+    )
+
+    groups.sync_googleworkspace_groups(
+        neo4j_session,
+        resources.cloudidentity,
+        config.update_tag,
+        common_job_parameters,
+    )
+    devices.sync_googleworkspace_devices(
+        neo4j_session,
+        resources.cloudidentity,
+        config.update_tag,
+        common_job_parameters,
+    )

cartography/intel/googleworkspace/devices.py

@@ -0,0 +1,254 @@
+import datetime
+import json
+import logging
+from typing import Any
+
+import neo4j
+from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.googleworkspace.device import GoogleWorkspaceDeviceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+GOOGLE_API_NUM_RETRIES = 5
+
+
+@timeit
+def get_devices(
+    cloudidentity: Resource,
+) -> list[dict[str, Any]]:
+    """
+    Fetch all devices from Google Cloud Identity API.
+    """
+    # Only fetch user synced in the last 90 days
+    from_date = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(
+        days=90
+    )
+    request = cloudidentity.devices().list(
+        customer="customers/my_customer",
+        pageSize=100,
+        orderBy="last_sync_time desc",
+        filter=f"sync:{from_date.strftime('%Y-%m-%dT%H:%M:%S')}..",
+    )
+    response_objects = []
+    while request is not None:
+        try:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            response_objects.extend(resp.get("devices", []))
+            request = cloudidentity.devices().list_next(request, resp)
+        except HttpError as e:
+            if (
+                e.resp.status == 403
+                and "Request had insufficient authentication scopes" in str(e)
+            ):
+                logger.error(
+                    "Missing required Google Workspace scopes. If using the gcloud CLI, "
+                    "run: gcloud auth application-default login --scopes="
+                    "https://www.googleapis.com/auth/admin.directory.customer.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.user.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.user.security,"
+                    "https://www.googleapis.com/auth/cloud-identity.devices.readonly,"
+                    "https://www.googleapis.com/auth/cloud-identity.groups.readonly,"
+                    "https://www.googleapis.com/auth/cloud-platform"
+                )
+            raise
+    return response_objects
+
+
+@timeit
+def get_device_users(
+    cloudidentity: Resource,
+) -> list[dict[str, Any]]:
+    """
+    Fetch all device users from Google Cloud Identity API.
+    """
+    # Only fetch user synced in the last 90 days
+    from_date = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(
+        days=90
+    )
+    request = (
+        cloudidentity.devices()
+        .deviceUsers()
+        .list(
+            customer="customers/my_customer",
+            parent="devices/-",
+            pageSize=100,
+            orderBy="last_sync_time desc",
+            filter=f"sync:{from_date.strftime('%Y-%m-%dT%H:%M:%S')}..",
+        )
+    )
+    response_objects = []
+    while request is not None:
+        try:
+            resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
+            response_objects.extend(resp.get("deviceUsers", []))
+            request = cloudidentity.devices().deviceUsers().list_next(request, resp)
+        except HttpError as e:
+            if (
+                e.resp.status == 403
+                and "Request had insufficient authentication scopes" in str(e)
+            ):
+                logger.error(
+                    "Missing required Google Workspace scopes. If using the gcloud CLI, "
+                    "run: gcloud auth application-default login --scopes="
+                    "https://www.googleapis.com/auth/admin.directory.customer.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.user.readonly,"
+                    "https://www.googleapis.com/auth/admin.directory.user.security,"
+                    "https://www.googleapis.com/auth/cloud-identity.devices.readonly,"
+                    "https://www.googleapis.com/auth/cloud-identity.groups.readonly,"
+                    "https://www.googleapis.com/auth/cloud-platform"
+                )
+            raise
+
+    return response_objects
+
+
+def transform_devices(
+    devices: list[dict[str, Any]], device_users: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """
+    Transform device data for Neo4j ingestion.
+    """
+    result = []
+
+    # First we need to create a mapping of device ID to its users
+    device_user_map: dict[str, str] = {}
+    for device_user in device_users:
+        # Ignore non approved device
+        if device_user.get("managementState") != "APPROVED":
+            continue
+        if device_user.get("userEmail") is None:
+            continue
+        # Extract device name from device user name path
+        device_name = device_user["name"].split("/deviceUsers/")[0]
+        if device_name not in device_user_map:
+            device_user_map[device_name] = device_user["userEmail"]
+        else:
+            logger.debug(
+                "Multiple users found for device %s, the most recent was kept",
+                device_name,
+            )
+
+    # Now transform each device, adding its users
+    for device in devices:
+        # Extract device ID from name path (devices/EiRlNzYzZjYyNC1...)
+        device_name = device["name"]
+
+        transformed = {
+            # Required fields
+            "deviceId": device.get("deviceId"),
+            "hostname": device.get("hostname"),
+            # Owner
+            "owner_email": device_user_map.get(device_name),
+            # Device information
+            "model": device.get("model"),
+            "manufacturer": device.get("manufacturer"),
+            "releaseVersion": device.get("releaseVersion"),
+            "brand": device.get("brand"),
+            "buildNumber": device.get("buildNumber"),
+            "kernelVersion": device.get("kernelVersion"),
+            "basebandVersion": device.get("basebandVersion"),
+            "deviceType": device.get("deviceType"),
+            "osVersion": device.get("osVersion"),
+            "ownerType": device.get("ownerType"),
+            # Hardware identifiers
+            "serialNumber": device.get("serialNumber"),
+            "assetTag": device.get("assetTag"),
+            "imei": device.get("imei"),
+            "meid": device.get("meid"),
+            "wifiMacAddresses": device.get("wifiMacAddresses"),
+            "networkOperator": device.get("networkOperator"),
+            # Security and state
+            "encryptionState": device.get("encryptionState"),
+            "compromisedState": device.get("compromisedState"),
+            "managementState": device.get("managementState"),
+            # Timestamps
+            "createTime": device.get("createTime"),
+            "lastSyncTime": device.get("lastSyncTime"),
+            "securityPatchTime": device.get("securityPatchTime"),
+            # Android specific attributes (stored as JSON string if present)
+            "androidSpecificAttributes": (
+                json.dumps(device.get("androidSpecificAttributes"))
+                if device.get("androidSpecificAttributes")
+                else None
+            ),
+            "enabledDeveloperOptions": device.get("enabledDeveloperOptions"),
+            "enabledUsbDebugging": device.get("enabledUsbDebugging"),
+            "bootloaderVersion": device.get("bootloaderVersion"),
+            "otherAccounts": device.get("otherAccounts"),
+            # Additional identifiers
+            "unifiedDeviceId": device.get("unifiedDeviceId"),
+            "endpointVerificationSpecificAttributes": (
+                json.dumps(device.get("endpointVerificationSpecificAttributes"))
+                if device.get("endpointVerificationSpecificAttributes")
+                else None
+            ),
+        }
+        result.append(transformed)
+
+    return result
+
+
+def load_devices(
+    neo4j_session: neo4j.Session,
+    devices: list[dict[str, Any]],
+    customer_id: str,
+    update_tag: int,
+) -> None:
+    """
+    Load device data into Neo4j.
+    """
+    logger.info("Loading %d Google Workspace devices", len(devices))
+    load(
+        neo4j_session,
+        GoogleWorkspaceDeviceSchema(),
+        devices,
+        lastupdated=update_tag,
+        CUSTOMER_ID=customer_id,
+    )
+
+
+def cleanup_devices(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    """
+    Remove devices that weren't updated in this sync run.
+    """
+    logger.debug("Running Google Workspace devices cleanup job")
+    GraphJob.from_node_schema(GoogleWorkspaceDeviceSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+
+
+@timeit
+def sync_googleworkspace_devices(
+    neo4j_session: neo4j.Session,
+    cloudidentity: Resource,
+    update_tag: int,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    """
+    Sync Google Workspace devices and device users.
+    """
+    logger.info("Starting Google Workspace devices sync")
+
+    customer_id = common_job_parameters["CUSTOMER_ID"]
+
+    # 1. GET - Fetch devices data
+    raw_devices = get_devices(cloudidentity)
+    raw_device_users = get_device_users(cloudidentity)
+
+    # 2. TRANSFORM - Shape data for ingestion
+    transformed_devices = transform_devices(raw_devices, raw_device_users)
+
+    # 3. LOAD - Ingest to Neo4j
+    load_devices(neo4j_session, transformed_devices, customer_id, update_tag)
+
+    # 4. CLEANUP - Remove stale data
+    cleanup_devices(neo4j_session, common_job_parameters)
+
+    logger.info("Completed Google Workspace devices sync")