cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/github/users.py

@@ -1,4 +1,5 @@
 import logging
+from copy import deepcopy
 from typing import Any
 from typing import Dict
 from typing import List
@@ -6,10 +7,14 @@ from typing import Tuple
 
 import neo4j
 
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
 from cartography.intel.github.util import fetch_all
+from cartography.models.github.orgs import GitHubOrganizationSchema
+from cartography.models.github.users import GitHubOrganizationUserSchema
+from cartography.models.github.users import GitHubUnaffiliatedUserSchema
 from cartography.stats import get_stats_client
 from cartography.util import merge_module_sync_metadata
-from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -32,6 +37,7 @@ GITHUB_ORG_USERS_PAGINATED_GRAPHQL = """
                         isSiteAdmin
                         email
                         company
+                        organizationVerifiedDomainEmails(login: $login)
                     }
                     role
                 }
@@ -44,83 +50,242 @@ GITHUB_ORG_USERS_PAGINATED_GRAPHQL = """
     }
     """
 
+GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL = """
+    query($login: String!, $cursor: String) {
+    organization(login: $login)
+        {
+            url
+            login
+            enterpriseOwners(first:100, after: $cursor){
+                edges {
+                    node {
+                        url
+                        login
+                        name
+                        isSiteAdmin
+                        email
+                        company
+                        organizationVerifiedDomainEmails(login: $login)
+                    }
+                    organizationRole
+                }
+                pageInfo{
+                    endCursor
+                    hasNextPage
+                }
+            }
+        }
+    }
+    """
+
 
 @timeit
-def get(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
+def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
     """
     Retrieve a list of users from the given GitHub organization as described in
     https://docs.github.com/en/graphql/reference/objects#organizationmemberedge.
     :param token: The Github API token as string.
     :param api_url: The Github v4 API endpoint as string.
     :param organization: The name of the target Github organization as string.
-    :return: A 2-tuple containing 1. a list of dicts representing users - see tests.data.github.users.GITHUB_USER_DATA
-    for shape, and 2. data on the owning GitHub organization - see tests.data.github.users.GITHUB_ORG_DATA for shape.
+    :return: A 2-tuple containing
+        1. a list of dicts representing users and
+        2. data on the owning GitHub organization
+        see tests.data.github.users.GITHUB_USER_DATA for shape of both
     """
+    logger.info(f"Retrieving users from GitHub organization {organization}")
     users, org = fetch_all(
         token,
         api_url,
         organization,
         GITHUB_ORG_USERS_PAGINATED_GRAPHQL,
-        'membersWithRole',
+        "membersWithRole",
     )
     return users.edges, org
 
 
+def get_enterprise_owners(
+    token: str,
+    api_url: str,
+    organization: str,
+) -> Tuple[List[Dict], Dict]:
+    """
+    Retrieve a list of enterprise owners from the given GitHub organization as described in
+    https://docs.github.com/en/graphql/reference/objects#organizationenterpriseowneredge.
+    :param token: The Github API token as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param organization: The name of the target Github organization as string.
+    :return: A 2-tuple containing
+        1. a list of dicts representing users who are enterprise owners
+        3. data on the owning GitHub organization
+        see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
+    """
+    logger.info(f"Retrieving enterprise owners from GitHub organization {organization}")
+    owners, org = fetch_all(
+        token,
+        api_url,
+        organization,
+        GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL,
+        "enterpriseOwners",
+    )
+    return owners.edges, org
+
+
+@timeit
+def transform_users(
+    user_data: List[Dict],
+    owners_data: List[Dict],
+    org_data: Dict,
+) -> Tuple[List[Dict], List[Dict]]:
+    """
+    Taking raw user and owner data, return two lists of processed user data:
+    * organization users aka affiliated users (users directly affiliated with an organization)
+    * unaffiliated users (user who, for example, are enterprise owners but not members of the target organization).
+
+    :param token: The Github API token as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param organization: The name of the target Github organization as string.
+    :return: A 2-tuple containing
+        1. a list of dicts representing users who are affiliated with the target org
+           see tests.data.github.users.GITHUB_USER_DATA for shape
+        2. a list of dicts representing users who are not affiliated (e.g. enterprise owners who are not also in
+           the target org) — see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
+        3. data on the owning GitHub organization
+    """
+
+    users_dict = {}
+    for user in user_data:
+        # all members get the 'MEMBER_OF' relationship
+        processed_user = deepcopy(user["node"])
+        # GitHub returns empty strings for email addresses, so we convert them to None
+        if "email" in processed_user and not processed_user["email"]:
+            processed_user["email"] = None
+        processed_user["hasTwoFactorEnabled"] = user["hasTwoFactorEnabled"]
+        processed_user["MEMBER_OF"] = org_data["url"]
+        # admins get a second relationship expressing them as such
+        if user["role"] == "ADMIN":
+            processed_user["ADMIN_OF"] = org_data["url"]
+        users_dict[processed_user["url"]] = processed_user
+
+    owners_dict = {}
+    for owner in owners_data:
+        processed_owner = deepcopy(owner["node"])
+        processed_owner["isEnterpriseOwner"] = True
+        if owner["organizationRole"] == "UNAFFILIATED":
+            processed_owner["UNAFFILIATED"] = org_data["url"]
+        else:
+            processed_owner["MEMBER_OF"] = org_data["url"]
+        owners_dict[processed_owner["url"]] = processed_owner
+
+    affiliated_users = []  # users affiliated with the target org
+    for url, user in users_dict.items():
+        user["isEnterpriseOwner"] = url in owners_dict
+        affiliated_users.append(user)
+
+    unaffiliated_users = []  # users not affiliated with the target org
+    for url, owner in owners_dict.items():
+        if url not in users_dict:
+            unaffiliated_users.append(owner)
+
+    return affiliated_users, unaffiliated_users
+
+
 @timeit
-def load_organization_users(
-    neo4j_session: neo4j.Session, user_data: List[Dict], org_data: Dict,
+def load_users(
+    neo4j_session: neo4j.Session,
+    node_schema: GitHubOrganizationUserSchema | GitHubUnaffiliatedUserSchema,
+    user_data: List[Dict],
+    org_data: Dict,
     update_tag: int,
 ) -> None:
-    query = """
-    MERGE (org:GitHubOrganization{id: $OrgUrl})
-    ON CREATE SET org.firstseen = timestamp()
-    SET org.username = $OrgLogin,
-    org.lastupdated = $UpdateTag
-    WITH org
-
-    UNWIND $UserData as user
-
-    MERGE (u:GitHubUser{id: user.node.url})
-    ON CREATE SET u.firstseen = timestamp()
-    SET u.fullname = user.node.name,
-    u.username = user.node.login,
-    u.has_2fa_enabled = user.hasTwoFactorEnabled,
-    u.role = user.role,
-    u.is_site_admin = user.node.isSiteAdmin,
-    u.email = user.node.email,
-    u.company = user.node.company,
-    u.lastupdated = $UpdateTag
-
-    MERGE (u)-[r:MEMBER_OF]->(org)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
-    """
-    neo4j_session.run(
-        query,
-        OrgUrl=org_data['url'],
-        OrgLogin=org_data['login'],
-        UserData=user_data,
-        UpdateTag=update_tag,
+    logger.info(f"Loading {len(user_data)} GitHub users to the graph")
+    load(
+        neo4j_session,
+        node_schema,
+        user_data,
+        lastupdated=update_tag,
+        org_url=org_data["url"],
+    )
+
+
+@timeit
+def load_organization(
+    neo4j_session: neo4j.Session,
+    node_schema: GitHubOrganizationSchema,
+    org_data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(org_data)} GitHub organization to the graph")
+    load(
+        neo4j_session,
+        node_schema,
+        org_data,
+        lastupdated=update_tag,
     )
 
 
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
+    logger.info("Cleaning up GitHub users")
+    GraphJob.from_node_schema(
+        GitHubOrganizationUserSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(
+        GitHubUnaffiliatedUserSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+
+
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        common_job_parameters: Dict[str, Any],
-        github_api_key: str,
-        github_url: str,
-        organization: str,
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+    github_api_key: str,
+    github_url: str,
+    organization: str,
 ) -> None:
     logger.info("Syncing GitHub users")
-    user_data, org_data = get(github_api_key, github_url, organization)
-    load_organization_users(neo4j_session, user_data, org_data, common_job_parameters['UPDATE_TAG'])
-    run_cleanup_job('github_users_cleanup.json', neo4j_session, common_job_parameters)
+    user_data, org_data = get_users(github_api_key, github_url, organization)
+    owners_data, org_data = get_enterprise_owners(
+        github_api_key,
+        github_url,
+        organization,
+    )
+    processed_affiliated_user_data, processed_unaffiliated_user_data = transform_users(
+        user_data,
+        owners_data,
+        org_data,
+    )
+    load_organization(
+        neo4j_session,
+        GitHubOrganizationSchema(),
+        [org_data],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_users(
+        neo4j_session,
+        GitHubOrganizationUserSchema(),
+        processed_affiliated_user_data,
+        org_data,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_users(
+        neo4j_session,
+        GitHubUnaffiliatedUserSchema(),
+        processed_unaffiliated_user_data,
+        org_data,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='GitHubOrganization',
-        group_id=org_data['url'],
-        synced_type='GitHubOrganization',
-        update_tag=common_job_parameters['UPDATE_TAG'],
+        group_type="GitHubOrganization",
+        group_id=org_data["url"],
+        synced_type="GitHubOrganization",
+        update_tag=common_job_parameters["UPDATE_TAG"],
         stat_handler=stat_handler,
     )

cartography/intel/github/util.py

@@ -13,7 +13,6 @@ from typing import Tuple
 
 import requests
 
-
 logger = logging.getLogger(__name__)
 # Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
 _TIMEOUT = (60, 60)
@@ -26,25 +25,28 @@ class PaginatedGraphqlData(NamedTuple):
 
 
 def handle_rate_limit_sleep(token: str) -> None:
-    '''
+    """
     Check the remaining rate limit and sleep if remaining is below threshold
     :param token: The Github API token as string.
-    '''
-    response = requests.get('https://api.github.com/rate_limit', headers={'Authorization': f"token {token}"})
+    """
+    response = requests.get(
+        "https://api.github.com/rate_limit",
+        headers={"Authorization": f"token {token}"},
+    )
     response.raise_for_status()
     response_json = response.json()
-    rate_limit_obj = response_json['resources']['graphql']
-    remaining = rate_limit_obj['remaining']
+    rate_limit_obj = response_json["resources"]["graphql"]
+    remaining = rate_limit_obj["remaining"]
     threshold = _GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD
     if remaining > threshold:
         return
-    reset_at = datetime.fromtimestamp(rate_limit_obj['reset'], tz=tz.utc)
+    reset_at = datetime.fromtimestamp(rate_limit_obj["reset"], tz=tz.utc)
     now = datetime.now(tz.utc)
     # add an extra minute for safety
     sleep_duration = reset_at - now + timedelta(minutes=1)
     logger.warning(
-        f'Github graphql ratelimit has {remaining} remaining and is under threshold {threshold},'
-        f' sleeping until reset at {reset_at} for {sleep_duration}',
+        f"Github graphql ratelimit has {remaining} remaining and is under threshold {threshold},"
+        f" sleeping until reset at {reset_at} for {sleep_duration}",
     )
     time.sleep(sleep_duration.seconds)
 
@@ -58,11 +60,11 @@ def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dic
     :param api_url: the URL to call for the API
     :return: query results json
     """
-    headers = {'Authorization': f"token {token}"}
+    headers = {"Authorization": f"token {token}"}
     try:
         response = requests.post(
             api_url,
-            json={'query': query, 'variables': variables},
+            json={"query": query, "variables": variables},
             headers=headers,
             timeout=_TIMEOUT,
         )
@@ -75,7 +77,7 @@ def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dic
     if "errors" in response_json:
         logger.warning(
             f'call_github_api() response has errors, please investigate. Raw response: {response_json["errors"]}; '
-            f'continuing sync.',
+            f"continuing sync.",
         )
     return response_json  # type: ignore
 
@@ -101,8 +103,8 @@ def fetch_page(
     """
     gql_vars = {
         **kwargs,
-        'login': organization,
-        'cursor': cursor,
+        "login": organization,
+        "cursor": cursor,
     }
     gql_vars_json = json.dumps(gql_vars)
     response = call_github_api(query, gql_vars_json, token, api_url)
@@ -110,14 +112,14 @@ def fetch_page(
 
 
 def fetch_all(
-        token: str,
-        api_url: str,
-        organization: str,
-        query: str,
-        resource_type: str,
-        retries: int = 5,
-        resource_inner_type: Optional[str] = None,
-        **kwargs: Any,
+    token: str,
+    api_url: str,
+    organization: str,
+    query: str,
+    resource_type: str,
+    retries: int = 5,
+    resource_inner_type: Optional[str] = None,
+    **kwargs: Any,
 ) -> Tuple[PaginatedGraphqlData, Dict[str, Any]]:
     """
     Fetch and return all data items of the given `resource_type` and `field_name` from Github's paginated GraphQL API as
@@ -155,6 +157,18 @@ def fetch_all(
             retry += 1
             exc = err
         except requests.exceptions.HTTPError as err:
+            if (
+                err.response is not None
+                and err.response.status_code == 502
+                and kwargs.get("count")
+                and kwargs["count"] > 1
+            ):
+                kwargs["count"] = max(1, kwargs["count"] // 2)
+                logger.warning(
+                    "GitHub: Received 502 response. Reducing page size to %s and retrying.",
+                    kwargs["count"],
+                )
+                continue
             retry += 1
             exc = err
         except requests.exceptions.ChunkedEncodingError as err:
@@ -163,37 +177,38 @@ def fetch_all(
 
         if retry >= retries:
             logger.error(
-                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error.",
+                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error "
+                f"after {retry} retries. Raising exception.",
                 exc_info=True,
             )
             raise exc
         elif retry > 0:
-            time.sleep(2 ** retry)
+            time.sleep(2**retry)
             continue
 
-        if 'data' not in resp:
+        if "data" not in resp:
             logger.warning(
                 f'Got no "data" attribute in response: {resp}. '
-                f'Stopping requests for organization: {organization} and '
-                f'resource_type: {resource_type}',
+                f"Stopping requests for organization: {organization} and "
+                f"resource_type: {resource_type}",
             )
             has_next_page = False
             continue
 
-        resource = resp['data']['organization'][resource_type]
+        resource = resp["data"]["organization"][resource_type]
         if resource_inner_type:
-            resource = resp['data']['organization'][resource_type][resource_inner_type]
+            resource = resp["data"]["organization"][resource_type][resource_inner_type]
 
         # Allow for paginating both nodes and edges fields of the GitHub GQL structure.
-        data.nodes.extend(resource.get('nodes', []))
-        data.edges.extend(resource.get('edges', []))
+        data.nodes.extend(resource.get("nodes", []))
+        data.edges.extend(resource.get("edges", []))
 
-        cursor = resource['pageInfo']['endCursor']
-        has_next_page = resource['pageInfo']['hasNextPage']
+        cursor = resource["pageInfo"]["endCursor"]
+        has_next_page = resource["pageInfo"]["hasNextPage"]
         if not org_data:
             org_data = {
-                'url': resp['data']['organization']['url'],
-                'login': resp['data']['organization']['login'],
+                "url": resp["data"]["organization"]["url"],
+                "login": resp["data"]["organization"]["login"],
             }
 
     if not org_data:

cartography/intel/googleworkspace/__init__.py

@@ -0,0 +1,193 @@
+import base64
+import json
+import logging
+import os
+from collections import namedtuple
+
+import googleapiclient.discovery
+import neo4j
+from google.auth import default
+from google.auth.exceptions import DefaultCredentialsError
+from google.auth.transport.requests import Request
+from google.oauth2 import credentials
+from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials as OAuth2Credentials
+from google.oauth2.service_account import Credentials as ServiceAccountCredentials
+
+from cartography.config import Config
+from cartography.intel.googleworkspace import devices
+from cartography.intel.googleworkspace import groups
+from cartography.intel.googleworkspace import oauth_apps
+from cartography.intel.googleworkspace import tenant
+from cartography.intel.googleworkspace import users
+from cartography.util import timeit
+
+OAUTH_SCOPES = [
+    "https://www.googleapis.com/auth/admin.directory.customer.readonly",
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+    "https://www.googleapis.com/auth/admin.directory.user.security",
+    "https://www.googleapis.com/auth/cloud-identity.devices.readonly",
+    "https://www.googleapis.com/auth/cloud-identity.groups.readonly",
+]
+
+logger = logging.getLogger(__name__)
+
+Resources = namedtuple("Resources", ["admin", "cloudidentity"])
+
+
+def _initialize_resources(
+    creds: OAuth2Credentials | ServiceAccountCredentials,
+) -> Resources:
+    """
+    Create namedtuple of all resource objects necessary for Google API data gathering.
+    :param credentials: The credentials object
+    :return: namedtuple of all resource objects
+    """
+
+    return Resources(
+        googleapiclient.discovery.build(
+            "admin",
+            "directory_v1",
+            credentials=creds,
+            cache_discovery=False,
+        ),
+        googleapiclient.discovery.build(
+            "cloudidentity",
+            "v1",
+            credentials=creds,
+            cache_discovery=False,
+        ),
+    )
+
+
+@timeit
+def start_googleworkspace_ingestion(
+    neo4j_session: neo4j.Session, config: Config
+) -> None:
+    """
+    Starts the Google Workspace ingestion process by initializing
+
+    :param neo4j_session: The Neo4j session
+    :param config: A `cartography.config` object
+    :return: Nothing
+    """
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    creds: OAuth2Credentials | ServiceAccountCredentials
+    if config.googleworkspace_auth_method == "delegated":  # Legacy delegated method
+        if config.googleworkspace_config is None or not os.path.isfile(
+            config.googleworkspace_config
+        ):
+            logger.warning(
+                (
+                    "The Google Workspace config file is not set or is not a valid file."
+                    "Skipping Google Workspace ingestion."
+                ),
+            )
+            return
+        logger.info(
+            "Attempting to authenticate to Google Workspace using legacy delegated method"
+        )
+        try:
+            creds = service_account.Credentials.from_service_account_file(
+                config.googleworkspace_config,
+                scopes=OAUTH_SCOPES,
+            )
+            creds = creds.with_subject(os.environ.get("GOOGLE_DELEGATED_ADMIN"))
+
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize Google Workspace creds. If you don't have Google Workspace data or don't want to load "
+                    "Google Workspace data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure your Google Workspace credentials file (if any) is valid. "
+                    "For more details see documentation."
+                ),
+                e,
+            )
+            return
+    elif config.googleworkspace_auth_method == "oauth":
+        auth_tokens = json.loads(
+            str(base64.b64decode(config.googleworkspace_config).decode())
+        )
+        logger.info("Attempting to authenticate to Google Workspace using OAuth")
+        try:
+            creds = credentials.Credentials(
+                token=None,
+                client_id=auth_tokens["client_id"],
+                client_secret=auth_tokens["client_secret"],
+                refresh_token=auth_tokens["refresh_token"],
+                expiry=None,
+                token_uri=auth_tokens["token_uri"],
+                scopes=OAUTH_SCOPES,
+            )
+            creds.refresh(Request())
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize Google Workspace creds. If you don't have Google Workspace data or don't want to load "
+                    "Google Workspace data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure your Google Workspace credentials are configured correctly, your credentials are valid. "
+                    "For more details see documentation."
+                ),
+                e,
+            )
+            return
+    elif config.googleworkspace_auth_method == "default":
+        logger.info(
+            "Attempting to authenticate to Google Workspace using default credentials"
+        )
+        try:
+            creds, _ = default(scopes=OAUTH_SCOPES)
+        except DefaultCredentialsError as e:
+            logger.error(
+                (
+                    "Unable to initialize Google Workspace creds using default credentials. If you don't have Google Workspace data or "
+                    "don't want to load Google Workspace data then you can ignore this message. Otherwise, the error code is: %s "
+                    "Make sure you have valid application default credentials configured. "
+                    "For more details see documentation."
+                ),
+                e,
+            )
+            return
+
+    resources = _initialize_resources(creds)
+    customer_id = tenant.sync_googleworkspace_tenant(
+        neo4j_session,
+        resources.admin,
+        config.update_tag,
+        common_job_parameters,
+    )
+    common_job_parameters["CUSTOMER_ID"] = customer_id
+
+    # Sync users and get the list of transformed users for OAuth token sync
+    user_ids = users.sync_googleworkspace_users(
+        neo4j_session,
+        resources.admin,
+        config.update_tag,
+        common_job_parameters,
+    )
+
+    # Sync OAuth apps for all users
+    oauth_apps.sync_googleworkspace_oauth_apps(
+        neo4j_session,
+        resources.admin,
+        user_ids,
+        config.update_tag,
+        common_job_parameters,
+    )
+
+    groups.sync_googleworkspace_groups(
+        neo4j_session,
+        resources.cloudidentity,
+        config.update_tag,
+        common_job_parameters,
+    )
+    devices.sync_googleworkspace_devices(
+        neo4j_session,
+        resources.cloudidentity,
+        config.update_tag,
+        common_job_parameters,
+    )