cartography 0.95.0__py3-none-any.whl → 0.96.0__py3-none-any.whl

cartography/intel/github/repos.py

@@ -1,5 +1,6 @@
 import configparser
 import logging
+from collections import namedtuple
 from string import Template
 from typing import Any
 from typing import Dict
@@ -12,11 +13,26 @@ from packaging.requirements import Requirement
 from packaging.utils import canonicalize_name
 
 from cartography.intel.github.util import fetch_all
+from cartography.intel.github.util import PaginatedGraphqlData
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+
+# Representation of a user's permission level and affiliation to a GitHub repo. See:
+# - Permission: https://docs.github.com/en/graphql/reference/enums#repositorypermission
+# - Affiliation: https://docs.github.com/en/graphql/reference/enums#collaboratoraffiliation
+UserAffiliationAndRepoPermission = namedtuple(
+    'UserAffiliationAndRepoPermission',
+    [
+        'user',  # Dict
+        'permission',  # 'WRITE', 'MAINTAIN', 'ADMIN', etc
+        'affiliation',  # 'OUTSIDE', 'DIRECT'
+    ],
+)
+
+
 GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
     query($login: String!, $cursor: String) {
     organization(login: $login)
@@ -59,17 +75,11 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
                         login
                         __typename
                     }
-                    collaborators(affiliation: OUTSIDE, first: 50) {
-                        edges {
-                            permission
-                        }
-                        nodes {
-                            url
-                            login
-                            name
-                            email
-                            company
-                        }
+                    directCollaborators: collaborators(first: 100, affiliation: DIRECT) {
+                        totalCount
+                    }
+                    outsideCollaborators: collaborators(first: 100, affiliation: OUTSIDE) {
+                        totalCount
                     }
                     requirements:object(expression: "HEAD:requirements.txt") {
                         ... on Blob {
@@ -89,6 +99,111 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
 # Note: In the above query, `HEAD` references the default branch.
 # See https://stackoverflow.com/questions/48935381/github-graphql-api-default-branch-in-repository
 
+GITHUB_REPO_COLLABS_PAGINATED_GRAPHQL = """
+    query($login: String!, $repo: String!, $affiliation: CollaboratorAffiliation!, $cursor: String) {
+        organization(login: $login) {
+            url
+            login
+            repository(name: $repo){
+                name
+                collaborators(first: 50, affiliation: $affiliation, after: $cursor) {
+                    edges {
+                        permission
+                    }
+                    nodes {
+                        url
+                        login
+                        name
+                        email
+                        company
+                    }
+                    pageInfo{
+                        endCursor
+                        hasNextPage
+                    }
+                }
+            }
+        }
+        rateLimit {
+            limit
+            cost
+            remaining
+            resetAt
+        }
+    }
+    """
+
+
+def _get_repo_collaborators_for_multiple_repos(
+        repo_raw_data: list[dict[str, Any]],
+        affiliation: str,
+        org: str,
+        api_url: str,
+        token: str,
+) -> dict[str, List[UserAffiliationAndRepoPermission]]:
+    """
+    For every repo in the given list, retrieve the collaborators.
+    :param repo_raw_data: A list of dicts representing repos. See tests.data.github.repos.GET_REPOS for data shape.
+    :param affiliation: The type of affiliation to retrieve collaborators for. Either 'DIRECT' or 'OUTSIDE'.
+      See https://docs.github.com/en/graphql/reference/enums#collaboratoraffiliation
+    :param org: The name of the target Github organization as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param token: The Github API token as string.
+    :return: A dictionary of repo URL to list of UserAffiliationAndRepoPermission
+    """
+    result: dict[str, List[UserAffiliationAndRepoPermission]] = {}
+    for repo in repo_raw_data:
+        repo_name = repo['name']
+        repo_url = repo['url']
+
+        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
+                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
+            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
+            result[repo_url] = []
+            continue
+
+        collab_users = []
+        collab_permission = []
+        collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
+        # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
+        for collab in collaborators.nodes:
+            collab_users.append(collab)
+        for perm in collaborators.edges:
+            collab_permission.append(perm['permission'])
+
+        result[repo_url] = [
+            UserAffiliationAndRepoPermission(user, permission, affiliation)
+            for user, permission in zip(collab_users, collab_permission)
+        ]
+    return result
+
+
+def _get_repo_collaborators(
+        token: str, api_url: str, organization: str, repo: str, affiliation: str,
+) -> PaginatedGraphqlData:
+    """
+    Retrieve a list of collaborators for a given repository, as described in
+    https://docs.github.com/en/graphql/reference/objects#repositorycollaboratorconnection.
+    :param token: The Github API token as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param organization: The name of the target Github organization as string.
+    :pram repo: The name of the target Github repository as string.
+    :param affiliation: The type of affiliation to retrieve collaborators for. Either 'DIRECT' or 'OUTSIDE'.
+      See https://docs.github.com/en/graphql/reference/enums#collaboratoraffiliation
+    :return: A list of dicts representing repos. See tests.data.github.repos for data shape.
+    """
+    collaborators, _ = fetch_all(
+        token,
+        api_url,
+        organization,
+        GITHUB_REPO_COLLABS_PAGINATED_GRAPHQL,
+        'repository',
+        resource_inner_type='collaborators',
+        repo=repo,
+        affiliation=affiliation,
+    )
+    return collaborators
+
 
 @timeit
 def get(token: str, api_url: str, organization: str) -> List[Dict]:
@@ -111,11 +226,18 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
     return repos.nodes
 
 
-def transform(repos_json: List[Dict]) -> Dict:
+def transform(
+    repos_json: List[Dict], direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+    outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+) -> Dict:
     """
     Parses the JSON returned from GitHub API to create data for graph ingestion
-    :param repos_json: the list of individual repository nodes from GitHub. See tests.data.github.repos.GET_REPOS for
-    data shape.
+    :param repos_json: the list of individual repository nodes from GitHub.
+        See tests.data.github.repos.GET_REPOS for data shape.
+    :param direct_collaborators: dict of repo URL to list of direct collaborators.
+        See tests.data.github.repos.DIRECT_COLLABORATORS for data shape.
+    :param outside_collaborators: dict of repo URL to list of outside collaborators.
+        See tests.data.github.repos.OUTSIDE_COLLABORATORS for data shape.
     :return: Dict containing the repos, repo->language mapping, owners->repo mapping, outside collaborators->repo
     mapping, and Python requirements files (if any) in a repo.
     """
@@ -123,7 +245,10 @@ def transform(repos_json: List[Dict]) -> Dict:
     transformed_repo_languages: List[Dict] = []
     transformed_repo_owners: List[Dict] = []
     # See https://docs.github.com/en/graphql/reference/enums#repositorypermission
-    transformed_collaborators: Dict[str, List[Any]] = {
+    transformed_outside_collaborators: Dict[str, List[Any]] = {
+        'ADMIN': [], 'MAINTAIN': [], 'READ': [], 'TRIAGE': [], 'WRITE': [],
+    }
+    transformed_direct_collaborators: Dict[str, List[Any]] = {
         'ADMIN': [], 'MAINTAIN': [], 'READ': [], 'TRIAGE': [], 'WRITE': [],
     }
     transformed_requirements_files: List[Dict] = []
@@ -131,14 +256,22 @@ def transform(repos_json: List[Dict]) -> Dict:
         _transform_repo_languages(repo_object['url'], repo_object, transformed_repo_languages)
         _transform_repo_objects(repo_object, transformed_repo_list)
         _transform_repo_owners(repo_object['owner']['url'], repo_object, transformed_repo_owners)
-        _transform_collaborators(repo_object['collaborators'], repo_object['url'], transformed_collaborators)
+        _transform_collaborators(
+            repo_object['url'], outside_collaborators[repo_object['url']],
+            transformed_outside_collaborators,
+        )
+        _transform_collaborators(
+            repo_object['url'], direct_collaborators[repo_object['url']],
+            transformed_direct_collaborators,
+        )
         _transform_requirements_txt(repo_object['requirements'], repo_object['url'], transformed_requirements_files)
         _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_object['url'], transformed_requirements_files)
     results = {
         'repos': transformed_repo_list,
         'repo_languages': transformed_repo_languages,
         'repo_owners': transformed_repo_owners,
-        'repo_collaborators': transformed_collaborators,
+        'repo_outside_collaborators': transformed_outside_collaborators,
+        'repo_direct_collaborators': transformed_direct_collaborators,
         'python_requirements': transformed_requirements_files,
     }
     return results
@@ -229,11 +362,15 @@ def _transform_repo_languages(repo_url: str, repo: Dict, repo_languages: List[Di
             })
 
 
-def _transform_collaborators(collaborators: Dict, repo_url: str, transformed_collaborators: Dict) -> None:
+def _transform_collaborators(
+        repo_url: str, collaborators: List[UserAffiliationAndRepoPermission], transformed_collaborators: Dict,
+) -> None:
     """
-    Performs data adjustments for outside collaborators in a GitHub repo.
+    Performs data adjustments for collaborators in a GitHub repo.
     Output data shape = [{permission, repo_url, url (the user's URL), login, name}, ...]
-    :param collaborators: See cartography.tests.data.github.repos for data shape.
+    :param collaborators: For data shape, see
+        cartography.tests.data.github.repos.DIRECT_COLLABORATORS
+        cartography.tests.data.github.repos.OUTSIDE_COLLABORATORS
     :param repo_url: The URL of the GitHub repo.
     :param transformed_collaborators: Output dict. Data shape =
     {'ADMIN': [{ user }, ...], 'MAINTAIN': [{ user }, ...], 'READ': [ ... ], 'TRIAGE': [ ... ], 'WRITE': [ ... ]}
@@ -241,10 +378,11 @@ def _transform_collaborators(collaborators: Dict, repo_url: str, transformed_col
     """
     # `collaborators` is sometimes None
     if collaborators:
-        for idx, user in enumerate(collaborators['nodes']):
-            user_permission = collaborators['edges'][idx]['permission']
+        for collaborator in collaborators:
+            user = collaborator.user
             user['repo_url'] = repo_url
-            transformed_collaborators[user_permission].append(user)
+            user['affiliation'] = collaborator.affiliation
+            transformed_collaborators[collaborator.permission].append(user)
 
 
 def _transform_requirements_txt(
@@ -482,7 +620,7 @@ def load_github_owners(neo4j_session: neo4j.Session, update_tag: int, repo_owner
 
 
 @timeit
-def load_collaborators(neo4j_session: neo4j.Session, update_tag: int, collaborators: Dict) -> None:
+def load_collaborators(neo4j_session: neo4j.Session, update_tag: int, collaborators: Dict, affiliation: str) -> None:
     query = Template("""
     UNWIND $UserData as user
 
@@ -502,7 +640,7 @@ def load_collaborators(neo4j_session: neo4j.Session, update_tag: int, collaborat
     SET o.lastupdated = $UpdateTag
     """)
     for collab_type in collaborators.keys():
-        relationship_label = f"OUTSIDE_COLLAB_{collab_type}"
+        relationship_label = f"{affiliation}_COLLAB_{collab_type}"
         neo4j_session.run(
             query.safe_substitute(rel_label=relationship_label),
             UserData=collaborators[collab_type],
@@ -515,7 +653,12 @@ def load(neo4j_session: neo4j.Session, common_job_parameters: Dict, repo_data: D
     load_github_repos(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repos'])
     load_github_owners(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_owners'])
     load_github_languages(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_languages'])
-    load_collaborators(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_collaborators'])
+    load_collaborators(
+        neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_direct_collaborators'], 'DIRECT',
+    )
+    load_collaborators(
+        neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_outside_collaborators'], 'OUTSIDE',
+    )
     load_python_requirements(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['python_requirements'])
 
 
@@ -561,6 +704,12 @@ def sync(
     """
     logger.info("Syncing GitHub repos")
     repos_json = get(github_api_key, github_url, organization)
-    repo_data = transform(repos_json)
+    direct_collabs = _get_repo_collaborators_for_multiple_repos(
+        repos_json, "DIRECT", organization, github_url, github_api_key,
+    )
+    outside_collabs = _get_repo_collaborators_for_multiple_repos(
+        repos_json, "OUTSIDE", organization, github_url, github_api_key,
+    )
+    repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
     run_cleanup_job('github_repos_cleanup.json', neo4j_session, common_job_parameters)

cartography/intel/github/users.py

@@ -1,4 +1,5 @@
 import logging
+from copy import deepcopy
 from typing import Any
 from typing import Dict
 from typing import List
@@ -6,7 +7,11 @@ from typing import Tuple
 
 import neo4j
 
+from cartography.client.core.tx import load
 from cartography.intel.github.util import fetch_all
+from cartography.models.github.orgs import GitHubOrganizationSchema
+from cartography.models.github.users import GitHubOrganizationUserSchema
+from cartography.models.github.users import GitHubUnaffiliatedUserSchema
 from cartography.stats import get_stats_client
 from cartography.util import merge_module_sync_metadata
 from cartography.util import run_cleanup_job
@@ -44,17 +49,46 @@ GITHUB_ORG_USERS_PAGINATED_GRAPHQL = """
     }
     """
 
+GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL = """
+    query($login: String!, $cursor: String) {
+    organization(login: $login)
+        {
+            url
+            login
+            enterpriseOwners(first:100, after: $cursor){
+                edges {
+                    node {
+                        url
+                        login
+                        name
+                        isSiteAdmin
+                        email
+                        company
+                    }
+                    organizationRole
+                }
+                pageInfo{
+                    endCursor
+                    hasNextPage
+                }
+            }
+        }
+    }
+    """
+
 
 @timeit
-def get(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
+def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
     """
     Retrieve a list of users from the given GitHub organization as described in
     https://docs.github.com/en/graphql/reference/objects#organizationmemberedge.
     :param token: The Github API token as string.
     :param api_url: The Github v4 API endpoint as string.
     :param organization: The name of the target Github organization as string.
-    :return: A 2-tuple containing 1. a list of dicts representing users - see tests.data.github.users.GITHUB_USER_DATA
-    for shape, and 2. data on the owning GitHub organization - see tests.data.github.users.GITHUB_ORG_DATA for shape.
+    :return: A 2-tuple containing
+        1. a list of dicts representing users and
+        2. data on the owning GitHub organization
+        see tests.data.github.users.GITHUB_USER_DATA for shape of both
     """
     users, org = fetch_all(
         token,
@@ -66,56 +100,139 @@ def get(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
     return users.edges, org
 
 
+def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
+    """
+    Retrieve a list of enterprise owners from the given GitHub organization as described in
+    https://docs.github.com/en/graphql/reference/objects#organizationenterpriseowneredge.
+    :param token: The Github API token as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param organization: The name of the target Github organization as string.
+    :return: A 2-tuple containing
+        1. a list of dicts representing users who are enterprise owners
+        3. data on the owning GitHub organization
+        see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
+    """
+    owners, org = fetch_all(
+        token,
+        api_url,
+        organization,
+        GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL,
+        'enterpriseOwners',
+    )
+    return owners.edges, org
+
+
 @timeit
-def load_organization_users(
-    neo4j_session: neo4j.Session, user_data: List[Dict], org_data: Dict,
+def transform_users(user_data: List[Dict], owners_data: List[Dict], org_data: Dict) -> Tuple[List[Dict], List[Dict]]:
+    """
+    Taking raw user and owner data, return two lists of processed user data:
+    * organization users aka affiliated users (users directly affiliated with an organization)
+    * unaffiliated users (user who, for example, are enterprise owners but not members of the target organization).
+
+    :param token: The Github API token as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param organization: The name of the target Github organization as string.
+    :return: A 2-tuple containing
+        1. a list of dicts representing users who are affiliated with the target org
+           see tests.data.github.users.GITHUB_USER_DATA for shape
+        2. a list of dicts representing users who are not affiliated (e.g. enterprise owners who are not also in
+           the target org) — see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
+        3. data on the owning GitHub organization
+    """
+
+    users_dict = {}
+    for user in user_data:
+        processed_user = deepcopy(user['node'])
+        processed_user['role'] = user['role']
+        processed_user['hasTwoFactorEnabled'] = user['hasTwoFactorEnabled']
+        processed_user['MEMBER_OF'] = org_data['url']
+        users_dict[processed_user['url']] = processed_user
+
+    owners_dict = {}
+    for owner in owners_data:
+        processed_owner = deepcopy(owner['node'])
+        processed_owner['isEnterpriseOwner'] = True
+        if owner['organizationRole'] == 'UNAFFILIATED':
+            processed_owner['UNAFFILIATED'] = org_data['url']
+        else:
+            processed_owner['MEMBER_OF'] = org_data['url']
+        owners_dict[processed_owner['url']] = processed_owner
+
+    affiliated_users = []  # users affiliated with the target org
+    for url, user in users_dict.items():
+        user['isEnterpriseOwner'] = url in owners_dict
+        affiliated_users.append(user)
+
+    unaffiliated_users = []  # users not affiliated with the target org
+    for url, owner in owners_dict.items():
+        if url not in users_dict:
+            unaffiliated_users.append(owner)
+
+    return affiliated_users, unaffiliated_users
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    node_schema: GitHubOrganizationUserSchema | GitHubUnaffiliatedUserSchema,
+    user_data: List[Dict],
+    org_data: Dict,
     update_tag: int,
 ) -> None:
-    query = """
-    MERGE (org:GitHubOrganization{id: $OrgUrl})
-    ON CREATE SET org.firstseen = timestamp()
-    SET org.username = $OrgLogin,
-    org.lastupdated = $UpdateTag
-    WITH org
-
-    UNWIND $UserData as user
-
-    MERGE (u:GitHubUser{id: user.node.url})
-    ON CREATE SET u.firstseen = timestamp()
-    SET u.fullname = user.node.name,
-    u.username = user.node.login,
-    u.has_2fa_enabled = user.hasTwoFactorEnabled,
-    u.role = user.role,
-    u.is_site_admin = user.node.isSiteAdmin,
-    u.email = user.node.email,
-    u.company = user.node.company,
-    u.lastupdated = $UpdateTag
-
-    MERGE (u)-[r:MEMBER_OF]->(org)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $UpdateTag
-    """
-    neo4j_session.run(
-        query,
-        OrgUrl=org_data['url'],
-        OrgLogin=org_data['login'],
-        UserData=user_data,
-        UpdateTag=update_tag,
+    logger.info(f"Loading {len(user_data)} GitHub users to the graph")
+    load(
+        neo4j_session,
+        node_schema,
+        user_data,
+        lastupdated=update_tag,
+        org_url=org_data['url'],
+    )
+
+
+@timeit
+def load_organization(
+    neo4j_session: neo4j.Session,
+    node_schema: GitHubOrganizationSchema,
+    org_data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info(f"Loading {len(org_data)} GitHub organization to the graph")
+    load(
+        neo4j_session,
+        node_schema,
+        org_data,
+        lastupdated=update_tag,
     )
 
 
 @timeit
 def sync(
         neo4j_session: neo4j.Session,
-        common_job_parameters: Dict[str, Any],
+        common_job_parameters: Dict,
         github_api_key: str,
         github_url: str,
         organization: str,
 ) -> None:
     logger.info("Syncing GitHub users")
-    user_data, org_data = get(github_api_key, github_url, organization)
-    load_organization_users(neo4j_session, user_data, org_data, common_job_parameters['UPDATE_TAG'])
-    run_cleanup_job('github_users_cleanup.json', neo4j_session, common_job_parameters)
+    user_data, org_data = get_users(github_api_key, github_url, organization)
+    owners_data, org_data = get_enterprise_owners(github_api_key, github_url, organization)
+    processed_affiliated_user_data, processed_unaffiliated_user_data = (
+        transform_users(user_data, owners_data, org_data)
+    )
+    load_organization(
+        neo4j_session, GitHubOrganizationSchema(), [org_data],
+        common_job_parameters['UPDATE_TAG'],
+    )
+    load_users(
+        neo4j_session, GitHubOrganizationUserSchema(), processed_affiliated_user_data, org_data,
+        common_job_parameters['UPDATE_TAG'],
+    )
+    load_users(
+        neo4j_session, GitHubUnaffiliatedUserSchema(), processed_unaffiliated_user_data, org_data,
+        common_job_parameters['UPDATE_TAG'],
+    )
+    # no automated cleanup job for users because user node has no sub_resource_relationship
+    run_cleanup_job('github_org_and_users_cleanup.json', neo4j_session, common_job_parameters)
     merge_module_sync_metadata(
         neo4j_session,
         group_type='GitHubOrganization',

cartography/intel/okta/users.py

@@ -150,7 +150,8 @@ def _load_okta_users(
     new_user.okta_last_updated = user_data.okta_last_updated,
     new_user.password_changed = user_data.password_changed,
     new_user.transition_to_status = user_data.transition_to_status,
-    new_user.lastupdated = $okta_update_tag
+    new_user.lastupdated = $okta_update_tag,
+    new_user :UserAccount
     WITH new_user, org
     MERGE (org)-[org_r:RESOURCE]->(new_user)
     ON CREATE SET org_r.firstseen = timestamp()

cartography/intel/semgrep/__init__.py

@@ -26,5 +26,5 @@ def start_semgrep_ingestion(
     # sync_deployment must be called first since it populates common_job_parameters
     # with the deployment ID and slug, which are required by the other sync functions
     sync_deployment(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
-    sync_dependencies(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+    sync_dependencies(neo4j_session, config.semgrep_app_token, config.semgrep_dependency_ecosystems, config.update_tag, common_job_parameters)  # noqa: E501
     sync_findings(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)