cartography 0.96.0rc3__py3-none-any.whl → 0.96.1__py3-none-any.whl

cartography/intel/github/repos.py

@@ -1,5 +1,6 @@
 import configparser
 import logging
+from collections import namedtuple
 from string import Template
 from typing import Any
 from typing import Dict
@@ -12,11 +13,28 @@ from packaging.requirements import Requirement
 from packaging.utils import canonicalize_name
 
 from cartography.intel.github.util import fetch_all
+from cartography.intel.github.util import PaginatedGraphqlData
+from cartography.util import backoff_handler
+from cartography.util import retries_with_backoff
 from cartography.util import run_cleanup_job
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+
+# Representation of a user's permission level and affiliation to a GitHub repo. See:
+# - Permission: https://docs.github.com/en/graphql/reference/enums#repositorypermission
+# - Affiliation: https://docs.github.com/en/graphql/reference/enums#collaboratoraffiliation
+UserAffiliationAndRepoPermission = namedtuple(
+    'UserAffiliationAndRepoPermission',
+    [
+        'user',  # Dict
+        'permission',  # 'WRITE', 'MAINTAIN', 'ADMIN', etc
+        'affiliation',  # 'OUTSIDE', 'DIRECT'
+    ],
+)
+
+
 GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
     query($login: String!, $cursor: String) {
     organization(login: $login)
@@ -59,17 +77,11 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
                         login
                         __typename
                     }
-                    collaborators(affiliation: OUTSIDE, first: 50) {
-                        edges {
-                            permission
-                        }
-                        nodes {
-                            url
-                            login
-                            name
-                            email
-                            company
-                        }
+                    directCollaborators: collaborators(first: 100, affiliation: DIRECT) {
+                        totalCount
+                    }
+                    outsideCollaborators: collaborators(first: 100, affiliation: OUTSIDE) {
+                        totalCount
                     }
                     requirements:object(expression: "HEAD:requirements.txt") {
                         ... on Blob {
@@ -89,6 +101,142 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
 # Note: In the above query, `HEAD` references the default branch.
 # See https://stackoverflow.com/questions/48935381/github-graphql-api-default-branch-in-repository
 
+GITHUB_REPO_COLLABS_PAGINATED_GRAPHQL = """
+    query($login: String!, $repo: String!, $affiliation: CollaboratorAffiliation!, $cursor: String) {
+        organization(login: $login) {
+            url
+            login
+            repository(name: $repo){
+                name
+                collaborators(first: 50, affiliation: $affiliation, after: $cursor) {
+                    edges {
+                        permission
+                    }
+                    nodes {
+                        url
+                        login
+                        name
+                        email
+                        company
+                    }
+                    pageInfo{
+                        endCursor
+                        hasNextPage
+                    }
+                }
+            }
+        }
+        rateLimit {
+            limit
+            cost
+            remaining
+            resetAt
+        }
+    }
+    """
+
+
+def _get_repo_collaborators_inner_func(
+        org: str,
+        api_url: str,
+        token: str,
+        repo_name: str,
+        affiliation: str,
+        collab_users: list[dict[str, Any]],
+        collab_permission: list[str],
+) -> None:
+    logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
+    collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
+
+    # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
+    # however sometimes GitHub returns None, as in issue 1334 and 1404.
+    for collab in collaborators.nodes or []:
+        collab_users.append(collab)
+
+    # The `or []` is because `.edges` can be None.
+    for perm in collaborators.edges or []:
+        collab_permission.append(perm['permission'])
+
+
+def _get_repo_collaborators_for_multiple_repos(
+        repo_raw_data: list[dict[str, Any]],
+        affiliation: str,
+        org: str,
+        api_url: str,
+        token: str,
+) -> dict[str, list[UserAffiliationAndRepoPermission]]:
+    """
+    For every repo in the given list, retrieve the collaborators.
+    :param repo_raw_data: A list of dicts representing repos. See tests.data.github.repos.GET_REPOS for data shape.
+    :param affiliation: The type of affiliation to retrieve collaborators for. Either 'DIRECT' or 'OUTSIDE'.
+      See https://docs.github.com/en/graphql/reference/enums#collaboratoraffiliation
+    :param org: The name of the target Github organization as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param token: The Github API token as string.
+    :return: A dictionary of repo URL to list of UserAffiliationAndRepoPermission
+    """
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    for repo in repo_raw_data:
+        repo_name = repo['name']
+        repo_url = repo['url']
+
+        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
+                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
+            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
+            result[repo_url] = []
+            continue
+
+        collab_users: List[dict[str, Any]] = []
+        collab_permission: List[str] = []
+
+        retries_with_backoff(
+            _get_repo_collaborators_inner_func,
+            TypeError,
+            5,
+            backoff_handler,
+        )(
+            org=org,
+            api_url=api_url,
+            token=token,
+            repo_name=repo_name,
+            affiliation=affiliation,
+            collab_users=collab_users,
+            collab_permission=collab_permission,
+        )
+
+        result[repo_url] = [
+            UserAffiliationAndRepoPermission(user, permission, affiliation)
+            for user, permission in zip(collab_users, collab_permission)
+        ]
+    return result
+
+
+def _get_repo_collaborators(
+        token: str, api_url: str, organization: str, repo: str, affiliation: str,
+) -> PaginatedGraphqlData:
+    """
+    Retrieve a list of collaborators for a given repository, as described in
+    https://docs.github.com/en/graphql/reference/objects#repositorycollaboratorconnection.
+    :param token: The Github API token as string.
+    :param api_url: The Github v4 API endpoint as string.
+    :param organization: The name of the target Github organization as string.
+    :pram repo: The name of the target Github repository as string.
+    :param affiliation: The type of affiliation to retrieve collaborators for. Either 'DIRECT' or 'OUTSIDE'.
+      See https://docs.github.com/en/graphql/reference/enums#collaboratoraffiliation
+    :return: A list of dicts representing repos. See tests.data.github.repos for data shape.
+    """
+    collaborators, _ = fetch_all(
+        token,
+        api_url,
+        organization,
+        GITHUB_REPO_COLLABS_PAGINATED_GRAPHQL,
+        'repository',
+        resource_inner_type='collaborators',
+        repo=repo,
+        affiliation=affiliation,
+    )
+    return collaborators
+
 
 @timeit
 def get(token: str, api_url: str, organization: str) -> List[Dict]:
@@ -111,11 +259,18 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
     return repos.nodes
 
 
-def transform(repos_json: List[Dict]) -> Dict:
+def transform(
+    repos_json: List[Dict], direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+    outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+) -> Dict:
     """
     Parses the JSON returned from GitHub API to create data for graph ingestion
-    :param repos_json: the list of individual repository nodes from GitHub. See tests.data.github.repos.GET_REPOS for
-    data shape.
+    :param repos_json: the list of individual repository nodes from GitHub.
+        See tests.data.github.repos.GET_REPOS for data shape.
+    :param direct_collaborators: dict of repo URL to list of direct collaborators.
+        See tests.data.github.repos.DIRECT_COLLABORATORS for data shape.
+    :param outside_collaborators: dict of repo URL to list of outside collaborators.
+        See tests.data.github.repos.OUTSIDE_COLLABORATORS for data shape.
     :return: Dict containing the repos, repo->language mapping, owners->repo mapping, outside collaborators->repo
     mapping, and Python requirements files (if any) in a repo.
     """
@@ -123,7 +278,10 @@ def transform(repos_json: List[Dict]) -> Dict:
     transformed_repo_languages: List[Dict] = []
     transformed_repo_owners: List[Dict] = []
     # See https://docs.github.com/en/graphql/reference/enums#repositorypermission
-    transformed_collaborators: Dict[str, List[Any]] = {
+    transformed_outside_collaborators: Dict[str, List[Any]] = {
+        'ADMIN': [], 'MAINTAIN': [], 'READ': [], 'TRIAGE': [], 'WRITE': [],
+    }
+    transformed_direct_collaborators: Dict[str, List[Any]] = {
         'ADMIN': [], 'MAINTAIN': [], 'READ': [], 'TRIAGE': [], 'WRITE': [],
     }
     transformed_requirements_files: List[Dict] = []
@@ -131,14 +289,22 @@ def transform(repos_json: List[Dict]) -> Dict:
         _transform_repo_languages(repo_object['url'], repo_object, transformed_repo_languages)
         _transform_repo_objects(repo_object, transformed_repo_list)
         _transform_repo_owners(repo_object['owner']['url'], repo_object, transformed_repo_owners)
-        _transform_collaborators(repo_object['collaborators'], repo_object['url'], transformed_collaborators)
+        _transform_collaborators(
+            repo_object['url'], outside_collaborators[repo_object['url']],
+            transformed_outside_collaborators,
+        )
+        _transform_collaborators(
+            repo_object['url'], direct_collaborators[repo_object['url']],
+            transformed_direct_collaborators,
+        )
         _transform_requirements_txt(repo_object['requirements'], repo_object['url'], transformed_requirements_files)
         _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_object['url'], transformed_requirements_files)
     results = {
         'repos': transformed_repo_list,
         'repo_languages': transformed_repo_languages,
         'repo_owners': transformed_repo_owners,
-        'repo_collaborators': transformed_collaborators,
+        'repo_outside_collaborators': transformed_outside_collaborators,
+        'repo_direct_collaborators': transformed_direct_collaborators,
         'python_requirements': transformed_requirements_files,
     }
     return results
@@ -229,11 +395,15 @@ def _transform_repo_languages(repo_url: str, repo: Dict, repo_languages: List[Di
             })
 
 
-def _transform_collaborators(collaborators: Dict, repo_url: str, transformed_collaborators: Dict) -> None:
+def _transform_collaborators(
+        repo_url: str, collaborators: List[UserAffiliationAndRepoPermission], transformed_collaborators: Dict,
+) -> None:
     """
-    Performs data adjustments for outside collaborators in a GitHub repo.
+    Performs data adjustments for collaborators in a GitHub repo.
     Output data shape = [{permission, repo_url, url (the user's URL), login, name}, ...]
-    :param collaborators: See cartography.tests.data.github.repos for data shape.
+    :param collaborators: For data shape, see
+        cartography.tests.data.github.repos.DIRECT_COLLABORATORS
+        cartography.tests.data.github.repos.OUTSIDE_COLLABORATORS
     :param repo_url: The URL of the GitHub repo.
     :param transformed_collaborators: Output dict. Data shape =
     {'ADMIN': [{ user }, ...], 'MAINTAIN': [{ user }, ...], 'READ': [ ... ], 'TRIAGE': [ ... ], 'WRITE': [ ... ]}
@@ -241,10 +411,11 @@ def _transform_collaborators(collaborators: Dict, repo_url: str, transformed_col
     """
     # `collaborators` is sometimes None
     if collaborators:
-        for idx, user in enumerate(collaborators['nodes']):
-            user_permission = collaborators['edges'][idx]['permission']
+        for collaborator in collaborators:
+            user = collaborator.user
             user['repo_url'] = repo_url
-            transformed_collaborators[user_permission].append(user)
+            user['affiliation'] = collaborator.affiliation
+            transformed_collaborators[collaborator.permission].append(user)
 
 
 def _transform_requirements_txt(
@@ -482,7 +653,7 @@ def load_github_owners(neo4j_session: neo4j.Session, update_tag: int, repo_owner
 
 
 @timeit
-def load_collaborators(neo4j_session: neo4j.Session, update_tag: int, collaborators: Dict) -> None:
+def load_collaborators(neo4j_session: neo4j.Session, update_tag: int, collaborators: Dict, affiliation: str) -> None:
     query = Template("""
     UNWIND $UserData as user
 
@@ -502,7 +673,7 @@ def load_collaborators(neo4j_session: neo4j.Session, update_tag: int, collaborat
     SET o.lastupdated = $UpdateTag
     """)
     for collab_type in collaborators.keys():
-        relationship_label = f"OUTSIDE_COLLAB_{collab_type}"
+        relationship_label = f"{affiliation}_COLLAB_{collab_type}"
         neo4j_session.run(
             query.safe_substitute(rel_label=relationship_label),
             UserData=collaborators[collab_type],
@@ -515,7 +686,12 @@ def load(neo4j_session: neo4j.Session, common_job_parameters: Dict, repo_data: D
     load_github_repos(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repos'])
     load_github_owners(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_owners'])
     load_github_languages(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_languages'])
-    load_collaborators(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_collaborators'])
+    load_collaborators(
+        neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_direct_collaborators'], 'DIRECT',
+    )
+    load_collaborators(
+        neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['repo_outside_collaborators'], 'OUTSIDE',
+    )
     load_python_requirements(neo4j_session, common_job_parameters['UPDATE_TAG'], repo_data['python_requirements'])
 
 
@@ -561,6 +737,12 @@ def sync(
     """
     logger.info("Syncing GitHub repos")
     repos_json = get(github_api_key, github_url, organization)
-    repo_data = transform(repos_json)
+    direct_collabs = _get_repo_collaborators_for_multiple_repos(
+        repos_json, "DIRECT", organization, github_url, github_api_key,
+    )
+    outside_collabs = _get_repo_collaborators_for_multiple_repos(
+        repos_json, "OUTSIDE", organization, github_url, github_api_key,
+    )
+    repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
     run_cleanup_job('github_repos_cleanup.json', neo4j_session, common_job_parameters)

cartography/intel/github/teams.py

@@ -1,6 +1,5 @@
 import logging
 from collections import namedtuple
-from time import sleep
 from typing import Any
 from typing import Dict
 from typing import List
@@ -13,11 +12,27 @@ from cartography.graph.job import GraphJob
 from cartography.intel.github.util import fetch_all
 from cartography.intel.github.util import PaginatedGraphqlData
 from cartography.models.github.teams import GitHubTeamSchema
+from cartography.util import retries_with_backoff
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
+# A team's permission on a repo: https://docs.github.com/en/graphql/reference/enums#repositorypermission
 RepoPermission = namedtuple('RepoPermission', ['repo_url', 'permission'])
+# A team member's role: https://docs.github.com/en/graphql/reference/enums#teammemberrole
+UserRole = namedtuple('UserRole', ['user_url', 'role'])
+
+
+def backoff_handler(details: Dict) -> None:
+    """
+    Custom backoff handler for GitHub calls in this module.
+    """
+    team_name = details['kwargs'].get('team_name') or 'not present in kwargs'
+    updated_details = {**details, 'team_name': team_name}
+    logger.warning(
+        "Backing off {wait:0.1f} seconds after {tries} tries. Calling function {target} for team {team_name}"
+        .format(**updated_details),
+    )
 
 
 @timeit
@@ -32,7 +47,10 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
                         slug
                         url
                         description
-                        repositories(first: 100) {
+                        repositories {
+                            totalCount
+                        }
+                        members(membership: IMMEDIATE) {
                             totalCount
                         }
                     }
@@ -47,6 +65,26 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
     return fetch_all(token, api_url, org, org_teams_gql, 'teams')
 
 
+def _get_teams_repos_inner_func(
+        org: str,
+        api_url: str,
+        token: str,
+        team_name: str,
+        repo_urls: list[str],
+        repo_permissions: list[str],
+) -> None:
+    logger.info(f"Loading team repos for {team_name}.")
+    team_repos = _get_team_repos(org, api_url, token, team_name)
+
+    # The `or []` is because `.nodes` can be None. See:
+    # https://docs.github.com/en/graphql/reference/objects#teamrepositoryconnection
+    for repo in team_repos.nodes or []:
+        repo_urls.append(repo['url'])
+    # The `or []` is because `.edges` can be None.
+    for edge in team_repos.edges or []:
+        repo_permissions.append(edge['permission'])
+
+
 @timeit
 def _get_team_repos_for_multiple_teams(
         team_raw_data: list[dict[str, Any]],
@@ -64,36 +102,22 @@ def _get_team_repos_for_multiple_teams(
             result[team_name] = []
             continue
 
-        repo_urls = []
-        repo_permissions = []
-
-        max_tries = 5
-
-        for current_try in range(1, max_tries + 1):
-            team_repos = _get_team_repos(org, api_url, token, team_name)
-
-            try:
-                # The `or []` is because `.nodes` can be None. See:
-                # https://docs.github.com/en/graphql/reference/objects#teamrepositoryconnection
-                for repo in team_repos.nodes or []:
-                    repo_urls.append(repo['url'])
-
-                # The `or []` is because `.edges` can be None.
-                for edge in team_repos.edges or []:
-                    repo_permissions.append(edge['permission'])
-                # We're done! Break out of the retry loop.
-                break
-
-            except TypeError:
-                # Handles issue #1334
-                logger.warning(
-                    f"GitHub returned None when trying to find repo or permission data for team {team_name}.",
-                    exc_info=True,
-                )
-                if current_try == max_tries:
-                    raise RuntimeError(f"GitHub returned a None repo url for team {team_name}, retries exhausted.")
-                sleep(current_try ** 2)
+        repo_urls: List[str] = []
+        repo_permissions: List[str] = []
 
+        retries_with_backoff(
+            _get_teams_repos_inner_func,
+            TypeError,
+            5,
+            backoff_handler,
+        )(
+            org=org,
+            api_url=api_url,
+            token=token,
+            team_name=team_name,
+            repo_urls=repo_urls,
+            repo_permissions=repo_permissions,
+        )
         # Shape = [(repo_url, 'WRITE'), ...]]
         result[team_name] = [RepoPermission(url, perm) for url, perm in zip(repo_urls, repo_permissions)]
     return result
@@ -142,10 +166,97 @@ def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedG
     return team_repos
 
 
+def _get_team_users_for_multiple_teams(
+        team_raw_data: list[dict[str, Any]],
+        org: str,
+        api_url: str,
+        token: str,
+) -> dict[str, list[UserRole]]:
+    result: dict[str, list[UserRole]] = {}
+    for team in team_raw_data:
+        team_name = team['slug']
+        user_count = team['members']['totalCount']
+
+        if user_count == 0:
+            # This team has no users so let's move on
+            result[team_name] = []
+            continue
+
+        user_urls: List[str] = []
+        user_roles: List[str] = []
+
+        def get_teams_users_inner_func(
+            org: str, api_url: str, token: str, team_name: str,
+            user_urls: List[str], user_roles: List[str],
+        ) -> None:
+            logger.info(f"Loading team users for {team_name}.")
+            team_users = _get_team_users(org, api_url, token, team_name)
+            # The `or []` is because `.nodes` can be None. See:
+            # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
+            for user in team_users.nodes or []:
+                user_urls.append(user['url'])
+            # The `or []` is because `.edges` can be None.
+            for edge in team_users.edges or []:
+                user_roles.append(edge['role'])
+
+        retries_with_backoff(get_teams_users_inner_func, TypeError, 5, backoff_handler)(
+            org=org, api_url=api_url, token=token, team_name=team_name, user_urls=user_urls, user_roles=user_roles,
+        )
+
+        # Shape = [(user_url, 'MAINTAINER'), ...]]
+        result[team_name] = [UserRole(url, role) for url, role in zip(user_urls, user_roles)]
+    return result
+
+
+@timeit
+def _get_team_users(org: str, api_url: str, token: str, team: str) -> PaginatedGraphqlData:
+    team_users_gql = """
+    query($login: String!, $team: String!, $cursor: String) {
+        organization(login: $login) {
+            url
+            login
+            team(slug: $team) {
+                slug
+                members(first: 100, after: $cursor, membership: IMMEDIATE) {
+                    totalCount
+                    nodes {
+                        url
+                    }
+                    edges {
+                        role
+                    }
+                    pageInfo {
+                        endCursor
+                        hasNextPage
+                    }
+                }
+            }
+        }
+        rateLimit {
+            limit
+            cost
+            remaining
+            resetAt
+        }
+    }
+    """
+    team_users, _ = fetch_all(
+        token,
+        api_url,
+        org,
+        team_users_gql,
+        'team',
+        resource_inner_type='members',
+        team=team,
+    )
+    return team_users
+
+
 def transform_teams(
         team_paginated_data: PaginatedGraphqlData,
         org_data: Dict[str, Any],
         team_repo_data: dict[str, list[RepoPermission]],
+        team_user_data: dict[str, list[UserRole]],
 ) -> list[dict[str, Any]]:
     result = []
     for team in team_paginated_data.nodes:
@@ -155,19 +266,29 @@ def transform_teams(
             'url': team['url'],
             'description': team['description'],
             'repo_count': team['repositories']['totalCount'],
+            'member_count': team['members']['totalCount'],
             'org_url': org_data['url'],
             'org_login': org_data['login'],
         }
         repo_permissions = team_repo_data[team_name]
-        if not repo_permissions:
+        user_roles = team_user_data[team_name]
+
+        if not repo_permissions and not user_roles:
             result.append(repo_info)
             continue
 
-        # `permission` can be one of ADMIN, READ, WRITE, TRIAGE, or MAINTAIN
-        for repo_url, permission in repo_permissions:
-            repo_info_copy = repo_info.copy()
-            repo_info_copy[permission] = repo_url
-            result.append(repo_info_copy)
+        if repo_permissions:
+            # `permission` can be one of ADMIN, READ, WRITE, TRIAGE, or MAINTAIN
+            for repo_url, permission in repo_permissions:
+                repo_info_copy = repo_info.copy()
+                repo_info_copy[permission] = repo_url
+                result.append(repo_info_copy)
+        if user_roles:
+            # `role` can be one of MAINTAINER, MEMBER
+            for user_url, role in user_roles:
+                repo_info_copy = repo_info.copy()
+                repo_info_copy[role] = user_url
+                result.append(repo_info_copy)
     return result
 
 
@@ -203,7 +324,8 @@ def sync_github_teams(
 ) -> None:
     teams_paginated, org_data = get_teams(organization, github_url, github_api_key)
     team_repos = _get_team_repos_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
-    processed_data = transform_teams(teams_paginated, org_data, team_repos)
+    team_users = _get_team_users_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
+    processed_data = transform_teams(teams_paginated, org_data, team_repos, team_users)
     load_team_repos(neo4j_session, processed_data, common_job_parameters['UPDATE_TAG'], org_data['url'])
     common_job_parameters['org_url'] = org_data['url']
     cleanup(neo4j_session, common_job_parameters)