cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/intel/github/teams.py

@@ -18,28 +18,33 @@ from cartography.util import timeit
 logger = logging.getLogger(__name__)
 
 # A team's permission on a repo: https://docs.github.com/en/graphql/reference/enums#repositorypermission
-RepoPermission = namedtuple('RepoPermission', ['repo_url', 'permission'])
+RepoPermission = namedtuple("RepoPermission", ["repo_url", "permission"])
 # A team member's role: https://docs.github.com/en/graphql/reference/enums#teammemberrole
-UserRole = namedtuple('UserRole', ['user_url', 'role'])
+UserRole = namedtuple("UserRole", ["user_url", "role"])
 # Unlike the other tuples here, there is no qualification (like 'role' or 'permission') to the relationship.
 # A child team is just a child team: https://docs.github.com/en/graphql/reference/objects#teamconnection
-ChildTeam = namedtuple('ChildTeam', ['team_url'])
+ChildTeam = namedtuple("ChildTeam", ["team_url"])
 
 
 def backoff_handler(details: Dict) -> None:
     """
     Custom backoff handler for GitHub calls in this module.
     """
-    team_name = details['kwargs'].get('team_name') or 'not present in kwargs'
-    updated_details = {**details, 'team_name': team_name}
+    team_name = details["kwargs"].get("team_name") or "not present in kwargs"
+    updated_details = {**details, "team_name": team_name}
     logger.warning(
-        "Backing off {wait:0.1f} seconds after {tries} tries. Calling function {target} for team {team_name}"
-        .format(**updated_details),
+        "Backing off {wait:0.1f} seconds after {tries} tries. Calling function {target} for team {team_name}".format(
+            **updated_details,
+        ),
     )
 
 
 @timeit
-def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData, Dict[str, Any]]:
+def get_teams(
+    org: str,
+    api_url: str,
+    token: str,
+) -> Tuple[PaginatedGraphqlData, Dict[str, Any]]:
     org_teams_gql = """
         query($login: String!, $cursor: String) {
             organization(login: $login) {
@@ -68,16 +73,16 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
             }
         }
     """
-    return fetch_all(token, api_url, org, org_teams_gql, 'teams')
+    return fetch_all(token, api_url, org, org_teams_gql, "teams")
 
 
 def _get_teams_repos_inner_func(
-        org: str,
-        api_url: str,
-        token: str,
-        team_name: str,
-        repo_urls: list[str],
-        repo_permissions: list[str],
+    org: str,
+    api_url: str,
+    token: str,
+    team_name: str,
+    repo_urls: list[str],
+    repo_permissions: list[str],
 ) -> None:
     logger.info(f"Loading team repos for {team_name}.")
     team_repos = _get_team_repos(org, api_url, token, team_name)
@@ -85,23 +90,23 @@ def _get_teams_repos_inner_func(
     # The `or []` is because `.nodes` can be None. See:
     # https://docs.github.com/en/graphql/reference/objects#teamrepositoryconnection
     for repo in team_repos.nodes or []:
-        repo_urls.append(repo['url'])
+        repo_urls.append(repo["url"])
     # The `or []` is because `.edges` can be None.
     for edge in team_repos.edges or []:
-        repo_permissions.append(edge['permission'])
+        repo_permissions.append(edge["permission"])
 
 
 @timeit
 def _get_team_repos_for_multiple_teams(
-        team_raw_data: list[dict[str, Any]],
-        org: str,
-        api_url: str,
-        token: str,
+    team_raw_data: list[dict[str, Any]],
+    org: str,
+    api_url: str,
+    token: str,
 ) -> dict[str, list[RepoPermission]]:
     result: dict[str, list[RepoPermission]] = {}
     for team in team_raw_data:
-        team_name = team['slug']
-        repo_count = team['repositories']['totalCount']
+        team_name = team["slug"]
+        repo_count = team["repositories"]["totalCount"]
 
         if repo_count == 0:
             # This team has access to no repos so let's move on
@@ -125,12 +130,19 @@ def _get_team_repos_for_multiple_teams(
             repo_permissions=repo_permissions,
         )
         # Shape = [(repo_url, 'WRITE'), ...]]
-        result[team_name] = [RepoPermission(url, perm) for url, perm in zip(repo_urls, repo_permissions)]
+        result[team_name] = [
+            RepoPermission(url, perm) for url, perm in zip(repo_urls, repo_permissions)
+        ]
     return result
 
 
 @timeit
-def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedGraphqlData:
+def _get_team_repos(
+    org: str,
+    api_url: str,
+    token: str,
+    team: str,
+) -> PaginatedGraphqlData:
     team_repos_gql = """
     query($login: String!, $team: String!, $cursor: String) {
         organization(login: $login) {
@@ -165,38 +177,42 @@ def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedG
         api_url,
         org,
         team_repos_gql,
-        'team',
-        resource_inner_type='repositories',
+        "team",
+        resource_inner_type="repositories",
         team=team,
     )
     return team_repos
 
 
 def _get_teams_users_inner_func(
-        org: str, api_url: str, token: str, team_name: str,
-        user_urls: List[str], user_roles: List[str],
+    org: str,
+    api_url: str,
+    token: str,
+    team_name: str,
+    user_urls: List[str],
+    user_roles: List[str],
 ) -> None:
     logger.info(f"Loading team users for {team_name}.")
     team_users = _get_team_users(org, api_url, token, team_name)
     # The `or []` is because `.nodes` can be None. See:
     # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
     for user in team_users.nodes or []:
-        user_urls.append(user['url'])
+        user_urls.append(user["url"])
     # The `or []` is because `.edges` can be None.
     for edge in team_users.edges or []:
-        user_roles.append(edge['role'])
+        user_roles.append(edge["role"])
 
 
 def _get_team_users_for_multiple_teams(
-        team_raw_data: list[dict[str, Any]],
-        org: str,
-        api_url: str,
-        token: str,
+    team_raw_data: list[dict[str, Any]],
+    org: str,
+    api_url: str,
+    token: str,
 ) -> dict[str, list[UserRole]]:
     result: dict[str, list[UserRole]] = {}
     for team in team_raw_data:
-        team_name = team['slug']
-        user_count = team['members']['totalCount']
+        team_name = team["slug"]
+        user_count = team["members"]["totalCount"]
 
         if user_count == 0:
             # This team has no users so let's move on
@@ -206,17 +222,34 @@ def _get_team_users_for_multiple_teams(
         user_urls: List[str] = []
         user_roles: List[str] = []
 
-        retries_with_backoff(_get_teams_users_inner_func, TypeError, 5, backoff_handler)(
-            org=org, api_url=api_url, token=token, team_name=team_name, user_urls=user_urls, user_roles=user_roles,
+        retries_with_backoff(
+            _get_teams_users_inner_func,
+            TypeError,
+            5,
+            backoff_handler,
+        )(
+            org=org,
+            api_url=api_url,
+            token=token,
+            team_name=team_name,
+            user_urls=user_urls,
+            user_roles=user_roles,
         )
 
         # Shape = [(user_url, 'MAINTAINER'), ...]]
-        result[team_name] = [UserRole(url, role) for url, role in zip(user_urls, user_roles)]
+        result[team_name] = [
+            UserRole(url, role) for url, role in zip(user_urls, user_roles)
+        ]
     return result
 
 
 @timeit
-def _get_team_users(org: str, api_url: str, token: str, team: str) -> PaginatedGraphqlData:
+def _get_team_users(
+    org: str,
+    api_url: str,
+    token: str,
+    team: str,
+) -> PaginatedGraphqlData:
     team_users_gql = """
     query($login: String!, $team: String!, $cursor: String) {
         organization(login: $login) {
@@ -252,35 +285,39 @@ def _get_team_users(org: str, api_url: str, token: str, team: str) -> PaginatedG
         api_url,
         org,
         team_users_gql,
-        'team',
-        resource_inner_type='members',
+        "team",
+        resource_inner_type="members",
         team=team,
     )
     return team_users
 
 
 def _get_child_teams_inner_func(
-        org: str, api_url: str, token: str, team_name: str, team_urls: List[str],
+    org: str,
+    api_url: str,
+    token: str,
+    team_name: str,
+    team_urls: List[str],
 ) -> None:
     logger.info(f"Loading child teams for {team_name}.")
     child_teams = _get_child_teams(org, api_url, token, team_name)
     # The `or []` is because `.nodes` can be None. See:
     # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
     for cteam in child_teams.nodes or []:
-        team_urls.append(cteam['url'])
+        team_urls.append(cteam["url"])
     # No edges to process here, the GitHub response for child teams has no relevant info in edges.
 
 
 def _get_child_teams_for_multiple_teams(
-        team_raw_data: list[dict[str, Any]],
-        org: str,
-        api_url: str,
-        token: str,
+    team_raw_data: list[dict[str, Any]],
+    org: str,
+    api_url: str,
+    token: str,
 ) -> dict[str, list[ChildTeam]]:
     result: dict[str, list[ChildTeam]] = {}
     for team in team_raw_data:
-        team_name = team['slug']
-        team_count = team['childTeams']['totalCount']
+        team_name = team["slug"]
+        team_count = team["childTeams"]["totalCount"]
 
         if team_count == 0:
             # This team has no child teams so let's move on
@@ -289,15 +326,29 @@ def _get_child_teams_for_multiple_teams(
 
         team_urls: List[str] = []
 
-        retries_with_backoff(_get_child_teams_inner_func, TypeError, 5, backoff_handler)(
-            org=org, api_url=api_url, token=token, team_name=team_name, team_urls=team_urls,
+        retries_with_backoff(
+            _get_child_teams_inner_func,
+            TypeError,
+            5,
+            backoff_handler,
+        )(
+            org=org,
+            api_url=api_url,
+            token=token,
+            team_name=team_name,
+            team_urls=team_urls,
         )
 
         result[team_name] = [ChildTeam(url) for url in team_urls]
     return result
 
 
-def _get_child_teams(org: str, api_url: str, token: str, team: str) -> PaginatedGraphqlData:
+def _get_child_teams(
+    org: str,
+    api_url: str,
+    token: str,
+    team: str,
+) -> PaginatedGraphqlData:
     team_users_gql = """
     query($login: String!, $team: String!, $cursor: String) {
         organization(login: $login) {
@@ -330,32 +381,32 @@ def _get_child_teams(org: str, api_url: str, token: str, team: str) -> Paginated
         api_url,
         org,
         team_users_gql,
-        'team',
-        resource_inner_type='childTeams',
+        "team",
+        resource_inner_type="childTeams",
         team=team,
     )
     return team_users
 
 
 def transform_teams(
-        team_paginated_data: PaginatedGraphqlData,
-        org_data: Dict[str, Any],
-        team_repo_data: dict[str, list[RepoPermission]],
-        team_user_data: dict[str, list[UserRole]],
-        team_child_team_data: dict[str, list[ChildTeam]],
+    team_paginated_data: PaginatedGraphqlData,
+    org_data: Dict[str, Any],
+    team_repo_data: dict[str, list[RepoPermission]],
+    team_user_data: dict[str, list[UserRole]],
+    team_child_team_data: dict[str, list[ChildTeam]],
 ) -> list[dict[str, Any]]:
     result = []
     for team in team_paginated_data.nodes:
-        team_name = team['slug']
+        team_name = team["slug"]
         repo_info = {
-            'name': team_name,
-            'url': team['url'],
-            'description': team['description'],
-            'repo_count': team['repositories']['totalCount'],
-            'member_count': team['members']['totalCount'],
-            'child_team_count': team['childTeams']['totalCount'],
-            'org_url': org_data['url'],
-            'org_login': org_data['login'],
+            "name": team_name,
+            "url": team["url"],
+            "description": team["description"],
+            "repo_count": team["repositories"]["totalCount"],
+            "member_count": team["members"]["totalCount"],
+            "child_team_count": team["childTeams"]["totalCount"],
+            "org_url": org_data["url"],
+            "org_login": org_data["login"],
         }
         repo_permissions = team_repo_data[team_name]
         user_roles = team_user_data[team_name]
@@ -384,17 +435,17 @@ def transform_teams(
                 # or here: https://docs.github.com/en/graphql/reference/enums#teammembershiptype
                 # We label the relationship as 'MEMBER_OF_TEAM' here because it is in line with
                 # other similar relationships in Cartography.
-                repo_info_copy['MEMBER_OF_TEAM'] = team_url
+                repo_info_copy["MEMBER_OF_TEAM"] = team_url
                 result.append(repo_info_copy)
     return result
 
 
 @timeit
 def load_team_repos(
-        neo4j_session: neo4j.Session,
-        data: List[Dict[str, Any]],
-        update_tag: int,
-        organization_url: str,
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+    organization_url: str,
 ) -> None:
     logger.info(f"Loading {len(data)} GitHub team-repos to the graph")
     load(
@@ -407,23 +458,54 @@ def load_team_repos(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
-    GraphJob.from_node_schema(GitHubTeamSchema(), common_job_parameters).run(neo4j_session)
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    GraphJob.from_node_schema(GitHubTeamSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
 def sync_github_teams(
-        neo4j_session: neo4j.Session,
-        common_job_parameters: Dict[str, Any],
-        github_api_key: str,
-        github_url: str,
-        organization: str,
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+    github_api_key: str,
+    github_url: str,
+    organization: str,
 ) -> None:
     teams_paginated, org_data = get_teams(organization, github_url, github_api_key)
-    team_repos = _get_team_repos_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
-    team_users = _get_team_users_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
-    team_children = _get_child_teams_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
-    processed_data = transform_teams(teams_paginated, org_data, team_repos, team_users, team_children)
-    load_team_repos(neo4j_session, processed_data, common_job_parameters['UPDATE_TAG'], org_data['url'])
-    common_job_parameters['org_url'] = org_data['url']
+    team_repos = _get_team_repos_for_multiple_teams(
+        teams_paginated.nodes,
+        organization,
+        github_url,
+        github_api_key,
+    )
+    team_users = _get_team_users_for_multiple_teams(
+        teams_paginated.nodes,
+        organization,
+        github_url,
+        github_api_key,
+    )
+    team_children = _get_child_teams_for_multiple_teams(
+        teams_paginated.nodes,
+        organization,
+        github_url,
+        github_api_key,
+    )
+    processed_data = transform_teams(
+        teams_paginated,
+        org_data,
+        team_repos,
+        team_users,
+        team_children,
+    )
+    load_team_repos(
+        neo4j_session,
+        processed_data,
+        common_job_parameters["UPDATE_TAG"],
+        org_data["url"],
+    )
+    common_job_parameters["org_url"] = org_data["url"]
     cleanup(neo4j_session, common_job_parameters)

cartography/intel/github/users.py

@@ -96,12 +96,16 @@ def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict],
         api_url,
         organization,
         GITHUB_ORG_USERS_PAGINATED_GRAPHQL,
-        'membersWithRole',
+        "membersWithRole",
     )
     return users.edges, org
 
 
-def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
+def get_enterprise_owners(
+    token: str,
+    api_url: str,
+    organization: str,
+) -> Tuple[List[Dict], Dict]:
     """
     Retrieve a list of enterprise owners from the given GitHub organization as described in
     https://docs.github.com/en/graphql/reference/objects#organizationenterpriseowneredge.
@@ -119,13 +123,17 @@ def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[
         api_url,
         organization,
         GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL,
-        'enterpriseOwners',
+        "enterpriseOwners",
     )
     return owners.edges, org
 
 
 @timeit
-def transform_users(user_data: List[Dict], owners_data: List[Dict], org_data: Dict) -> Tuple[List[Dict], List[Dict]]:
+def transform_users(
+    user_data: List[Dict],
+    owners_data: List[Dict],
+    org_data: Dict,
+) -> Tuple[List[Dict], List[Dict]]:
     """
     Taking raw user and owner data, return two lists of processed user data:
     * organization users aka affiliated users (users directly affiliated with an organization)
@@ -145,27 +153,27 @@ def transform_users(user_data: List[Dict], owners_data: List[Dict], org_data: Di
     users_dict = {}
     for user in user_data:
         # all members get the 'MEMBER_OF' relationship
-        processed_user = deepcopy(user['node'])
-        processed_user['hasTwoFactorEnabled'] = user['hasTwoFactorEnabled']
-        processed_user['MEMBER_OF'] = org_data['url']
+        processed_user = deepcopy(user["node"])
+        processed_user["hasTwoFactorEnabled"] = user["hasTwoFactorEnabled"]
+        processed_user["MEMBER_OF"] = org_data["url"]
         # admins get a second relationship expressing them as such
-        if user['role'] == 'ADMIN':
-            processed_user['ADMIN_OF'] = org_data['url']
-        users_dict[processed_user['url']] = processed_user
+        if user["role"] == "ADMIN":
+            processed_user["ADMIN_OF"] = org_data["url"]
+        users_dict[processed_user["url"]] = processed_user
 
     owners_dict = {}
     for owner in owners_data:
-        processed_owner = deepcopy(owner['node'])
-        processed_owner['isEnterpriseOwner'] = True
-        if owner['organizationRole'] == 'UNAFFILIATED':
-            processed_owner['UNAFFILIATED'] = org_data['url']
+        processed_owner = deepcopy(owner["node"])
+        processed_owner["isEnterpriseOwner"] = True
+        if owner["organizationRole"] == "UNAFFILIATED":
+            processed_owner["UNAFFILIATED"] = org_data["url"]
         else:
-            processed_owner['MEMBER_OF'] = org_data['url']
-        owners_dict[processed_owner['url']] = processed_owner
+            processed_owner["MEMBER_OF"] = org_data["url"]
+        owners_dict[processed_owner["url"]] = processed_owner
 
     affiliated_users = []  # users affiliated with the target org
     for url, user in users_dict.items():
-        user['isEnterpriseOwner'] = url in owners_dict
+        user["isEnterpriseOwner"] = url in owners_dict
         affiliated_users.append(user)
 
     unaffiliated_users = []  # users not affiliated with the target org
@@ -190,7 +198,7 @@ def load_users(
         node_schema,
         user_data,
         lastupdated=update_tag,
-        org_url=org_data['url'],
+        org_url=org_data["url"],
     )
 
 
@@ -211,45 +219,68 @@ def load_organization(
 
 
 @timeit
-def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+def cleanup(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: dict[str, Any],
+) -> None:
     logger.info("Cleaning up GitHub users")
-    GraphJob.from_node_schema(GitHubOrganizationUserSchema(), common_job_parameters).run(neo4j_session)
-    GraphJob.from_node_schema(GitHubUnaffiliatedUserSchema(), common_job_parameters).run(neo4j_session)
+    GraphJob.from_node_schema(
+        GitHubOrganizationUserSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
+    GraphJob.from_node_schema(
+        GitHubUnaffiliatedUserSchema(),
+        common_job_parameters,
+    ).run(neo4j_session)
 
 
 @timeit
 def sync(
-        neo4j_session: neo4j.Session,
-        common_job_parameters: Dict,
-        github_api_key: str,
-        github_url: str,
-        organization: str,
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+    github_api_key: str,
+    github_url: str,
+    organization: str,
 ) -> None:
     logger.info("Syncing GitHub users")
     user_data, org_data = get_users(github_api_key, github_url, organization)
-    owners_data, org_data = get_enterprise_owners(github_api_key, github_url, organization)
-    processed_affiliated_user_data, processed_unaffiliated_user_data = (
-        transform_users(user_data, owners_data, org_data)
+    owners_data, org_data = get_enterprise_owners(
+        github_api_key,
+        github_url,
+        organization,
+    )
+    processed_affiliated_user_data, processed_unaffiliated_user_data = transform_users(
+        user_data,
+        owners_data,
+        org_data,
     )
     load_organization(
-        neo4j_session, GitHubOrganizationSchema(), [org_data],
-        common_job_parameters['UPDATE_TAG'],
+        neo4j_session,
+        GitHubOrganizationSchema(),
+        [org_data],
+        common_job_parameters["UPDATE_TAG"],
     )
     load_users(
-        neo4j_session, GitHubOrganizationUserSchema(), processed_affiliated_user_data, org_data,
-        common_job_parameters['UPDATE_TAG'],
+        neo4j_session,
+        GitHubOrganizationUserSchema(),
+        processed_affiliated_user_data,
+        org_data,
+        common_job_parameters["UPDATE_TAG"],
     )
     load_users(
-        neo4j_session, GitHubUnaffiliatedUserSchema(), processed_unaffiliated_user_data, org_data,
-        common_job_parameters['UPDATE_TAG'],
+        neo4j_session,
+        GitHubUnaffiliatedUserSchema(),
+        processed_unaffiliated_user_data,
+        org_data,
+        common_job_parameters["UPDATE_TAG"],
     )
     cleanup(neo4j_session, common_job_parameters)
 
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='GitHubOrganization',
-        group_id=org_data['url'],
-        synced_type='GitHubOrganization',
-        update_tag=common_job_parameters['UPDATE_TAG'],
+        group_type="GitHubOrganization",
+        group_id=org_data["url"],
+        synced_type="GitHubOrganization",
+        update_tag=common_job_parameters["UPDATE_TAG"],
         stat_handler=stat_handler,
     )