cartography 0.96.1__tar.gz → 0.97.0__tar.gz

{cartography-0.96.1 → cartography-0.97.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.96.1
+Version: 0.97.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/aws/identitycenter.py

@@ -56,6 +56,7 @@ def load_identity_center_instances(
 
 
 @timeit
+@aws_handle_regions
 def get_permission_sets(boto3_session: boto3.session.Session, instance_arn: str, region: str) -> List[Dict]:
     """
     Get all permission sets for a given Identity Center instance

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/cve/__init__.py

@@ -2,6 +2,9 @@ import logging
 from datetime import datetime
 
 import neo4j
+from requests import Session
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
 
 from cartography.config import Config
 from cartography.intel.cve import feed
@@ -13,28 +16,34 @@ logger = logging.getLogger(__name__)
 stat_handler = get_stats_client(__name__)
 
 
-@timeit
-def start_cve_ingestion(
-    neo4j_session: neo4j.Session, config: Config,
-) -> None:
-    """
-    Perform ingestion of CVE data from NIST APIs.
-    :param neo4j_session: Neo4J session for database interface
-    :param config: A cartography.config object
-    :return: None
-    """
-    if not config.cve_enabled:
-        return
-    cve_api_key: str | None = config.cve_api_key if config.cve_api_key else None
+def _retryable_session() -> Session:
+    session = Session()
+    retry_policy = Retry(
+        total=8,
+        connect=1,
+        backoff_factor=1,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["GET"],
+    )
+    session.mount("https://", HTTPAdapter(max_retries=retry_policy))
+    logger.info(f"Configured session with retry policy: {retry_policy}")
+    return session
 
-    # sync CVE year archives, if not yet synced
+
+def _sync_year_archives(
+    http_session: Session,
+    neo4j_session: neo4j.Session,
+    config: Config,
+    cve_api_key: str | None,
+) -> None:
     existing_years = feed.get_cve_sync_metadata(neo4j_session)
     current_year = datetime.now().year
-    for year in range(2002, current_year + 1):
+    logger.info(f"Syncing CVE data for year archives. Existing years: {existing_years}. Current year: {current_year}")
+    for year in range(1999, current_year + 1):
         if year in existing_years:
             continue
         logger.info(f"Syncing CVE data for year {year}")
-        cves = feed.get_published_cves_per_year(config.nist_cve_url, str(year), cve_api_key)
+        cves = feed.get_published_cves_per_year(http_session, config.nist_cve_url, str(year), cve_api_key)
         feed_metadata = feed.transform_cve_feed(cves)
         feed.load_cve_feed(neo4j_session, [feed_metadata], config.update_tag)
         published_cves = feed.transform_cves(cves)
@@ -48,10 +57,16 @@ def start_cve_ingestion(
             stat_handler=stat_handler,
         )
 
-    # sync modified data
+
+def _sync_modified_data(
+    http_session: Session,
+    neo4j_session: neo4j.Session,
+    config: Config,
+    cve_api_key: str | None,
+) -> None:
     logger.info("Syncing CVE data for modified data")
     last_modified_date = feed.get_last_modified_cve_date(neo4j_session)
-    cves = feed.get_modified_cves(config.nist_cve_url, last_modified_date, cve_api_key)
+    cves = feed.get_modified_cves(http_session, config.nist_cve_url, last_modified_date, cve_api_key)
     feed_metadata = feed.transform_cve_feed(cves)
     feed.load_cve_feed(neo4j_session, [feed_metadata], config.update_tag)
     modified_cves = feed.transform_cves(cves)
@@ -65,4 +80,21 @@ def start_cve_ingestion(
         stat_handler=stat_handler,
     )
 
-    # CVEs are never deleted, so we don't need to run a cleanup job
+
+@timeit
+def start_cve_ingestion(
+    neo4j_session: neo4j.Session, config: Config,
+) -> None:
+    """
+    Perform ingestion of CVE data from NIST APIs.
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if not config.cve_enabled:
+        return
+    cve_api_key: str | None = config.cve_api_key if config.cve_api_key else None
+    with _retryable_session() as http_session:
+        _sync_year_archives(http_session, neo4j_session=neo4j_session, config=config, cve_api_key=cve_api_key)
+        _sync_modified_data(http_session, neo4j_session=neo4j_session, config=config, cve_api_key=cve_api_key)
+        # CVEs are never deleted, so we don't need to run a cleanup job

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/cve/feed.py

@@ -11,7 +11,7 @@ from typing import List
 from typing import Optional
 
 import neo4j
-import requests
+from requests import Session
 
 from cartography.client.core.tx import load
 from cartography.client.core.tx import read_list_of_values_tx
@@ -22,7 +22,6 @@ from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
 
-MAX_RETRIES = 8
 # Connect and read timeouts of 120 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
 CONNECT_AND_READ_TIMEOUT = (30, 120)
 CVE_FEED_ID = "NIST_NVD"
@@ -68,51 +67,36 @@ def _map_cve_dict(cve_dict: Dict[Any, Any], data: Dict[Any, Any]) -> None:
     cve_dict["startIndex"] = data["startIndex"]
 
 
-def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dict[Any, Any]:
-    totalResults = 0
-    sleep_time = DEFAULT_SLEEP_TIME
-    retries = 0
+def _call_cves_api(http_session: Session, url: str, api_key: str | None, params: Dict[str, Any]) -> Dict[Any, Any]:
+    total_results = 0
     params["startIndex"] = 0
     params["resultsPerPage"] = RESULTS_PER_PAGE
-    headers = {}
-    headers["Content-Type"] = "application/json"
+    headers = {"Content-Type": "application/json"}
     if api_key:
+        sleep_between_requests = DEFAULT_SLEEP_TIME
         headers["apiKey"] = api_key
     else:
-        sleep_time = DELAYED_SLEEP_TIME  # Sleep for 6 seconds between each request to avoid rate limiting
+        sleep_between_requests = DELAYED_SLEEP_TIME
         logger.warning(
-            f"No NIST NVD API key provided. Increasing sleep time to {sleep_time}.",
+            f"No NIST NVD API key provided. Increasing sleep time to {sleep_between_requests}.",
         )
     results: Dict[Any, Any] = dict()
 
-    while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
-        try:
-            res = requests.get(
-                url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
-            )
-            res.raise_for_status()
-        except requests.exceptions.HTTPError:
-            logger.error(
-                f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
-            )
-            retries += 1
-            if retries >= MAX_RETRIES:
-                raise
-            # Exponential backoff
-            sleep_time *= 2
-            time.sleep(sleep_time)
-            continue
+    while params["resultsPerPage"] > 0 or params["startIndex"] < total_results:
+        logger.info(f"Calling NIST NVD API at {url} with params {params}")
+        res = http_session.get(url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT)
+        res.raise_for_status()
         data = res.json()
         _map_cve_dict(results, data)
-        totalResults = data["totalResults"]
+        total_results = data["totalResults"]
         params["resultsPerPage"] = data["resultsPerPage"]
         params["startIndex"] += data["resultsPerPage"]
-        retries = 0
-        time.sleep(sleep_time)
+        time.sleep(sleep_between_requests)
     return results
 
 
 def get_cves_in_batches(
+    http_session: Session,
     nist_cve_url: str,
     start_date: datetime,
     end_date: datetime,
@@ -145,7 +129,7 @@ def get_cves_in_batches(
         logger.info(
             f"Querying CVE data between {current_start_date} and {current_end_date}",
         )
-        batch_cves = _call_cves_api(nist_cve_url, api_key, params)
+        batch_cves = _call_cves_api(http_session, nist_cve_url, api_key, params)
         _map_cve_dict(cves, batch_cves)
         current_start_date = current_end_date
         new_end_date = current_start_date + batch_size
@@ -156,9 +140,8 @@ def get_cves_in_batches(
 
 
 def get_modified_cves(
-    nist_cve_url: str, last_modified_date: str, api_key: str | None,
+    http_session: Session, nist_cve_url: str, last_modified_date: str, api_key: str | None,
 ) -> Dict[Any, Any]:
-    cves = dict()
     end_date = datetime.now(tz=timezone.utc)
     start_date = datetime.strptime(last_modified_date, "%Y-%m-%dT%H:%M:%S").replace(
         tzinfo=timezone.utc,
@@ -168,15 +151,14 @@ def get_modified_cves(
         "end": "lastModEndDate",
     }
     cves = get_cves_in_batches(
-        nist_cve_url, start_date, end_date, date_param_names, api_key,
+        http_session, nist_cve_url, start_date, end_date, date_param_names, api_key,
     )
     return cves
 
 
 def get_published_cves_per_year(
-    nist_cve_url: str, year: str, api_key: str | None,
+    http_session: Session, nist_cve_url: str, year: str, api_key: str | None,
 ) -> Dict[Any, Any]:
-    cves = {}
     start_of_year = datetime.strptime(f"{year}-01-01", "%Y-%m-%d")
     next_year = int(year) + 1
     end_of_next_year = datetime.strptime(f"{next_year}-01-01", "%Y-%m-%d")
@@ -185,7 +167,7 @@ def get_published_cves_per_year(
         "end": "pubEndDate",
     }
     cves = get_cves_in_batches(
-        nist_cve_url, start_of_year, end_of_next_year, date_param_names, api_key,
+        http_session, nist_cve_url, start_of_year, end_of_next_year, date_param_names, api_key,
     )
     return cves
 

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/github/repos.py

@@ -140,22 +140,40 @@ def _get_repo_collaborators_inner_func(
         org: str,
         api_url: str,
         token: str,
-        repo_name: str,
+        repo_raw_data: list[dict[str, Any]],
         affiliation: str,
         collab_users: list[dict[str, Any]],
         collab_permission: list[str],
-) -> None:
-    logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
-    collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
+) -> dict[str, list[UserAffiliationAndRepoPermission]]:
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+
+    for repo in repo_raw_data:
+        repo_name = repo['name']
+        repo_url = repo['url']
+
+        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
+                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
+            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
+            result[repo_url] = []
+            continue
+
+        logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
+        collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
 
-    # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
-    # however sometimes GitHub returns None, as in issue 1334 and 1404.
-    for collab in collaborators.nodes or []:
-        collab_users.append(collab)
+        # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
+        # however sometimes GitHub returns None, as in issue 1334 and 1404.
+        for collab in collaborators.nodes or []:
+            collab_users.append(collab)
 
-    # The `or []` is because `.edges` can be None.
-    for perm in collaborators.edges or []:
-        collab_permission.append(perm['permission'])
+        # The `or []` is because `.edges` can be None.
+        for perm in collaborators.edges or []:
+            collab_permission.append(perm['permission'])
+
+        result[repo_url] = [
+            UserAffiliationAndRepoPermission(user, permission, affiliation)
+            for user, permission in zip(collab_users, collab_permission)
+        ]
+    return result
 
 
 def _get_repo_collaborators_for_multiple_repos(
@@ -175,39 +193,24 @@ def _get_repo_collaborators_for_multiple_repos(
     :param token: The Github API token as string.
     :return: A dictionary of repo URL to list of UserAffiliationAndRepoPermission
     """
-    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
-    for repo in repo_raw_data:
-        repo_name = repo['name']
-        repo_url = repo['url']
-
-        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
-                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
-            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
-            result[repo_url] = []
-            continue
-
-        collab_users: List[dict[str, Any]] = []
-        collab_permission: List[str] = []
-
-        retries_with_backoff(
-            _get_repo_collaborators_inner_func,
-            TypeError,
-            5,
-            backoff_handler,
-        )(
-            org=org,
-            api_url=api_url,
-            token=token,
-            repo_name=repo_name,
-            affiliation=affiliation,
-            collab_users=collab_users,
-            collab_permission=collab_permission,
-        )
-
-        result[repo_url] = [
-            UserAffiliationAndRepoPermission(user, permission, affiliation)
-            for user, permission in zip(collab_users, collab_permission)
-        ]
+    logger.info(f'Retrieving repo collaborators for affiliation "{affiliation}" on org "{org}".')
+    collab_users: List[dict[str, Any]] = []
+    collab_permission: List[str] = []
+
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = retries_with_backoff(
+        _get_repo_collaborators_inner_func,
+        TypeError,
+        5,
+        backoff_handler,
+    )(
+        org=org,
+        api_url=api_url,
+        token=token,
+        repo_raw_data=repo_raw_data,
+        affiliation=affiliation,
+        collab_users=collab_users,
+        collab_permission=collab_permission,
+    )
     return result
 
 
@@ -260,8 +263,9 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
 
 
 def transform(
-    repos_json: List[Dict], direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
-    outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+        repos_json: List[Dict],
+        direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+        outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
 ) -> Dict:
     """
     Parses the JSON returned from GitHub API to create data for graph ingestion
@@ -289,16 +293,24 @@ def transform(
         _transform_repo_languages(repo_object['url'], repo_object, transformed_repo_languages)
         _transform_repo_objects(repo_object, transformed_repo_list)
         _transform_repo_owners(repo_object['owner']['url'], repo_object, transformed_repo_owners)
-        _transform_collaborators(
-            repo_object['url'], outside_collaborators[repo_object['url']],
-            transformed_outside_collaborators,
-        )
-        _transform_collaborators(
-            repo_object['url'], direct_collaborators[repo_object['url']],
-            transformed_direct_collaborators,
-        )
-        _transform_requirements_txt(repo_object['requirements'], repo_object['url'], transformed_requirements_files)
-        _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_object['url'], transformed_requirements_files)
+
+        # Allow sync to continue if we didn't have permissions to list collaborators
+        repo_url = repo_object['url']
+        if repo_url in outside_collaborators:
+            _transform_collaborators(
+                repo_object['url'],
+                outside_collaborators[repo_object['url']],
+                transformed_outside_collaborators,
+            )
+        if repo_url in direct_collaborators:
+            _transform_collaborators(
+                repo_object['url'],
+                direct_collaborators[repo_object['url']],
+                transformed_direct_collaborators,
+            )
+
+        _transform_requirements_txt(repo_object['requirements'], repo_url, transformed_requirements_files)
+        _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_url, transformed_requirements_files)
     results = {
         'repos': transformed_repo_list,
         'repo_languages': transformed_repo_languages,
@@ -737,12 +749,18 @@ def sync(
     """
     logger.info("Syncing GitHub repos")
     repos_json = get(github_api_key, github_url, organization)
-    direct_collabs = _get_repo_collaborators_for_multiple_repos(
-        repos_json, "DIRECT", organization, github_url, github_api_key,
-    )
-    outside_collabs = _get_repo_collaborators_for_multiple_repos(
-        repos_json, "OUTSIDE", organization, github_url, github_api_key,
-    )
+    direct_collabs: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    outside_collabs: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    try:
+        direct_collabs = _get_repo_collaborators_for_multiple_repos(
+            repos_json, "DIRECT", organization, github_url, github_api_key,
+        )
+        outside_collabs = _get_repo_collaborators_for_multiple_repos(
+            repos_json, "OUTSIDE", organization, github_url, github_api_key,
+        )
+    except TypeError:
+        # due to permission errors or transient network error or some other nonsense
+        logger.warning('Unable to list repo collaborators due to permission errors; continuing on.', exc_info=True)
     repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
     run_cleanup_job('github_repos_cleanup.json', neo4j_session, common_job_parameters)

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/github/teams.py

@@ -21,6 +21,9 @@ logger = logging.getLogger(__name__)
 RepoPermission = namedtuple('RepoPermission', ['repo_url', 'permission'])
 # A team member's role: https://docs.github.com/en/graphql/reference/enums#teammemberrole
 UserRole = namedtuple('UserRole', ['user_url', 'role'])
+# Unlike the other tuples here, there is no qualification (like 'role' or 'permission') to the relationship.
+# A child team is just a child team: https://docs.github.com/en/graphql/reference/objects#teamconnection
+ChildTeam = namedtuple('ChildTeam', ['team_url'])
 
 
 def backoff_handler(details: Dict) -> None:
@@ -53,6 +56,9 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
                         members(membership: IMMEDIATE) {
                             totalCount
                         }
+                        childTeams {
+                            totalCount
+                        }
                     }
                     pageInfo{
                         endCursor
@@ -166,6 +172,21 @@ def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedG
     return team_repos
 
 
+def _get_teams_users_inner_func(
+        org: str, api_url: str, token: str, team_name: str,
+        user_urls: List[str], user_roles: List[str],
+) -> None:
+    logger.info(f"Loading team users for {team_name}.")
+    team_users = _get_team_users(org, api_url, token, team_name)
+    # The `or []` is because `.nodes` can be None. See:
+    # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
+    for user in team_users.nodes or []:
+        user_urls.append(user['url'])
+    # The `or []` is because `.edges` can be None.
+    for edge in team_users.edges or []:
+        user_roles.append(edge['role'])
+
+
 def _get_team_users_for_multiple_teams(
         team_raw_data: list[dict[str, Any]],
         org: str,
@@ -185,21 +206,7 @@ def _get_team_users_for_multiple_teams(
         user_urls: List[str] = []
         user_roles: List[str] = []
 
-        def get_teams_users_inner_func(
-            org: str, api_url: str, token: str, team_name: str,
-            user_urls: List[str], user_roles: List[str],
-        ) -> None:
-            logger.info(f"Loading team users for {team_name}.")
-            team_users = _get_team_users(org, api_url, token, team_name)
-            # The `or []` is because `.nodes` can be None. See:
-            # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
-            for user in team_users.nodes or []:
-                user_urls.append(user['url'])
-            # The `or []` is because `.edges` can be None.
-            for edge in team_users.edges or []:
-                user_roles.append(edge['role'])
-
-        retries_with_backoff(get_teams_users_inner_func, TypeError, 5, backoff_handler)(
+        retries_with_backoff(_get_teams_users_inner_func, TypeError, 5, backoff_handler)(
             org=org, api_url=api_url, token=token, team_name=team_name, user_urls=user_urls, user_roles=user_roles,
         )
 
@@ -252,11 +259,90 @@ def _get_team_users(org: str, api_url: str, token: str, team: str) -> PaginatedG
     return team_users
 
 
+def _get_child_teams_inner_func(
+        org: str, api_url: str, token: str, team_name: str, team_urls: List[str],
+) -> None:
+    logger.info(f"Loading child teams for {team_name}.")
+    child_teams = _get_child_teams(org, api_url, token, team_name)
+    # The `or []` is because `.nodes` can be None. See:
+    # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
+    for cteam in child_teams.nodes or []:
+        team_urls.append(cteam['url'])
+    # No edges to process here, the GitHub response for child teams has no relevant info in edges.
+
+
+def _get_child_teams_for_multiple_teams(
+        team_raw_data: list[dict[str, Any]],
+        org: str,
+        api_url: str,
+        token: str,
+) -> dict[str, list[ChildTeam]]:
+    result: dict[str, list[ChildTeam]] = {}
+    for team in team_raw_data:
+        team_name = team['slug']
+        team_count = team['childTeams']['totalCount']
+
+        if team_count == 0:
+            # This team has no child teams so let's move on
+            result[team_name] = []
+            continue
+
+        team_urls: List[str] = []
+
+        retries_with_backoff(_get_child_teams_inner_func, TypeError, 5, backoff_handler)(
+            org=org, api_url=api_url, token=token, team_name=team_name, team_urls=team_urls,
+        )
+
+        result[team_name] = [ChildTeam(url) for url in team_urls]
+    return result
+
+
+def _get_child_teams(org: str, api_url: str, token: str, team: str) -> PaginatedGraphqlData:
+    team_users_gql = """
+    query($login: String!, $team: String!, $cursor: String) {
+        organization(login: $login) {
+            url
+            login
+            team(slug: $team) {
+                slug
+                childTeams(first: 100, after: $cursor) {
+                    totalCount
+                    nodes {
+                        url
+                    }
+                    pageInfo {
+                        endCursor
+                        hasNextPage
+                    }
+                }
+            }
+        }
+        rateLimit {
+            limit
+            cost
+            remaining
+            resetAt
+        }
+    }
+    """
+    team_users, _ = fetch_all(
+        token,
+        api_url,
+        org,
+        team_users_gql,
+        'team',
+        resource_inner_type='childTeams',
+        team=team,
+    )
+    return team_users
+
+
 def transform_teams(
         team_paginated_data: PaginatedGraphqlData,
         org_data: Dict[str, Any],
         team_repo_data: dict[str, list[RepoPermission]],
         team_user_data: dict[str, list[UserRole]],
+        team_child_team_data: dict[str, list[ChildTeam]],
 ) -> list[dict[str, Any]]:
     result = []
     for team in team_paginated_data.nodes:
@@ -267,13 +353,15 @@ def transform_teams(
             'description': team['description'],
             'repo_count': team['repositories']['totalCount'],
             'member_count': team['members']['totalCount'],
+            'child_team_count': team['childTeams']['totalCount'],
             'org_url': org_data['url'],
             'org_login': org_data['login'],
         }
         repo_permissions = team_repo_data[team_name]
         user_roles = team_user_data[team_name]
+        child_teams = team_child_team_data[team_name]
 
-        if not repo_permissions and not user_roles:
+        if not repo_permissions and not user_roles and not child_teams:
             result.append(repo_info)
             continue
 
@@ -289,6 +377,15 @@ def transform_teams(
                 repo_info_copy = repo_info.copy()
                 repo_info_copy[role] = user_url
                 result.append(repo_info_copy)
+        if child_teams:
+            for (team_url,) in child_teams:
+                repo_info_copy = repo_info.copy()
+                # GitHub speaks of team-team relationships as 'child teams', as in the graphql query
+                # or here: https://docs.github.com/en/graphql/reference/enums#teammembershiptype
+                # We label the relationship as 'MEMBER_OF_TEAM' here because it is in line with
+                # other similar relationships in Cartography.
+                repo_info_copy['MEMBER_OF_TEAM'] = team_url
+                result.append(repo_info_copy)
     return result
 
 
@@ -325,7 +422,8 @@ def sync_github_teams(
     teams_paginated, org_data = get_teams(organization, github_url, github_api_key)
     team_repos = _get_team_repos_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
     team_users = _get_team_users_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
-    processed_data = transform_teams(teams_paginated, org_data, team_repos, team_users)
+    team_children = _get_child_teams_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
+    processed_data = transform_teams(teams_paginated, org_data, team_repos, team_users, team_children)
     load_team_repos(neo4j_session, processed_data, common_job_parameters['UPDATE_TAG'], org_data['url'])
     common_job_parameters['org_url'] = org_data['url']
     cleanup(neo4j_session, common_job_parameters)

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/github/users.py

@@ -90,6 +90,7 @@ def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict],
         2. data on the owning GitHub organization
         see tests.data.github.users.GITHUB_USER_DATA for shape of both
     """
+    logger.info(f"Retrieving users from GitHub organization {organization}")
     users, org = fetch_all(
         token,
         api_url,
@@ -112,6 +113,7 @@ def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[
         3. data on the owning GitHub organization
         see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
     """
+    logger.info(f"Retrieving enterprise owners from GitHub organization {organization}")
     owners, org = fetch_all(
         token,
         api_url,
@@ -142,10 +144,13 @@ def transform_users(user_data: List[Dict], owners_data: List[Dict], org_data: Di
 
     users_dict = {}
     for user in user_data:
+        # all members get the 'MEMBER_OF' relationship
         processed_user = deepcopy(user['node'])
-        processed_user['role'] = user['role']
         processed_user['hasTwoFactorEnabled'] = user['hasTwoFactorEnabled']
         processed_user['MEMBER_OF'] = org_data['url']
+        # admins get a second relationship expressing them as such
+        if user['role'] == 'ADMIN':
+            processed_user['ADMIN_OF'] = org_data['url']
         users_dict[processed_user['url']] = processed_user
 
     owners_dict = {}

{cartography-0.96.1 → cartography-0.97.0}/cartography/intel/github/util.py

@@ -163,7 +163,8 @@ def fetch_all(
 
         if retry >= retries:
             logger.error(
-                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error.",
+                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error "
+                f"after {retry} retries. Raising exception.",
                 exc_info=True,
             )
             raise exc