cartography 0.96.1__tar.gz → 0.96.2__tar.gz

{cartography-0.96.1 → cartography-0.96.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.96.1
+Version: 0.96.2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.96.1 → cartography-0.96.2}/cartography/intel/cve/feed.py

@@ -85,30 +85,32 @@ def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dic
         )
     results: Dict[Any, Any] = dict()
 
-    while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
-        try:
-            res = requests.get(
-                url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
-            )
-            res.raise_for_status()
-        except requests.exceptions.HTTPError:
-            logger.error(
-                f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
-            )
-            retries += 1
-            if retries >= MAX_RETRIES:
-                raise
-            # Exponential backoff
-            sleep_time *= 2
+    with requests.Session() as session:
+        while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
+            logger.info(f"Calling NIST NVD API at {url} with params {params}")
+            try:
+                res = session.get(
+                    url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
+                )
+                res.raise_for_status()
+                data = res.json()
+            except requests.exceptions.HTTPError:
+                logger.error(
+                    f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
+                )
+                retries += 1
+                if retries >= MAX_RETRIES:
+                    raise
+                # Exponential backoff
+                sleep_time *= 2
+                time.sleep(sleep_time)
+                continue
+            _map_cve_dict(results, data)
+            totalResults = data["totalResults"]
+            params["resultsPerPage"] = data["resultsPerPage"]
+            params["startIndex"] += data["resultsPerPage"]
+            retries = 0
             time.sleep(sleep_time)
-            continue
-        data = res.json()
-        _map_cve_dict(results, data)
-        totalResults = data["totalResults"]
-        params["resultsPerPage"] = data["resultsPerPage"]
-        params["startIndex"] += data["resultsPerPage"]
-        retries = 0
-        time.sleep(sleep_time)
     return results
 
 

{cartography-0.96.1 → cartography-0.96.2}/cartography/intel/github/repos.py

@@ -140,22 +140,40 @@ def _get_repo_collaborators_inner_func(
         org: str,
         api_url: str,
         token: str,
-        repo_name: str,
+        repo_raw_data: list[dict[str, Any]],
         affiliation: str,
         collab_users: list[dict[str, Any]],
         collab_permission: list[str],
-) -> None:
-    logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
-    collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
+) -> dict[str, list[UserAffiliationAndRepoPermission]]:
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+
+    for repo in repo_raw_data:
+        repo_name = repo['name']
+        repo_url = repo['url']
+
+        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
+                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
+            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
+            result[repo_url] = []
+            continue
+
+        logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
+        collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
 
-    # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
-    # however sometimes GitHub returns None, as in issue 1334 and 1404.
-    for collab in collaborators.nodes or []:
-        collab_users.append(collab)
+        # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
+        # however sometimes GitHub returns None, as in issue 1334 and 1404.
+        for collab in collaborators.nodes or []:
+            collab_users.append(collab)
 
-    # The `or []` is because `.edges` can be None.
-    for perm in collaborators.edges or []:
-        collab_permission.append(perm['permission'])
+        # The `or []` is because `.edges` can be None.
+        for perm in collaborators.edges or []:
+            collab_permission.append(perm['permission'])
+
+        result[repo_url] = [
+            UserAffiliationAndRepoPermission(user, permission, affiliation)
+            for user, permission in zip(collab_users, collab_permission)
+        ]
+    return result
 
 
 def _get_repo_collaborators_for_multiple_repos(
@@ -175,39 +193,24 @@ def _get_repo_collaborators_for_multiple_repos(
     :param token: The Github API token as string.
     :return: A dictionary of repo URL to list of UserAffiliationAndRepoPermission
     """
-    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
-    for repo in repo_raw_data:
-        repo_name = repo['name']
-        repo_url = repo['url']
-
-        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
-                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
-            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
-            result[repo_url] = []
-            continue
-
-        collab_users: List[dict[str, Any]] = []
-        collab_permission: List[str] = []
-
-        retries_with_backoff(
-            _get_repo_collaborators_inner_func,
-            TypeError,
-            5,
-            backoff_handler,
-        )(
-            org=org,
-            api_url=api_url,
-            token=token,
-            repo_name=repo_name,
-            affiliation=affiliation,
-            collab_users=collab_users,
-            collab_permission=collab_permission,
-        )
-
-        result[repo_url] = [
-            UserAffiliationAndRepoPermission(user, permission, affiliation)
-            for user, permission in zip(collab_users, collab_permission)
-        ]
+    logger.info(f'Retrieving repo collaborators for affiliation "{affiliation}" on org "{org}".')
+    collab_users: List[dict[str, Any]] = []
+    collab_permission: List[str] = []
+
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = retries_with_backoff(
+        _get_repo_collaborators_inner_func,
+        TypeError,
+        5,
+        backoff_handler,
+    )(
+        org=org,
+        api_url=api_url,
+        token=token,
+        repo_raw_data=repo_raw_data,
+        affiliation=affiliation,
+        collab_users=collab_users,
+        collab_permission=collab_permission,
+    )
     return result
 
 
@@ -260,8 +263,9 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
 
 
 def transform(
-    repos_json: List[Dict], direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
-    outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+        repos_json: List[Dict],
+        direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+        outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
 ) -> Dict:
     """
     Parses the JSON returned from GitHub API to create data for graph ingestion
@@ -289,16 +293,24 @@ def transform(
         _transform_repo_languages(repo_object['url'], repo_object, transformed_repo_languages)
         _transform_repo_objects(repo_object, transformed_repo_list)
         _transform_repo_owners(repo_object['owner']['url'], repo_object, transformed_repo_owners)
-        _transform_collaborators(
-            repo_object['url'], outside_collaborators[repo_object['url']],
-            transformed_outside_collaborators,
-        )
-        _transform_collaborators(
-            repo_object['url'], direct_collaborators[repo_object['url']],
-            transformed_direct_collaborators,
-        )
-        _transform_requirements_txt(repo_object['requirements'], repo_object['url'], transformed_requirements_files)
-        _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_object['url'], transformed_requirements_files)
+
+        # Allow sync to continue if we didn't have permissions to list collaborators
+        repo_url = repo_object['url']
+        if repo_url in outside_collaborators:
+            _transform_collaborators(
+                repo_object['url'],
+                outside_collaborators[repo_object['url']],
+                transformed_outside_collaborators,
+            )
+        if repo_url in direct_collaborators:
+            _transform_collaborators(
+                repo_object['url'],
+                direct_collaborators[repo_object['url']],
+                transformed_direct_collaborators,
+            )
+
+        _transform_requirements_txt(repo_object['requirements'], repo_url, transformed_requirements_files)
+        _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_url, transformed_requirements_files)
     results = {
         'repos': transformed_repo_list,
         'repo_languages': transformed_repo_languages,
@@ -737,12 +749,18 @@ def sync(
     """
     logger.info("Syncing GitHub repos")
     repos_json = get(github_api_key, github_url, organization)
-    direct_collabs = _get_repo_collaborators_for_multiple_repos(
-        repos_json, "DIRECT", organization, github_url, github_api_key,
-    )
-    outside_collabs = _get_repo_collaborators_for_multiple_repos(
-        repos_json, "OUTSIDE", organization, github_url, github_api_key,
-    )
+    direct_collabs: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    outside_collabs: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    try:
+        direct_collabs = _get_repo_collaborators_for_multiple_repos(
+            repos_json, "DIRECT", organization, github_url, github_api_key,
+        )
+        outside_collabs = _get_repo_collaborators_for_multiple_repos(
+            repos_json, "OUTSIDE", organization, github_url, github_api_key,
+        )
+    except TypeError:
+        # due to permission errors or transient network error or some other nonsense
+        logger.warning('Unable to list repo collaborators due to permission errors; continuing on.', exc_info=True)
     repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
     run_cleanup_job('github_repos_cleanup.json', neo4j_session, common_job_parameters)

{cartography-0.96.1 → cartography-0.96.2}/cartography/intel/github/users.py

@@ -90,6 +90,7 @@ def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict],
         2. data on the owning GitHub organization
         see tests.data.github.users.GITHUB_USER_DATA for shape of both
     """
+    logger.info(f"Retrieving users from GitHub organization {organization}")
     users, org = fetch_all(
         token,
         api_url,
@@ -112,6 +113,7 @@ def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[
         3. data on the owning GitHub organization
         see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
     """
+    logger.info(f"Retrieving enterprise owners from GitHub organization {organization}")
     owners, org = fetch_all(
         token,
         api_url,

{cartography-0.96.1 → cartography-0.96.2}/cartography/intel/github/util.py

@@ -163,7 +163,8 @@ def fetch_all(
 
         if retry >= retries:
             logger.error(
-                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error.",
+                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error "
+                f"after {retry} retries. Raising exception.",
                 exc_info=True,
             )
             raise exc

{cartography-0.96.1 → cartography-0.96.2}/cartography.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.96.1
+Version: 0.96.2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.96.1 → cartography-0.96.2}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = '0.96.1'
+__version__ = '0.96.2'
 
 
 setup(