cartography 0.96.1__py3-none-any.whl → 0.96.2__py3-none-any.whl

cartography/intel/cve/feed.py

@@ -85,30 +85,32 @@ def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dic
         )
     results: Dict[Any, Any] = dict()
 
-    while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
-        try:
-            res = requests.get(
-                url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
-            )
-            res.raise_for_status()
-        except requests.exceptions.HTTPError:
-            logger.error(
-                f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
-            )
-            retries += 1
-            if retries >= MAX_RETRIES:
-                raise
-            # Exponential backoff
-            sleep_time *= 2
+    with requests.Session() as session:
+        while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
+            logger.info(f"Calling NIST NVD API at {url} with params {params}")
+            try:
+                res = session.get(
+                    url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
+                )
+                res.raise_for_status()
+                data = res.json()
+            except requests.exceptions.HTTPError:
+                logger.error(
+                    f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
+                )
+                retries += 1
+                if retries >= MAX_RETRIES:
+                    raise
+                # Exponential backoff
+                sleep_time *= 2
+                time.sleep(sleep_time)
+                continue
+            _map_cve_dict(results, data)
+            totalResults = data["totalResults"]
+            params["resultsPerPage"] = data["resultsPerPage"]
+            params["startIndex"] += data["resultsPerPage"]
+            retries = 0
             time.sleep(sleep_time)
-            continue
-        data = res.json()
-        _map_cve_dict(results, data)
-        totalResults = data["totalResults"]
-        params["resultsPerPage"] = data["resultsPerPage"]
-        params["startIndex"] += data["resultsPerPage"]
-        retries = 0
-        time.sleep(sleep_time)
     return results
 
 

cartography/intel/github/repos.py

@@ -140,22 +140,40 @@ def _get_repo_collaborators_inner_func(
         org: str,
         api_url: str,
         token: str,
-        repo_name: str,
+        repo_raw_data: list[dict[str, Any]],
         affiliation: str,
         collab_users: list[dict[str, Any]],
         collab_permission: list[str],
-) -> None:
-    logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
-    collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
+) -> dict[str, list[UserAffiliationAndRepoPermission]]:
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+
+    for repo in repo_raw_data:
+        repo_name = repo['name']
+        repo_url = repo['url']
+
+        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
+                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
+            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
+            result[repo_url] = []
+            continue
+
+        logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
+        collaborators = _get_repo_collaborators(token, api_url, org, repo_name, affiliation)
 
-    # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
-    # however sometimes GitHub returns None, as in issue 1334 and 1404.
-    for collab in collaborators.nodes or []:
-        collab_users.append(collab)
+        # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
+        # however sometimes GitHub returns None, as in issue 1334 and 1404.
+        for collab in collaborators.nodes or []:
+            collab_users.append(collab)
 
-    # The `or []` is because `.edges` can be None.
-    for perm in collaborators.edges or []:
-        collab_permission.append(perm['permission'])
+        # The `or []` is because `.edges` can be None.
+        for perm in collaborators.edges or []:
+            collab_permission.append(perm['permission'])
+
+        result[repo_url] = [
+            UserAffiliationAndRepoPermission(user, permission, affiliation)
+            for user, permission in zip(collab_users, collab_permission)
+        ]
+    return result
 
 
 def _get_repo_collaborators_for_multiple_repos(
@@ -175,39 +193,24 @@ def _get_repo_collaborators_for_multiple_repos(
     :param token: The Github API token as string.
     :return: A dictionary of repo URL to list of UserAffiliationAndRepoPermission
     """
-    result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
-    for repo in repo_raw_data:
-        repo_name = repo['name']
-        repo_url = repo['url']
-
-        if ((affiliation == 'OUTSIDE' and repo['outsideCollaborators']['totalCount'] == 0) or
-                (affiliation == 'DIRECT' and repo['directCollaborators']['totalCount'] == 0)):
-            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
-            result[repo_url] = []
-            continue
-
-        collab_users: List[dict[str, Any]] = []
-        collab_permission: List[str] = []
-
-        retries_with_backoff(
-            _get_repo_collaborators_inner_func,
-            TypeError,
-            5,
-            backoff_handler,
-        )(
-            org=org,
-            api_url=api_url,
-            token=token,
-            repo_name=repo_name,
-            affiliation=affiliation,
-            collab_users=collab_users,
-            collab_permission=collab_permission,
-        )
-
-        result[repo_url] = [
-            UserAffiliationAndRepoPermission(user, permission, affiliation)
-            for user, permission in zip(collab_users, collab_permission)
-        ]
+    logger.info(f'Retrieving repo collaborators for affiliation "{affiliation}" on org "{org}".')
+    collab_users: List[dict[str, Any]] = []
+    collab_permission: List[str] = []
+
+    result: dict[str, list[UserAffiliationAndRepoPermission]] = retries_with_backoff(
+        _get_repo_collaborators_inner_func,
+        TypeError,
+        5,
+        backoff_handler,
+    )(
+        org=org,
+        api_url=api_url,
+        token=token,
+        repo_raw_data=repo_raw_data,
+        affiliation=affiliation,
+        collab_users=collab_users,
+        collab_permission=collab_permission,
+    )
     return result
 
 
@@ -260,8 +263,9 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
 
 
 def transform(
-    repos_json: List[Dict], direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
-    outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+        repos_json: List[Dict],
+        direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
+        outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
 ) -> Dict:
     """
     Parses the JSON returned from GitHub API to create data for graph ingestion
@@ -289,16 +293,24 @@ def transform(
         _transform_repo_languages(repo_object['url'], repo_object, transformed_repo_languages)
         _transform_repo_objects(repo_object, transformed_repo_list)
         _transform_repo_owners(repo_object['owner']['url'], repo_object, transformed_repo_owners)
-        _transform_collaborators(
-            repo_object['url'], outside_collaborators[repo_object['url']],
-            transformed_outside_collaborators,
-        )
-        _transform_collaborators(
-            repo_object['url'], direct_collaborators[repo_object['url']],
-            transformed_direct_collaborators,
-        )
-        _transform_requirements_txt(repo_object['requirements'], repo_object['url'], transformed_requirements_files)
-        _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_object['url'], transformed_requirements_files)
+
+        # Allow sync to continue if we didn't have permissions to list collaborators
+        repo_url = repo_object['url']
+        if repo_url in outside_collaborators:
+            _transform_collaborators(
+                repo_object['url'],
+                outside_collaborators[repo_object['url']],
+                transformed_outside_collaborators,
+            )
+        if repo_url in direct_collaborators:
+            _transform_collaborators(
+                repo_object['url'],
+                direct_collaborators[repo_object['url']],
+                transformed_direct_collaborators,
+            )
+
+        _transform_requirements_txt(repo_object['requirements'], repo_url, transformed_requirements_files)
+        _transform_setup_cfg_requirements(repo_object['setupCfg'], repo_url, transformed_requirements_files)
     results = {
         'repos': transformed_repo_list,
         'repo_languages': transformed_repo_languages,
@@ -737,12 +749,18 @@ def sync(
     """
     logger.info("Syncing GitHub repos")
     repos_json = get(github_api_key, github_url, organization)
-    direct_collabs = _get_repo_collaborators_for_multiple_repos(
-        repos_json, "DIRECT", organization, github_url, github_api_key,
-    )
-    outside_collabs = _get_repo_collaborators_for_multiple_repos(
-        repos_json, "OUTSIDE", organization, github_url, github_api_key,
-    )
+    direct_collabs: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    outside_collabs: dict[str, list[UserAffiliationAndRepoPermission]] = {}
+    try:
+        direct_collabs = _get_repo_collaborators_for_multiple_repos(
+            repos_json, "DIRECT", organization, github_url, github_api_key,
+        )
+        outside_collabs = _get_repo_collaborators_for_multiple_repos(
+            repos_json, "OUTSIDE", organization, github_url, github_api_key,
+        )
+    except TypeError:
+        # due to permission errors or transient network error or some other nonsense
+        logger.warning('Unable to list repo collaborators due to permission errors; continuing on.', exc_info=True)
     repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
     run_cleanup_job('github_repos_cleanup.json', neo4j_session, common_job_parameters)

cartography/intel/github/users.py

@@ -90,6 +90,7 @@ def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict],
         2. data on the owning GitHub organization
         see tests.data.github.users.GITHUB_USER_DATA for shape of both
     """
+    logger.info(f"Retrieving users from GitHub organization {organization}")
     users, org = fetch_all(
         token,
         api_url,
@@ -112,6 +113,7 @@ def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[
         3. data on the owning GitHub organization
         see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
     """
+    logger.info(f"Retrieving enterprise owners from GitHub organization {organization}")
     owners, org = fetch_all(
         token,
         api_url,

cartography/intel/github/util.py

@@ -163,7 +163,8 @@ def fetch_all(
 
         if retry >= retries:
             logger.error(
-                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error.",
+                f"GitHub: Could not retrieve page of resource `{resource_type}` due to HTTP error "
+                f"after {retry} retries. Raising exception.",
                 exc_info=True,
             )
             raise exc

{cartography-0.96.1.dist-info → cartography-0.96.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.96.1
+Version: 0.96.2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/cartography-cncf/cartography
 Maintainer: Cartography Contributors

{cartography-0.96.1.dist-info → cartography-0.96.2.dist-info}/RECORD

@@ -209,7 +209,7 @@ cartography/intel/crowdstrike/endpoints.py,sha256=tdqokMDW3p4fK3dHKKb2T1DTogvOJB
 cartography/intel/crowdstrike/spotlight.py,sha256=yNhj44-RYF6ubck-hHMKhKiNU0fCfhQf4Oagopc31EM,4754
 cartography/intel/crowdstrike/util.py,sha256=gfJ6Ptr6YdbBS9Qj9a_-Jc-IJroADDRcXqjh5TW0qXE,277
 cartography/intel/cve/__init__.py,sha256=3PLAhQ36g-aq40IHvba789WANsA-TY9B-Oe9mpQweQ8,2516
-cartography/intel/cve/feed.py,sha256=bKzetBVGYz8e2GKupUh2BilBCZuIwSfZdN23ArxYLZU,10147
+cartography/intel/cve/feed.py,sha256=1q9ZxbA4Hp2V84h7kadTImyCk7w0dmV7ROsZQ-kxKwE,10365
 cartography/intel/digitalocean/__init__.py,sha256=SMYB7LGIQOj_EgGSGVjWZk7SJNbP43hQuOfgOu6xYm4,1526
 cartography/intel/digitalocean/compute.py,sha256=9XctwMjq9h5dExFgExvawoqyiEwSoocNgaMm3Fgl5GM,4911
 cartography/intel/digitalocean/management.py,sha256=YWRnBLLL_bAP1vefIAQgm_-QzefGH0sZKmyU_EokHfA,3764
@@ -229,10 +229,10 @@ cartography/intel/gcp/dns.py,sha256=y2pvbmV04cnrMyuu_nbW3oc7uwHX6yEzn1n7veCsjmk,
 cartography/intel/gcp/gke.py,sha256=qaTwsVaxkwNhW5_Mw4bedOk7fgJK8y0LwwcYlUABXDg,7966
 cartography/intel/gcp/storage.py,sha256=oO_ayEhkXlj2Gn7T5MU41ZXiqwRwe6Ud4wzqyRTsyf4,9075
 cartography/intel/github/__init__.py,sha256=y876JJGTDJZEOFCDiNCJfcLNxN24pVj4s2N0YmuuoaE,1914
-cartography/intel/github/repos.py,sha256=FaNJK3Hs2MywuQaEyx_TTjHTtSXxfqs0jiV79aU_p7g,28959
+cartography/intel/github/repos.py,sha256=MmpxZASDJFQxDeSMxX3pZcpxCHFPos4_uYC_cX9KjOg,29865
 cartography/intel/github/teams.py,sha256=uQTiOfUBCk4qk0uw7jEPevAq-b2fvynRJ3t-62RZW2c,10659
-cartography/intel/github/users.py,sha256=zkMLVNfEMZIYYtPfqTsMXs2BNFzK8SfFRO7qnnlHy_s,8399
-cartography/intel/github/util.py,sha256=K6hbxypy4luKhIE1Uh5VWZc9OyjMK2OuO00vBAQfloA,8049
+cartography/intel/github/users.py,sha256=kkxk0TqPp8HKQAd3RvJs-N_ySFIxHqLsvkvJf0WoGyc,8565
+cartography/intel/github/util.py,sha256=K0cXOPuhnGvN-aqcSUBO3vTuKQLjufVal9kn2HwOpbo,8110
 cartography/intel/gsuite/__init__.py,sha256=AGIUskGlLCVGHbnQicNpNWi9AvmV7_7hUKTt-hsB2J8,4306
 cartography/intel/gsuite/api.py,sha256=J0dkNdfBVMrEv8vvStQu7YKVxXSyV45WueFhUS4aOG4,10310
 cartography/intel/jamf/__init__.py,sha256=Nof-LrUeevoieo6oP2GyfTwx8k5TUIgreW6hSj53YjQ,419
@@ -353,9 +353,9 @@ cartography/models/snipeit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 cartography/models/snipeit/asset.py,sha256=FyRAaeXuZjMy0eUQcSDFcgEAF5lbLMlvqp1Tv9d3Lv4,3238
 cartography/models/snipeit/tenant.py,sha256=p4rFnpNNuF1W5ilGBbexDaETWTwavfb38RcQGoImkQI,679
 cartography/models/snipeit/user.py,sha256=MsB4MiCVNTH6JpESime7cOkB89autZOXQpL6Z0l7L6o,2113
-cartography-0.96.1.dist-info/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
-cartography-0.96.1.dist-info/METADATA,sha256=6IcNRa-SP4o3DT8IBROqO5fdVzRMOlub-1VKVaEeWR0,1938
-cartography-0.96.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-cartography-0.96.1.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
-cartography-0.96.1.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
-cartography-0.96.1.dist-info/RECORD,,
+cartography-0.96.2.dist-info/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
+cartography-0.96.2.dist-info/METADATA,sha256=JPfNQ2Z_bulMqksx5unRej-WGueUKhi1Q9wFWqaOHk8,1938
+cartography-0.96.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+cartography-0.96.2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
+cartography-0.96.2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
+cartography-0.96.2.dist-info/RECORD,,