cartography 0.85.2__tar.gz → 0.85.3__tar.gz

{cartography-0.85.2/cartography.egg-info → cartography-0.85.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.85.2
+Version: 0.85.3
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.85.2 → cartography-0.85.3}/cartography/intel/github/repos.py

@@ -23,7 +23,7 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
         {
             url
             login
-            repositories(first: 100, after: $cursor){
+            repositories(first: 50, after: $cursor){
                 pageInfo{
                     endCursor
                     hasNextPage
@@ -59,7 +59,7 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
                         login
                         __typename
                     }
-                    collaborators(affiliation: OUTSIDE, first: 100) {
+                    collaborators(affiliation: OUTSIDE, first: 50) {
                         edges {
                             permission
                         }

{cartography-0.85.2 → cartography-0.85.3}/cartography/intel/github/util.py

@@ -1,6 +1,9 @@
 import json
 import logging
 import time
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone as tz
 from typing import Any
 from typing import Dict
 from typing import List
@@ -10,9 +13,11 @@ from typing import Tuple
 
 import requests
 
+
 logger = logging.getLogger(__name__)
 # Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
 _TIMEOUT = (60, 60)
+_GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD = 500
 
 
 class PaginatedGraphqlData(NamedTuple):
@@ -20,6 +25,30 @@ class PaginatedGraphqlData(NamedTuple):
     edges: List[Dict[str, Any]]
 
 
+def handle_rate_limit_sleep(token: str) -> None:
+    '''
+    Check the remaining rate limit and sleep if remaining is below threshold
+    :param token: The Github API token as string.
+    '''
+    response = requests.get('https://api.github.com/rate_limit', headers={'Authorization': f"token {token}"})
+    response.raise_for_status()
+    response_json = response.json()
+    rate_limit_obj = response_json['resources']['graphql']
+    remaining = rate_limit_obj['remaining']
+    threshold = _GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD
+    if remaining > threshold:
+        return
+    reset_at = datetime.fromtimestamp(rate_limit_obj['reset'], tz=tz.utc)
+    now = datetime.now(tz.utc)
+    # add an extra minute for safety
+    sleep_duration = reset_at - now + timedelta(minutes=1)
+    logger.warning(
+        f'Github graphql ratelimit has {remaining} remaining and is under threshold {threshold},'
+        f' sleeping until reset at {reset_at} for {sleep_duration}',
+    )
+    time.sleep(sleep_duration.seconds)
+
+
 def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dict:
     """
     Calls the GitHub v4 API and executes a query
@@ -116,6 +145,9 @@ def fetch_all(
     while has_next_page:
         exc: Any = None
         try:
+            # In the future, we may use also use the rateLimit object from the graphql response.
+            # But we still need at least one call to the REST endpoint in case the graphql remaining is already 0
+            handle_rate_limit_sleep(token)
             resp = fetch_page(token, api_url, organization, query, cursor, **kwargs)
             retry = 0
         except requests.exceptions.Timeout as err:
@@ -135,7 +167,7 @@ def fetch_all(
             )
             raise exc
         elif retry > 0:
-            time.sleep(1 * retry)
+            time.sleep(2 ** retry)
             continue
 
         resource = resp['data']['organization'][resource_type]
@@ -148,5 +180,6 @@ def fetch_all(
 
         cursor = resource['pageInfo']['endCursor']
         has_next_page = resource['pageInfo']['hasNextPage']
+
     org_data = {'url': resp['data']['organization']['url'], 'login': resp['data']['organization']['login']}
     return data, org_data

{cartography-0.85.2 → cartography-0.85.3/cartography.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.85.2
+Version: 0.85.3
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.85.2 → cartography-0.85.3}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import find_packages
 from setuptools import setup
 
-__version__ = '0.85.2'
+__version__ = '0.85.3'
 
 
 setup(