cartography 0.85.2__py3-none-any.whl → 0.85.3__py3-none-any.whl

cartography/intel/github/repos.py

@@ -23,7 +23,7 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
         {
             url
             login
-            repositories(first: 100, after: $cursor){
+            repositories(first: 50, after: $cursor){
                 pageInfo{
                     endCursor
                     hasNextPage
@@ -59,7 +59,7 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
                         login
                         __typename
                     }
-                    collaborators(affiliation: OUTSIDE, first: 100) {
+                    collaborators(affiliation: OUTSIDE, first: 50) {
                         edges {
                             permission
                         }

cartography/intel/github/util.py

@@ -1,6 +1,9 @@
 import json
 import logging
 import time
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone as tz
 from typing import Any
 from typing import Dict
 from typing import List
@@ -10,9 +13,11 @@ from typing import Tuple
 
 import requests
 
+
 logger = logging.getLogger(__name__)
 # Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
 _TIMEOUT = (60, 60)
+_GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD = 500
 
 
 class PaginatedGraphqlData(NamedTuple):
@@ -20,6 +25,30 @@ class PaginatedGraphqlData(NamedTuple):
     edges: List[Dict[str, Any]]
 
 
+def handle_rate_limit_sleep(token: str) -> None:
+    '''
+    Check the remaining rate limit and sleep if remaining is below threshold
+    :param token: The Github API token as string.
+    '''
+    response = requests.get('https://api.github.com/rate_limit', headers={'Authorization': f"token {token}"})
+    response.raise_for_status()
+    response_json = response.json()
+    rate_limit_obj = response_json['resources']['graphql']
+    remaining = rate_limit_obj['remaining']
+    threshold = _GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD
+    if remaining > threshold:
+        return
+    reset_at = datetime.fromtimestamp(rate_limit_obj['reset'], tz=tz.utc)
+    now = datetime.now(tz.utc)
+    # add an extra minute for safety
+    sleep_duration = reset_at - now + timedelta(minutes=1)
+    logger.warning(
+        f'Github graphql ratelimit has {remaining} remaining and is under threshold {threshold},'
+        f' sleeping until reset at {reset_at} for {sleep_duration}',
+    )
+    time.sleep(sleep_duration.seconds)
+
+
 def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dict:
     """
     Calls the GitHub v4 API and executes a query
@@ -116,6 +145,9 @@ def fetch_all(
     while has_next_page:
         exc: Any = None
         try:
+            # In the future, we may use also use the rateLimit object from the graphql response.
+            # But we still need at least one call to the REST endpoint in case the graphql remaining is already 0
+            handle_rate_limit_sleep(token)
             resp = fetch_page(token, api_url, organization, query, cursor, **kwargs)
             retry = 0
         except requests.exceptions.Timeout as err:
@@ -135,7 +167,7 @@ def fetch_all(
             )
             raise exc
         elif retry > 0:
-            time.sleep(1 * retry)
+            time.sleep(2 ** retry)
             continue
 
         resource = resp['data']['organization'][resource_type]
@@ -148,5 +180,6 @@ def fetch_all(
 
         cursor = resource['pageInfo']['endCursor']
         has_next_page = resource['pageInfo']['hasNextPage']
+
     org_data = {'url': resp['data']['organization']['url'], 'login': resp['data']['organization']['login']}
     return data, org_data

{cartography-0.85.2.dist-info → cartography-0.85.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.85.2
+Version: 0.85.3
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.85.2.dist-info → cartography-0.85.3.dist-info}/RECORD

@@ -228,10 +228,10 @@ cartography/intel/gcp/dns.py,sha256=y2pvbmV04cnrMyuu_nbW3oc7uwHX6yEzn1n7veCsjmk,
 cartography/intel/gcp/gke.py,sha256=qaTwsVaxkwNhW5_Mw4bedOk7fgJK8y0LwwcYlUABXDg,7966
 cartography/intel/gcp/storage.py,sha256=oO_ayEhkXlj2Gn7T5MU41ZXiqwRwe6Ud4wzqyRTsyf4,9075
 cartography/intel/github/__init__.py,sha256=y876JJGTDJZEOFCDiNCJfcLNxN24pVj4s2N0YmuuoaE,1914
-cartography/intel/github/repos.py,sha256=-4bX9U2G3FJhBW2c_NqbR2QcRUbS1FfSd2Fg8FRjNho,21768
+cartography/intel/github/repos.py,sha256=YPDdBMk6NkZjwPcqPW5LlCy_OS9tKcrZD6ygiUG93J0,21766
 cartography/intel/github/teams.py,sha256=aX1dj7HHVjw0hv_VvG1dee3WzTfmVMraP1khlwvN7Xs,5287
 cartography/intel/github/users.py,sha256=kQp0dxzP08DVrdvfVeCciQbrKPbbFvwbR_p_I_XGt7s,3826
-cartography/intel/github/util.py,sha256=pPKqawBuMoz12_gsnNkmtts_s_kQscRX5u_m6Zaxur0,6056
+cartography/intel/github/util.py,sha256=trigWITP7C44RjKpBT12fCcxtqo5WFqUOcB9VqtzifA,7465
 cartography/intel/gsuite/__init__.py,sha256=AGIUskGlLCVGHbnQicNpNWi9AvmV7_7hUKTt-hsB2J8,4306
 cartography/intel/gsuite/api.py,sha256=J0dkNdfBVMrEv8vvStQu7YKVxXSyV45WueFhUS4aOG4,10310
 cartography/intel/jamf/__init__.py,sha256=Nof-LrUeevoieo6oP2GyfTwx8k5TUIgreW6hSj53YjQ,419
@@ -321,10 +321,10 @@ cartography/models/semgrep/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 cartography/models/semgrep/deployment.py,sha256=or5qZDuR51MXzINpH15jZrqmSUvXQevCNYWJ7D6v-JI,745
 cartography/models/semgrep/findings.py,sha256=fzoCbn3QPNAapellnkXT-SSngqaXtJo7ArEHSoCekM8,3883
 cartography/models/semgrep/locations.py,sha256=kSk7Nn5Mn4Ob84MVZOo2GR0YFi-9Okq9pgA3FfC6_bk,3061
-cartography-0.85.2.dist-info/LICENSE,sha256=489ZXeW9G90up6ep-D1n-lJgk9ciNT2yxXpFgRSidtk,11341
-cartography-0.85.2.dist-info/METADATA,sha256=qiS3adJc2OK6_0bPGwUtH0XkoDY6ZBtoDpIQPjKCvog,1987
-cartography-0.85.2.dist-info/NOTICE,sha256=YOGAsjFtbyKj5tslYIg6V5jEYRuEvnSsIuDOUKj0Qj4,97
-cartography-0.85.2.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-cartography-0.85.2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
-cartography-0.85.2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
-cartography-0.85.2.dist-info/RECORD,,
+cartography-0.85.3.dist-info/LICENSE,sha256=489ZXeW9G90up6ep-D1n-lJgk9ciNT2yxXpFgRSidtk,11341
+cartography-0.85.3.dist-info/METADATA,sha256=uP8WvTjJDL7jKi7ztSmgZSdaqbnn-47IPcsmeVlw53A,1987
+cartography-0.85.3.dist-info/NOTICE,sha256=YOGAsjFtbyKj5tslYIg6V5jEYRuEvnSsIuDOUKj0Qj4,97
+cartography-0.85.3.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+cartography-0.85.3.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
+cartography-0.85.3.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
+cartography-0.85.3.dist-info/RECORD,,