ingestr 0.6.2__tar.gz → 0.6.4__tar.gz

{ingestr-0.6.2 → ingestr-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.2
+Version: 0.6.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.6.2 → ingestr-0.6.4}/ingestr/src/gorgias/helpers.py

@@ -1,5 +1,6 @@
 """Gorgias source helpers"""
 
+import time
 from typing import Any, Iterable, Optional, Tuple
 
 from dlt.common.pendulum import pendulum
@@ -9,6 +10,8 @@ from dlt.sources.helpers import requests
 from pyrate_limiter import Duration, Limiter, Rate
 from requests.auth import HTTPBasicAuth
 
+RETRY_COUNT = 10
+
 
 def get_max_datetime_from_datetime_fields(
     item: Dict[str, Any],
@@ -104,10 +107,24 @@ class GorgiasApi:
 
         while True:
             limiter.try_acquire(f"gorgias-{self.domain}")
-            response = requests.get(
-                url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
-            )
-            response.raise_for_status()
+
+            # this is to retry a back-off if we get a 429
+            for i in range(RETRY_COUNT):
+                try:
+                    response = requests.get(
+                        url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
+                    )
+                except Exception as e:
+                    retry_after = int(response.headers.get("Retry-After", 10))
+                    print(
+                        f"Got an error from Gorgias API, retrying after {retry_after} seconds",
+                        e,
+                    )
+                    time.sleep(retry_after)
+                    continue
+
+                break
+
             if len(response.json()["data"]) == 0:
                 break
 

ingestr-0.6.4/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.6.4"

ingestr-0.6.2/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.6.2"