ingestr 0.6.1__tar.gz → 0.6.3__tar.gz

{ingestr-0.6.1 → ingestr-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.1
+Version: 0.6.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.6.1 → ingestr-0.6.3}/ingestr/src/gorgias/__init__.py

@@ -15,7 +15,7 @@ def gorgias_source(
     domain: str = dlt.secrets.value,
     email: str = dlt.config.value,
     api_key: str = dlt.secrets.value,
-    start_date: TAnyDateTime = "2024-06-15",
+    start_date: TAnyDateTime = "2000-01-01",
     end_date: TAnyDateTime = None,
 ) -> Iterable[DltResource]:
     """

{ingestr-0.6.1 → ingestr-0.6.3}/ingestr/src/gorgias/helpers.py

@@ -1,5 +1,6 @@
 """Gorgias source helpers"""
 
+import time
 from typing import Any, Iterable, Optional, Tuple
 
 from dlt.common.pendulum import pendulum
@@ -9,6 +10,8 @@ from dlt.sources.helpers import requests
 from pyrate_limiter import Duration, Limiter, Rate
 from requests.auth import HTTPBasicAuth
 
+RETRY_COUNT = 3
+
 
 def get_max_datetime_from_datetime_fields(
     item: Dict[str, Any],
@@ -104,10 +107,21 @@ class GorgiasApi:
 
         while True:
             limiter.try_acquire(f"gorgias-{self.domain}")
-            response = requests.get(
-                url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
-            )
-            response.raise_for_status()
+
+            # this is to retry a back-off if we get a 429
+            for i in range(RETRY_COUNT):
+                response = requests.get(
+                    url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
+                )
+
+                if response.status_code == 429:
+                    retry_after = int(response.headers.get("Retry-After", 2))
+                    time.sleep(retry_after)
+                    continue
+
+                response.raise_for_status()
+                break
+
             if len(response.json()["data"]) == 0:
                 break
 

ingestr-0.6.3/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.6.3"

ingestr-0.6.1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.6.1"