ingestr 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

ingestr/src/gorgias/helpers.py

@@ -1,5 +1,6 @@
 """Gorgias source helpers"""
 
+import time
 from typing import Any, Iterable, Optional, Tuple
 
 from dlt.common.pendulum import pendulum
@@ -9,6 +10,8 @@ from dlt.sources.helpers import requests
 from pyrate_limiter import Duration, Limiter, Rate
 from requests.auth import HTTPBasicAuth
 
+RETRY_COUNT = 10
+
 
 def get_max_datetime_from_datetime_fields(
     item: Dict[str, Any],
@@ -104,10 +107,24 @@ class GorgiasApi:
 
         while True:
             limiter.try_acquire(f"gorgias-{self.domain}")
-            response = requests.get(
-                url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
-            )
-            response.raise_for_status()
+
+            # this is to retry a back-off if we get a 429
+            for i in range(RETRY_COUNT):
+                try:
+                    response = requests.get(
+                        url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
+                    )
+                except Exception as e:
+                    retry_after = int(response.headers.get("Retry-After", 10))
+                    print(
+                        f"Got an error from Gorgias API, retrying after {retry_after} seconds",
+                        e,
+                    )
+                    time.sleep(retry_after)
+                    continue
+
+                break
+
             if len(response.json()["data"]) == 0:
                 break
 

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.6.2"
+__version__ = "0.6.4"

{ingestr-0.6.2.dist-info → ingestr-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.6.2
+Version: 0.6.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.6.2.dist-info → ingestr-0.6.4.dist-info}/RECORD

@@ -6,14 +6,14 @@ ingestr/src/factory.py,sha256=7skwetBXFIwAuKyTFfffGPgSo_PRRZ5uEnxOHUv28yQ,3517
 ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
 ingestr/src/sources.py,sha256=v0oFu-Pt_Zv06FypBb9_rHKP_iIMpkgm9MMaVpOUjPg,10313
 ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
-ingestr/src/version.py,sha256=jFlbxEJFS0G44LE-yXXVSwXACA1J_NyYDk5E20_2zpc,22
+ingestr/src/version.py,sha256=WMmvm2Keb76yMz8OL_h4fKT34Xpi-1BVfCiTn2QGzz4,22
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
 ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=iASGYJhdx6yZ0uDgIaQQfW91l3p0XOhowalIByLFkvw,21228
-ingestr/src/gorgias/helpers.py,sha256=dQ56CpZJobBVZP-vCM56vKHZdrfCHqla5peUt0bVr1c,4905
+ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
 ingestr/src/gorgias/helpers_test.py,sha256=kSR2nhB8U8HZ8pgDnd7HvXlzojmBnpOm8fTKHJvvKGY,1580
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
@@ -40,8 +40,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.6.2.dist-info/METADATA,sha256=UitFFYGnAlTIH9cR6_fSynXNnUsa9YVDxfvXVvkujq8,5699
-ingestr-0.6.2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.6.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.6.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.6.2.dist-info/RECORD,,
+ingestr-0.6.4.dist-info/METADATA,sha256=P9HvhFDdyABsqWup6JK4-LgdaCDH_UBvTByLtFvxbW4,5699
+ingestr-0.6.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.6.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.6.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.6.4.dist-info/RECORD,,