ingestr 0.13.69__py3-none-any.whl → 0.13.71__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.69"
+version = "v0.13.71"

ingestr/src/kafka/__init__.py

@@ -83,7 +83,7 @@ def kafka_consumer(
     # read messages up to the maximum offsets,
     # not waiting for new messages
     with closing(consumer):
-        while tracker.has_unread:
+        while True:
             messages = consumer.consume(batch_size, timeout=batch_timeout)
             if not messages:
                 break
@@ -101,3 +101,6 @@ def kafka_consumer(
                     tracker.renew(msg)
 
             yield batch
+
+            if tracker.has_unread is False:
+                return

ingestr/src/solidgate/__init__.py

@@ -86,16 +86,12 @@ COLUMN_HINTS = {
         "transaction_datetime_provider": {"data_type": "timestamp"},
         "transaction_datetime_utc": {"data_type": "timestamp"},
         "accounting_date": {"data_type": "date"},
-        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount": {"data_type": "double"},
+        "amount_in_major_units": {"data_type": "double"},
         "currency": {"data_type": "text"},
         "currency_minor_units": {"data_type": "bigint"},
-        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "payout_amount_in_major_units": {
-            "data_type": "decimal",
-            "precision": 18,
-            "scale": 4,
-        },
+        "payout_amount": {"data_type": "double"},
+        "payout_amount_in_major_units": {"data_type": "double"},
         "payout_currency": {"data_type": "text"},
         "payout_currency_minor_units": {"data_type": "bigint"},
         "record_type_key": {"data_type": "text"},

ingestr/src/solidgate/helpers.py

@@ -1,4 +1,5 @@
 import base64
+import csv
 import hashlib
 import hmac
 import json
@@ -123,11 +124,18 @@ class SolidgateClient:
                 except json.JSONDecodeError:
                     try:
                         csv_data = get_response.content.decode("utf-8")
-                        df = pd.read_csv(StringIO(csv_data))
-                        df["created_at"] = df["created_at"].apply(
-                            lambda x: pendulum.parse(x)
-                        )
-                        return df
+                        reader = csv.DictReader(StringIO(csv_data))
+                        rows = []
+                        for row in reader:
+                            if row["created_at"]:
+                                row["created_at"] = pendulum.parse(row["created_at"])
+                            else:
+                                row["created_at"] = None
+
+                            row2 = {k: v for k, v in row.items() if v != ''}
+                            rows.append(row2)
+                        
+                        return rows
                     except Exception as e:
                         raise Exception(f"Error reading CSV: {e}")
             else:

ingestr/src/tiktok_ads/__init__.py

@@ -112,7 +112,8 @@ def tiktok_source(
         datetime=(
             dlt.sources.incremental(
                 incremental_loading_param,
-                start_date,
+                initial_value=start_date,
+                end_value=end_date,
                 range_end="closed",
                 range_start="closed",
             )
@@ -120,15 +121,16 @@ def tiktok_source(
             else None
         ),
     ) -> Iterable[TDataItem]:
-        current_date = start_date.in_tz(timezone)
+        start_date_tz_adjusted = start_date.in_tz(timezone)
+        end_date_tz_adjusted = end_date.in_tz(timezone)
 
         if datetime is not None:
-            datetime_str = datetime.last_value
-            current_date = ensure_pendulum_datetime(datetime_str).in_tz(timezone)
+            start_date_tz_adjusted = ensure_pendulum_datetime(datetime.last_value).in_tz(timezone)
+            end_date_tz_adjusted = ensure_pendulum_datetime(datetime.end_value).in_tz(timezone)
 
         list_of_interval = find_intervals(
-            current_date=current_date,
-            end_date=end_date,
+            current_date=start_date_tz_adjusted,
+            end_date=end_date_tz_adjusted,
             interval_days=interval_days,
         )
 

{ingestr-0.13.69.dist-info → ingestr-0.13.71.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.69
+Version: 0.13.71
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.69.dist-info → ingestr-0.13.71.dist-info}/RECORD

@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=wu4c_4nit3c5v2K1af68DjT2hIfEiCvzQ8EZJxOkJjo,26369
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=Ke17ObaAtICcxQMwBjBfKvBywicwCyo9pp8D2NmSsjk,21
+ingestr/src/buildinfo.py,sha256=GlvBwB8aV5mufGGW4MafbR067L8Zr8wModuobUsy3Gs,21
 ingestr/src/destinations.py,sha256=ivTPio0zzqLRx22i597pxZHMNClz-XvYSyCaCPuGd8g,22248
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=AJCvlK4M1sIpAAks1K-xsR_uxziIxru74mj572zixhg,6546
@@ -74,7 +74,7 @@ ingestr/src/hubspot/__init__.py,sha256=wqHefhc_YRI5dNFCcpvH-UUilNThE49sbGouSBiHY
 ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/isoc_pulse/__init__.py,sha256=9b4eN4faatpiwTuRNPuYcEt1hEFDEjua9XhfakUigBk,4648
-ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
+ingestr/src/kafka/__init__.py,sha256=QUHsGmdv5_E-3z0GDHXvbk39puwuGDBsyYSDhvbA89E,3595
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=YretSz4F28tbkcPhd55mBp2Xk7XE9unyWx0nmvl8iEc,6235
 ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0s,3152
@@ -117,8 +117,8 @@ ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
-ingestr/src/solidgate/__init__.py,sha256=as3m-lrs8teI3BW_8__5SY7BY8Af7SetzuU8Cj3L80U,8170
-ingestr/src/solidgate/helpers.py,sha256=j8FjYHyUJTwkJ8VY8nre2qMesLeRq3eTwU22Ff-2K8M,5426
+ingestr/src/solidgate/__init__.py,sha256=Ts83j-JSnFsFuF4tDhVOfZKg7H0-bIpfn3kg1ZOR58A,8003
+ingestr/src/solidgate/helpers.py,sha256=yZRBrBZlofuTvKOs_KQVoH1K60shSeDMyiBYzXYfMT0,5750
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
 ingestr/src/stripe_analytics/__init__.py,sha256=mK8dGKAlMPVqGE9gG30XfbvOvgVD0yWhNpt-D3iavDY,6385
@@ -126,7 +126,7 @@ ingestr/src/stripe_analytics/helpers.py,sha256=KGtRcSrhKEqzJ3AWpgDV2o4cuBFaIwu2G
 ingestr/src/stripe_analytics/settings.py,sha256=xt1-ljwP4nLTNUa8l3KwFbtK8FtQHgHpzGF5uPKfRsw,2246
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
+ingestr/src/tiktok_ads/__init__.py,sha256=KFaN_s7VIj9AW6z-4Fcq0PXYcoX7OtKUgrdcHEitvUk,4794
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=jmWHvZzN1Vt_PWrJkgq5a2wIwon-OBEzXoZx0jEy-74,3905
 ingestr/src/trustpilot/__init__.py,sha256=ofhjep4qRPIi8q41qc97QVex8UbWF-Fd7gUsqeQlQX8,1279
 ingestr/src/trustpilot/client.py,sha256=zKYt5C7nrR83Id0KN49EPmtml8MEtlSPlAosEFU3VXY,1616
@@ -147,8 +147,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.69.dist-info/METADATA,sha256=DNrbX4KACx_XBhLJuMTig_hG3m_jjPw9GQrOXlsG6BQ,15022
-ingestr-0.13.69.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.69.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.69.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.69.dist-info/RECORD,,
+ingestr-0.13.71.dist-info/METADATA,sha256=YIzmk8sSvhEUF2Yb7VRLRyGJpcigSaJrExacU-QtXlU,15022
+ingestr-0.13.71.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.71.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.71.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.71.dist-info/RECORD,,