ingestr 0.13.69__tar.gz → 0.13.71__tar.gz

{ingestr-0.13.69 → ingestr-0.13.71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.69
+Version: 0.13.71
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

ingestr-0.13.71/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.71"

{ingestr-0.13.69 → ingestr-0.13.71}/ingestr/src/kafka/__init__.py

@@ -83,7 +83,7 @@ def kafka_consumer(
     # read messages up to the maximum offsets,
     # not waiting for new messages
     with closing(consumer):
-        while tracker.has_unread:
+        while True:
             messages = consumer.consume(batch_size, timeout=batch_timeout)
             if not messages:
                 break
@@ -101,3 +101,6 @@ def kafka_consumer(
                     tracker.renew(msg)
 
             yield batch
+
+            if tracker.has_unread is False:
+                return

{ingestr-0.13.69 → ingestr-0.13.71}/ingestr/src/solidgate/__init__.py

@@ -86,16 +86,12 @@ COLUMN_HINTS = {
         "transaction_datetime_provider": {"data_type": "timestamp"},
         "transaction_datetime_utc": {"data_type": "timestamp"},
         "accounting_date": {"data_type": "date"},
-        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount": {"data_type": "double"},
+        "amount_in_major_units": {"data_type": "double"},
         "currency": {"data_type": "text"},
         "currency_minor_units": {"data_type": "bigint"},
-        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "payout_amount_in_major_units": {
-            "data_type": "decimal",
-            "precision": 18,
-            "scale": 4,
-        },
+        "payout_amount": {"data_type": "double"},
+        "payout_amount_in_major_units": {"data_type": "double"},
         "payout_currency": {"data_type": "text"},
         "payout_currency_minor_units": {"data_type": "bigint"},
         "record_type_key": {"data_type": "text"},

{ingestr-0.13.69 → ingestr-0.13.71}/ingestr/src/solidgate/helpers.py

@@ -1,4 +1,5 @@
 import base64
+import csv
 import hashlib
 import hmac
 import json
@@ -123,11 +124,18 @@ class SolidgateClient:
                 except json.JSONDecodeError:
                     try:
                         csv_data = get_response.content.decode("utf-8")
-                        df = pd.read_csv(StringIO(csv_data))
-                        df["created_at"] = df["created_at"].apply(
-                            lambda x: pendulum.parse(x)
-                        )
-                        return df
+                        reader = csv.DictReader(StringIO(csv_data))
+                        rows = []
+                        for row in reader:
+                            if row["created_at"]:
+                                row["created_at"] = pendulum.parse(row["created_at"])
+                            else:
+                                row["created_at"] = None
+
+                            row2 = {k: v for k, v in row.items() if v != ''}
+                            rows.append(row2)
+                        
+                        return rows
                     except Exception as e:
                         raise Exception(f"Error reading CSV: {e}")
             else:

{ingestr-0.13.69 → ingestr-0.13.71}/ingestr/src/tiktok_ads/__init__.py

@@ -112,7 +112,8 @@ def tiktok_source(
         datetime=(
             dlt.sources.incremental(
                 incremental_loading_param,
-                start_date,
+                initial_value=start_date,
+                end_value=end_date,
                 range_end="closed",
                 range_start="closed",
             )
@@ -120,15 +121,16 @@ def tiktok_source(
             else None
         ),
     ) -> Iterable[TDataItem]:
-        current_date = start_date.in_tz(timezone)
+        start_date_tz_adjusted = start_date.in_tz(timezone)
+        end_date_tz_adjusted = end_date.in_tz(timezone)
 
         if datetime is not None:
-            datetime_str = datetime.last_value
-            current_date = ensure_pendulum_datetime(datetime_str).in_tz(timezone)
+            start_date_tz_adjusted = ensure_pendulum_datetime(datetime.last_value).in_tz(timezone)
+            end_date_tz_adjusted = ensure_pendulum_datetime(datetime.end_value).in_tz(timezone)
 
         list_of_interval = find_intervals(
-            current_date=current_date,
-            end_date=end_date,
+            current_date=start_date_tz_adjusted,
+            end_date=end_date_tz_adjusted,
             interval_days=interval_days,
         )
 

ingestr-0.13.69/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.69"