ingestr 0.13.68__tar.gz → 0.13.70__tar.gz

ingestr-0.13.70/.dlt/config.toml

@@ -0,0 +1,3 @@
+[runtime]
+
+dlthub_telemetry=false

{ingestr-0.13.68 → ingestr-0.13.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.68
+Version: 0.13.70
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.68 → ingestr-0.13.70}/ingestr/conftest.py

@@ -2,6 +2,7 @@ import os
 import tempfile
 from concurrent.futures import ThreadPoolExecutor
 
+import pytest
 from main_test import DESTINATIONS, SOURCES  # type: ignore
 
 
@@ -15,6 +16,14 @@ def pytest_configure_node(node):
     node.workerinput["shared_directory"] = node.config.shared_directory
 
 
+@pytest.fixture(scope="session")
+def shared_directory(request):
+    if is_master(request.config):
+        return request.config.shared_directory
+    else:
+        return request.config.workerinput["shared_directory"]
+
+
 def is_master(config):
     """True if the code running the given pytest.config object is running in a xdist master
     node or not running xdist at all.

{ingestr-0.13.68 → ingestr-0.13.70}/ingestr/main.py

@@ -256,7 +256,7 @@ def ingest(
         Optional[list[str]],
         typer.Option(
             help="The column types to be used for the destination table in the format of 'column_name:column_type'",
-            envvar=["COLUMNS", "INGESTR_COLUMNS"],
+            envvar=["INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
     yield_limit: Annotated[

ingestr-0.13.70/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.70"

{ingestr-0.13.68 → ingestr-0.13.70}/ingestr/src/kafka/__init__.py

@@ -83,7 +83,7 @@ def kafka_consumer(
     # read messages up to the maximum offsets,
     # not waiting for new messages
     with closing(consumer):
-        while tracker.has_unread:
+        while True:
             messages = consumer.consume(batch_size, timeout=batch_timeout)
             if not messages:
                 break
@@ -101,3 +101,6 @@ def kafka_consumer(
                     tracker.renew(msg)
 
             yield batch
+
+            if tracker.has_unread is False:
+                return

ingestr-0.13.70/ingestr/src/solidgate/__init__.py

@@ -0,0 +1,223 @@
+from typing import Iterable, Iterator
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .helpers import SolidgateClient
+
+COLUMN_HINTS = {
+    "subscriptions": {
+        "id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "status": {"data_type": "text"},
+        "started_at": {"data_type": "timestamp"},
+        "updated_at": {"data_type": "timestamp"},
+        "expired_at": {"data_type": "timestamp"},
+        "next_charge_at": {"data_type": "timestamp"},
+        "payment_type": {"data_type": "text"},
+        "trial": {"data_type": "bool"},
+        "cancelled_at": {"data_type": "timestamp"},
+        "cancellation_requested_at": {"data_type": "timestamp"},
+        "cancel_code": {"data_type": "text"},
+        "cancel_message": {"data_type": "text"},
+        "customer": {"data_type": "json"},
+        "product": {"data_type": "json"},
+        "invoices": {"data_type": "json"},
+    },
+    "apm_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "method": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+    },
+    "card_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "psp_order_id": {"data_type": "text"},
+        "provider_payment_id": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "payment_type": {"data_type": "text"},
+        "type": {"data_type": "text"},
+        "is_secured": {"data_type": "bool"},
+        "routing": {"data_type": "json"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "customer_first_name": {"data_type": "text"},
+        "customer_last_name": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "mid": {"data_type": "text"},
+        "traffic_source": {"data_type": "text"},
+        "platform": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+        "fraudulent": {"data_type": "bool"},
+    },
+    "financial_entries": {
+        "id": {
+            "data_type": "text",
+            "nullable": False,
+            "primary_key": True,
+        },
+        "order_id": {"data_type": "text"},
+        "external_psp_order_id": {"data_type": "text"},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "transaction_datetime_provider": {"data_type": "timestamp"},
+        "transaction_datetime_utc": {"data_type": "timestamp"},
+        "accounting_date": {"data_type": "date"},
+        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "currency": {"data_type": "text"},
+        "currency_minor_units": {"data_type": "bigint"},
+        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "payout_amount_in_major_units": {
+            "data_type": "decimal",
+            "precision": 18,
+            "scale": 4,
+        },
+        "payout_currency": {"data_type": "text"},
+        "payout_currency_minor_units": {"data_type": "bigint"},
+        "record_type_key": {"data_type": "text"},
+        "provider": {"data_type": "text"},
+        "payment_method": {"data_type": "text"},
+        "card_brand": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "issuing_country": {"data_type": "text"},
+        "transaction_id": {"data_type": "text"},
+        "chargeback_id": {"data_type": "text"},
+        "legal_entity": {"data_type": "text"},
+    },
+}
+
+
+@dlt.source(max_table_nesting=0)
+def solidgate_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None,
+    public_key: str,
+    secret_key: str,
+) -> Iterable[DltResource]:
+    solidgate_client = SolidgateClient(public_key, secret_key)
+
+    @dlt.resource(
+        name="subscriptions",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["subscriptions"],  # type: ignore
+    )
+    def fetch_all_subscriptions(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "subscriptions"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="apm_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["apm_orders"],  # type: ignore
+    )
+    def fetch_apm_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "apm-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="card_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["card_orders"],  # type: ignore
+    )
+    def fetch_card_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "card-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="financial_entries",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["financial_entries"],  # type: ignore
+    )
+    def fetch_financial_entries(
+        dateTime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ):
+        if dateTime.end_value is None:
+            end_date = pendulum.now(tz="UTC")
+        else:
+            end_date = dateTime.end_value
+
+        start_date = dateTime.last_value
+        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
+
+    return (
+        fetch_all_subscriptions,
+        fetch_apm_orders,
+        fetch_card_orders,
+        fetch_financial_entries,
+    )

{ingestr-0.13.68 → ingestr-0.13.70}/ingestr/src/solidgate/helpers.py

@@ -92,9 +92,6 @@ class SolidgateClient:
         if not report_url:
             return f"Report URL not found in the response: {post_response.json()}", 400
 
-        # Wait for 5 seconds before attempting to download the report as report may not be immediately available
-        time.sleep(5)
-
         data = self.public_key + self.public_key
         hmac_hash = hmac.new(
             self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
@@ -109,27 +106,38 @@ class SolidgateClient:
             "Content-Type": "application/json",
         }
 
-        get_response = self.client.get(report_url, headers=headers_get)
+        # Retry getting the report for up to 10 minutes (600 seconds) with 5-second intervals
+        max_retries = 120  # 10 minutes / 5 seconds = 120 attempts
+        retry_count = 0
+
+        while retry_count < max_retries:
+            get_response = self.client.get(report_url, headers=headers_get)
 
-        if get_response.status_code == 200:
-            try:
-                response_json = json.loads(get_response.content)
-                if "error" in response_json:
-                    raise Exception(f"API Error: {response_json['error']['messages']}")
-            except json.JSONDecodeError:
+            if get_response.status_code == 200:
                 try:
-                    csv_data = get_response.content.decode("utf-8")
-                    df = pd.read_csv(StringIO(csv_data))
-                    df["created_at"] = df["created_at"].apply(
-                        lambda x: pendulum.parse(x)
+                    response_json = json.loads(get_response.content)
+                    if "error" in response_json:
+                        raise Exception(
+                            f"API Error: {response_json['error']['messages']}"
+                        )
+                except json.JSONDecodeError:
+                    try:
+                        csv_data = get_response.content.decode("utf-8")
+                        df = pd.read_csv(StringIO(csv_data))
+                        df["created_at"] = df["created_at"].apply(
+                            lambda x: pendulum.parse(x)
+                        )
+                        return df
+                    except Exception as e:
+                        raise Exception(f"Error reading CSV: {e}")
+            else:
+                # Report might not be ready yet, wait and retry
+                retry_count += 1
+                if retry_count >= max_retries:
+                    raise Exception(
+                        f"Failed to get report after {max_retries} attempts. Status code: {get_response.status_code}"
                     )
-                    return df
-                except Exception as e:
-                    raise Exception(f"Error reading CSV: {e}")
-        else:
-            raise Exception(
-                f"Failed to get report. Status code: {get_response.status_code}"
-            )
+                time.sleep(5)  # Wait 5 seconds before retrying
 
     def generateSignature(self, json_string):
         data = self.public_key + json_string + self.public_key

{ingestr-0.13.68 → ingestr-0.13.70}/ingestr/src/sources.py

@@ -80,18 +80,18 @@ class SqlSource:
         if uri.startswith("snowflake://"):
             parsed_uri = urlparse(uri)
             query_params = parse_qs(parsed_uri.query)
-            
+
             if "private_key" in query_params:
                 from dlt.common.libs.cryptography import decode_private_key
-                
+
                 private_key = query_params["private_key"][0]
                 passphrase = query_params.get("private_key_passphrase", [None])[0]
                 decoded_key = decode_private_key(private_key, passphrase)
-                
+
                 query_params["private_key"] = [base64.b64encode(decoded_key).decode()]
                 if "private_key_passphrase" in query_params:
                     del query_params["private_key_passphrase"]
-                
+
                 # Rebuild URI
                 uri = parsed_uri._replace(
                     query=urlencode(query_params, doseq=True)

ingestr-0.13.68/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.68"

ingestr-0.13.68/ingestr/src/solidgate/__init__.py

@@ -1,127 +0,0 @@
-from typing import Iterable, Iterator
-
-import dlt
-import pendulum
-from dlt.sources import DltResource
-
-from .helpers import SolidgateClient
-
-
-@dlt.source(max_table_nesting=0)
-def solidgate_source(
-    start_date: pendulum.DateTime,
-    end_date: pendulum.DateTime | None,
-    public_key: str,
-    secret_key: str,
-) -> Iterable[DltResource]:
-    solidgate_client = SolidgateClient(public_key, secret_key)
-
-    @dlt.resource(
-        name="subscriptions",
-        write_disposition="merge",
-        primary_key="id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
-    )
-    def fetch_all_subscriptions(
-        dateTime=dlt.sources.incremental(
-            "updated_at",
-            initial_value=start_date,
-            end_value=end_date,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[dict]:
-        path = "subscriptions"
-        if dateTime.end_value is None:
-            end_dt = pendulum.now(tz="UTC")
-        else:
-            end_dt = dateTime.end_value
-
-        start_dt = dateTime.last_value
-        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
-
-    @dlt.resource(
-        name="apm_orders",
-        write_disposition="merge",
-        primary_key="order_id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
-    )
-    def fetch_apm_orders(
-        dateTime=dlt.sources.incremental(
-            "updated_at",
-            initial_value=start_date,
-            end_value=end_date,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[dict]:
-        path = "apm-orders"
-        if dateTime.end_value is None:
-            end_dt = pendulum.now(tz="UTC")
-        else:
-            end_dt = dateTime.end_value
-
-        start_dt = dateTime.last_value
-        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
-
-    @dlt.resource(
-        name="card_orders",
-        write_disposition="merge",
-        primary_key="order_id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
-    )
-    def fetch_card_orders(
-        dateTime=dlt.sources.incremental(
-            "updated_at",
-            initial_value=start_date,
-            end_value=end_date,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[dict]:
-        path = "card-orders"
-        if dateTime.end_value is None:
-            end_dt = pendulum.now(tz="UTC")
-        else:
-            end_dt = dateTime.end_value
-
-        start_dt = dateTime.last_value
-        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
-
-    @dlt.resource(
-        name="financial_entries",
-        write_disposition="merge",
-        primary_key="id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
-    )
-    def fetch_financial_entries(
-        dateTime=dlt.sources.incremental(
-            "created_at",
-            initial_value=start_date,
-            end_value=end_date,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ):
-        if dateTime.end_value is None:
-            end_date = pendulum.now(tz="UTC")
-        else:
-            end_date = dateTime.end_value
-
-        start_date = dateTime.last_value
-        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
-
-    return (
-        fetch_all_subscriptions,
-        fetch_apm_orders,
-        fetch_card_orders,
-        fetch_financial_entries,
-    )