ingestr 0.13.68__py3-none-any.whl → 0.13.70__py3-none-any.whl

ingestr/conftest.py

@@ -2,6 +2,7 @@ import os
 import tempfile
 from concurrent.futures import ThreadPoolExecutor
 
+import pytest
 from main_test import DESTINATIONS, SOURCES  # type: ignore
 
 
@@ -15,6 +16,14 @@ def pytest_configure_node(node):
     node.workerinput["shared_directory"] = node.config.shared_directory
 
 
+@pytest.fixture(scope="session")
+def shared_directory(request):
+    if is_master(request.config):
+        return request.config.shared_directory
+    else:
+        return request.config.workerinput["shared_directory"]
+
+
 def is_master(config):
     """True if the code running the given pytest.config object is running in a xdist master
     node or not running xdist at all.

ingestr/main.py

@@ -256,7 +256,7 @@ def ingest(
         Optional[list[str]],
         typer.Option(
             help="The column types to be used for the destination table in the format of 'column_name:column_type'",
-            envvar=["COLUMNS", "INGESTR_COLUMNS"],
+            envvar=["INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
     yield_limit: Annotated[

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.68"
+version = "v0.13.70"

ingestr/src/kafka/__init__.py

@@ -83,7 +83,7 @@ def kafka_consumer(
     # read messages up to the maximum offsets,
     # not waiting for new messages
     with closing(consumer):
-        while tracker.has_unread:
+        while True:
             messages = consumer.consume(batch_size, timeout=batch_timeout)
             if not messages:
                 break
@@ -101,3 +101,6 @@ def kafka_consumer(
                     tracker.renew(msg)
 
             yield batch
+
+            if tracker.has_unread is False:
+                return

ingestr/src/solidgate/__init__.py

@@ -6,6 +6,110 @@ from dlt.sources import DltResource
 
 from .helpers import SolidgateClient
 
+COLUMN_HINTS = {
+    "subscriptions": {
+        "id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "status": {"data_type": "text"},
+        "started_at": {"data_type": "timestamp"},
+        "updated_at": {"data_type": "timestamp"},
+        "expired_at": {"data_type": "timestamp"},
+        "next_charge_at": {"data_type": "timestamp"},
+        "payment_type": {"data_type": "text"},
+        "trial": {"data_type": "bool"},
+        "cancelled_at": {"data_type": "timestamp"},
+        "cancellation_requested_at": {"data_type": "timestamp"},
+        "cancel_code": {"data_type": "text"},
+        "cancel_message": {"data_type": "text"},
+        "customer": {"data_type": "json"},
+        "product": {"data_type": "json"},
+        "invoices": {"data_type": "json"},
+    },
+    "apm_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "method": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+    },
+    "card_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "psp_order_id": {"data_type": "text"},
+        "provider_payment_id": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "payment_type": {"data_type": "text"},
+        "type": {"data_type": "text"},
+        "is_secured": {"data_type": "bool"},
+        "routing": {"data_type": "json"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "customer_first_name": {"data_type": "text"},
+        "customer_last_name": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "mid": {"data_type": "text"},
+        "traffic_source": {"data_type": "text"},
+        "platform": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+        "fraudulent": {"data_type": "bool"},
+    },
+    "financial_entries": {
+        "id": {
+            "data_type": "text",
+            "nullable": False,
+            "primary_key": True,
+        },
+        "order_id": {"data_type": "text"},
+        "external_psp_order_id": {"data_type": "text"},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "transaction_datetime_provider": {"data_type": "timestamp"},
+        "transaction_datetime_utc": {"data_type": "timestamp"},
+        "accounting_date": {"data_type": "date"},
+        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "currency": {"data_type": "text"},
+        "currency_minor_units": {"data_type": "bigint"},
+        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "payout_amount_in_major_units": {
+            "data_type": "decimal",
+            "precision": 18,
+            "scale": 4,
+        },
+        "payout_currency": {"data_type": "text"},
+        "payout_currency_minor_units": {"data_type": "bigint"},
+        "record_type_key": {"data_type": "text"},
+        "provider": {"data_type": "text"},
+        "payment_method": {"data_type": "text"},
+        "card_brand": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "issuing_country": {"data_type": "text"},
+        "transaction_id": {"data_type": "text"},
+        "chargeback_id": {"data_type": "text"},
+        "legal_entity": {"data_type": "text"},
+    },
+}
+
 
 @dlt.source(max_table_nesting=0)
 def solidgate_source(
@@ -20,9 +124,7 @@ def solidgate_source(
         name="subscriptions",
         write_disposition="merge",
         primary_key="id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
+        columns=COLUMN_HINTS["subscriptions"],  # type: ignore
     )
     def fetch_all_subscriptions(
         dateTime=dlt.sources.incremental(
@@ -46,9 +148,7 @@ def solidgate_source(
         name="apm_orders",
         write_disposition="merge",
         primary_key="order_id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
+        columns=COLUMN_HINTS["apm_orders"],  # type: ignore
     )
     def fetch_apm_orders(
         dateTime=dlt.sources.incremental(
@@ -72,9 +172,7 @@ def solidgate_source(
         name="card_orders",
         write_disposition="merge",
         primary_key="order_id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
+        columns=COLUMN_HINTS["card_orders"],  # type: ignore
     )
     def fetch_card_orders(
         dateTime=dlt.sources.incremental(
@@ -98,9 +196,7 @@ def solidgate_source(
         name="financial_entries",
         write_disposition="merge",
         primary_key="id",
-        columns={
-            "created_at": {"data_type": "timestamp", "partition": True},
-        },
+        columns=COLUMN_HINTS["financial_entries"],  # type: ignore
     )
     def fetch_financial_entries(
         dateTime=dlt.sources.incremental(

ingestr/src/solidgate/helpers.py

@@ -92,9 +92,6 @@ class SolidgateClient:
         if not report_url:
             return f"Report URL not found in the response: {post_response.json()}", 400
 
-        # Wait for 5 seconds before attempting to download the report as report may not be immediately available
-        time.sleep(5)
-
         data = self.public_key + self.public_key
         hmac_hash = hmac.new(
             self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
@@ -109,27 +106,38 @@ class SolidgateClient:
             "Content-Type": "application/json",
         }
 
-        get_response = self.client.get(report_url, headers=headers_get)
+        # Retry getting the report for up to 10 minutes (600 seconds) with 5-second intervals
+        max_retries = 120  # 10 minutes / 5 seconds = 120 attempts
+        retry_count = 0
+
+        while retry_count < max_retries:
+            get_response = self.client.get(report_url, headers=headers_get)
 
-        if get_response.status_code == 200:
-            try:
-                response_json = json.loads(get_response.content)
-                if "error" in response_json:
-                    raise Exception(f"API Error: {response_json['error']['messages']}")
-            except json.JSONDecodeError:
+            if get_response.status_code == 200:
                 try:
-                    csv_data = get_response.content.decode("utf-8")
-                    df = pd.read_csv(StringIO(csv_data))
-                    df["created_at"] = df["created_at"].apply(
-                        lambda x: pendulum.parse(x)
+                    response_json = json.loads(get_response.content)
+                    if "error" in response_json:
+                        raise Exception(
+                            f"API Error: {response_json['error']['messages']}"
+                        )
+                except json.JSONDecodeError:
+                    try:
+                        csv_data = get_response.content.decode("utf-8")
+                        df = pd.read_csv(StringIO(csv_data))
+                        df["created_at"] = df["created_at"].apply(
+                            lambda x: pendulum.parse(x)
+                        )
+                        return df
+                    except Exception as e:
+                        raise Exception(f"Error reading CSV: {e}")
+            else:
+                # Report might not be ready yet, wait and retry
+                retry_count += 1
+                if retry_count >= max_retries:
+                    raise Exception(
+                        f"Failed to get report after {max_retries} attempts. Status code: {get_response.status_code}"
                     )
-                    return df
-                except Exception as e:
-                    raise Exception(f"Error reading CSV: {e}")
-        else:
-            raise Exception(
-                f"Failed to get report. Status code: {get_response.status_code}"
-            )
+                time.sleep(5)  # Wait 5 seconds before retrying
 
     def generateSignature(self, json_string):
         data = self.public_key + json_string + self.public_key

ingestr/src/sources.py

@@ -80,18 +80,18 @@ class SqlSource:
         if uri.startswith("snowflake://"):
             parsed_uri = urlparse(uri)
             query_params = parse_qs(parsed_uri.query)
-            
+
             if "private_key" in query_params:
                 from dlt.common.libs.cryptography import decode_private_key
-                
+
                 private_key = query_params["private_key"][0]
                 passphrase = query_params.get("private_key_passphrase", [None])[0]
                 decoded_key = decode_private_key(private_key, passphrase)
-                
+
                 query_params["private_key"] = [base64.b64encode(decoded_key).decode()]
                 if "private_key_passphrase" in query_params:
                     del query_params["private_key_passphrase"]
-                
+
                 # Rebuild URI
                 uri = parsed_uri._replace(
                     query=urlencode(query_params, doseq=True)

{ingestr-0.13.68.dist-info → ingestr-0.13.70.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.68
+Version: 0.13.70
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.68.dist-info → ingestr-0.13.70.dist-info}/RECORD

@@ -1,8 +1,8 @@
-ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=8EduMJdPnj6Z8p5xMzCJ8KZPq-STaPhIzhVTzoSz_sw,26380
+ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
+ingestr/main.py,sha256=wu4c_4nit3c5v2K1af68DjT2hIfEiCvzQ8EZJxOkJjo,26369
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=AmQ_QOtiF5_x4B8nPBudbz-zX_sJrDJElSgNVFmAdKM,21
+ingestr/src/buildinfo.py,sha256=Aj6WfU5V5JSNLm519zySNpHSLoEZIUyuu2KBZ12S60w,21
 ingestr/src/destinations.py,sha256=ivTPio0zzqLRx22i597pxZHMNClz-XvYSyCaCPuGd8g,22248
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=AJCvlK4M1sIpAAks1K-xsR_uxziIxru74mj572zixhg,6546
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=IGFRqH1f9x2W7GUnAHoVYtrBYvvJbuDcHAhULCXn5i8,104352
+ingestr/src/sources.py,sha256=Ey3fI-SZ08fVMZf5_uCdLku87UuT3bjM8SmesgoSSY8,104292
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -74,7 +74,7 @@ ingestr/src/hubspot/__init__.py,sha256=wqHefhc_YRI5dNFCcpvH-UUilNThE49sbGouSBiHY
 ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/isoc_pulse/__init__.py,sha256=9b4eN4faatpiwTuRNPuYcEt1hEFDEjua9XhfakUigBk,4648
-ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
+ingestr/src/kafka/__init__.py,sha256=QUHsGmdv5_E-3z0GDHXvbk39puwuGDBsyYSDhvbA89E,3595
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=YretSz4F28tbkcPhd55mBp2Xk7XE9unyWx0nmvl8iEc,6235
 ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0s,3152
@@ -117,8 +117,8 @@ ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
-ingestr/src/solidgate/__init__.py,sha256=JdaXvAu5QGuf9-FY294vwCQCEmfrqIld9oqbzqCJS3g,3626
-ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY4Fc,4952
+ingestr/src/solidgate/__init__.py,sha256=as3m-lrs8teI3BW_8__5SY7BY8Af7SetzuU8Cj3L80U,8170
+ingestr/src/solidgate/helpers.py,sha256=j8FjYHyUJTwkJ8VY8nre2qMesLeRq3eTwU22Ff-2K8M,5426
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
 ingestr/src/stripe_analytics/__init__.py,sha256=mK8dGKAlMPVqGE9gG30XfbvOvgVD0yWhNpt-D3iavDY,6385
@@ -147,8 +147,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.68.dist-info/METADATA,sha256=l2t9KyoQ_YqRie-2IItPzZSzHCIzk26wW16ZZ5SA4d8,15022
-ingestr-0.13.68.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.68.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.68.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.68.dist-info/RECORD,,
+ingestr-0.13.70.dist-info/METADATA,sha256=kO40q7I3oWYT79mXepM4rslDJnvNOtkTSqtU3Pgt168,15022
+ingestr-0.13.70.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.70.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.70.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.70.dist-info/RECORD,,