ingestr 0.13.50__py3-none-any.whl → 0.13.52__py3-none-any.whl

ingestr/main.py

@@ -58,6 +58,7 @@ class LoaderFileFormat(str, Enum):
 
 
 class SqlBackend(str, Enum):
+    default = "default"
     sqlalchemy = "sqlalchemy"
     pyarrow = "pyarrow"
     connectorx = "connectorx"
@@ -187,7 +188,7 @@ def ingest(
             help="The SQL backend to use",
             envvar=["SQL_BACKEND", "INGESTR_SQL_BACKEND"],
         ),
-    ] = SqlBackend.pyarrow,  # type: ignore
+    ] = SqlBackend.default,  # type: ignore
     loader_file_format: Annotated[
         Optional[LoaderFileFormat],
         typer.Option(
@@ -289,7 +290,11 @@ def ingest(
     from ingestr.src.collector.spinner import SpinnerCollector
     from ingestr.src.destinations import AthenaDestination
     from ingestr.src.factory import SourceDestinationFactory
-    from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
+    from ingestr.src.filters import (
+        cast_set_to_list,
+        cast_spanner_types,
+        handle_mysql_empty_dates,
+    )
     from ingestr.src.sources import MongoDbSource
 
     def report_errors(run_info: LoadInfo):
@@ -517,6 +522,15 @@ def ingest(
             if interval_end:
                 interval_end = interval_end.date()  # type: ignore
 
+        if factory.source_scheme.startswith("spanner"):
+            # we tend to use the 'pyarrow' backend in general, however, it has issues with JSON objects, so we override it to 'sqlalchemy' for Spanner.
+            if sql_backend.value == SqlBackend.default:
+                sql_backend = SqlBackend.sqlalchemy
+
+        # this allows us to identify the cases where the user does not have a preference, so that for some sources we can override it.
+        if sql_backend == SqlBackend.default:
+            sql_backend = SqlBackend.pyarrow
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -535,6 +549,9 @@ def ingest(
         if factory.source_scheme.startswith("mysql"):
             resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
+        if factory.source_scheme.startswith("spanner"):
+            resource.for_each(dlt_source, lambda x: x.add_map(cast_spanner_types))
+
         if yield_limit:
             resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
 

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.50"
+version = "v0.13.52"

ingestr/src/destinations.py

@@ -468,3 +468,15 @@ class S3Destination:
 
     def post_load(self) -> None:
         pass
+
+
+class SqliteDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        return {
+            #https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            "dataset_name": "main",
+            "table_name": table,
+        }

ingestr/src/factory.py

@@ -15,6 +15,7 @@ from ingestr.src.destinations import (
     RedshiftDestination,
     S3Destination,
     SnowflakeDestination,
+    SqliteDestination,
     SynapseDestination,
 )
 from ingestr.src.sources import (
@@ -54,6 +55,7 @@ from ingestr.src.sources import (
     SalesforceSource,
     ShopifySource,
     SlackSource,
+    SmartsheetSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
@@ -161,6 +163,7 @@ class SourceDestinationFactory:
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,
         "solidgate": SolidgateSource,
+        "smartsheet": SmartsheetSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
@@ -180,6 +183,7 @@ class SourceDestinationFactory:
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
         "s3": S3Destination,
+        "sqlite": SqliteDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

ingestr/src/filters.py

@@ -7,6 +7,20 @@ def cast_set_to_list(row):
     return row
 
 
+def cast_spanner_types(row):
+    if not isinstance(row, dict):
+        return row
+
+    from google.cloud.spanner_v1.data_types import JsonObject
+
+    for key in row.keys():
+        if isinstance(row[key], JsonObject):
+            import json
+
+            row[key] = json.loads(row[key].serialize())
+    return row
+
+
 def handle_mysql_empty_dates(row):
     # MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
     if not isinstance(row, dict):

ingestr/src/http_client.py

@@ -2,17 +2,23 @@ import requests
 from dlt.sources.helpers.requests import Client
 
 
-def create_client() -> requests.Session:
+def create_client(retry_status_codes: list[int] | None = None) -> requests.Session:
+    if retry_status_codes is None:
+        retry_status_codes = [502]
     return Client(
         raise_for_status=False,
-        retry_condition=retry_on_limit,
+        retry_condition=retry_on_status_code(retry_status_codes),
         request_max_attempts=12,
+        request_backoff_factor=10,
     ).session
 
 
-def retry_on_limit(
-    response: requests.Response | None, exception: BaseException | None
-) -> bool:
-    if response is None:
-        return False
-    return response.status_code == 502
+def retry_on_status_code(retry_status_codes: list[int]):
+    def retry_on_limit(
+        response: requests.Response | None, exception: BaseException | None
+    ) -> bool:
+        if response is None:
+            return False
+        return response.status_code in retry_status_codes
+
+    return retry_on_limit

ingestr/src/solidgate/__init__.py

@@ -43,7 +43,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="apm-orders",
+        name="apm_orders",
         write_disposition="merge",
         primary_key="order_id",
         columns={
@@ -69,7 +69,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="card-orders",
+        name="card_orders",
         write_disposition="merge",
         primary_key="order_id",
         columns={
@@ -94,4 +94,34 @@ def solidgate_source(
         start_dt = dateTime.last_value
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
-    return fetch_all_subscriptions, fetch_apm_orders, fetch_card_orders
+    @dlt.resource(
+        name="financial_entries",
+        write_disposition="merge",
+        primary_key="id",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_financial_entries(
+        dateTime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ):
+        if dateTime.end_value is None:
+            end_date = pendulum.now(tz="UTC")
+        else:
+            end_date = dateTime.end_value
+
+        start_date = dateTime.last_value
+        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
+
+    return (
+        fetch_all_subscriptions,
+        fetch_apm_orders,
+        fetch_card_orders,
+        fetch_financial_entries,
+    )

ingestr/src/solidgate/helpers.py

@@ -2,7 +2,10 @@ import base64
 import hashlib
 import hmac
 import json
+import time
+from io import StringIO
 
+import pandas as pd  # type: ignore
 import pendulum
 
 from ingestr.src.http_client import create_client
@@ -13,7 +16,7 @@ class SolidgateClient:
         self.base_url = "https://reports.solidgate.com/api/v1"
         self.public_key = public_key
         self.secret_key = secret_key
-        self.client = create_client()
+        self.client = create_client(retry_status_codes=[204])
 
     def fetch_data(
         self,
@@ -66,6 +69,68 @@ class SolidgateClient:
             if not next_page_iterator or next_page_iterator == "None":
                 break
 
+    def fetch_financial_entry_data(
+        self, date_from: pendulum.DateTime, date_to: pendulum.DateTime
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        url = f"{self.base_url}/finance/financial_entries"
+        post_response = self.client.post(url, headers=headers, json=request_payload)
+        post_response.raise_for_status()
+        report_url = post_response.json().get("report_url")
+        if not report_url:
+            return f"Report URL not found in the response: {post_response.json()}", 400
+
+        # Wait for 5 seconds before attempting to download the report as report may not be immediately available
+        time.sleep(5)
+
+        data = self.public_key + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        signature_get = base64.b64encode(hmac_hash.hex().encode("utf-8")).decode(
+            "utf-8"
+        )
+
+        headers_get = {
+            "merchant": self.public_key,
+            "Signature": signature_get,
+            "Content-Type": "application/json",
+        }
+
+        get_response = self.client.get(report_url, headers=headers_get)
+
+        if get_response.status_code == 200:
+            try:
+                response_json = json.loads(get_response.content)
+                if "error" in response_json:
+                    raise Exception(f"API Error: {response_json['error']['messages']}")
+            except json.JSONDecodeError:
+                try:
+                    csv_data = get_response.content.decode("utf-8")
+                    df = pd.read_csv(StringIO(csv_data))
+                    df["created_at"] = df["created_at"].apply(
+                        lambda x: pendulum.parse(x)
+                    )
+                    return df
+                except Exception as e:
+                    raise Exception(f"Error reading CSV: {e}")
+        else:
+            raise Exception(
+                f"Failed to get report. Status code: {get_response.status_code}"
+            )
+
     def generateSignature(self, json_string):
         data = self.public_key + json_string + self.public_key
         hmac_hash = hmac.new(

ingestr/src/stripe_analytics/settings.py

@@ -3,24 +3,71 @@
 # the most popular endpoints
 # Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
 ENDPOINTS = {
-    "subscription": "Subscription",
     "account": "Account",
+    "applepaydomain": "ApplePayDomain",
+    "apple_pay_domain": "ApplePayDomain",
+    "applicationfee": "ApplicationFee",
+    "application_fee": "ApplicationFee",
+    "checkoutsession": "CheckoutSession",
+    "checkout_session": "CheckoutSession",
     "coupon": "Coupon",
     "customer": "Customer",
-    "product": "Product",
+    "dispute": "Dispute",
+    "paymentintent": "PaymentIntent",
+    "payment_intent": "PaymentIntent",
+    "paymentlink": "PaymentLink",
+    "payment_link": "PaymentLink",
+    "paymentmethod": "PaymentMethod",
+    "payment_method": "PaymentMethod",
+    "paymentmethoddomain": "PaymentMethodDomain",
+    "payment_method_domain": "PaymentMethodDomain",
+    "payout": "Payout",
+    "plan": "Plan",
     "price": "Price",
+    "product": "Product",
+    "promotioncode": "PromotionCode",
+    "promotion_code": "PromotionCode",
+    "quote": "Quote",
+    "refund": "Refund",
+    "review": "Review",
+    "setupattempt": "SetupAttempt",
+    "setup_attempt": "SetupAttempt",
+    "setupintent": "SetupIntent",
+    "setup_intent": "SetupIntent",
     "shippingrate": "ShippingRate",
-    "dispute": "Dispute",
+    "shipping_rate": "ShippingRate",
+    "subscription": "Subscription",
     "subscriptionitem": "SubscriptionItem",
-    "checkoutsession": "CheckoutSession",
+    "subscription_item": "SubscriptionItem",
+    "subscriptionschedule": "SubscriptionSchedule",
+    "subscription_schedule": "SubscriptionSchedule",
+    "transfer": "Transfer",
+    "taxcode": "TaxCode",
+    "tax_code": "TaxCode",
+    "taxid": "TaxId",
+    "tax_id": "TaxId",
+    "taxrate": "TaxRate",
+    "tax_rate": "TaxRate",
+    "topup": "Topup",
+    "top_up": "Topup",
+    "webhookendpoint": "WebhookEndpoint",
+    "webhook_endpoint": "WebhookEndpoint",
 }
 # possible incremental endpoints
 INCREMENTAL_ENDPOINTS = {
-    "event": "Event",
-    "invoice": "Invoice",
+    "applicationfee": "ApplicationFee",
+    "application_fee": "ApplicationFee",
     "balancetransaction": "BalanceTransaction",
+    "balance_transaction": "BalanceTransaction",
     "charge": "Charge",
-    "applicationfee": "ApplicationFee",
-    "setupattempt": "SetupAttempt",
     "creditnote": "CreditNote",
+    "credit_note": "CreditNote",
+    "event": "Event",
+    "invoice": "Invoice",
+    "invoiceitem": "InvoiceItem",
+    "invoice_item": "InvoiceItem",
+    "invoicelineitem": "InvoiceLineItem",
+    "invoice_line_item": "InvoiceLineItem",
+    "setupattempt": "SetupAttempt",
+    "setup_attempt": "SetupAttempt",
 }

{ingestr-0.13.50.dist-info → ingestr-0.13.52.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.50
+Version: 0.13.52
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.50.dist-info → ingestr-0.13.52.dist-info}/RECORD

@@ -1,13 +1,13 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=rHxHQAbd0ccW2e2kQSWlv7-5qcc2ZB6Eh3vyjm4Nzns,25550
+ingestr/main.py,sha256=GkC1hdq8AVGrvolc95zMfjmibI95p2pmFkbgCOVf-Og,26311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=SIygmTZiIS9bHiKi-Fr3gXHmpPssVhCeEF_SCAtF8Ww,21
-ingestr/src/destinations.py,sha256=41Bj1UgxR8a2KcZWqtGw74AKZKnSBrueQRnBdrf3c-A,16003
+ingestr/src/buildinfo.py,sha256=qcKqSPI861BBapk7rbcYkc9M_18QQnT44YM1Zr_Sg80,21
+ingestr/src/destinations.py,sha256=axSm0GLM_bKQmWYb74fnHebNktWgtfAyyqdHV-zBOsA,16405
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=FXWJLFfBsKMzUwtsyaaruZU-_OLFKivobj6Olse9vSI,5741
-ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
-ingestr/src/http_client.py,sha256=UwPiv95EfHPdT4525xeLFJ1AYlf-cyaHKRU-2QnZt2o,435
+ingestr/src/factory.py,sha256=5yrg-XkaixuCkiTz3B7mraE8LaANXtzItenbx8TdPrE,5863
+ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
+ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
@@ -109,13 +109,13 @@ ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
-ingestr/src/solidgate/__init__.py,sha256=vpoXu0Ox3zE_WPSzdsA6iUG1_XBa9OaA5F7eFBbZYuQ,2819
-ingestr/src/solidgate/helpers.py,sha256=_PuHmKZ-jpNEsPxRgXCzu39PsaygVDCpnoMTZAYSpHE,2432
+ingestr/src/solidgate/__init__.py,sha256=JdaXvAu5QGuf9-FY294vwCQCEmfrqIld9oqbzqCJS3g,3626
+ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY4Fc,4952
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
 ingestr/src/stripe_analytics/__init__.py,sha256=FBkZu5op5Z-FceEi4zG7qcAgZfUYJRPMVPPrPMjvmXw,4502
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
-ingestr/src/stripe_analytics/settings.py,sha256=fA2j_6FquEmyRyB799P4SncwLwK1S1u9WFNjbzu91kY,786
+ingestr/src/stripe_analytics/settings.py,sha256=ZahhZg3Sq2KnvnDcfSaXO494Csy3tElBDEHnvA1AVmA,2461
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
@@ -135,8 +135,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.50.dist-info/METADATA,sha256=cYNN5jAD-1Fistktk9x6ggyzbHIbmKEoOkwUgkLLosI,13983
-ingestr-0.13.50.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.50.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.50.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.50.dist-info/RECORD,,
+ingestr-0.13.52.dist-info/METADATA,sha256=jfSWJ6mE6vPz-5XsIIjqdLxd45-OrOUKbhbwJqiZlSc,13983
+ingestr-0.13.52.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.52.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.52.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.52.dist-info/RECORD,,