ingestr 0.13.49__py3-none-any.whl → 0.13.51__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.49"
+version = "v0.13.51"

ingestr/src/factory.py

@@ -59,6 +59,7 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
+    SmartsheetSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -161,6 +162,7 @@ class SourceDestinationFactory:
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,
         "solidgate": SolidgateSource,
+        "smartsheet": SmartsheetSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/http_client.py

@@ -2,17 +2,23 @@ import requests
 from dlt.sources.helpers.requests import Client
 
 
-def create_client() -> requests.Session:
+def create_client(retry_status_codes: list[int] | None = None) -> requests.Session:
+    if retry_status_codes is None:
+        retry_status_codes = [502]
     return Client(
         raise_for_status=False,
-        retry_condition=retry_on_limit,
+        retry_condition=retry_on_status_code(retry_status_codes),
         request_max_attempts=12,
+        request_backoff_factor=10,
     ).session
 
 
-def retry_on_limit(
-    response: requests.Response | None, exception: BaseException | None
-) -> bool:
-    if response is None:
-        return False
-    return response.status_code == 502
+def retry_on_status_code(retry_status_codes: list[int]):
+    def retry_on_limit(
+        response: requests.Response | None, exception: BaseException | None
+    ) -> bool:
+        if response is None:
+            return False
+        return response.status_code in retry_status_codes
+
+    return retry_on_limit

ingestr/src/solidgate/__init__.py

@@ -94,4 +94,34 @@ def solidgate_source(
         start_dt = dateTime.last_value
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
-    return fetch_all_subscriptions, fetch_apm_orders, fetch_card_orders
+    @dlt.resource(
+        name="financial-entries",
+        write_disposition="merge",
+        primary_key="id",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_financial_entries(
+        dateTime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ):
+        if dateTime.end_value is None:
+            end_date = pendulum.now(tz="UTC")
+        else:
+            end_date = dateTime.end_value
+
+        start_date = dateTime.last_value
+        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
+
+    return (
+        fetch_all_subscriptions,
+        fetch_apm_orders,
+        fetch_card_orders,
+        fetch_financial_entries,
+    )

ingestr/src/solidgate/helpers.py

@@ -2,7 +2,10 @@ import base64
 import hashlib
 import hmac
 import json
+import time
+from io import StringIO
 
+import pandas as pd  # type: ignore
 import pendulum
 
 from ingestr.src.http_client import create_client
@@ -13,7 +16,7 @@ class SolidgateClient:
         self.base_url = "https://reports.solidgate.com/api/v1"
         self.public_key = public_key
         self.secret_key = secret_key
-        self.client = create_client()
+        self.client = create_client(retry_status_codes=[204])
 
     def fetch_data(
         self,
@@ -66,6 +69,68 @@ class SolidgateClient:
             if not next_page_iterator or next_page_iterator == "None":
                 break
 
+    def fetch_financial_entry_data(
+        self, date_from: pendulum.DateTime, date_to: pendulum.DateTime
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        url = f"{self.base_url}/finance/financial_entries"
+        post_response = self.client.post(url, headers=headers, json=request_payload)
+        post_response.raise_for_status()
+        report_url = post_response.json().get("report_url")
+        if not report_url:
+            return f"Report URL not found in the response: {post_response.json()}", 400
+
+        # Wait for 5 seconds before attempting to download the report as report may not be immediately available
+        time.sleep(5)
+
+        data = self.public_key + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        signature_get = base64.b64encode(hmac_hash.hex().encode("utf-8")).decode(
+            "utf-8"
+        )
+
+        headers_get = {
+            "merchant": self.public_key,
+            "Signature": signature_get,
+            "Content-Type": "application/json",
+        }
+
+        get_response = self.client.get(report_url, headers=headers_get)
+
+        if get_response.status_code == 200:
+            try:
+                response_json = json.loads(get_response.content)
+                if "error" in response_json:
+                    raise Exception(f"API Error: {response_json['error']['messages']}")
+            except json.JSONDecodeError:
+                try:
+                    csv_data = get_response.content.decode("utf-8")
+                    df = pd.read_csv(StringIO(csv_data))
+                    df["created_at"] = df["created_at"].apply(
+                        lambda x: pendulum.parse(x)
+                    )
+                    return df
+                except Exception as e:
+                    raise Exception(f"Error reading CSV: {e}")
+        else:
+            raise Exception(
+                f"Failed to get report. Status code: {get_response.status_code}"
+            )
+
     def generateSignature(self, json_string):
         data = self.public_key + json_string + self.public_key
         hmac_hash = hmac.new(

ingestr/src/stripe_analytics/settings.py

@@ -3,24 +3,71 @@
 # the most popular endpoints
 # Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
 ENDPOINTS = {
-    "subscription": "Subscription",
     "account": "Account",
+    "applepaydomain": "ApplePayDomain",
+    "apple_pay_domain": "ApplePayDomain",
+    "applicationfee": "ApplicationFee",
+    "application_fee": "ApplicationFee",
+    "checkoutsession": "CheckoutSession",
+    "checkout_session": "CheckoutSession",
     "coupon": "Coupon",
     "customer": "Customer",
-    "product": "Product",
+    "dispute": "Dispute",
+    "paymentintent": "PaymentIntent",
+    "payment_intent": "PaymentIntent",
+    "paymentlink": "PaymentLink",
+    "payment_link": "PaymentLink",
+    "paymentmethod": "PaymentMethod",
+    "payment_method": "PaymentMethod",
+    "paymentmethoddomain": "PaymentMethodDomain",
+    "payment_method_domain": "PaymentMethodDomain",
+    "payout": "Payout",
+    "plan": "Plan",
     "price": "Price",
+    "product": "Product",
+    "promotioncode": "PromotionCode",
+    "promotion_code": "PromotionCode",
+    "quote": "Quote",
+    "refund": "Refund",
+    "review": "Review",
+    "setupattempt": "SetupAttempt",
+    "setup_attempt": "SetupAttempt",
+    "setupintent": "SetupIntent",
+    "setup_intent": "SetupIntent",
     "shippingrate": "ShippingRate",
-    "dispute": "Dispute",
+    "shipping_rate": "ShippingRate",
+    "subscription": "Subscription",
     "subscriptionitem": "SubscriptionItem",
-    "checkoutsession": "CheckoutSession",
+    "subscription_item": "SubscriptionItem",
+    "subscriptionschedule": "SubscriptionSchedule",
+    "subscription_schedule": "SubscriptionSchedule",
+    "transfer": "Transfer",
+    "taxcode": "TaxCode",
+    "tax_code": "TaxCode",
+    "taxid": "TaxId",
+    "tax_id": "TaxId",
+    "taxrate": "TaxRate",
+    "tax_rate": "TaxRate",
+    "topup": "Topup",
+    "top_up": "Topup",
+    "webhookendpoint": "WebhookEndpoint",
+    "webhook_endpoint": "WebhookEndpoint",
 }
 # possible incremental endpoints
 INCREMENTAL_ENDPOINTS = {
-    "event": "Event",
-    "invoice": "Invoice",
+    "applicationfee": "ApplicationFee",
+    "application_fee": "ApplicationFee",
     "balancetransaction": "BalanceTransaction",
+    "balance_transaction": "BalanceTransaction",
     "charge": "Charge",
-    "applicationfee": "ApplicationFee",
-    "setupattempt": "SetupAttempt",
     "creditnote": "CreditNote",
+    "credit_note": "CreditNote",
+    "event": "Event",
+    "invoice": "Invoice",
+    "invoiceitem": "InvoiceItem",
+    "invoice_item": "InvoiceItem",
+    "invoicelineitem": "InvoiceLineItem",
+    "invoice_line_item": "InvoiceLineItem",
+    "setupattempt": "SetupAttempt",
+    "setup_attempt": "SetupAttempt",
 }

{ingestr-0.13.49.dist-info → ingestr-0.13.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.49
+Version: 0.13.51
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.49.dist-info → ingestr-0.13.51.dist-info}/RECORD

@@ -2,12 +2,12 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=rHxHQAbd0ccW2e2kQSWlv7-5qcc2ZB6Eh3vyjm4Nzns,25550
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=Q38dT9-xAEHMT3iRCHcD0SOhlVbTR8sRnjHj5TYEUfo,21
+ingestr/src/buildinfo.py,sha256=ZgugQZPIlr-_7kRH5qIupQ5e48JTOHSlRWEsmNx4wXw,21
 ingestr/src/destinations.py,sha256=41Bj1UgxR8a2KcZWqtGw74AKZKnSBrueQRnBdrf3c-A,16003
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=FXWJLFfBsKMzUwtsyaaruZU-_OLFKivobj6Olse9vSI,5741
+ingestr/src/factory.py,sha256=JeK0M8C05-_KxxIhtevs-MJGLlngXUQupd-Pm4Fg904,5803
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
-ingestr/src/http_client.py,sha256=UwPiv95EfHPdT4525xeLFJ1AYlf-cyaHKRU-2QnZt2o,435
+ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
@@ -109,13 +109,13 @@ ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
-ingestr/src/solidgate/__init__.py,sha256=vpoXu0Ox3zE_WPSzdsA6iUG1_XBa9OaA5F7eFBbZYuQ,2819
-ingestr/src/solidgate/helpers.py,sha256=_PuHmKZ-jpNEsPxRgXCzu39PsaygVDCpnoMTZAYSpHE,2432
+ingestr/src/solidgate/__init__.py,sha256=DZYQ4M3Cc7AIbdQcNQm_6yX2whnFhE-iM10-ACJ3W3A,3626
+ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY4Fc,4952
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
 ingestr/src/stripe_analytics/__init__.py,sha256=FBkZu5op5Z-FceEi4zG7qcAgZfUYJRPMVPPrPMjvmXw,4502
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
-ingestr/src/stripe_analytics/settings.py,sha256=fA2j_6FquEmyRyB799P4SncwLwK1S1u9WFNjbzu91kY,786
+ingestr/src/stripe_analytics/settings.py,sha256=ZahhZg3Sq2KnvnDcfSaXO494Csy3tElBDEHnvA1AVmA,2461
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
@@ -135,8 +135,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.49.dist-info/METADATA,sha256=pkrtX6z_Vkjc4GnSTW_6SmskX8MQm_ASZOX6hdFlJgM,13983
-ingestr-0.13.49.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.49.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.49.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.49.dist-info/RECORD,,
+ingestr-0.13.51.dist-info/METADATA,sha256=danPRyBhqlvyOwtw8kBmBh5FEY_xH6upsy5GrsFaMg4,13983
+ingestr-0.13.51.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.51.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.51.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.51.dist-info/RECORD,,