ingestr 0.13.47__py3-none-any.whl → 0.13.48__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.47"
+version = "v0.13.48"

ingestr/src/factory.py

@@ -54,6 +54,7 @@ from ingestr.src.sources import (
     SalesforceSource,
     ShopifySource,
     SlackSource,
+    SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
@@ -159,6 +160,7 @@ class SourceDestinationFactory:
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,
+        "solidgate": SolidgateSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/solidgate/__init__.py

@@ -0,0 +1,97 @@
+from typing import Iterable, Iterator
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .helpers import SolidgateClient
+
+
+@dlt.source(max_table_nesting=0)
+def solidgate_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None,
+    public_key: str,
+    secret_key: str,
+) -> Iterable[DltResource]:
+    solidgate_client = SolidgateClient(public_key, secret_key)
+
+    @dlt.resource(
+        name="subscriptions",
+        write_disposition="merge",
+        primary_key="id",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_all_subscriptions(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "subscriptions"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="apm-orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_apm_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "apm-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="card-orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_card_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "card-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    return fetch_all_subscriptions, fetch_apm_orders, fetch_card_orders

ingestr/src/solidgate/helpers.py

@@ -0,0 +1,78 @@
+import base64
+import hashlib
+import hmac
+import json
+
+import pendulum
+
+from ingestr.src.http_client import create_client
+
+
+class SolidgateClient:
+    def __init__(self, public_key, secret_key):
+        self.base_url = "https://reports.solidgate.com/api/v1"
+        self.public_key = public_key
+        self.secret_key = secret_key
+        self.client = create_client()
+
+    def fetch_data(
+        self,
+        path: str,
+        date_from: pendulum.DateTime,
+        date_to: pendulum.DateTime,
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        next_page_iterator = None
+        url = f"{self.base_url}/{path}"
+
+        while True:
+            payload = request_payload.copy()
+            if next_page_iterator:
+                payload["page_iterator"] = next_page_iterator
+
+            response = self.client.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            response_json = response.json()
+
+            if path == "subscriptions":
+                data = response_json["subscriptions"]
+                for _, value in data.items():
+                    if "updated_at" in value:
+                        value["updated_at"] = pendulum.parse(
+                            value["updated_at"]
+                        )
+                    yield value
+
+            else:
+                data = response_json["orders"]
+                for value in data:
+                    if "updated_at" in value:
+                        value["updated_at"] = pendulum.parse(
+                        value["updated_at"]
+                    )
+                    yield value
+
+            next_page_iterator = response_json.get("metadata", {}).get(
+                "next_page_iterator"
+            )
+            if not next_page_iterator or next_page_iterator == "None":
+                break
+
+    def generateSignature(self, json_string):
+        data = self.public_key + json_string + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        return base64.b64encode(hmac_hash.hex().encode("utf-8")).decode("utf-8")

ingestr/src/sources.py

@@ -702,7 +702,7 @@ class StripeAnalyticsSource:
             table = "BalanceTransaction"
         else:
             table = table.capitalize()
-        
+
         if table in [
             "Subscription",
             "Account",
@@ -2431,3 +2431,45 @@ class AttioSource:
             )
         except ResourcesNotFoundError:
             raise UnsupportedResourceError(table_name, "Attio")
+
+
+class SolidgateSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        public_key = query_params.get("public_key")
+        secret_key = query_params.get("secret_key")
+
+        if public_key is None:
+            raise MissingValueError("public_key", "Solidgate")
+
+        if secret_key is None:
+            raise MissingValueError("secret_key", "Solidgate")
+
+        table_name = table.replace(" ", "")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is None:
+            start_date = pendulum.yesterday().in_tz("UTC")
+        else:
+            start_date = ensure_pendulum_datetime(start_date).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.solidgate import solidgate_source
+
+        try:
+            return solidgate_source(
+                public_key=public_key[0],
+                secret_key=secret_key[0],
+                start_date=start_date,
+                end_date=end_date,
+            ).with_resources(table_name)
+        except ResourcesNotFoundError:
+            raise UnsupportedResourceError(table_name, "Solidgate")

{ingestr-0.13.47.dist-info → ingestr-0.13.48.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.47
+Version: 0.13.48
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.47.dist-info → ingestr-0.13.48.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=Pe_rzwcDRKIYa7baEVUAAPOHyqQbX29RUexMl0F_S1k,25273
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=SV5EmllSWi3j2_r09IqnWLNtpDD-P5xWoRY9SX2JAE4,21
+ingestr/src/buildinfo.py,sha256=yhRWZauFAWKFe7lRKiLr-5OUeP2iQLHekroxm2J_e1c,21
 ingestr/src/destinations.py,sha256=41Bj1UgxR8a2KcZWqtGw74AKZKnSBrueQRnBdrf3c-A,16003
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=c5WfqmRrXFj1PddnBOzTzzZUHJ-Fb42cvCvsBEqn6Yo,5682
+ingestr/src/factory.py,sha256=FXWJLFfBsKMzUwtsyaaruZU-_OLFKivobj6Olse9vSI,5741
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/http_client.py,sha256=UwPiv95EfHPdT4525xeLFJ1AYlf-cyaHKRU-2QnZt2o,435
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=XtMZsuA2LO4WBg1pQeJQdSI8Z76oMt7Ki-a5ubuJQTw,83840
+ingestr/src/sources.py,sha256=kT5XtFQyPPE0dEBXuPaPfEVTqCi0l-6Fa2DWWFCUMeg,85205
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -108,6 +108,8 @@ ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSs
 ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM,10113
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
+ingestr/src/solidgate/__init__.py,sha256=vpoXu0Ox3zE_WPSzdsA6iUG1_XBa9OaA5F7eFBbZYuQ,2819
+ingestr/src/solidgate/helpers.py,sha256=0TP71RQDo6ub-q0goivQOPiUxWIQU7SgQLUSXVctoVI,2532
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
 ingestr/src/stripe_analytics/__init__.py,sha256=0HCL0qsrh_si1RR3a4k9XS94VWQ4v9aG7CqXF-V-57M,4593
@@ -131,8 +133,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.47.dist-info/METADATA,sha256=uYpb1Z1exz85lRRKwkVunHVbSlWN11MGci7sPj4W4Mg,13852
-ingestr-0.13.47.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.47.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.47.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.47.dist-info/RECORD,,
+ingestr-0.13.48.dist-info/METADATA,sha256=wSTbG4xvQuRvVBXVzF0Q_0w0J9uVCKfJmyVmkQDbg50,13852
+ingestr-0.13.48.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.48.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.48.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.48.dist-info/RECORD,,