ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/socrata_source/settings.py

@@ -0,0 +1,8 @@
+"""Socrata API settings and constants"""
+
+# Request timeout in seconds
+REQUEST_TIMEOUT = 30
+
+# Maximum number of records to fetch per page
+# Socrata API supports up to 50000 records per request
+DEFAULT_PAGE_SIZE = 50000

ingestr/src/solidgate/__init__.py

@@ -0,0 +1,219 @@
+from typing import Iterable, Iterator
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .helpers import SolidgateClient
+
+COLUMN_HINTS = {
+    "subscriptions": {
+        "id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "status": {"data_type": "text"},
+        "started_at": {"data_type": "timestamp"},
+        "updated_at": {"data_type": "timestamp"},
+        "expired_at": {"data_type": "timestamp"},
+        "next_charge_at": {"data_type": "timestamp"},
+        "payment_type": {"data_type": "text"},
+        "trial": {"data_type": "bool"},
+        "cancelled_at": {"data_type": "timestamp"},
+        "cancellation_requested_at": {"data_type": "timestamp"},
+        "cancel_code": {"data_type": "text"},
+        "cancel_message": {"data_type": "text"},
+        "customer": {"data_type": "json"},
+        "product": {"data_type": "json"},
+        "invoices": {"data_type": "json"},
+    },
+    "apm_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "method": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+    },
+    "card_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "psp_order_id": {"data_type": "text"},
+        "provider_payment_id": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "payment_type": {"data_type": "text"},
+        "type": {"data_type": "text"},
+        "is_secured": {"data_type": "bool"},
+        "routing": {"data_type": "json"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "customer_first_name": {"data_type": "text"},
+        "customer_last_name": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "mid": {"data_type": "text"},
+        "traffic_source": {"data_type": "text"},
+        "platform": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+        "fraudulent": {"data_type": "bool"},
+    },
+    "financial_entries": {
+        "id": {
+            "data_type": "text",
+            "nullable": False,
+            "primary_key": True,
+        },
+        "order_id": {"data_type": "text"},
+        "external_psp_order_id": {"data_type": "text"},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "transaction_datetime_provider": {"data_type": "timestamp"},
+        "transaction_datetime_utc": {"data_type": "timestamp"},
+        "accounting_date": {"data_type": "date"},
+        "amount": {"data_type": "double"},
+        "amount_in_major_units": {"data_type": "double"},
+        "currency": {"data_type": "text"},
+        "currency_minor_units": {"data_type": "bigint"},
+        "payout_amount": {"data_type": "double"},
+        "payout_amount_in_major_units": {"data_type": "double"},
+        "payout_currency": {"data_type": "text"},
+        "payout_currency_minor_units": {"data_type": "bigint"},
+        "record_type_key": {"data_type": "text"},
+        "provider": {"data_type": "text"},
+        "payment_method": {"data_type": "text"},
+        "card_brand": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "issuing_country": {"data_type": "text"},
+        "transaction_id": {"data_type": "text"},
+        "chargeback_id": {"data_type": "text"},
+        "legal_entity": {"data_type": "text"},
+    },
+}
+
+
+@dlt.source(max_table_nesting=0)
+def solidgate_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None,
+    public_key: str,
+    secret_key: str,
+) -> Iterable[DltResource]:
+    solidgate_client = SolidgateClient(public_key, secret_key)
+
+    @dlt.resource(
+        name="subscriptions",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["subscriptions"],  # type: ignore
+    )
+    def fetch_all_subscriptions(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "subscriptions"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="apm_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["apm_orders"],  # type: ignore
+    )
+    def fetch_apm_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "apm-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="card_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["card_orders"],  # type: ignore
+    )
+    def fetch_card_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "card-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="financial_entries",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["financial_entries"],  # type: ignore
+    )
+    def fetch_financial_entries(
+        dateTime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ):
+        if dateTime.end_value is None:
+            end_date = pendulum.now(tz="UTC")
+        else:
+            end_date = dateTime.end_value
+
+        start_date = dateTime.last_value
+        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
+
+    return (
+        fetch_all_subscriptions,
+        fetch_apm_orders,
+        fetch_card_orders,
+        fetch_financial_entries,
+    )

ingestr/src/solidgate/helpers.py

@@ -0,0 +1,154 @@
+import base64
+import csv
+import hashlib
+import hmac
+import json
+import time
+from io import StringIO
+
+import pendulum
+
+from ingestr.src.http_client import create_client
+
+
+class SolidgateClient:
+    def __init__(self, public_key, secret_key):
+        self.base_url = "https://reports.solidgate.com/api/v1"
+        self.public_key = public_key
+        self.secret_key = secret_key
+        self.client = create_client(retry_status_codes=[204])
+
+    def fetch_data(
+        self,
+        path: str,
+        date_from: pendulum.DateTime,
+        date_to: pendulum.DateTime,
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        next_page_iterator = None
+        url = f"{self.base_url}/{path}"
+
+        while True:
+            payload = request_payload.copy()
+            if next_page_iterator:
+                payload["page_iterator"] = next_page_iterator
+
+            response = self.client.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            response_json = response.json()
+
+            if path == "subscriptions":
+                data = response_json["subscriptions"]
+                for _, value in data.items():
+                    if "updated_at" in value:
+                        value["updated_at"] = pendulum.parse(value["updated_at"])
+                    yield value
+
+            else:
+                data = response_json["orders"]
+                for value in data:
+                    if "updated_at" in value:
+                        value["updated_at"] = pendulum.parse(value["updated_at"])
+                    yield value
+
+            next_page_iterator = response_json.get("metadata", {}).get(
+                "next_page_iterator"
+            )
+            if not next_page_iterator or next_page_iterator == "None":
+                break
+
+    def fetch_financial_entry_data(
+        self, date_from: pendulum.DateTime, date_to: pendulum.DateTime
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        url = f"{self.base_url}/finance/financial_entries"
+        post_response = self.client.post(url, headers=headers, json=request_payload)
+        post_response.raise_for_status()
+        report_url = post_response.json().get("report_url")
+        if not report_url:
+            return f"Report URL not found in the response: {post_response.json()}", 400
+
+        data = self.public_key + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        signature_get = base64.b64encode(hmac_hash.hex().encode("utf-8")).decode(
+            "utf-8"
+        )
+
+        headers_get = {
+            "merchant": self.public_key,
+            "Signature": signature_get,
+            "Content-Type": "application/json",
+        }
+
+        # Retry getting the report for up to 10 minutes (600 seconds) with 5-second intervals
+        max_retries = 120  # 10 minutes / 5 seconds = 120 attempts
+        retry_count = 0
+
+        while retry_count < max_retries:
+            get_response = self.client.get(report_url, headers=headers_get)
+
+            if get_response.status_code == 200:
+                try:
+                    response_json = json.loads(get_response.content)
+                    if "error" in response_json:
+                        raise Exception(
+                            f"API Error: {response_json['error']['messages']}"
+                        )
+                except json.JSONDecodeError:
+                    try:
+                        csv_data = get_response.content.decode("utf-8")
+                        reader = csv.DictReader(StringIO(csv_data))
+                        rows = []
+                        for row in reader:
+                            if row["created_at"]:
+                                row["created_at"] = pendulum.parse(row["created_at"])
+                            else:
+                                row["created_at"] = None
+
+                            row2 = {k: v for k, v in row.items() if v != ""}
+                            rows.append(row2)
+
+                        return rows
+                    except Exception as e:
+                        raise Exception(f"Error reading CSV: {e}")
+            else:
+                # Report might not be ready yet, wait and retry
+                retry_count += 1
+                if retry_count >= max_retries:
+                    raise Exception(
+                        f"Failed to get report after {max_retries} attempts. Status code: {get_response.status_code}"
+                    )
+                time.sleep(5)  # Wait 5 seconds before retrying
+
+    def generateSignature(self, json_string):
+        data = self.public_key + json_string + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        return base64.b64encode(hmac_hash.hex().encode("utf-8")).decode("utf-8")