ingestr 0.13.67__tar.gz → 0.13.69__tar.gz

ingestr-0.13.69/.dlt/config.toml

@@ -0,0 +1,3 @@
+[runtime]
+
+dlthub_telemetry=false

{ingestr-0.13.67 → ingestr-0.13.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.67
+Version: 0.13.69
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -48,7 +48,7 @@ Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
 Requires-Dist: dlt-cratedb==0.0.1
-Requires-Dist: dlt==1.10.0
+Requires-Dist: dlt==1.11.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/conftest.py

@@ -2,6 +2,7 @@ import os
 import tempfile
 from concurrent.futures import ThreadPoolExecutor
 
+import pytest
 from main_test import DESTINATIONS, SOURCES  # type: ignore
 
 
@@ -15,6 +16,14 @@ def pytest_configure_node(node):
     node.workerinput["shared_directory"] = node.config.shared_directory
 
 
+@pytest.fixture(scope="session")
+def shared_directory(request):
+    if is_master(request.config):
+        return request.config.shared_directory
+    else:
+        return request.config.workerinput["shared_directory"]
+
+
 def is_master(config):
     """True if the code running the given pytest.config object is running in a xdist master
     node or not running xdist at all.

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/main.py

@@ -256,7 +256,7 @@ def ingest(
         Optional[list[str]],
         typer.Option(
             help="The column types to be used for the destination table in the format of 'column_name:column_type'",
-            envvar=["COLUMNS", "INGESTR_COLUMNS"],
+            envvar=["INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
     yield_limit: Annotated[
@@ -455,7 +455,7 @@ def ingest(
 
                 column_hints[key]["primary_key"] = True
 
-        pipeline = dlt.pipeline(
+        pipeline = dlt.pipeline(  # type: ignore
             pipeline_name=m.hexdigest(),
             destination=dlt_dest,
             progress=progressInstance,

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/applovin_max/__init__.py

@@ -113,4 +113,5 @@ def get_data(
     df = pd.read_csv(response_url)
     df["Date"] = pd.to_datetime(df["Date"])
     df["partition_date"] = df["Date"].dt.date
+    df["platform"] = platform
     return df

ingestr-0.13.69/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.69"

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/destinations.py

@@ -90,7 +90,7 @@ class BigQueryDestination:
 
         return dlt.destinations.bigquery(
             credentials=credentials,  # type: ignore
-            location=location,
+            location=location,  # type: ignore
             project_id=project_id,
             **kwargs,
         )
@@ -398,7 +398,7 @@ class AthenaDestination:
 
         return dlt.destinations.athena(
             query_result_bucket=query_result_path,
-            athena_work_group=source_params.get("workgroup", [None])[0],
+            athena_work_group=source_params.get("workgroup", [None])[0],  # type: ignore
             credentials=AwsCredentials(
                 aws_access_key_id=access_key_id,  # type: ignore
                 aws_secret_access_key=secret_access_key,  # type: ignore

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/linear/__init__.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, Iterable, Iterator
 import dlt
 import pendulum
 
-from .helpers import  _paginate, _normalize_issue, _normalize_team
+from .helpers import _normalize_issue, _normalize_team, _paginate
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -143,13 +143,15 @@ def linear_source(
                     yield item
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams( updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+    def teams(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "updatedAt",
             initial_value=start_date.isoformat(),
             end_value=end_date.isoformat() if end_date else None,
             range_start="closed",
             range_end="closed",
-        ),) -> Iterator[Dict[str, Any]]:
+        ),
+    ) -> Iterator[Dict[str, Any]]:
         print(start_date)
         if updated_at.last_value:
             current_start_date = pendulum.parse(updated_at.last_value)

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/linear/helpers.py

@@ -1,10 +1,10 @@
-import json
 from typing import Any, Dict, Iterator, Optional
 
 import requests
 
 LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
 
+
 def _graphql(
     api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
 ) -> Dict[str, Any]:
@@ -20,6 +20,7 @@ def _graphql(
         raise ValueError(str(payload["errors"]))
     return payload["data"]
 
+
 def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
     cursor: Optional[str] = None
     while True:
@@ -30,6 +31,7 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
             break
         cursor = data["pageInfo"]["endCursor"]
 
+
 def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
     field_mapping = {
         "assignee": "assignee_id",
@@ -45,16 +47,26 @@ def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
         else:
             item[value] = None
             del item[key]
-    json_fields = ["comments", "subscribers", "attachments", "labels", "subtasks","projects", "memberships", "members"]
+    json_fields = [
+        "comments",
+        "subscribers",
+        "attachments",
+        "labels",
+        "subtasks",
+        "projects",
+        "memberships",
+        "members",
+    ]
     for field in json_fields:
         if item.get(field):
             item[f"{field}"] = item[field].get("nodes", [])
-                
+
     return item
 
+
 def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
     json_fields = ["memberships", "members", "projects"]
     for field in json_fields:
         if item.get(field):
             item[f"{field}"] = item[field].get("nodes", [])
-    return item
+    return item

ingestr-0.13.69/ingestr/src/solidgate/__init__.py

@@ -0,0 +1,223 @@
+from typing import Iterable, Iterator
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .helpers import SolidgateClient
+
+COLUMN_HINTS = {
+    "subscriptions": {
+        "id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "status": {"data_type": "text"},
+        "started_at": {"data_type": "timestamp"},
+        "updated_at": {"data_type": "timestamp"},
+        "expired_at": {"data_type": "timestamp"},
+        "next_charge_at": {"data_type": "timestamp"},
+        "payment_type": {"data_type": "text"},
+        "trial": {"data_type": "bool"},
+        "cancelled_at": {"data_type": "timestamp"},
+        "cancellation_requested_at": {"data_type": "timestamp"},
+        "cancel_code": {"data_type": "text"},
+        "cancel_message": {"data_type": "text"},
+        "customer": {"data_type": "json"},
+        "product": {"data_type": "json"},
+        "invoices": {"data_type": "json"},
+    },
+    "apm_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "method": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+    },
+    "card_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "psp_order_id": {"data_type": "text"},
+        "provider_payment_id": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "payment_type": {"data_type": "text"},
+        "type": {"data_type": "text"},
+        "is_secured": {"data_type": "bool"},
+        "routing": {"data_type": "json"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "customer_first_name": {"data_type": "text"},
+        "customer_last_name": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "mid": {"data_type": "text"},
+        "traffic_source": {"data_type": "text"},
+        "platform": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+        "fraudulent": {"data_type": "bool"},
+    },
+    "financial_entries": {
+        "id": {
+            "data_type": "text",
+            "nullable": False,
+            "primary_key": True,
+        },
+        "order_id": {"data_type": "text"},
+        "external_psp_order_id": {"data_type": "text"},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "transaction_datetime_provider": {"data_type": "timestamp"},
+        "transaction_datetime_utc": {"data_type": "timestamp"},
+        "accounting_date": {"data_type": "date"},
+        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "currency": {"data_type": "text"},
+        "currency_minor_units": {"data_type": "bigint"},
+        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "payout_amount_in_major_units": {
+            "data_type": "decimal",
+            "precision": 18,
+            "scale": 4,
+        },
+        "payout_currency": {"data_type": "text"},
+        "payout_currency_minor_units": {"data_type": "bigint"},
+        "record_type_key": {"data_type": "text"},
+        "provider": {"data_type": "text"},
+        "payment_method": {"data_type": "text"},
+        "card_brand": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "issuing_country": {"data_type": "text"},
+        "transaction_id": {"data_type": "text"},
+        "chargeback_id": {"data_type": "text"},
+        "legal_entity": {"data_type": "text"},
+    },
+}
+
+
+@dlt.source(max_table_nesting=0)
+def solidgate_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None,
+    public_key: str,
+    secret_key: str,
+) -> Iterable[DltResource]:
+    solidgate_client = SolidgateClient(public_key, secret_key)
+
+    @dlt.resource(
+        name="subscriptions",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["subscriptions"],  # type: ignore
+    )
+    def fetch_all_subscriptions(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "subscriptions"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="apm_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["apm_orders"],  # type: ignore
+    )
+    def fetch_apm_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "apm-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="card_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["card_orders"],  # type: ignore
+    )
+    def fetch_card_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "card-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="financial_entries",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["financial_entries"],  # type: ignore
+    )
+    def fetch_financial_entries(
+        dateTime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ):
+        if dateTime.end_value is None:
+            end_date = pendulum.now(tz="UTC")
+        else:
+            end_date = dateTime.end_value
+
+        start_date = dateTime.last_value
+        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
+
+    return (
+        fetch_all_subscriptions,
+        fetch_apm_orders,
+        fetch_card_orders,
+        fetch_financial_entries,
+    )

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/solidgate/helpers.py

@@ -92,9 +92,6 @@ class SolidgateClient:
         if not report_url:
             return f"Report URL not found in the response: {post_response.json()}", 400
 
-        # Wait for 5 seconds before attempting to download the report as report may not be immediately available
-        time.sleep(5)
-
         data = self.public_key + self.public_key
         hmac_hash = hmac.new(
             self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
@@ -109,27 +106,38 @@ class SolidgateClient:
             "Content-Type": "application/json",
         }
 
-        get_response = self.client.get(report_url, headers=headers_get)
+        # Retry getting the report for up to 10 minutes (600 seconds) with 5-second intervals
+        max_retries = 120  # 10 minutes / 5 seconds = 120 attempts
+        retry_count = 0
+
+        while retry_count < max_retries:
+            get_response = self.client.get(report_url, headers=headers_get)
 
-        if get_response.status_code == 200:
-            try:
-                response_json = json.loads(get_response.content)
-                if "error" in response_json:
-                    raise Exception(f"API Error: {response_json['error']['messages']}")
-            except json.JSONDecodeError:
+            if get_response.status_code == 200:
                 try:
-                    csv_data = get_response.content.decode("utf-8")
-                    df = pd.read_csv(StringIO(csv_data))
-                    df["created_at"] = df["created_at"].apply(
-                        lambda x: pendulum.parse(x)
+                    response_json = json.loads(get_response.content)
+                    if "error" in response_json:
+                        raise Exception(
+                            f"API Error: {response_json['error']['messages']}"
+                        )
+                except json.JSONDecodeError:
+                    try:
+                        csv_data = get_response.content.decode("utf-8")
+                        df = pd.read_csv(StringIO(csv_data))
+                        df["created_at"] = df["created_at"].apply(
+                            lambda x: pendulum.parse(x)
+                        )
+                        return df
+                    except Exception as e:
+                        raise Exception(f"Error reading CSV: {e}")
+            else:
+                # Report might not be ready yet, wait and retry
+                retry_count += 1
+                if retry_count >= max_retries:
+                    raise Exception(
+                        f"Failed to get report after {max_retries} attempts. Status code: {get_response.status_code}"
                     )
-                    return df
-                except Exception as e:
-                    raise Exception(f"Error reading CSV: {e}")
-        else:
-            raise Exception(
-                f"Failed to get report. Status code: {get_response.status_code}"
-            )
+                time.sleep(5)  # Wait 5 seconds before retrying
 
     def generateSignature(self, json_string):
         data = self.public_key + json_string + self.public_key

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/sources.py

@@ -76,6 +76,27 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        # Process Snowflake private key authentication
+        if uri.startswith("snowflake://"):
+            parsed_uri = urlparse(uri)
+            query_params = parse_qs(parsed_uri.query)
+
+            if "private_key" in query_params:
+                from dlt.common.libs.cryptography import decode_private_key
+
+                private_key = query_params["private_key"][0]
+                passphrase = query_params.get("private_key_passphrase", [None])[0]
+                decoded_key = decode_private_key(private_key, passphrase)
+
+                query_params["private_key"] = [base64.b64encode(decoded_key).decode()]
+                if "private_key_passphrase" in query_params:
+                    del query_params["private_key_passphrase"]
+
+                # Rebuild URI
+                uri = parsed_uri._replace(
+                    query=urlencode(query_params, doseq=True)
+                ).geturl()
+
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
@@ -139,7 +160,6 @@ class SqlSource:
                 return engine.execution_options(read_only=True)
 
             engine_adapter_callback = eng_callback
-
         from dlt.common.libs.sql_alchemy import (
             Engine,
             MetaData,

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/stripe_analytics/helpers.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import math
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from typing import Any, Dict, Iterable, List, Optional, Union
 

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/zoom/__init__.py

@@ -42,14 +42,14 @@ def zoom_source(
             end_dt = pendulum.now("UTC")
         else:
             end_dt = pendulum.parse(datetime.end_value)
-        
+
         base_params: Dict[str, Any] = {
             "type": "scheduled",
             "page_size": 300,
             "from": start_dt.to_date_string(),
             "to": end_dt.to_date_string(),
         }
-        
+
         for user in client.get_users():
             user_id = user["id"]
             yield from client.get_meetings(user_id, base_params)

{ingestr-0.13.67 → ingestr-0.13.69}/ingestr/src/zoom/helpers.py

@@ -62,7 +62,7 @@ class ZoomClient:
                 break
             params["next_page_token"] = token
 
-#https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
+    # https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
     def get_meetings(
         self, user_id: str, params: Dict[str, Any]
     ) -> Iterator[Dict[str, Any]]:

{ingestr-0.13.67 → ingestr-0.13.69}/requirements.in

@@ -2,7 +2,7 @@ asana==3.2.3
 confluent-kafka==2.8.0
 databricks-sql-connector==2.9.3
 dataclasses-json==0.6.7
-dlt==1.10.0
+dlt==1.11.0
 dlt-cratedb>=0.0.1
 duckdb_engine==0.17.0
 duckdb==1.2.1

{ingestr-0.13.67 → ingestr-0.13.69}/requirements.txt

@@ -97,7 +97,7 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.10.0
+dlt==1.11.0
     # via
     #   -r requirements.in
     #   dlt-cratedb

{ingestr-0.13.67 → ingestr-0.13.69}/requirements_arm64.txt

@@ -97,7 +97,7 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.10.0
+dlt==1.11.0
     # via
     #   -r requirements.in
     #   dlt-cratedb

ingestr-0.13.67/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.67"