ingestr 0.13.67__tar.gz → 0.13.68__tar.gz

{ingestr-0.13.67 → ingestr-0.13.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.67
+Version: 0.13.68
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -48,7 +48,7 @@ Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
 Requires-Dist: dlt-cratedb==0.0.1
-Requires-Dist: dlt==1.10.0
+Requires-Dist: dlt==1.11.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/main.py

@@ -455,7 +455,7 @@ def ingest(
 
                 column_hints[key]["primary_key"] = True
 
-        pipeline = dlt.pipeline(
+        pipeline = dlt.pipeline(  # type: ignore
             pipeline_name=m.hexdigest(),
             destination=dlt_dest,
             progress=progressInstance,

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/applovin_max/__init__.py

@@ -113,4 +113,5 @@ def get_data(
     df = pd.read_csv(response_url)
     df["Date"] = pd.to_datetime(df["Date"])
     df["partition_date"] = df["Date"].dt.date
+    df["platform"] = platform
     return df

ingestr-0.13.68/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.68"

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/destinations.py

@@ -90,7 +90,7 @@ class BigQueryDestination:
 
         return dlt.destinations.bigquery(
             credentials=credentials,  # type: ignore
-            location=location,
+            location=location,  # type: ignore
             project_id=project_id,
             **kwargs,
         )
@@ -398,7 +398,7 @@ class AthenaDestination:
 
         return dlt.destinations.athena(
             query_result_bucket=query_result_path,
-            athena_work_group=source_params.get("workgroup", [None])[0],
+            athena_work_group=source_params.get("workgroup", [None])[0],  # type: ignore
             credentials=AwsCredentials(
                 aws_access_key_id=access_key_id,  # type: ignore
                 aws_secret_access_key=secret_access_key,  # type: ignore

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/linear/__init__.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, Iterable, Iterator
 import dlt
 import pendulum
 
-from .helpers import  _paginate, _normalize_issue, _normalize_team
+from .helpers import _normalize_issue, _normalize_team, _paginate
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -143,13 +143,15 @@ def linear_source(
                     yield item
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams( updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+    def teams(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "updatedAt",
             initial_value=start_date.isoformat(),
             end_value=end_date.isoformat() if end_date else None,
             range_start="closed",
             range_end="closed",
-        ),) -> Iterator[Dict[str, Any]]:
+        ),
+    ) -> Iterator[Dict[str, Any]]:
         print(start_date)
         if updated_at.last_value:
             current_start_date = pendulum.parse(updated_at.last_value)

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/linear/helpers.py

@@ -1,10 +1,10 @@
-import json
 from typing import Any, Dict, Iterator, Optional
 
 import requests
 
 LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
 
+
 def _graphql(
     api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
 ) -> Dict[str, Any]:
@@ -20,6 +20,7 @@ def _graphql(
         raise ValueError(str(payload["errors"]))
     return payload["data"]
 
+
 def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
     cursor: Optional[str] = None
     while True:
@@ -30,6 +31,7 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
             break
         cursor = data["pageInfo"]["endCursor"]
 
+
 def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
     field_mapping = {
         "assignee": "assignee_id",
@@ -45,16 +47,26 @@ def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
         else:
             item[value] = None
             del item[key]
-    json_fields = ["comments", "subscribers", "attachments", "labels", "subtasks","projects", "memberships", "members"]
+    json_fields = [
+        "comments",
+        "subscribers",
+        "attachments",
+        "labels",
+        "subtasks",
+        "projects",
+        "memberships",
+        "members",
+    ]
     for field in json_fields:
         if item.get(field):
             item[f"{field}"] = item[field].get("nodes", [])
-                
+
     return item
 
+
 def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
     json_fields = ["memberships", "members", "projects"]
     for field in json_fields:
         if item.get(field):
             item[f"{field}"] = item[field].get("nodes", [])
-    return item
+    return item

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/sources.py

@@ -76,6 +76,27 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        # Process Snowflake private key authentication
+        if uri.startswith("snowflake://"):
+            parsed_uri = urlparse(uri)
+            query_params = parse_qs(parsed_uri.query)
+            
+            if "private_key" in query_params:
+                from dlt.common.libs.cryptography import decode_private_key
+                
+                private_key = query_params["private_key"][0]
+                passphrase = query_params.get("private_key_passphrase", [None])[0]
+                decoded_key = decode_private_key(private_key, passphrase)
+                
+                query_params["private_key"] = [base64.b64encode(decoded_key).decode()]
+                if "private_key_passphrase" in query_params:
+                    del query_params["private_key_passphrase"]
+                
+                # Rebuild URI
+                uri = parsed_uri._replace(
+                    query=urlencode(query_params, doseq=True)
+                ).geturl()
+
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
@@ -139,7 +160,6 @@ class SqlSource:
                 return engine.execution_options(read_only=True)
 
             engine_adapter_callback = eng_callback
-
         from dlt.common.libs.sql_alchemy import (
             Engine,
             MetaData,

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/stripe_analytics/helpers.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import math
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from typing import Any, Dict, Iterable, List, Optional, Union
 

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/zoom/__init__.py

@@ -42,14 +42,14 @@ def zoom_source(
             end_dt = pendulum.now("UTC")
         else:
             end_dt = pendulum.parse(datetime.end_value)
-        
+
         base_params: Dict[str, Any] = {
             "type": "scheduled",
             "page_size": 300,
             "from": start_dt.to_date_string(),
             "to": end_dt.to_date_string(),
         }
-        
+
         for user in client.get_users():
             user_id = user["id"]
             yield from client.get_meetings(user_id, base_params)

{ingestr-0.13.67 → ingestr-0.13.68}/ingestr/src/zoom/helpers.py

@@ -62,7 +62,7 @@ class ZoomClient:
                 break
             params["next_page_token"] = token
 
-#https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
+    # https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
     def get_meetings(
         self, user_id: str, params: Dict[str, Any]
     ) -> Iterator[Dict[str, Any]]:

{ingestr-0.13.67 → ingestr-0.13.68}/requirements.in

@@ -2,7 +2,7 @@ asana==3.2.3
 confluent-kafka==2.8.0
 databricks-sql-connector==2.9.3
 dataclasses-json==0.6.7
-dlt==1.10.0
+dlt==1.11.0
 dlt-cratedb>=0.0.1
 duckdb_engine==0.17.0
 duckdb==1.2.1

{ingestr-0.13.67 → ingestr-0.13.68}/requirements.txt

@@ -97,7 +97,7 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.10.0
+dlt==1.11.0
     # via
     #   -r requirements.in
     #   dlt-cratedb

{ingestr-0.13.67 → ingestr-0.13.68}/requirements_arm64.txt

@@ -97,7 +97,7 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.10.0
+dlt==1.11.0
     # via
     #   -r requirements.in
     #   dlt-cratedb

ingestr-0.13.67/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.67"