ingestr 0.13.67__py3-none-any.whl → 0.13.68__py3-none-any.whl

ingestr/main.py

@@ -455,7 +455,7 @@ def ingest(
 
                 column_hints[key]["primary_key"] = True
 
-        pipeline = dlt.pipeline(
+        pipeline = dlt.pipeline(  # type: ignore
             pipeline_name=m.hexdigest(),
             destination=dlt_dest,
             progress=progressInstance,

ingestr/src/applovin_max/__init__.py

@@ -113,4 +113,5 @@ def get_data(
     df = pd.read_csv(response_url)
     df["Date"] = pd.to_datetime(df["Date"])
     df["partition_date"] = df["Date"].dt.date
+    df["platform"] = platform
     return df

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.67"
+version = "v0.13.68"

ingestr/src/destinations.py

@@ -90,7 +90,7 @@ class BigQueryDestination:
 
         return dlt.destinations.bigquery(
             credentials=credentials,  # type: ignore
-            location=location,
+            location=location,  # type: ignore
             project_id=project_id,
             **kwargs,
         )
@@ -398,7 +398,7 @@ class AthenaDestination:
 
         return dlt.destinations.athena(
             query_result_bucket=query_result_path,
-            athena_work_group=source_params.get("workgroup", [None])[0],
+            athena_work_group=source_params.get("workgroup", [None])[0],  # type: ignore
             credentials=AwsCredentials(
                 aws_access_key_id=access_key_id,  # type: ignore
                 aws_secret_access_key=secret_access_key,  # type: ignore

ingestr/src/linear/__init__.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, Iterable, Iterator
 import dlt
 import pendulum
 
-from .helpers import  _paginate, _normalize_issue, _normalize_team
+from .helpers import _normalize_issue, _normalize_team, _paginate
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -143,13 +143,15 @@ def linear_source(
                     yield item
 
     @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams( updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+    def teams(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "updatedAt",
             initial_value=start_date.isoformat(),
             end_value=end_date.isoformat() if end_date else None,
             range_start="closed",
             range_end="closed",
-        ),) -> Iterator[Dict[str, Any]]:
+        ),
+    ) -> Iterator[Dict[str, Any]]:
         print(start_date)
         if updated_at.last_value:
             current_start_date = pendulum.parse(updated_at.last_value)

ingestr/src/linear/helpers.py

@@ -1,10 +1,10 @@
-import json
 from typing import Any, Dict, Iterator, Optional
 
 import requests
 
 LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
 
+
 def _graphql(
     api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
 ) -> Dict[str, Any]:
@@ -20,6 +20,7 @@ def _graphql(
         raise ValueError(str(payload["errors"]))
     return payload["data"]
 
+
 def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
     cursor: Optional[str] = None
     while True:
@@ -30,6 +31,7 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
             break
         cursor = data["pageInfo"]["endCursor"]
 
+
 def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
     field_mapping = {
         "assignee": "assignee_id",
@@ -45,16 +47,26 @@ def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
         else:
             item[value] = None
             del item[key]
-    json_fields = ["comments", "subscribers", "attachments", "labels", "subtasks","projects", "memberships", "members"]
+    json_fields = [
+        "comments",
+        "subscribers",
+        "attachments",
+        "labels",
+        "subtasks",
+        "projects",
+        "memberships",
+        "members",
+    ]
     for field in json_fields:
         if item.get(field):
             item[f"{field}"] = item[field].get("nodes", [])
-                
+
     return item
 
+
 def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
     json_fields = ["memberships", "members", "projects"]
     for field in json_fields:
         if item.get(field):
             item[f"{field}"] = item[field].get("nodes", [])
-    return item
+    return item

ingestr/src/sources.py

@@ -76,6 +76,27 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        # Process Snowflake private key authentication
+        if uri.startswith("snowflake://"):
+            parsed_uri = urlparse(uri)
+            query_params = parse_qs(parsed_uri.query)
+            
+            if "private_key" in query_params:
+                from dlt.common.libs.cryptography import decode_private_key
+                
+                private_key = query_params["private_key"][0]
+                passphrase = query_params.get("private_key_passphrase", [None])[0]
+                decoded_key = decode_private_key(private_key, passphrase)
+                
+                query_params["private_key"] = [base64.b64encode(decoded_key).decode()]
+                if "private_key_passphrase" in query_params:
+                    del query_params["private_key_passphrase"]
+                
+                # Rebuild URI
+                uri = parsed_uri._replace(
+                    query=urlencode(query_params, doseq=True)
+                ).geturl()
+
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
@@ -139,7 +160,6 @@ class SqlSource:
                 return engine.execution_options(read_only=True)
 
             engine_adapter_callback = eng_callback
-
         from dlt.common.libs.sql_alchemy import (
             Engine,
             MetaData,

ingestr/src/stripe_analytics/helpers.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import math
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from typing import Any, Dict, Iterable, List, Optional, Union
 

ingestr/src/zoom/__init__.py

@@ -42,14 +42,14 @@ def zoom_source(
             end_dt = pendulum.now("UTC")
         else:
             end_dt = pendulum.parse(datetime.end_value)
-        
+
         base_params: Dict[str, Any] = {
             "type": "scheduled",
             "page_size": 300,
             "from": start_dt.to_date_string(),
             "to": end_dt.to_date_string(),
         }
-        
+
         for user in client.get_users():
             user_id = user["id"]
             yield from client.get_meetings(user_id, base_params)

ingestr/src/zoom/helpers.py

@@ -62,7 +62,7 @@ class ZoomClient:
                 break
             params["next_page_token"] = token
 
-#https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
+    # https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
     def get_meetings(
         self, user_id: str, params: Dict[str, Any]
     ) -> Iterator[Dict[str, Any]]:

{ingestr-0.13.67.dist-info → ingestr-0.13.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.67
+Version: 0.13.68
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -48,7 +48,7 @@ Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
 Requires-Dist: dlt-cratedb==0.0.1
-Requires-Dist: dlt==1.10.0
+Requires-Dist: dlt==1.11.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1

{ingestr-0.13.67.dist-info → ingestr-0.13.68.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=taDyHyaVSpB17iNLl8zA0gmr4CqDO-MSTQX1CaRBB9U,26364
+ingestr/main.py,sha256=8EduMJdPnj6Z8p5xMzCJ8KZPq-STaPhIzhVTzoSz_sw,26380
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=sLhX7e2Ae9uo6JOAie2f4CiqQivKOIkv83SFKQF-YU8,21
-ingestr/src/destinations.py,sha256=rNKtrfWlH7tk4kXZlEpzIgZm1tG8fV-BimBlJ4CJG4E,22216
+ingestr/src/buildinfo.py,sha256=AmQ_QOtiF5_x4B8nPBudbz-zX_sJrDJElSgNVFmAdKM,21
+ingestr/src/destinations.py,sha256=ivTPio0zzqLRx22i597pxZHMNClz-XvYSyCaCPuGd8g,22248
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=AJCvlK4M1sIpAAks1K-xsR_uxziIxru74mj572zixhg,6546
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=gNayvEcWHppb5SLs2QFMxwXxPlUpbmoJTX6cwl9_dkk,103384
+ingestr/src/sources.py,sha256=IGFRqH1f9x2W7GUnAHoVYtrBYvvJbuDcHAhULCXn5i8,104352
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -19,7 +19,7 @@ ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3
 ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
 ingestr/src/airtable/__init__.py,sha256=XzRsS39xszUlh_s7P1_zq5v8vLfjz3m-NtTPaa8TTZU,2818
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
-ingestr/src/applovin_max/__init__.py,sha256=ZrxOUSirGxkGDmM9wsQO3anwNVzqtoCwN_OuCXfPkXE,3285
+ingestr/src/applovin_max/__init__.py,sha256=fxXqsIibJarp5NOGe08G964HftwLDymTtYS_LqPJht4,3315
 ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
 ingestr/src/appsflyer/client.py,sha256=E6xPW4KlbBnQZ0K4eq2Xgb3AmGrtrzIX9bX8EnQr-D4,3615
 ingestr/src/appstore/__init__.py,sha256=3P4VZH2WJF477QjW19jMTwu6L8DXcLkYSdutnvp3AmM,4742
@@ -81,8 +81,8 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
-ingestr/src/linear/__init__.py,sha256=attlRyodShvAZ5dmDJXgoKrYhwElpLMQTSaRaAGEqC0,5941
-ingestr/src/linear/helpers.py,sha256=VR_CBgTfMVTH6ULcSLKyrssGoJpJx8VFZrmBeYZzFfc,1995
+ingestr/src/linear/__init__.py,sha256=Qbf8EPHVh-8pVNe_fqLVinds7qQ3O4ymDuPPPD560Ng,5953
+ingestr/src/linear/helpers.py,sha256=Mb7oKpUTRnHl-CvO1fubjJlJFDkhTuA7PUldWglvagI,2044
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
@@ -122,7 +122,7 @@ ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
 ingestr/src/stripe_analytics/__init__.py,sha256=mK8dGKAlMPVqGE9gG30XfbvOvgVD0yWhNpt-D3iavDY,6385
-ingestr/src/stripe_analytics/helpers.py,sha256=O5ow8xORcyLhw1Yn6vFm__tASfmPOgR0TMVU9gXmxcE,11828
+ingestr/src/stripe_analytics/helpers.py,sha256=KGtRcSrhKEqzJ3AWpgDV2o4cuBFaIwu2Gc1KgvVWTtg,11764
 ingestr/src/stripe_analytics/settings.py,sha256=xt1-ljwP4nLTNUa8l3KwFbtK8FtQHgHpzGF5uPKfRsw,2246
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
@@ -136,8 +136,8 @@ ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
 ingestr/src/zendesk/helpers/credentials.py,sha256=EWyi0ZlxWFgd1huD86KNF4dApLHgmabqWksFpEg1cf0,1332
 ingestr/src/zendesk/helpers/talk_api.py,sha256=TSVSOErsBZvxcX91LMhAgvy6yLSYvpuVfOyKViOHtvA,4718
-ingestr/src/zoom/__init__.py,sha256=xkkqeWeNkD3K45j1WvCrn0YAwA1U04z01193YJMcqwU,3280
-ingestr/src/zoom/helpers.py,sha256=63DypaYdeEUpq7W9AJkKRVpE0z0YmlzOZP-9c1La6cc,3710
+ingestr/src/zoom/__init__.py,sha256=hgx_27kQTE9p26cXdp6yE6x5tjw0g8gY5pTB1qeg4L0,3264
+ingestr/src/zoom/helpers.py,sha256=jJuBkiK8OEUVYnulbemIFOO5rfg_iKI-7wql8hGNbZ0,3715
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
 ingestr/testdata/create_replace.csv,sha256=TQDbOSkRKq9ZZv1d68Qjwh94aIyUQ-oEwxpJIrd3YK8,1060
 ingestr/testdata/delete_insert_expected.csv,sha256=wbj7uboVWwm3sNMh1n7f4-OKFEQJv1s96snjEHp9nkg,336
@@ -147,8 +147,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.67.dist-info/METADATA,sha256=97obKQ7ltYvn_6SfwmBV3QrSH5mit8GbE9uQiJenqCY,15022
-ingestr-0.13.67.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.67.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.67.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.67.dist-info/RECORD,,
+ingestr-0.13.68.dist-info/METADATA,sha256=l2t9KyoQ_YqRie-2IItPzZSzHCIzk26wW16ZZ5SA4d8,15022
+ingestr-0.13.68.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.68.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.68.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.68.dist-info/RECORD,,