ingestr 0.13.88__py3-none-any.whl → 0.13.89__py3-none-any.whl

ingestr/src/applovin/__init__.py

@@ -224,7 +224,7 @@ def resource(
 def custom_report_from_spec(spec: str) -> EndpointResource:
     parts = spec.split(":")
     if len(parts) != 4:
-        raise InvalidCustomReportError()
+        raise InvalidCustomReportError() 
 
     _, endpoint, report, dims = parts
     report_type = ReportType(report.strip())

ingestr/src/asana_source/__init__.py

@@ -182,7 +182,7 @@ def tasks(
 
 @dlt.transformer(
     data_from=tasks,
-    write_disposition="append",
+    write_disposition="replace",
 )
 @dlt.defer
 def stories(

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.88"
+version = "v0.13.89"

ingestr/src/destinations.py

@@ -25,7 +25,20 @@ from ingestr.src.loader import load_dlt_file
 
 class GenericSqlDestination:
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
-        table_fields = table.split(".")
+        
+        if uri.startswith("databricks://"):
+            p = urlparse(uri)
+            q = parse_qs(p.query)
+            schema = q.get("schema", [None])[0]
+            if not schema:
+                raise ValueError("Databricks requires schema in the URI.")
+            res = {
+                "dataset_name": schema,
+                "table_name": table,
+            }
+            return res
+        
+        table_fields = table.split(".") 
         if len(table_fields) != 2:
             raise ValueError("Table name must be in the format <schema>.<table>")
 
@@ -270,8 +283,30 @@ class MsSQLDestination(GenericSqlDestination):
 
 class DatabricksDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):
-        return dlt.destinations.databricks(credentials=uri, **kwargs)
+        p = urlparse(uri)
+        q = parse_qs(p.query)
+        access_token = p.password
+        server_hostname = p.hostname
+        http_path = q.get("http_path", [None])[0]
+        catalog = q.get("catalog", [None])[0]
+        schema = q.get("schema", [None])[0]
+        
+
+        creds = {
+            "access_token": access_token,
+            "server_hostname": server_hostname,
+            "http_path": http_path,
+            "catalog": catalog,
+            "schema": schema,
+        }
+       
+        return dlt.destinations.databricks(
+            credentials=creds,
+            **kwargs,
+        )
+    
 
+    
 
 class SynapseDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):

ingestr/src/filesystem/__init__.py

@@ -37,9 +37,14 @@ def readers(
         file_glob (str, optional): The filter to apply to the files in glob format. by default lists all files in bucket_url non-recursively
     """
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
-    filesystem_resource.apply_hints(
-        incremental=dlt.sources.incremental("modification_date"),
-    )
+
+    # NOTE: incremental support is disabled until we can figure out
+    #       how to support incremental loads per matching file, rather
+    #       than a blanket threshold.
+    #
+    # filesystem_resource.apply_hints(
+    #     incremental=dlt.sources.incremental("modification_date"),
+    # )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),
         filesystem_resource | dlt.transformer(name="read_jsonl")(_read_jsonl),

ingestr/src/salesforce/__init__.py

@@ -13,6 +13,7 @@ def salesforce_source(
     username: str,
     password: str,
     token: str,
+    domain: str,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from Salesforce using the Salesforce API.
@@ -26,7 +27,7 @@ def salesforce_source(
         DltResource: Data resources from Salesforce.
     """
 
-    client = Salesforce(username, password, token)
+    client = Salesforce(username, password, token, domain=domain)
 
     # define resources
     @dlt.resource(write_disposition="replace")
@@ -37,7 +38,7 @@ def salesforce_source(
     def user_role() -> Iterable[TDataItem]:
         yield get_records(client, "UserRole")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -47,7 +48,7 @@ def salesforce_source(
             client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity_line_item(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -57,7 +58,7 @@ def salesforce_source(
             client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity_contact_role(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -70,7 +71,7 @@ def salesforce_source(
             "SystemModstamp",
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def account(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "LastModifiedDate", initial_value=None
@@ -92,7 +93,7 @@ def salesforce_source(
     def campaign() -> Iterable[TDataItem]:
         yield get_records(client, "Campaign")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def campaign_member(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -114,7 +115,7 @@ def salesforce_source(
     def pricebook_entry() -> Iterable[TDataItem]:
         yield get_records(client, "PricebookEntry")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def task(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -122,7 +123,7 @@ def salesforce_source(
     ) -> Iterable[TDataItem]:
         yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def event(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None

ingestr/src/sources.py

@@ -2515,6 +2515,7 @@ class SalesforceSource:
             "username": params.get("username", [None])[0],
             "password": params.get("password", [None])[0],
             "token": params.get("token", [None])[0],
+            "domain": params.get("domain", [None])[0],
         }
         for k, v in creds.items():
             if v is None:

{ingestr-0.13.88.dist-info → ingestr-0.13.89.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.88
+Version: 0.13.89
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -42,7 +42,7 @@ Requires-Dist: confluent-kafka==2.8.0
 Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1
-Requires-Dist: databricks-sql-connector==2.9.3
+Requires-Dist: databricks-sql-connector==4.0.5
 Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1

{ingestr-0.13.88.dist-info → ingestr-0.13.89.dist-info}/RECORD

@@ -2,8 +2,8 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=HKIWe5l7QAN_f0qXt18bMVKJYb_guRTpX7gXDtwcRlc,21
-ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
+ingestr/src/buildinfo.py,sha256=YfKg385xjqfTlxcN3drjtO_64R31p1f28goiV70TrQY,21
+ingestr/src/destinations.py,sha256=QNT2rm91cZmY1_Zyj4VnbI14qGmZOUQOQUg9xUTVVYs,23799
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=hC5E_XgrgTHMqwqPc6ihUYvRGTGMTzdPfQhrgPyD0tY,6945
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
@@ -12,14 +12,14 @@ ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=YtqbkrF_z5n6Ccmj6kiYgjGMPL08r_1vc9YOvNhXlcw,125121
+ingestr/src/sources.py,sha256=MM_-6ZmIwFLS_L4kBkwJJc0XDyjDyHUkxMMnQaRfuRA,125176
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
 ingestr/src/airtable/__init__.py,sha256=XzRsS39xszUlh_s7P1_zq5v8vLfjz3m-NtTPaa8TTZU,2818
-ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
+ingestr/src/applovin/__init__.py,sha256=Y02ysL2vRiDoP3uN9ven2OVcH9zTd8PbmIjqIHds4zU,6985
 ingestr/src/applovin_max/__init__.py,sha256=fxXqsIibJarp5NOGe08G964HftwLDymTtYS_LqPJht4,3315
 ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
 ingestr/src/appsflyer/client.py,sha256=E6xPW4KlbBnQZ0K4eq2Xgb3AmGrtrzIX9bX8EnQr-D4,3615
@@ -29,7 +29,7 @@ ingestr/src/appstore/errors.py,sha256=KVpPWth5qlv6_QWEm3aJAt3cdf6miPJs0UDzxknx2M
 ingestr/src/appstore/models.py,sha256=tW1JSATHBIxZ6a77-RTCBQptJk6iRC8fWcmx4NW7SVA,1716
 ingestr/src/appstore/resources.py,sha256=DJxnNrBohVV0uSeruGV-N_e7UHSlhMhjhYNYdBuqECU,5375
 ingestr/src/arrow/__init__.py,sha256=8fEntgHseKjFMiPQIzxYzw_raicNsEgnveLi1IzBca0,2848
-ingestr/src/asana_source/__init__.py,sha256=QwQTCb5PXts8I4wLHG9UfRP-5ChfjSe88XAVfxMV5Ag,8183
+ingestr/src/asana_source/__init__.py,sha256=p9p89e62Qd3YmrrCCkIclswciSX51pBOMCuT7Ukeq2I,8184
 ingestr/src/asana_source/helpers.py,sha256=PukcdDQWIGqnGxuuobbLw4hUy4-t6gxXg_XywR7Lg9M,375
 ingestr/src/asana_source/settings.py,sha256=-2tpdkwh04RvLKFvwQodnFLYn9MaxOO1hsebGnDQMTU,2829
 ingestr/src/attio/__init__.py,sha256=CLejJjp5vGkt6r18nfNNZ-Xjc1SZgQ5IlcBW5XFQR90,3243
@@ -47,7 +47,7 @@ ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfE
 ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
 ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
 ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
-ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
+ingestr/src/filesystem/__init__.py,sha256=42YAOHQxZ7TkTXC1eeaLUJpjqJ3l7DH7C8j927pV4pc,4353
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
 ingestr/src/fluxx/__init__.py,sha256=Ei8BE0KAEzpadJT9RO5-8zMA7LvnIPhNPDKF4EyBcLo,328980
@@ -116,7 +116,7 @@ ingestr/src/pipedrive/helpers/pages.py,sha256=Klpjw2OnMuhzit3PpiHKsfzGcJ3rQPSQBl
 ingestr/src/quickbooks/__init__.py,sha256=cZUuVCOTGPHTscRj6i0DytO63_fWF-4ieMxoU4PcyTg,3727
 ingestr/src/revenuecat/__init__.py,sha256=5HbyZuEOekkbeeT72sM_bnGygSyYdmd_vczfAUz7xoM,4029
 ingestr/src/revenuecat/helpers.py,sha256=CYU6l79kplnfL87GfdxyGeEBrBSWEZfGP0GyjPHuVDk,9619
-ingestr/src/salesforce/__init__.py,sha256=2hik5pRrxVODdDTlUEMoyccNC07zozjnxkMHcjMT1qA,4558
+ingestr/src/salesforce/__init__.py,sha256=HVHY8pDngB498B6g6KDzwq-q2KPU4PxuEd9Y_8tDDFs,4716
 ingestr/src/salesforce/helpers.py,sha256=QTdazBt-qRTBbCQMZnyclIaDQFmBixBy_RDKD00Lt-8,2492
 ingestr/src/shopify/__init__.py,sha256=RzSSG93g-Qlkz6TAxi1XasFDdxxtVXIo53ZTtjGczW4,62602
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
@@ -158,8 +158,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.88.dist-info/METADATA,sha256=IypTsrgDspKt59K01ip36dHQYNCqAkj4ROGhuoj1kGk,15182
-ingestr-0.13.88.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.88.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.88.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.88.dist-info/RECORD,,
+ingestr-0.13.89.dist-info/METADATA,sha256=EfmN2TdrNG9oBfU-U78YNWe9tSZr6smMS5kynvFUBZE,15182
+ingestr-0.13.89.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.89.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.89.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.89.dist-info/RECORD,,