ingestr 0.13.34__py3-none-any.whl → 0.13.35__py3-none-any.whl

ingestr/src/airtable/__init__.py

@@ -9,7 +9,7 @@ import pyairtable
 from dlt.sources import DltResource
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def airtable_source(
     base_id: str = dlt.config.value,
     table_names: Optional[List[str]] = dlt.config.value,
@@ -50,12 +50,13 @@ def airtable_resource(
             It starts with "app". See https://support.airtable.com/docs/finding-airtable-ids
         table (Dict[str, Any]): Metadata about an airtable, does not contain the actual records
     """
+
     primary_key_id = table["primaryFieldId"]
     primary_key_field = [
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [f"fields__{primary_key_field['name']}"]
+    primary_key: List[str] = [f"fields__{primary_key_field['name']}".lower()]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.34"
+version = "v0.13.35"

ingestr/src/destinations.py

@@ -235,12 +235,19 @@ class AthenaDestination:
         if not bucket.startswith("s3://"):
             bucket = f"s3://{bucket}"
 
-        query_result_path = source_params.get("query_results_path", [None])[0]
-        if query_result_path:
-            if not query_result_path.startswith("s3://"):
-                query_result_path = f"s3://{query_result_path}"
-        else:
-            query_result_path = bucket
+        bucket = bucket.rstrip("/")
+
+        dest_table = kwargs.get("dest_table", None)
+        if not dest_table:
+            raise ValueError("A destination table is required to connect to Athena.")
+
+        dest_table_fields = dest_table.split(".")
+        if len(dest_table_fields) != 2:
+            raise ValueError(
+                f"Table name must be in the format <schema>.<table>, given: {dest_table}"
+            )
+
+        query_result_path = f"{bucket}/{dest_table_fields[0]}_staging/metadata"
 
         access_key_id = source_params.get("access_key_id", [None])[0]
         secret_access_key = source_params.get("secret_access_key", [None])[0]
@@ -285,6 +292,7 @@ class AthenaDestination:
                 region_name=region_name,
             ),
             destination_name=bucket,
+            force_iceberg=True,
         )
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

ingestr/src/sources.py

@@ -852,22 +852,31 @@ class AirtableSource:
         if not table:
             raise ValueError("Source table is required to connect to Airtable")
 
-        tables = table.split(",")
-
         source_parts = urlparse(uri)
         source_fields = parse_qs(source_parts.query)
-        base_id = source_fields.get("base_id")
         access_token = source_fields.get("access_token")
 
-        if not base_id or not access_token:
+        if not access_token:
             raise ValueError(
-                "base_id and access_token in the URI are required to connect to Airtable"
+                "access_token in the URI is required to connect to Airtable"
             )
 
+        base_id = source_fields.get("base_id", [None])[0]
+        clean_table = table
+
+        table_fields = table.split("/")
+        if len(table_fields) == 2:
+            clean_table = table_fields[1]
+            if not base_id:
+                base_id = table_fields[0]
+
+        if not base_id:
+            raise ValueError("base_id in the URI is required to connect to Airtable")
+
         from ingestr.src.airtable import airtable_source
 
         return airtable_source(
-            base_id=base_id[0], table_names=tables, access_token=access_token[0]
+            base_id=base_id, table_names=[clean_table], access_token=access_token[0]
         )
 
 

{ingestr-0.13.34.dist-info → ingestr-0.13.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.34
+Version: 0.13.35
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -46,7 +46,7 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt==1.9.0
+Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1

{ingestr-0.13.34.dist-info → ingestr-0.13.35.dist-info}/RECORD

@@ -2,21 +2,21 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=mRlGSqi2sHcZ2AKlwn5MqoMvFxXlSjcZxmPJr76rmRk,25187
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=i3Tz80qXUH6VzMC8jzlySZd05zyoaaBcvoyLd2q-wKg,21
-ingestr/src/destinations.py,sha256=0fEwLY78SQDXbHcX4iz4Xc7H8FXN-QhVJL9uoUTZOs4,12924
+ingestr/src/buildinfo.py,sha256=-fdK0r3dEieckm9FbunVnN7VEWpVvtyhbo9bq89u0Es,21
+ingestr/src/destinations.py,sha256=Z79f01BSmEaXnQno2IQVt4Th4dmD-BiOQXlibZJ5sTw,13180
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=uRERygJ41y0MNXF3-FJvHr4btxlEM93ZeWr_Liz3N2M,76181
+ingestr/src/sources.py,sha256=Xinebylg-PqzyQ-r2wFukqhsPsv611fEoTvTWY1L-B4,76461
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
-ingestr/src/airtable/__init__.py,sha256=mdzeaq0g12HR8gbhtVR_aS_5GVWPZn6XD-zHUE5FunI,2788
+ingestr/src/airtable/__init__.py,sha256=XzRsS39xszUlh_s7P1_zq5v8vLfjz3m-NtTPaa8TTZU,2818
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
 ingestr/src/applovin_max/__init__.py,sha256=ZrxOUSirGxkGDmM9wsQO3anwNVzqtoCwN_OuCXfPkXE,3285
 ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
@@ -122,8 +122,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.34.dist-info/METADATA,sha256=84NPfN9LSTGrw79p3116CXH9BZGjnXgEvglsXpVhEY0,13574
-ingestr-0.13.34.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.34.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.34.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.34.dist-info/RECORD,,
+ingestr-0.13.35.dist-info/METADATA,sha256=HazXK_VyPcaappMDArhp7cBeRRaVc1oOTzgo3S7Gtr0,13575
+ingestr-0.13.35.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.35.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.35.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.35.dist-info/RECORD,,