ingestr 0.13.33__py3-none-any.whl → 0.13.35__py3-none-any.whl

ingestr/src/airtable/__init__.py

@@ -9,7 +9,7 @@ import pyairtable
 from dlt.sources import DltResource
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def airtable_source(
     base_id: str = dlt.config.value,
     table_names: Optional[List[str]] = dlt.config.value,
@@ -50,12 +50,13 @@ def airtable_resource(
             It starts with "app". See https://support.airtable.com/docs/finding-airtable-ids
         table (Dict[str, Any]): Metadata about an airtable, does not contain the actual records
     """
+
     primary_key_id = table["primaryFieldId"]
     primary_key_field = [
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [f"fields__{primary_key_field['name']}"]
+    primary_key: List[str] = [f"fields__{primary_key_field['name']}".lower()]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.33"
+version = "v0.13.35"

ingestr/src/destinations.py

@@ -235,12 +235,19 @@ class AthenaDestination:
         if not bucket.startswith("s3://"):
             bucket = f"s3://{bucket}"
 
-        query_result_path = source_params.get("query_results_path", [None])[0]
-        if query_result_path:
-            if not query_result_path.startswith("s3://"):
-                query_result_path = f"s3://{query_result_path}"
-        else:
-            query_result_path = bucket
+        bucket = bucket.rstrip("/")
+
+        dest_table = kwargs.get("dest_table", None)
+        if not dest_table:
+            raise ValueError("A destination table is required to connect to Athena.")
+
+        dest_table_fields = dest_table.split(".")
+        if len(dest_table_fields) != 2:
+            raise ValueError(
+                f"Table name must be in the format <schema>.<table>, given: {dest_table}"
+            )
+
+        query_result_path = f"{bucket}/{dest_table_fields[0]}_staging/metadata"
 
         access_key_id = source_params.get("access_key_id", [None])[0]
         secret_access_key = source_params.get("secret_access_key", [None])[0]
@@ -285,6 +292,7 @@ class AthenaDestination:
                 region_name=region_name,
             ),
             destination_name=bucket,
+            force_iceberg=True,
         )
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

ingestr/src/hubspot/__init__.py

@@ -197,11 +197,18 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-        get_custom_object = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
+        custom_objects = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
         object_type_id = None
+        associations = None
+        if ":" in custom_object_name:
+            fields = custom_object_name.split(":")
+            if len(fields) == 2:
+                custom_object_name = fields[0]
+                associations = fields[1]
 
         custom_object_lowercase = custom_object_name.lower()
-        for custom_object in get_custom_object["results"]:
+
+        for custom_object in custom_objects["results"]:
             if custom_object["name"].lower() == custom_object_lowercase:
                 object_type_id = custom_object["objectTypeId"]
                 break
@@ -223,6 +230,8 @@ def hubspot(
         props = ",".join(sorted(list(set(props))))
 
         custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+        if associations:
+            custom_object_endpoint += f"&associations={associations}"
 
         """Hubspot custom object details resource"""
         yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")

ingestr/src/hubspot/helpers.py

@@ -148,10 +148,6 @@ def fetch_data(
                             "updatedAt": _result.get("updatedAt", ""),
                         }
                     )
-                elif resource_name == "custom":
-                    _objects.append(
-                        _result.get("properties", ""),
-                    )
                 else:
                     _obj = _result.get("properties", _result)
                     if "id" not in _obj and "id" in _result:

ingestr/src/sources.py

@@ -722,7 +722,7 @@ class FacebookAdsSource:
             ).with_resources("facebook_insights")
         else:
             raise ValueError(
-                "fResource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                f"Resource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
         return facebook_ads_source(
@@ -804,11 +804,16 @@ class HubspotSource:
 
         if table.startswith("custom:"):
             fields = table.split(":", 2)
-            if len(fields) != 2:
+            if len(fields) != 2 and len(fields) != 3:
                 raise ValueError(
-                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type>"
+                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type> or custom:<custom_object_type>:<associations>"
                 )
-            endpoint = fields[1]
+
+            if len(fields) == 2:
+                endpoint = fields[1]
+            else:
+                endpoint = f"{fields[1]}:{fields[2]}"
+
             return hubspot(
                 api_key=api_key[0],
                 custom_object=endpoint,
@@ -847,22 +852,31 @@ class AirtableSource:
         if not table:
             raise ValueError("Source table is required to connect to Airtable")
 
-        tables = table.split(",")
-
         source_parts = urlparse(uri)
         source_fields = parse_qs(source_parts.query)
-        base_id = source_fields.get("base_id")
         access_token = source_fields.get("access_token")
 
-        if not base_id or not access_token:
+        if not access_token:
             raise ValueError(
-                "base_id and access_token in the URI are required to connect to Airtable"
+                "access_token in the URI is required to connect to Airtable"
             )
 
+        base_id = source_fields.get("base_id", [None])[0]
+        clean_table = table
+
+        table_fields = table.split("/")
+        if len(table_fields) == 2:
+            clean_table = table_fields[1]
+            if not base_id:
+                base_id = table_fields[0]
+
+        if not base_id:
+            raise ValueError("base_id in the URI is required to connect to Airtable")
+
         from ingestr.src.airtable import airtable_source
 
         return airtable_source(
-            base_id=base_id[0], table_names=tables, access_token=access_token[0]
+            base_id=base_id, table_names=[clean_table], access_token=access_token[0]
         )
 
 
@@ -1175,7 +1189,7 @@ class ZendeskSource:
             ).with_resources(table)
         else:
             raise ValueError(
-                "fResource '{table}' is not supported for Zendesk source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                f"Resource '{table}' is not supported for Zendesk source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
 

{ingestr-0.13.33.dist-info → ingestr-0.13.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.33
+Version: 0.13.35
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -46,7 +46,7 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt==1.9.0
+Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1

{ingestr-0.13.33.dist-info → ingestr-0.13.35.dist-info}/RECORD

@@ -2,21 +2,21 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=mRlGSqi2sHcZ2AKlwn5MqoMvFxXlSjcZxmPJr76rmRk,25187
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=0FHynPZSrkbfDnEUgBysgvemGcO2crTfn654je6lyHY,21
-ingestr/src/destinations.py,sha256=0fEwLY78SQDXbHcX4iz4Xc7H8FXN-QhVJL9uoUTZOs4,12924
+ingestr/src/buildinfo.py,sha256=-fdK0r3dEieckm9FbunVnN7VEWpVvtyhbo9bq89u0Es,21
+ingestr/src/destinations.py,sha256=Z79f01BSmEaXnQno2IQVt4Th4dmD-BiOQXlibZJ5sTw,13180
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=JiSqOyuvun0T3JQriYDmp_8EPe_2cepwo0wWtI-jOrU,76003
+ingestr/src/sources.py,sha256=Xinebylg-PqzyQ-r2wFukqhsPsv611fEoTvTWY1L-B4,76461
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
-ingestr/src/airtable/__init__.py,sha256=mdzeaq0g12HR8gbhtVR_aS_5GVWPZn6XD-zHUE5FunI,2788
+ingestr/src/airtable/__init__.py,sha256=XzRsS39xszUlh_s7P1_zq5v8vLfjz3m-NtTPaa8TTZU,2818
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
 ingestr/src/applovin_max/__init__.py,sha256=ZrxOUSirGxkGDmM9wsQO3anwNVzqtoCwN_OuCXfPkXE,3285
 ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
@@ -62,8 +62,8 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=QheZb_F2TEBdzd29SEgkU3AMdIIF7Gpj-t27EXnSIZ4,11448
-ingestr/src/hubspot/helpers.py,sha256=4aVOSzIsQV3RemqRJEJLu7BWMdcOuubwNvrjuMu87rg,8045
+ingestr/src/hubspot/__init__.py,sha256=wqHefhc_YRI5dNFCcpvH-UUilNThE49sbGouSBiHYsw,11776
+ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
@@ -122,8 +122,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.33.dist-info/METADATA,sha256=EZJ3I_1bEu7BeaeNBzWKNePpaLkw56jWoPiJBSUJF_Y,13574
-ingestr-0.13.33.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.33.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.33.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.33.dist-info/RECORD,,
+ingestr-0.13.35.dist-info/METADATA,sha256=HazXK_VyPcaappMDArhp7cBeRRaVc1oOTzgo3S7Gtr0,13575
+ingestr-0.13.35.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.35.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.35.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.35.dist-info/RECORD,,