ingestr 0.13.33__tar.gz → 0.13.35__tar.gz

{ingestr-0.13.33 → ingestr-0.13.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.33
+Version: 0.13.35
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -46,7 +46,7 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt==1.9.0
+Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1

{ingestr-0.13.33 → ingestr-0.13.35}/docs/supported-sources/airtable.md

@@ -9,12 +9,11 @@ ingestr supports Airtable as a source.
 The URI format for Airtable is as follows:
 
 ```plaintext
- airtable://?access_token=<access_token>&base_id=<base_id>
+ airtable://?access_token=<access_token>
 ```
 
 URI parameters:
 
-- `base_id`: A unique identifier for an Airtable base.
 - `access_token`: A personal access token for authentication with the Airtable API.
 
 The URI is used to connect to the Airtable API for extracting data. More details on setting up Airtable integrations can be found [here](https://airtable.com/developers/web/api).
@@ -23,15 +22,19 @@ The URI is used to connect to the Airtable API for extracting data. More details
 
 Airtable requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/airtable#setup-guide).
 
-Once you complete the guide, you should have an Access Token and Base Id. Let's say your Access Token is `patr123.abc` and Base Id is `appXYZ`, here's a sample command that will copy the data from Airtable into a DuckDB database:
+Once you complete the guide, you should have an Access Token and a Base ID. The source table you'll use for ingestr will be `<base_id>/<table_name>`.
+
+Let's say your access token is `patr123.abc` and the base ID is `appXYZ`, here's a sample command that will copy the data from Airtable into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 'airtable://?base_id=appXYc&access_token=patr123.abc' --source-table 'employee' --dest-uri 'duckdb:///airtable.duckdb' --dest-table 'des.employee'
+ingestr ingest 
+    --source-uri 'airtable://?access_token=patr123.abc' 
+    --source-table 'appXYZ/employee' 
+    --dest-uri 'duckdb:///airtable.duckdb' 
+    --dest-table 'des.employee'
 ```
 
-The result of this command will be an `employee` table containing data from the `employee` source in the `Airtable.duckdb` database.
-
-The `source-table` can include multiple table names that share the same `base_id`, e.g. `--source-table 'employee,users'`, but this will merge all the data from the specified tables into a single destination table.
+The result of this command will be an `employee` table containing data from the `employee` source in the `airtable.duckdb` database.
 
 > [!CAUTION]
 > Airtable does not support incremental loading, which means every time you run the command, the entire table will be copied from Airtable to the destination. This can be slow for large tables.

{ingestr-0.13.33 → ingestr-0.13.35}/docs/supported-sources/athena.md

@@ -10,7 +10,6 @@ The URI format for Athena is as follows:
 
 ```plaintext
 athena://?bucket=<your-destination-bucket> \
-    query_results_path=<your-query-results-location> \
     access_key_id=<your-aws-access-key-id> \
     secret_access_key=<your-aws-secret-access-key> \
     region_name=<your-aws-region>
@@ -20,7 +19,6 @@ URI parameters:
 - `access_key_id` and `secret_access_key` (required): These are AWS credentials that will be used to authenticate with AWS services like S3 and Athena.
 - `session_token` (optional): The session token for temporary credentials.
 - `region_name` (required if there's no local profile found): The AWS region of the Athena service and S3 buckets, e.g. `eu-central-1`
-- `query_results_path` (optional): The query location path where the results of Athena queries will be saved, e.g. `dest_path` or `s3://dest_path`. If not provided, it will default to the bucket specified in the `bucket` parameter.
 - `workgroup` (optional): The name of the Athena workgroup, e.g. `my_group`
 - `profile` (optional): The name of the AWS profile to use, e.g. `my_profile`
 

{ingestr-0.13.33 → ingestr-0.13.35}/docs/supported-sources/hubspot.md

@@ -42,3 +42,23 @@ HubSpot source allows ingesting the following sources into separate tables:
 - `contacts`: Retrieves information about visitors, potential customers, and leads.
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+## Custom Objects
+
+HubSpot custom objects are supported by using the `custom` table.
+
+The format for the custom object is as follows:
+
+```plaintext
+custom:<custom_object_name>:<optional associations>
+```
+
+For example, to ingest the `course` custom object with the `contacts` and `companies` associations, the format would be:
+
+```plaintext
+custom:course:contacts,companies
+```
+
+This would pull all the data for the `course` custom object and include the `contacts` and `companies` associations in the resulting table.
+
+

{ingestr-0.13.33 → ingestr-0.13.35}/ingestr/src/airtable/__init__.py

@@ -9,7 +9,7 @@ import pyairtable
 from dlt.sources import DltResource
 
 
-@dlt.source
+@dlt.source(max_table_nesting=1)
 def airtable_source(
     base_id: str = dlt.config.value,
     table_names: Optional[List[str]] = dlt.config.value,
@@ -50,12 +50,13 @@ def airtable_resource(
             It starts with "app". See https://support.airtable.com/docs/finding-airtable-ids
         table (Dict[str, Any]): Metadata about an airtable, does not contain the actual records
     """
+
     primary_key_id = table["primaryFieldId"]
     primary_key_field = [
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [f"fields__{primary_key_field['name']}"]
+    primary_key: List[str] = [f"fields__{primary_key_field['name']}".lower()]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

ingestr-0.13.35/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.35"

{ingestr-0.13.33 → ingestr-0.13.35}/ingestr/src/destinations.py

@@ -235,12 +235,19 @@ class AthenaDestination:
         if not bucket.startswith("s3://"):
             bucket = f"s3://{bucket}"
 
-        query_result_path = source_params.get("query_results_path", [None])[0]
-        if query_result_path:
-            if not query_result_path.startswith("s3://"):
-                query_result_path = f"s3://{query_result_path}"
-        else:
-            query_result_path = bucket
+        bucket = bucket.rstrip("/")
+
+        dest_table = kwargs.get("dest_table", None)
+        if not dest_table:
+            raise ValueError("A destination table is required to connect to Athena.")
+
+        dest_table_fields = dest_table.split(".")
+        if len(dest_table_fields) != 2:
+            raise ValueError(
+                f"Table name must be in the format <schema>.<table>, given: {dest_table}"
+            )
+
+        query_result_path = f"{bucket}/{dest_table_fields[0]}_staging/metadata"
 
         access_key_id = source_params.get("access_key_id", [None])[0]
         secret_access_key = source_params.get("secret_access_key", [None])[0]
@@ -285,6 +292,7 @@ class AthenaDestination:
                 region_name=region_name,
             ),
             destination_name=bucket,
+            force_iceberg=True,
         )
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:

{ingestr-0.13.33 → ingestr-0.13.35}/ingestr/src/hubspot/__init__.py

@@ -197,11 +197,18 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-        get_custom_object = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
+        custom_objects = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
         object_type_id = None
+        associations = None
+        if ":" in custom_object_name:
+            fields = custom_object_name.split(":")
+            if len(fields) == 2:
+                custom_object_name = fields[0]
+                associations = fields[1]
 
         custom_object_lowercase = custom_object_name.lower()
-        for custom_object in get_custom_object["results"]:
+
+        for custom_object in custom_objects["results"]:
             if custom_object["name"].lower() == custom_object_lowercase:
                 object_type_id = custom_object["objectTypeId"]
                 break
@@ -223,6 +230,8 @@ def hubspot(
         props = ",".join(sorted(list(set(props))))
 
         custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+        if associations:
+            custom_object_endpoint += f"&associations={associations}"
 
         """Hubspot custom object details resource"""
         yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")

{ingestr-0.13.33 → ingestr-0.13.35}/ingestr/src/hubspot/helpers.py

@@ -148,10 +148,6 @@ def fetch_data(
                             "updatedAt": _result.get("updatedAt", ""),
                         }
                     )
-                elif resource_name == "custom":
-                    _objects.append(
-                        _result.get("properties", ""),
-                    )
                 else:
                     _obj = _result.get("properties", _result)
                     if "id" not in _obj and "id" in _result:

{ingestr-0.13.33 → ingestr-0.13.35}/ingestr/src/sources.py

@@ -722,7 +722,7 @@ class FacebookAdsSource:
             ).with_resources("facebook_insights")
         else:
             raise ValueError(
-                "fResource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                f"Resource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
         return facebook_ads_source(
@@ -804,11 +804,16 @@ class HubspotSource:
 
         if table.startswith("custom:"):
             fields = table.split(":", 2)
-            if len(fields) != 2:
+            if len(fields) != 2 and len(fields) != 3:
                 raise ValueError(
-                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type>"
+                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type> or custom:<custom_object_type>:<associations>"
                 )
-            endpoint = fields[1]
+
+            if len(fields) == 2:
+                endpoint = fields[1]
+            else:
+                endpoint = f"{fields[1]}:{fields[2]}"
+
             return hubspot(
                 api_key=api_key[0],
                 custom_object=endpoint,
@@ -847,22 +852,31 @@ class AirtableSource:
         if not table:
             raise ValueError("Source table is required to connect to Airtable")
 
-        tables = table.split(",")
-
         source_parts = urlparse(uri)
         source_fields = parse_qs(source_parts.query)
-        base_id = source_fields.get("base_id")
         access_token = source_fields.get("access_token")
 
-        if not base_id or not access_token:
+        if not access_token:
             raise ValueError(
-                "base_id and access_token in the URI are required to connect to Airtable"
+                "access_token in the URI is required to connect to Airtable"
             )
 
+        base_id = source_fields.get("base_id", [None])[0]
+        clean_table = table
+
+        table_fields = table.split("/")
+        if len(table_fields) == 2:
+            clean_table = table_fields[1]
+            if not base_id:
+                base_id = table_fields[0]
+
+        if not base_id:
+            raise ValueError("base_id in the URI is required to connect to Airtable")
+
         from ingestr.src.airtable import airtable_source
 
         return airtable_source(
-            base_id=base_id[0], table_names=tables, access_token=access_token[0]
+            base_id=base_id, table_names=[clean_table], access_token=access_token[0]
         )
 
 
@@ -1175,7 +1189,7 @@ class ZendeskSource:
             ).with_resources(table)
         else:
             raise ValueError(
-                "fResource '{table}' is not supported for Zendesk source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                f"Resource '{table}' is not supported for Zendesk source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
 

{ingestr-0.13.33 → ingestr-0.13.35}/requirements.in

@@ -2,7 +2,7 @@ asana==3.2.3
 confluent-kafka==2.8.0
 databricks-sql-connector==2.9.3
 dataclasses-json==0.6.7
-dlt==1.9.0
+dlt==1.10.0
 duckdb_engine==0.17.0
 duckdb==1.2.1
 google-ads==25.1.0

{ingestr-0.13.33 → ingestr-0.13.35}/requirements.txt

@@ -94,7 +94,7 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.9.0
+dlt==1.10.0
     # via -r requirements.in
 dnspython==2.7.0
     # via pymongo

{ingestr-0.13.33 → ingestr-0.13.35}/requirements_arm64.txt

@@ -94,7 +94,7 @@ decorator==5.2.1
     # via gcsfs
 deprecation==2.1.0
     # via rudder-sdk-python
-dlt==1.9.0
+dlt==1.10.0
     # via -r requirements.in
 dnspython==2.7.0
     # via pymongo

ingestr-0.13.33/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.33"