ingestr 0.10.0rc5__tar.gz → 0.10.2__tar.gz

{ingestr-0.10.0rc5 → ingestr-0.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc5
+Version: 0.10.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,13 +26,15 @@ Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
+Requires-Dist: pyarrow==18.1.0
+Requires-Dist: pyathena==3.9.0
 Requires-Dist: pymongo==4.10.1
 Requires-Dist: pymysql==1.1.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: redshift-connector==2.1.3
 Requires-Dist: rich==13.9.4
 Requires-Dist: rudder-sdk-python==2.1.4
-Requires-Dist: s3fs==2024.9.0
+Requires-Dist: s3fs==2024.10.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sqlalchemy-bigquery==1.12.0
 Requires-Dist: sqlalchemy-hana==2.0.0

{ingestr-0.10.0rc5 → ingestr-0.10.2}/docs/.vitepress/config.mjs

@@ -57,6 +57,7 @@ export default defineConfig({
             text: "Databases",
             collapsed: false,
             items: [
+              { text: "Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },

ingestr-0.10.2/docs/supported-sources/athena.md

@@ -0,0 +1,36 @@
+# Athena
+[Athena](https://aws.amazon.com/athena/) is an interactive query service that allows users to analyze data directly in Amazon S3 using standard SQL.
+
+The Athena destination stores data as Parquet files in S3 buckets and creates external tables in AWS Glue Catalog.
+
+ingestr supports Athena as a destination.
+
+## URI format
+The URI format for Athena is as follows:
+
+```plaintext
+athena://?bucket=<your-destination-bucket> \
+    query_results_path=<your-query-results-location> \
+    access_key_id=<your-aws-access-key-id> \
+    secret_access_key=<your-aws-secret-access-key> \
+    region_name=<your-aws-region>
+```
+URI parameters:
+- `bucket` (required): The name of the bucket where the data will be stored, containing the Parquet files that Athena will work with, e.g. `your_bucket_name` or `s3://your_bucket_name`.
+- `access_key_id` and `secret_access_key` (required): These are AWS credentials that will be used to authenticate with AWS services like S3 and Athena.
+- `region_name` (required): The AWS region of the Athena service and S3 buckets, e.g. `eu-central-1`
+- `query_results_path` (optional): The query location path where the results of Athena queries will be saved, e.g. `dest_path` or `s3://dest_path`. If not provided, it will default to the bucket specified in the `bucket` parameter.
+- `workgroup` (optional): The name of the Athena workgroup, e.g. `my_group`
+
+## Setting up an Athena Integration
+Athena requires a `bucket`, `access_key_id`, `secret_access_key` and `region_name` to access the S3 bucket. Please follow the guide on dltHub to obtain [credentials](https://dlthub.com/docs/dlt-ecosystem/destinations/athena#2-setup-bucket-storage-and-athena-credentials). Once you've completed the guide, you should have all the above-mentioned credentials.
+```
+ingestr ingest \
+  --source-uri "stripe://?api_key=key123" \
+  --source-table 'event' \
+  --dest-uri "athena://?bucket=bucket_123&access_key_id=access_123&secret_access_key=secret_123&region_name=eu-central-1" \
+  --dest-table 'stripe.event'
+```
+This is a sample command that will copy the data from the Stripe source into Athena.
+
+<img alt="athena_img" src="../media/athena.png" />

{ingestr-0.10.0rc5 → ingestr-0.10.2}/ingestr/main.py

@@ -348,6 +348,14 @@ def ingest(
             )
             raise typer.Abort()
 
+    def run_on_resource(source, executable):
+        if hasattr(source, "selected_resources") and source.selected_resources:
+            resource_names = list(source.selected_resources.keys())
+            for res in resource_names:
+                executable(source.resources[res])
+        else:
+            executable(source)
+
     track(
         "command_triggered",
         {
@@ -487,10 +495,20 @@ def ingest(
             sql_exclude_columns=sql_exclude_columns,
         )
 
-        dlt_source.add_map(cast_set_to_list)
+        run_on_resource(dlt_source, lambda x: x.add_map(cast_set_to_list))
+
+        def col_h(x):
+            if column_hints:
+                x.apply_hints(columns=column_hints)
+
+        run_on_resource(dlt_source, col_h)
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
-            dlt_source.incremental.primary_key = ()
+
+            def set_primary_key(x):
+                x.incremental.primary_key = ()
+
+            run_on_resource(dlt_source, set_primary_key)
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
@@ -522,7 +540,6 @@ def ingest(
             loader_file_format=(
                 loader_file_format.value if loader_file_format is not None else None  # type: ignore
             ),  # type: ignore
-            columns=column_hints,
         )
 
         report_errors(run_info)

{ingestr-0.10.0rc5 → ingestr-0.10.2}/ingestr/src/adjust/__init__.py

@@ -82,7 +82,7 @@ def adjust_source(
             type_hints[metric] = KNOWN_TYPE_HINTS[metric]
 
     @dlt.resource(
-        write_disposition={"disposition": "merge", "strategy": "delete+insert"},
+        write_disposition={"disposition": "merge", "strategy": "delete-insert"},
         merge_key=merge_key,
         primary_key=dimensions,
         columns=type_hints,

{ingestr-0.10.0rc5 → ingestr-0.10.2}/ingestr/src/destinations.py

@@ -5,9 +5,10 @@ import json
 import os
 import shutil
 import tempfile
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+from dlt.common.configuration.specs import AwsCredentials
 
 
 class GenericSqlDestination:
@@ -194,3 +195,62 @@ class CsvDestination(GenericSqlDestination):
                     csv_writer.writerow(json_obj)
 
         shutil.rmtree(self.temp_path)
+
+
+class AthenaDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        encoded_uri = quote(uri, safe=":/?&=")
+        source_fields = urlparse(encoded_uri)
+        source_params = parse_qs(source_fields.query)
+
+        bucket = source_params.get("bucket", [None])[0]
+        if not bucket:
+            raise ValueError("A bucket is required to connect to Athena.")
+
+        if not bucket.startswith("s3://"):
+            bucket = f"s3://{bucket}"
+
+        query_result_path = source_params.get("query_results_path", [None])[0]
+        if query_result_path:
+            if not query_result_path.startswith("s3://"):
+                query_result_path = f"s3://{query_result_path}"
+        else:
+            query_result_path = bucket
+
+        access_key_id = source_params.get("access_key_id", [None])[0]
+        if not access_key_id:
+            raise ValueError("The AWS access_key_id is required to connect to Athena.")
+
+        secret_access_key = source_params.get("secret_access_key", [None])[0]
+        if not secret_access_key:
+            raise ValueError("The AWS secret_access_key is required to connect Athena")
+
+        work_group = source_params.get("workgroup", [None])[0]
+
+        region_name = source_params.get("region_name", [None])[0]
+        if not region_name:
+            raise ValueError("The region_name is required to connect to Athena.")
+
+        os.environ["DESTINATION__BUCKET_URL"] = bucket
+        os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+        os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+            secret_access_key
+        )
+
+        credentials = AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            region_name=region_name,
+        )
+        return dlt.destinations.athena(
+            query_result_bucket=query_result_path,
+            athena_work_group=work_group,
+            credentials=credentials,
+            destination_name=bucket,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        return {}
+
+    def post_load(self):
+        pass

{ingestr-0.10.0rc5 → ingestr-0.10.2}/ingestr/src/factory.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 from dlt.common.destination import Destination
 
 from ingestr.src.destinations import (
+    AthenaDestination,
     BigQueryDestination,
     CsvDestination,
     DatabricksDestination,
@@ -159,6 +160,7 @@ class SourceDestinationFactory:
             "snowflake": SnowflakeDestination(),
             "synapse": SynapseDestination(),
             "csv": CsvDestination(),
+            "athena": AthenaDestination(),
         }
 
         if self.destination_scheme in match:

{ingestr-0.10.0rc5 → ingestr-0.10.2}/ingestr/src/filters.py

@@ -11,8 +11,6 @@ def cast_set_to_list(row):
 
 
 def table_adapter_exclude_columns(cols: list[str]):
-    print("given cols", cols)
-
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:

{ingestr-0.10.0rc5 → ingestr-0.10.2}/ingestr/src/sources.py

@@ -76,14 +76,17 @@ class SqlSource:
         if kwargs.get("sql_limit"):
 
             def query_adapter_callback(query, table):
-                return query.limit(kwargs.get("sql_limit"))
+                query = query.limit(kwargs.get("sql_limit"))
+                if kwargs.get("incremental_key"):
+                    query = query.order_by(kwargs.get("incremental_key"))
+                return query
 
         def type_adapter_callback(sql_type):
             if isinstance(sql_type, mysql.SET):
                 return sa.JSON
             return sql_type
 
-        table_instance = self.table_builder(
+        builder_res = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
             table=table_fields.table,
@@ -98,7 +101,7 @@ class SqlSource:
             ),
         )
 
-        return table_instance
+        return builder_res
 
 
 class ArrowMemoryMappedSource:
@@ -744,7 +747,7 @@ class KafkaSource:
 
 class AdjustSource:
     def handles_incrementality(self) -> bool:
-        return False
+        return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
         if kwargs.get("incremental_key") and not table.startswith("custom:"):
@@ -806,7 +809,7 @@ class AdjustSource:
                 filters_raw = fields[3]
                 filters = parse_filters(filters_raw)
 
-        return adjust_source(
+        src = adjust_source(
             start_date=start_date,
             end_date=end_date,
             api_key=api_key[0],
@@ -814,7 +817,9 @@ class AdjustSource:
             metrics=metrics,
             merge_key=kwargs.get("merge_key"),
             filters=filters,
-        ).with_resources(table)
+        )
+
+        return src.with_resources(table)
 
 
 class AppsflyerSource:

ingestr-0.10.2/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.10.2"