ingestr 0.10.0rc5__py3-none-any.whl → 0.10.2__py3-none-any.whl

ingestr/main.py

@@ -348,6 +348,14 @@ def ingest(
             )
             raise typer.Abort()
 
+    def run_on_resource(source, executable):
+        if hasattr(source, "selected_resources") and source.selected_resources:
+            resource_names = list(source.selected_resources.keys())
+            for res in resource_names:
+                executable(source.resources[res])
+        else:
+            executable(source)
+
     track(
         "command_triggered",
         {
@@ -487,10 +495,20 @@ def ingest(
             sql_exclude_columns=sql_exclude_columns,
         )
 
-        dlt_source.add_map(cast_set_to_list)
+        run_on_resource(dlt_source, lambda x: x.add_map(cast_set_to_list))
+
+        def col_h(x):
+            if column_hints:
+                x.apply_hints(columns=column_hints)
+
+        run_on_resource(dlt_source, col_h)
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
-            dlt_source.incremental.primary_key = ()
+
+            def set_primary_key(x):
+                x.incremental.primary_key = ()
+
+            run_on_resource(dlt_source, set_primary_key)
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS
@@ -522,7 +540,6 @@ def ingest(
             loader_file_format=(
                 loader_file_format.value if loader_file_format is not None else None  # type: ignore
             ),  # type: ignore
-            columns=column_hints,
         )
 
         report_errors(run_info)

ingestr/src/adjust/__init__.py

@@ -82,7 +82,7 @@ def adjust_source(
             type_hints[metric] = KNOWN_TYPE_HINTS[metric]
 
     @dlt.resource(
-        write_disposition={"disposition": "merge", "strategy": "delete+insert"},
+        write_disposition={"disposition": "merge", "strategy": "delete-insert"},
         merge_key=merge_key,
         primary_key=dimensions,
         columns=type_hints,

ingestr/src/destinations.py

@@ -5,9 +5,10 @@ import json
 import os
 import shutil
 import tempfile
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+from dlt.common.configuration.specs import AwsCredentials
 
 
 class GenericSqlDestination:
@@ -194,3 +195,62 @@ class CsvDestination(GenericSqlDestination):
                     csv_writer.writerow(json_obj)
 
         shutil.rmtree(self.temp_path)
+
+
+class AthenaDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        encoded_uri = quote(uri, safe=":/?&=")
+        source_fields = urlparse(encoded_uri)
+        source_params = parse_qs(source_fields.query)
+
+        bucket = source_params.get("bucket", [None])[0]
+        if not bucket:
+            raise ValueError("A bucket is required to connect to Athena.")
+
+        if not bucket.startswith("s3://"):
+            bucket = f"s3://{bucket}"
+
+        query_result_path = source_params.get("query_results_path", [None])[0]
+        if query_result_path:
+            if not query_result_path.startswith("s3://"):
+                query_result_path = f"s3://{query_result_path}"
+        else:
+            query_result_path = bucket
+
+        access_key_id = source_params.get("access_key_id", [None])[0]
+        if not access_key_id:
+            raise ValueError("The AWS access_key_id is required to connect to Athena.")
+
+        secret_access_key = source_params.get("secret_access_key", [None])[0]
+        if not secret_access_key:
+            raise ValueError("The AWS secret_access_key is required to connect Athena")
+
+        work_group = source_params.get("workgroup", [None])[0]
+
+        region_name = source_params.get("region_name", [None])[0]
+        if not region_name:
+            raise ValueError("The region_name is required to connect to Athena.")
+
+        os.environ["DESTINATION__BUCKET_URL"] = bucket
+        os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+        os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+            secret_access_key
+        )
+
+        credentials = AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            region_name=region_name,
+        )
+        return dlt.destinations.athena(
+            query_result_bucket=query_result_path,
+            athena_work_group=work_group,
+            credentials=credentials,
+            destination_name=bucket,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        return {}
+
+    def post_load(self):
+        pass

ingestr/src/factory.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 from dlt.common.destination import Destination
 
 from ingestr.src.destinations import (
+    AthenaDestination,
     BigQueryDestination,
     CsvDestination,
     DatabricksDestination,
@@ -159,6 +160,7 @@ class SourceDestinationFactory:
             "snowflake": SnowflakeDestination(),
             "synapse": SynapseDestination(),
             "csv": CsvDestination(),
+            "athena": AthenaDestination(),
         }
 
         if self.destination_scheme in match:

ingestr/src/filters.py

@@ -11,8 +11,6 @@ def cast_set_to_list(row):
 
 
 def table_adapter_exclude_columns(cols: list[str]):
-    print("given cols", cols)
-
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:

ingestr/src/sources.py

@@ -76,14 +76,17 @@ class SqlSource:
         if kwargs.get("sql_limit"):
 
             def query_adapter_callback(query, table):
-                return query.limit(kwargs.get("sql_limit"))
+                query = query.limit(kwargs.get("sql_limit"))
+                if kwargs.get("incremental_key"):
+                    query = query.order_by(kwargs.get("incremental_key"))
+                return query
 
         def type_adapter_callback(sql_type):
             if isinstance(sql_type, mysql.SET):
                 return sa.JSON
             return sql_type
 
-        table_instance = self.table_builder(
+        builder_res = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
             schema=table_fields.dataset,
             table=table_fields.table,
@@ -98,7 +101,7 @@ class SqlSource:
             ),
         )
 
-        return table_instance
+        return builder_res
 
 
 class ArrowMemoryMappedSource:
@@ -744,7 +747,7 @@ class KafkaSource:
 
 class AdjustSource:
     def handles_incrementality(self) -> bool:
-        return False
+        return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
         if kwargs.get("incremental_key") and not table.startswith("custom:"):
@@ -806,7 +809,7 @@ class AdjustSource:
                 filters_raw = fields[3]
                 filters = parse_filters(filters_raw)
 
-        return adjust_source(
+        src = adjust_source(
             start_date=start_date,
             end_date=end_date,
             api_key=api_key[0],
@@ -814,7 +817,9 @@ class AdjustSource:
             metrics=metrics,
             merge_key=kwargs.get("merge_key"),
             filters=filters,
-        ).with_resources(table)
+        )
+
+        return src.with_resources(table)
 
 
 class AppsflyerSource:

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.0rc5"
+__version__ = "0.10.2"

{ingestr-0.10.0rc5.dist-info → ingestr-0.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.0rc5
+Version: 0.10.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,13 +26,15 @@ Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
+Requires-Dist: pyarrow==18.1.0
+Requires-Dist: pyathena==3.9.0
 Requires-Dist: pymongo==4.10.1
 Requires-Dist: pymysql==1.1.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: redshift-connector==2.1.3
 Requires-Dist: rich==13.9.4
 Requires-Dist: rudder-sdk-python==2.1.4
-Requires-Dist: s3fs==2024.9.0
+Requires-Dist: s3fs==2024.10.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sqlalchemy-bigquery==1.12.0
 Requires-Dist: sqlalchemy-hana==2.0.0

{ingestr-0.10.0rc5.dist-info → ingestr-0.10.2.dist-info}/RECORD

@@ -1,12 +1,12 @@
-ingestr/main.py,sha256=UqSuqOMJPoq0fkcrba8D5gBfFIMKPEBEh0n19PJrP1c,21439
+ingestr/main.py,sha256=Uq0GTfCtlYu94Iw41AWhexZjgNdaEi7SqxkRAl0iTCg,22001
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
-ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
-ingestr/src/filters.py,sha256=9ckkhNF0lGVvSbw24VKQGXdtvxDCOfnSIFjswMjIu-A,611
-ingestr/src/sources.py,sha256=lvp_xAsa0EUvzEq0Txbc4gNe-JPAn_47jyHdSHKwLOQ,34644
+ingestr/src/destinations.py,sha256=wT76Pi3JBbzfKj2goy4-L_XDPfjyPK6b95zyRxksr9g,8555
+ingestr/src/factory.py,sha256=nYWgWQINQEQKPeELwGY7MCeiOSoCP6JDPozfKKyGNXk,5013
+ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
+ingestr/src/sources.py,sha256=KhY6AH91zZoSthi7AbFd4_OsrPmxP3Q4ratA7ZscsZU,34810
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=FipU1D3j5OsHv2DjFcCBccEi2EiXUNKnXRgNJjIA7Vo,26
-ingestr/src/adjust/__init__.py,sha256=oTM7XozDcMuUiCZ0w4gWEBXuCCtMZ0iBfkKdd2pVa1E,3007
+ingestr/src/version.py,sha256=A_AARqtxTOj_AQTpjpgOxNx-UOBio5wYFfZ2mrdMKfs,23
+ingestr/src/adjust/__init__.py,sha256=I_G90D260OPIWCS716k0U4aeztlAieW9zi0R9-oW7TA,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
@@ -70,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.0rc5.dist-info/METADATA,sha256=GdCvj3lzj-P3rQ0kW27B18lP_XeikycDnOSU-Hp01wQ,7063
-ingestr-0.10.0rc5.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.0rc5.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.0rc5.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.0rc5.dist-info/RECORD,,
+ingestr-0.10.2.dist-info/METADATA,sha256=3_ilZkg36lUCtkfUBRlI2LnVj4Vl5OKq_R8NhvKwWk4,7123
+ingestr-0.10.2.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.2.dist-info/RECORD,,