ingestr 0.10.1__py3-none-any.whl → 0.10.2__py3-none-any.whl

ingestr/src/destinations.py

@@ -5,9 +5,10 @@ import json
 import os
 import shutil
 import tempfile
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+from dlt.common.configuration.specs import AwsCredentials
 
 
 class GenericSqlDestination:
@@ -194,3 +195,62 @@ class CsvDestination(GenericSqlDestination):
                     csv_writer.writerow(json_obj)
 
         shutil.rmtree(self.temp_path)
+
+
+class AthenaDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        encoded_uri = quote(uri, safe=":/?&=")
+        source_fields = urlparse(encoded_uri)
+        source_params = parse_qs(source_fields.query)
+
+        bucket = source_params.get("bucket", [None])[0]
+        if not bucket:
+            raise ValueError("A bucket is required to connect to Athena.")
+
+        if not bucket.startswith("s3://"):
+            bucket = f"s3://{bucket}"
+
+        query_result_path = source_params.get("query_results_path", [None])[0]
+        if query_result_path:
+            if not query_result_path.startswith("s3://"):
+                query_result_path = f"s3://{query_result_path}"
+        else:
+            query_result_path = bucket
+
+        access_key_id = source_params.get("access_key_id", [None])[0]
+        if not access_key_id:
+            raise ValueError("The AWS access_key_id is required to connect to Athena.")
+
+        secret_access_key = source_params.get("secret_access_key", [None])[0]
+        if not secret_access_key:
+            raise ValueError("The AWS secret_access_key is required to connect Athena")
+
+        work_group = source_params.get("workgroup", [None])[0]
+
+        region_name = source_params.get("region_name", [None])[0]
+        if not region_name:
+            raise ValueError("The region_name is required to connect to Athena.")
+
+        os.environ["DESTINATION__BUCKET_URL"] = bucket
+        os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+        os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+            secret_access_key
+        )
+
+        credentials = AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            region_name=region_name,
+        )
+        return dlt.destinations.athena(
+            query_result_bucket=query_result_path,
+            athena_work_group=work_group,
+            credentials=credentials,
+            destination_name=bucket,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        return {}
+
+    def post_load(self):
+        pass

ingestr/src/factory.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 from dlt.common.destination import Destination
 
 from ingestr.src.destinations import (
+    AthenaDestination,
     BigQueryDestination,
     CsvDestination,
     DatabricksDestination,
@@ -159,6 +160,7 @@ class SourceDestinationFactory:
             "snowflake": SnowflakeDestination(),
             "synapse": SynapseDestination(),
             "csv": CsvDestination(),
+            "athena": AthenaDestination(),
         }
 
         if self.destination_scheme in match:

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.1"
+__version__ = "0.10.2"

{ingestr-0.10.1.dist-info → ingestr-0.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.1
+Version: 0.10.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,13 +26,15 @@ Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
+Requires-Dist: pyarrow==18.1.0
+Requires-Dist: pyathena==3.9.0
 Requires-Dist: pymongo==4.10.1
 Requires-Dist: pymysql==1.1.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: redshift-connector==2.1.3
 Requires-Dist: rich==13.9.4
 Requires-Dist: rudder-sdk-python==2.1.4
-Requires-Dist: s3fs==2024.9.0
+Requires-Dist: s3fs==2024.10.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sqlalchemy-bigquery==1.12.0
 Requires-Dist: sqlalchemy-hana==2.0.0

{ingestr-0.10.1.dist-info → ingestr-0.10.2.dist-info}/RECORD

@@ -1,11 +1,11 @@
 ingestr/main.py,sha256=Uq0GTfCtlYu94Iw41AWhexZjgNdaEi7SqxkRAl0iTCg,22001
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
-ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
+ingestr/src/destinations.py,sha256=wT76Pi3JBbzfKj2goy4-L_XDPfjyPK6b95zyRxksr9g,8555
+ingestr/src/factory.py,sha256=nYWgWQINQEQKPeELwGY7MCeiOSoCP6JDPozfKKyGNXk,5013
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/sources.py,sha256=KhY6AH91zZoSthi7AbFd4_OsrPmxP3Q4ratA7ZscsZU,34810
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=v7Gyp89umFzDtY45tTjCdXqZnQ2RN01AibdYNxEvxYo,23
+ingestr/src/version.py,sha256=A_AARqtxTOj_AQTpjpgOxNx-UOBio5wYFfZ2mrdMKfs,23
 ingestr/src/adjust/__init__.py,sha256=I_G90D260OPIWCS716k0U4aeztlAieW9zi0R9-oW7TA,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -70,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.1.dist-info/METADATA,sha256=FcFG3P5z-yvC3TiZSvf07DLaLVoJsGL3cEDqLqHFCeY,7060
-ingestr-0.10.1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.1.dist-info/RECORD,,
+ingestr-0.10.2.dist-info/METADATA,sha256=3_ilZkg36lUCtkfUBRlI2LnVj4Vl5OKq_R8NhvKwWk4,7123
+ingestr-0.10.2.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.2.dist-info/RECORD,,