ingestr 0.10.1__py3-none-any.whl → 0.10.3__py3-none-any.whl

ingestr/main.py

@@ -288,6 +288,8 @@ def ingest(
         ),
     ] = [],  # type: ignore
 ):
+
+    # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
     from datetime import datetime
@@ -383,6 +385,15 @@ def ingest(
         )
 
         factory = SourceDestinationFactory(source_uri, dest_uri)
+        track(
+            "command_running", 
+            {
+                "command": "ingest",
+                "source_type": factory.source_scheme,
+                "destination_type": factory.destination_scheme,
+            },
+        )
+
         source = factory.get_source()
         destination = factory.get_destination()
 

ingestr/src/destinations.py

@@ -5,9 +5,10 @@ import json
 import os
 import shutil
 import tempfile
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
+from dlt.common.configuration.specs import AwsCredentials
 
 
 class GenericSqlDestination:
@@ -194,3 +195,62 @@ class CsvDestination(GenericSqlDestination):
                     csv_writer.writerow(json_obj)
 
         shutil.rmtree(self.temp_path)
+
+
+class AthenaDestination:
+    def dlt_dest(self, uri: str, **kwargs):
+        encoded_uri = quote(uri, safe=":/?&=")
+        source_fields = urlparse(encoded_uri)
+        source_params = parse_qs(source_fields.query)
+
+        bucket = source_params.get("bucket", [None])[0]
+        if not bucket:
+            raise ValueError("A bucket is required to connect to Athena.")
+
+        if not bucket.startswith("s3://"):
+            bucket = f"s3://{bucket}"
+
+        query_result_path = source_params.get("query_results_path", [None])[0]
+        if query_result_path:
+            if not query_result_path.startswith("s3://"):
+                query_result_path = f"s3://{query_result_path}"
+        else:
+            query_result_path = bucket
+
+        access_key_id = source_params.get("access_key_id", [None])[0]
+        if not access_key_id:
+            raise ValueError("The AWS access_key_id is required to connect to Athena.")
+
+        secret_access_key = source_params.get("secret_access_key", [None])[0]
+        if not secret_access_key:
+            raise ValueError("The AWS secret_access_key is required to connect Athena")
+
+        work_group = source_params.get("workgroup", [None])[0]
+
+        region_name = source_params.get("region_name", [None])[0]
+        if not region_name:
+            raise ValueError("The region_name is required to connect to Athena.")
+
+        os.environ["DESTINATION__BUCKET_URL"] = bucket
+        os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+        os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+            secret_access_key
+        )
+
+        credentials = AwsCredentials(
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            region_name=region_name,
+        )
+        return dlt.destinations.athena(
+            query_result_bucket=query_result_path,
+            athena_work_group=work_group,
+            credentials=credentials,
+            destination_name=bucket,
+        )
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
+        return {}
+
+    def post_load(self):
+        pass

ingestr/src/factory.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 from dlt.common.destination import Destination
 
 from ingestr.src.destinations import (
+    AthenaDestination,
     BigQueryDestination,
     CsvDestination,
     DatabricksDestination,
@@ -159,6 +160,7 @@ class SourceDestinationFactory:
             "snowflake": SnowflakeDestination(),
             "synapse": SynapseDestination(),
             "csv": CsvDestination(),
+            "athena": AthenaDestination(),
         }
 
         if self.destination_scheme in match:

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.1"
+__version__ = "0.10.3"

{ingestr-0.10.1.dist-info → ingestr-0.10.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.1
+Version: 0.10.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,13 +26,15 @@ Requires-Dist: pendulum==3.0.0
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
+Requires-Dist: pyarrow==18.1.0
+Requires-Dist: pyathena==3.9.0
 Requires-Dist: pymongo==4.10.1
 Requires-Dist: pymysql==1.1.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: redshift-connector==2.1.3
 Requires-Dist: rich==13.9.4
 Requires-Dist: rudder-sdk-python==2.1.4
-Requires-Dist: s3fs==2024.9.0
+Requires-Dist: s3fs==2024.10.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sqlalchemy-bigquery==1.12.0
 Requires-Dist: sqlalchemy-hana==2.0.0
@@ -72,11 +74,20 @@ ingestr is a command-line app that allows you to ingest data from any source int
 ingestr takes away the complexity of managing any backend or writing any code for ingesting data, simply run the command and watch the data land on its destination.
 
 ## Installation
+We recommend using [uv](https://github.com/astral-sh/uv) to run `ingestr`.
 
 ```
-pip install ingestr
+pip install uv
+uvx ingestr
 ```
 
+Alternatively, if you'd like to install it globally:
+```
+uv pip install --system ingestr
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
+
 ## Quickstart
 
 ```bash
@@ -102,6 +113,13 @@ You can see the full documentation [here](https://bruin-data.github.io/ingestr/g
 
 Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shared_invite/zt-2dl2i8foy-bVsuMUauHeN9M2laVm3ZVg).
 
+## Contributing
+
+Pull requests are welcome. However, please open an issue first to discuss what you would like to change. We maybe able to offer you help and feedback regarding any changes you would like to make.
+
+> [!NOTE]
+> After cloning `ingestr` make sure to run `make setup` to install githooks.
+
 ## Supported sources & destinations
 
 <table>

{ingestr-0.10.1.dist-info → ingestr-0.10.3.dist-info}/RECORD

@@ -1,11 +1,11 @@
-ingestr/main.py,sha256=Uq0GTfCtlYu94Iw41AWhexZjgNdaEi7SqxkRAl0iTCg,22001
+ingestr/main.py,sha256=KR43c1BeIZUOjV8XJUvYEyFHi1nRaTcKdrmwAn79hQk,22311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
-ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
+ingestr/src/destinations.py,sha256=wT76Pi3JBbzfKj2goy4-L_XDPfjyPK6b95zyRxksr9g,8555
+ingestr/src/factory.py,sha256=nYWgWQINQEQKPeELwGY7MCeiOSoCP6JDPozfKKyGNXk,5013
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/sources.py,sha256=KhY6AH91zZoSthi7AbFd4_OsrPmxP3Q4ratA7ZscsZU,34810
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=v7Gyp89umFzDtY45tTjCdXqZnQ2RN01AibdYNxEvxYo,23
+ingestr/src/version.py,sha256=0C8KcY1dzs3hdkAre06v0NCQ0Uxcqv6g9a93bRcVLW0,23
 ingestr/src/adjust/__init__.py,sha256=I_G90D260OPIWCS716k0U4aeztlAieW9zi0R9-oW7TA,3007
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -70,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.1.dist-info/METADATA,sha256=FcFG3P5z-yvC3TiZSvf07DLaLVoJsGL3cEDqLqHFCeY,7060
-ingestr-0.10.1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.1.dist-info/RECORD,,
+ingestr-0.10.3.dist-info/METADATA,sha256=bQgQaW4dtrtPYMTyJRyR6OG-6rLobsyxCfIf04quiIA,7688
+ingestr-0.10.3.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.3.dist-info/RECORD,,