ingestr 0.10.2__py3-none-any.whl → 0.10.4__py3-none-any.whl

ingestr/main.py

@@ -288,6 +288,7 @@ def ingest(
         ),
     ] = [],  # type: ignore
 ):
+    # TODO(turtledev): can't we move this to the top of this file?
     import hashlib
     import tempfile
     from datetime import datetime
@@ -383,6 +384,15 @@ def ingest(
         )
 
         factory = SourceDestinationFactory(source_uri, dest_uri)
+        track(
+            "command_running",
+            {
+                "command": "ingest",
+                "source_type": factory.source_scheme,
+                "destination_type": factory.destination_scheme,
+            },
+        )
+
         source = factory.get_source()
         destination = factory.get_destination()
 

ingestr/src/adjust/__init__.py

@@ -67,13 +67,16 @@ def adjust_source(
             filters=filters,
         )
 
+    if not dimensions:
+        return campaigns, creatives
+
     merge_key = merge_key
+    type_hints = {}
     for dimension in REQUIRED_CUSTOM_DIMENSIONS:
         if dimension in dimensions:
             merge_key = dimension
             break
 
-    type_hints = {}
     for dimension in dimensions:
         if dimension in KNOWN_TYPE_HINTS:
             type_hints[dimension] = KNOWN_TYPE_HINTS[dimension]

ingestr/src/chess/__init__.py

@@ -12,7 +12,7 @@ from .helpers import get_path_with_retry, get_url_with_retry, validate_month_str
 from .settings import UNOFFICIAL_CHESS_API_URL
 
 
-@dlt.source(name="chess")
+@dlt.source(name="chess", max_table_nesting=0)
 def source(
     players: List[str], start_month: str = None, end_month: str = None
 ) -> Sequence[DltResource]:

ingestr/src/destinations.py

@@ -250,7 +250,14 @@ class AthenaDestination:
         )
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
-        return {}
+        table_fields = table.split(".")
+        if len(table_fields) != 2:
+            raise ValueError("Table name must be in the format <schema>.<table>")
+        return {
+            "table_format": "iceberg",
+            "dataset_name": table_fields[-2],
+            "table_name": table_fields[-1],
+        }
 
     def post_load(self):
         pass

ingestr/src/hubspot/__init__.py

@@ -50,7 +50,7 @@ from .settings import (
 THubspotObjectType = Literal["company", "contact", "deal", "ticket", "product", "quote"]
 
 
-@dlt.source(name="hubspot")
+@dlt.source(name="hubspot", max_table_nesting=0)
 def hubspot(
     api_key: str = dlt.secrets.value,
     include_history: bool = False,

ingestr/src/sources.py

@@ -622,7 +622,7 @@ class HubspotSource:
 
 class AirtableSource:
     def handles_incrementality(self) -> bool:
-        return True
+        return False
 
     # airtable://?access_token=<access_token>&base_id=<base_id>
 

ingestr/src/stripe_analytics/__init__.py

@@ -11,7 +11,7 @@ from .helpers import pagination, transform_date
 from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def stripe_source(
     endpoints: Tuple[str, ...] = ENDPOINTS,
     stripe_secret_key: str = dlt.secrets.value,

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.10.2"
+__version__ = "0.10.4"

ingestr/src/zendesk/__init__.py

@@ -23,7 +23,6 @@ from .settings import (
     TALK_ENDPOINTS,
 )
 
-
 @dlt.source(max_table_nesting=0)
 def zendesk_talk(
     credentials: TZendeskCredentials = dlt.secrets.value,

{ingestr-0.10.2.dist-info → ingestr-0.10.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.10.2
+Version: 0.10.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,11 +74,20 @@ ingestr is a command-line app that allows you to ingest data from any source int
 ingestr takes away the complexity of managing any backend or writing any code for ingesting data, simply run the command and watch the data land on its destination.
 
 ## Installation
+We recommend using [uv](https://github.com/astral-sh/uv) to run `ingestr`.
 
 ```
-pip install ingestr
+pip install uv
+uvx ingestr
 ```
 
+Alternatively, if you'd like to install it globally:
+```
+uv pip install --system ingestr
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
+
 ## Quickstart
 
 ```bash
@@ -104,6 +113,13 @@ You can see the full documentation [here](https://bruin-data.github.io/ingestr/g
 
 Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shared_invite/zt-2dl2i8foy-bVsuMUauHeN9M2laVm3ZVg).
 
+## Contributing
+
+Pull requests are welcome. However, please open an issue first to discuss what you would like to change. We maybe able to offer you help and feedback regarding any changes you would like to make.
+
+> [!NOTE]
+> After cloning `ingestr` make sure to run `make setup` to install githooks.
+
 ## Supported sources & destinations
 
 <table>

{ingestr-0.10.2.dist-info → ingestr-0.10.4.dist-info}/RECORD

@@ -1,18 +1,18 @@
-ingestr/main.py,sha256=Uq0GTfCtlYu94Iw41AWhexZjgNdaEi7SqxkRAl0iTCg,22001
+ingestr/main.py,sha256=wkU2uLMy1q8YarJ9mXNfJepeRjp6AuPDeNDOmMUt6n0,22309
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
-ingestr/src/destinations.py,sha256=wT76Pi3JBbzfKj2goy4-L_XDPfjyPK6b95zyRxksr9g,8555
+ingestr/src/destinations.py,sha256=zcHJIIHAZmcD9sJomd6G1Bc-1KsxnBD2aByOSV_9L3g,8850
 ingestr/src/factory.py,sha256=nYWgWQINQEQKPeELwGY7MCeiOSoCP6JDPozfKKyGNXk,5013
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
-ingestr/src/sources.py,sha256=KhY6AH91zZoSthi7AbFd4_OsrPmxP3Q4ratA7ZscsZU,34810
+ingestr/src/sources.py,sha256=Zw8bcAUhD_QLFNau5V768RBUhUuKhfkrV7ZXohrkaRE,34811
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=A_AARqtxTOj_AQTpjpgOxNx-UOBio5wYFfZ2mrdMKfs,23
-ingestr/src/adjust/__init__.py,sha256=I_G90D260OPIWCS716k0U4aeztlAieW9zi0R9-oW7TA,3007
+ingestr/src/version.py,sha256=fGZMaoPHZfTX9I4TDkr07gp-kj_1U_SD-gjQC_2flQs,23
+ingestr/src/adjust/__init__.py,sha256=NaRNwDhItG8Q7vUHw7zQvyfWjmT32M0CSc5ufjmBM9U,3067
 ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/arrow/__init__.py,sha256=AgU7S9Ra3ZeeG00Mf32zxO5sgMFfRnTdOSirUJ1Pu10,2976
-ingestr/src/chess/__init__.py,sha256=PaxT2DObudOGlhyoENE5LjR6rTdsxiqKKpAZeyzVLCA,6791
+ingestr/src/chess/__init__.py,sha256=y0Q8aKBigeKf3N7wuB_gadMQjVJzBPUT8Jhp1ObEWjk,6812
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
 ingestr/src/facebook_ads/__init__.py,sha256=ZZyogV48gmhDcC3CYQEsC4qT3Q6JI9IOnMff2NS1M-A,9207
@@ -29,7 +29,7 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=LZ3m6aGuhLVI3eNjvQE0rT4o_wbSPkY_SDKsM-g0V5U,21176
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=eSD_lEIEd16YijAtUATFG8FGO8YGPm-MtAk94KKsx6o,9740
+ingestr/src/hubspot/__init__.py,sha256=LshHlFzzs8trAOxSg7C9F7zIBakqsg8XfyNBouip09w,9761
 ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
 ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
@@ -51,12 +51,12 @@ ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSs
 ingestr/src/slack/__init__.py,sha256=UF-ficQ6K32u1EHytW3P35suACo9wuc6nMrAPViyZL8,9981
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
-ingestr/src/stripe_analytics/__init__.py,sha256=8yy6i4DAhUqY4ZForetQ0DWc_YQrY0FBH6yk0Z3m-Mw,4493
+ingestr/src/stripe_analytics/__init__.py,sha256=VEXH4to2vNojN4rk3qsypR7urtTzaxSBB3IBiD5tuoE,4514
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
-ingestr/src/zendesk/__init__.py,sha256=C7HkN195DGdOHId2_Sa_kAlcBrUmnVYZUa_tPkiyf1Q,17564
+ingestr/src/zendesk/__init__.py,sha256=x3RNxF0kqAu81nZkg1STmb1OPhd3Gdd4-wC8gUMNLic,17563
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
@@ -70,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.10.2.dist-info/METADATA,sha256=3_ilZkg36lUCtkfUBRlI2LnVj4Vl5OKq_R8NhvKwWk4,7123
-ingestr-0.10.2.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-ingestr-0.10.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.10.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.10.2.dist-info/RECORD,,
+ingestr-0.10.4.dist-info/METADATA,sha256=ARKr36Ta0VKvviovDwWmDVXeGC4l4h9b3HTaNUUDpYg,7688
+ingestr-0.10.4.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.10.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.10.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.10.4.dist-info/RECORD,,