ingestr 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

ingestr/main.py

@@ -323,10 +323,12 @@ def ingest(
             else "Platform-specific"
         )
 
+        source_table_print = source_table.split(":")[0]
+
         print()
         print("[bold green]Initiated the pipeline with the following:[/bold green]")
         print(
-            f"[bold yellow]  Source:[/bold yellow] {factory.source_scheme} / {source_table}"
+            f"[bold yellow]  Source:[/bold yellow] {factory.source_scheme} / {source_table_print}"
         )
         print(
             f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"

ingestr/src/adjust/__init__.py

@@ -0,0 +1,100 @@
+from typing import Optional, Sequence
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .adjust_helpers import DEFAULT_DIMENSIONS, DEFAULT_METRICS, AdjustAPI
+
+REQUIRED_CUSTOM_DIMENSIONS = [
+    "hour",
+    "day",
+    "week",
+    "month",
+    "quarter",
+    "year",
+]
+KNOWN_TYPE_HINTS = {
+    "hour": {"data_type": "timestamp"},
+    "day": {"data_type": "date"},
+    "week": {"data_type": "text"},
+    "month": {"data_type": "text"},
+    "quarter": {"data_type": "text"},
+    "year": {"data_type": "text"},
+    "campaign": {"data_type": "text"},
+    "adgroup": {"data_type": "text"},
+    "creative": {"data_type": "text"},
+    # metrics
+    "installs": {"data_type": "bigint"},
+    "clicks": {"data_type": "bigint"},
+    "cost": {"data_type": "decimal"},
+    "network_cost": {"data_type": "decimal"},
+    "impressions": {"data_type": "bigint"},
+    "ad_revenue": {"data_type": "decimal"},
+    "all_revenue": {"data_type": "decimal"},
+}
+
+
+@dlt.source(max_table_nesting=0)
+def adjust_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
+    api_key: str,
+    dimensions: Optional[list[str]] = None,
+    metrics: Optional[list[str]] = None,
+    merge_key: Optional[str] = None,
+    filters: Optional[dict] = None,
+) -> Sequence[DltResource]:
+    @dlt.resource(write_disposition="merge", merge_key="day")
+    def campaigns():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date,
+            end_date=end_date,
+            dimensions=DEFAULT_DIMENSIONS,
+            metrics=DEFAULT_METRICS,
+            filters=filters,
+        )
+
+    @dlt.resource(write_disposition="merge", merge_key="day")
+    def creatives():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date,
+            end_date=end_date,
+            dimensions=DEFAULT_DIMENSIONS + ["adgroup", "creative"],
+            metrics=DEFAULT_METRICS,
+            filters=filters,
+        )
+
+    merge_key = merge_key
+    for dimension in REQUIRED_CUSTOM_DIMENSIONS:
+        if dimension in dimensions:
+            merge_key = dimension
+            break
+
+    type_hints = {}
+    for dimension in dimensions:
+        if dimension in KNOWN_TYPE_HINTS:
+            type_hints[dimension] = KNOWN_TYPE_HINTS[dimension]
+    for metric in metrics:
+        if metric in KNOWN_TYPE_HINTS:
+            type_hints[metric] = KNOWN_TYPE_HINTS[metric]
+
+    @dlt.resource(
+        write_disposition={"disposition": "merge", "strategy": "delete+insert"},
+        merge_key=merge_key,
+        primary_key=dimensions,
+        columns=type_hints,
+    )
+    def custom():
+        adjust_api = AdjustAPI(api_key=api_key)
+        yield from adjust_api.fetch_report_data(
+            start_date=start_date,
+            end_date=end_date,
+            dimensions=dimensions,
+            metrics=metrics,
+            filters=filters,
+        )
+
+    return campaigns, creatives, custom

ingestr/src/adjust/{helpers.py → adjust_helpers.py}

@@ -1,3 +1,6 @@
+from typing import Optional
+
+import pendulum
 import requests
 from dlt.sources.helpers.requests import Client
 from requests.exceptions import HTTPError
@@ -32,33 +35,32 @@ class AdjustAPI:
 
     def fetch_report_data(
         self,
-        start_date,
-        end_date,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
         dimensions=DEFAULT_DIMENSIONS,
         metrics=DEFAULT_METRICS,
-        utc_offset="+00:00",
-        ad_spend_mode="network",
-        attribution_source="first",
-        attribution_type="all",
-        cohort_maturity="immature",
-        reattributed="all",
-        sandbox="false",
+        filters: Optional[dict] = None,
     ):
         headers = {"Authorization": f"Bearer {self.api_key}"}
-        comma_separated_dimensions = ",".join(dimensions)
-        comma_separated_metrics = ",".join(metrics)
-        params = {
-            "date_period": f"{start_date}:{end_date}",
-            "dimensions": comma_separated_dimensions,
-            "metrics": comma_separated_metrics,
-            "utc_offset": utc_offset,
-            "ad_spend_mode": ad_spend_mode,
-            "attribution_source": attribution_source,
-            "attribution_type": attribution_type,
-            "cohort_maturity": cohort_maturity,
-            "reattributed": reattributed,
-            "sandbox": sandbox,
-        }
+        params = {}
+
+        if filters:
+            for key, value in filters.items():
+                if isinstance(value, list):
+                    params[key] = ",".join(value)
+                else:
+                    params[key] = value
+
+        params["date_period"] = (
+            f"{start_date.format('YYYY-MM-DD')}:{end_date.format('YYYY-MM-DD')}"
+        )
+        params["dimensions"] = ",".join(dimensions)
+        params["metrics"] = ",".join(metrics)
+
+        if start_date > end_date:
+            raise ValueError(
+                f"Invalid date range: Start date ({start_date}) must be earlier than end date ({end_date})."
+            )
 
         def retry_on_limit(
             response: requests.Response, exception: BaseException
@@ -80,3 +82,24 @@ class AdjustAPI:
             yield items
         else:
             raise HTTPError(f"Request failed with status code: {response.status_code}")
+
+
+def parse_filters(filters_raw: str) -> dict:
+    # Parse filter string like "key1=value1,key2=value2,value3,value4"
+    filters = {}
+    current_key = None
+
+    for item in filters_raw.split(","):
+        if "=" in item:
+            # Start of a new key-value pair
+            key, value = item.split("=")
+            filters[key] = [value]  # Always start with a list
+            current_key = key
+        elif current_key is not None:
+            # Additional value for the current key
+            filters[current_key].append(item)
+
+    # Convert single-item lists to simple values
+    filters = {k: v[0] if len(v) == 1 else v for k, v in filters.items()}
+
+    return filters

ingestr/src/sources.py

@@ -1,15 +1,18 @@
 import base64
 import csv
 import json
-from datetime import date, datetime
+from datetime import date
 from typing import Any, Callable, Optional
 from urllib.parse import parse_qs, urlparse
 
 import dlt
+import pendulum
 from dlt.common.configuration.specs import AwsCredentials
+from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TSecretStrValue
 
-from ingestr.src.adjust._init_ import adjust_source
+from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
+from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
 from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.arrow import memory_mapped_arrow
@@ -719,10 +722,10 @@ class KafkaSource:
 
 class AdjustSource:
     def handles_incrementality(self) -> bool:
-        return True
+        return False
 
     def dlt_source(self, uri: str, table: str, **kwargs):
-        if kwargs.get("incremental_key"):
+        if kwargs.get("incremental_key") and not table.startswith("custom:"):
             raise ValueError(
                 "Adjust takes care of incrementality on its own, you should not provide incremental_key"
             )
@@ -734,25 +737,62 @@ class AdjustSource:
         if not api_key:
             raise ValueError("api_key in the URI is required to connect to Adjust")
 
-        interval_start = kwargs.get("interval_start")
-        interval_end = kwargs.get("interval_end")
+        lookback_days = int(source_params.get("lookback_days", [30])[0])
 
         start_date = (
-            interval_start.strftime("%Y-%m-%d") if interval_start else "2000-01-01"
-        )
-        end_date = (
-            interval_end.strftime("%Y-%m-%d")
-            if interval_end
-            else datetime.now().strftime("%Y-%m-%d")
+            pendulum.now()
+            .replace(hour=0, minute=0, second=0, microsecond=0)
+            .subtract(days=lookback_days)
         )
+        if kwargs.get("interval_start"):
+            start_date = (
+                ensure_pendulum_datetime(str(kwargs.get("interval_start")))
+                .replace(hour=0, minute=0, second=0, microsecond=0)
+                .subtract(days=lookback_days)
+            )
 
-        Endpoint = None
-        if table in ["campaigns", "creatives"]:
-            Endpoint = table
+        end_date = pendulum.now()
+        if kwargs.get("interval_end"):
+            end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
+
+        dimensions = None
+        metrics = None
+        filters = []
+        if table.startswith("custom:"):
+            fields = table.split(":")
+            if len(fields) != 3 and len(fields) != 4:
+                raise ValueError(
+                    "Invalid Adjust custom table format. Expected format: custom:<dimensions>,<metrics> or custom:<dimensions>:<metrics>:<filters>"
+                )
+
+            dimensions = fields[1].split(",")
+            metrics = fields[2].split(",")
+            table = "custom"
+
+            found = False
+            for dimension in dimensions:
+                if dimension in REQUIRED_CUSTOM_DIMENSIONS:
+                    found = True
+                    break
+
+            if not found:
+                raise ValueError(
+                    f"At least one of the required dimensions is missing for custom Adjust report: {REQUIRED_CUSTOM_DIMENSIONS}"
+                )
+
+            if len(fields) == 4:
+                filters_raw = fields[3]
+                filters = parse_filters(filters_raw)
 
         return adjust_source(
-            start_date=start_date, end_date=end_date, api_key=api_key[0]
-        ).with_resources(Endpoint)
+            start_date=start_date,
+            end_date=end_date,
+            api_key=api_key[0],
+            dimensions=dimensions,
+            metrics=metrics,
+            merge_key=kwargs.get("merge_key"),
+            filters=filters,
+        ).with_resources(table)
 
 
 class AppsflyerSource:

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.9.2"
+__version__ = "0.9.4"

{ingestr-0.9.2.dist-info → ingestr-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.9.2
+Version: 0.9.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -51,7 +51,7 @@ Description-Content-Type: text/markdown
 
 <div align="center">
     <img src="https://github.com/bruin-data/ingestr/blob/main/resources/ingestr.svg?raw=true" width="500" />
-    <p>Ingest & copy data from any source to any destination without any code</p>
+    <p>Copy data from any source to any destination without any code</p>
     <img src="https://github.com/bruin-data/ingestr/blob/main/resources/demo.gif?raw=true" width="750" />
 </div>
 
@@ -63,7 +63,7 @@ Description-Content-Type: text/markdown
 
 ---
 
-Ingestr is a command-line application that allows you to ingest data from any source into any destination using simple command-line flags, no code necessary.
+ingestr is a command-line app that allows you to ingest data from any source into any destination using simple command-line flags, no code necessary.
 
 - ✨ copy data from your database into any destination
 - ➕ incremental loading: `append`, `merge` or `delete+insert`
@@ -89,10 +89,10 @@ ingestr ingest \
 
 That's it.
 
-This command will:
+This command:
 
-- get the table `public.some_data` from the Postgres instance.
-- upload this data to your BigQuery warehouse under the schema `ingestr` and table `some_data`.
+- gets the table `public.some_data` from the Postgres instance.
+- uploads this data to your BigQuery warehouse under the schema `ingestr` and table `some_data`.
 
 ## Documentation
 
@@ -102,7 +102,7 @@ You can see the full documentation [here](https://bruin-data.github.io/ingestr/g
 
 Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shared_invite/zt-2dl2i8foy-bVsuMUauHeN9M2laVm3ZVg).
 
-## Supported Sources & Destinations
+## Supported sources & destinations
 
 <table>
     <tr>
@@ -256,7 +256,7 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     </tr>
 </table>
 
-More to come soon!
+Feel free to create an issue if you'd like to see support for another source or destination.
 
 ## Acknowledgements
 

{ingestr-0.9.2.dist-info → ingestr-0.9.4.dist-info}/RECORD

@@ -1,12 +1,12 @@
-ingestr/main.py,sha256=U66TM57ePv-RdoAftQ0pFZx8woZUQnLepKa50C-bA5I,17655
+ingestr/main.py,sha256=B8TAQotJoYSvmaQQm33o2lv99OVLYNz-1Aw_fgQahwE,17718
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
 ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
-ingestr/src/sources.py,sha256=iZbCY-pzv6jbgdHOh0Vdsl3cBoC71eiFZgu_a5RoaDE,32188
+ingestr/src/sources.py,sha256=0eLrkd3oVsK9bWG7rp2Asu5QNrAUP2ZeMIIkduEuLL8,33756
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=gqT-BGoeEItda9fICQDvLbxEjWRIBhFJxPxxKvmHLUo,22
-ingestr/src/adjust/_init_.py,sha256=_jJE3Ywvv-YyJ7ywICdht_X2Gnd1cKm6F1wAfnpXuWM,890
-ingestr/src/adjust/helpers.py,sha256=kkYC3MqMHLNucuQ50klZWrvd3o8VfUysNtZTQSsKZ_c,2588
+ingestr/src/version.py,sha256=e56AvHfJCtG2ZwwINqsxINVbehWdKxMYgIDbjd7P-II,22
+ingestr/src/adjust/__init__.py,sha256=oTM7XozDcMuUiCZ0w4gWEBXuCCtMZ0iBfkKdd2pVa1E,3007
+ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
@@ -74,8 +74,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.9.2.dist-info/METADATA,sha256=ZnA32SeV-3jKU3g13UsczdNiKtvu1lXvF-Gb2mZUlzw,7004
-ingestr-0.9.2.dist-info/WHEEL,sha256=wukiCwsxxsuzcQTdnC_ZWHZECE4wwOh3xCCrap6i6Ts,87
-ingestr-0.9.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.9.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.9.2.dist-info/RECORD,,
+ingestr-0.9.4.dist-info/METADATA,sha256=mQKVpJ_Z7Ur3CLMbieKRmUrTEV0KIUe3MSeH2A1AUoE,7058
+ingestr-0.9.4.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+ingestr-0.9.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.9.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.9.4.dist-info/RECORD,,

{ingestr-0.9.2.dist-info → ingestr-0.9.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.26.0
+Generator: hatchling 1.26.3
 Root-Is-Purelib: true
 Tag: py3-none-any

ingestr/src/adjust/_init_.py

@@ -1,31 +0,0 @@
-from typing import Sequence
-
-import dlt
-from dlt.sources import DltResource
-
-from .helpers import DEFAULT_DIMENSIONS, AdjustAPI
-
-
-@dlt.source(max_table_nesting=0)
-def adjust_source(
-    start_date: str,
-    end_date: str,
-    api_key: str,
-) -> Sequence[DltResource]:
-    @dlt.resource(write_disposition="merge", merge_key="day")
-    def campaigns():
-        adjust_api = AdjustAPI(api_key=api_key)
-        yield from adjust_api.fetch_report_data(
-            start_date=start_date,
-            end_date=end_date,
-        )
-
-    @dlt.resource(write_disposition="merge", merge_key="day")
-    def creatives():
-        dimensions = DEFAULT_DIMENSIONS + ["adgroup", "creative"]
-        adjust_api = AdjustAPI(api_key=api_key)
-        yield from adjust_api.fetch_report_data(
-            start_date=start_date, end_date=end_date, dimensions=dimensions
-        )
-
-    return campaigns, creatives