ingestr 0.13.28__py3-none-any.whl → 0.13.30__py3-none-any.whl

ingestr/main.py

@@ -302,6 +302,13 @@ def ingest(
             envvar=["COLUMNS", "INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
+    yield_limit: Annotated[
+        Optional[int],
+        typer.Option(
+            help="Limit the number of pages yielded from the source",
+            envvar=["YIELD_LIMIT", "INGESTR_YIELD_LIMIT"],
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -556,6 +563,9 @@ def ingest(
         if factory.source_scheme.startswith("mysql"):
             resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
+        if yield_limit:
+            resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
+
         def col_h(x):
             if column_hints:
                 x.apply_hints(columns=column_hints)

ingestr/src/appsflyer/__init__.py

@@ -0,0 +1,325 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from ingestr.src.appsflyer.client import AppsflyerClient
+
+DIMENSION_RESPONSE_MAPPING = {
+    "c": "campaign",
+    "af_adset_id": "adset_id",
+    "af_adset": "adset",
+    "af_ad_id": "ad_id",
+}
+HINTS = {
+    "app_id": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "campaign": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "geo": {
+        "data_type": "text",
+        "nullable": False,
+    },
+    "cost": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "clicks": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "impressions": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "average_ecpi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "installs": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "retention_day_7": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "retention_day_14": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": False,
+    },
+    "cohort_day_1_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_1_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_3_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_7_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_14_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "cohort_day_21_total_revenue_per_user": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "install_time": {
+        "data_type": "date",
+        "nullable": False,
+    },
+    "loyal_users": {
+        "data_type": "bigint",
+        "nullable": False,
+    },
+    "revenue": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "roi": {
+        "data_type": "decimal",
+        "precision": 30,
+        "scale": 5,
+        "nullable": True,
+    },
+    "uninstalls": {
+        "data_type": "bigint",
+        "nullable": True,
+    },
+}
+
+CAMPAIGNS_DIMENSIONS = ["c", "geo", "app_id", "install_time"]
+CAMPAIGNS_METRICS = [
+    "average_ecpi",
+    "clicks",
+    "cohort_day_1_revenue_per_user",
+    "cohort_day_1_total_revenue_per_user",
+    "cohort_day_14_revenue_per_user",
+    "cohort_day_14_total_revenue_per_user",
+    "cohort_day_21_revenue_per_user",
+    "cohort_day_21_total_revenue_per_user",
+    "cohort_day_3_revenue_per_user",
+    "cohort_day_3_total_revenue_per_user",
+    "cohort_day_7_revenue_per_user",
+    "cohort_day_7_total_revenue_per_user",
+    "cost",
+    "impressions",
+    "installs",
+    "loyal_users",
+    "retention_day_7",
+    "revenue",
+    "roi",
+    "uninstalls",
+]
+
+CREATIVES_DIMENSIONS = [
+    "c",
+    "geo",
+    "app_id",
+    "install_time",
+    "af_adset_id",
+    "af_adset",
+    "af_ad_id",
+]
+CREATIVES_METRICS = [
+    "impressions",
+    "clicks",
+    "installs",
+    "cost",
+    "revenue",
+    "average_ecpi",
+    "loyal_users",
+    "uninstalls",
+    "roi",
+]
+
+
+@dlt.source(max_table_nesting=0)
+def appsflyer_source(
+    api_key: str,
+    start_date: str,
+    end_date: str,
+    dimensions: list[str],
+    metrics: list[str],
+) -> Iterable[DltResource]:
+    client = AppsflyerClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CAMPAIGNS_DIMENSIONS, CAMPAIGNS_METRICS),
+    )
+    def campaigns(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+
+        yield from client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=CAMPAIGNS_DIMENSIONS,
+            metrics=CAMPAIGNS_METRICS,
+        )
+
+    @dlt.resource(
+        write_disposition="merge",
+        merge_key="install_time",
+        columns=make_hints(CREATIVES_DIMENSIONS, CREATIVES_METRICS),
+    )
+    def creatives(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        yield from client._fetch_data(
+            datetime.last_value,
+            end,
+            dimensions=CREATIVES_DIMENSIONS,
+            metrics=CREATIVES_METRICS,
+        )
+
+    primary_keys = []
+    if "install_time" not in dimensions:
+        dimensions.append("install_time")
+        primary_keys.append("install_time")
+
+    for dimension in dimensions:
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            primary_keys.append(DIMENSION_RESPONSE_MAPPING[dimension])
+        else:
+            primary_keys.append(dimension)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key=primary_keys,
+        columns=make_hints(dimensions, metrics),
+    )
+    def custom(
+        datetime=dlt.sources.incremental(
+            "install_time",
+            initial_value=(
+                start_date
+                if start_date
+                else pendulum.today().subtract(days=30).format("YYYY-MM-DD")
+            ),
+            end_value=end_date,
+        ),
+    ):
+        end = (
+            datetime.end_value
+            if datetime.end_value
+            else pendulum.now().format("YYYY-MM-DD")
+        )
+        res = client._fetch_data(
+            from_date=datetime.last_value,
+            to_date=end,
+            dimensions=dimensions,
+            metrics=metrics,
+        )
+        yield from res
+
+    return campaigns, creatives, custom
+
+
+def make_hints(dimensions: list[str], metrics: list[str]):
+    campaign_hints = {}
+    for dimension in dimensions:
+        resp_key = dimension
+        if dimension in DIMENSION_RESPONSE_MAPPING:
+            resp_key = DIMENSION_RESPONSE_MAPPING[dimension]
+
+        if resp_key in HINTS:
+            campaign_hints[resp_key] = HINTS[resp_key]
+
+    for metric in metrics:
+        if metric in HINTS:
+            campaign_hints[metric] = HINTS[metric]
+
+    return campaign_hints

ingestr/src/appsflyer/client.py

@@ -4,19 +4,6 @@ import requests
 from dlt.sources.helpers.requests import Client
 from requests.exceptions import HTTPError
 
-DEFAULT_GROUPING = ["c", "geo", "app_id", "install_time"]
-DEFAULT_KPIS = [
-    "impressions",
-    "clicks",
-    "installs",
-    "cost",
-    "revenue",
-    "average_ecpi",
-    "loyal_users",
-    "uninstalls",
-    "roi",
-]
-
 
 class AppsflyerClient:
     def __init__(self, api_key: str):
@@ -33,15 +20,20 @@ class AppsflyerClient:
         self,
         from_date: str,
         to_date: str,
+        dimensions: list[str],
+        metrics: list[str],
         maximum_rows=1000000,
-        dimensions=DEFAULT_GROUPING,
-        metrics=DEFAULT_KPIS,
     ):
+        excluded_metrics = exclude_metrics_for_date_range(metrics, from_date, to_date)
+        included_metrics = [
+            metric for metric in metrics if metric not in excluded_metrics
+        ]
+
         params = {
             "from": from_date,
             "to": to_date,
             "groupings": ",".join(dimensions),
-            "kpis": ",".join(metrics),
+            "kpis": ",".join(included_metrics),
             "format": "json",
             "maximum_rows": maximum_rows,
         }
@@ -68,39 +60,52 @@ class AppsflyerClient:
 
             if response.status_code == 200:
                 result = response.json()
-                yield result
+                yield standardize_keys(result, excluded_metrics)
             else:
                 raise HTTPError(
-                    f"Request failed with status code: {response.status_code}"
+                    f"Request failed with status code: {response.status_code}: {response.text}"
                 )
 
         except requests.RequestException as e:
             raise HTTPError(f"Request failed: {e}")
 
-    def fetch_campaigns(
-        self,
-        start_date: str,
-        end_date: str,
-    ):
-        metrics = DEFAULT_KPIS + [
-            "cohort_day_1_revenue_per_user",
-            "cohort_day_1_total_revenue_per_user",
-            "cohort_day_3_revenue_per_user",
-            "cohort_day_3_total_revenue_per_user",
-            "cohort_day_7_total_revenue_per_user",
-            "cohort_day_7_revenue_per_user",
-            "cohort_day_14_total_revenue_per_user",
-            "cohort_day_14_revenue_per_user",
-            "cohort_day_21_total_revenue_per_user",
-            "cohort_day_21_revenue_per_user",
-            "retention_day_7",
-        ]
-        return self._fetch_data(start_date, end_date, metrics=metrics)
 
-    def fetch_creatives(
-        self,
-        start_date: str,
-        end_date: str,
-    ):
-        dimensions = DEFAULT_GROUPING + ["af_adset_id", "af_adset", "af_ad_id"]
-        return self._fetch_data(start_date, end_date, dimensions=dimensions)
+def standardize_keys(data: list[dict], excluded_metrics: list[str]) -> list[dict]:
+    def fix_key(key: str) -> str:
+        return key.lower().replace("-", "").replace("  ", "_").replace(" ", "_")
+
+    standardized = []
+    for item in data:
+        standardized_item = {}
+        for key, value in item.items():
+            standardized_item[fix_key(key)] = value
+
+        for metric in excluded_metrics:
+            if metric not in standardized_item:
+                standardized_item[fix_key(metric)] = None
+
+        standardized.append(standardized_item)
+
+    return standardized
+
+
+def exclude_metrics_for_date_range(
+    metrics: list[str], from_date: str, to_date: str
+) -> list[str]:
+    """
+    Some of the cohort metrics are not available if there hasn't been enough time to have data for that cohort.
+    This means if you request data for yesterday with cohort day 7 metrics, you will get an error because 7 days hasn't passed yet.
+    One would expect the API to handle this gracefully, but it doesn't.
+
+    This function will exclude the metrics that are not available for the given date range.
+    """
+    import pendulum
+
+    excluded_metrics = []
+    days_between_today_and_end = (pendulum.now() - pendulum.parse(to_date)).days  # type: ignore
+    for metric in metrics:
+        if "cohort_day_" in metric:
+            day_count = int(metric.split("_")[2])
+            if days_between_today_and_end <= day_count:
+                excluded_metrics.append(metric)
+    return excluded_metrics

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.28"
+version = "v0.13.30"

ingestr/src/hubspot/__init__.py

@@ -169,7 +169,7 @@ def hubspot(
         api_key: str = api_key,
     ) -> Iterator[TDataItems]:
         """Hubspot schemas resource"""
-        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key,resource_name="schemas")
+        yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key, resource_name="schemas")
 
     @dlt.resource(name="quotes", write_disposition="replace")
     def quotes(
@@ -192,8 +192,7 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-
-        get_custom_object =  schemas(api_key)
+        get_custom_object = schemas(api_key)
         object_type_id = None
         for custom_object in get_custom_object:
             if custom_object["name"] == custom_object_name.capitalize():
@@ -201,18 +200,18 @@ def hubspot(
                 break
         if object_type_id is None:
             raise ValueError(f"There is no such custom object as {custom_object_name}")
-        custom_object_properties= f"crm/v3/properties/{object_type_id}"
-        
+        custom_object_properties = f"crm/v3/properties/{object_type_id}"
+
         props_pages = fetch_data(custom_object_properties, api_key)
         props = []
         for page in props_pages:
             props.extend([prop["name"] for prop in page])
         props = ",".join(sorted(list(set(props))))
-       
-        custom_object_endpoint= f"crm/v3/objects/{object_type_id}/?properties={props}"
-        
+
+        custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+
         """Hubspot custom object details resource"""
-        yield from fetch_data(custom_object_endpoint, api_key,resource_name="custom")
+        yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")
 
     return companies, contacts, deals, tickets, products, quotes, schemas, custom
 

ingestr/src/hubspot/helpers.py

@@ -90,7 +90,10 @@ def fetch_property_history(
 
 
 def fetch_data(
-    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None,resource_name: str = None
+    endpoint: str,
+    api_key: str,
+    params: Optional[Dict[str, Any]] = None,
+    resource_name: str = None,
 ) -> Iterator[List[Dict[str, Any]]]:
     """
     Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result.
@@ -133,15 +136,17 @@ def fetch_data(
             _objects: List[Dict[str, Any]] = []
             for _result in _data["results"]:
                 if resource_name == "schemas":
-                    _objects.append({
-                        "name": _result["labels"].get("singular", ""),
-                        "objectTypeId": _result.get("objectTypeId", ""),
-                        "id": _result.get("id", ""),
-                        "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
-                        "properties": _result.get("properties", ""),
-                        "createdAt": _result.get("createdAt", ""),
-                        "updatedAt": _result.get("updatedAt", "")
-                    })
+                    _objects.append(
+                        {
+                            "name": _result["labels"].get("singular", ""),
+                            "objectTypeId": _result.get("objectTypeId", ""),
+                            "id": _result.get("id", ""),
+                            "fullyQualifiedName": _result.get("fullyQualifiedName", ""),
+                            "properties": _result.get("properties", ""),
+                            "createdAt": _result.get("createdAt", ""),
+                            "updatedAt": _result.get("updatedAt", ""),
+                        }
+                    )
                 elif resource_name == "custom":
                     _objects.append(
                         _result.get("properties", ""),
@@ -157,9 +162,11 @@ def fetch_data(
                                 {
                                     "value": _obj["hs_object_id"],
                                     f"{association}_id": __r["id"],
-                            }
-                            for __r in _result["associations"][association]["results"]
-                        ]
+                                }
+                                for __r in _result["associations"][association][
+                                    "results"
+                                ]
+                            ]
 
                         # remove duplicates from list of dicts
                         __values = [

ingestr/src/hubspot/settings.py

@@ -5,15 +5,21 @@ from dlt.common import pendulum
 STARTDATE = pendulum.datetime(year=2000, month=1, day=1)
 
 CRM_CONTACTS_ENDPOINT = (
-    "/crm/v3/objects/contacts?associations=deals,products,tickets,quotes"
+    "/crm/v3/objects/contacts?associations=companies,deals,products,tickets,quotes"
 )
-CRM_COMPANIES_ENDPOINT = (
-    "/crm/v3/objects/companies?associations=contacts,deals,products,tickets,quotes"
+CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=products"
+CRM_DEALS_ENDPOINT = (
+    "/crm/v3/objects/deals?associations=companies,contacts,products,tickets,quotes"
+)
+CRM_PRODUCTS_ENDPOINT = (
+    "/crm/v3/objects/products?associations=companies,contacts,deals,tickets,quotes"
+)
+CRM_TICKETS_ENDPOINT = (
+    "/crm/v3/objects/tickets?associations=companies,contacts,deals,products,quotes"
+)
+CRM_QUOTES_ENDPOINT = (
+    "/crm/v3/objects/quotes?associations=companies,contacts,deals,products,tickets"
 )
-CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
-CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
-CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
-CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
 CRM_SCHEMAS_ENDPOINT = "/crm/v3/schemas"
 
 CRM_OBJECT_ENDPOINTS = {

ingestr/src/sources.py

@@ -52,7 +52,6 @@ from ingestr.src.adjust.adjust_helpers import parse_filters
 from ingestr.src.airtable import airtable_source
 from ingestr.src.applovin import applovin_source
 from ingestr.src.applovin_max import applovin_max_source
-from ingestr.src.appsflyer._init_ import appsflyer_source
 from ingestr.src.appstore import app_store
 from ingestr.src.appstore.client import AppStoreConnectClient
 from ingestr.src.arrow import memory_mapped_arrow
@@ -816,7 +815,15 @@ class HubspotSource:
                 custom_object=endpoint,
             ).with_resources("custom")
 
-        elif table in ["contacts", "companies", "deals", "tickets", "products", "quotes", "schemas"]:
+        elif table in [
+            "contacts",
+            "companies",
+            "deals",
+            "tickets",
+            "products",
+            "quotes",
+            "schemas",
+        ]:
             endpoint = table
         else:
             raise ValueError(
@@ -1035,6 +1042,8 @@ class AppsflyerSource:
         return True
 
     def dlt_source(self, uri: str, table: str, **kwargs):
+        from ingestr.src.appsflyer import appsflyer_source
+
         if kwargs.get("incremental_key"):
             raise ValueError(
                 "Appsflyer_Source takes care of incrementality on its own, you should not provide incremental_key"
@@ -1047,22 +1056,27 @@ class AppsflyerSource:
         if not api_key:
             raise ValueError("api_key in the URI is required to connect to Appsflyer")
 
-        resource = None
-        if table in ["campaigns", "creatives"]:
-            resource = table
-        else:
-            raise ValueError(
-                f"Resource '{table}' is not supported for Appsflyer source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
-            )
-
-        start_date = kwargs.get("interval_start") or "2024-01-02"
-        end_date = kwargs.get("interval_end") or "2024-01-29"
+        start_date = kwargs.get("interval_start")
+        end_date = kwargs.get("interval_end")
+        dimensions = []
+        metrics = []
+        if table.startswith("custom:"):
+            fields = table.split(":", 3)
+            if len(fields) != 3:
+                raise ValueError(
+                    "Invalid Adjust custom table format. Expected format: custom:<dimensions>:<metrics>"
+                )
+            dimensions = fields[1].split(",")
+            metrics = fields[2].split(",")
+            table = "custom"
 
         return appsflyer_source(
             api_key=api_key[0],
-            start_date=start_date,
-            end_date=end_date,
-        ).with_resources(resource)
+            start_date=start_date.strftime("%Y-%m-%d") if start_date else None,  # type: ignore
+            end_date=end_date.strftime("%Y-%m-%d") if end_date else None,  # type: ignore
+            dimensions=dimensions,
+            metrics=metrics,
+        ).with_resources(table)
 
 
 class ZendeskSource:
@@ -1414,16 +1428,16 @@ class GoogleAnalyticsSource:
         cred_base64 = source_fields.get("credentials_base64")
 
         if not cred_path and not cred_base64:
-            raise ValueError("credentials_path or credentials_base64 is required to connect Google Analytics")
+            raise ValueError(
+                "credentials_path or credentials_base64 is required to connect Google Analytics"
+            )
 
         credentials = {}
         if cred_path:
             with open(cred_path[0], "r") as f:
                 credentials = json.load(f)
         elif cred_base64:
-            credentials = json.loads(
-                base64.b64decode(cred_base64[0]).decode("utf-8")
-            )
+            credentials = json.loads(base64.b64decode(cred_base64[0]).decode("utf-8"))
 
         property_id = source_fields.get("property_id")
         if not property_id:

{ingestr-0.13.28.dist-info → ingestr-0.13.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.28
+Version: 0.13.30
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.28.dist-info → ingestr-0.13.30.dist-info}/RECORD

@@ -1,8 +1,8 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
+ingestr/main.py,sha256=Vt5NFN59BUlkYrOwiF9xF5MCFYp9r-aVSnQ99f1ypRE,25964
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=oC8nu6AzLZ_pxgRfz88wvC4OwlhMFeJZSkZgNXnuiSA,21
+ingestr/src/buildinfo.py,sha256=4uN8Z1CmXZ2T6X2YHEC3ZWFuS5uL_sTMUZiwgVyotHo,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
@@ -10,7 +10,7 @@ ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=GNX3dLFFqNxe6-humD46PZI9nh84tyuiwh9Cl0RwUg8,74309
+ingestr/src/sources.py,sha256=oCqYQPGUrGVASJyZpDe7b4hz6SqwcaFtF6hXHxx4pzo,74650
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -19,8 +19,8 @@ ingestr/src/adjust/adjust_helpers.py,sha256=G_EvNuvA7CsaOtbV3g249iAyggMDMZYbtWOz
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
 ingestr/src/applovin_max/__init__.py,sha256=CBMADQ23gi0oxAsxe-RV67GGb8I4EFOX_It45Vv9dj4,3315
-ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
-ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
+ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
+ingestr/src/appsflyer/client.py,sha256=5lIbiVG_l1_n2uMZx6_rw3FyM0yZUM9-sOa20utNtYU,3649
 ingestr/src/appstore/__init__.py,sha256=3P4VZH2WJF477QjW19jMTwu6L8DXcLkYSdutnvp3AmM,4742
 ingestr/src/appstore/client.py,sha256=qY9nBZPNIAveR-Dn-pW141Mr9xi9LMOz2HHfnfueHvE,3975
 ingestr/src/appstore/errors.py,sha256=KVpPWth5qlv6_QWEm3aJAt3cdf6miPJs0UDzxknx2Ms,481
@@ -61,9 +61,9 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=seijVlwBXE3-5SRtLjfJwbk1ZvckUj0IYBL0w69k464,10983
-ingestr/src/hubspot/helpers.py,sha256=jwWQy0AjVaMq4B033dAiQh2oehhYdFWLkynA43QsQzM,7573
-ingestr/src/hubspot/settings.py,sha256=hXnIhE4_FlhtgUiCjMLGGZJ_lBlBQJ0YWi6lHJ6Q3WU,2234
+ingestr/src/hubspot/__init__.py,sha256=rSYmN8h6qqxhWCW6elD-pC7iqHXlIofb1F9wvTzziUE,10962
+ingestr/src/hubspot/helpers.py,sha256=fscilfO_K7HS2XQxzf7MeZwVXLTP0WdqnV-NhdeqQAA,7748
+ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
@@ -121,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.28.dist-info/METADATA,sha256=-jYfvk0dlu8EvEDi2df5163aCHLX-JTkru0dmQroa0M,13659
-ingestr-0.13.28.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.28.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.28.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.28.dist-info/RECORD,,
+ingestr-0.13.30.dist-info/METADATA,sha256=h7uzURTF300XZEFjInqhpEjO6b-RFeBHWgS5jf4qCM0,13659
+ingestr-0.13.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.30.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.30.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.30.dist-info/RECORD,,

ingestr/src/appsflyer/_init_.py

@@ -1,24 +0,0 @@
-from typing import Iterable
-
-import dlt
-from dlt.common.typing import TDataItem
-from dlt.sources import DltResource
-
-from ingestr.src.appsflyer.client import AppsflyerClient
-
-
-@dlt.source(max_table_nesting=0)
-def appsflyer_source(
-    api_key: str, start_date: str, end_date: str
-) -> Iterable[DltResource]:
-    client = AppsflyerClient(api_key)
-
-    @dlt.resource(write_disposition="merge", merge_key="install_time")
-    def campaigns() -> Iterable[TDataItem]:
-        yield from client.fetch_campaigns(start_date, end_date)
-
-    @dlt.resource(write_disposition="merge", merge_key="install_time")
-    def creatives() -> Iterable[TDataItem]:
-        yield from client.fetch_creatives(start_date, end_date)
-
-    return campaigns, creatives